diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fd5b36e9..ea89afaf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -182,7 +182,7 @@ jobs: CIBW_MUSLLINUX_X86_64_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_2' }} CIBW_MUSLLINUX_I686_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_2' }} CIBW_MUSLLINUX_AARCH64_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_2' }} - CIBW_TEST_REQUIRES: pytest setuptools # 3.12+ no longer includes distutils, just always ensure setuptools is present + CIBW_TEST_REQUIRES: pytest setuptools meson-python ninja # 3.12+ no longer includes distutils, just always ensure setuptools is present CIBW_TEST_COMMAND: PYTHONUNBUFFERED=1 python -m pytest ${{ matrix.test_args || '{project}' }} # default to test all run: | set -eux @@ -268,7 +268,7 @@ jobs: id: build env: CIBW_BUILD: ${{ matrix.spec }} - CIBW_TEST_REQUIRES: pytest setuptools + CIBW_TEST_REQUIRES: pytest setuptools meson-python ninja CIBW_TEST_COMMAND: pip install pip --upgrade; cd {project}; PYTHONUNBUFFERED=1 pytest MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.13' }} SDKROOT: ${{ matrix.sdkroot || 'macosx' }} @@ -351,7 +351,7 @@ jobs: id: build env: CIBW_BUILD: ${{ matrix.spec }} - CIBW_TEST_REQUIRES: pytest setuptools + CIBW_TEST_REQUIRES: pytest setuptools meson-python ninja CIBW_TEST_COMMAND: ${{ matrix.test_cmd || 'python -m pytest {package}/src/c' }} # FIXME: /testing takes ~45min on Windows and has some failures... # CIBW_TEST_COMMAND='python -m pytest {package}/src/c {package}/testing' @@ -527,7 +527,7 @@ jobs: - name: build and install run: | - python -m pip install pytest setuptools pytest-run-parallel + python -m pip install pytest setuptools meson-python ninja pytest-run-parallel python -m pip install . - name: run tests under pytest-run-parallel @@ -542,7 +542,7 @@ jobs: - name: build and install run: | - python -m pip install setuptools pytest pytest-run-parallel + python -m pip install setuptools meson-python ninja pytest pytest-run-parallel CFLAGS="-g -O3 -fsanitize=thread" python -m pip install -v . - name: run tests under pytest-run-parallel diff --git a/MANIFEST.in b/MANIFEST.in index a7616ffe..0590e1e5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ recursive-include src/cffi *.py *.h recursive-include src/c *.c *.h *.asm *.py win64.obj ffi.lib -recursive-include testing *.py *.c *.h +recursive-include testing *.py *.c *.h meson.build pyproject.toml *.txt recursive-include doc *.py *.rst Makefile *.bat recursive-include demo py.cleanup *.py embedding_test.c manual.c include AUTHORS LICENSE setup.py setup_base.py diff --git a/doc/source/buildtool.rst b/doc/source/buildtool.rst new file mode 100644 index 00000000..8d084ad6 --- /dev/null +++ b/doc/source/buildtool.rst @@ -0,0 +1,351 @@ +.. _buildtool_docs: + +========================================= +Building and Distributing CFFI Extensions +========================================= + +.. contents:: + +CFFI ships a small subpackage, :mod:`cffi.buildtool`, together with a +command-line program, ``gen-cffi-src``. Both produce the same output as +:meth:`FFI.emit_c_code`: a ``.c`` source file ready to be compiled into +a CPython extension module. What they add is two convenient front-ends +-- one that executes an existing "build" Python script, and one that +reads a ``cdef`` and C prelude from two files. This tool enables +integrating with and build backend, such as `meson-python +`_, `scikit-build-core +`_, or similar. + +The rest of this page uses meson-python in the examples, but any PEP +517 backend that lets you run a helper program during the build can +drive ``gen-cffi-src`` the same way. + +The ``cffi.buildtool`` subpackage was integrated from the +`cffi-buildtool`_ project by Rose Davidson (@inklesspen on GitHub). + +.. _cffi-buildtool: https://github.com/inklesspen/cffi-buildtool + + +Python API for ``cffi.buildtool`` +================================= + +.. py:module:: cffi.buildtool + +.. py:function:: find_ffi_in_python_script(pysrc, filename, ffivar) + + Execute a Python build script and return the :class:`cffi.FFI` + object it defines. ``pysrc`` is the text of the script, + ``filename`` is used for diagnostics, and ``ffivar`` is the name + the script binds to the :class:`FFI` (or to a callable returning + one -- typical ``ffibuilder`` names are supported). The script is + executed with ``__name__`` set to ``"gen-cffi-src"`` so a trailing + ``if __name__ == "__main__": ffibuilder.compile()`` block is + skipped. + +.. py:function:: make_ffi_from_sources(modulename, cdef, csrc) + + Build an :class:`cffi.FFI` from a ``cdef`` string and a C source + prelude. Equivalent to:: + + ffi = FFI() + ffi.cdef(cdef) + ffi.set_source(modulename, csrc) + +.. py:function:: generate_c_source(ffi) + + Return the C source that :meth:`FFI.emit_c_code` would write for + the given :class:`cffi.FFI`, as a :class:`str`. + + +The ``gen-cffi-src`` Command-line Tool +====================================== + +``gen-cffi-src`` has two subcommands. In both, the final positional +argument is the path to the ``.c`` file to generate. + +.. note:: + + When you drive the build from a build backend, the + ``libraries=``, ``library_dirs=``, ``include_dirs=``, + ``extra_compile_args=`` etc. arguments you pass to + :meth:`FFI.set_source` are *ignored*. Link and include settings are + the build backend's responsibility; for meson-python you express + them through the ``dependencies:`` / ``include_directories:`` + arguments of ``py.extension_module()``. + + +``gen-cffi-src exec-python`` +---------------------------- + +This mode takes the Python build script you would normally run by +hand -- the one the CFFI docs show under "Main mode of usage" -- and +generates the ``.c`` source for you. For example, given this +``_squared_build.py``:: + + from cffi import FFI + + ffibuilder = FFI() + + ffibuilder.cdef("int square(int n);") + + ffibuilder.set_source( + "squared._squared", + '#include "square.h"', + ) + + if __name__ == "__main__": + ffibuilder.compile(verbose=True) + +you run: + +.. code-block:: console + + $ gen-cffi-src exec-python _squared_build.py _squared.c + +If the :class:`cffi.FFI` is bound to a name other than ``ffibuilder``, +pass ``--ffi-var``: + +.. code-block:: console + + $ gen-cffi-src exec-python --ffi-var=make_ffi _squared_build.py _squared.c + +``gen-cffi-src read-sources`` +----------------------------- + +For larger modules, keeping the ``cdef`` and the C source prelude in +separate files tends to be easier to work with -- your editor +treats them as plain C, and presubmit tooling doesn't have to parse +them out of a string literal. + +Given ``squared.cdef.txt``: + +.. code-block:: C + + int square(int n); + +and ``squared.csrc.c``: + +.. code-block:: C + + #include "square.h" + +you run: + +.. code-block:: console + + $ gen-cffi-src read-sources squared._squared squared.cdef.txt squared.csrc.c _squared.c + +The first positional argument is the fully qualified module name that +will be embedded in the generated source (equivalent to the first +argument to :meth:`FFI.set_source`). + + +A Worked Example Using ``meson-python`` +======================================= + +Project layout: + +.. code-block:: text + + squared/ + ├── pyproject.toml + ├── meson.build + └── src/ + ├── squared/ + │ ├── __init__.py + │ └── _squared_build.py + └── csrc/ + ├── square.h + └── square.c + +``pyproject.toml``: + +.. code-block:: toml + + [build-system] + build-backend = 'mesonpy' + requires = ['meson-python', 'cffi'] + + [project] + name = 'squared' + version = '0.1.0' + requires-python = '>=3.9' + dependencies = ['cffi'] + +``meson.build``: + +.. code-block:: meson + + project( + 'squared', + 'c', + version: '0.1.0', + ) + + py = import('python').find_installation(pure: false) + + install_subdir('src/squared', install_dir: py.get_install_dir()) + + gen_cffi_src = find_program('gen-cffi-src') + + square_lib = static_library( + 'square', + 'src/csrc/square.c', + include_directories: include_directories('src/csrc'), + ) + square_dep = declare_dependency( + link_with: square_lib, + include_directories: include_directories('src/csrc'), + ) + + squared_ext_src = custom_target( + 'squared-cffi-src', + command: [ + gen_cffi_src, + 'exec-python', + '@INPUT@', + '@OUTPUT@', + ], + output: '_squared.c', + input: ['src/squared/_squared_build.py'], + ) + + py.extension_module( + '_squared', + squared_ext_src, + subdir: 'squared', + install: true, + dependencies: [square_dep, py.dependency()], + ) + +``src/squared/__init__.py``: + +.. code-block:: python + + from ._squared import ffi, lib + + + def squared(n): + return lib.square(n) + +``src/squared/_squared_build.py``, ``src/csrc/square.h`` and +``src/csrc/square.c`` contain the snippets shown above. + +Build and install the project with any PEP 517 front-end. For +example: + +.. code-block:: console + + $ python -m pip install . + $ python -c "from squared import squared; print(squared(7))" + 49 + +To switch this project to ``read-sources`` mode, replace +``_squared_build.py`` with two files (``_squared.cdef.txt`` and +``_squared.csrc.c``), then change the ``custom_target`` command to: + +.. code-block:: meson + + command: [ + gen_cffi_src, + 'read-sources', + 'squared._squared', + '@INPUT0@', + '@INPUT1@', + '@OUTPUT@', + ], + +and list both files under ``input:``: + +.. code-block:: meson + + input: ['src/squared/_squared.cdef.txt', '_squared.csrc.c'] + +Distributing CFFI Extensions using Setuptools +============================================= + +.. _distutils-setuptools: + + You can (but don't have to) use CFFI's **Distutils** or + **Setuptools integration** when writing a ``setup.py``. For + Distutils (only in out-of-line API mode; deprecated since + Python 3.10): + + .. code-block:: python + + # setup.py (requires CFFI to be installed first) + from distutils.core import setup + + import foo_build # possibly with sys.path tricks to find it + + setup( + ..., + ext_modules=[foo_build.ffibuilder.distutils_extension()], + ) + + For Setuptools (out-of-line only, but works in ABI or API mode; + recommended): + + .. code-block:: python + + # setup.py (with automatic dependency tracking) + from setuptools import setup + + setup( + ..., + setup_requires=["cffi>=1.0.0"], + cffi_modules=["package/foo_build.py:ffibuilder"], + install_requires=["cffi>=1.0.0"], + ) + + Note again that the ``foo_build.py`` example contains the following + lines, which mean that the ``ffibuilder`` is not actually compiled + when ``package.foo_build`` is merely imported---it will be compiled + independently by the Setuptools logic, using compilation parameters + provided by Setuptools: + + .. code-block:: python + + if __name__ == "__main__": # not when running with setuptools + ffibuilder.compile(verbose=True) + +* Note that some bundler tools that try to find all modules used by a + project, like PyInstaller, will miss ``_cffi_backend`` in the + out-of-line mode because your program contains no explicit ``import + cffi`` or ``import _cffi_backend``. You need to add + ``_cffi_backend`` explicitly (as a "hidden import" in PyInstaller, + but it can also be done more generally by adding the line ``import + _cffi_backend`` in your main program). + +Note that CFFI actually contains two different ``FFI`` classes. The +page `Using the ffi/lib objects`_ describes the common functionality. +It is what you get in the ``from package._foo import ffi`` lines above. +On the other hand, the extended ``FFI`` class is the one you get from +``import cffi; ffi_or_ffibuilder = cffi.FFI()``. It has the same +functionality (for in-line use), but also the extra methods described +below (to prepare the FFI). NOTE: We use the name ``ffibuilder`` +instead of ``ffi`` in the out-of-line context, when the code is about +producing a ``_foo.so`` file; this is an attempt to distinguish it +from the different ``ffi`` object that you get by later saying +``from _foo import ffi``. + +.. _`Using the ffi/lib objects`: using.html + +The reason for this split of functionality is that a regular program +using CFFI out-of-line does not need to import the ``cffi`` pure +Python package at all. (Internally it still needs ``_cffi_backend``, +a C extension module that comes with CFFI; this is why CFFI is also +listed in ``install_requires=..`` above. In the future this might be +split into a different PyPI package that only installs +``_cffi_backend``.) + +Note that a few small differences do exist: notably, ``from _foo import +ffi`` returns an object of a type written in C, which does not let you +add random attributes to it (nor does it have all the +underscore-prefixed internal attributes of the Python version). +Similarly, the ``lib`` objects returned by the C version are read-only, +apart from writes to global variables. Also, ``lib.__dict__`` does +not work before version 1.2 or if ``lib`` happens to declare a name +called ``__dict__`` (use instead ``dir(lib)``). The same is true +for ``lib.__class__``, ``lib.__all__`` and ``lib.__name__`` added +in successive versions. diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst index 15be27fa..9a18dc9a 100644 --- a/doc/source/cdef.rst +++ b/doc/source/cdef.rst @@ -1,9 +1,15 @@ -====================================== -Preparing and Distributing modules -====================================== +========================= +Preparing Wrapper Modules +========================= .. contents:: +.. note:: + + This covers how to create wrapper modules. See :ref:`buildtool_docs` + for instructions on how to integrate with a Python build backend and + distribute wrapper modules. + There are three or four different ways to use CFFI in a project. In order of complexity: @@ -76,92 +82,6 @@ In order of complexity: # use ffi and lib here -.. _distutils-setuptools: - -* Finally, you can (but don't have to) use CFFI's **Distutils** or - **Setuptools integration** when writing a ``setup.py``. For - Distutils (only in out-of-line API mode; deprecated since - Python 3.10): - - .. code-block:: python - - # setup.py (requires CFFI to be installed first) - from distutils.core import setup - - import foo_build # possibly with sys.path tricks to find it - - setup( - ..., - ext_modules=[foo_build.ffibuilder.distutils_extension()], - ) - - For Setuptools (out-of-line only, but works in ABI or API mode; - recommended): - - .. code-block:: python - - # setup.py (with automatic dependency tracking) - from setuptools import setup - - setup( - ..., - setup_requires=["cffi>=1.0.0"], - cffi_modules=["package/foo_build.py:ffibuilder"], - install_requires=["cffi>=1.0.0"], - ) - - Note again that the ``foo_build.py`` example contains the following - lines, which mean that the ``ffibuilder`` is not actually compiled - when ``package.foo_build`` is merely imported---it will be compiled - independently by the Setuptools logic, using compilation parameters - provided by Setuptools: - - .. code-block:: python - - if __name__ == "__main__": # not when running with setuptools - ffibuilder.compile(verbose=True) - -* Note that some bundler tools that try to find all modules used by a - project, like PyInstaller, will miss ``_cffi_backend`` in the - out-of-line mode because your program contains no explicit ``import - cffi`` or ``import _cffi_backend``. You need to add - ``_cffi_backend`` explicitly (as a "hidden import" in PyInstaller, - but it can also be done more generally by adding the line ``import - _cffi_backend`` in your main program). - -Note that CFFI actually contains two different ``FFI`` classes. The -page `Using the ffi/lib objects`_ describes the common functionality. -It is what you get in the ``from package._foo import ffi`` lines above. -On the other hand, the extended ``FFI`` class is the one you get from -``import cffi; ffi_or_ffibuilder = cffi.FFI()``. It has the same -functionality (for in-line use), but also the extra methods described -below (to prepare the FFI). NOTE: We use the name ``ffibuilder`` -instead of ``ffi`` in the out-of-line context, when the code is about -producing a ``_foo.so`` file; this is an attempt to distinguish it -from the different ``ffi`` object that you get by later saying -``from _foo import ffi``. - -.. _`Using the ffi/lib objects`: using.html - -The reason for this split of functionality is that a regular program -using CFFI out-of-line does not need to import the ``cffi`` pure -Python package at all. (Internally it still needs ``_cffi_backend``, -a C extension module that comes with CFFI; this is why CFFI is also -listed in ``install_requires=..`` above. In the future this might be -split into a different PyPI package that only installs -``_cffi_backend``.) - -Note that a few small differences do exist: notably, ``from _foo import -ffi`` returns an object of a type written in C, which does not let you -add random attributes to it (nor does it have all the -underscore-prefixed internal attributes of the Python version). -Similarly, the ``lib`` objects returned by the C version are read-only, -apart from writes to global variables. Also, ``lib.__dict__`` does -not work before version 1.2 or if ``lib`` happens to declare a name -called ``__dict__`` (use instead ``dir(lib)``). The same is true -for ``lib.__class__``, ``lib.__all__`` and ``lib.__name__`` added -in successive versions. - .. _cdef: @@ -935,10 +855,10 @@ steps. and *if* the "stuff" part is big enough that import time is a concern, then rewrite it as described in `the out-of-line but still ABI mode`__ -above. Optionally, see also the `setuptools integration`__ paragraph. +above. Optionally, see also the :ref:`build backend and distrubution +` documentation. .. __: out-of-line-abi_ -.. __: distutils-setuptools_ **API mode** if your CFFI project uses ``ffi.verify()``: @@ -951,16 +871,15 @@ above. Optionally, see also the `setuptools integration`__ paragraph. ffi.cdef("stuff") lib = ffi.verify("real C code") -then you should really rewrite it as described in `the out-of-line, -API mode`__ above. It avoids a number of issues that have caused +then you should really rewrite it as described in `the out-of-line, API +mode`__ above. It avoids a number of issues that have caused ``ffi.verify()`` to grow a number of extra arguments over time. Then -see the `distutils or setuptools`__ paragraph. Also, remember to -remove the ``ext_package=".."`` from your ``setup.py``, which was -sometimes needed with ``verify()`` but is just creating confusion with -``set_source()``. +see the :ref:`build backend and distrubution ` +documentation. Also, remember to remove the ``ext_package=".."`` from +your ``setup.py``, which was sometimes needed with ``verify()`` but is +just creating confusion with ``set_source()``. .. __: out-of-line-api_ -.. __: distutils-setuptools_ The following example should work both with old (pre-1.0) and new versions of CFFI---supporting both is important to run on old diff --git a/doc/source/index.rst b/doc/source/index.rst index 54934f22..f5939d0e 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -16,4 +16,5 @@ copy-paste from header files or documentation. using ref cdef + buildtool embedding diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst index db9352d0..df7755ba 100644 --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -10,8 +10,16 @@ v2.0.0.dev0 with the limited API, so you must set py_limited_api=False when building extensions for the free-threaded build. * Added support for Python 3.14. (`#177`_) +* Added the :mod:`cffi.buildtool` subpackage and the ``gen-cffi-src`` + command-line tool, which let build backends such as `meson-python`_ + generate CFFI extension C source without depending on setuptools. + See :doc:`buildtool` for details. Integrated from the + `cffi-buildtool`_ project by Rose Davidson. * WIP +.. _`meson-python`: https://meson-python.readthedocs.io/ +.. _`cffi-buildtool`: https://github.com/inklesspen/cffi-buildtool + .. _`#177`: https://github.com/python-cffi/cffi/pull/177 .. _`#178`: https://github.com/python-cffi/cffi/pull/178 diff --git a/pyproject.toml b/pyproject.toml index 447621fb..0f2ac4cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,9 @@ maintainers = [ [project.entry-points."distutils.setup_keywords"] cffi_modules = "cffi.setuptools_ext:cffi_modules" +[project.scripts] +gen-cffi-src = "cffi.buildtool._cli:run" + [project.urls] Documentation = "https://cffi.readthedocs.io/" Changelog = "https://cffi.readthedocs.io/en/latest/whatsnew.html" diff --git a/setup.py b/setup.py index c75812c2..29ea5be2 100644 --- a/setup.py +++ b/setup.py @@ -183,7 +183,7 @@ def has_ext_modules(self): cpython = ('_cffi_backend' not in sys.builtin_module_names) setup( - packages=['cffi'] if cpython else [], + packages=['cffi', 'cffi.buildtool'] if cpython else [], package_dir={"": "src"}, package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', '_embedding.h', '_cffi_errors.h']} diff --git a/src/cffi/buildtool/__init__.py b/src/cffi/buildtool/__init__.py new file mode 100644 index 00000000..597777d2 --- /dev/null +++ b/src/cffi/buildtool/__init__.py @@ -0,0 +1,26 @@ +"""Helpers for generating CFFI C source without invoking external dependencies. + +This subpackage exposes a small API and a command-line entry point +(``gen-cffi-src``) that build backends can invoke during a build to +produce the ``.c`` source file for a CFFI extension module.: + +* :func:`find_ffi_in_python_script` -- execute an "exec-python" build + script and return the :class:`cffi.FFI` object it defines. +* :func:`make_ffi_from_sources` -- construct an :class:`cffi.FFI` + from a ``cdef`` string and a C source prelude. +* :func:`generate_c_source` -- emit the generated C source for an + :class:`cffi.FFI` as a string. + +""" + +from ._gen import ( + find_ffi_in_python_script, + generate_c_source, + make_ffi_from_sources, +) + +__all__ = [ + 'find_ffi_in_python_script', + 'generate_c_source', + 'make_ffi_from_sources', +] diff --git a/src/cffi/buildtool/__main__.py b/src/cffi/buildtool/__main__.py new file mode 100644 index 00000000..21c97640 --- /dev/null +++ b/src/cffi/buildtool/__main__.py @@ -0,0 +1,18 @@ +# Integrated from the cffi-buildtool project by Rose Davidson +# (https://github.com/inklesspen/cffi-buildtool), MIT-licensed. +""" +Entrypoint module, in case you use `python -m cffi.buildtool`. + + +Why does this file exist, and why __main__? For more info, read: + +- https://www.python.org/dev/peps/pep-0338/ +- https://docs.python.org/2/using/cmdline.html#cmdoption-m +- https://docs.python.org/3/using/cmdline.html#cmdoption-m +""" + + +from ._cli import run + +if __name__ == '__main__': + run() diff --git a/src/cffi/buildtool/_cli.py b/src/cffi/buildtool/_cli.py new file mode 100644 index 00000000..94c414f0 --- /dev/null +++ b/src/cffi/buildtool/_cli.py @@ -0,0 +1,110 @@ +# Integrated from the cffi-buildtool project by Rose Davidson +# (https://github.com/inklesspen/cffi-buildtool), MIT-licensed. +"""Command-line entry point for ``gen-cffi-src``. + +Two subcommands: + +``exec-python`` + Execute a Python build script that constructs a :class:`cffi.FFI` + (the same kind of script that the CFFI docs' "Main mode of usage" + describes) and emit the generated C source. + +``read-sources`` + Build the :class:`cffi.FFI` from a separate ``cdef`` file and C + source prelude, then emit the generated C source. +""" + +import argparse + +from ._gen import ( + find_ffi_in_python_script, + generate_c_source, + make_ffi_from_sources, +) + + +def exec_python(*, output, pyfile, ffi_var): + with pyfile: + ffi = find_ffi_in_python_script(pyfile.read(), pyfile.name, ffi_var) + generated = generate_c_source(ffi) + with output: + output.write(generated) + + +def read_sources(*, output, module_name, cdef_input, csrc_input): + with csrc_input, cdef_input: + csrc = csrc_input.read() + cdef = cdef_input.read() + ffi = make_ffi_from_sources(module_name, cdef, csrc) + generated = generate_c_source(ffi) + with output: + output.write(generated) + + +parser = argparse.ArgumentParser( + prog='gen-cffi-src', + description='Generate CFFI C source for a build backend (e.g. meson-python).', +) +subparsers = parser.add_subparsers(dest='mode') + +exec_python_parser = subparsers.add_parser( + 'exec-python', + help='Execute a Python script to build an FFI object', +) +exec_python_parser.add_argument( + '--ffi-var', + default='ffibuilder', + help="Name of the FFI object in the Python script; defaults to 'ffibuilder'.", +) +exec_python_parser.add_argument( + 'pyfile', + type=argparse.FileType('r', encoding='utf-8'), + help='Path to the Python script', +) +exec_python_parser.add_argument( + 'output', + type=argparse.FileType('w', encoding='utf-8'), + help='Output path for the C source', +) + +read_sources_parser = subparsers.add_parser( + 'read-sources', + help='Read cdef and C source prelude files to build an FFI object', +) +read_sources_parser.add_argument( + 'module_name', + help='Full name of the generated module, including packages', +) +read_sources_parser.add_argument( + 'cdef', + type=argparse.FileType('r', encoding='utf-8'), + help='File containing C definitions', +) +read_sources_parser.add_argument( + 'csrc', + type=argparse.FileType('r', encoding='utf-8'), + help='File containing C source prelude', +) +read_sources_parser.add_argument( + 'output', + type=argparse.FileType('w', encoding='utf-8'), + help='Output path for the C source', +) + + +def run(args=None): + args = parser.parse_args(args=args) + if args.mode == 'exec-python': + exec_python(output=args.output, pyfile=args.pyfile, ffi_var=args.ffi_var) + elif args.mode == 'read-sources': + if args.cdef is args.csrc: + parser.error('cdef and csrc are the same file and should not be') + read_sources( + output=args.output, + module_name=args.module_name, + cdef_input=args.cdef, + csrc_input=args.csrc, + ) + else: + parser.error('a subcommand is required: exec-python or read-sources') + parser.exit(0) diff --git a/src/cffi/buildtool/_gen.py b/src/cffi/buildtool/_gen.py new file mode 100644 index 00000000..4ad2070f --- /dev/null +++ b/src/cffi/buildtool/_gen.py @@ -0,0 +1,58 @@ +# Integrated from the cffi-buildtool project by Rose Davidson +# (https://github.com/inklesspen/cffi-buildtool), MIT-licensed. +import io + +from ..api import FFI + + +def _execfile(pysrc, filename, globs): + compiled = compile(source=pysrc, filename=filename, mode='exec') + exec(compiled, globs, globs) + + +def find_ffi_in_python_script(pysrc, filename, ffivar): + """Execute ``pysrc`` and return the :class:`FFI` object it defines. + + The script is executed with ``__name__`` set to ``"gen-cffi-src"``, + so a trailing ``if __name__ == "__main__": ffibuilder.compile()`` + block in the script is skipped. + + ``ffivar`` is the name bound by the script to the :class:`FFI` + object, or to a callable that returns one. + + Raises :class:`NameError` if the name is not bound by the script, + or :class:`TypeError` if the name does not resolve to an + :class:`FFI` instance. + """ + globs = {'__name__': 'gen-cffi-src'} + _execfile(pysrc, filename, globs) + if ffivar not in globs: + raise NameError( + "Expected to find the FFI object with the name %r, " + "but it was not found." % (ffivar,) + ) + ffi = globs[ffivar] + if not isinstance(ffi, FFI) and callable(ffi): + # Maybe it's a callable that returns a FFI + ffi = ffi() + if not isinstance(ffi, FFI): + raise TypeError( + "Found an object with the name %r but it was not an " + "instance of cffi.api.FFI" % (ffivar,) + ) + return ffi + + +def make_ffi_from_sources(modulename, cdef, csrc): + """Build an :class:`FFI` from ``cdef`` text and a C source prelude.""" + ffibuilder = FFI() + ffibuilder.cdef(cdef) + ffibuilder.set_source(modulename, csrc) + return ffibuilder + + +def generate_c_source(ffi): + """Return the C source that :meth:`FFI.emit_c_code` would write.""" + output = io.StringIO() + ffi.emit_c_code(output) + return output.getvalue() diff --git a/testing/cffi1/buildtool_example/meson.build b/testing/cffi1/buildtool_example/meson.build new file mode 100644 index 00000000..a0c15107 --- /dev/null +++ b/testing/cffi1/buildtool_example/meson.build @@ -0,0 +1,42 @@ +project( + 'squared', + 'c', + version: '0.1.0', + default_options: ['warning_level=2'], +) + +py = import('python').find_installation(pure: false) + +install_subdir('src/squared', install_dir: py.get_install_dir()) + +gen_cffi_src = find_program('gen-cffi-src') + +square_lib = static_library( + 'square', + 'src/csrc/square.c', + include_directories: include_directories('src/csrc'), +) +square_dep = declare_dependency( + link_with: square_lib, + include_directories: include_directories('src/csrc'), +) + +squared_ext_src = custom_target( + 'squared-cffi-src', + command: [ + gen_cffi_src, + 'exec-python', + '@INPUT@', + '@OUTPUT@', + ], + output: '_squared.c', + input: ['src/squared/_squared_build.py'], +) + +py.extension_module( + '_squared', + squared_ext_src, + subdir: 'squared', + install: true, + dependencies: [square_dep, py.dependency()], +) diff --git a/testing/cffi1/buildtool_example/pyproject.toml b/testing/cffi1/buildtool_example/pyproject.toml new file mode 100644 index 00000000..45b21900 --- /dev/null +++ b/testing/cffi1/buildtool_example/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +build-backend = 'mesonpy' +requires = ['meson-python', 'cffi'] + +[project] +name = 'squared' +version = '0.1.0' +description = 'Small self-contained example project that builds a CFFI extension via meson-python.' +requires-python = '>=3.9' +dependencies = ['cffi'] diff --git a/testing/cffi1/buildtool_example/src/csrc/square.c b/testing/cffi1/buildtool_example/src/csrc/square.c new file mode 100644 index 00000000..05d6092f --- /dev/null +++ b/testing/cffi1/buildtool_example/src/csrc/square.c @@ -0,0 +1,5 @@ +#include "square.h" + +int square(int n) { + return n * n; +} diff --git a/testing/cffi1/buildtool_example/src/csrc/square.h b/testing/cffi1/buildtool_example/src/csrc/square.h new file mode 100644 index 00000000..356a5771 --- /dev/null +++ b/testing/cffi1/buildtool_example/src/csrc/square.h @@ -0,0 +1,6 @@ +#ifndef SQUARE_H +#define SQUARE_H + +int square(int n); + +#endif diff --git a/testing/cffi1/buildtool_example/src/squared/__init__.py b/testing/cffi1/buildtool_example/src/squared/__init__.py new file mode 100644 index 00000000..156e67d5 --- /dev/null +++ b/testing/cffi1/buildtool_example/src/squared/__init__.py @@ -0,0 +1,5 @@ +from ._squared import ffi, lib + + +def squared(n): + return lib.square(n) diff --git a/testing/cffi1/buildtool_example/src/squared/_squared_build.py b/testing/cffi1/buildtool_example/src/squared/_squared_build.py new file mode 100644 index 00000000..f44e92c5 --- /dev/null +++ b/testing/cffi1/buildtool_example/src/squared/_squared_build.py @@ -0,0 +1,13 @@ +from cffi import FFI + +ffibuilder = FFI() + +ffibuilder.cdef("int square(int n);") + +ffibuilder.set_source( + "squared._squared", + '#include "square.h"', +) + +if __name__ == "__main__": + ffibuilder.compile(verbose=True) diff --git a/testing/cffi1/buildtool_example2/meson.build b/testing/cffi1/buildtool_example2/meson.build new file mode 100644 index 00000000..64491047 --- /dev/null +++ b/testing/cffi1/buildtool_example2/meson.build @@ -0,0 +1,44 @@ +project( + 'squared', + 'c', + version: '0.1.0', + default_options: ['warning_level=2'], +) + +py = import('python').find_installation(pure: false) + +install_subdir('src/squared', install_dir: py.get_install_dir()) + +gen_cffi_src = find_program('gen-cffi-src') + +square_lib = static_library( + 'square', + 'src/csrc/square.c', + include_directories: include_directories('src/csrc'), +) +square_dep = declare_dependency( + link_with: square_lib, + include_directories: include_directories('src/csrc'), +) + +squared_ext_src = custom_target( + 'squared-cffi-src', + command: [ + gen_cffi_src, + 'read-sources', + 'squared._squared', + '@INPUT0@', + '@INPUT1@', + '@OUTPUT@', + ], + output: '_squared.c', + input: ['src/squared/squared.cdef.txt', 'src/squared/squared.csrc.c'], +) + +py.extension_module( + '_squared', + squared_ext_src, + subdir: 'squared', + install: true, + dependencies: [square_dep, py.dependency()], +) diff --git a/testing/cffi1/buildtool_example2/pyproject.toml b/testing/cffi1/buildtool_example2/pyproject.toml new file mode 100644 index 00000000..45b21900 --- /dev/null +++ b/testing/cffi1/buildtool_example2/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +build-backend = 'mesonpy' +requires = ['meson-python', 'cffi'] + +[project] +name = 'squared' +version = '0.1.0' +description = 'Small self-contained example project that builds a CFFI extension via meson-python.' +requires-python = '>=3.9' +dependencies = ['cffi'] diff --git a/testing/cffi1/buildtool_example2/src/csrc/square.c b/testing/cffi1/buildtool_example2/src/csrc/square.c new file mode 100644 index 00000000..05d6092f --- /dev/null +++ b/testing/cffi1/buildtool_example2/src/csrc/square.c @@ -0,0 +1,5 @@ +#include "square.h" + +int square(int n) { + return n * n; +} diff --git a/testing/cffi1/buildtool_example2/src/csrc/square.h b/testing/cffi1/buildtool_example2/src/csrc/square.h new file mode 100644 index 00000000..356a5771 --- /dev/null +++ b/testing/cffi1/buildtool_example2/src/csrc/square.h @@ -0,0 +1,6 @@ +#ifndef SQUARE_H +#define SQUARE_H + +int square(int n); + +#endif diff --git a/testing/cffi1/buildtool_example2/src/squared/__init__.py b/testing/cffi1/buildtool_example2/src/squared/__init__.py new file mode 100644 index 00000000..156e67d5 --- /dev/null +++ b/testing/cffi1/buildtool_example2/src/squared/__init__.py @@ -0,0 +1,5 @@ +from ._squared import ffi, lib + + +def squared(n): + return lib.square(n) diff --git a/testing/cffi1/buildtool_example2/src/squared/squared.cdef.txt b/testing/cffi1/buildtool_example2/src/squared/squared.cdef.txt new file mode 100644 index 00000000..72afcbd7 --- /dev/null +++ b/testing/cffi1/buildtool_example2/src/squared/squared.cdef.txt @@ -0,0 +1 @@ +int square(int n); \ No newline at end of file diff --git a/testing/cffi1/buildtool_example2/src/squared/squared.csrc.c b/testing/cffi1/buildtool_example2/src/squared/squared.csrc.c new file mode 100644 index 00000000..930ea7ee --- /dev/null +++ b/testing/cffi1/buildtool_example2/src/squared/squared.csrc.c @@ -0,0 +1 @@ +#include "square.h" diff --git a/testing/cffi1/test_buildtool.py b/testing/cffi1/test_buildtool.py new file mode 100644 index 00000000..1792060f --- /dev/null +++ b/testing/cffi1/test_buildtool.py @@ -0,0 +1,131 @@ +import pytest + +from cffi.buildtool import ( + find_ffi_in_python_script, + generate_c_source, + make_ffi_from_sources, +) +from cffi.buildtool import _cli as buildtool_cli + + +SIMPLE_SCRIPT = """\ +from cffi import FFI + +ffibuilder = FFI() + +ffibuilder.cdef("int square(int n);") + +ffibuilder.set_source("squared._squared", '#include "square.h"') + +something_else = 42 + +if __name__ == "__main__": + ffibuilder.compile(verbose=True) +""" + +CALLABLE_SCRIPT = """\ +from cffi import FFI + +def make_ffi(): + ffibuilder = FFI() + ffibuilder.cdef("int square(int n);") + ffibuilder.set_source("squared._squared", '#include "square.h"') + return ffibuilder + +def something_else(): + return 42 +""" + + +def _dont_exit(_status): + pass + + +def test_find_ffi_simple(): + ffi = find_ffi_in_python_script(SIMPLE_SCRIPT, "_squared_build.py", "ffibuilder") + module_name, csrc, _source_extension, _kwds = ffi._assigned_source + assert module_name == "squared._squared" + assert csrc.strip() == '#include "square.h"' + cdef = "\n".join(ffi._cdefsources) + assert "int square" in cdef + + +def test_find_ffi_callable(): + ffi = find_ffi_in_python_script(CALLABLE_SCRIPT, "_squared_build.py", "make_ffi") + module_name, csrc, _source_extension, _kwds = ffi._assigned_source + assert module_name == "squared._squared" + assert csrc.strip() == '#include "square.h"' + cdef = "\n".join(ffi._cdefsources) + assert "int square" in cdef + + +def test_find_ffi_name_not_found(): + with pytest.raises(NameError, match="'notfound'"): + find_ffi_in_python_script(SIMPLE_SCRIPT, "_squared_build.py", "notfound") + + +def test_find_ffi_wrong_type(): + with pytest.raises(TypeError, match="not an instance of cffi.api.FFI"): + find_ffi_in_python_script(SIMPLE_SCRIPT, "_squared_build.py", "something_else") + + +def test_find_ffi_callable_wrong_type(): + with pytest.raises(TypeError, match="not an instance of cffi.api.FFI"): + find_ffi_in_python_script(CALLABLE_SCRIPT, "_squared_build.py", "something_else") + + +def test_make_ffi_from_sources_and_generate(): + ffi = make_ffi_from_sources( + "squared._squared", + "int square(int n);", + '#include "square.h"', + ) + c_source = generate_c_source(ffi) + assert "square" in c_source + # sanity check: the emitted C source is the thing meson-python will compile + assert "PyInit" in c_source or "_cffi_f_" in c_source + + +def test_cli_exec_python(tmp_path, monkeypatch): + monkeypatch.setattr(buildtool_cli.parser, "exit", _dont_exit) + pyfile = tmp_path / "_squared_build.py" + pyfile.write_text(SIMPLE_SCRIPT) + output = tmp_path / "out.c" + buildtool_cli.run(["exec-python", str(pyfile), str(output)]) + generated = output.read_text() + assert "square" in generated + + +def test_cli_exec_python_ffi_var(tmp_path, monkeypatch): + monkeypatch.setattr(buildtool_cli.parser, "exit", _dont_exit) + pyfile = tmp_path / "_squared_build.py" + pyfile.write_text(CALLABLE_SCRIPT) + output = tmp_path / "out.c" + buildtool_cli.run(["exec-python", "--ffi-var", "make_ffi", str(pyfile), str(output)]) + generated = output.read_text() + assert "square" in generated + + +def test_cli_read_sources(tmp_path, monkeypatch): + monkeypatch.setattr(buildtool_cli.parser, "exit", _dont_exit) + cdef = tmp_path / "squared.cdef.txt" + cdef.write_text("int square(int n);\n") + csrc = tmp_path / "squared.csrc.c" + csrc.write_text('#include "square.h"\n') + output = tmp_path / "out.c" + buildtool_cli.run([ + "read-sources", + "squared._squared", + str(cdef), + str(csrc), + str(output), + ]) + generated = output.read_text() + assert "square" in generated + + +def test_cli_read_sources_same_input_fails(capsys): + with pytest.raises(SystemExit): + buildtool_cli.run(["read-sources", "_squared", "-", "-", "-"]) + _stdout, stderr = capsys.readouterr() + assert "are the same file and should not be" in stderr diff --git a/testing/cffi1/test_buildtool_meson.py b/testing/cffi1/test_buildtool_meson.py new file mode 100644 index 00000000..fa086c70 --- /dev/null +++ b/testing/cffi1/test_buildtool_meson.py @@ -0,0 +1,81 @@ +"""End-to-end test: build a self-contained CFFI extension with meson-python. + +The test provisions a fresh nested venv under ``tmp_path`` using the +stdlib :mod:`venv` module, installs ``cffi`` (from the current source +tree) and ``meson-python`` into it, installs the small example project +that lives under ``testing/cffi1/buildtool_example/``, and then +imports the built extension to confirm it works. + +The test does not use ``uv``. It only relies on the running Python +interpreter having access to ``pip`` (which is true for any venv +created by :mod:`venv`) and on a working C compiler being on ``PATH``. +""" + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +import cffi + +pytestmark = [ + pytest.mark.thread_unsafe(reason="spawns subprocesses, slow"), +] + +try: + import mesonpy +except ImportError: + pytest.skip("Test requires meson-python", allow_module_level=True) + + +HERE = Path(__file__).resolve().parent +EXAMPLE_PROJECT = HERE / "buildtool_example" +EXAMPLE_PROJECT2 = HERE / "buildtool_example2" +CFFI_DIR = HERE.parent.parent + + +def _venv_python(venv_dir): + if sys.platform == "win32": + return venv_dir / "Scripts" / "python.exe" + return venv_dir / "bin" / "python" + + +@pytest.mark.parametrize("project", [EXAMPLE_PROJECT, EXAMPLE_PROJECT2]) +def test_meson_python_build(tmp_path, project): + venv_dir = tmp_path / "venv" + subprocess.check_call([sys.executable, "-m", "venv", str(venv_dir)]) + venv_python = _venv_python(venv_dir) + assert venv_python.exists(), venv_python + + # Upgrade pip so --no-build-isolation behaves consistently with recent + # resolver behaviour on older base images. + subprocess.check_call([ + str(venv_python), "-m", "pip", "install", "--upgrade", "pip", + ]) + + # Install build-time deps into the nested venv. + subprocess.check_call([ + str(venv_python), "-m", "pip", "install", "meson-python", CFFI_DIR + ]) + + # Copy the example project so nothing is written back into the + # source tree + project_dir = tmp_path / "project" + shutil.copytree(project, project_dir) + + # --no-build-isolation to ensure the test runs against the CFFI build we want to test + subprocess.check_call([ + str(venv_python), "-m", "pip", "install", + "--no-build-isolation", str(project_dir), + ]) + + # Confirm the built extension imports and behaves as expected. + subprocess.check_call([ + str(venv_python), "-c", + "from squared import squared; " + "assert squared(7) == 49; " + "assert squared(-3) == 9", + ])