Skip to content

Commit d680401

Browse files
committed
Add python stubs to built wheels
Adds support for IDE type completion and static type analysis Signed-off-by: Chad Dombrova <chadrik@gmail.com>
1 parent 6c15d45 commit d680401

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+3175
-134
lines changed

.github/workflows/wheel_workflow.yml

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,14 +144,21 @@ jobs:
144144
- name: Build wheels
145145
uses: pypa/cibuildwheel@v2.22.0
146146
env:
147+
# pass GITHUB_ACTIONS through to the build container so that custom
148+
# processes can tell they are running in CI.
149+
CIBW_ENVIRONMENT_PASS_LINUX: GITHUB_ACTIONS
147150
CIBW_BUILD: ${{ matrix.python }}
148151
CIBW_ARCHS: ${{ matrix.arch }}
149152
CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux }}
150153

151154
- uses: actions/upload-artifact@v4
152155
with:
153156
name: cibw-wheels-${{ matrix.python }}-${{ matrix.manylinux }}
154-
path: ./wheelhouse/*.whl
157+
path: |
158+
./wheelhouse/*.whl
159+
./wheelhouse/PyOpenColorIO/__init__.pyi
160+
# if stub validation fails we want to upload the stubs for users to review
161+
if: success() || failure()
155162

156163
# ---------------------------------------------------------------------------
157164
# Linux ARM Wheels
@@ -213,7 +220,11 @@ jobs:
213220
- uses: actions/upload-artifact@v4
214221
with:
215222
name: cibw-wheels-${{ matrix.python }}-${{ matrix.manylinux }}
216-
path: ./wheelhouse/*.whl
223+
path: |
224+
./wheelhouse/*.whl
225+
./wheelhouse/PyOpenColorIO/__init__.pyi
226+
# if stub validation fails we want to upload the stubs for users to review
227+
if: success() || failure()
217228

218229
# ---------------------------------------------------------------------------
219230
# macOS Wheels

docs/quick_start/installation.rst

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -536,3 +536,51 @@ Note: For other user facing environment variables, see :ref:`using_env_vars`.
536536
for Windows, Linux and macOS respectively. For systems that supports it,
537537
it is also possible to edit the RPATH of the plugin to add the location of
538538
the shared OpenColorIO lib.
539+
540+
Updating python type stubs
541+
==========================
542+
543+
The python stubs are built by analyzing the docstrings generated by pybind11
544+
and are used to drive IDE code completion and static analysis. Changes to
545+
the bindings that result in changes to the stubs should be reviewed together
546+
in a single pull request, so that regressions are not unwittingly introduced.
547+
548+
The workflow looks like this:
549+
550+
- Install `uv <https://docs.astral.sh/uv/getting-started/installation/>`_ and
551+
``docker``
552+
- Create the temp build directory, cd into it, and run ``cmake /path/to/source``
553+
- Run ``cmake --build . --target pystubs`` to generate updated stubs in
554+
``src/bindings/python/stubs/__init__.pyi``
555+
- Commit the new stubs and push to Github
556+
- In CI, the stubs will be included in the wheels built by ``cibuildwheel``, as
557+
defined in ``.github/workflows/wheel_workflow.yml``
558+
- In CI, one of the ``cibuildwheel`` Github actions will rebuild the stubs to a
559+
temp location and verify that they match what has been committed to the repo.
560+
This step ensures that if changes to the C++ source code and bindings results
561+
in a change to the stubs, developers are notified of the need to regenerate
562+
the stubs, so that changes can be reviewed and the rules in ``generate_stubs.py``
563+
can be updated, if necessary.
564+
565+
If you want to bypass the use of docker, you can use the following steps to
566+
build the python bindings and run the stub generator directly (requires python
567+
3.11+):
568+
569+
.. code-block:: bash
570+
571+
python -m venv .venv
572+
. .venv/bin/activate
573+
mkdir build
574+
cd build
575+
cmake -DPython_EXECUTABLE=$(which python) -DCMAKE_INSTALL_PREFIX=../dist ..
576+
cmake --build . --target=install
577+
# extract the deps from the pyproject.toml and install them into the virtualenv
578+
PY_DEPS=$(python -c "import tomllib;print(tomllib.load(open('../pyproject.toml', 'rb'))['tool']['cibuildwheel']['overrides'][0]['test-requires'])")
579+
pip install ${PY_DEPS}
580+
PY_VER=$(python -c "import sys;print(f'{sys.version_info[0]}.{sys.version_info[1]}')")
581+
# generate stubs
582+
PYTHONPATH=../dist/lib/python$PY_VER/site-packages python ../src/bindings/python/stubs/generate_stubs.py
583+
584+
If all else fails and you can't build the stubs locally, you can push changes
585+
to CI and download an artifact containing the wheel and ``__init__.pyi`` file from
586+
any job that fails the stub validation.

pyproject.toml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,3 +40,33 @@ before-build = "share/ci/scripts/macos/install_docs_env.sh"
4040
# Disable for now as wheels are built with static dependencies.
4141
environment = { PATH="$GITHUB_WORKSPACE/doxygen;$PATH", OCIO_PYTHON_LOAD_DLLS_FROM_PATH="0" }
4242
before-build = 'bash -c "share/ci/scripts/windows/install_docs_env.sh $GITHUB_WORKSPACE/doxygen"'
43+
44+
[[tool.cibuildwheel.overrides]]
45+
# Trigger the build & validation of the python stubs for certain platforms.
46+
# The test command acts as a kind of "post-build" callback where it's possible
47+
# for the stub-generator to import the freshly-built wheel.
48+
# There are two entry-points which are designed to call generate_stubs.py through
49+
# this test command:
50+
# - `cmake /path/to/source; cmake --build . --target pystubs` is called during
51+
# local development to generate the stubs and copy them into the git repo to
52+
# be committed and reviewed.
53+
# - in CI, the cibuildwheel action is used to validate that the stubs match what
54+
# has been committed to the repo.
55+
test-requires = "mypy~=1.15.0 stubgenlib~=0.2.0"
56+
# Note: the python version here must be kept in sync with src/bindings/python/CMakeLists.txt
57+
select = "cp311-manylinux_*64"
58+
inherit.test-command = "append"
59+
test-command = [
60+
"python {project}/src/bindings/python/stubs/generate_stubs.py --out-path '/output' --validate-path '{project}/src/bindings/python/stubs/PyOpenColorIO/__init__.pyi'",
61+
]
62+
63+
[tool.mypy]
64+
files = [
65+
"tests/python/",
66+
"src/bindings/python/stubs",
67+
]
68+
allow_redefinition = true
69+
check_untyped_defs = true
70+
disable_error_code = [
71+
"func-returns-value",
72+
]

src/bindings/python/CMakeLists.txt

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,3 +259,30 @@ install(TARGETS PyOpenColorIO
259259
)
260260

261261
install(FILES package/__init__.py DESTINATION ${_PyOpenColorIO_SITE_PACKAGE_DIR})
262+
install(FILES stubs/PyOpenColorIO/__init__.pyi DESTINATION ${_PyOpenColorIO_SITE_PACKAGE_DIR})
263+
install(FILES stubs/PyOpenColorIO/py.typed DESTINATION ${_PyOpenColorIO_SITE_PACKAGE_DIR})
264+
265+
###############################################################################
266+
267+
# Note: the python version must be kept in sync with `[[tool.cibuildwheel.overrides]]` in pyproject.toml.
268+
# The stubs are generated within a container so the version of python does not need to match
269+
# the version of python that OpenColorIO is being built against.
270+
# Note: the version of cibuildwheel should be kept in sync with .github/workflows/wheel.yml
271+
add_custom_command (COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/src/bindings/python/stubs/generate_stubs_local.py
272+
--repo-root ${CMAKE_SOURCE_DIR} --python-version="3.11" --cibuildwheel-version="2.22.0"
273+
--output-dir "${CMAKE_BINARY_DIR}/wheelhouse"
274+
OUTPUT "${CMAKE_BINARY_DIR}/wheelhouse/PyOpenColorIO/__init__.pyi"
275+
DEPENDS "${CMAKE_SOURCE_DIR}/src/bindings/python/stubs/generate_stubs.py"
276+
DEPENDS "${CMAKE_SOURCE_DIR}/src/bindings/python/stubs/generate_stubs_local.py"
277+
COMMENT "pystubs: Generating python stubs"
278+
)
279+
280+
add_custom_command (COMMAND ${CMAKE_COMMAND} -E copy
281+
"${CMAKE_BINARY_DIR}/wheelhouse/PyOpenColorIO/__init__.pyi"
282+
"${CMAKE_SOURCE_DIR}/src/bindings/python/stubs/PyOpenColorIO/__init__.pyi"
283+
OUTPUT "${CMAKE_SOURCE_DIR}/src/bindings/python/stubs/PyOpenColorIO/__init__.pyi"
284+
DEPENDS "${CMAKE_BINARY_DIR}/wheelhouse/PyOpenColorIO/__init__.pyi"
285+
COMMENT "pystubs: Copying generated stubs to source"
286+
)
287+
288+
add_custom_target (pystubs DEPENDS "${CMAKE_SOURCE_DIR}/src/bindings/python/stubs/PyOpenColorIO/__init__.pyi")

0 commit comments

Comments
 (0)