From 56ac28a24ac11dbceb05cd9bdb908466b6cb0aac Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 5 Feb 2025 21:50:54 -0800 Subject: [PATCH 1/6] try nightly --- .github/workflows/release_python_nightly.yml | 137 +++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 .github/workflows/release_python_nightly.yml diff --git a/.github/workflows/release_python_nightly.yml b/.github/workflows/release_python_nightly.yml new file mode 100644 index 0000000000..0e4a6289ba --- /dev/null +++ b/.github/workflows/release_python_nightly.yml @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: "Nightly PyPI Build" + +on: + schedule: + - cron: "0 0 * * *" # Runs at midnight UTC every day + workflow_dispatch: # Allows manual triggering + +env: + rust_msrv: "1.77.1" + +permissions: + contents: read + +jobs: + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install toml + run: pip install toml + + # Override the library version + - name: Get current version + run: | + CURRENT_VERSION=$(python -c "import toml; print(toml.load('bindings/python/pyproject.toml')['project']['version'])") + echo "CURRENT_VERSION=$CURRENT_VERSION" >> $GITHUB_ENV + + - name: Update version with timestamp + run: | + TIMESTAMP=$(date +%Y%m%d%H%M%S) + NEW_VERSION="${CURRENT_VERSION}.dev${TIMESTAMP}" + NEW_VERSION=$NEW_VERSION python -c " + import toml + import os + config = toml.load('bindings/python/pyproject.toml') + config['project']['version'] = os.environ['NEW_VERSION'] + with open('bindings/python/pyproject.toml', 'w') as f: + toml.dump(config, f) + print(f'Updated version to: {config[\"project\"][\"version\"]}') + " + + - uses: PyO3/maturin-action@v1 + with: + working-directory: "bindings/python" + command: sdist + args: -o dist + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: wheels-sdist + path: bindings/python/dist + + wheels: + runs-on: "${{ matrix.os }}" + strategy: + matrix: + include: + - { os: windows-latest } + - { os: macos-latest, target: "universal2-apple-darwin" } + - { os: ubuntu-latest, target: "x86_64" } + - { os: ubuntu-latest, target: "aarch64" } + - { os: ubuntu-latest, target: "armv7l" } + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: ${{ env.rust_msrv }} + - uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + manylinux: auto + working-directory: "bindings/python" + command: build + args: --release -o dist + env: + # Workaround ring 0.17 build issue + CFLAGS_aarch64_unknown_linux_gnu: "-D__ARM_ARCH=8" + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.os }}-${{ matrix.target }} + path: bindings/python/dist + + testpypi-publish: + name: Publish Python 🐍 distribution 📦 to TestPypi + needs: [ sdist, wheels ] + runs-on: ubuntu-latest + + environment: + name: testpypi + url: https://test.pypi.org/p/pyiceberg_core + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + pattern: wheels-* + merge-multiple: true + path: bindings/python/dist + - name: List downloaded artifacts + run: ls -R bindings/python/dist + # - name: Publish to TestPyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # repository-url: https://test.pypi.org/legacy/ + # skip-existing: true + # packages-dir: bindings/python/dist From 58e3e55915d196d2ddb5e333cc5f6cebf53ce5a8 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 6 Feb 2025 08:57:47 -0800 Subject: [PATCH 2/6] use action for both jobs --- .../overwrite-package-version/action.yml | 46 +++++++++++++++++++ .github/workflows/release_python_nightly.yml | 30 +----------- 2 files changed, 48 insertions(+), 28 deletions(-) create mode 100644 .github/actions/overwrite-package-version/action.yml diff --git a/.github/actions/overwrite-package-version/action.yml b/.github/actions/overwrite-package-version/action.yml new file mode 100644 index 0000000000..087e6401d1 --- /dev/null +++ b/.github/actions/overwrite-package-version/action.yml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Update Package Version' +description: 'Updates pyproject.toml version with timestamp' +runs: + using: "composite" + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install toml + run: pip install toml + shell: bash + + - name: Get and update version + shell: bash + run: | + CURRENT_VERSION=$(python -c "import toml; print(toml.load('bindings/python/pyproject.toml')['project']['version'])") + TIMESTAMP=$(date +%Y%m%d%H%M%S) + NEW_VERSION="${CURRENT_VERSION}.dev${TIMESTAMP}" + NEW_VERSION=$NEW_VERSION python -c " + import toml + import os + config = toml.load('bindings/python/pyproject.toml') + config['project']['version'] = os.environ['NEW_VERSION'] + with open('bindings/python/pyproject.toml', 'w') as f: + toml.dump(config, f) + print(f'Updated version to: {config[\"project\"][\"version\"]}') + " \ No newline at end of file diff --git a/.github/workflows/release_python_nightly.yml b/.github/workflows/release_python_nightly.yml index 0e4a6289ba..1d62e6b933 100644 --- a/.github/workflows/release_python_nightly.yml +++ b/.github/workflows/release_python_nightly.yml @@ -34,33 +34,7 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install toml - run: pip install toml - - # Override the library version - - name: Get current version - run: | - CURRENT_VERSION=$(python -c "import toml; print(toml.load('bindings/python/pyproject.toml')['project']['version'])") - echo "CURRENT_VERSION=$CURRENT_VERSION" >> $GITHUB_ENV - - - name: Update version with timestamp - run: | - TIMESTAMP=$(date +%Y%m%d%H%M%S) - NEW_VERSION="${CURRENT_VERSION}.dev${TIMESTAMP}" - NEW_VERSION=$NEW_VERSION python -c " - import toml - import os - config = toml.load('bindings/python/pyproject.toml') - config['project']['version'] = os.environ['NEW_VERSION'] - with open('bindings/python/pyproject.toml', 'w') as f: - toml.dump(config, f) - print(f'Updated version to: {config[\"project\"][\"version\"]}') - " + - uses: ./.github/actions/overwrite-package-version # overwrite the pacakge version with current timestamp - uses: PyO3/maturin-action@v1 with: @@ -85,6 +59,7 @@ jobs: - { os: ubuntu-latest, target: "armv7l" } steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/overwrite-package-version # overwrite the pacakge version with current timestamp - uses: actions/setup-python@v5 with: python-version: 3.9 @@ -109,7 +84,6 @@ jobs: path: bindings/python/dist testpypi-publish: - name: Publish Python 🐍 distribution 📦 to TestPypi needs: [ sdist, wheels ] runs-on: ubuntu-latest From 3211928ff412f47b99ac10b87fdb31840e5ff70c Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 17 Feb 2025 17:18:31 -0800 Subject: [PATCH 3/6] add publish step --- .github/workflows/release_python_nightly.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release_python_nightly.yml b/.github/workflows/release_python_nightly.yml index 1d62e6b933..db1d39f49c 100644 --- a/.github/workflows/release_python_nightly.yml +++ b/.github/workflows/release_python_nightly.yml @@ -103,9 +103,9 @@ jobs: path: bindings/python/dist - name: List downloaded artifacts run: ls -R bindings/python/dist - # - name: Publish to TestPyPI - # uses: pypa/gh-action-pypi-publish@release/v1 - # with: - # repository-url: https://test.pypi.org/legacy/ - # skip-existing: true - # packages-dir: bindings/python/dist + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + skip-existing: true + packages-dir: bindings/python/dist From 28ba9fb5a25886ff9241bcb9e1ccf8c29d5de42a Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 17 Feb 2025 17:48:28 -0800 Subject: [PATCH 4/6] add gate for only apache repo --- .github/workflows/release_python_nightly.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release_python_nightly.yml b/.github/workflows/release_python_nightly.yml index db1d39f49c..4de209a78a 100644 --- a/.github/workflows/release_python_nightly.yml +++ b/.github/workflows/release_python_nightly.yml @@ -30,6 +30,7 @@ permissions: jobs: sdist: + if: github.repository == 'apache/iceberg-rust' # Only run for apache repo runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -48,6 +49,7 @@ jobs: path: bindings/python/dist wheels: + if: github.repository == 'apache/iceberg-rust' # Only run for apache repo runs-on: "${{ matrix.os }}" strategy: matrix: From 613a3ff158a28e9c52f9dafc25d66896ae2565f7 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Tue, 18 Feb 2025 07:26:03 -0800 Subject: [PATCH 5/6] remove publishing to testpypi on regular release workflow --- .github/workflows/release_python.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/.github/workflows/release_python.yml b/.github/workflows/release_python.yml index 9fc6f2c97f..b5ee3a50a1 100644 --- a/.github/workflows/release_python.yml +++ b/.github/workflows/release_python.yml @@ -113,31 +113,3 @@ jobs: with: skip-existing: true packages-dir: bindings/python/dist - - testpypi-publish: - name: Publish Python 🐍 distribution 📦 to TestPypi - needs: [ sdist, wheels ] - runs-on: ubuntu-latest - # Only publish to TestPyPi if the tag is a pre-release - if: ${{ startsWith(github.ref, 'refs/tags/') && contains(github.ref, '-')}} - - environment: - name: testpypi - url: https://test.pypi.org/p/pyiceberg_core - - permissions: - id-token: write # IMPORTANT: mandatory for trusted publishing - - steps: - - name: Download all the dists - uses: actions/download-artifact@v4 - with: - pattern: wheels-* - merge-multiple: true - path: bindings/python/dist - - name: Publish to TestPyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: https://test.pypi.org/legacy/ - skip-existing: true - packages-dir: bindings/python/dist From c85454913bb750cf4b5432d53eebd7718f5b2e20 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Tue, 18 Feb 2025 07:41:01 -0800 Subject: [PATCH 6/6] apply fix --- .github/workflows/release_python_nightly.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release_python_nightly.yml b/.github/workflows/release_python_nightly.yml index 4de209a78a..5091d0e38d 100644 --- a/.github/workflows/release_python_nightly.yml +++ b/.github/workflows/release_python_nightly.yml @@ -57,7 +57,7 @@ jobs: - { os: windows-latest } - { os: macos-latest, target: "universal2-apple-darwin" } - { os: ubuntu-latest, target: "x86_64" } - - { os: ubuntu-latest, target: "aarch64" } + - { os: ubuntu-latest, target: "aarch64", manylinux: "manylinux_2_28" } - { os: ubuntu-latest, target: "armv7l" } steps: - uses: actions/checkout@v4 @@ -72,13 +72,10 @@ jobs: - uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} - manylinux: auto + manylinux: ${{ matrix.manylinux || 'auto' }} working-directory: "bindings/python" command: build args: --release -o dist - env: - # Workaround ring 0.17 build issue - CFLAGS_aarch64_unknown_linux_gnu: "-D__ARM_ARCH=8" - name: Upload wheels uses: actions/upload-artifact@v4 with: