From 7f192dde5b0b9cdfcd5a5e30e7d4df2d0b74225e Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 2 Feb 2025 13:50:36 -0800 Subject: [PATCH 1/7] add nightly build --- .github/workflows/nightly-pypi-build.yml | 56 ++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/nightly-pypi-build.yml diff --git a/.github/workflows/nightly-pypi-build.yml b/.github/workflows/nightly-pypi-build.yml new file mode 100644 index 0000000000..df49acfb9f --- /dev/null +++ b/.github/workflows/nightly-pypi-build.yml @@ -0,0 +1,56 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: "Nightly PyPI Build" + +on: + schedule: + - cron: "0 0 * * *" # Runs at midnight UTC every day + workflow_dispatch: # Allows manual triggering + +jobs: + nightly-build: + uses: ./.github/workflows/pypi-build-artifacts.yml # Reference the PyPI build workflow + with: + VERSION: "0.0.0" # Generate nightly version + RC: "0" # Reset RC for nightly builds + + testpypi-publish: + name: Publish to TestPypi + needs: + - nightly-build + runs-on: ubuntu-latest + environment: + name: testpypi + url: https://test.pypi.org/p/pyiceberg-kevinliu + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the artifacts + uses: actions/download-artifact@v4 + with: + merge-multiple: true + path: dist/ + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + skip-existing: true From dadfe3b7d294cd17c3e65f1767ca070192be2b18 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 3 Feb 2025 08:05:57 -0800 Subject: [PATCH 2/7] mac 14 and 15 are the same and produce the same artifacts --- .github/workflows/pypi-build-artifacts.yml | 2 +- .github/workflows/svn-build-artifacts.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypi-build-artifacts.yml b/.github/workflows/pypi-build-artifacts.yml index 274528c3bd..897d640525 100644 --- a/.github/workflows/pypi-build-artifacts.yml +++ b/.github/workflows/pypi-build-artifacts.yml @@ -35,7 +35,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ] + os: [ ubuntu-22.04, windows-2022, macos-13, macos-14 ] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/svn-build-artifacts.yml b/.github/workflows/svn-build-artifacts.yml index 336ece68f4..81737df833 100644 --- a/.github/workflows/svn-build-artifacts.yml +++ b/.github/workflows/svn-build-artifacts.yml @@ -35,7 +35,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ] + os: [ ubuntu-22.04, windows-2022, macos-13, macos-14 ] steps: - uses: actions/checkout@v4 From f7d82ccc364f03a041a2d2d164a44eb13b8d9b10 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 3 Feb 2025 08:06:03 -0800 Subject: [PATCH 3/7] tmp --- .github/workflows/nightly-pypi-build.yml | 4 +++- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly-pypi-build.yml b/.github/workflows/nightly-pypi-build.yml index df49acfb9f..661d060eb2 100644 --- a/.github/workflows/nightly-pypi-build.yml +++ b/.github/workflows/nightly-pypi-build.yml @@ -29,7 +29,7 @@ jobs: uses: ./.github/workflows/pypi-build-artifacts.yml # Reference the PyPI build workflow with: VERSION: "0.0.0" # Generate nightly version - RC: "0" # Reset RC for nightly builds + RC: "1" # Reset RC for nightly builds testpypi-publish: name: Publish to TestPypi @@ -49,6 +49,8 @@ jobs: with: merge-multiple: true path: dist/ + - name: List downloaded artifacts + run: ls -R dist/ - name: Publish to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/pyproject.toml b/pyproject.toml index 7d28e89832..342313711e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. [tool.poetry] -name = "pyiceberg" +name = "pyiceberg-kevinliu" version = "0.9.0" readme = "README.md" homepage = "https://py.iceberg.apache.org/" From 3cc5ca12e1a87ef11a62e763a8f8061d8d9bcd50 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 3 Feb 2025 09:00:57 -0800 Subject: [PATCH 4/7] try new version format --- .github/workflows/nightly-pypi-build.yml | 36 ++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly-pypi-build.yml b/.github/workflows/nightly-pypi-build.yml index 661d060eb2..328b6e54d5 100644 --- a/.github/workflows/nightly-pypi-build.yml +++ b/.github/workflows/nightly-pypi-build.yml @@ -25,11 +25,40 @@ on: workflow_dispatch: # Allows manual triggering jobs: + set-version: + runs-on: ubuntu-latest + outputs: + VERSION: ${{ steps.set-version.outputs.VERSION }} + RC: ${{ steps.set-version.outputs.RC }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Install Poetry + run: make install-poetry + + - name: Extract version and rc + id: set-version + run: | + VERSION=$(poetry version --short) + RC="$(date +%Y%m%d)" + echo "VERSION=$VERSION" >> "$GITHUB_OUTPUT" + echo "RC=$RC" >> "$GITHUB_OUTPUT" + + - name: Debug version and rc + run: echo "Publishing version ${{ steps.set-version.outputs.VERSION }}rc${{ steps.set-version.outputs.RC }}" + nightly-build: - uses: ./.github/workflows/pypi-build-artifacts.yml # Reference the PyPI build workflow + needs: set-version + uses: ./.github/workflows/pypi-build-artifacts.yml with: - VERSION: "0.0.0" # Generate nightly version - RC: "1" # Reset RC for nightly builds + version: ${{ needs.set-version.outputs.VERSION }} # i.e. 0.9.0 + rc: ${{ needs.set-version.outputs.RC }} # i.e. 20250203 testpypi-publish: name: Publish to TestPypi @@ -56,3 +85,4 @@ jobs: with: repository-url: https://test.pypi.org/legacy/ skip-existing: true + verbose: true From 9f2786e44af6d539e99c1cf6f61def83adee6ba7 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 3 Feb 2025 13:51:58 -0800 Subject: [PATCH 5/7] use actual repo --- .github/workflows/nightly-pypi-build.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly-pypi-build.yml b/.github/workflows/nightly-pypi-build.yml index 328b6e54d5..87638a099c 100644 --- a/.github/workflows/nightly-pypi-build.yml +++ b/.github/workflows/nightly-pypi-build.yml @@ -67,7 +67,7 @@ jobs: runs-on: ubuntu-latest environment: name: testpypi - url: https://test.pypi.org/p/pyiceberg-kevinliu + url: https://test.pypi.org/p/pyiceberg permissions: id-token: write # IMPORTANT: mandatory for trusted publishing diff --git a/pyproject.toml b/pyproject.toml index 342313711e..7d28e89832 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. [tool.poetry] -name = "pyiceberg-kevinliu" +name = "pyiceberg" version = "0.9.0" readme = "README.md" homepage = "https://py.iceberg.apache.org/" From 62a0072ce6a3f0d5ec2251812614cb7f1d3f0ecd Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 3 Feb 2025 13:52:54 -0800 Subject: [PATCH 6/7] gate on apache repo --- .github/workflows/nightly-pypi-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nightly-pypi-build.yml b/.github/workflows/nightly-pypi-build.yml index 87638a099c..8253180b12 100644 --- a/.github/workflows/nightly-pypi-build.yml +++ b/.github/workflows/nightly-pypi-build.yml @@ -26,6 +26,7 @@ on: jobs: set-version: + if: github.repository == 'apache/iceberg-python' # Only run for apache repo runs-on: ubuntu-latest outputs: VERSION: ${{ steps.set-version.outputs.VERSION }} From d7349433f3a5bbd341b62d10ec8f908f8330ec42 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 5 Feb 2025 20:51:38 -0800 Subject: [PATCH 7/7] refactor remove RC --- .github/workflows/nightly-pypi-build.yml | 18 +++++++----------- .github/workflows/pypi-build-artifacts.yml | 8 ++------ .github/workflows/python-release.yml | 6 ++---- .github/workflows/svn-build-artifacts.yml | 5 +---- 4 files changed, 12 insertions(+), 25 deletions(-) diff --git a/.github/workflows/nightly-pypi-build.yml b/.github/workflows/nightly-pypi-build.yml index 8253180b12..e7a32a3d0a 100644 --- a/.github/workflows/nightly-pypi-build.yml +++ b/.github/workflows/nightly-pypi-build.yml @@ -30,7 +30,6 @@ jobs: runs-on: ubuntu-latest outputs: VERSION: ${{ steps.set-version.outputs.VERSION }} - RC: ${{ steps.set-version.outputs.RC }} steps: - uses: actions/checkout@v4 with: @@ -43,24 +42,21 @@ jobs: - name: Install Poetry run: make install-poetry - - name: Extract version and rc + - name: Set version id: set-version run: | - VERSION=$(poetry version --short) - RC="$(date +%Y%m%d)" - echo "VERSION=$VERSION" >> "$GITHUB_OUTPUT" - echo "RC=$RC" >> "$GITHUB_OUTPUT" + CURRENT_VERSION=$(poetry version --short) + TIMESTAMP=$(date +%Y%m%d%H%M%S) + echo "VERSION=${CURRENT_VERSION}.dev${TIMESTAMP}" >> "$GITHUB_OUTPUT" - - name: Debug version and rc - run: echo "Publishing version ${{ steps.set-version.outputs.VERSION }}rc${{ steps.set-version.outputs.RC }}" + - name: Debug version + run: echo "Publishing version ${{ steps.set-version.outputs.VERSION }}" nightly-build: needs: set-version uses: ./.github/workflows/pypi-build-artifacts.yml with: - version: ${{ needs.set-version.outputs.VERSION }} # i.e. 0.9.0 - rc: ${{ needs.set-version.outputs.RC }} # i.e. 20250203 - + version: ${{ needs.set-version.outputs.VERSION }} testpypi-publish: name: Publish to TestPypi needs: diff --git a/.github/workflows/pypi-build-artifacts.yml b/.github/workflows/pypi-build-artifacts.yml index 897d640525..288e5c0046 100644 --- a/.github/workflows/pypi-build-artifacts.yml +++ b/.github/workflows/pypi-build-artifacts.yml @@ -25,9 +25,6 @@ on: VERSION: required: true type: string - RC: - required: true - type: string jobs: pypi-build-artifacts: @@ -56,8 +53,7 @@ jobs: - name: Set version with RC env: VERSION: ${{ inputs.VERSION }} - RC: ${{ inputs.RC }} - run: python -m poetry version "${{ env.VERSION }}rc${{ env.RC }}" # e.g., 0.8.0rc1 + run: python -m poetry version "${{ env.VERSION }}" # Publish the source distribution with the version that's in # the repository, otherwise the tests will fail @@ -97,6 +93,6 @@ jobs: - name: Merge Artifacts uses: actions/upload-artifact/merge@v4 with: - name: "pypi-release-candidate-${{ inputs.VERSION }}rc${{ inputs.RC }}" + name: "pypi-release-candidate-${{ inputs.VERSION }}" pattern: pypi-release-candidate* delete-merged: true diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 8c2216d8a7..a6175ead9e 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -121,8 +121,7 @@ jobs: - validate-library-version uses: ./.github/workflows/svn-build-artifacts.yml with: - version: ${{ needs.validate-inputs.outputs.VERSION }} - rc: ${{ needs.validate-inputs.outputs.RC }} + version: ${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }} # PyPi pypi-build-artifacts: @@ -131,5 +130,4 @@ jobs: - validate-library-version uses: ./.github/workflows/pypi-build-artifacts.yml with: - version: ${{ needs.validate-inputs.outputs.VERSION }} - rc: ${{ needs.validate-inputs.outputs.RC }} + version: ${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }} diff --git a/.github/workflows/svn-build-artifacts.yml b/.github/workflows/svn-build-artifacts.yml index 81737df833..8336b46940 100644 --- a/.github/workflows/svn-build-artifacts.yml +++ b/.github/workflows/svn-build-artifacts.yml @@ -25,9 +25,6 @@ on: VERSION: required: true type: string - RC: - required: true - type: string jobs: svn-build-artifacts: @@ -91,6 +88,6 @@ jobs: - name: Merge Artifacts uses: actions/upload-artifact/merge@v4 with: - name: "svn-release-candidate-${{ inputs.VERSION }}rc${{ inputs.RC }}" + name: "svn-release-candidate-${{ inputs.VERSION }}" pattern: svn-release-candidate* delete-merged: true