From 30229e6f80306ddb30a1ef159a05fec878c20fc1 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 27 Jan 2025 10:18:04 -0600 Subject: [PATCH 01/15] PYTHON-5047 Improve testing of publish workflows --- .github/workflows/release-python.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml index d2a3c754c..e1259ec8d 100644 --- a/.github/workflows/release-python.yml +++ b/.github/workflows/release-python.yml @@ -13,6 +13,8 @@ on: description: "Dry Run?" default: false type: boolean + schedule: + - cron: '30 5 * * *' env: # Changes per repo @@ -20,6 +22,10 @@ env: # Changes per branch SILK_ASSET_GROUP: pymongocrypt EVERGREEN_PROJECT: libmongocrypt + # Constant + DRY_RUN: ${{ inputs.dry_run || 'true' }} + FOLLOWING_VERSION: ${{ inputs.following_version || '' }} + VERSION: ${{ inputs.version || '10.10.10.10' }} defaults: run: @@ -48,9 +54,9 @@ jobs: - uses: mongodb-labs/drivers-github-tools/python/pre-publish@v2 id: pre-publish with: - version: ${{ inputs.version }} + version: ${{ env.VERSION }} working_directory: ./bindings/python - dry_run: ${{ inputs.dry_run }} + dry_run: ${{ env.DRY_RUN }} tag_template: "pymongocrypt-${VERSION}" tag_message_template: "Release PyMongoCrypt ${VERSION}" @@ -81,7 +87,7 @@ jobs: name: all-dist-${{ github.run_id }} path: dist/ - name: Publish package distributions to PyPI - if: startsWith(inputs.dry_run, 'false') + if: startsWith(env.DRY_RUN, 'false') uses: pypa/gh-action-pypi-publish@release/v1 post-publish: @@ -106,12 +112,12 @@ jobs: artifactory_username: ${{ vars.ARTIFACTORY_USERNAME }} - uses: mongodb-labs/drivers-github-tools/python/post-publish@v2 with: - version: ${{ inputs.version }} - following_version: ${{ inputs.following_version }} + version: ${{ env.VERSION }} + following_version: ${{ env.FOLLOWING_VERSION }} working_directory: ./bindings/python product_name: ${{ env.PRODUCT_NAME }} silk_asset_group: ${{ env.SILK_ASSET_GROUP }} evergreen_project: ${{ env.EVERGREEN_PROJECT }} tag_template: "pymongocrypt-${VERSION}" token: ${{ github.token }} - dry_run: ${{ inputs.dry_run }} \ No newline at end of file + dry_run: ${{ env.DRY_RUN }} \ No newline at end of file From cbe6efb896bee370da15ae65d4a8dd172d9a18b0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 27 Jan 2025 13:23:13 -0600 Subject: [PATCH 02/15] fix handling of testpypi --- .github/workflows/release-python.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml index e1259ec8d..0061a4fb8 100644 --- a/.github/workflows/release-python.yml +++ b/.github/workflows/release-python.yml @@ -86,6 +86,12 @@ jobs: with: name: all-dist-${{ github.run_id }} path: dist/ + - name: Publish package distributions to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + skip-existing: true + attestations: ${{ env.DRY_RUN }} - name: Publish package distributions to PyPI if: startsWith(env.DRY_RUN, 'false') uses: pypa/gh-action-pypi-publish@release/v1 From 53f2817bb77107e5e7d20cbcad1d2f82bc2ff576 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 28 Jan 2025 09:17:32 -0600 Subject: [PATCH 03/15] fix dry run logic --- .github/workflows/release-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml index 0061a4fb8..e97e3d53b 100644 --- a/.github/workflows/release-python.yml +++ b/.github/workflows/release-python.yml @@ -23,7 +23,7 @@ env: SILK_ASSET_GROUP: pymongocrypt EVERGREEN_PROJECT: libmongocrypt # Constant - DRY_RUN: ${{ inputs.dry_run || 'true' }} + DRY_RUN: ${{ inputs.dry_run == 'true' }} FOLLOWING_VERSION: ${{ inputs.following_version || '' }} VERSION: ${{ inputs.version || '10.10.10.10' }} From 0744e01427e7004e7f06ec625ddd6ce2788c08b0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Jan 2025 09:38:32 -0600 Subject: [PATCH 04/15] fix dry run logic --- .github/workflows/release-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml index e97e3d53b..470b19523 100644 --- a/.github/workflows/release-python.yml +++ b/.github/workflows/release-python.yml @@ -23,7 +23,7 @@ env: SILK_ASSET_GROUP: pymongocrypt EVERGREEN_PROJECT: libmongocrypt # Constant - DRY_RUN: ${{ inputs.dry_run == 'true' }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || 'true' }} FOLLOWING_VERSION: ${{ inputs.following_version || '' }} VERSION: ${{ inputs.version || '10.10.10.10' }} From fb1e6980fdbb777d8d9236d5032121d177f2f75c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Jan 2025 13:24:11 -0600 Subject: [PATCH 05/15] fix integ test --- bindings/python/.evergreen/integ-test.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bindings/python/.evergreen/integ-test.sh b/bindings/python/.evergreen/integ-test.sh index 2efdc9a07..84cf894fd 100755 --- a/bindings/python/.evergreen/integ-test.sh +++ b/bindings/python/.evergreen/integ-test.sh @@ -32,11 +32,14 @@ pushd $PYMONGO_DIR pip install -e ".[test,encryption]" source ${DRIVERS_TOOLS}/.evergreen/csfle/secrets-export.sh set -x -TEST_CRYPT_SHARED=1 DYLD_FALLBACK_LIBRARY_PATH=$CRYPT_SHARED_DIR:${DYLD_FALLBACK_LIBRARY_PATH:-} \ - LD_LIBRARY_PATH=$CRYPT_SHARED_DIR:${LD_LIBRARY_PATH-} \ - PATH=$CRYPT_SHARED_DIR:$PATH \ - AUTH=auth SSL=ssl \ - .evergreen/run-tests.sh -m encryption +export DB_USER="bob" +export DB_PASSWORD="pwd123" +export CLIENT_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem" +export CA_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem" +export DYLD_FALLBACK_LIBRARY_PATH=$CRYPT_SHARED_DIR:${DYLD_FALLBACK_LIBRARY_PATH:-} +export LD_LIBRARY_PATH=$CRYPT_SHARED_DIR:${LD_LIBRARY_PATH-} +export PATH=$CRYPT_SHARED_DIR:$PATH +pytest -m encryption popd deactivate From 18c1e7234b74d6b28489abcb632410ba580ab536 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Jan 2025 14:06:06 -0600 Subject: [PATCH 06/15] debug --- bindings/python/.evergreen/integ-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/.evergreen/integ-test.sh b/bindings/python/.evergreen/integ-test.sh index 84cf894fd..d99954b35 100755 --- a/bindings/python/.evergreen/integ-test.sh +++ b/bindings/python/.evergreen/integ-test.sh @@ -39,7 +39,7 @@ export CA_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem" export DYLD_FALLBACK_LIBRARY_PATH=$CRYPT_SHARED_DIR:${DYLD_FALLBACK_LIBRARY_PATH:-} export LD_LIBRARY_PATH=$CRYPT_SHARED_DIR:${LD_LIBRARY_PATH-} export PATH=$CRYPT_SHARED_DIR:$PATH -pytest -m encryption +pytest --maxfail=10 -v -m encryption popd deactivate From 69e5305c8fe8f4b9d277b5d6ce2e3abb1c4007d9 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Jan 2025 14:44:25 -0600 Subject: [PATCH 07/15] fix integ test --- bindings/python/.evergreen/integ-setup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bindings/python/.evergreen/integ-setup.sh b/bindings/python/.evergreen/integ-setup.sh index 60fa5e577..a23ba1544 100755 --- a/bindings/python/.evergreen/integ-setup.sh +++ b/bindings/python/.evergreen/integ-setup.sh @@ -38,6 +38,8 @@ EOT # Get the secrets bash $DRIVERS_TOOLS/.evergreen/csfle/setup-secrets.sh +# Start the csfle servers +bash $DRIVERS_TOOLS/.evergreen/csfle/start-servers.sh # Clone mongo-python-driver git clone https://github.com/mongodb/mongo-python-driver.git ${PYMONGO_DIR} From cf52604d199d13b875a3845c59226b78dfb38832 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Jan 2025 14:57:55 -0600 Subject: [PATCH 08/15] update crypt_shared_dir --- .github/workflows/release-python.yml | 3 +-- bindings/python/.evergreen/integ-test.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml index 470b19523..3fc1e5ab9 100644 --- a/.github/workflows/release-python.yml +++ b/.github/workflows/release-python.yml @@ -8,7 +8,6 @@ on: required: true following_version: description: "The post (dev) version to set" - required: true dry_run: description: "Dry Run?" default: false @@ -23,7 +22,7 @@ env: SILK_ASSET_GROUP: pymongocrypt EVERGREEN_PROJECT: libmongocrypt # Constant - DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || 'true' }} + DRY_RUN: ${{ ! contains(inputs.dry_run, 'false') }} FOLLOWING_VERSION: ${{ inputs.following_version || '' }} VERSION: ${{ inputs.version || '10.10.10.10' }} diff --git a/bindings/python/.evergreen/integ-test.sh b/bindings/python/.evergreen/integ-test.sh index d99954b35..a16951cfa 100755 --- a/bindings/python/.evergreen/integ-test.sh +++ b/bindings/python/.evergreen/integ-test.sh @@ -10,7 +10,7 @@ BASE_PYTHON=$(find_python3) # MONGOCRYPT_DIR is set by libmongocrypt/.evergreen/config.yml MONGOCRYPT_DIR="$MONGOCRYPT_DIR" -CRYPT_SHARED_DIR="$DRIVERS_TOOLS" +CRYPT_SHARED_DIR="$DRIVERS_TOOLS/mongodb/bin" MACHINE=$(uname -m) if [ $MACHINE == "aarch64" ]; then From 714f60daff8b410c89011d8081c4a36c95329425 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Jan 2025 19:10:29 -0600 Subject: [PATCH 09/15] test --- .github/workflows/release-python.yml | 2 ++ bindings/python/.evergreen/integ-setup.sh | 2 +- bindings/python/.evergreen/integ-test.sh | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml index 3fc1e5ab9..047577b9a 100644 --- a/.github/workflows/release-python.yml +++ b/.github/workflows/release-python.yml @@ -22,6 +22,8 @@ env: SILK_ASSET_GROUP: pymongocrypt EVERGREEN_PROJECT: libmongocrypt # Constant + # inputs will be empty on a scheduled run. so, we only set dry_run + # to 'false' when the input is set to 'false'. DRY_RUN: ${{ ! contains(inputs.dry_run, 'false') }} FOLLOWING_VERSION: ${{ inputs.following_version || '' }} VERSION: ${{ inputs.version || '10.10.10.10' }} diff --git a/bindings/python/.evergreen/integ-setup.sh b/bindings/python/.evergreen/integ-setup.sh index a23ba1544..ef29b6b28 100755 --- a/bindings/python/.evergreen/integ-setup.sh +++ b/bindings/python/.evergreen/integ-setup.sh @@ -28,7 +28,7 @@ MONGOCRYPT_DIR: "$MONGOCRYPT_DIR" EOT # Set up drivers-tools with a .env file. -git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git ${DRIVERS_TOOLS} +git clone --branch DRIVERS-3032-fix-crypt_shared_loc https://github.com/blink1073/drivers-evergreen-tools.git ${DRIVERS_TOOLS} cat < ${DRIVERS_TOOLS}/.env DRIVERS_TOOLS="$DRIVERS_TOOLS" MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" diff --git a/bindings/python/.evergreen/integ-test.sh b/bindings/python/.evergreen/integ-test.sh index a16951cfa..d99954b35 100755 --- a/bindings/python/.evergreen/integ-test.sh +++ b/bindings/python/.evergreen/integ-test.sh @@ -10,7 +10,7 @@ BASE_PYTHON=$(find_python3) # MONGOCRYPT_DIR is set by libmongocrypt/.evergreen/config.yml MONGOCRYPT_DIR="$MONGOCRYPT_DIR" -CRYPT_SHARED_DIR="$DRIVERS_TOOLS/mongodb/bin" +CRYPT_SHARED_DIR="$DRIVERS_TOOLS" MACHINE=$(uname -m) if [ $MACHINE == "aarch64" ]; then From 190de26394dd15f0332fc53488ac971b5aa8869a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Jan 2025 19:49:01 -0600 Subject: [PATCH 10/15] update path --- bindings/python/.evergreen/integ-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bindings/python/.evergreen/integ-test.sh b/bindings/python/.evergreen/integ-test.sh index d99954b35..323c2709d 100755 --- a/bindings/python/.evergreen/integ-test.sh +++ b/bindings/python/.evergreen/integ-test.sh @@ -11,6 +11,7 @@ BASE_PYTHON=$(find_python3) # MONGOCRYPT_DIR is set by libmongocrypt/.evergreen/config.yml MONGOCRYPT_DIR="$MONGOCRYPT_DIR" CRYPT_SHARED_DIR="$DRIVERS_TOOLS" +MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" MACHINE=$(uname -m) if [ $MACHINE == "aarch64" ]; then @@ -38,7 +39,7 @@ export CLIENT_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem" export CA_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem" export DYLD_FALLBACK_LIBRARY_PATH=$CRYPT_SHARED_DIR:${DYLD_FALLBACK_LIBRARY_PATH:-} export LD_LIBRARY_PATH=$CRYPT_SHARED_DIR:${LD_LIBRARY_PATH-} -export PATH=$CRYPT_SHARED_DIR:$PATH +export PATH=$CRYPT_SHARED_DIR:$MONGODB_BINARIES:$PATH pytest --maxfail=10 -v -m encryption popd From fd64fc948d43bbc1349a4eba8c293e0fac695710 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Jan 2025 20:30:28 -0600 Subject: [PATCH 11/15] update test --- bindings/python/.evergreen/integ-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bindings/python/.evergreen/integ-test.sh b/bindings/python/.evergreen/integ-test.sh index 323c2709d..53c36cf10 100755 --- a/bindings/python/.evergreen/integ-test.sh +++ b/bindings/python/.evergreen/integ-test.sh @@ -40,6 +40,7 @@ export CA_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem" export DYLD_FALLBACK_LIBRARY_PATH=$CRYPT_SHARED_DIR:${DYLD_FALLBACK_LIBRARY_PATH:-} export LD_LIBRARY_PATH=$CRYPT_SHARED_DIR:${LD_LIBRARY_PATH-} export PATH=$CRYPT_SHARED_DIR:$MONGODB_BINARIES:$PATH +export TEST_CRYPT_SHARED="1" pytest --maxfail=10 -v -m encryption popd From b77463410eed2dc930614a83b0dc14afe47340a7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Jan 2025 20:30:51 -0600 Subject: [PATCH 12/15] use upstream --- bindings/python/.evergreen/integ-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/.evergreen/integ-setup.sh b/bindings/python/.evergreen/integ-setup.sh index ef29b6b28..667f98a90 100755 --- a/bindings/python/.evergreen/integ-setup.sh +++ b/bindings/python/.evergreen/integ-setup.sh @@ -28,7 +28,7 @@ MONGOCRYPT_DIR: "$MONGOCRYPT_DIR" EOT # Set up drivers-tools with a .env file. -git clone --branch DRIVERS-3032-fix-crypt_shared_loc https://github.com/blink1073/drivers-evergreen-tools.git ${DRIVERS_TOOLS} +git clone --branch https://github.com/mongodb-labs/drivers-evergreen-tools.git ${DRIVERS_TOOLS} cat < ${DRIVERS_TOOLS}/.env DRIVERS_TOOLS="$DRIVERS_TOOLS" MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" From 5e3d5e7cca763ee7751e09512b058b9acabdac45 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Jan 2025 20:49:08 -0600 Subject: [PATCH 13/15] use upstream --- bindings/python/.evergreen/integ-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/.evergreen/integ-setup.sh b/bindings/python/.evergreen/integ-setup.sh index 667f98a90..a23ba1544 100755 --- a/bindings/python/.evergreen/integ-setup.sh +++ b/bindings/python/.evergreen/integ-setup.sh @@ -28,7 +28,7 @@ MONGOCRYPT_DIR: "$MONGOCRYPT_DIR" EOT # Set up drivers-tools with a .env file. -git clone --branch https://github.com/mongodb-labs/drivers-evergreen-tools.git ${DRIVERS_TOOLS} +git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git ${DRIVERS_TOOLS} cat < ${DRIVERS_TOOLS}/.env DRIVERS_TOOLS="$DRIVERS_TOOLS" MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" From b2b82d48a22775e6faafdbe6d35086cdeeb8e4d3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 29 Jan 2025 21:21:59 -0600 Subject: [PATCH 14/15] use fork --- bindings/python/.evergreen/integ-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/.evergreen/integ-setup.sh b/bindings/python/.evergreen/integ-setup.sh index a23ba1544..ef29b6b28 100755 --- a/bindings/python/.evergreen/integ-setup.sh +++ b/bindings/python/.evergreen/integ-setup.sh @@ -28,7 +28,7 @@ MONGOCRYPT_DIR: "$MONGOCRYPT_DIR" EOT # Set up drivers-tools with a .env file. -git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git ${DRIVERS_TOOLS} +git clone --branch DRIVERS-3032-fix-crypt_shared_loc https://github.com/blink1073/drivers-evergreen-tools.git ${DRIVERS_TOOLS} cat < ${DRIVERS_TOOLS}/.env DRIVERS_TOOLS="$DRIVERS_TOOLS" MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" From 7bf72f6e28344291ad1e10cf827cf69bd8137fde Mon Sep 17 00:00:00 2001 From: Casey Clements Date: Fri, 31 Jan 2025 17:54:17 -0500 Subject: [PATCH 15/15] Get drivers-evergreen-tools from master again --- bindings/python/.evergreen/integ-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/.evergreen/integ-setup.sh b/bindings/python/.evergreen/integ-setup.sh index ef29b6b28..a23ba1544 100755 --- a/bindings/python/.evergreen/integ-setup.sh +++ b/bindings/python/.evergreen/integ-setup.sh @@ -28,7 +28,7 @@ MONGOCRYPT_DIR: "$MONGOCRYPT_DIR" EOT # Set up drivers-tools with a .env file. -git clone --branch DRIVERS-3032-fix-crypt_shared_loc https://github.com/blink1073/drivers-evergreen-tools.git ${DRIVERS_TOOLS} +git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git ${DRIVERS_TOOLS} cat < ${DRIVERS_TOOLS}/.env DRIVERS_TOOLS="$DRIVERS_TOOLS" MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"