Add support for PyMongo Async and N+1 Dereference/select_related using Pipeline

.github/workflows/github-actions.yml

@@ -13,112 +13,60 @@ on:
     tags:
       - 'v[0-9]+\.[0-9]+\.[0-9]+*'
 env:
-  MONGODB_4_4: "4.4.29"
-  MONGODB_5_0: "5.0.31"
-  MONGODB_6_0: "6.0.22"
-  MONGODB_7_0: "7.0.19"
-  MONGODB_8_0: "8.0.9"
-
-  PYMONGO_3_12: "3.12.3"
-  PYMONGO_3_13: "3.13.0"
-  PYMONGO_4_0:  "4.0.2"
-  PYMONGO_4_3:  "4.3.3"
-  PYMONGO_4_4:  "4.4.1"
-  PYMONGO_4_6:  "4.6.2"
-  PYMONGO_4_7:  "4.7.3"
-  PYMONGO_4_8:  "4.8.0"
-  PYMONGO_4_9:  "4.9.2"
-  PYMONGO_4_10: "4.10.1"
-  PYMONGO_4_11: "4.11.2"
-
-  MAIN_PYTHON_VERSION: "3.9"
+  MAIN_PYTHON_VERSION: "3.14"
 
 jobs:
   linting:
     # Run pre-commit (https://pre-commit.com/)
     # which runs pre-configured linter & autoformatter
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: ${{ env.MAIN_PYTHON_VERSION }}
-        check-latest: true
-    - run: bash .github/workflows/install_ci_python_dep.sh
-    - run: pre-commit run -a
+      - uses: actions/checkout@v6
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v7
+      - name: Install Dependencies for lint
+        run: uv sync --only-group dev
+      - name: Install Dependencies for lint
+        run: uv run pre-commit run -a
 
   test:
     # Test suite run against recent python versions
     # and against a few combination of MongoDB and pymongo
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "pypy3.9", "pypy3.10"]
-        MONGODB: [$MONGODB_4_4]
-        PYMONGO: [$PYMONGO_3_12]
-        include:
-          - python-version: "3.9"
-            MONGODB: $MONGODB_4_4
-            PYMONGO: $PYMONGO_3_13
-          - python-version: "3.10"
-            MONGODB: $MONGODB_4_4
-            PYMONGO: $PYMONGO_4_0
-          - python-version: "3.11"
-            MONGODB: $MONGODB_5_0
-            PYMONGO: $PYMONGO_4_3
-          - python-version: "3.11"
-            MONGODB: $MONGODB_6_0
-            PYMONGO: $PYMONGO_4_4
-          - python-version: "3.11"
-            MONGODB: $MONGODB_7_0
-            PYMONGO: $PYMONGO_4_6
-          - python-version: "3.11"
-            MONGODB: $MONGODB_7_0
-            PYMONGO: $PYMONGO_4_7
-          - python-version: "3.11"
-            MONGODB: $MONGODB_7_0
-            PYMONGO: $PYMONGO_4_8
-          - python-version: "3.11"
-            MONGODB: $MONGODB_7_0
-            PYMONGO: $PYMONGO_4_9
-          - python-version: "3.12"
-            MONGODB: $MONGODB_7_0
-            PYMONGO: $PYMONGO_4_9
-          - python-version: "3.12"
-            MONGODB: $MONGODB_8_0
-            PYMONGO: $PYMONGO_4_9
-          - python-version: "3.13"
-            MONGODB: $MONGODB_8_0
-            PYMONGO: $PYMONGO_4_9
-          - python-version: "3.13"
-            MONGODB: $MONGODB_8_0
-            PYMONGO: $PYMONGO_4_10
-          - python-version: "3.13"
-            MONGODB: $MONGODB_8_0
-            PYMONGO: $PYMONGO_4_11
+        python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ]
+        mongodb-version: ["4.4", "5.0", "6.0", "7.0", "8.0" ]
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-        check-latest: true
-    - name: install mongo and ci dependencies
-      run: |
-        bash .github/workflows/install_mongo.sh ${{ matrix.MONGODB }}
-        bash .github/workflows/install_ci_python_dep.sh
-        bash .github/workflows/start_mongo.sh ${{ matrix.MONGODB }}
-    - name: tox dry-run (to pre-install venv)
-      run: tox -e $(echo py${{ matrix.python-version }}-mg${{ matrix.PYMONGO }} | tr -d . | sed -e 's/pypypy/pypy/') -- "-k=test_ci_placeholder"
-    - name: Run test suite
-      run: tox -e $(echo py${{ matrix.python-version }}-mg${{ matrix.PYMONGO }} | tr -d . | sed -e 's/pypypy/pypy/') -- "--cov=mongoengine"
-    - name: Send coverage to Coveralls
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        COVERALLS_SERVICE_NAME: github
-      if: ${{ matrix.python-version == env.MAIN_PYTHON_VERSION }}
-      run: coveralls
+      - uses: actions/checkout@v6
+      - name: Install uv and set the Python version ${{ matrix.python-version }}
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Start MongoDB
+        uses: supercharge/mongodb-github-action@1.12.1
+        with:
+          mongodb-version: ${{ matrix.mongodb-version }}
+          mongodb-replica-set: mongoengine
+      - name: Install Dependencies
+        run: uv sync --only-group test
+      - name: Run test suite
+        run: |
+          envs="$(uv run tox -a | grep py$(echo "${{ matrix.python-version }}" | tr -d . ) || true)"
+          if [ -z "$envs" ]; then
+            echo "Error: No matching tox envs found" >&2
+            exit 1
+          fi
+          echo "Running with: $envs"
+          uv run tox run-parallel -e "$envs" -- "--cov=mongoengine --cov-report="
+          uv run coverage combine
+          uv run coverage report
+  #    - name: Send coverage to Coveralls
+  #      env:
+  #        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  #        COVERALLS_SERVICE_NAME: github
+  #      run: uv run coveralls
 
   build_doc_dryrun:
     # ensures that readthedocs can be built continuously
@@ -127,53 +75,45 @@ jobs:
     # builds are visible at https://readthedocs.org/projects/mongoengine-odm/builds/
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ env.MAIN_PYTHON_VERSION }}
-        check-latest: true
-    - name: install python dep
-      run: |
-        pip install -e .
-        pip install -r docs/requirements.txt
-    - name: build doc
-      run: |
-        cd docs
-        make html-readthedocs
+      - uses: actions/checkout@v6
+      - name: Install uv and set the Python version ${{ env.MAIN_PYTHON_VERSION }}
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ env.MAIN_PYTHON_VERSION }}
+      - name: install python dep
+        run: uv sync --only-group docs
+      - name: build doc
+        run: |
+          cd docs
+          make html-readthedocs
 
   build-dryrun:
     runs-on: ubuntu-latest
-    needs: [linting, test, build_doc_dryrun]
+    needs: [ linting, test, build_doc_dryrun ]
     steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: ${{ env.MAIN_PYTHON_VERSION }}
-        check-latest: true
-    - name: build dummy wheel for test-pypi
-      run: |
-        pip install wheel
-        python setup.py sdist bdist_wheel
+      - uses: actions/checkout@v6
+      - name: Install uv and set the Python version ${{ env.MAIN_PYTHON_VERSION }}
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ env.MAIN_PYTHON_VERSION }}
+      - name: build dummy wheel for test-pypi
+        run: uv build
 
   build-n-publish:
     runs-on: ubuntu-latest
-    needs: [linting, test, build_doc_dryrun, build-dryrun]
+    needs: [ linting, test, build_doc_dryrun, build-dryrun ]
     if: github.event_name == 'create' && startsWith(github.ref, 'refs/tags/v')
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python ${{ env.MAIN_PYTHON_VERSION }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ env.MAIN_PYTHON_VERSION }}
-        check-latest: true
-    # todo separate build from publish
-    # https://stackoverflow.com/questions/59349905/which-properties-does-github-event-in-a-github-workflow-have
-    - name: build dummy wheel for test-pypi
-      run: |
-        pip install wheel
-        python setup.py sdist bdist_wheel
-    - name: publish pypi
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        password: ${{ secrets.pypi_token }}
+      - uses: actions/checkout@v6
+      - name: Install uv and set the Python version ${{ env.MAIN_PYTHON_VERSION }}
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ env.MAIN_PYTHON_VERSION }}
+      # todo separate build from publish
+      # https://stackoverflow.com/questions/59349905/which-properties-does-github-event-in-a-github-workflow-have
+      - name: build dummy wheel for test-pypi
+        run: uv build
+      - name: publish pypi
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.pypi_token }}

.pre-commit-config.yaml

@@ -1,28 +1,22 @@
 fail_fast: false
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: check-merge-conflict
       - id: debug-statements
       - id: trailing-whitespace
       - id: end-of-file-fixer
-  - repo: https://github.com/ambv/black
-    rev: 25.1.0
-    hooks:
-      - id: black
-  - repo: https://github.com/pycqa/flake8
-    rev: 7.2.0
-    hooks:
-      - id: flake8
-        additional_dependencies:
-          - importlib_metadata<5
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.19.1
+    rev: v3.21.2
     hooks:
       - id: pyupgrade
         args: [--py36-plus]
-  - repo: https://github.com/pycqa/isort
-    rev: 6.0.1
-    hooks:
-      - id: isort
+# TODO: Fix lint issues and enable ruff pre-commit
+#  - repo: https://github.com/astral-sh/ruff-pre-commit
+#    rev: v0.14.10
+#    hooks:
+#      # Run the linter.
+#      - id: ruff-check
+#      # Run the formatter.
+#      - id: ruff-format