ci: use pytest-split in ci

.coveragerc

@@ -0,0 +1,38 @@
+# SPDX-FileCopyrightText: Copyright (C) 2025 Opal Health Informatics Group at the Research Institute of the McGill University Health Centre <john.kildea@mcgill.ca>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+[run]
+source = opal
+# store relative paths to make it easier to combine different coverages
+relative_files = True
+omit =
+    .*,
+    **/tests/*,
+    # **/migrations/*,
+    opal/wsgi.py,
+    opal/asgi.py,
+    # omit prod settings
+    opal/settings_prod.py,
+    # omit sidebar menu
+    opal/templates/sidebar_menu.html,
+    opal/templates/components/menu_group.html,
+    opal/templates/components/menu_item.html
+branch = True
+plugins =
+    django_coverage_plugin
+
+[report]
+fail_under = 100
+precision = 2
+show_missing = True
+skip_empty = True
+skip_covered = True
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+    # Don't require coverage on TYPE_CHECKING imports
+    if TYPE_CHECKING:
+
+    # Don't complain if non-runnable code isn't run:
+    if __name__ == .__main__.:

.github/workflows/ci.yml

@@ -114,76 +114,6 @@ jobs:
       #   if: failure()
       #   run: exit 1
 
-  test:
-    runs-on: ubuntu-latest
-    env:
-      DB_ROOT_PASSWORD: "root-password"
-      DB_PASSWORD: "user-password"
-      DB_USER: citest
-    container: python:3.12.9-alpine3.20
-    services:
-      db:
-        image: mariadb:10.11.11-jammy
-        env:
-          MARIADB_ROOT_PASSWORD: ${{ env.DB_ROOT_PASSWORD }}
-          # ensure that user has permissions for test DB to be used by pytest
-          MARIADB_DATABASE: test_opal
-          MARIADB_USER: ${{ env.DB_USER }}
-          MARIADB_PASSWORD: ${{ env.DB_PASSWORD }}
-
-    steps:
-      - name: Install Git
-        run: |
-          apk add --no-cache git git-lfs
-      - uses: actions/checkout@v4.2.2
-        with:
-          persist-credentials: false
-          lfs: true
-      - name: Install dependencies
-        run: |
-          pip install uv
-          echo "Installed uv version is $(uv --version)"
-          # install dependencies for mysqlclient
-          apk add --no-cache build-base mariadb-dev mariadb-client chromium
-          uv sync --locked
-      - name: Prepare environment
-        # set up env file for DB service
-        # use sample env file
-        # create additional DBs for legacy DB tests (OpalDB & QuestionnaireDB)
-        run: |
-          cp .env.sample .env
-          sed -i "s/^DATABASE_USER=.*/DATABASE_USER=$DB_USER/" .env
-          sed -i "s/^DATABASE_PASSWORD=.*/DATABASE_PASSWORD=$DB_PASSWORD/" .env
-          sed -i "s/^DATABASE_HOST=.*/DATABASE_HOST=db/" .env
-          # set up legacy DB connection
-          # reuse the same database as for the other tests to make it easier
-          sed -i "s/^LEGACY_DATABASE_HOST=.*/LEGACY_DATABASE_HOST=db/" .env
-          sed -i "s/^LEGACY_DATABASE_PORT=.*/LEGACY_DATABASE_PORT=3306/" .env
-          sed -i "s/^LEGACY_DATABASE_USER=.*/LEGACY_DATABASE_USER=$DB_USER/" .env
-          sed -i "s/^LEGACY_DATABASE_PASSWORD=.*/LEGACY_DATABASE_PASSWORD=$DB_PASSWORD/" .env
-          # generate secret key
-          SECRET_KEY=$(python -c "import secrets; print(secrets.token_urlsafe())")
-          sed -i "s/^SECRET_KEY=.*/SECRET_KEY=$SECRET_KEY/" .env
-          MYSQL_PWD=$DB_ROOT_PASSWORD mariadb -u root -h db --skip-ssl -e "GRANT ALL PRIVILEGES ON \`test_OpalDB\`.* TO \`$DB_USER\`@\`%\`;"
-          MYSQL_PWD=$DB_ROOT_PASSWORD mariadb -u root -h db --skip-ssl -e "GRANT ALL PRIVILEGES ON \`test_QuestionnaireDB\`.* TO \`$DB_USER\`@\`%\`;"
-      - name: Run pytest
-        run: |
-          uv run pytest --version
-          # -m "" runs all tests, even the ones marked as slow
-          uv run coverage run -m pytest -m "" -v --junitxml=test-report.xml
-      # see: https://github.com/dorny/test-reporter/issues/244
-      # - name: Publish Test Results
-      #   uses: dorny/test-reporter@v1.9.1
-      #   if: '!cancelled()'
-      #   with:
-      #     name: Tests
-      #     path: ./test-report.xml
-      #     reporter: java-junit
-      - name: Check coverage
-        run: |
-          uv run coverage report
-
-
   markdownlint:
     permissions:
       contents: read

.github/workflows/test.yml

@@ -0,0 +1,131 @@
+# SPDX-FileCopyrightText: Copyright (C) 2025 Opal Health Informatics Group at the Research Institute of the McGill University Health Centre <john.kildea@mcgill.ca>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+name: test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+  merge_group:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  seed:
+    runs-on: ubuntu-latest
+    outputs:
+      seed: ${{ steps.calculate-seed.outputs.seed }}
+    steps:
+      - name: Create seed
+        id: calculate-seed
+        run: echo "seed=$(python3 -c 'import random; print(random.Random().getrandbits(32));')" >> "$GITHUB_OUTPUT"
+
+  # usage of pytest-split and matrix strategy based on:
+  # https://github.com/jerry-git/pytest-split-gh-actions-demo/blob/master/.github/workflows/test.yml
+  pytest:
+    runs-on: ubuntu-latest
+    needs: seed
+    strategy:
+      fail-fast: false
+      matrix:
+        group: [1, 2, 3, 4]
+    env:
+      DB_ROOT_PASSWORD: "root-password"
+      DB_PASSWORD: "user-password"
+      DB_USER: citest
+      SEED: ${{ needs.seed.outputs.seed }}
+    container: python:3.12.9-alpine3.20
+    services:
+      db:
+        image: mariadb:10.11.11-jammy
+        env:
+          MARIADB_ROOT_PASSWORD: ${{ env.DB_ROOT_PASSWORD }}
+          # ensure that user has permissions for test DB to be used by pytest
+          MARIADB_DATABASE: test_opal
+          MARIADB_USER: ${{ env.DB_USER }}
+          MARIADB_PASSWORD: ${{ env.DB_PASSWORD }}
+
+    steps:
+      - name: Install Git
+        run: |
+          apk add --no-cache git git-lfs
+      - uses: actions/checkout@v4.2.2
+        with:
+          persist-credentials: false
+          lfs: true
+      - name: Install dependencies
+        run: |
+          pip install uv
+          echo "Installed uv version is $(uv --version)"
+          # install dependencies for mysqlclient
+          apk add --no-cache build-base mariadb-dev mariadb-client chromium
+          uv sync --locked
+      - name: Prepare environment
+        # set up env file for DB service
+        # use sample env file
+        # create additional DBs for legacy DB tests (OpalDB & QuestionnaireDB)
+        run: |
+          cp .env.sample .env
+          sed -i "s/^DATABASE_USER=.*/DATABASE_USER=$DB_USER/" .env
+          sed -i "s/^DATABASE_PASSWORD=.*/DATABASE_PASSWORD=$DB_PASSWORD/" .env
+          sed -i "s/^DATABASE_HOST=.*/DATABASE_HOST=db/" .env
+          # set up legacy DB connection
+          # reuse the same database as for the other tests to make it easier
+          sed -i "s/^LEGACY_DATABASE_HOST=.*/LEGACY_DATABASE_HOST=db/" .env
+          sed -i "s/^LEGACY_DATABASE_PORT=.*/LEGACY_DATABASE_PORT=3306/" .env
+          sed -i "s/^LEGACY_DATABASE_USER=.*/LEGACY_DATABASE_USER=$DB_USER/" .env
+          sed -i "s/^LEGACY_DATABASE_PASSWORD=.*/LEGACY_DATABASE_PASSWORD=$DB_PASSWORD/" .env
+          # generate secret key
+          SECRET_KEY=$(python -c "import secrets; print(secrets.token_urlsafe())")
+          sed -i "s/^SECRET_KEY=.*/SECRET_KEY=$SECRET_KEY/" .env
+          MYSQL_PWD=$DB_ROOT_PASSWORD mariadb -u root -h db --skip-ssl -e "GRANT ALL PRIVILEGES ON \`test_OpalDB\`.* TO \`$DB_USER\`@\`%\`;"
+          MYSQL_PWD=$DB_ROOT_PASSWORD mariadb -u root -h db --skip-ssl -e "GRANT ALL PRIVILEGES ON \`test_QuestionnaireDB\`.* TO \`$DB_USER\`@\`%\`;"
+      - name: Run pytest
+        # use pytest-cov to disable coverage fail
+        # use the same seed for each group
+        run: |
+          uv run pytest --version
+          # -m "" runs all tests, even the ones marked as slow
+          uv run pytest --cov --cov-fail-under=0 --cov-report= --randomly-seed="$SEED" --splitting-algorithm=least_duration --splits 4 --group ${{ matrix.group }} -m ""
+          pwd
+      - name: Upload coverage
+        uses: actions/upload-artifact@v4.6.2
+        with:
+          name: coverage-${{ matrix.group }}
+          path: .coverage
+          include-hidden-files: true
+          if-no-files-found: error
+
+  coverage:
+    needs: pytest
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4.2.2
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@v5.4.1
+        id: setup-uv
+        with:
+          # renovate: datasource=pypi dependency=uv
+          version: "0.6.12"
+      - name: Install dependencies
+        run: uv sync --locked --only-dev
+      - name: Download coverage reports
+        uses: actions/download-artifact@v4.2.1
+      - name: Check coverage
+        run: |
+          pwd
+          ls -la coverage-*
+          ls -la opal/
+          uv run coverage combine coverage*/.coverage*
+          ls -la
+          uv run coverage report

pyproject.toml

@@ -63,10 +63,12 @@ dev = [
     "pandas-stubs==2.2.3.250308",
     "pre-commit==4.2.0",
     "pytest==8.3.5",
+    "pytest-cov==6.1.1",
     "pytest-django==4.11.1",
     "pytest-mock==3.14.0",
     "pytest-randomly==3.16.0",
     "pytest-socket==0.7.0",
+    "pytest-split==0.10.0",
     "pytest-sugar==1.0.0",
     "ruff==0.11.4",
     "types-beautifulsoup4==4.12.0.20250204",

setup.cfg

@@ -44,37 +44,3 @@ warn_unreachable = False
 
 [mypy-*.factories]
 disallow_untyped_calls = False
-
-[coverage:run]
-source = opal
-; include = opal/*
-omit =
-    .*,
-    **/tests/*,
-    # **/migrations/*,
-    opal/wsgi.py,
-    opal/asgi.py,
-    # omit prod settings
-    opal/settings_prod.py,
-    # omit sidebar menu
-    opal/templates/sidebar_menu.html,
-    opal/templates/components/menu_group.html,
-    opal/templates/components/menu_item.html
-branch = True
-plugins =
-    django_coverage_plugin
-
-[coverage:report]
-fail_under = 100
-precision = 2
-show_missing = True
-skip_empty = True
-skip_covered = True
-exclude_lines =
-    # Have to re-enable the standard pragma
-    pragma: no cover
-    # Don't require coverage on TYPE_CHECKING imports
-    if TYPE_CHECKING:
-
-    # Don't complain if non-runnable code isn't run:
-    if __name__ == .__main__.: