diff --git a/.github/RELEASE_CHECKLIST.md b/.github/RELEASE_CHECKLIST.md new file mode 100644 index 000000000..613bb0a46 --- /dev/null +++ b/.github/RELEASE_CHECKLIST.md @@ -0,0 +1,25 @@ +# FerretDB's DocumentDB Release Checklist + +## Preparation + +1. Create draft release on GitHub to see a list of merged PRs. +2. Update CHANGELOG.md manually. It will point to versions of DocumentDB and FerretDB that are not released yet. +3. Update `packaging/debian_files/changelog` and `packaging/rpm_files/documentdb.spec`. +4. Send PR with changes, merge it. + +## Git tag + +1. Make a signed tag with `git tag -s --cleanup=verbatim vX.Y.Z-ferretdb-A.B.C(-p)` (like `v0.103.0-ferretdb-2.2.0-beta.1`), + where `X.Y.Z` is the SemVar formatted version of DocumentDB (like `0.103.0`), + and `A.B.C(-p)` is the compatible FerretDB version (like `2.2.0-beta.1`). +2. Check `git status` output. +3. Push it! + +## Release + +1. Find [Packages CI build](https://github.com/FerretDB/documentdb/actions/workflows/ferretdb_packages.yml?query=event%3Apush) + for the tag to release. +2. Check Docker images. +3. Upload `.deb` and `.rpm` packages to the draft release. +4. Update release notes with the content of CHANGELOG.md. +5. Publish release on GitHub. diff --git a/.github/containers/Build-Ubuntu/Dockerfile b/.github/containers/Build-Ubuntu/Dockerfile index db2d9494c..69c82aa09 100644 --- a/.github/containers/Build-Ubuntu/Dockerfile +++ b/.github/containers/Build-Ubuntu/Dockerfile @@ -130,4 +130,4 @@ WORKDIR /home/documentdb LABEL org.opencontainers.image.source=https://github.com/microsoft/documentdb LABEL org.opencontainers.image.description="DocumentDB ubuntu build image" -LABEL org.opencontainers.image.licenses=MIT \ No newline at end of file +LABEL org.opencontainers.image.licenses=MIT diff --git a/.github/mergify.yml b/.github/mergify.yml new file mode 100644 index 000000000..e722a16b2 --- /dev/null +++ b/.github/mergify.yml @@ -0,0 +1,27 @@ +--- +# see https://docs.mergify.com/ + +pull_request_rules: + - name: "Assign PRs" + conditions: + - "-closed" + - "#assignee = 0" + actions: + assign: + add_users: ["{{ author }}"] + + - name: "Add label on conflicts" + conditions: + - "conflict" + actions: + comment: + message: "@{{author}} this pull request has merge conflicts." + label: + add: [conflict] + + - name: "Remove label when conflicts were resolved" + conditions: + - "-conflict" + actions: + label: + remove: [conflict] diff --git a/.github/settings.yml b/.github/settings.yml new file mode 100644 index 000000000..9edfd2909 --- /dev/null +++ b/.github/settings.yml @@ -0,0 +1,34 @@ +--- +# https://github.com/repository-settings/app + +repository: + allow_squash_merge: true + allow_merge_commit: true + allow_rebase_merge: false + allow_auto_merge: true + allow_update_branch: true + delete_branch_on_merge: true + enable_automated_security_fixes: true + enable_vulnerability_alerts: true + +# https://docs.github.com/en/rest/issues/labels +labels: + - name: conflict + color: "#FF0000" + description: PRs that have merge conflicts + + - name: deps + color: "#D4C5F9" + description: PRs that update dependencies + + - name: not ready + color: "#000000" + description: Issues that are not ready to be worked on; PRs that should skip CI + + - name: packages + color: "#9B022C" + description: PRs that should build packages + + - name: trust + color: "#00FF00" + description: PRs that can access Actions secrets diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 8cf5a2bad..c7db78a99 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -54,7 +54,7 @@ jobs: postgresql-16-cron \ postgresql-16-pgvector \ postgresql-16-postgis-3 \ - postgresql-16-rum + postgresql-16-rum export CLEAN_SETUP=1 export INSTALL_DEPENDENCIES_ROOT=/tmp/install_setup mkdir -p /tmp/install_setup @@ -70,4 +70,4 @@ jobs: sudo make install - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 \ No newline at end of file + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/ferretdb_go_tests.yml b/.github/workflows/ferretdb_go_tests.yml new file mode 100644 index 000000000..dc5230084 --- /dev/null +++ b/.github/workflows/ferretdb_go_tests.yml @@ -0,0 +1,59 @@ +--- +name: Go +on: + pull_request: + types: + - unlabeled # if GitHub Actions stuck, add and remove "not ready" label to force rebuild + - opened + - reopened + - synchronize + push: + branches: + - ferretdb + tags: + - "*" + schedule: + - cron: "12 0 * * *" + +env: + GOPATH: /home/runner/go + GOCACHE: /home/runner/go/cache + GOLANGCI_LINT_CACHE: /home/runner/go/cache/lint + GOMODCACHE: /home/runner/go/mod + GOPROXY: https://proxy.golang.org + GOTOOLCHAIN: local + +jobs: + test: + name: Test + runs-on: ubuntu-24.04 + timeout-minutes: 15 + + # Do not run this job in parallel for any PR change or branch push. + concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + + if: github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'not ready') + + steps: + # TODO https://github.com/FerretDB/github-actions/issues/211 + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: FerretDB/github-actions/setup-go@main + + - name: Run tests + run: | + cd packaging/defineversion + go mod tidy + go mod verify + go test ./... + + - name: Check dirty + if: always() + run: | + git status --untracked-files --ignored + git status + git diff --exit-code diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml new file mode 100644 index 000000000..f18307fdc --- /dev/null +++ b/.github/workflows/ferretdb_packages.yml @@ -0,0 +1,294 @@ +--- +# This and other workflows that use `pull_request_target` event are dangerous +# and should be handled with a lot of care to avoid security problems. +# We use this event to give pull requests access to secrets with permissions to login into Docker registries. +# But rogue PR authors could try to steal our secrets. +# We prevent that with the following: +# +# * We require approval for PRs from first-time contributors. That's a built-in feature for all actions. +# * For workflows that checkout source code, +# we require the `trust` label to be assigned to PRs by FerretDB maintainers after reviewing changes. +# Only a few trusted people have permission to do that. +# * Thanks to the way `pull_request_target` trigger works, +# PR changes in the workflow itself are not run until they are merged. +# * We use short-lived automatic `GITHUB_TOKEN`s instead of a long-living personal access tokens (PAT) where possible. +# * Both `GITHUB_TOKEN`s and PATs have minimal permissions. +# * We publish Docker images from PRs as separate packages that should not be run by users. +# * We limit what third-party actions can be used. +# +# Relevant GitHub documentation is a bit scattered. The first article gives a good overview: +# * https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ +# * https://docs.github.com/en/actions/security-guides/automatic-token-authentication +# * https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions + +name: Packages +on: + pull_request_target: + types: + - unlabeled # if GitHub Actions stuck, add and remove "not ready" label to force rebuild + - opened + - reopened + - synchronize + push: + branches: + - ferretdb + tags: + - "*" + schedule: + - cron: "10 8 * * 1" + +env: + GOPATH: /home/runner/go + GOCACHE: /home/runner/go/cache + GOLANGCI_LINT_CACHE: /home/runner/go/cache/lint + GOMODCACHE: /home/runner/go/mod + GOPROXY: https://proxy.golang.org + GOTOOLCHAIN: local + +# Do not run this workflow in parallel for any PR change or branch/tag push +# to save some resources. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: false + +jobs: + packages: + strategy: + fail-fast: false + matrix: + os: [deb11, deb12, ubuntu22.04, ubuntu24.04, rhel8, rhel9] + arch: [amd64, arm64] + pg: [15, 16, 17] + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + exclude: + # No PostgreSQL 15 for RHEL. + - os: rhel8 + pg: 15 + - os: rhel9 + pg: 15 + # TODO https://github.com/microsoft/documentdb/issues/259 + - arch: arm64 + os: rhel8 + - arch: arm64 + os: rhel9 + + name: ${{ matrix.os }}, ${{ matrix.arch }}, Pg${{ matrix.pg }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 40 + + if: > + github.event_name != 'pull_request_target' || + ( + contains(github.event.pull_request.labels.*.name, 'trust') && + !contains(github.event.pull_request.labels.*.name, 'not ready') && + contains(github.event.pull_request.labels.*.name, 'packages') + ) + + permissions: {} + + steps: + # TODO https://github.com/FerretDB/github-actions/issues/211 + - name: Checkout code + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v4 + with: + fetch-depth: 0 # for `generate_extension_version.sh` to work + + # TODO https://github.com/FerretDB/github-actions/issues/211 + - name: Checkout pull request code + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v4 + with: + fetch-depth: 0 # for `generate_extension_version.sh` to work + ref: ${{ github.event.pull_request.head.sha }} + + - name: Fetch annotated tags + run: | + git fetch --tags --force + git status + + - name: Name branch + if: github.event_name == 'pull_request_target' + env: + BRANCH: ${{ github.head_ref }} # see https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + run: git checkout -b $BRANCH + + - name: Setup Go + uses: FerretDB/github-actions/setup-go@main + + - name: Define version + id: version + run: | + cd packaging/defineversion + go mod tidy + go mod verify + go run . -control-file ../../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} -package-version-only + + - name: Build ${{ steps.version.outputs.package_version }} + if: steps.version.outputs.package_version != '' + run: ./packaging/build_packages.sh --os ${{ matrix.os }} --pg ${{ matrix.pg }} --version ${{ steps.version.outputs.package_version }} --test-clean-install + + - name: Upload packages + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.pg }}-${{ steps.version.outputs.package_version }} + path: | + packaging/*.rpm + packaging/*.deb + retention-days: 1 + if-no-files-found: error + compression-level: 0 + overwrite: false + + - name: Check dirty + run: | + git status + git diff --exit-code + + docker: + name: Docker Pg${{ matrix.pg }} + runs-on: ubuntu-24.04 + timeout-minutes: 40 + + needs: packages + + if: > + github.event_name != 'pull_request_target' || + ( + contains(github.event.pull_request.labels.*.name, 'trust') && + !contains(github.event.pull_request.labels.*.name, 'not ready') && + contains(github.event.pull_request.labels.*.name, 'packages') + ) + + permissions: + packages: write + + strategy: + fail-fast: false + matrix: + pg: [15, 16, 17] + + steps: + # TODO https://github.com/FerretDB/github-actions/issues/211 + - name: Checkout code + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v4 + with: + fetch-depth: 0 # for `generate_extension_version.sh` to work + + # TODO https://github.com/FerretDB/github-actions/issues/211 + - name: Checkout pull request code + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v4 + with: + fetch-depth: 0 # for `generate_extension_version.sh` to work + ref: ${{ github.event.pull_request.head.sha }} + + - name: Fetch annotated tags + run: | + git fetch --tags --force + git status + + - name: Name branch + if: github.event_name == 'pull_request_target' + env: + BRANCH: ${{ github.head_ref }} # see https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + run: git checkout -b $BRANCH + + - name: Setup Go + uses: FerretDB/github-actions/setup-go@main + + - name: Define version + id: version + run: | + cd packaging/defineversion + go mod tidy + go mod verify + go run . -control-file ../../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} + + # We should use deb13 once we switch to Debian 13 Trixie. + # TODO https://github.com/FerretDB/FerretDB/issues/5449 + - name: Download deb12-${{ matrix.pg }}-${{ steps.version.outputs.package_version }} + uses: actions/download-artifact@v4 + with: + pattern: deb12-*-${{ matrix.pg }}-${{ steps.version.outputs.package_version }} + path: packaging + merge-multiple: true + + - name: List files + run: ls -l packaging + + - name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Initialize Docker builder + run: make -C packaging docker-init + + - name: Build local development Docker images + if: steps.version.outputs.docker_development_tag_flags != '' + run: > + make -C packaging docker-build + POSTGRES_VERSION=${{ matrix.pg }} + DOCUMENTDB_VERSION=${{ steps.version.outputs.package_version }} + FILE=development + OUTPUT='type=image' + TAGS='${{ steps.version.outputs.docker_development_tag_flags }}' + + - name: Build local production Docker images + if: steps.version.outputs.docker_production_tag_flags != '' + run: > + make -C packaging docker-build + POSTGRES_VERSION=${{ matrix.pg }} + DOCUMENTDB_VERSION=${{ steps.version.outputs.package_version }} + FILE=production + OUTPUT='type=image' + TAGS='${{ steps.version.outputs.docker_production_tag_flags }}' + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ferretdbbot + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to Quay.io + uses: docker/login-action@v3 + with: + registry: quay.io + username: ferretdb+ferretdbbot + password: ${{ secrets.QUAY_TOKEN }} + + - name: Build and push development Docker images + if: steps.version.outputs.docker_development_tag_flags != '' + run: > + make -C packaging docker-build + POSTGRES_VERSION=${{ matrix.pg }} + DOCUMENTDB_VERSION=${{ steps.version.outputs.package_version }} + FILE=development + OUTPUT='type=image,push=true' + TAGS='${{ steps.version.outputs.docker_development_tag_flags }}' + + - name: Build and push production Docker images + if: steps.version.outputs.docker_production_tag_flags != '' + run: > + make -C packaging docker-build + POSTGRES_VERSION=${{ matrix.pg }} + DOCUMENTDB_VERSION=${{ steps.version.outputs.package_version }} + FILE=production + OUTPUT='type=image,push=true' + TAGS='${{ steps.version.outputs.docker_production_tag_flags }}' + + - name: Check dirty + run: | + git status + git diff --exit-code diff --git a/.github/workflows/regress_tests.yml b/.github/workflows/regress_tests.yml index ecb858942..b468f61cc 100644 --- a/.github/workflows/regress_tests.yml +++ b/.github/workflows/regress_tests.yml @@ -9,6 +9,7 @@ on: push: branches: - 'main' + - 'ferretdb' paths-ignore: - 'docs/**' - '.devcontainer/**' @@ -161,6 +162,6 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: logs_$${{ matrix.runner }}_PG${{ matrix.pg_version }} + name: logs_${{ matrix.runner }}_PG${{ matrix.pg_version }} overwrite: true path: "**/*.log" diff --git a/.gitignore b/.gitignore index 2013d0adc..e6f66eb57 100755 --- a/.gitignore +++ b/.gitignore @@ -70,7 +70,8 @@ build/ # temp schedules *.tmp -# deb packages +# packages *.deb +*.rpm -site/ \ No newline at end of file +site/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 021732afd..d3928ccee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,27 +1,80 @@ -### documentdb v1.107-0 (Unreleased) ### +### DocumentDB v0.107.0-ferretdb-2.7.0 (November 9, 2025) ### + +This version works best with FerretDB v2.7.0. +(We skipped v2.6.0 to align DocumentDB and FerretDB version numbers.) + +Docker images are available +[in the registry](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). +`.deb` packages for Debian and Ubuntu, and `.rpm` packages for Red Hat Enterprise Linux (RHEL) are provided +[on the release page](https://github.com/FerretDB/documentdb/releases/tag/v0.107.0-ferretdb-2.7.0). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/). + +### documentdb v1.107-0 (August 20, 2025) ### - Support sort by _id against the _id index using the enableIndexOrderbyPushdown flag *[Feature]*. - Improvements to explain for various scan types *[Feature]*. -### documentdb v1.106-0 (Unreleased) ### -* Add internal extension that provides extensions to the `rum` index. *[Feature]* +### DocumentDB v0.106.0-ferretdb-2.5.0 (August 12, 2025) ### + +This version works best with FerretDB v2.5.0. + +Docker images are available +[in the registry](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). +`.deb` packages for Debian and Ubuntu, and `.rpm` packages for Red Hat Enterprise Linux (RHEL) are provided +[on the release page](https://github.com/FerretDB/documentdb/releases/tag/v0.106.0-ferretdb-2.5.0). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/). + +### documentdb v1.106-0 (August 07, 2025) ### +* Add internal extension that provides extensions to the `rum` index *[Feature]* * Enable let support for update queries *[Feature]*. Requires `EnableVariablesSupportForWriteCommands` to be `on`. * Enable let support for findAndModify queries *[Feature]*. Requires `EnableVariablesSupportForWriteCommands` to be `on`. -* Add internal extension that provides extensions to the `rum` index. *[Feature]* -* Optimized query for `usersInfo` command. +* Add internal extension that provides extensions to the `rum` index *[Feature]* +* Optimized query for `usersInfo` command * Support collation with `delete` *[Feature]*. Requires `EnableCollation` to be `on`. * Support for index hints for find/aggregate/count/distinct *[Feature]* * Support `createRole` command *[Feature]* * Add schema changes for Role CRUD APIs *[Feature]* * Add support for using EntraId tokens via Plain Auth -### documentdb v0.105-0 (July 28, 2025) ### +### DocumentDB v0.105.0-ferretdb-2.4.0 (July 15, 2025) ### + +This version works best with FerretDB v2.4.0. + +Debian and Ubuntu `.deb` packages are provided +[on the release page](https://github.com/FerretDB/documentdb/releases/tag/v0.105.0-ferretdb-2.4.0). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/deb/). + +Docker images are available +[in the registry](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/docker/). +We always recommend specifying the full image tag (e.g., `17-0.105.0-ferretdb-2.4.0`, not just `17` or `17-0.105.0`) +to avoid unexpected updates. + +### documentdb v0.105-0 (July 09, 2025) ### * Support `$bucketAuto` aggregation stage, with granularity types: `POWERSOF2`, `1-2-5`, `R5`, `R10`, `R20`, `R40`, `R80`, `E6`, `E12`, `E24`, `E48`, `E96`, `E192` *[Feature]* -* Support `conectionStatus` command *[Feature]*. +* Support `connectionStatus` command *[Feature]*. + +### DocumentDB v0.104.0-ferretdb-2.3.0 (June 10, 2025) ### + +This version works best with FerretDB v2.3.0 and v2.3.1. + +> [!NOTE] +> Docker tags `XX-0.104.0-ferretdb-2.3.0` and `XX-0.104.0-ferretdb-2.3.1` point to the same images. +> Additional tags were added to accommodate FerretDB hotfix release 2.3.1. + +Debian and Ubuntu `.deb` packages are provided +[on the release page](https://github.com/FerretDB/documentdb/releases/tag/v0.104.0-ferretdb-2.3.0). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/deb/). + +Docker images are available +[in the registry](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/docker/). +We always recommend specifying the full image tag (e.g., `17-0.104.0-ferretdb-2.3.0`, not just `17` or `17-0.104.0`) +to avoid unexpected updates. ### documentdb v0.104-0 (June 09, 2025) ### * Add string case support for `$toDate` operator -* Support `sort` with collation in runtime*[Feature]* -* Support collation with `$indexOfArray` aggregation operator. *[Feature]* +* Support `sort` with collation in runtime *[Feature]* +* Support collation with `$indexOfArray` aggregation operator *[Feature]* * Support collation with arrays and objects comparisons *[Feature]* * Support background index builds *[Bugfix]* (#36) * Enable user CRUD by default *[Feature]* @@ -29,7 +82,21 @@ * Enable rum_enable_index_scan as default on *[Perf]* * Add public `documentdb-local` Docker image with gateway to GHCR * Support `compact` command *[Feature]*. Requires `documentdb.enablecompact` GUC to be `on`. -* Enable role privileges for `usersInfo` command *[Feature]* +* Enable role privileges for `usersInfo` command *[Feature]* + +### DocumentDB v0.103.0-ferretdb-2.2.0 (May 09, 2025) ### + +This version works best with FerretDB v2.2.0. + +Debian and Ubuntu `.deb` packages are provided +[on the release page](https://github.com/FerretDB/documentdb/releases/tag/v0.103.0-ferretdb-2.2.0). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/deb/). + +Docker images are available +[in the registry](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/docker/). +We always recommend specifying the full image tag (e.g., `17-0.103.0-ferretdb-2.2.0`, not just `17` or `17-0.103.0`) +to avoid unexpected updates. ### documentdb v0.103-0 (May 09, 2025) ### * Support collation with aggregation and find on sharded collections *[Feature]* @@ -40,6 +107,37 @@ * Support collation with `$documents` and `$replaceWith` stage of the aggregation pipeline *[Feature]* * Push pg_documentdb_gw for documentdb connections *[Feature]* +### DocumentDB v0.102.0-ferretdb-2.1.0 (April 2, 2025) ### + +> [!CAUTION] +> Please note that due to incompatibilities in our previous releases, they can't be updated in place, +> even with a manual `ALTER EXTENSION UPDATE` query or other means. +> A new clean installation into an empty data directory/volume is required. +> All data should be backed up with `mongodump`/`mongoexport` before +> and restored with `mongorestore`/`mongoimport` after. +> +> We expect future updates to be much smoother. + +This version works best with the upcoming FerretDB v2.1.0. + +Debian and Ubuntu `.deb` packages are provided [on the release page](https://github.com/FerretDB/documentdb/releases/tag/v0.102.0-ferretdb-2.1.0). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/deb/). + +Docker images are available [in the registry](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/docker/). +We always recommend specifying the full image tag (e.g., `17-0.102.0-ferretdb-2.1.0`, not just `17` or `17-0.102.0`) to avoid unexpected updates. + +### DocumentDB v0.102.0-ferretdb-2.0.0 (GA) (March 5, 2025) ### + +This version works best with [FerretDB v2.0.0 (GA)](https://github.com/FerretDB/FerretDB/releases/tag/v2.0.0). + +Debian and Ubuntu `.deb` packages are provided [on the release page](https://github.com/FerretDB/documentdb/releases/tag/v0.102.0-ferretdb-2.0.0). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/deb/). + +Docker images are available [in the registry](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/docker/). +We always recommend specifying the full image tag (e.g., `17-0.102.0-ferretdb-2.0.0`, not just `17` or `17-0.102.0`) to avoid unexpected updates. + ### documentdb v0.102-0 (March 26, 2025) ### * Support index pushdown for vector search queries *[Bugfix]* * Support exact search for vector search queries *[Feature]* @@ -55,10 +153,10 @@ * Support unique index truncation by default with new operator class *[Feature]* * Top level aggregate command `let` variables support for `$geoNear` stage *[Feature]* * Enable Backend Command support for Statement Timeout *[Feature]* -* Support type aggregation operator `$toUUID`. *[Feature]* +* Support type aggregation operator `$toUUID` *[Feature]* * Support Partial filter pushdown for `$in` predicates *[Perf]* * Support the $dateFromString operator with full functionality *[Feature]* -* Support extended syntax for `$getField` aggregation operator. Now the value of 'field' could be an expression that resolves to a string. *[Feature]* +* Support extended syntax for `$getField` aggregation operator (field as expression) *[Feature]* ### documentdb v0.101-0 (February 12, 2025) ### * Push $graphlookup recursive CTE JOIN filters to index *[Perf]* @@ -66,7 +164,7 @@ * Enable support of currentOp aggregation stage, along with collstats, dbstats, and indexStats *[Commands]* (#52) * Allow inlining $unwind with $lookup with `preserveNullAndEmptyArrays` *[Perf]* * Skip loading documents if group expression is constant *[Perf]* -* Fix Merge stage not outputing to target collection *[Bugfix]* (#20) +* Fix Merge stage not outputting to target collection *[Bugfix]* (#20) ### documentdb v0.100-0 (January 23rd, 2025) ### Initial Release diff --git a/CODEOWNERS b/CODEOWNERS index 6c6098c52..8d47a1941 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,11 +1 @@ -* @microsoft/documentdb-contributors - -/.pipelines/ @safern @visridha @shuaitian-git @lichoil -/.github/workflows/ @safern @visridha @shuaitian-git @lichoil - -*.csv @microsoft/documentdb-engine-reviewers -*.sql @microsoft/documentdb-engine-reviewers -Make* @microsoft/documentdb-engine-reviewers -/internal/ @microsoft/documentdb-engine-reviewers -/pg_documentdb/ @microsoft/documentdb-engine-reviewers -/pg_documentdb_core/ @microsoft/documentdb-engine-reviewers \ No newline at end of file +* @AlekSi diff --git a/packaging/.gitignore b/packaging/.gitignore new file mode 100644 index 000000000..6216440af --- /dev/null +++ b/packaging/.gitignore @@ -0,0 +1,2 @@ +# defineversion's +!go.mod diff --git a/packaging/10-preload.sh b/packaging/10-preload.sh new file mode 100755 index 000000000..1e156df7c --- /dev/null +++ b/packaging/10-preload.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e + +# https://github.com/microsoft/documentdb/tree/main/pg_documentdb/src/configs + +cat <> $PGDATA/postgresql.conf +shared_preload_libraries = 'pg_cron,pg_documentdb_core,pg_documentdb' +cron.database_name = 'postgres' + +documentdb.enableCompact = true + +documentdb.enableLetAndCollationForQueryMatch = true +documentdb.enableNowSystemVariable = true +documentdb.enableSortbyIdPushDownToPrimaryKey = true + +documentdb.enableSchemaValidation = true +documentdb.enableBypassDocumentValidation = true + +documentdb.enableUserCrud = true +documentdb.maxUserLimit = 100 +EOT + +source /usr/local/bin/docker-entrypoint.sh + +# restarting is needed to create extension +docker_temp_server_stop +docker_temp_server_start "$@" diff --git a/packaging/20-install.sql b/packaging/20-install.sql new file mode 100644 index 000000000..34c7b1140 --- /dev/null +++ b/packaging/20-install.sql @@ -0,0 +1 @@ +CREATE EXTENSION documentdb CASCADE; diff --git a/packaging/90-install-development.sql b/packaging/90-install-development.sql new file mode 100644 index 000000000..070b769a5 --- /dev/null +++ b/packaging/90-install-development.sql @@ -0,0 +1 @@ +CREATE EXTENSION pgtap; diff --git a/packaging/Dockerfile_build_deb_packages b/packaging/Dockerfile_build_deb_packages old mode 100755 new mode 100644 index 9fa4ff630..ca1553f56 --- a/packaging/Dockerfile_build_deb_packages +++ b/packaging/Dockerfile_build_deb_packages @@ -2,7 +2,7 @@ ARG BASE_IMAGE=debian:bookworm FROM ${BASE_IMAGE} ARG DEBIAN_FRONTEND=noninteractive -ARG POSTGRES_VERSION=16 +ARG POSTGRES_VERSION ARG DOCUMENTDB_VERSION RUN test -n "$DOCUMENTDB_VERSION" || (echo "DOCUMENTDB_VERSION not set" && false) diff --git a/packaging/Makefile b/packaging/Makefile new file mode 100644 index 000000000..025e04c04 --- /dev/null +++ b/packaging/Makefile @@ -0,0 +1,37 @@ +.PHONY: docker-init docker-build docker-cleanup + +docker-init: + docker buildx create \ + --driver=docker-container \ + --name=documentdb \ + --bootstrap=true \ + --use=false \ + --config=./buildkitd.toml \ + --driver-opt network=host \ + --driver-opt env.JAEGER_TRACE=127.0.0.1:6831 \ + --driver-opt env.BUILDKIT_STEP_LOG_MAX_SIZE=-1 \ + --driver-opt env.BUILDKIT_STEP_LOG_MAX_SPEED=-1 \ + || true + docker buildx ls + +docker-build: + test -n "$(POSTGRES_VERSION)" || (echo "POSTGRES_VERSION not set" && false) + test -n "$(DOCUMENTDB_VERSION)" || (echo "DOCUMENTDB_VERSION not set" && false) + test -n "$(FILE)" || (echo "FILE not set" && false) + test -n "$(OUTPUT)" || (echo "OUTPUT not set" && false) + test -n "$(TAGS)" || (echo "TAGS not set" && false) + docker buildx build --builder=documentdb \ + --file=$(FILE).Dockerfile \ + --build-arg='POSTGRES_VERSION=$(POSTGRES_VERSION)' \ + --build-arg='DOCUMENTDB_VERSION=$(DOCUMENTDB_VERSION)' \ + --platform=linux/amd64,linux/arm64 \ + --output='$(OUTPUT)' \ + $(TAGS) \ + .. + +docker-cleanup: + docker system df + docker buildx --builder=documentdb du || true + docker buildx --builder=documentdb rm --force || true + docker system prune --force + docker system df diff --git a/packaging/README.md b/packaging/README.md old mode 100755 new mode 100644 index ff9dc1a30..00b7812c2 --- a/packaging/README.md +++ b/packaging/README.md @@ -3,10 +3,10 @@ ## Building Debian/Ubuntu Packages Run `./packaging/build_packages.sh -h` and follow the instructions. -E.g. to build for Debian 12 and PostgreSQL 16, run: +E.g. to build for Debian 12 and PostgreSQL 16 for `0.102.0~ferretdb~2.0.0~rc.2` version, run: ```sh -./packaging/build_packages.sh --os deb12 --pg 16 +./packaging/build_packages.sh --os deb12 --pg 16 --version 0.102.0~ferretdb~2.0.0~rc.2 ``` ## Building RPM Packages @@ -50,4 +50,4 @@ This script checks: Packages can be found at the `packages` directory by default, but it can be configured with the `--output-dir` option. -**Note:** The packages do not include pg_documentdb_distributed in the `internal` directory. \ No newline at end of file +**Note:** The packages do not include pg_documentdb_distributed in the `internal` directory. diff --git a/packaging/build_packages.sh b/packaging/build_packages.sh index 6624cc069..8449198ed 100755 --- a/packaging/build_packages.sh +++ b/packaging/build_packages.sh @@ -4,7 +4,7 @@ set -euo pipefail # Function to display help message function show_help { - echo "Usage: $0 --os --pg [--test-clean-install] [--output-dir ] [-h|--help]" + echo "Usage: $0 --os --pg --version [--test-clean-install] [--output-dir ] [-h|--help]" echo "" echo "Description:" echo " This script builds extension packages (DEB/RPM) using Docker." @@ -12,9 +12,9 @@ function show_help { echo "Mandatory Arguments:" echo " --os OS to build packages for. Possible values: [deb11, deb12, ubuntu22.04, ubuntu24.04, rhel8, rhel9]" echo " --pg PG version to build packages for. Possible values: [15, 16, 17]" + echo " --version The version of documentdb to build. Examples: [0.100.0, 0.101.0]" echo "" echo "Optional Arguments:" - echo " --version The version of documentdb to build. Examples: [0.100.0, 0.101.0]" echo " --test-clean-install Test installing the packages in a clean Docker container." echo " --output-dir Relative path from the repo root of the directory where to drop the packages. The directory will be created if it doesn't exist. Default: packaging" echo " -h, --help Display this help message." @@ -97,16 +97,10 @@ if [[ -z "$PG" ]]; then exit 1 fi -# get the version from control file if [[ -z "$DOCUMENTDB_VERSION" ]]; then - DOCUMENTDB_VERSION=$(grep -E "^default_version" pg_documentdb_core/documentdb_core.control | sed -E "s/.*'([0-9]+\.[0-9]+-[0-9]+)'.*/\1/") - DOCUMENTDB_VERSION=$(echo $DOCUMENTDB_VERSION | sed "s/-/./g") - echo "DOCUMENTDB_VERSION extracted from control file: $DOCUMENTDB_VERSION" - if [[ -z "$DOCUMENTDB_VERSION" ]]; then - echo "Error: --version is required and could not be found in the control file." - show_help - exit 1 - fi + echo "Error: --version is required." + show_help + exit 1 fi # Set the appropriate Docker image and configuration based on the OS @@ -198,22 +192,22 @@ if [[ $TEST_CLEAN_INSTALL == true ]]; then docker run --rm documentdb-test-packages:latest elif [[ "$PACKAGE_TYPE" == "rpm" ]]; then - rpm_package_name=$(ls "$abs_output_dir" | grep -E "${OS}-postgresql${PG}-documentdb-${DOCUMENTDB_VERSION}.*\.x86_64\.rpm" | head -n 1) + rpm_package_name=$(ls "$abs_output_dir" | grep -E "${OS}-postgresql${PG}-documentdb-${DOCUMENTDB_VERSION}.*\.rpm" | head -n 1) if [[ -z "$rpm_package_name" ]]; then echo "Error: Could not find the built RPM package in $abs_output_dir for testing." exit 1 fi package_rel_path="$OUTPUT_DIR/$rpm_package_name" - + echo "RPM package path for testing: $package_rel_path" - + # Build the Docker image while showing the output to the console docker build -t documentdb-test-rpm-packages:latest -f packaging/test_packages/Dockerfile_test_install_rpm_packages \ --build-arg BASE_IMAGE="$DOCKER_IMAGE" \ --build-arg POSTGRES_VERSION="$PG" \ --build-arg RPM_PACKAGE_REL_PATH="$package_rel_path" \ --build-arg OS_VERSION_ARG="$OS_VERSION_NUMBER" . - + # Run the Docker container to test the packages docker run --rm --env POSTGRES_VERSION="$PG" documentdb-test-rpm-packages:latest fi @@ -221,4 +215,4 @@ if [[ $TEST_CLEAN_INSTALL == true ]]; then echo "Clean installation test successful!!" fi -echo "Packages are available in $abs_output_dir" \ No newline at end of file +echo "Packages are available in $abs_output_dir" diff --git a/packaging/buildkitd.toml b/packaging/buildkitd.toml new file mode 100644 index 000000000..1add01ddb --- /dev/null +++ b/packaging/buildkitd.toml @@ -0,0 +1,5 @@ +[worker.oci] + gc = false + +[worker.containerd] + gc = false diff --git a/packaging/debian_files/changelog b/packaging/debian_files/changelog old mode 100755 new mode 100644 index 3b2e60738..572056329 --- a/packaging/debian_files/changelog +++ b/packaging/debian_files/changelog @@ -1,37 +1,51 @@ +documentdb (DOCUMENTDB_VERSION) unstable; urgency=medium + + * Add internal extension that provides extensions to the rum index [Feature] + * Enable let support for update queries [Feature]. Requires EnableVariablesSupportForWriteCommands to be on. + * Enable let support for findAndModify queries [Feature]. Requires EnableVariablesSupportForWriteCommands to be on. + * Optimized query for usersInfo command + * Support collation with delete [Feature]. Requires EnableCollation to be on. + * Support for index hints for find/aggregate/count/distinct [Feature] + * Support createRole command [Feature] + * Add schema changes for Role CRUD APIs [Feature] + * Add support for using EntraId tokens via Plain Auth + + -- FerretDB Packages Thu, 07 Aug 2025 12:00:00 +0000 + documentdb (0.105-0) unstable; urgency=medium -* Support `$bucketAuto` aggregation stage, with granularity types: `POWERSOF2`, `1-2-5`, `R5`, `R10`, `R20`, `R40`, `R80`, `E6`, `E12`, `E24`, `E48`, `E96`, `E192` *[Feature]* -* Support `conectionStatus` command *[Feature]*. + * Support $bucketAuto aggregation stage, with granularity types: POWERSOF2, 1-2-5, R5, R10, R20, R40, R80, E6, E12, E24, E48, E96, E192 [Feature] + * Support connectionStatus command [Feature] - -- Shuai Tian Mon, 28 Jul 2025 12:00:00 +0000 + -- FerretDB Packages Wed, 09 Jul 2025 12:00:00 +0000 documentdb (0.104-0) unstable; urgency=medium -* Add string case support for `$toDate` operator -* Support `sort` with collation in runtime*[Feature]* -* Support collation with `$indexOfArray` aggregation operator. *[Feature]* -* Support collation with arrays and objects comparisons *[Feature]* -* Support background index builds *[Bugfix]* (#36) -* Enable user CRUD by default *[Feature]* -* Enable let support for delete queries *[Feature]*. Requires `EnableVariablesSupportForWriteCommands` to be `on`. -* Enable rum_enable_index_scan as default on *[Perf]* -* Add public `documentdb-local` Docker image with gateway to GHCR -* Support `compact` command *[Feature]*. Requires `documentdb.enablecompact` GUC to be `on`. -* Enable role privileges for `usersInfo` command *[Feature]* + * Add string case support for $toDate operator + * Support sort with collation in runtime [Feature] + * Support collation with $indexOfArray aggregation operator [Feature] + * Support collation with arrays and objects comparisons [Feature] + * Support background index builds [Bugfix] (#36) + * Enable user CRUD by default [Feature] + * Enable let support for delete queries [Feature]. Requires EnableVariablesSupportForWriteCommands to be on. + * Enable rum_enable_index_scan as default on [Perf] + * Add public documentdb-local Docker image with gateway to GHCR + * Support compact command [Feature]. Requires documentdb.enablecompact GUC to be on. + * Enable role privileges for usersInfo command [Feature] - -- Shuai Tian Mon, 09 Jun 2025 12:00:00 +0000 + -- FerretDB Packages Mon, 09 Jun 2025 12:00:00 +0000 documentdb (0.103-0) unstable; urgency=medium -* Support collation with aggregation and find on sharded collections *[Feature]* -* Support `$convert` on `binData` to `binData`, `string` to `binData` and `binData` to `string` (except with `format: auto`) *[Feature]* -* Fix list_databases for databases with size > 2 GB *[Bugfix]* (#119) -* Support half-precision vector indexing, vectors can have up to 4,000 dimensions *[Feature]* -* Support ARM64 architecture when building docker container *[Preview]* -* Support collation with `$documents` and `$replaceWith` stage of the aggregation pipeline *[Feature]* -* Push pg_documentdb_gw for documentdb connections *[Feature]* + * Support collation with aggregation and find on sharded collections [Feature] + * Support $convert on binData to binData, string to binData and binData to string (except with format: auto) [Feature] + * Fix list_databases for databases with size > 2 GB [Bugfix] (#119) + * Support half-precision vector indexing, vectors can have up to 4,000 dimensions [Feature] + * Support ARM64 architecture when building docker container [Preview] + * Support collation with $documents and $replaceWith stage of the aggregation pipeline [Feature] + * Push pg_documentdb_gw for documentdb connections [Feature] - -- Shuai Tian Fri, 09 May 2025 12:00:00 +0000 + -- FerretDB Packages Fri, 09 May 2025 12:00:00 +0000 documentdb (0.102-0) unstable; urgency=medium @@ -54,7 +68,7 @@ documentdb (0.102-0) unstable; urgency=medium * Support the $dateFromString operator with full functionality [Feature] * Support extended syntax for $getField aggregation operator (field as expression) [Feature] - -- Shuai Tian Wed, 26 Mar 2025 10:00:00 +0000 + -- FerretDB Packages Wed, 26 Mar 2025 10:00:00 +0000 documentdb (0.101-0) unstable; urgency=medium @@ -65,10 +79,10 @@ documentdb (0.101-0) unstable; urgency=medium * Skip loading documents if group expression is constant [Perf] * Fix Merge stage not outputting to target collection [Bugfix] (#20) - -- Shuai Tian Tue, 12 Feb 2025 10:00:00 +0000 + -- FerretDB Packages Tue, 12 Feb 2025 10:00:00 +0000 documentdb (0.100-0) unstable; urgency=medium * Initial release - -- Shuai Tian Thu, 23 Jan 2025 10:00:00 +0000 \ No newline at end of file + -- FerretDB Packages Thu, 23 Jan 2025 10:00:00 +0000 diff --git a/packaging/debian_files/compat b/packaging/debian_files/compat old mode 100755 new mode 100644 index 3cacc0b93..48082f72f --- a/packaging/debian_files/compat +++ b/packaging/debian_files/compat @@ -1 +1 @@ -12 \ No newline at end of file +12 diff --git a/packaging/debian_files/control b/packaging/debian_files/control old mode 100755 new mode 100644 index 959ba2e7e..c7719d2eb --- a/packaging/debian_files/control +++ b/packaging/debian_files/control @@ -2,9 +2,9 @@ Source: documentdb Section: database Priority: optional Build-Depends: debhelper (>= 11), postgresql-server-dev-POSTGRES_VERSION -Maintainer: Shuai Tian +Maintainer: FerretDB Packages Package: postgresql-POSTGRES_VERSION-documentdb Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends}, postgresql-POSTGRES_VERSION, postgresql-POSTGRES_VERSION-cron, postgresql-POSTGRES_VERSION-pgvector, postgresql-POSTGRES_VERSION-postgis-3, postgresql-POSTGRES_VERSION-rum -Description: DocumentDB is the open-source engine powering vCore-based Azure Cosmos DB for MongoDB. It offers a native implementation of document-oriented NoSQL database, enabling seamless CRUD operations on BSON data types within a PostgreSQL framework. \ No newline at end of file +Description: DocumentDB is the open-source engine powering vCore-based Azure Cosmos DB for MongoDB. It offers a native implementation of document-oriented NoSQL database, enabling seamless CRUD operations on BSON data types within a PostgreSQL framework. diff --git a/packaging/debian_files/rules b/packaging/debian_files/rules old mode 100755 new mode 100644 index 3478d0506..363dd034d --- a/packaging/debian_files/rules +++ b/packaging/debian_files/rules @@ -7,4 +7,4 @@ override_dh_auto_test: make install adduser --disabled-password --gecos "" documentdb chown -R documentdb:documentdb . - su documentdb -c "make check" \ No newline at end of file + su documentdb -c "make check" diff --git a/packaging/defineversion/docker.go b/packaging/defineversion/docker.go new file mode 100644 index 000000000..ebe2c47e8 --- /dev/null +++ b/packaging/defineversion/docker.go @@ -0,0 +1,21 @@ +package main + +import ( + "fmt" + "strings" +) + +// dockerImageURL returns HTML page URL for the given image name and tag. +func dockerImageURL(name string) string { + switch { + case strings.HasPrefix(name, "ghcr.io/"): + return fmt.Sprintf("https://%s", name) + case strings.HasPrefix(name, "quay.io/"): + return fmt.Sprintf("https://%s", name) + } + + name, _, _ = strings.Cut(name, ":") + + // there is no easy way to get Docker Hub URL for the given tag + return fmt.Sprintf("https://hub.docker.com/r/%s/tags", name) +} diff --git a/packaging/defineversion/docker_test.go b/packaging/defineversion/docker_test.go new file mode 100644 index 000000000..9911ba38b --- /dev/null +++ b/packaging/defineversion/docker_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestImageURL(t *testing.T) { + // expected URLs should work + assert.Equal( + t, + "https://ghcr.io/ferretdb/postgres-documentdb-dev:pr-docker-tag", + dockerImageURL("ghcr.io/ferretdb/postgres-documentdb-dev:pr-docker-tag"), + ) + assert.Equal( + t, + "https://quay.io/ferretdb/postgres-documentdb-dev:pr-docker-tag", + dockerImageURL("quay.io/ferretdb/postgres-documentdb-dev:pr-docker-tag"), + ) + assert.Equal( + t, + "https://hub.docker.com/r/ferretdb/postgres-documentdb-dev/tags", + dockerImageURL("ferretdb/postgres-documentdb-dev:pr-docker-tag"), + ) +} diff --git a/packaging/defineversion/go.mod b/packaging/defineversion/go.mod new file mode 100644 index 000000000..4aae104ae --- /dev/null +++ b/packaging/defineversion/go.mod @@ -0,0 +1,16 @@ +module github.com/FerretDB/documentdb/packaging/defineversion + +go 1.25 + +toolchain go1.25.4 + +require ( + github.com/sethvargo/go-githubactions v1.3.1 + github.com/stretchr/testify v1.11.1 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/packaging/defineversion/go.sum b/packaging/defineversion/go.sum new file mode 100644 index 000000000..2402ca0e4 --- /dev/null +++ b/packaging/defineversion/go.sum @@ -0,0 +1,12 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sethvargo/go-githubactions v1.3.1 h1:rlwwLRUaunWLQ1aN2o5Y+3s0xhaTC30YObCnilRx448= +github.com/sethvargo/go-githubactions v1.3.1/go.mod h1:7/4WeHgYfSz9U5vwuToCK9KPnELVHAhGtRwLREOQV80= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packaging/defineversion/main.go b/packaging/defineversion/main.go new file mode 100644 index 000000000..ef2501117 --- /dev/null +++ b/packaging/defineversion/main.go @@ -0,0 +1,346 @@ +// Package main defines version numbers for DocumentDB. +package main + +import ( + "flag" + "fmt" + "os" + "regexp" + "slices" + "strconv" + "strings" + "text/tabwriter" + + "github.com/sethvargo/go-githubactions" +) + +// semVerTag is a https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string, +// but with a leading `v`. +var semVerTag = regexp.MustCompile(`^v(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`) + +// versions represents Docker image names and tags, and package versions used by Debian and RPM. +type versions struct { + dockerDevelopmentImages []string + dockerProductionImages []string + packageVersion string +} + +// parseGitTag parses git tag in specific format and returns SemVer components. +// +// Expected format is v0.103.0-ferretdb-2.2.0-beta.1, +// where v0.103.0 is a DocumentDB version (0.103-0 -> 0.103.0), +// and ferretdb-2.2.0-beta.1 is a compatible FerretDB version. +func parseGitTag(tag string) (major, minor, patch int, prerelease string, err error) { + match := semVerTag.FindStringSubmatch(tag) + if match == nil || len(match) != semVerTag.NumSubexp()+1 { + err = fmt.Errorf("unexpected git tag format %q", tag) + return + } + + if major, err = strconv.Atoi(match[semVerTag.SubexpIndex("major")]); err != nil { + return + } + if minor, err = strconv.Atoi(match[semVerTag.SubexpIndex("minor")]); err != nil { + return + } + if patch, err = strconv.Atoi(match[semVerTag.SubexpIndex("patch")]); err != nil { + return + } + prerelease = match[semVerTag.SubexpIndex("prerelease")] + buildmetadata := match[semVerTag.SubexpIndex("buildmetadata")] + + if !strings.HasPrefix(prerelease, "ferretdb-") { + err = fmt.Errorf("prerelease %q should start with 'ferretdb-'", prerelease) + return + } + + if buildmetadata != "" { + err = fmt.Errorf("buildmetadata %q is present", buildmetadata) + return + } + + return +} + +// debugEnv logs all environment variables that start with `GITHUB_` or `INPUT_` +// in debug level. +func debugEnv(action *githubactions.Action) { + res := make([]string, 0, 30) + + for _, l := range os.Environ() { + if strings.HasPrefix(l, "GITHUB_") || strings.HasPrefix(l, "INPUT_") { + res = append(res, l) + } + } + + slices.Sort(res) + + action.Debugf("Dumping environment variables:") + + for _, l := range res { + action.Debugf("\t%s", l) + } +} + +// defineVersion extracts Docker image names and tags, and package versions used by Debian and RPM +// using the environment variables defined by GitHub Actions. +// +// The Debian and RPM package versions are based on `default_version` in the control file. +// See https://www.debian.org/doc/debian-policy/ch-controlfields.html#version, +// and https://fedoraproject.org/wiki/PackagingDrafts/TildeVersioning#Basic_versioning_rules. +// The Debian uses `upstream_version` only. +// For that reason, we can't use `-`, so we replace it with `~`. +func defineVersion(controlDefaultVersion, pgVersion string, getenv githubactions.GetenvFunc) (*versions, error) { + repo := getenv("GITHUB_REPOSITORY") + + // to support GitHub forks + parts := strings.Split(strings.ToLower(repo), "/") + if len(parts) != 2 { + return nil, fmt.Errorf("failed to split %q into owner and name", repo) + } + owner := parts[0] + repo = parts[1] + + var res *versions + var err error + + switch event := getenv("GITHUB_EVENT_NAME"); event { + case "pull_request", "pull_request_target": + branch := strings.ToLower(getenv("GITHUB_HEAD_REF")) + res = defineVersionForPR(controlDefaultVersion, pgVersion, owner, repo, branch) + + case "push", "schedule", "workflow_run": + refName := strings.ToLower(getenv("GITHUB_REF_NAME")) + + switch refType := strings.ToLower(getenv("GITHUB_REF_TYPE")); refType { + case "branch": + res, err = defineVersionForBranch(controlDefaultVersion, pgVersion, owner, repo, refName) + + case "tag": + res, err = defineVersionForTag(controlDefaultVersion, pgVersion, owner, repo, refName) + + default: + err = fmt.Errorf("unhandled ref type %q for event %q", refType, event) + } + + default: + err = fmt.Errorf("unhandled event type %q", event) + } + + if err != nil { + return nil, err + } + + if res == nil { + return nil, fmt.Errorf("both res and err are nil") + } + + slices.Sort(res.dockerDevelopmentImages) + slices.Sort(res.dockerProductionImages) + + return res, nil +} + +// defineVersionForPR defines Docker image names and tags, and package version for PR. +// See [defineVersion]. +func defineVersionForPR(controlDefaultVersion, pgVersion, owner, repo, branch string) *versions { + // for branches like "dependabot/submodules/XXX" + parts := strings.Split(branch, "/") + branch = parts[len(parts)-1] + + packageVersion := fmt.Sprintf("%s-pr-%s", controlDefaultVersion, branch) + packageVersion = disallowedDebian.ReplaceAllString(packageVersion, "~") + packageVersion = disallowedRPM.ReplaceAllString(packageVersion, "~") + + res := &versions{ + dockerDevelopmentImages: []string{ + fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-pr-%s", owner, repo, pgVersion, branch), + }, + dockerProductionImages: []string{ + fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-pr-%s-prod", owner, repo, pgVersion, branch), + }, + packageVersion: packageVersion, + } + + // PRs are only for testing; no Quay.io and Docker Hub repos + + return res +} + +// defineVersionForBranch defines Docker image names and tags, and package version for branch. +// See [defineVersion]. +func defineVersionForBranch(controlDefaultVersion, pgVersion, owner, repo, branch string) (*versions, error) { + if branch != "ferretdb" { + return nil, fmt.Errorf("unhandled branch %q", branch) + } + + res := &versions{ + dockerDevelopmentImages: []string{ + fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-ferretdb", owner, repo, pgVersion), + }, + dockerProductionImages: []string{ + fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-ferretdb-prod", owner, repo, pgVersion), + }, + packageVersion: fmt.Sprintf("%s~ferretdb", controlDefaultVersion), + } + + // forks don't have Quay.io and Docker Hub orgs + if owner != "ferretdb" { + return res, nil + } + + // we don't have Quay.io and Docker Hub repos for other GitHub repos + if repo != "documentdb" { + return res, nil + } + + res.dockerDevelopmentImages = append(res.dockerDevelopmentImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb-dev:%s-ferretdb", pgVersion)) + res.dockerProductionImages = append(res.dockerProductionImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb-dev:%s-ferretdb-prod", pgVersion)) + res.dockerDevelopmentImages = append(res.dockerDevelopmentImages, fmt.Sprintf("ferretdb/postgres-documentdb-dev:%s-ferretdb", pgVersion)) + res.dockerProductionImages = append(res.dockerProductionImages, fmt.Sprintf("ferretdb/postgres-documentdb-dev:%s-ferretdb-prod", pgVersion)) + + return res, nil +} + +// defineVersionForTag defines Docker image names and tags, and package version for tag. +// See [defineVersion]. +func defineVersionForTag(controlDefaultVersion, pgVersion, owner, repo, tag string) (*versions, error) { + major, minor, patch, prerelease, err := parseGitTag(tag) + if err != nil { + return nil, err + } + + tagVersion := fmt.Sprintf("%d.%d.%d", major, minor, patch) + if tagVersion != controlDefaultVersion { + return nil, fmt.Errorf("git tag version %q does not match the control file default version %q", tagVersion, controlDefaultVersion) + } + + tags := []string{ + fmt.Sprintf("%s-%s-%s", pgVersion, tagVersion, prerelease), + fmt.Sprintf("%s-%s", pgVersion, tagVersion), + fmt.Sprintf("%s", pgVersion), + } + + if pgVersion == "17" { + tags = append(tags, "latest") + } + + packageVersion := fmt.Sprintf("%s-%s", tagVersion, prerelease) + packageVersion = disallowedDebian.ReplaceAllString(packageVersion, "~") + packageVersion = disallowedRPM.ReplaceAllString(packageVersion, "~") + + res := versions{ + packageVersion: packageVersion, + } + + for _, t := range tags { + res.dockerDevelopmentImages = append(res.dockerDevelopmentImages, fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s", owner, repo, t)) + res.dockerProductionImages = append(res.dockerProductionImages, fmt.Sprintf("ghcr.io/%s/postgres-%s:%s", owner, repo, t)) + } + + // forks don't have Quay.io and Docker Hub orgs + if owner != "ferretdb" { + return &res, nil + } + + // we don't have Quay.io and Docker Hub repos for other GitHub repos + if repo != "documentdb" { + return &res, nil + } + + for _, t := range tags { + res.dockerDevelopmentImages = append(res.dockerDevelopmentImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb-dev:%s", t)) + res.dockerProductionImages = append(res.dockerProductionImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb:%s", t)) + + res.dockerDevelopmentImages = append(res.dockerDevelopmentImages, fmt.Sprintf("ferretdb/postgres-documentdb-dev:%s", t)) + res.dockerProductionImages = append(res.dockerProductionImages, fmt.Sprintf("ferretdb/postgres-documentdb:%s", t)) + } + + return &res, nil +} + +// setSummary sets action summary. +func setSummary(action *githubactions.Action, version *versions) { + var buf strings.Builder + + fmt.Fprintf(&buf, "Package version (Debian with `upstream_version` only, or RPM): `%s`\n\n", version.packageVersion) + + w := tabwriter.NewWriter(&buf, 1, 1, 1, ' ', tabwriter.Debug) + fmt.Fprintf(w, "\tType\tDocker image\t\n") + fmt.Fprintf(w, "\t----\t------------\t\n") + + for _, image := range version.dockerDevelopmentImages { + u := dockerImageURL(image) + _, _ = fmt.Fprintf(w, "\tDevelopment\t[`%s`](%s)\t\n", image, u) + } + + for _, image := range version.dockerProductionImages { + u := dockerImageURL(image) + _, _ = fmt.Fprintf(w, "\tProduction\t[`%s`](%s)\t\n", image, u) + } + + _ = w.Flush() + + action.AddStepSummary(buf.String()) + action.Infof("%s", buf.String()) +} + +func main() { + controlFileF := flag.String("control-file", "../pg_documentdb/documentdb.control", "pg_documentdb/documentdb.control file path") + pgVersionF := flag.String("pg-version", "17", "Major PostgreSQL version") + packageVersionOnlyF := flag.Bool("package-version-only", false, "Only set output for package version") + + flag.Parse() + + action := githubactions.New() + + debugEnv(action) + + if *controlFileF == "" { + action.Fatalf("%s", "-control-file flag is empty.") + } + + switch *pgVersionF { + case "15", "16", "17": + // nothing + default: + action.Fatalf("%s", fmt.Sprintf("Invalid PostgreSQL version %q.", *pgVersionF)) + } + + controlDefaultVersion, err := getControlDefaultVersion(*controlFileF) + if err != nil { + action.Fatalf("%s", err) + } + + res, err := defineVersion(controlDefaultVersion, *pgVersionF, action.Getenv) + if err != nil { + action.Fatalf("%s", err) + } + + action.SetOutput("package_version", res.packageVersion) + + if *packageVersionOnlyF { + // Only 3 summaries are shown in the GitHub Actions UI by default, + // and Docker summaries are more important (and include package version anyway). + action.Infof("package version (Debian with `upstream_version` only, or RPM): `%s`", res.packageVersion) + + return + } + + setSummary(action, res) + + action.SetOutput("docker_development_images", strings.Join(res.dockerDevelopmentImages, ",")) + action.SetOutput("docker_production_images", strings.Join(res.dockerProductionImages, ",")) + + developmentTagFlags := make([]string, len(res.dockerDevelopmentImages)) + for i, image := range res.dockerDevelopmentImages { + developmentTagFlags[i] = fmt.Sprintf("--tag %s", image) + } + action.SetOutput("docker_development_tag_flags", strings.Join(developmentTagFlags, " ")) + + productionTagFlags := make([]string, len(res.dockerProductionImages)) + for i, image := range res.dockerProductionImages { + productionTagFlags[i] = fmt.Sprintf("--tag %s", image) + } + action.SetOutput("docker_production_tag_flags", strings.Join(productionTagFlags, " ")) +} diff --git a/packaging/defineversion/main_test.go b/packaging/defineversion/main_test.go new file mode 100644 index 000000000..b56b0631c --- /dev/null +++ b/packaging/defineversion/main_test.go @@ -0,0 +1,478 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "os" + "strings" + "testing" + + "github.com/sethvargo/go-githubactions" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// getEnvFunc implements [os.Getenv] for testing. +func getEnvFunc(t *testing.T, env map[string]string) func(string) string { + t.Helper() + + return func(key string) string { + val, ok := env[key] + require.True(t, ok, "missing key %q", key) + + return val + } +} + +func TestParseGitTag(t *testing.T) { + tests := map[string]struct { + major int + minor int + patch int + prerelease string + err string + }{ + "v0.103.0-ferretdb-2.2.0-beta.1": { + major: 0, + minor: 103, + patch: 0, + prerelease: "ferretdb-2.2.0-beta.1", + }, + "0.103.0-ferretdb-2.2.0-beta.1": { + err: `unexpected git tag format "0.103.0-ferretdb-2.2.0-beta.1"`, + }, + "v0.103.0-ferretdb": { + err: `prerelease "ferretdb" should start with 'ferretdb-'`, + }, + "v0.103.0": { + err: `prerelease "" should start with 'ferretdb-'`, + }, + } + + for tag, tc := range tests { + t.Run(tag, func(t *testing.T) { + major, minor, patch, prerelease, err := parseGitTag(tag) + if tc.err != "" { + require.EqualError(t, err, tc.err) + return + } + + require.NoError(t, err) + + assert.Equal(t, tc.major, major) + assert.Equal(t, tc.minor, minor) + assert.Equal(t, tc.patch, patch) + assert.Equal(t, tc.prerelease, prerelease) + }) + } +} + +func TestDefineVersion(t *testing.T) { + const pgVersion = "17" + + for name, tc := range map[string]struct { + env map[string]string + controlDefaultVersion string + expected *versions + expectedErr error + }{ + "pull_request": { + env: map[string]string{ + "GITHUB_BASE_REF": "ferretdb", + "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_HEAD_REF": "define-version", + "GITHUB_REF_NAME": "1/merge", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + }, + controlDefaultVersion: "0.103.0", + expected: &versions{ + dockerDevelopmentImages: []string{ + "ghcr.io/ferretdb/postgres-documentdb-dev:17-pr-define-version", + }, + dockerProductionImages: []string{ + "ghcr.io/ferretdb/postgres-documentdb-dev:17-pr-define-version-prod", + }, + packageVersion: "0.103.0~pr~define~version", + }, + }, + "pull_request-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "ferretdb", + "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_HEAD_REF": "define-version", + "GITHUB_REF_NAME": "1/merge", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + }, + controlDefaultVersion: "0.103.0", + expected: &versions{ + dockerDevelopmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17-pr-define-version", + }, + dockerProductionImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17-pr-define-version-prod", + }, + packageVersion: "0.103.0~pr~define~version", + }, + }, + + "pull_request_target": { + env: map[string]string{ + "GITHUB_BASE_REF": "ferretdb", + "GITHUB_EVENT_NAME": "pull_request_target", + "GITHUB_HEAD_REF": "define-version", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + }, + controlDefaultVersion: "0.103.0", + expected: &versions{ + dockerDevelopmentImages: []string{ + "ghcr.io/ferretdb/postgres-documentdb-dev:17-pr-define-version", + }, + dockerProductionImages: []string{ + "ghcr.io/ferretdb/postgres-documentdb-dev:17-pr-define-version-prod", + }, + packageVersion: "0.103.0~pr~define~version", + }, + }, + "pull_request_target-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "ferretdb", + "GITHUB_EVENT_NAME": "pull_request_target", + "GITHUB_HEAD_REF": "define-version", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + }, + controlDefaultVersion: "0.103.0", + expected: &versions{ + dockerDevelopmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17-pr-define-version", + }, + dockerProductionImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17-pr-define-version-prod", + }, + packageVersion: "0.103.0~pr~define~version", + }, + }, + + "push/ferretdb": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + }, + controlDefaultVersion: "0.103.0", + expected: &versions{ + dockerDevelopmentImages: []string{ + "ferretdb/postgres-documentdb-dev:17-ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb", + "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb", + }, + dockerProductionImages: []string{ + "ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + }, + packageVersion: "0.103.0~ferretdb", + }, + }, + "push/ferretdb-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + }, + controlDefaultVersion: "0.103.0", + expected: &versions{ + dockerDevelopmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", + }, + dockerProductionImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb-prod", + }, + packageVersion: "0.103.0~ferretdb", + }, + }, + + "push/main": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "main", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + }, + controlDefaultVersion: "0.103.0", + expectedErr: fmt.Errorf(`unhandled branch "main"`), + }, + "push/main-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "main", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + }, + controlDefaultVersion: "0.103.0", + expectedErr: fmt.Errorf(`unhandled branch "main"`), + }, + + "push/tag/release": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "v0.103.0-ferretdb-2.2.0-beta.1", + "GITHUB_REF_TYPE": "tag", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + }, + controlDefaultVersion: "0.103.0", + expected: &versions{ + dockerDevelopmentImages: []string{ + "ferretdb/postgres-documentdb-dev:17", + "ferretdb/postgres-documentdb-dev:17-0.103.0", + "ferretdb/postgres-documentdb-dev:17-0.103.0-ferretdb-2.2.0-beta.1", + "ferretdb/postgres-documentdb-dev:latest", + "ghcr.io/ferretdb/postgres-documentdb-dev:17", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-0.103.0", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-0.103.0-ferretdb-2.2.0-beta.1", + "ghcr.io/ferretdb/postgres-documentdb-dev:latest", + "quay.io/ferretdb/postgres-documentdb-dev:17", + "quay.io/ferretdb/postgres-documentdb-dev:17-0.103.0", + "quay.io/ferretdb/postgres-documentdb-dev:17-0.103.0-ferretdb-2.2.0-beta.1", + "quay.io/ferretdb/postgres-documentdb-dev:latest", + }, + dockerProductionImages: []string{ + "ferretdb/postgres-documentdb:17", + "ferretdb/postgres-documentdb:17-0.103.0", + "ferretdb/postgres-documentdb:17-0.103.0-ferretdb-2.2.0-beta.1", + "ferretdb/postgres-documentdb:latest", + "ghcr.io/ferretdb/postgres-documentdb:17", + "ghcr.io/ferretdb/postgres-documentdb:17-0.103.0", + "ghcr.io/ferretdb/postgres-documentdb:17-0.103.0-ferretdb-2.2.0-beta.1", + "ghcr.io/ferretdb/postgres-documentdb:latest", + "quay.io/ferretdb/postgres-documentdb:17", + "quay.io/ferretdb/postgres-documentdb:17-0.103.0", + "quay.io/ferretdb/postgres-documentdb:17-0.103.0-ferretdb-2.2.0-beta.1", + "quay.io/ferretdb/postgres-documentdb:latest", + }, + packageVersion: "0.103.0~ferretdb~2.2.0~beta.1", + }, + }, + "push/tag/release-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "v0.103.0-ferretdb-2.2.0-beta.1", + "GITHUB_REF_TYPE": "tag", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + }, + controlDefaultVersion: "0.103.0", + expected: &versions{ + dockerDevelopmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17", + "ghcr.io/otherorg/postgres-otherrepo-dev:17-0.103.0", + "ghcr.io/otherorg/postgres-otherrepo-dev:17-0.103.0-ferretdb-2.2.0-beta.1", + "ghcr.io/otherorg/postgres-otherrepo-dev:latest", + }, + dockerProductionImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo:17", + "ghcr.io/otherorg/postgres-otherrepo:17-0.103.0", + "ghcr.io/otherorg/postgres-otherrepo:17-0.103.0-ferretdb-2.2.0-beta.1", + "ghcr.io/otherorg/postgres-otherrepo:latest", + }, + packageVersion: "0.103.0~ferretdb~2.2.0~beta.1", + }, + }, + + "push/tag/release-wrong-control-default-version": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "v0.103.0-ferretdb-2.2.0-beta.1", + "GITHUB_REF_TYPE": "tag", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + }, + controlDefaultVersion: "0.104.0", + expectedErr: fmt.Errorf(`git tag version "0.103.0" does not match the control file default version "0.104.0"`), + }, + + "schedule": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "schedule", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + }, + controlDefaultVersion: "0.103.0", + expected: &versions{ + dockerDevelopmentImages: []string{ + "ferretdb/postgres-documentdb-dev:17-ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb", + "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb", + }, + dockerProductionImages: []string{ + "ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + }, + packageVersion: "0.103.0~ferretdb", + }, + }, + "schedule-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "schedule", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + }, + controlDefaultVersion: "0.103.0", + expected: &versions{ + dockerDevelopmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", + }, + dockerProductionImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb-prod", + }, + packageVersion: "0.103.0~ferretdb", + }, + }, + + "workflow_run": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "workflow_run", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + }, + controlDefaultVersion: "0.103.0", + expected: &versions{ + dockerDevelopmentImages: []string{ + "ferretdb/postgres-documentdb-dev:17-ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb", + "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb", + }, + dockerProductionImages: []string{ + "ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + }, + packageVersion: "0.103.0~ferretdb", + }, + }, + "workflow_run-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "workflow_run", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + }, + controlDefaultVersion: "0.103.0", + expected: &versions{ + dockerDevelopmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", + }, + dockerProductionImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb-prod", + }, + packageVersion: "0.103.0~ferretdb", + }, + }, + } { + t.Run(name, func(t *testing.T) { + docker, err := defineVersion(tc.controlDefaultVersion, pgVersion, getEnvFunc(t, tc.env)) + if tc.expected == nil { + require.Error(t, tc.expectedErr) + require.Equal(t, err, tc.expectedErr) + return + } + + require.NoError(t, tc.expectedErr) + require.NoError(t, err) + + assert.Equal(t, tc.expected, docker) + }) + } +} + +func TestSummary(t *testing.T) { + dir := t.TempDir() + + summaryF, err := os.CreateTemp(dir, "summary") + require.NoError(t, err) + defer summaryF.Close() + + outputF, err := os.CreateTemp(dir, "output") + require.NoError(t, err) + defer outputF.Close() + + var stdout bytes.Buffer + getenv := getEnvFunc(t, map[string]string{ + "GITHUB_STEP_SUMMARY": summaryF.Name(), + "GITHUB_OUTPUT": outputF.Name(), + }) + action := githubactions.New(githubactions.WithGetenv(getenv), githubactions.WithWriter(&stdout)) + + result := &versions{ + dockerDevelopmentImages: []string{ + "ghcr.io/ferretdb/postgres-documentdb-dev:17-0.103.0-ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:latest", + }, + dockerProductionImages: []string{ + "quay.io/ferretdb/postgres-documentdb:latest", + }, + packageVersion: "0.103.0~ferretdb", + } + + setSummary(action, result) + + expectedStdout := strings.ReplaceAll(` +Package version (Debian with 'upstream_version' only, or RPM): '0.103.0~ferretdb' + + |Type |Docker image | + |---- |------------ | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:17-0.103.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:17-0.103.0-ferretdb) | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:latest'](https://ghcr.io/ferretdb/postgres-documentdb-dev:latest) | + |Production |['quay.io/ferretdb/postgres-documentdb:latest'](https://quay.io/ferretdb/postgres-documentdb:latest) | + +`[1:], "'", "`", + ) + assert.Equal(t, expectedStdout, stdout.String(), "stdout does not match") + + expectedSummary := strings.ReplaceAll(` +Package version (Debian with 'upstream_version' only, or RPM): '0.103.0~ferretdb' + + |Type |Docker image | + |---- |------------ | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:17-0.103.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:17-0.103.0-ferretdb) | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:latest'](https://ghcr.io/ferretdb/postgres-documentdb-dev:latest) | + |Production |['quay.io/ferretdb/postgres-documentdb:latest'](https://quay.io/ferretdb/postgres-documentdb:latest) | + +`[1:], "'", "`", + ) + b, err := io.ReadAll(summaryF) + require.NoError(t, err) + assert.Equal(t, expectedSummary, string(b), "summary does not match") +} diff --git a/packaging/defineversion/package_version.go b/packaging/defineversion/package_version.go new file mode 100644 index 000000000..636b815ea --- /dev/null +++ b/packaging/defineversion/package_version.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "os" + "regexp" +) + +// controlDefaultVer matches major, minor and "patch" from `default_version` field in control file, +// see pg_documentdb_core/documentdb_core.control. +var controlDefaultVer = regexp.MustCompile(`(?m)^default_version = '(?P[0-9]+)\.(?P[0-9]+)-(?P[0-9]+)'$`) + +// disallowedDebian matches disallowed characters of Debian `upstream_version` when used without `debian_revision`. +// See https://www.debian.org/doc/debian-policy/ch-controlfields.html#version. +var disallowedDebian = regexp.MustCompile(`[^A-Za-z0-9\.+~]`) + +// disallowedRPM matches disallowed characters of pre-release string. +// See https://fedoraproject.org/wiki/PackagingDrafts/TildeVersioning#Basic_versioning_rules. +var disallowedRPM = regexp.MustCompile(`[^A-Za-z0-9\._+~]`) + +// getControlDefaultVersion returns the default_version field from the control file +// in SemVer format (0.103-0 -> 0.103.0). +func getControlDefaultVersion(f string) (string, error) { + b, err := os.ReadFile(f) + if err != nil { + return "", err + } + + match := controlDefaultVer.FindSubmatch(b) + if match == nil || len(match) != controlDefaultVer.NumSubexp()+1 { + return "", fmt.Errorf("control file did not find default_version: %s", f) + } + + major := match[controlDefaultVer.SubexpIndex("major")] + minor := match[controlDefaultVer.SubexpIndex("minor")] + patch := match[controlDefaultVer.SubexpIndex("patch")] + + return fmt.Sprintf("%s.%s.%s", major, minor, patch), nil +} diff --git a/packaging/defineversion/package_version_test.go b/packaging/defineversion/package_version_test.go new file mode 100644 index 000000000..c98e6f1bc --- /dev/null +++ b/packaging/defineversion/package_version_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "io" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestReadControlDefaultVersion(t *testing.T) { + controlF, err := os.CreateTemp(t.TempDir(), "test.control") + require.NoError(t, err) + + defer controlF.Close() //nolint:errcheck // temporary file for testing + + buf := `comment = 'API surface for DocumentDB for PostgreSQL' +default_version = '0.103-0' +module_pathname = '$libdir/pg_documentdb' +relocatable = false +superuser = true +requires = 'documentdb_core, pg_cron, tsm_system_rows, vector, postgis, rum'` + _, err = io.WriteString(controlF, buf) + require.NoError(t, err) + + controlDefaultVersion, err := getControlDefaultVersion(controlF.Name()) + require.NoError(t, err) + + require.Equal(t, "0.103.0", controlDefaultVersion) +} diff --git a/packaging/development.Dockerfile b/packaging/development.Dockerfile new file mode 100644 index 000000000..b6f6e1a4a --- /dev/null +++ b/packaging/development.Dockerfile @@ -0,0 +1,72 @@ +# syntax=docker/dockerfile:1 + +ARG POSTGRES_VERSION + +# TODO https://github.com/FerretDB/FerretDB/issues/5449 +FROM postgres:${POSTGRES_VERSION}-bookworm + +ARG TARGETARCH +ARG POSTGRES_VERSION +ARG DOCUMENTDB_VERSION + +# common steps for production and development + +RUN --mount=type=cache,sharing=locked,target=/var/cache/apt < - 0.105-0-1 +* Thu Aug 07 2025 FerretDB Packages - 0.106-0-1 +- Add internal extension that provides extensions to the rum index *[Feature]* +- Enable let support for update queries *[Feature]*. Requires EnableVariablesSupportForWriteCommands to be on. +- Enable let support for findAndModify queries *[Feature]*. Requires EnableVariablesSupportForWriteCommands to be on. +- Optimized query for usersInfo command +- Support collation with delete *[Feature]*. Requires EnableCollation to be on. +- Support for index hints for find/aggregate/count/distinct *[Feature]* +- Support createRole command *[Feature]* +- Add schema changes for Role CRUD APIs *[Feature]* +- Add support for using EntraId tokens via Plain Auth + +* Wed Jul 09 2025 FerretDB Packages - 0.105-0-1 - Support $bucketAuto aggregation stage, with granularity types: POWERSOF2, 1-2-5, R5, R10, R20, R40, R80, E6, E12, E24, E48, E96, E192 *[Feature]* -- Support conectionStatus command *[Feature]*. +- Support connectionStatus command *[Feature]*. -* Mon Jun 09 2025 Shuai Tian - 0.104-0-1 +* Mon Jun 09 2025 FerretDB Packages - 0.104-0-1 - Add string case support for $toDate operator -- Support sort with collation in runtime*[Feature]* -- Support collation with $indexOfArray aggregation operator. *[Feature]* +- Support sort with collation in runtime *[Feature]* +- Support collation with $indexOfArray aggregation operator *[Feature]* - Support collation with arrays and objects comparisons *[Feature]* - Support background index builds *[Bugfix]* (#36) - Enable user CRUD by default *[Feature]* - Enable let support for delete queries *[Feature]*. Requires EnableVariablesSupportForWriteCommands to be on. - Enable rum_enable_index_scan as default on *[Perf]* - Add public documentdb-local Docker image with gateway to GHCR +- Support compact command *[Feature]*. Requires documentdb.enablecompact GUC to be on. +- Enable role privileges for usersInfo command *[Feature]* -* Fri May 09 2025 Shuai Tian - 0.103-0-1 +* Fri May 09 2025 FerretDB Packages - 0.103-0-1 - Support collation with aggregation and find on sharded collections *[Feature]* - Support $convert on binData to binData, string to binData and binData to string (except with format: auto) *[Feature]* - Fix list_databases for databases with size > 2 GB *[Bugfix]* (#119) @@ -127,7 +140,7 @@ rm -rf %{buildroot}/usr/src/documentdb/build - Support collation with $documents and $replaceWith stage of the aggregation pipeline *[Feature]* - Push pg_documentdb_gw for documentdb connections *[Feature]* -* Wed Mar 26 2025 Shuai Tian - 0.102-0-1 +* Wed Mar 26 2025 FerretDB Packages - 0.102-0-1 - Support index pushdown for vector search queries *[Bugfix]* - Support exact search for vector search queries *[Feature]* - Inline $match with let in $lookup pipelines as JOIN Filter *[Perf]* @@ -147,7 +160,7 @@ rm -rf %{buildroot}/usr/src/documentdb/build - Support the $dateFromString operator with full functionality *[Feature]* - Support extended syntax for $getField aggregation operator. Now the value of 'field' could be an expression that resolves to a string. *[Feature]* -* Wed Feb 12 2025 Shuai Tian - 0.101-0-1 +* Wed Feb 12 2025 FerretDB Packages - 0.101-0-1 - Push $graphlookup recursive CTE JOIN filters to index *[Perf]* - Build pg_documentdb for PostgreSQL 17 *[Infra]* (#13) - Enable support of currentOp aggregation stage, along with collstats, dbstats, and indexStats *[Commands]* (#52) @@ -155,5 +168,5 @@ rm -rf %{buildroot}/usr/src/documentdb/build - Skip loading documents if group expression is constant *[Perf]* - Fix Merge stage not outputing to target collection *[Bugfix]* (#20) -* Thu Jan 23 2025 Shuai Tian - 0.100-0-1 -- Initial Release \ No newline at end of file +* Thu Jan 23 2025 FerretDB Packages - 0.100-0-1 +- Initial Release diff --git a/packaging/test_packages/Dockerfile_test_install_deb_packages b/packaging/test_packages/Dockerfile_test_install_deb_packages old mode 100755 new mode 100644 index e52cd6543..c0792fc4b --- a/packaging/test_packages/Dockerfile_test_install_deb_packages +++ b/packaging/test_packages/Dockerfile_test_install_deb_packages @@ -44,4 +44,4 @@ RUN dpkg -i ${DEB_PACKAGE_REL_PATH} COPY packaging/test_packages/test-install-entrypoint.sh /usr/local/bin/test-install-entrypoint.sh -ENTRYPOINT ["test-install-entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["test-install-entrypoint.sh"] diff --git a/packaging/test_packages/test-install-entrypoint.sh b/packaging/test_packages/test-install-entrypoint.sh index a894e6e28..fd22dfc86 100755 --- a/packaging/test_packages/test-install-entrypoint.sh +++ b/packaging/test_packages/test-install-entrypoint.sh @@ -10,4 +10,4 @@ sed -i '/internal/d' Makefile # Run the test adduser --disabled-password --gecos "" documentdb chown -R documentdb:documentdb . -su documentdb -c "make check" \ No newline at end of file +su documentdb -c "make check"