From d6794c9211bb5411f4615a36da39b8c2a75a7435 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 19 Feb 2025 15:06:54 +0400 Subject: [PATCH 01/59] Update CODEOWNERS (#4) --- CODEOWNERS | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 6c6098c52..9a9db1207 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -8,4 +8,6 @@ Make* @microsoft/documentdb-engine-reviewers /internal/ @microsoft/documentdb-engine-reviewers /pg_documentdb/ @microsoft/documentdb-engine-reviewers -/pg_documentdb_core/ @microsoft/documentdb-engine-reviewers \ No newline at end of file +/pg_documentdb_core/ @microsoft/documentdb-engine-reviewers + +* @AlekSi From fc8f917fb36fc882c668c8d344a239a3c05d8a89 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Thu, 20 Feb 2025 03:12:19 +0900 Subject: [PATCH 02/59] Add versioning tool (#3) --- .github/workflows/ferretdb_go_tests.yml | 62 +++++ ferretdb_packaging/.gitignore | 2 + ferretdb_packaging/defineversion/main.go | 210 ++++++++++++++++ ferretdb_packaging/defineversion/main_test.go | 232 ++++++++++++++++++ ferretdb_packaging/go.mod | 16 ++ ferretdb_packaging/go.sum | 12 + 6 files changed, 534 insertions(+) create mode 100644 .github/workflows/ferretdb_go_tests.yml create mode 100644 ferretdb_packaging/.gitignore create mode 100644 ferretdb_packaging/defineversion/main.go create mode 100644 ferretdb_packaging/defineversion/main_test.go create mode 100644 ferretdb_packaging/go.mod create mode 100644 ferretdb_packaging/go.sum diff --git a/.github/workflows/ferretdb_go_tests.yml b/.github/workflows/ferretdb_go_tests.yml new file mode 100644 index 000000000..2c3c01211 --- /dev/null +++ b/.github/workflows/ferretdb_go_tests.yml @@ -0,0 +1,62 @@ +--- +name: Go +on: + pull_request: + types: + - unlabeled # if GitHub Actions stuck, add and remove "not ready" label to force rebuild + - opened + - reopened + - synchronize + push: + branches: + - ferretdb + tags: + - "*" + schedule: + - cron: "12 0 * * *" + +env: + GOPATH: /home/runner/go + GOCACHE: /home/runner/go/cache + GOLANGCI_LINT_CACHE: /home/runner/go/cache/lint + GOMODCACHE: /home/runner/go/mod + GOPROXY: https://proxy.golang.org + GOTOOLCHAIN: local + +jobs: + test: + name: Test + # https://www.ubicloud.com/docs/about/pricing#github-actions + # https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#per-minute-rates-for-larger-runners + runs-on: ubicloud-standard-4 + + timeout-minutes: 15 + + # Do not run this job in parallel for any PR change or branch push. + concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + + if: github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'not ready') + + steps: + # TODO https://github.com/FerretDB/github-actions/issues/211 + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: FerretDB/github-actions/setup-go@main + + - name: Run tests + run: | + cd ferretdb_packaging + go mod tidy + go mod verify + go test ./... + + - name: Check dirty + if: always() + run: | + git status --untracked-files --ignored + git status + git diff --exit-code diff --git a/ferretdb_packaging/.gitignore b/ferretdb_packaging/.gitignore new file mode 100644 index 000000000..353ce6cb8 --- /dev/null +++ b/ferretdb_packaging/.gitignore @@ -0,0 +1,2 @@ +# FerretDB packaging +!*go.mod diff --git a/ferretdb_packaging/defineversion/main.go b/ferretdb_packaging/defineversion/main.go new file mode 100644 index 000000000..aeafdcd69 --- /dev/null +++ b/ferretdb_packaging/defineversion/main.go @@ -0,0 +1,210 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main defines debian version number of DocumentDB for CI builds. +package main + +import ( + "flag" + "fmt" + "os" + "regexp" + "slices" + "strings" + + "github.com/sethvargo/go-githubactions" +) + +func main() { + controlFileF := flag.String("control-file", "../pg_documentdb/documentdb.control", "pg_documentdb/documentdb.control file path") + + flag.Parse() + + action := githubactions.New() + + if *controlFileF == "" { + action.Fatalf("%s", "-control-file flag is empty.") + } + + controlDefaultVersion, err := getControlDefaultVersion(*controlFileF) + if err != nil { + action.Fatalf("%s", err) + } + + debugEnv(action) + + packageVersion, err := definePackageVersion(controlDefaultVersion, action.Getenv) + if err != nil { + action.Fatalf("%s", err) + } + + setResults(action, packageVersion) +} + +// controlDefaultVer matches major, minor and "patch" from default_version field in control file, +// see pg_documentdb_core/documentdb_core.control. +var controlDefaultVer = regexp.MustCompile(`(?m)^default_version = '(?P[0-9]+)\.(?P[0-9]+)-(?P[0-9]+)'$`) + +// semVerTag is a https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string, +// but with a leading `v`. +var semVerTag = regexp.MustCompile(`^v(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`) + +// disallowedVer matches disallowed characters of Debian `upstream_version` when used without `debian_revision`. +// See https://www.debian.org/doc/debian-policy/ch-controlfields.html#version. +var disallowedVer = regexp.MustCompile(`[^A-Za-z0-9~.+]`) + +// debugEnv logs all environment variables that start with `GITHUB_` or `INPUT_` +// in debug level. +func debugEnv(action *githubactions.Action) { + res := make([]string, 0, 30) + + for _, l := range os.Environ() { + if strings.HasPrefix(l, "GITHUB_") || strings.HasPrefix(l, "INPUT_") { + res = append(res, l) + } + } + + slices.Sort(res) + + action.Debugf("Dumping environment variables:") + + for _, l := range res { + action.Debugf("\t%s", l) + } +} + +// getControlDefaultVersion returns the default_version field from the control file +// in SemVer format (0.100-0 -> 0.100.0). +func getControlDefaultVersion(f string) (string, error) { + b, err := os.ReadFile(f) + if err != nil { + return "", err + } + + match := controlDefaultVer.FindSubmatch(b) + if match == nil || len(match) != controlDefaultVer.NumSubexp()+1 { + return "", fmt.Errorf("control file did not find default_version: %s", f) + } + + major := match[controlDefaultVer.SubexpIndex("major")] + minor := match[controlDefaultVer.SubexpIndex("minor")] + patch := match[controlDefaultVer.SubexpIndex("patch")] + + return fmt.Sprintf("%s.%s.%s", major, minor, patch), nil +} + +// definePackageVersion returns valid Debian package version, +// based on `default_version` in the control file and environment variables set by GitHub Actions. +// +// See https://www.debian.org/doc/debian-policy/ch-controlfields.html#version. +// We use `upstream_version` only. +// For that reason, we can't use `-`, so we replace it with `~`. +func definePackageVersion(controlDefaultVersion string, getenv githubactions.GetenvFunc) (string, error) { + var packageVersion string + var err error + + switch event := getenv("GITHUB_EVENT_NAME"); event { + case "pull_request", "pull_request_target": + branch := strings.ToLower(getenv("GITHUB_HEAD_REF")) + packageVersion = definePackageVersionForPR(controlDefaultVersion, branch) + + case "push", "schedule", "workflow_run": + refName := strings.ToLower(getenv("GITHUB_REF_NAME")) + + switch refType := strings.ToLower(getenv("GITHUB_REF_TYPE")); refType { + case "branch": + packageVersion, err = definePackageVersionForBranch(controlDefaultVersion, refName) + + case "tag": + packageVersion, err = definePackagerVersionForTag(refName) + + default: + err = fmt.Errorf("unhandled ref type %q for event %q", refType, event) + } + + default: + err = fmt.Errorf("unhandled event type %q", event) + } + + if err != nil { + return "", err + } + + if packageVersion == "" { + return "", fmt.Errorf("both packageVersion and err are nil") + } + + return packageVersion, nil +} + +// definePackageVersionForPR returns valid Debian package version for PR. +// See [definePackageVersion]. +func definePackageVersionForPR(controlDefaultVersion, branch string) string { + // for branches like "dependabot/submodules/XXX" + parts := strings.Split(branch, "/") + branch = parts[len(parts)-1] + res := fmt.Sprintf("%s-pr-%s", controlDefaultVersion, branch) + + return disallowedVer.ReplaceAllString(res, "~") +} + +// definePackageVersionForBranch returns valid Debian package version for branch. +// See [definePackageVersion]. +func definePackageVersionForBranch(controlDefaultVersion, branch string) (string, error) { + switch branch { + case "ferretdb": + return fmt.Sprintf("%s~branch~%s", controlDefaultVersion, branch), nil + default: + return "", fmt.Errorf("unhandled branch %q", branch) + } +} + +// definePackagerVersionForTag returns valid Debian package version for tag. +// See [definePackageVersion]. +func definePackagerVersionForTag(tag string) (string, error) { + match := semVerTag.FindStringSubmatch(tag) + if match == nil || len(match) != semVerTag.NumSubexp()+1 { + return "", fmt.Errorf("unexpected tag syntax %q", tag) + } + + major := match[semVerTag.SubexpIndex("major")] + minor := match[semVerTag.SubexpIndex("minor")] + patch := match[semVerTag.SubexpIndex("patch")] + prerelease := match[semVerTag.SubexpIndex("prerelease")] + buildmetadata := match[semVerTag.SubexpIndex("buildmetadata")] + + if prerelease == "" { + return "", fmt.Errorf("prerelease is empty") + } + + if !strings.Contains(prerelease, "ferretdb") { + return "", fmt.Errorf("prerelease %q should include `ferretdb`", prerelease) + } + + if buildmetadata != "" { + return "", fmt.Errorf("buildmetadata %q is present", buildmetadata) + } + + res := fmt.Sprintf("%s.%s.%s-%s", major, minor, patch, prerelease) + return disallowedVer.ReplaceAllString(res, "~"), nil +} + +// setResults sets action output parameters, summary, etc. +func setResults(action *githubactions.Action, res string) { + output := fmt.Sprintf("version: %s", res) + + action.AddStepSummary(output) + action.Infof("%s", output) + action.SetOutput("version", res) +} diff --git a/ferretdb_packaging/defineversion/main_test.go b/ferretdb_packaging/defineversion/main_test.go new file mode 100644 index 000000000..74a5edbb9 --- /dev/null +++ b/ferretdb_packaging/defineversion/main_test.go @@ -0,0 +1,232 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "io" + "os" + "testing" + + "github.com/sethvargo/go-githubactions" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// getEnvFunc implements [os.Getenv] for testing. +func getEnvFunc(t *testing.T, env map[string]string) func(string) string { + t.Helper() + + return func(key string) string { + val, ok := env[key] + require.True(t, ok, "missing key %q", key) + + return val + } +} + +func TestDefine(t *testing.T) { + for name, tc := range map[string]struct { + env map[string]string + controlDefaultVersion string + expected string + }{ + "pull_request": { + env: map[string]string{ + "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_HEAD_REF": "define-version", + "GITHUB_REF_NAME": "1/merge", + "GITHUB_REF_TYPE": "branch", + }, + controlDefaultVersion: "0.100.0", + expected: "0.100.0~pr~define~version", + }, + + "pull_request_target": { + env: map[string]string{ + "GITHUB_EVENT_NAME": "pull_request_target", + "GITHUB_HEAD_REF": "define-version", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + }, + controlDefaultVersion: "0.100.0", + expected: "0.100.0~pr~define~version", + }, + + "push/ferretdb": { + env: map[string]string{ + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + }, + controlDefaultVersion: "0.100.0", + expected: "0.100.0~branch~ferretdb", + }, + "push/other": { + env: map[string]string{ + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "releases", + "GITHUB_REF_TYPE": "other", // not ferretdb branch + }, + }, + + "push/tag/v0.100.0-ferretdb": { + env: map[string]string{ + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "v0.100.0-ferretdb", + "GITHUB_REF_TYPE": "tag", + }, + expected: "0.100.0~ferretdb", + }, + "push/tag/v0.100.0-ferretdb-2.0.1": { + env: map[string]string{ + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "v0.100.0-ferretdb-2.0.1", + "GITHUB_REF_TYPE": "tag", + }, + expected: "0.100.0~ferretdb~2.0.1", + }, + + "push/tag/missing-prerelease": { + env: map[string]string{ + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "v0.100.0", // missing prerelease + "GITHUB_REF_TYPE": "tag", + }, + }, + "push/tag/not-ferretdb-prerelease": { + env: map[string]string{ + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "v0.100.0-other", // missing ferretdb in prerelease + "GITHUB_REF_TYPE": "tag", + }, + }, + "push/tag/missing-v": { + env: map[string]string{ + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "0.100.0-ferretdb", + "GITHUB_REF_TYPE": "tag", + }, + }, + "push/tag/not-semvar": { + env: map[string]string{ + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "v0.100-0-ferretdb", + "GITHUB_REF_TYPE": "tag", + }, + }, + + "schedule": { + env: map[string]string{ + "GITHUB_EVENT_NAME": "schedule", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + }, + controlDefaultVersion: "0.100.0", + expected: "0.100.0~branch~ferretdb", + }, + + "workflow_run": { + env: map[string]string{ + "GITHUB_EVENT_NAME": "workflow_run", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + }, + controlDefaultVersion: "0.100.0", + expected: "0.100.0~branch~ferretdb", + }, + } { + t.Run(name, func(t *testing.T) { + actual, err := definePackageVersion(tc.controlDefaultVersion, getEnvFunc(t, tc.env)) + if tc.expected == "" { + require.Error(t, err) + return + } + + require.NoError(t, err) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestResults(t *testing.T) { + dir := t.TempDir() + + summaryF, err := os.CreateTemp(dir, "summary") + require.NoError(t, err) + defer summaryF.Close() //nolint:errcheck // temporary file for testing + + outputF, err := os.CreateTemp(dir, "output") + require.NoError(t, err) + defer outputF.Close() //nolint:errcheck // temporary file for testing + + var stdout bytes.Buffer + getenv := getEnvFunc(t, map[string]string{ + "GITHUB_STEP_SUMMARY": summaryF.Name(), + "GITHUB_OUTPUT": outputF.Name(), + }) + action := githubactions.New(githubactions.WithGetenv(getenv), githubactions.WithWriter(&stdout)) + + version := "0.100.0~ferretdb" + + setResults(action, version) + + expected := "version: 0.100.0~ferretdb\n" + assert.Equal(t, expected, stdout.String(), "stdout does not match") + + b, err := io.ReadAll(summaryF) + require.NoError(t, err) + assert.Equal(t, expected, string(b), "summary does not match") + + expectedOutput := ` +version<<_GitHubActionsFileCommandDelimeter_ +0.100.0~ferretdb +_GitHubActionsFileCommandDelimeter_ +`[1:] + b, err = io.ReadAll(outputF) + require.NoError(t, err) + assert.Equal(t, expectedOutput, string(b), "output parameters does not match") +} + +func TestReadControlDefaultVersion(t *testing.T) { + dir := t.TempDir() + + controlF, err := os.CreateTemp(dir, "test.control") + defer controlF.Close() //nolint:errcheck // temporary file for testing + + buf := `comment = 'API surface for DocumentDB for PostgreSQL' +default_version = '0.100-0' +module_pathname = '$libdir/pg_documentdb' +relocatable = false +superuser = true +requires = 'documentdb_core, pg_cron, tsm_system_rows, vector, postgis, rum'` + _, err = io.WriteString(controlF, buf) + require.NoError(t, err) + + controlDefaultVersion, err := getControlDefaultVersion(controlF.Name()) + require.NoError(t, err) + + require.Equal(t, "0.100.0", controlDefaultVersion) +} diff --git a/ferretdb_packaging/go.mod b/ferretdb_packaging/go.mod new file mode 100644 index 000000000..50a865286 --- /dev/null +++ b/ferretdb_packaging/go.mod @@ -0,0 +1,16 @@ +module github.com/FerretDB/documentdb/ferretdb_packaging + +go 1.24 + +toolchain go1.24.0 + +require ( + github.com/sethvargo/go-githubactions v1.3.0 + github.com/stretchr/testify v1.10.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/ferretdb_packaging/go.sum b/ferretdb_packaging/go.sum new file mode 100644 index 000000000..0051d23f1 --- /dev/null +++ b/ferretdb_packaging/go.sum @@ -0,0 +1,12 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sethvargo/go-githubactions v1.3.0 h1:Kg633LIUV2IrJsqy2MfveiED/Ouo+H2P0itWS0eLh8A= +github.com/sethvargo/go-githubactions v1.3.0/go.mod h1:7/4WeHgYfSz9U5vwuToCK9KPnELVHAhGtRwLREOQV80= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 8a85bdd9830e2b3c5babcdd7289179d0137188e5 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 20 Feb 2025 14:37:54 +0400 Subject: [PATCH 03/59] Add basic repository and Mergify configuration (#6) --- .github/mergify.yml | 27 +++++++++++++++++++++++++++ .github/settings.yml | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 .github/mergify.yml create mode 100644 .github/settings.yml diff --git a/.github/mergify.yml b/.github/mergify.yml new file mode 100644 index 000000000..e722a16b2 --- /dev/null +++ b/.github/mergify.yml @@ -0,0 +1,27 @@ +--- +# see https://docs.mergify.com/ + +pull_request_rules: + - name: "Assign PRs" + conditions: + - "-closed" + - "#assignee = 0" + actions: + assign: + add_users: ["{{ author }}"] + + - name: "Add label on conflicts" + conditions: + - "conflict" + actions: + comment: + message: "@{{author}} this pull request has merge conflicts." + label: + add: [conflict] + + - name: "Remove label when conflicts were resolved" + conditions: + - "-conflict" + actions: + label: + remove: [conflict] diff --git a/.github/settings.yml b/.github/settings.yml new file mode 100644 index 000000000..974cae3f9 --- /dev/null +++ b/.github/settings.yml @@ -0,0 +1,38 @@ +--- +# https://github.com/repository-settings/app + +repository: + allow_squash_merge: true + allow_merge_commit: true + allow_rebase_merge: false + allow_auto_merge: true + allow_update_branch: true + delete_branch_on_merge: true + enable_automated_security_fixes: true + enable_vulnerability_alerts: true + +# https://docs.github.com/en/rest/issues/labels +labels: + - name: conflict + color: "#FF0000" + description: PRs that have merge conflicts + + - name: deps + color: "#D4C5F9" + description: PRs that update dependencies + + - name: do not merge + color: "#0052CC" + description: PRs that should not be merged + + - name: not ready + color: "#000000" + description: Issues that are not ready to be worked on; PRs that should skip CI + + - name: packages + color: "#9B022C" + description: PRs that should build packages + + - name: trust + color: "#00FF00" + description: PRs that can access Actions secrets From dfb4bdb534bd57cbddecf9596813b84fc9f89020 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Thu, 20 Feb 2025 20:02:08 +0900 Subject: [PATCH 04/59] Build `.deb` packages on CI (#2) --- .github/workflows/ferretdb_packages.yml | 88 ++++++++++ .gitignore | 3 + ferretdb_packaging/.gitignore | 1 + .../Dockerfile_build_deb_packages | 79 +++++++++ ferretdb_packaging/README.md | 10 ++ ferretdb_packaging/build_packages.sh | 155 ++++++++++++++++++ ferretdb_packaging/debian_files/changelog | 5 + ferretdb_packaging/debian_files/compat | 1 + ferretdb_packaging/debian_files/control | 10 ++ ferretdb_packaging/debian_files/rules | 10 ++ ferretdb_packaging/packaging-entrypoint.sh | 20 +++ .../Dockerfile_test_install_deb_packages | 46 ++++++ .../test_packages/test-install-entrypoint.sh | 13 ++ 13 files changed, 441 insertions(+) create mode 100644 .github/workflows/ferretdb_packages.yml create mode 100644 ferretdb_packaging/Dockerfile_build_deb_packages create mode 100644 ferretdb_packaging/README.md create mode 100755 ferretdb_packaging/build_packages.sh create mode 100644 ferretdb_packaging/debian_files/changelog create mode 100644 ferretdb_packaging/debian_files/compat create mode 100644 ferretdb_packaging/debian_files/control create mode 100755 ferretdb_packaging/debian_files/rules create mode 100755 ferretdb_packaging/packaging-entrypoint.sh create mode 100644 ferretdb_packaging/test_packages/Dockerfile_test_install_deb_packages create mode 100755 ferretdb_packaging/test_packages/test-install-entrypoint.sh diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml new file mode 100644 index 000000000..a962fe40b --- /dev/null +++ b/.github/workflows/ferretdb_packages.yml @@ -0,0 +1,88 @@ +--- +name: Packages +on: + pull_request: + types: + - unlabeled # if GitHub Actions stuck, add and remove "not ready" label to force rebuild + - opened + - reopened + - synchronize + push: + branches: + - ferretdb + tags: + - "*" + schedule: + - cron: "10 8 * * 1" + +env: + GOPATH: /home/runner/go + GOCACHE: /home/runner/go/cache + GOLANGCI_LINT_CACHE: /home/runner/go/cache/lint + GOMODCACHE: /home/runner/go/mod + GOPROXY: https://proxy.golang.org + GOTOOLCHAIN: local + +# Do not run this workflow in parallel for any PR change or branch/tag push +# to save some resources. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: false + +jobs: + build: + name: Build .debs (${{ matrix.os }}, Pg${{ matrix.pg }}) + + # https://www.ubicloud.com/docs/about/pricing#github-actions + # https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#per-minute-rates-for-larger-runners + runs-on: ubicloud-standard-4 + + timeout-minutes: 40 + + if: > + github.event_name != 'pull_request' || + ( + !contains(github.event.pull_request.labels.*.name, 'not ready') && + contains(github.event.pull_request.labels.*.name, 'packages') + ) + + strategy: + fail-fast: false + matrix: + os: [deb11, deb12, ubuntu20.04, ubuntu22.04, ubuntu24.04] + pg: [15, 16] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: FerretDB/github-actions/setup-go@main + + - name: Define Debian package version + id: version + run: | + cd ferretdb_packaging + go mod tidy + go mod verify + go build -v -o=bin/ ./defineversion + bin/defineversion --control-file ../pg_documentdb_core/documentdb_core.control + + - name: Build ${{ steps.version.outputs.version }} + if: steps.version.outputs.version != '' + run: ./ferretdb_packaging/build_packages.sh --os ${{ matrix.os }} --pg ${{ matrix.pg }} --version ${{ steps.version.outputs.version }} --test-clean-install + + - name: Upload + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}-${{ matrix.pg }}-debs + path: packaging/*.deb + retention-days: 1 + if-no-files-found: error + compression-level: 0 + overwrite: false + + - name: Check dirty + run: | + git status + git diff --exit-code diff --git a/.gitignore b/.gitignore index 86c6c8324..626a206cb 100755 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,9 @@ *.idb *.pdb +# FerretDB packaging +*.deb + # Kernel Module Compile Results *.mod* *.cmd diff --git a/ferretdb_packaging/.gitignore b/ferretdb_packaging/.gitignore index 353ce6cb8..43758e150 100644 --- a/ferretdb_packaging/.gitignore +++ b/ferretdb_packaging/.gitignore @@ -1,2 +1,3 @@ # FerretDB packaging !*go.mod +/bin diff --git a/ferretdb_packaging/Dockerfile_build_deb_packages b/ferretdb_packaging/Dockerfile_build_deb_packages new file mode 100644 index 000000000..d6caf9d7d --- /dev/null +++ b/ferretdb_packaging/Dockerfile_build_deb_packages @@ -0,0 +1,79 @@ +ARG BASE_IMAGE=debian:bookworm +FROM --platform=linux/amd64 ${BASE_IMAGE} +ARG DEBIAN_FRONTEND=noninteractive + +ARG POSTGRES_VERSION=16 +ARG DOCUMENTDB_VERSION + +RUN test -n "$DOCUMENTDB_VERSION" || (echo "DOCUMENTDB_VERSION not set" && false) + +RUN apt-get update + +RUN apt-get install -y --no-install-recommends \ + wget \ + gnupg2 \ + lsb-release \ + ca-certificates \ + locales + +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ + locale-gen en_US.UTF-8 + +ENV LC_ALL=en_US.UTF-8 +ENV LANGUAGE=en_US +ENV LC_COLLATE=en_US.UTF-8 +ENV LC_CTYPE=en_US.UTF-8 +ENV LANG=en_US.UTF-8 + +RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main ${POSTGRES_VERSION}" > /etc/apt/sources.list.d/pgdg.list && \ + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + tzdata \ + build-essential \ + pkg-config \ + cmake \ + git \ + postgresql-${POSTGRES_VERSION} \ + postgresql-server-dev-${POSTGRES_VERSION} \ + libpq-dev \ + libicu-dev \ + libkrb5-dev \ + postgresql-${POSTGRES_VERSION}-cron \ + postgresql-${POSTGRES_VERSION}-pgvector \ + postgresql-${POSTGRES_VERSION}-postgis-3 \ + postgresql-${POSTGRES_VERSION}-rum \ + devscripts \ + debhelper \ + dpkg-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY scripts /tmp/install_setup + +RUN CLEAN_SETUP=1 INSTALL_DEPENDENCIES_ROOT=/tmp/install_setup /tmp/install_setup/install_setup_pcre2.sh +RUN CLEAN_SETUP=1 INSTALL_DEPENDENCIES_ROOT=/tmp/install_setup /tmp/install_setup/install_setup_intel_decimal_math_lib.sh +RUN CLEAN_SETUP=1 INSTALL_DEPENDENCIES_ROOT=/tmp/install_setup /tmp/install_setup/install_citus_indent.sh + +RUN export CLEAN_SETUP=1 && \ + export INSTALL_DEPENDENCIES_ROOT=/tmp/install_setup && \ + MAKE_PROGRAM=cmake /tmp/install_setup/install_setup_libbson.sh && \ + /tmp/install_setup/install_setup_pcre2.sh && \ + /tmp/install_setup/install_setup_intel_decimal_math_lib.sh && \ + /tmp/install_setup/install_citus_indent.sh + +# Set the working directory inside the container +WORKDIR /build + +# Copy the source code into the container +COPY . /build + +# Setup the debian packaging +COPY ferretdb_packaging/debian_files /build/debian +RUN sed -i "s/POSTGRES_VERSION/${POSTGRES_VERSION}/g" /build/debian/control +RUN sed -i "s/DOCUMENTDB_VERSION/${DOCUMENTDB_VERSION}/g" /build/debian/changelog + +COPY ferretdb_packaging/packaging-entrypoint.sh /usr/local/bin/packaging-entrypoint.sh + +# Set the entrypoint +ENTRYPOINT ["packaging-entrypoint.sh"] diff --git a/ferretdb_packaging/README.md b/ferretdb_packaging/README.md new file mode 100644 index 000000000..01ad60eda --- /dev/null +++ b/ferretdb_packaging/README.md @@ -0,0 +1,10 @@ +# To Build Your Own Debian Packages With Docker +Run `./ferretdb_packaging/build_packages.sh -h` and follow the instructions. +E.g. to build for Debian 12 and PostgreSQL 16 for 0.100.0~ferretdb~2.0.1 version, run: +``` +./ferretdb_packaging/build_packages.sh --os deb12 --pg 16 --version 0.100.0~ferretdb~2.0.1 +``` + +Packages can be found at the `packages` directory by default, but that can be configured with the `--output-dir` option. + +**Note:** The packages do not include pg_documentdb_distributed in the `internal` directory. diff --git a/ferretdb_packaging/build_packages.sh b/ferretdb_packaging/build_packages.sh new file mode 100755 index 000000000..46cca1b7e --- /dev/null +++ b/ferretdb_packaging/build_packages.sh @@ -0,0 +1,155 @@ +#!/bin/bash + +set -euo pipefail + +# Function to display help message +function show_help { + echo "Usage: $0 --os --pg [--test-clean-install] [--output-dir ] [-h|--help]" + echo "" + echo "Description:" + echo " This script builds extension packages using Docker." + echo "" + echo "Mandatory Arguments:" + echo " --os OS to build packages for. Possible values: [deb11, deb12, ubuntu20.04, ubuntu22.04, ubuntu24.04]" + echo " --pg PG version to build packages for. Possible values: [15, 16]" + echo " --version The debian conformed version of documentdb to build. Examples: [0.100.0, 0.100.0~ferretdb]" + echo "" + echo "Optional Arguments:" + echo " --test-clean-install Test installing the packages in a clean Docker container." + echo " --output-dir Relative path from the repo root of the directory where to drop the packages. The directory will be created if it doesn't exist. Default: packaging" + echo " -h, --help Display this help message." + exit 0 +} + +# Initialize variables +OS="" +PG="" +DOCUMENTDB_VERSION="" +TEST_CLEAN_INSTALL=false +OUTPUT_DIR="packaging" # Default value for output directory + +# Process arguments to convert long options to short ones +while [[ $# -gt 0 ]]; do + case "$1" in + --os) + shift + case $1 in + deb11|deb12|ubuntu20.04|ubuntu22.04|ubuntu24.04) + OS=$1 + ;; + *) + error_exit "Invalid --os value. Allowed values are [deb11, deb12, ubuntu20.04, ubuntu22.04, ubuntu24.04]" + ;; + esac + ;; + --pg) + shift + case $1 in + 15|16) + PG=$1 + ;; + *) + error_exit "Invalid --pg value. Allowed values are [15, 16]" + ;; + esac + ;; + --version) + shift + DOCUMENTDB_VERSION=$1 + ;; + --test-clean-install) + TEST_CLEAN_INSTALL=true + ;; + --output-dir) + shift + OUTPUT_DIR=$1 + ;; + -h|--help) + show_help + ;; + *) + echo "Unknown argument: $1" + show_help + exit 1 + ;; + esac + shift +done + +# Check mandatory arguments +if [[ -z "$OS" ]]; then + echo "Error: --os is required." + show_help + exit 1 +fi + +if [[ -z "$PG" ]]; then + echo "Error: --pg is required." + show_help + exit 1 +fi + +if [[ -z "$DOCUMENTDB_VERSION" ]]; then + echo "Error: --version is required." + show_help + exit 1 +fi + + +# Set the appropriate Docker image based on the OS +case $OS in + deb11) + DOCKER_IMAGE="debian:bullseye" + ;; + deb12) + DOCKER_IMAGE="debian:bookworm" + ;; + ubuntu20.04) + DOCKER_IMAGE="ubuntu:20.04" + ;; + ubuntu22.04) + DOCKER_IMAGE="ubuntu:22.04" + ;; + ubuntu24.04) + DOCKER_IMAGE="ubuntu:24.04" + ;; +esac + +repo_root=$(git rev-parse --show-toplevel) +abs_output_dir="$repo_root/$OUTPUT_DIR" +cd $repo_root + +echo "Building packages for OS: $OS, PostgreSQL version: $PG, DOCUMENTDB version: $DOCUMENTDB_VERSION" +echo "Output directory: $abs_output_dir" + +# Create the output directory if it doesn't exist +mkdir -p $abs_output_dir + +# Build the Docker image while showing the output to the console +docker build --platform linux/amd64 -t documentdb-build-packages:latest -f ferretdb_packaging/Dockerfile_build_deb_packages \ + --build-arg BASE_IMAGE=$DOCKER_IMAGE --build-arg POSTGRES_VERSION=$PG --build-arg DOCUMENTDB_VERSION=$DOCUMENTDB_VERSION . + +# Run the Docker container to build the packages +docker run --platform linux/amd64 --rm -v $abs_output_dir:/output documentdb-build-packages:latest + +echo "Packages built successfully!!" + +if [[ $TEST_CLEAN_INSTALL == true ]]; then + echo "Testing clean installation in a Docker container..." + + deb_package_name=$(ls $abs_output_dir | grep -E "postgresql-$PG-documentdb_${DOCUMENTDB_VERSION}_amd64.deb" | grep -v "dbg" | head -n 1) + deb_package_rel_path="$OUTPUT_DIR/$deb_package_name" + + echo "Debian package path: $deb_package_rel_path" + + # Build the Docker image while showing the output to the console + docker build --platform linux/amd64 -t documentdb-test-packages:latest -f ferretdb_packaging/test_packages/Dockerfile_test_install_deb_packages \ + --build-arg BASE_IMAGE=$DOCKER_IMAGE --build-arg POSTGRES_VERSION=$PG --build-arg DEB_PACKAGE_REL_PATH=$deb_package_rel_path . + + # Run the Docker container to test the packages + docker run --platform linux/amd64 --rm documentdb-test-packages:latest + + echo "Clean installation test successful!!" +fi + +echo "Packages are available in $abs_output_dir" diff --git a/ferretdb_packaging/debian_files/changelog b/ferretdb_packaging/debian_files/changelog new file mode 100644 index 000000000..e8710acec --- /dev/null +++ b/ferretdb_packaging/debian_files/changelog @@ -0,0 +1,5 @@ +documentdb (DOCUMENTDB_VERSION) unstable; urgency=low + + * Initial release. + + -- FerretDB Packages Fri, 24 Jan 2025 12:00:00 +0000 diff --git a/ferretdb_packaging/debian_files/compat b/ferretdb_packaging/debian_files/compat new file mode 100644 index 000000000..48082f72f --- /dev/null +++ b/ferretdb_packaging/debian_files/compat @@ -0,0 +1 @@ +12 diff --git a/ferretdb_packaging/debian_files/control b/ferretdb_packaging/debian_files/control new file mode 100644 index 000000000..4574d1cb7 --- /dev/null +++ b/ferretdb_packaging/debian_files/control @@ -0,0 +1,10 @@ +Source: documentdb +Section: database +Priority: optional +Build-Depends: debhelper (>= 11), postgresql-server-dev-POSTGRES_VERSION +Maintainer: FerretDB Packages + +Package: postgresql-POSTGRES_VERSION-documentdb +Architecture: any +Depends: ${shlibs:Depends}, ${misc:Depends}, postgresql-POSTGRES_VERSION, postgresql-POSTGRES_VERSION-cron, postgresql-POSTGRES_VERSION-pgvector, postgresql-POSTGRES_VERSION-postgis-3, postgresql-POSTGRES_VERSION-rum +Description: DOCUMENTDB diff --git a/ferretdb_packaging/debian_files/rules b/ferretdb_packaging/debian_files/rules new file mode 100755 index 000000000..363dd034d --- /dev/null +++ b/ferretdb_packaging/debian_files/rules @@ -0,0 +1,10 @@ +#!/usr/bin/make -f + +%: + dh $@ + +override_dh_auto_test: + make install + adduser --disabled-password --gecos "" documentdb + chown -R documentdb:documentdb . + su documentdb -c "make check" diff --git a/ferretdb_packaging/packaging-entrypoint.sh b/ferretdb_packaging/packaging-entrypoint.sh new file mode 100755 index 000000000..2aaeaf5d0 --- /dev/null +++ b/ferretdb_packaging/packaging-entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +# Change to the build directory +cd /build + +# Keep the internal directory out of the Debian package +sed -i '/internal/d' Makefile + +# Build the Debian package +debuild -us -uc + +# Create the output directory if it doesn't exist +mkdir -p /output + +# Copy the built packages to the output directory +cp ../*.deb /output/ + +# Change ownership of the output files to match the host user's UID and GID +chown -R $(stat -c "%u:%g" /output) /output diff --git a/ferretdb_packaging/test_packages/Dockerfile_test_install_deb_packages b/ferretdb_packaging/test_packages/Dockerfile_test_install_deb_packages new file mode 100644 index 000000000..c6231dc35 --- /dev/null +++ b/ferretdb_packaging/test_packages/Dockerfile_test_install_deb_packages @@ -0,0 +1,46 @@ +ARG BASE_IMAGE=debian:bookworm +FROM --platform=linux/amd64 ${BASE_IMAGE} +ARG DEBIAN_FRONTEND=noninteractive + +ARG POSTGRES_VERSION=16 +ARG DEB_PACKAGE_REL_PATH=packages/postgresql-16-documentdb-1_1.0.0_amd64.deb + +RUN apt-get update + +RUN apt-get install -y --no-install-recommends \ + make \ + wget \ + gnupg2 \ + lsb-release \ + ca-certificates \ + locales + +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ + locale-gen en_US.UTF-8 + +ENV LC_ALL=en_US.UTF-8 +ENV LANGUAGE=en_US +ENV LC_COLLATE=en_US.UTF-8 +ENV LC_CTYPE=en_US.UTF-8 +ENV LANG=en_US.UTF-8 + +RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main ${POSTGRES_VERSION}" > /etc/apt/sources.list.d/pgdg.list && \ + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + +# actual dependencies of the package +RUN apt-get update && apt-get install -y \ + postgresql-${POSTGRES_VERSION} \ + postgresql-${POSTGRES_VERSION}-cron \ + postgresql-${POSTGRES_VERSION}-pgvector \ + postgresql-${POSTGRES_VERSION}-postgis-3 \ + postgresql-${POSTGRES_VERSION}-rum + +WORKDIR /test-install + +COPY . /test-install + +RUN dpkg -i ${DEB_PACKAGE_REL_PATH} + +COPY ferretdb_packaging/test_packages/test-install-entrypoint.sh /usr/local/bin/test-install-entrypoint.sh + +ENTRYPOINT ["test-install-entrypoint.sh"] diff --git a/ferretdb_packaging/test_packages/test-install-entrypoint.sh b/ferretdb_packaging/test_packages/test-install-entrypoint.sh new file mode 100755 index 000000000..fd22dfc86 --- /dev/null +++ b/ferretdb_packaging/test_packages/test-install-entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +# Change to the test directory +cd /test-install + +# Keep the internal directory out of the testing +sed -i '/internal/d' Makefile + +# Run the test +adduser --disabled-password --gecos "" documentdb +chown -R documentdb:documentdb . +su documentdb -c "make check" From 3cfbfa12f63acf0976ff66045cfe0a2ba8edc77e Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Fri, 21 Feb 2025 04:47:02 +0400 Subject: [PATCH 05/59] Set better artifact name (#7) --- .github/workflows/ferretdb_go_tests.yml | 5 +---- .github/workflows/ferretdb_packages.yml | 11 +++-------- ferretdb_packaging/.gitignore | 1 - 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ferretdb_go_tests.yml b/.github/workflows/ferretdb_go_tests.yml index 2c3c01211..8a36fd108 100644 --- a/.github/workflows/ferretdb_go_tests.yml +++ b/.github/workflows/ferretdb_go_tests.yml @@ -26,10 +26,7 @@ env: jobs: test: name: Test - # https://www.ubicloud.com/docs/about/pricing#github-actions - # https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#per-minute-rates-for-larger-runners - runs-on: ubicloud-standard-4 - + runs-on: ubuntu-24.04 timeout-minutes: 15 # Do not run this job in parallel for any PR change or branch push. diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index a962fe40b..30b936b4b 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -32,11 +32,7 @@ concurrency: jobs: build: name: Build .debs (${{ matrix.os }}, Pg${{ matrix.pg }}) - - # https://www.ubicloud.com/docs/about/pricing#github-actions - # https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#per-minute-rates-for-larger-runners - runs-on: ubicloud-standard-4 - + runs-on: ubuntu-24.04 timeout-minutes: 40 if: > @@ -65,8 +61,7 @@ jobs: cd ferretdb_packaging go mod tidy go mod verify - go build -v -o=bin/ ./defineversion - bin/defineversion --control-file ../pg_documentdb_core/documentdb_core.control + go run ./defineversion --control-file ../pg_documentdb_core/documentdb_core.control - name: Build ${{ steps.version.outputs.version }} if: steps.version.outputs.version != '' @@ -75,7 +70,7 @@ jobs: - name: Upload uses: actions/upload-artifact@v4 with: - name: ${{ matrix.os }}-${{ matrix.pg }}-debs + name: ${{ matrix.os }}-${{ matrix.pg }}-${{ steps.version.outputs.version }} path: packaging/*.deb retention-days: 1 if-no-files-found: error diff --git a/ferretdb_packaging/.gitignore b/ferretdb_packaging/.gitignore index 43758e150..353ce6cb8 100644 --- a/ferretdb_packaging/.gitignore +++ b/ferretdb_packaging/.gitignore @@ -1,3 +1,2 @@ # FerretDB packaging !*go.mod -/bin From 04821c5c611e4f15572011974dd2d46196e1a024 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Fri, 21 Feb 2025 12:49:22 +0400 Subject: [PATCH 06/59] Install python3 for test installation image (#8) --- .../test_packages/Dockerfile_test_install_deb_packages | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ferretdb_packaging/test_packages/Dockerfile_test_install_deb_packages b/ferretdb_packaging/test_packages/Dockerfile_test_install_deb_packages index c6231dc35..89f918e54 100644 --- a/ferretdb_packaging/test_packages/Dockerfile_test_install_deb_packages +++ b/ferretdb_packaging/test_packages/Dockerfile_test_install_deb_packages @@ -13,7 +13,8 @@ RUN apt-get install -y --no-install-recommends \ gnupg2 \ lsb-release \ ca-certificates \ - locales + locales \ + python3 RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ locale-gen en_US.UTF-8 From 299aa443b7926e7dbda5908d1028ba84d8f8d744 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Mon, 24 Feb 2025 16:17:55 +0900 Subject: [PATCH 07/59] Add release checklist (#9) Closes FerretDB/FerretDB#4724. --- .github/RELEASE_CHECKLIST.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/RELEASE_CHECKLIST.md diff --git a/.github/RELEASE_CHECKLIST.md b/.github/RELEASE_CHECKLIST.md new file mode 100644 index 000000000..d653d9737 --- /dev/null +++ b/.github/RELEASE_CHECKLIST.md @@ -0,0 +1,20 @@ +# FerretDB Release Checklist + +## Preparation + +1. Create draft release on GitHub with autogenerated release notes. +2. Add link to [CHANGELOG.md](../CHANGELOG.md) in the release description. + +## Git tag + +1. On `ferretdb` branch, make a signed tag `vX.Y.Z-ferretdb` (or `vX.Y.Z-ferretdb-A.B.C`) + where `X.Y.Z` is the SemVar formatted version of documentdb, `A.B.C` is the compatible FerretDB version. +2. Check `git status` output. +3. Push it! + +## Release + +1. Find [ferretdb_packages CI build](https://github.com/FerretDB/documentdb/actions/workflows/ferretdb_packages.yml?query=event%3Apush) + for the tag to release. +2. Upload `.deb` packages to the draft release. +3. Publish release on GitHub. From 9e6fe15794233552608e211f1663b12579cc9165 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Mon, 24 Feb 2025 14:39:13 +0400 Subject: [PATCH 08/59] Update release checklist (#10) --- .github/RELEASE_CHECKLIST.md | 17 ++++++++++------- .github/containers/Build-Ubuntu/Dockerfile | 4 ++-- .github/workflows/codeql.yml | 5 ++--- .github/workflows/ferretdb_packages.yml | 2 +- CHANGELOG.md | 4 ++-- ferretdb_packaging/README.md | 11 +++++++---- ferretdb_packaging/build_packages.sh | 17 +++++++---------- ferretdb_packaging/defineversion/main_test.go | 4 ++-- 8 files changed, 33 insertions(+), 31 deletions(-) diff --git a/.github/RELEASE_CHECKLIST.md b/.github/RELEASE_CHECKLIST.md index d653d9737..4f1cc394a 100644 --- a/.github/RELEASE_CHECKLIST.md +++ b/.github/RELEASE_CHECKLIST.md @@ -1,20 +1,23 @@ -# FerretDB Release Checklist +# FerretDB's DocumentDB Release Checklist ## Preparation -1. Create draft release on GitHub with autogenerated release notes. -2. Add link to [CHANGELOG.md](../CHANGELOG.md) in the release description. +1. Create draft release on GitHub to see a list of merged PRs. +2. Update CHANGELOG.md manually. +3. Push changes. ## Git tag -1. On `ferretdb` branch, make a signed tag `vX.Y.Z-ferretdb` (or `vX.Y.Z-ferretdb-A.B.C`) - where `X.Y.Z` is the SemVar formatted version of documentdb, `A.B.C` is the compatible FerretDB version. +1. Make a signed tag `vX.Y.Z-ferretdb-A.B.C(-p)` (like `v0.102.0-ferretdb-2.0.0-rc.2`), + where `X.Y.Z` is the SemVar formatted version of DocumentDB (like `0.102.0`), + and `A.B.C(-p)` is the compatible FerretDB version (like `2.0.0-rc.2`). 2. Check `git status` output. 3. Push it! ## Release -1. Find [ferretdb_packages CI build](https://github.com/FerretDB/documentdb/actions/workflows/ferretdb_packages.yml?query=event%3Apush) +1. Find [Packages CI build](https://github.com/FerretDB/documentdb/actions/workflows/ferretdb_packages.yml?query=event%3Apush) for the tag to release. 2. Upload `.deb` packages to the draft release. -3. Publish release on GitHub. +3. Update release notes with the list of changes from CHANGELOG.md. +4. Publish release on GitHub. diff --git a/.github/containers/Build-Ubuntu/Dockerfile b/.github/containers/Build-Ubuntu/Dockerfile index e3542b2eb..325ae51e1 100644 --- a/.github/containers/Build-Ubuntu/Dockerfile +++ b/.github/containers/Build-Ubuntu/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=linux/amd64 mcr.microsoft.com/mirror/docker/library/ubuntu:20.04 +FROM --platform=linux/amd64 mcr.microsoft.com/mirror/docker/library/ubuntu:22.04 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ && apt-get install -qy \ @@ -62,4 +62,4 @@ USER runner LABEL org.opencontainers.image.source=https://github.com/microsoft/documentdb LABEL org.opencontainers.image.description="DocumentDB ubuntu build image" -LABEL org.opencontainers.image.licenses=MIT \ No newline at end of file +LABEL org.opencontainers.image.licenses=MIT diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e39481ca7..d0b37aa13 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -8,7 +8,7 @@ on: jobs: analyze: name: Analyze - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 permissions: actions: read contents: read @@ -54,7 +54,7 @@ jobs: postgresql-15-cron \ postgresql-15-pgvector \ postgresql-15-postgis-3 \ - postgresql-15-rum + postgresql-15-rum export CLEAN_SETUP=1 export INSTALL_DEPENDENCIES_ROOT=/tmp/install_setup mkdir -p /tmp/install_setup @@ -71,4 +71,3 @@ jobs: - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 - diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index 30b936b4b..abb2d88b1 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -45,7 +45,7 @@ jobs: strategy: fail-fast: false matrix: - os: [deb11, deb12, ubuntu20.04, ubuntu22.04, ubuntu24.04] + os: [deb11, deb12, ubuntu22.04, ubuntu24.04] pg: [15, 16] steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index a8cda3d21..db160586b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,8 @@ * Build pg_documentdb for PostgreSQL 17 *[Infra]* (#13) * Enable support of currentOp aggregation stage, along with collstats, dbstats, and indexStats *[Commands]* (#52) * Allow inlining $unwind with $lookup with `preserveNullAndEmptyArrays` *[Perf]* -* Skip loading documents if group expression is constant *[Perf]* +* Skip loading documents if group expression is constant *[Perf]* * Fix Merge stage not outputing to target collection *[Bugfix]* (#20) ### documentdb v0.100-0 (January 23rd, 2025) ### -Initial Release \ No newline at end of file +Initial Release diff --git a/ferretdb_packaging/README.md b/ferretdb_packaging/README.md index 01ad60eda..18f9dbe53 100644 --- a/ferretdb_packaging/README.md +++ b/ferretdb_packaging/README.md @@ -1,10 +1,13 @@ # To Build Your Own Debian Packages With Docker + Run `./ferretdb_packaging/build_packages.sh -h` and follow the instructions. -E.g. to build for Debian 12 and PostgreSQL 16 for 0.100.0~ferretdb~2.0.1 version, run: -``` -./ferretdb_packaging/build_packages.sh --os deb12 --pg 16 --version 0.100.0~ferretdb~2.0.1 +E.g. to build for Debian 12 and PostgreSQL 16 for `0.102.0~ferretdb~2.0.0~rc.2` version, run: + +```sh +./ferretdb_packaging/build_packages.sh --os deb12 --pg 16 --version 0.102.0~ferretdb~2.0.0~rc.2 ``` -Packages can be found at the `packages` directory by default, but that can be configured with the `--output-dir` option. +Packages can be found at the `packages` directory by default, +but that can be configured with the `--output-dir` option. **Note:** The packages do not include pg_documentdb_distributed in the `internal` directory. diff --git a/ferretdb_packaging/build_packages.sh b/ferretdb_packaging/build_packages.sh index 46cca1b7e..27a4320fa 100755 --- a/ferretdb_packaging/build_packages.sh +++ b/ferretdb_packaging/build_packages.sh @@ -10,9 +10,9 @@ function show_help { echo " This script builds extension packages using Docker." echo "" echo "Mandatory Arguments:" - echo " --os OS to build packages for. Possible values: [deb11, deb12, ubuntu20.04, ubuntu22.04, ubuntu24.04]" - echo " --pg PG version to build packages for. Possible values: [15, 16]" - echo " --version The debian conformed version of documentdb to build. Examples: [0.100.0, 0.100.0~ferretdb]" + echo " --os OS to build packages for. Possible values: [deb11, deb12, ubuntu22.04, ubuntu24.04]" + echo " --pg PG version to build packages for. Possible values: [15, 16, 17]" + echo " --version The debian conformed version of documentdb to build. Example: 0.102.0~ferretdb~2.0.0~rc.2" echo "" echo "Optional Arguments:" echo " --test-clean-install Test installing the packages in a clean Docker container." @@ -34,22 +34,22 @@ while [[ $# -gt 0 ]]; do --os) shift case $1 in - deb11|deb12|ubuntu20.04|ubuntu22.04|ubuntu24.04) + deb11|deb12|ubuntu22.04|ubuntu24.04) OS=$1 ;; *) - error_exit "Invalid --os value. Allowed values are [deb11, deb12, ubuntu20.04, ubuntu22.04, ubuntu24.04]" + error_exit "Invalid --os value. Allowed values are [deb11, deb12, ubuntu22.04, ubuntu24.04]" ;; esac ;; --pg) shift case $1 in - 15|16) + 15|16|17) PG=$1 ;; *) - error_exit "Invalid --pg value. Allowed values are [15, 16]" + error_exit "Invalid --pg value. Allowed values are [15, 16, 17]" ;; esac ;; @@ -104,9 +104,6 @@ case $OS in deb12) DOCKER_IMAGE="debian:bookworm" ;; - ubuntu20.04) - DOCKER_IMAGE="ubuntu:20.04" - ;; ubuntu22.04) DOCKER_IMAGE="ubuntu:22.04" ;; diff --git a/ferretdb_packaging/defineversion/main_test.go b/ferretdb_packaging/defineversion/main_test.go index 74a5edbb9..765c999b7 100644 --- a/ferretdb_packaging/defineversion/main_test.go +++ b/ferretdb_packaging/defineversion/main_test.go @@ -211,9 +211,9 @@ _GitHubActionsFileCommandDelimeter_ } func TestReadControlDefaultVersion(t *testing.T) { - dir := t.TempDir() + controlF, err := os.CreateTemp(t.TempDir(), "test.control") + require.NoError(t, err) - controlF, err := os.CreateTemp(dir, "test.control") defer controlF.Close() //nolint:errcheck // temporary file for testing buf := `comment = 'API surface for DocumentDB for PostgreSQL' From 39ec23dfdc92c8a40772a931a53b7dc67deb3f62 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Mon, 24 Feb 2025 14:43:46 +0400 Subject: [PATCH 09/59] Update CHANGELOG.md --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index db160586b..1c103efa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +### documentdb v0.102.0-ferretdb-2.0.0-rc.2 (February 24, 2025) ### + +This version works best with FerretDB v2.0.0-rc.2. + +Debian and Ubuntu `.deb` packages are now provided. + ### documentdb v0.102-0 (Unreleased) ### * Support index pushdown for vector search queries *[Bugfix]* * Support exact search for vector search queries *[Feature]* From ee54dd962fe140628c22337df163f4bb2d189141 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 25 Feb 2025 21:08:13 +0400 Subject: [PATCH 10/59] Update CHANGELOG (#14) --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c103efa1..f064d0676 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ -### documentdb v0.102.0-ferretdb-2.0.0-rc.2 (February 24, 2025) ### +### DocumentDB v0.102.0-ferretdb-2.0.0-rc.2 (February 24, 2025) ### -This version works best with FerretDB v2.0.0-rc.2. +This version works best with [FerretDB v2.0.0-rc.2](https://github.com/FerretDB/FerretDB/releases/tag/v2.0.0-rc.2). -Debian and Ubuntu `.deb` packages are now provided. +Debian and Ubuntu `.deb` packages are now [provided](https://github.com/FerretDB/documentdb/releases/tag/v0.102.0-ferretdb-2.0.0-rc.2). +Docker images are available [here](https://github.com/FerretDB/FerretDB/pkgs/container/postgres-documentdb). ### documentdb v0.102-0 (Unreleased) ### * Support index pushdown for vector search queries *[Bugfix]* From 9bca0e3bf1d71dc3433e900a46dd1500c55047cd Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Thu, 27 Feb 2025 21:46:35 +0900 Subject: [PATCH 11/59] Make `.deb` package names unique (#12) Closes FerretDB/FerretDB#4724. --- ferretdb_packaging/build_packages.sh | 5 ++--- ferretdb_packaging/packaging-entrypoint.sh | 12 +++++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/ferretdb_packaging/build_packages.sh b/ferretdb_packaging/build_packages.sh index 27a4320fa..86b451b00 100755 --- a/ferretdb_packaging/build_packages.sh +++ b/ferretdb_packaging/build_packages.sh @@ -95,7 +95,6 @@ if [[ -z "$DOCUMENTDB_VERSION" ]]; then exit 1 fi - # Set the appropriate Docker image based on the OS case $OS in deb11) @@ -127,14 +126,14 @@ docker build --platform linux/amd64 -t documentdb-build-packages:latest -f ferre --build-arg BASE_IMAGE=$DOCKER_IMAGE --build-arg POSTGRES_VERSION=$PG --build-arg DOCUMENTDB_VERSION=$DOCUMENTDB_VERSION . # Run the Docker container to build the packages -docker run --platform linux/amd64 --rm -v $abs_output_dir:/output documentdb-build-packages:latest +docker run --platform linux/amd64 --rm --env OS=$OS -v $abs_output_dir:/output documentdb-build-packages:latest echo "Packages built successfully!!" if [[ $TEST_CLEAN_INSTALL == true ]]; then echo "Testing clean installation in a Docker container..." - deb_package_name=$(ls $abs_output_dir | grep -E "postgresql-$PG-documentdb_${DOCUMENTDB_VERSION}_amd64.deb" | grep -v "dbg" | head -n 1) + deb_package_name=$(ls $abs_output_dir | grep -E "${OS}-postgresql-${PG}-documentdb_${DOCUMENTDB_VERSION}_amd64.deb" | grep -v "dbg" | head -n 1) deb_package_rel_path="$OUTPUT_DIR/$deb_package_name" echo "Debian package path: $deb_package_rel_path" diff --git a/ferretdb_packaging/packaging-entrypoint.sh b/ferretdb_packaging/packaging-entrypoint.sh index 2aaeaf5d0..0bd083629 100755 --- a/ferretdb_packaging/packaging-entrypoint.sh +++ b/ferretdb_packaging/packaging-entrypoint.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e +test -n "$OS" || (echo "OS not set" && false) + # Change to the build directory cd /build @@ -10,11 +12,19 @@ sed -i '/internal/d' Makefile # Build the Debian package debuild -us -uc +# Change to the root to make file renaming expression simpler +cd / + +# Rename .deb files to include the OS name prefix +for f in *.deb; do + mv $f $OS-$f; +done + # Create the output directory if it doesn't exist mkdir -p /output # Copy the built packages to the output directory -cp ../*.deb /output/ +cp *.deb /output/ # Change ownership of the output files to match the host user's UID and GID chown -R $(stat -c "%u:%g" /output) /output From 8fe521952aa7d8da9ffd813ad3bdea05f60c35c2 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 27 Feb 2025 17:58:24 +0400 Subject: [PATCH 12/59] Small tweaks (#17) --- .github/settings.yml | 4 ---- ferretdb_packaging/defineversion/main.go | 2 +- ferretdb_packaging/defineversion/main_test.go | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/settings.yml b/.github/settings.yml index 974cae3f9..5779efa2f 100644 --- a/.github/settings.yml +++ b/.github/settings.yml @@ -32,7 +32,3 @@ labels: - name: packages color: "#9B022C" description: PRs that should build packages - - - name: trust - color: "#00FF00" - description: PRs that can access Actions secrets diff --git a/ferretdb_packaging/defineversion/main.go b/ferretdb_packaging/defineversion/main.go index aeafdcd69..ec7f689b0 100644 --- a/ferretdb_packaging/defineversion/main.go +++ b/ferretdb_packaging/defineversion/main.go @@ -202,7 +202,7 @@ func definePackagerVersionForTag(tag string) (string, error) { // setResults sets action output parameters, summary, etc. func setResults(action *githubactions.Action, res string) { - output := fmt.Sprintf("version: %s", res) + output := fmt.Sprintf("version: `%s`", res) action.AddStepSummary(output) action.Infof("%s", output) diff --git a/ferretdb_packaging/defineversion/main_test.go b/ferretdb_packaging/defineversion/main_test.go index 765c999b7..dcbba4524 100644 --- a/ferretdb_packaging/defineversion/main_test.go +++ b/ferretdb_packaging/defineversion/main_test.go @@ -193,7 +193,7 @@ func TestResults(t *testing.T) { setResults(action, version) - expected := "version: 0.100.0~ferretdb\n" + expected := "version: `0.100.0~ferretdb`\n" assert.Equal(t, expected, stdout.String(), "stdout does not match") b, err := io.ReadAll(summaryF) From 4063590e57d59e01e9aef8f292457c5748136e79 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 27 Feb 2025 22:42:54 +0400 Subject: [PATCH 13/59] Build packages for PostgreSQL 17 (#11) --- .github/workflows/ferretdb_packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index abb2d88b1..8aa49b76a 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -46,7 +46,7 @@ jobs: fail-fast: false matrix: os: [deb11, deb12, ubuntu22.04, ubuntu24.04] - pg: [15, 16] + pg: [15, 16, 17] steps: - name: Checkout code From c9c13d6881d9172c1f2feff3ffd6f129e88a47cc Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Mon, 3 Mar 2025 22:14:28 +0900 Subject: [PATCH 14/59] Define Docker build tags (#16) Closes FerretDB/FerretDB#4725. --- .github/workflows/ferretdb_packages.yml | 45 +- .../defineversion/docker_tags.go | 225 ++++++++ .../defineversion/docker_tags_test.go | 488 ++++++++++++++++++ ferretdb_packaging/defineversion/main.go | 80 ++- ferretdb_packaging/defineversion/main_test.go | 54 +- 5 files changed, 864 insertions(+), 28 deletions(-) create mode 100644 ferretdb_packaging/defineversion/docker_tags.go create mode 100644 ferretdb_packaging/defineversion/docker_tags_test.go diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index 8aa49b76a..3ad1854aa 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -30,7 +30,7 @@ concurrency: cancel-in-progress: false jobs: - build: + deb: name: Build .debs (${{ matrix.os }}, Pg${{ matrix.pg }}) runs-on: ubuntu-24.04 timeout-minutes: 40 @@ -61,7 +61,7 @@ jobs: cd ferretdb_packaging go mod tidy go mod verify - go run ./defineversion --control-file ../pg_documentdb_core/documentdb_core.control + go run ./defineversion --command deb-version --control-file ../pg_documentdb_core/documentdb_core.control - name: Build ${{ steps.version.outputs.version }} if: steps.version.outputs.version != '' @@ -81,3 +81,44 @@ jobs: run: | git status git diff --exit-code + + docker: + name: Build Docker (Pg${{ matrix.pg }}) + runs-on: ubuntu-24.04 + timeout-minutes: 40 + + if: > + github.event_name != 'pull_request' || + ( + !contains(github.event.pull_request.labels.*.name, 'not ready') && + contains(github.event.pull_request.labels.*.name, 'packages') + ) + + strategy: + fail-fast: false + matrix: + pg: [15.12, 16.8, 17.4] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: FerretDB/github-actions/setup-go@main + + - name: Define Docker tags + id: tag + run: | + cd ferretdb_packaging + go mod tidy + go mod verify + go run ./defineversion --command docker-tags + env: + INPUT_PG_VERSION: ${{ matrix.pg }} + + # Build and push Docker images + + - name: Check dirty + run: | + git status + git diff --exit-code diff --git a/ferretdb_packaging/defineversion/docker_tags.go b/ferretdb_packaging/defineversion/docker_tags.go new file mode 100644 index 000000000..9cdddbc5d --- /dev/null +++ b/ferretdb_packaging/defineversion/docker_tags.go @@ -0,0 +1,225 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "regexp" + "slices" + "strings" + "text/tabwriter" + + "github.com/sethvargo/go-githubactions" +) + +// images represents Docker image names and tags extracted from the environment. +type images struct { + developmentImages []string + productionImages []string +} + +// pgVer is the version of PostgreSQL. +var pgVer = regexp.MustCompile(`^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)$`) + +// defineDockerTags extracts Docker image names and tags from the environment variables defined by GitHub Actions. +func defineDockerTags(getenv githubactions.GetenvFunc) (*images, error) { + repo := getenv("GITHUB_REPOSITORY") + + // to support GitHub forks + parts := strings.Split(strings.ToLower(repo), "/") + if len(parts) != 2 { + return nil, fmt.Errorf("failed to split %q into owner and name", repo) + } + owner := parts[0] + repo = parts[1] + + var res *images + var err error + + switch event := getenv("GITHUB_EVENT_NAME"); event { + case "pull_request", "pull_request_target": + branch := strings.ToLower(getenv("GITHUB_HEAD_REF")) + res = defineForPR(owner, repo, branch) + + case "push", "schedule", "workflow_run": + refName := strings.ToLower(getenv("GITHUB_REF_NAME")) + + switch refType := strings.ToLower(getenv("GITHUB_REF_TYPE")); refType { + case "branch": + res, err = defineForBranch(owner, repo, refName) + + case "tag": + var major, minor, patch, prerelease string + if major, minor, patch, prerelease, err = semVar(refName); err != nil { + return nil, err + } + + pgVersion := getenv("INPUT_PG_VERSION") + pgMatch := pgVer.FindStringSubmatch(pgVersion) + if pgMatch == nil || len(pgMatch) != pgVer.NumSubexp()+1 { + return nil, fmt.Errorf("unexpected PostgreSQL version %q", pgVersion) + } + + pgMajor := pgMatch[pgVer.SubexpIndex("major")] + pgMinor := pgMatch[pgVer.SubexpIndex("minor")] + + tags := []string{ + fmt.Sprintf("%s-%s.%s.%s-%s", pgMajor, major, minor, patch, prerelease), + "latest", + } + + if pgMinor != "" { + tags = append(tags, fmt.Sprintf("%s.%s-%s.%s.%s-%s", pgMajor, pgMinor, major, minor, patch, prerelease)) + } + + res = defineForTag(owner, repo, tags) + + default: + err = fmt.Errorf("unhandled ref type %q for event %q", refType, event) + } + + default: + err = fmt.Errorf("unhandled event type %q", event) + } + + if err != nil { + return nil, err + } + + if res == nil { + return nil, fmt.Errorf("both res and err are nil") + } + + slices.Sort(res.developmentImages) + slices.Sort(res.productionImages) + + return res, nil +} + +// defineForPR defines Docker image names and tags for pull requests. +func defineForPR(owner, repo, branch string) *images { + // for branches like "dependabot/submodules/XXX" + parts := strings.Split(branch, "/") + branch = parts[len(parts)-1] + + res := &images{ + developmentImages: []string{ + fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:pr-%s", owner, repo, branch), + }, + } + + // PRs are only for testing; no Quay.io and Docker Hub repos + + return res +} + +// defineForBranch defines Docker image names and tags for branch builds. +func defineForBranch(owner, repo, branch string) (*images, error) { + if branch != "ferretdb" { + return nil, fmt.Errorf("unhandled branch %q", branch) + } + + res := &images{ + developmentImages: []string{ + fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:ferretdb", owner, repo), + }, + } + + // forks don't have Quay.io and Docker Hub orgs + if owner != "ferretdb" { + return res, nil + } + + // we don't have Quay.io and Docker Hub repos for other GitHub repos + if repo != "documentdb" { + return res, nil + } + + //res.developmentImages = append(res.developmentImages, "quay.io/ferretdb/postgres-documentdb-dev:ferretdb") + //res.developmentImages = append(res.developmentImages, "ferretdb/postgres-documentdb-dev:ferretdb") + + return res, nil +} + +// defineForTag defines Docker image names and tags for prerelease tag builds. +func defineForTag(owner, repo string, tags []string) *images { + res := new(images) + + for _, t := range tags { + res.developmentImages = append(res.developmentImages, fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s", owner, repo, t)) + res.productionImages = append(res.productionImages, fmt.Sprintf("ghcr.io/%s/postgres-%s:%s", owner, repo, t)) + } + + // forks don't have Quay.io and Docker Hub orgs + if owner != "ferretdb" { + return res + } + + // we don't have Quay.io and Docker Hub repos for other GitHub repos + if repo != "documentdb" { + return res + } + + //for _, t := range tags { + // res.developmentImages = append(res.developmentImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb-dev:%s", t)) + // res.productionImages = append(res.productionImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb:%s", t)) + // + // res.developmentImages = append(res.developmentImages, fmt.Sprintf("ferretdb/postgres-documentdb-dev:%s", t)) + // res.productionImages = append(res.productionImages, fmt.Sprintf("ferretdb/postgres-documentdb:%s", t)) + //} + + return res +} + +// setDockerTagsResults sets action output parameters, summary, etc. +func setDockerTagsResults(action *githubactions.Action, res *images) { + var buf strings.Builder + w := tabwriter.NewWriter(&buf, 1, 1, 1, ' ', tabwriter.Debug) + fmt.Fprintf(w, "\tType\tImage\t\n") + fmt.Fprintf(w, "\t----\t-----\t\n") + + for _, image := range res.developmentImages { + u := imageURL(image) + _, _ = fmt.Fprintf(w, "\tDevelopment\t[`%s`](%s)\t\n", image, u) + } + + for _, image := range res.productionImages { + u := imageURL(image) + _, _ = fmt.Fprintf(w, "\tProduction\t[`%s`](%s)\t\n", image, u) + } + + _ = w.Flush() + + action.AddStepSummary(buf.String()) + action.Infof("%s", buf.String()) + + action.SetOutput("development_images", strings.Join(res.developmentImages, ",")) + action.SetOutput("production_images", strings.Join(res.productionImages, ",")) +} + +// imageURL returns HTML page URL for the given image name and tag. +func imageURL(name string) string { + switch { + case strings.HasPrefix(name, "ghcr.io/"): + return fmt.Sprintf("https://%s", name) + case strings.HasPrefix(name, "quay.io/"): + return fmt.Sprintf("https://%s", name) + } + + name, _, _ = strings.Cut(name, ":") + + // there is no easy way to get Docker Hub URL for the given tag + return fmt.Sprintf("https://hub.docker.com/r/%s/tags", name) +} diff --git a/ferretdb_packaging/defineversion/docker_tags_test.go b/ferretdb_packaging/defineversion/docker_tags_test.go new file mode 100644 index 000000000..d7eecae7b --- /dev/null +++ b/ferretdb_packaging/defineversion/docker_tags_test.go @@ -0,0 +1,488 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "github.com/sethvargo/go-githubactions" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "io" + "os" + "strings" + "testing" +) + +func TestDefineDockerTags(t *testing.T) { + for name, tc := range map[string]struct { + env map[string]string + expected *images + }{ + "pull_request": { + env: map[string]string{ + "GITHUB_BASE_REF": "ferretdb", + "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_HEAD_REF": "docker-tag", + "GITHUB_REF_NAME": "1/merge", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + "INPUT_PG_VERSION": "16", + }, + expected: &images{ + developmentImages: []string{ + "ghcr.io/ferretdb/postgres-documentdb-dev:pr-docker-tag", + }, + }, + }, + "pull_request-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "ferretdb", + "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_HEAD_REF": "docker-tag", + "GITHUB_REF_NAME": "1/merge", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + "INPUT_PG_VERSION": "16", + }, + expected: &images{ + developmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:pr-docker-tag", + }, + }, + }, + + "pull_request_target": { + env: map[string]string{ + "GITHUB_BASE_REF": "ferretdb", + "GITHUB_EVENT_NAME": "pull_request_target", + "GITHUB_HEAD_REF": "docker-tag", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + "INPUT_PG_VERSION": "16", + }, + expected: &images{ + developmentImages: []string{ + "ghcr.io/ferretdb/postgres-documentdb-dev:pr-docker-tag", + }, + }, + }, + "pull_request_target-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "ferretdb", + "GITHUB_EVENT_NAME": "pull_request_target", + "GITHUB_HEAD_REF": "docker-tag", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + "INPUT_PG_VERSION": "16", + }, + expected: &images{ + developmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:pr-docker-tag", + }, + }, + }, + + "push/ferretdb": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + "INPUT_PG_VERSION": "16", + }, + expected: &images{ + developmentImages: []string{ + //"ferretdb/postgres-documentdb-dev:ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:ferretdb", + //"quay.io/ferretdb/postgres-documentdb-dev:ferretdb", + }, + }, + }, + "push/ferretdb-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + "INPUT_PG_VERSION": "16", + }, + expected: &images{ + developmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:ferretdb", + }, + }, + }, + + "push/main": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "main", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + "INPUT_PG_VERSION": "16", + }, + }, + "push/main-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "main", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + "INPUT_PG_VERSION": "16", + }, + }, + + "push/tag/release": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "v0.102.0-ferretdb", + "GITHUB_REF_TYPE": "tag", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + "INPUT_PG_VERSION": "16.7", + }, + expected: &images{ + developmentImages: []string{ + //"ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb", + //"ferretdb/postgres-documentdb-dev:16.7-0.102.0-ferretdb", + //"ferretdb/postgres-documentdb-dev:latest", + "ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:16.7-0.102.0-ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:latest", + //"quay.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb", + //"quay.io/ferretdb/postgres-documentdb-dev:16.7-0.102.0-ferretdb", + //"quay.io/ferretdb/postgres-documentdb-dev:latest", + }, + productionImages: []string{ + //"ferretdb/postgres-documentdb:16-0.102.0-ferretdb", + //"ferretdb/postgres-documentdb:16.7-0.102.0-ferretdb", + //"ferretdb/postgres-documentdb:latest", + "ghcr.io/ferretdb/postgres-documentdb:16-0.102.0-ferretdb", + "ghcr.io/ferretdb/postgres-documentdb:16.7-0.102.0-ferretdb", + "ghcr.io/ferretdb/postgres-documentdb:latest", + //"quay.io/ferretdb/postgres-documentdb:16-0.102.0-ferretdb", + //"quay.io/ferretdb/postgres-documentdb:16.7-0.102.0-ferretdb", + //"quay.io/ferretdb/postgres-documentdb:latest", + }, + }, + }, + "push/tag/release-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "v0.102.0-ferretdb", + "GITHUB_REF_TYPE": "tag", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + "INPUT_PG_VERSION": "16.7", + }, + expected: &images{ + developmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:16-0.102.0-ferretdb", + "ghcr.io/otherorg/postgres-otherrepo-dev:16.7-0.102.0-ferretdb", + "ghcr.io/otherorg/postgres-otherrepo-dev:latest", + }, + productionImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo:16-0.102.0-ferretdb", + "ghcr.io/otherorg/postgres-otherrepo:16.7-0.102.0-ferretdb", + "ghcr.io/otherorg/postgres-otherrepo:latest", + }, + }, + }, + + "push/tag/release-rc": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "v0.102.0-ferretdb-2.0.0-rc2", + "GITHUB_REF_TYPE": "tag", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + "INPUT_PG_VERSION": "16.7", // set major and minor version + }, + expected: &images{ + developmentImages: []string{ + //"ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb-2.0.0-rc2", + //"ferretdb/postgres-documentdb-dev:16.7-0.102.0-ferretdb-2.0.0-rc2", + //"ferretdb/postgres-documentdb-dev:latest", + "ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb-2.0.0-rc2", + "ghcr.io/ferretdb/postgres-documentdb-dev:16.7-0.102.0-ferretdb-2.0.0-rc2", + "ghcr.io/ferretdb/postgres-documentdb-dev:latest", + //"quay.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb-2.0.0-rc2", + //"quay.io/ferretdb/postgres-documentdb-dev:16.7-0.102.0-ferretdb-2.0.0-rc2", + //"quay.io/ferretdb/postgres-documentdb-dev:latest", + }, + productionImages: []string{ + //"ferretdb/postgres-documentdb:16-0.102.0-ferretdb-2.0.0-rc2", + //"ferretdb/postgres-documentdb:16.7-0.102.0-ferretdb-2.0.0-rc2", + //"ferretdb/postgres-documentdb:latest", + "ghcr.io/ferretdb/postgres-documentdb:16-0.102.0-ferretdb-2.0.0-rc2", + "ghcr.io/ferretdb/postgres-documentdb:16.7-0.102.0-ferretdb-2.0.0-rc2", + "ghcr.io/ferretdb/postgres-documentdb:latest", + //"quay.io/ferretdb/postgres-documentdb:16-0.102.0-ferretdb-2.0.0-rc2", + //"quay.io/ferretdb/postgres-documentdb:16.7-0.102.0-ferretdb-2.0.0-rc2", + //"quay.io/ferretdb/postgres-documentdb:latest", + }, + }, + }, + "push/tag/release-rc-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "v0.102.0-ferretdb-2.0.0-rc2", + "GITHUB_REF_TYPE": "tag", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + "INPUT_PG_VERSION": "16.7", // set major and minor version + }, + expected: &images{ + developmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:16-0.102.0-ferretdb-2.0.0-rc2", + "ghcr.io/otherorg/postgres-otherrepo-dev:16.7-0.102.0-ferretdb-2.0.0-rc2", + "ghcr.io/otherorg/postgres-otherrepo-dev:latest", + }, + productionImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo:16-0.102.0-ferretdb-2.0.0-rc2", + "ghcr.io/otherorg/postgres-otherrepo:16.7-0.102.0-ferretdb-2.0.0-rc2", + "ghcr.io/otherorg/postgres-otherrepo:latest", + }, + }, + }, + + "push/tag/release// missing minor": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "v0.102.0-ferretdb-2.0.0-rc2", + "GITHUB_REF_TYPE": "tag", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + "INPUT_PG_VERSION": "16", // missing minor + }, + }, + "push/tag/release-missing-minor-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "v0.102.0-ferretdb-2.0.0-rc2", + "GITHUB_REF_TYPE": "tag", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + "INPUT_PG_VERSION": "16", // missing minor + }, + }, + + "push/tag/wrong": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "0.102.0-ferretdb-2.0.0-rc2", // no leading v + "GITHUB_REF_TYPE": "tag", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + "INPUT_PG_VERSION": "16", + }, + }, + "push/tag/wrong-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "0.102.0-ferretdb-2.0.0-rc2", // no leading v + "GITHUB_REF_TYPE": "tag", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + "INPUT_PG_VERSION": "16", + }, + }, + + "schedule": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "schedule", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + "INPUT_PG_VERSION": "16", + }, + expected: &images{ + developmentImages: []string{ + //"ferretdb/postgres-documentdb-dev:ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:ferretdb", + //"quay.io/ferretdb/postgres-documentdb-dev:ferretdb", + }, + }, + }, + "schedule-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "schedule", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + "INPUT_PG_VERSION": "16", + }, + expected: &images{ + developmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:ferretdb", + }, + }, + }, + + "workflow_run": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "workflow_run", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + "INPUT_PG_VERSION": "16", + }, + expected: &images{ + developmentImages: []string{ + //"ferretdb/postgres-documentdb-dev:ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:ferretdb", + //"quay.io/ferretdb/postgres-documentdb-dev:ferretdb", + }, + }, + }, + "workflow_run-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "workflow_run", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + "INPUT_PG_VERSION": "16", + }, + expected: &images{ + developmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:ferretdb", + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + actual, err := defineDockerTags(getEnvFunc(t, tc.env)) + if tc.expected == nil { + require.Error(t, err) + return + } + + require.NoError(t, err) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestImageURL(t *testing.T) { + // expected URLs should work + assert.Equal( + t, + "https://ghcr.io/ferretdb/postgres-documentdb-dev:pr-docker-tag", + imageURL("ghcr.io/ferretdb/postgres-documentdb-dev:pr-docker-tag"), + ) + assert.Equal( + t, + "https://quay.io/ferretdb/postgres-documentdb-dev:pr-docker-tag", + imageURL("quay.io/ferretdb/postgres-documentdb-dev:pr-docker-tag"), + ) + assert.Equal( + t, + "https://hub.docker.com/r/ferretdb/postgres-documentdb-dev/tags", + imageURL("ferretdb/postgres-documentdb-dev:pr-docker-tag"), + ) +} + +func TestDockerTagsResults(t *testing.T) { + dir := t.TempDir() + + summaryF, err := os.CreateTemp(dir, "summary") + require.NoError(t, err) + defer summaryF.Close() + + outputF, err := os.CreateTemp(dir, "output") + require.NoError(t, err) + defer outputF.Close() + + var stdout bytes.Buffer + getenv := getEnvFunc(t, map[string]string{ + "GITHUB_STEP_SUMMARY": summaryF.Name(), + "GITHUB_OUTPUT": outputF.Name(), + }) + action := githubactions.New(githubactions.WithGetenv(getenv), githubactions.WithWriter(&stdout)) + + result := &images{ + developmentImages: []string{ + "ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:latest", + }, + productionImages: []string{ + "quay.io/ferretdb/postgres-documentdb:latest", + }, + } + + setDockerTagsResults(action, result) + + expectedStdout := strings.ReplaceAll(` + |Type |Image | + |---- |----- | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb) | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:latest'](https://ghcr.io/ferretdb/postgres-documentdb-dev:latest) | + |Production |['quay.io/ferretdb/postgres-documentdb:latest'](https://quay.io/ferretdb/postgres-documentdb:latest) | + +`[1:], "'", "`", + ) + assert.Equal(t, expectedStdout, stdout.String(), "stdout does not match") + + expectedSummary := strings.ReplaceAll(` + |Type |Image | + |---- |----- | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb) | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:latest'](https://ghcr.io/ferretdb/postgres-documentdb-dev:latest) | + |Production |['quay.io/ferretdb/postgres-documentdb:latest'](https://quay.io/ferretdb/postgres-documentdb:latest) | + +`[1:], "'", "`", + ) + b, err := io.ReadAll(summaryF) + require.NoError(t, err) + assert.Equal(t, expectedSummary, string(b), "summary does not match") + + expectedOutput := ` +development_images<<_GitHubActionsFileCommandDelimeter_ +ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb,ghcr.io/ferretdb/postgres-documentdb-dev:latest +_GitHubActionsFileCommandDelimeter_ +production_images<<_GitHubActionsFileCommandDelimeter_ +quay.io/ferretdb/postgres-documentdb:latest +_GitHubActionsFileCommandDelimeter_ +`[1:] + b, err = io.ReadAll(outputF) + require.NoError(t, err) + assert.Equal(t, expectedOutput, string(b), "output parameters does not match") +} diff --git a/ferretdb_packaging/defineversion/main.go b/ferretdb_packaging/defineversion/main.go index ec7f689b0..d3976c370 100644 --- a/ferretdb_packaging/defineversion/main.go +++ b/ferretdb_packaging/defineversion/main.go @@ -27,29 +27,47 @@ import ( ) func main() { + commandF := flag.String("command", "", "command to run, possible values: [deb-version, docker-tags]") + controlFileF := flag.String("control-file", "../pg_documentdb/documentdb.control", "pg_documentdb/documentdb.control file path") flag.Parse() action := githubactions.New() - if *controlFileF == "" { - action.Fatalf("%s", "-control-file flag is empty.") - } + debugEnv(action) - controlDefaultVersion, err := getControlDefaultVersion(*controlFileF) - if err != nil { - action.Fatalf("%s", err) + if *commandF == "" { + action.Fatalf("-command flag is empty.") } - debugEnv(action) + switch *commandF { + case "deb-version": + if *controlFileF == "" { + action.Fatalf("%s", "-control-file flag is empty.") + } - packageVersion, err := definePackageVersion(controlDefaultVersion, action.Getenv) - if err != nil { - action.Fatalf("%s", err) - } + controlDefaultVersion, err := getControlDefaultVersion(*controlFileF) + if err != nil { + action.Fatalf("%s", err) + } + + packageVersion, err := definePackageVersion(controlDefaultVersion, action.Getenv) + if err != nil { + action.Fatalf("%s", err) + } - setResults(action, packageVersion) + setDebianVersionResults(action, packageVersion) + case "docker-tags": + res, err := defineDockerTags(action.Getenv) + if err != nil { + action.Fatalf("%s", err) + } + + setDockerTagsResults(action, res) + default: + action.Fatalf("unhandled command %q", *commandF) + } } // controlDefaultVer matches major, minor and "patch" from default_version field in control file, @@ -170,38 +188,50 @@ func definePackageVersionForBranch(controlDefaultVersion, branch string) (string } } -// definePackagerVersionForTag returns valid Debian package version for tag. -// See [definePackageVersion]. -func definePackagerVersionForTag(tag string) (string, error) { +// semVar parses tag and returns version components. +// +// It returns error for invalid tag syntax, prerelease is missing `ferretdb` or if it has buildmetadata. +func semVar(tag string) (major, minor, patch, prerelease string, err error) { match := semVerTag.FindStringSubmatch(tag) if match == nil || len(match) != semVerTag.NumSubexp()+1 { - return "", fmt.Errorf("unexpected tag syntax %q", tag) + return "", "", "", "", fmt.Errorf("unexpected tag syntax %q", tag) } - major := match[semVerTag.SubexpIndex("major")] - minor := match[semVerTag.SubexpIndex("minor")] - patch := match[semVerTag.SubexpIndex("patch")] - prerelease := match[semVerTag.SubexpIndex("prerelease")] + major = match[semVerTag.SubexpIndex("major")] + minor = match[semVerTag.SubexpIndex("minor")] + patch = match[semVerTag.SubexpIndex("patch")] + prerelease = match[semVerTag.SubexpIndex("prerelease")] buildmetadata := match[semVerTag.SubexpIndex("buildmetadata")] if prerelease == "" { - return "", fmt.Errorf("prerelease is empty") + return "", "", "", "", fmt.Errorf("prerelease is empty") } if !strings.Contains(prerelease, "ferretdb") { - return "", fmt.Errorf("prerelease %q should include `ferretdb`", prerelease) + return "", "", "", "", fmt.Errorf("prerelease %q should include `ferretdb`", prerelease) } if buildmetadata != "" { - return "", fmt.Errorf("buildmetadata %q is present", buildmetadata) + return "", "", "", "", fmt.Errorf("buildmetadata %q is present", buildmetadata) + } + + return +} + +// definePackagerVersionForTag returns valid Debian package version for tag. +// See [definePackageVersion]. +func definePackagerVersionForTag(tag string) (string, error) { + major, minor, patch, prerelease, err := semVar(tag) + if err != nil { + return "", err } res := fmt.Sprintf("%s.%s.%s-%s", major, minor, patch, prerelease) return disallowedVer.ReplaceAllString(res, "~"), nil } -// setResults sets action output parameters, summary, etc. -func setResults(action *githubactions.Action, res string) { +// setDebianVersionResults sets action output parameters, summary, etc. +func setDebianVersionResults(action *githubactions.Action, res string) { output := fmt.Sprintf("version: `%s`", res) action.AddStepSummary(output) diff --git a/ferretdb_packaging/defineversion/main_test.go b/ferretdb_packaging/defineversion/main_test.go index dcbba4524..71eea4877 100644 --- a/ferretdb_packaging/defineversion/main_test.go +++ b/ferretdb_packaging/defineversion/main_test.go @@ -191,7 +191,7 @@ func TestResults(t *testing.T) { version := "0.100.0~ferretdb" - setResults(action, version) + setDebianVersionResults(action, version) expected := "version: `0.100.0~ferretdb`\n" assert.Equal(t, expected, stdout.String(), "stdout does not match") @@ -230,3 +230,55 @@ requires = 'documentdb_core, pg_cron, tsm_system_rows, vector, postgis, rum'` require.Equal(t, "0.100.0", controlDefaultVersion) } + +func TestSemVar(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + tag string + major string + minor string + patch string + prerelease string + err string + }{ + "Valid": { + tag: "v1.100.0-ferretdb", + major: "1", + minor: "100", + patch: "0", + prerelease: "ferretdb", + }, + "SpecificVersion": { + tag: "v1.100.0-ferretdb-2.0.1", + major: "1", + minor: "100", + patch: "0", + prerelease: "ferretdb-2.0.1", + }, + "MissingV": { + tag: "0.100.0-ferretdb", + err: `unexpected tag syntax "0.100.0-ferretdb"`, + }, + "MissingFerretDB": { + tag: "v0.100.0", + err: "prerelease is empty", + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + major, minor, patch, prerelease, err := semVar(tc.tag) + + if tc.err != "" { + require.EqualError(t, err, tc.err) + return + } + + require.Equal(t, tc.major, major) + require.Equal(t, tc.minor, minor) + require.Equal(t, tc.patch, patch) + require.Equal(t, tc.prerelease, prerelease) + }) + } +} From 46b3f5a8294e5be9bb9a33cc214f011c76c24e5f Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Mon, 3 Mar 2025 17:51:33 +0400 Subject: [PATCH 15/59] Add and use `trust` label (#22) --- .github/settings.yml | 4 ++ .github/workflows/ferretdb_packages.yml | 80 ++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/.github/settings.yml b/.github/settings.yml index 5779efa2f..974cae3f9 100644 --- a/.github/settings.yml +++ b/.github/settings.yml @@ -32,3 +32,7 @@ labels: - name: packages color: "#9B022C" description: PRs that should build packages + + - name: trust + color: "#00FF00" + description: PRs that can access Actions secrets diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index 3ad1854aa..227512acc 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -1,7 +1,29 @@ --- +# This and other workflows that use `pull_request_target` event are dangerous +# and should be handled with a lot of care to avoid security problems. +# We use this event to give pull requests access to secrets with permissions to login into Docker registries. +# But rogue PR authors could try to steal our secrets. +# We prevent that with the following: +# +# * We require approval for PRs from first-time contributors. That's a built-in feature for all actions. +# * For workflows that checkout source code, +# we require the `trust` label to be assigned to PRs by FerretDB maintainers after reviewing changes. +# Only a few trusted people have permission to do that. +# * Thanks to the way `pull_request_target` trigger works, +# PR changes in the workflow itself are not run until they are merged. +# * We use short-lived automatic `GITHUB_TOKEN`s instead of a long-living personal access tokens (PAT) where possible. +# * Both `GITHUB_TOKEN`s and PATs have minimal permissions. +# * We publish Docker images from PRs as separate packages that should not be run by users. +# * We limit what third-party actions can be used. +# +# Relevant GitHub documentation is a bit scattered. The first article gives a good overview: +# * https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ +# * https://docs.github.com/en/actions/security-guides/automatic-token-authentication +# * https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions + name: Packages on: - pull_request: + pull_request_target: types: - unlabeled # if GitHub Actions stuck, add and remove "not ready" label to force rebuild - opened @@ -36,12 +58,14 @@ jobs: timeout-minutes: 40 if: > - github.event_name != 'pull_request' || + github.event_name != 'pull_request_target' || ( !contains(github.event.pull_request.labels.*.name, 'not ready') && contains(github.event.pull_request.labels.*.name, 'packages') ) + permissions: {} + strategy: fail-fast: false matrix: @@ -49,8 +73,31 @@ jobs: pg: [15, 16, 17] steps: + # TODO https://github.com/FerretDB/github-actions/issues/211 - name: Checkout code + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v4 + with: + fetch-depth: 0 # for `generate_extension_version.sh` to work + + # TODO https://github.com/FerretDB/github-actions/issues/211 + - name: Checkout pull request code + if: github.event_name == 'pull_request_target' uses: actions/checkout@v4 + with: + fetch-depth: 0 # for `generate_extension_version.sh` to work + ref: ${{ github.event.pull_request.head.sha }} + + - name: Fetch annotated tags + run: | + git fetch --tags --force + git status + + - name: Name branch + if: github.event_name == 'pull_request_target' + env: + BRANCH: ${{ github.head_ref }} # see https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + run: git checkout -b $BRANCH - name: Setup Go uses: FerretDB/github-actions/setup-go@main @@ -88,20 +135,47 @@ jobs: timeout-minutes: 40 if: > - github.event_name != 'pull_request' || + github.event_name != 'pull_request_target' || ( + contains(github.event.pull_request.labels.*.name, 'trust') && !contains(github.event.pull_request.labels.*.name, 'not ready') && contains(github.event.pull_request.labels.*.name, 'packages') ) + permissions: + packages: write + strategy: fail-fast: false matrix: pg: [15.12, 16.8, 17.4] steps: + # TODO https://github.com/FerretDB/github-actions/issues/211 - name: Checkout code + if: github.event_name != 'pull_request_target' uses: actions/checkout@v4 + with: + fetch-depth: 0 # for `generate_extension_version.sh` to work + + # TODO https://github.com/FerretDB/github-actions/issues/211 + - name: Checkout pull request code + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v4 + with: + fetch-depth: 0 # for `generate_extension_version.sh` to work + ref: ${{ github.event.pull_request.head.sha }} + + - name: Fetch annotated tags + run: | + git fetch --tags --force + git status + + - name: Name branch + if: github.event_name == 'pull_request_target' + env: + BRANCH: ${{ github.head_ref }} # see https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + run: git checkout -b $BRANCH - name: Setup Go uses: FerretDB/github-actions/setup-go@main From 99efe6b6a234dbe1f0d08623c14cfde01b81c7d2 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Mon, 3 Mar 2025 22:55:31 +0400 Subject: [PATCH 16/59] Update `defineversion` logic (#23) --- ferretdb_packaging/defineversion/debian.go | 124 +++++ .../defineversion/debian_test.go | 30 ++ .../{docker_tags.go => docker.go} | 109 ++-- .../defineversion/docker_tags_test.go | 488 ------------------ .../defineversion/docker_test.go | 87 ++++ ferretdb_packaging/defineversion/main.go | 249 +++------ ferretdb_packaging/defineversion/main_test.go | 402 ++++++++------- 7 files changed, 581 insertions(+), 908 deletions(-) create mode 100644 ferretdb_packaging/defineversion/debian.go create mode 100644 ferretdb_packaging/defineversion/debian_test.go rename ferretdb_packaging/defineversion/{docker_tags.go => docker.go} (57%) delete mode 100644 ferretdb_packaging/defineversion/docker_tags_test.go create mode 100644 ferretdb_packaging/defineversion/docker_test.go diff --git a/ferretdb_packaging/defineversion/debian.go b/ferretdb_packaging/defineversion/debian.go new file mode 100644 index 000000000..0bde19a79 --- /dev/null +++ b/ferretdb_packaging/defineversion/debian.go @@ -0,0 +1,124 @@ +package main + +import ( + "fmt" + "os" + "regexp" + "strings" + + "github.com/sethvargo/go-githubactions" +) + +// controlDefaultVer matches major, minor and "patch" from `default_version` field in control file, +// see pg_documentdb_core/documentdb_core.control. +var controlDefaultVer = regexp.MustCompile(`(?m)^default_version = '(?P[0-9]+)\.(?P[0-9]+)-(?P[0-9]+)'$`) + +// disallowedDebian matches disallowed characters of Debian `upstream_version` when used without `debian_revision`. +// See https://www.debian.org/doc/debian-policy/ch-controlfields.html#version. +var disallowedDebian = regexp.MustCompile(`[^A-Za-z0-9\.+~]`) + +// getControlDefaultVersion returns the default_version field from the control file +// in SemVer format (0.100-0 -> 0.100.0). +func getControlDefaultVersion(f string) (string, error) { + b, err := os.ReadFile(f) + if err != nil { + return "", err + } + + match := controlDefaultVer.FindSubmatch(b) + if match == nil || len(match) != controlDefaultVer.NumSubexp()+1 { + return "", fmt.Errorf("control file did not find default_version: %s", f) + } + + major := match[controlDefaultVer.SubexpIndex("major")] + minor := match[controlDefaultVer.SubexpIndex("minor")] + patch := match[controlDefaultVer.SubexpIndex("patch")] + + return fmt.Sprintf("%s.%s.%s", major, minor, patch), nil +} + +// defineDebianVersion returns valid Debian package version, +// based on `default_version` in the control file and environment variables set by GitHub Actions. +// +// See https://www.debian.org/doc/debian-policy/ch-controlfields.html#version. +// We use `upstream_version` only. +// For that reason, we can't use `-`, so we replace it with `~`. +func defineDebianVersion(controlDefaultVersion, pgVersion string, getenv githubactions.GetenvFunc) (string, error) { + var res string + var err error + + switch event := getenv("GITHUB_EVENT_NAME"); event { + case "pull_request", "pull_request_target": + branch := strings.ToLower(getenv("GITHUB_HEAD_REF")) + res = defineDebianVersionForPR(controlDefaultVersion, branch) + + case "push", "schedule", "workflow_run": + refName := strings.ToLower(getenv("GITHUB_REF_NAME")) + + switch refType := strings.ToLower(getenv("GITHUB_REF_TYPE")); refType { + case "branch": + res, err = defineDebianVersionForBranch(controlDefaultVersion, refName) + + case "tag": + res, err = defineDebianVersionForTag(refName) + + default: + err = fmt.Errorf("unhandled ref type %q for event %q", refType, event) + } + + default: + err = fmt.Errorf("unhandled event type %q", event) + } + + if err != nil { + return "", err + } + + if res == "" { + return "", fmt.Errorf("both packageVersion and err are nil") + } + + return res, nil +} + +// defineDebianVersionForPR returns valid Debian package version for PR. +// See [defineDebianVersion]. +func defineDebianVersionForPR(controlDefaultVersion, branch string) string { + // for branches like "dependabot/submodules/XXX" + parts := strings.Split(branch, "/") + branch = parts[len(parts)-1] + res := fmt.Sprintf("%s-pr-%s", controlDefaultVersion, branch) + + return disallowedDebian.ReplaceAllString(res, "~") +} + +// defineDebianVersionForBranch returns valid Debian package version for branch. +// See [defineDebianVersion]. +func defineDebianVersionForBranch(controlDefaultVersion, branch string) (string, error) { + switch branch { + case "ferretdb": + return fmt.Sprintf("%s~branch~%s", controlDefaultVersion, branch), nil + default: + return "", fmt.Errorf("unhandled branch %q", branch) + } +} + +// defineDebianVersionForTag returns valid Debian package version for tag. +// See [defineDebianVersion]. +func defineDebianVersionForTag(tag string) (string, error) { + major, minor, patch, prerelease, err := parseGitTag(tag) + if err != nil { + return "", err + } + + res := fmt.Sprintf("%d.%d.%d-%s", major, minor, patch, prerelease) + return disallowedDebian.ReplaceAllString(res, "~"), nil +} + +// debianSummary sets action summary. +func debianSummary(action *githubactions.Action, version string) { + output := fmt.Sprintf("Debian package version (`upstream_version` only): `%s`", version) + + action.AddStepSummary(output) + action.Infof("%s", output) +} diff --git a/ferretdb_packaging/defineversion/debian_test.go b/ferretdb_packaging/defineversion/debian_test.go new file mode 100644 index 000000000..f8f2cd855 --- /dev/null +++ b/ferretdb_packaging/defineversion/debian_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "io" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestReadControlDefaultVersion(t *testing.T) { + controlF, err := os.CreateTemp(t.TempDir(), "test.control") + require.NoError(t, err) + + defer controlF.Close() //nolint:errcheck // temporary file for testing + + buf := `comment = 'API surface for DocumentDB for PostgreSQL' +default_version = '0.100-0' +module_pathname = '$libdir/pg_documentdb' +relocatable = false +superuser = true +requires = 'documentdb_core, pg_cron, tsm_system_rows, vector, postgis, rum'` + _, err = io.WriteString(controlF, buf) + require.NoError(t, err) + + controlDefaultVersion, err := getControlDefaultVersion(controlF.Name()) + require.NoError(t, err) + + require.Equal(t, "0.100.0", controlDefaultVersion) +} diff --git a/ferretdb_packaging/defineversion/docker_tags.go b/ferretdb_packaging/defineversion/docker.go similarity index 57% rename from ferretdb_packaging/defineversion/docker_tags.go rename to ferretdb_packaging/defineversion/docker.go index 9cdddbc5d..ffb9db561 100644 --- a/ferretdb_packaging/defineversion/docker_tags.go +++ b/ferretdb_packaging/defineversion/docker.go @@ -1,22 +1,7 @@ -// Copyright 2021 FerretDB Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package main import ( "fmt" - "regexp" "slices" "strings" "text/tabwriter" @@ -30,11 +15,8 @@ type images struct { productionImages []string } -// pgVer is the version of PostgreSQL. -var pgVer = regexp.MustCompile(`^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)$`) - -// defineDockerTags extracts Docker image names and tags from the environment variables defined by GitHub Actions. -func defineDockerTags(getenv githubactions.GetenvFunc) (*images, error) { +// defineDockerVersion extracts Docker image names and tags from the environment variables defined by GitHub Actions. +func defineDockerVersion(controlDefaultVersion, pgVersion string, getenv githubactions.GetenvFunc) (*images, error) { repo := getenv("GITHUB_REPOSITORY") // to support GitHub forks @@ -51,40 +33,28 @@ func defineDockerTags(getenv githubactions.GetenvFunc) (*images, error) { switch event := getenv("GITHUB_EVENT_NAME"); event { case "pull_request", "pull_request_target": branch := strings.ToLower(getenv("GITHUB_HEAD_REF")) - res = defineForPR(owner, repo, branch) + res = defineDockerVersionForPR(owner, repo, branch) case "push", "schedule", "workflow_run": refName := strings.ToLower(getenv("GITHUB_REF_NAME")) switch refType := strings.ToLower(getenv("GITHUB_REF_TYPE")); refType { case "branch": - res, err = defineForBranch(owner, repo, refName) + res, err = defineDockerVersionForBranch(owner, repo, refName) case "tag": - var major, minor, patch, prerelease string - if major, minor, patch, prerelease, err = semVar(refName); err != nil { + var major, minor, patch int + var prerelease string + if major, minor, patch, prerelease, err = parseGitTag(refName); err != nil { return nil, err } - pgVersion := getenv("INPUT_PG_VERSION") - pgMatch := pgVer.FindStringSubmatch(pgVersion) - if pgMatch == nil || len(pgMatch) != pgVer.NumSubexp()+1 { - return nil, fmt.Errorf("unexpected PostgreSQL version %q", pgVersion) - } - - pgMajor := pgMatch[pgVer.SubexpIndex("major")] - pgMinor := pgMatch[pgVer.SubexpIndex("minor")] - tags := []string{ - fmt.Sprintf("%s-%s.%s.%s-%s", pgMajor, major, minor, patch, prerelease), + fmt.Sprintf("%s-%d.%d.%d-%s", pgVersion, major, minor, patch, prerelease), "latest", } - if pgMinor != "" { - tags = append(tags, fmt.Sprintf("%s.%s-%s.%s.%s-%s", pgMajor, pgMinor, major, minor, patch, prerelease)) - } - - res = defineForTag(owner, repo, tags) + res = defineDockerVersionForTag(owner, repo, tags) default: err = fmt.Errorf("unhandled ref type %q for event %q", refType, event) @@ -108,8 +78,8 @@ func defineDockerTags(getenv githubactions.GetenvFunc) (*images, error) { return res, nil } -// defineForPR defines Docker image names and tags for pull requests. -func defineForPR(owner, repo, branch string) *images { +// defineDockerVersionForPR defines Docker image names and tags for pull requests. +func defineDockerVersionForPR(owner, repo, branch string) *images { // for branches like "dependabot/submodules/XXX" parts := strings.Split(branch, "/") branch = parts[len(parts)-1] @@ -125,8 +95,8 @@ func defineForPR(owner, repo, branch string) *images { return res } -// defineForBranch defines Docker image names and tags for branch builds. -func defineForBranch(owner, repo, branch string) (*images, error) { +// defineDockerVersionForBranch defines Docker image names and tags for branch builds. +func defineDockerVersionForBranch(owner, repo, branch string) (*images, error) { if branch != "ferretdb" { return nil, fmt.Errorf("unhandled branch %q", branch) } @@ -147,14 +117,14 @@ func defineForBranch(owner, repo, branch string) (*images, error) { return res, nil } - //res.developmentImages = append(res.developmentImages, "quay.io/ferretdb/postgres-documentdb-dev:ferretdb") - //res.developmentImages = append(res.developmentImages, "ferretdb/postgres-documentdb-dev:ferretdb") + // res.developmentImages = append(res.developmentImages, "quay.io/ferretdb/postgres-documentdb-dev:ferretdb") + // res.developmentImages = append(res.developmentImages, "ferretdb/postgres-documentdb-dev:ferretdb") return res, nil } -// defineForTag defines Docker image names and tags for prerelease tag builds. -func defineForTag(owner, repo string, tags []string) *images { +// defineDockerVersionForTag defines Docker image names and tags for prerelease tag builds. +func defineDockerVersionForTag(owner, repo string, tags []string) *images { res := new(images) for _, t := range tags { @@ -183,20 +153,35 @@ func defineForTag(owner, repo string, tags []string) *images { return res } -// setDockerTagsResults sets action output parameters, summary, etc. -func setDockerTagsResults(action *githubactions.Action, res *images) { +// dockerImageURL returns HTML page URL for the given image name and tag. +func dockerImageURL(name string) string { + switch { + case strings.HasPrefix(name, "ghcr.io/"): + return fmt.Sprintf("https://%s", name) + case strings.HasPrefix(name, "quay.io/"): + return fmt.Sprintf("https://%s", name) + } + + name, _, _ = strings.Cut(name, ":") + + // there is no easy way to get Docker Hub URL for the given tag + return fmt.Sprintf("https://hub.docker.com/r/%s/tags", name) +} + +// dockerSummary sets action summary. +func dockerSummary(action *githubactions.Action, version *images) { var buf strings.Builder w := tabwriter.NewWriter(&buf, 1, 1, 1, ' ', tabwriter.Debug) fmt.Fprintf(w, "\tType\tImage\t\n") fmt.Fprintf(w, "\t----\t-----\t\n") - for _, image := range res.developmentImages { - u := imageURL(image) + for _, image := range version.developmentImages { + u := dockerImageURL(image) _, _ = fmt.Fprintf(w, "\tDevelopment\t[`%s`](%s)\t\n", image, u) } - for _, image := range res.productionImages { - u := imageURL(image) + for _, image := range version.productionImages { + u := dockerImageURL(image) _, _ = fmt.Fprintf(w, "\tProduction\t[`%s`](%s)\t\n", image, u) } @@ -204,22 +189,4 @@ func setDockerTagsResults(action *githubactions.Action, res *images) { action.AddStepSummary(buf.String()) action.Infof("%s", buf.String()) - - action.SetOutput("development_images", strings.Join(res.developmentImages, ",")) - action.SetOutput("production_images", strings.Join(res.productionImages, ",")) -} - -// imageURL returns HTML page URL for the given image name and tag. -func imageURL(name string) string { - switch { - case strings.HasPrefix(name, "ghcr.io/"): - return fmt.Sprintf("https://%s", name) - case strings.HasPrefix(name, "quay.io/"): - return fmt.Sprintf("https://%s", name) - } - - name, _, _ = strings.Cut(name, ":") - - // there is no easy way to get Docker Hub URL for the given tag - return fmt.Sprintf("https://hub.docker.com/r/%s/tags", name) } diff --git a/ferretdb_packaging/defineversion/docker_tags_test.go b/ferretdb_packaging/defineversion/docker_tags_test.go deleted file mode 100644 index d7eecae7b..000000000 --- a/ferretdb_packaging/defineversion/docker_tags_test.go +++ /dev/null @@ -1,488 +0,0 @@ -// Copyright 2021 FerretDB Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bytes" - "github.com/sethvargo/go-githubactions" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "io" - "os" - "strings" - "testing" -) - -func TestDefineDockerTags(t *testing.T) { - for name, tc := range map[string]struct { - env map[string]string - expected *images - }{ - "pull_request": { - env: map[string]string{ - "GITHUB_BASE_REF": "ferretdb", - "GITHUB_EVENT_NAME": "pull_request", - "GITHUB_HEAD_REF": "docker-tag", - "GITHUB_REF_NAME": "1/merge", - "GITHUB_REF_TYPE": "branch", - "GITHUB_REPOSITORY": "FerretDB/documentdb", - "INPUT_PG_VERSION": "16", - }, - expected: &images{ - developmentImages: []string{ - "ghcr.io/ferretdb/postgres-documentdb-dev:pr-docker-tag", - }, - }, - }, - "pull_request-other": { - env: map[string]string{ - "GITHUB_BASE_REF": "ferretdb", - "GITHUB_EVENT_NAME": "pull_request", - "GITHUB_HEAD_REF": "docker-tag", - "GITHUB_REF_NAME": "1/merge", - "GITHUB_REF_TYPE": "branch", - "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", - "INPUT_PG_VERSION": "16", - }, - expected: &images{ - developmentImages: []string{ - "ghcr.io/otherorg/postgres-otherrepo-dev:pr-docker-tag", - }, - }, - }, - - "pull_request_target": { - env: map[string]string{ - "GITHUB_BASE_REF": "ferretdb", - "GITHUB_EVENT_NAME": "pull_request_target", - "GITHUB_HEAD_REF": "docker-tag", - "GITHUB_REF_NAME": "ferretdb", - "GITHUB_REF_TYPE": "branch", - "GITHUB_REPOSITORY": "FerretDB/documentdb", - "INPUT_PG_VERSION": "16", - }, - expected: &images{ - developmentImages: []string{ - "ghcr.io/ferretdb/postgres-documentdb-dev:pr-docker-tag", - }, - }, - }, - "pull_request_target-other": { - env: map[string]string{ - "GITHUB_BASE_REF": "ferretdb", - "GITHUB_EVENT_NAME": "pull_request_target", - "GITHUB_HEAD_REF": "docker-tag", - "GITHUB_REF_NAME": "ferretdb", - "GITHUB_REF_TYPE": "branch", - "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", - "INPUT_PG_VERSION": "16", - }, - expected: &images{ - developmentImages: []string{ - "ghcr.io/otherorg/postgres-otherrepo-dev:pr-docker-tag", - }, - }, - }, - - "push/ferretdb": { - env: map[string]string{ - "GITHUB_BASE_REF": "", - "GITHUB_EVENT_NAME": "push", - "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "ferretdb", - "GITHUB_REF_TYPE": "branch", - "GITHUB_REPOSITORY": "FerretDB/documentdb", - "INPUT_PG_VERSION": "16", - }, - expected: &images{ - developmentImages: []string{ - //"ferretdb/postgres-documentdb-dev:ferretdb", - "ghcr.io/ferretdb/postgres-documentdb-dev:ferretdb", - //"quay.io/ferretdb/postgres-documentdb-dev:ferretdb", - }, - }, - }, - "push/ferretdb-other": { - env: map[string]string{ - "GITHUB_BASE_REF": "", - "GITHUB_EVENT_NAME": "push", - "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "ferretdb", - "GITHUB_REF_TYPE": "branch", - "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", - "INPUT_PG_VERSION": "16", - }, - expected: &images{ - developmentImages: []string{ - "ghcr.io/otherorg/postgres-otherrepo-dev:ferretdb", - }, - }, - }, - - "push/main": { - env: map[string]string{ - "GITHUB_BASE_REF": "", - "GITHUB_EVENT_NAME": "push", - "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "main", - "GITHUB_REF_TYPE": "branch", - "GITHUB_REPOSITORY": "FerretDB/documentdb", - "INPUT_PG_VERSION": "16", - }, - }, - "push/main-other": { - env: map[string]string{ - "GITHUB_BASE_REF": "", - "GITHUB_EVENT_NAME": "push", - "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "main", - "GITHUB_REF_TYPE": "branch", - "GITHUB_REPOSITORY": "FerretDB/documentdb", - "INPUT_PG_VERSION": "16", - }, - }, - - "push/tag/release": { - env: map[string]string{ - "GITHUB_BASE_REF": "", - "GITHUB_EVENT_NAME": "push", - "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "v0.102.0-ferretdb", - "GITHUB_REF_TYPE": "tag", - "GITHUB_REPOSITORY": "FerretDB/documentdb", - "INPUT_PG_VERSION": "16.7", - }, - expected: &images{ - developmentImages: []string{ - //"ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb", - //"ferretdb/postgres-documentdb-dev:16.7-0.102.0-ferretdb", - //"ferretdb/postgres-documentdb-dev:latest", - "ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb", - "ghcr.io/ferretdb/postgres-documentdb-dev:16.7-0.102.0-ferretdb", - "ghcr.io/ferretdb/postgres-documentdb-dev:latest", - //"quay.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb", - //"quay.io/ferretdb/postgres-documentdb-dev:16.7-0.102.0-ferretdb", - //"quay.io/ferretdb/postgres-documentdb-dev:latest", - }, - productionImages: []string{ - //"ferretdb/postgres-documentdb:16-0.102.0-ferretdb", - //"ferretdb/postgres-documentdb:16.7-0.102.0-ferretdb", - //"ferretdb/postgres-documentdb:latest", - "ghcr.io/ferretdb/postgres-documentdb:16-0.102.0-ferretdb", - "ghcr.io/ferretdb/postgres-documentdb:16.7-0.102.0-ferretdb", - "ghcr.io/ferretdb/postgres-documentdb:latest", - //"quay.io/ferretdb/postgres-documentdb:16-0.102.0-ferretdb", - //"quay.io/ferretdb/postgres-documentdb:16.7-0.102.0-ferretdb", - //"quay.io/ferretdb/postgres-documentdb:latest", - }, - }, - }, - "push/tag/release-other": { - env: map[string]string{ - "GITHUB_BASE_REF": "", - "GITHUB_EVENT_NAME": "push", - "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "v0.102.0-ferretdb", - "GITHUB_REF_TYPE": "tag", - "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", - "INPUT_PG_VERSION": "16.7", - }, - expected: &images{ - developmentImages: []string{ - "ghcr.io/otherorg/postgres-otherrepo-dev:16-0.102.0-ferretdb", - "ghcr.io/otherorg/postgres-otherrepo-dev:16.7-0.102.0-ferretdb", - "ghcr.io/otherorg/postgres-otherrepo-dev:latest", - }, - productionImages: []string{ - "ghcr.io/otherorg/postgres-otherrepo:16-0.102.0-ferretdb", - "ghcr.io/otherorg/postgres-otherrepo:16.7-0.102.0-ferretdb", - "ghcr.io/otherorg/postgres-otherrepo:latest", - }, - }, - }, - - "push/tag/release-rc": { - env: map[string]string{ - "GITHUB_BASE_REF": "", - "GITHUB_EVENT_NAME": "push", - "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "v0.102.0-ferretdb-2.0.0-rc2", - "GITHUB_REF_TYPE": "tag", - "GITHUB_REPOSITORY": "FerretDB/documentdb", - "INPUT_PG_VERSION": "16.7", // set major and minor version - }, - expected: &images{ - developmentImages: []string{ - //"ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb-2.0.0-rc2", - //"ferretdb/postgres-documentdb-dev:16.7-0.102.0-ferretdb-2.0.0-rc2", - //"ferretdb/postgres-documentdb-dev:latest", - "ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb-2.0.0-rc2", - "ghcr.io/ferretdb/postgres-documentdb-dev:16.7-0.102.0-ferretdb-2.0.0-rc2", - "ghcr.io/ferretdb/postgres-documentdb-dev:latest", - //"quay.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb-2.0.0-rc2", - //"quay.io/ferretdb/postgres-documentdb-dev:16.7-0.102.0-ferretdb-2.0.0-rc2", - //"quay.io/ferretdb/postgres-documentdb-dev:latest", - }, - productionImages: []string{ - //"ferretdb/postgres-documentdb:16-0.102.0-ferretdb-2.0.0-rc2", - //"ferretdb/postgres-documentdb:16.7-0.102.0-ferretdb-2.0.0-rc2", - //"ferretdb/postgres-documentdb:latest", - "ghcr.io/ferretdb/postgres-documentdb:16-0.102.0-ferretdb-2.0.0-rc2", - "ghcr.io/ferretdb/postgres-documentdb:16.7-0.102.0-ferretdb-2.0.0-rc2", - "ghcr.io/ferretdb/postgres-documentdb:latest", - //"quay.io/ferretdb/postgres-documentdb:16-0.102.0-ferretdb-2.0.0-rc2", - //"quay.io/ferretdb/postgres-documentdb:16.7-0.102.0-ferretdb-2.0.0-rc2", - //"quay.io/ferretdb/postgres-documentdb:latest", - }, - }, - }, - "push/tag/release-rc-other": { - env: map[string]string{ - "GITHUB_BASE_REF": "", - "GITHUB_EVENT_NAME": "push", - "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "v0.102.0-ferretdb-2.0.0-rc2", - "GITHUB_REF_TYPE": "tag", - "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", - "INPUT_PG_VERSION": "16.7", // set major and minor version - }, - expected: &images{ - developmentImages: []string{ - "ghcr.io/otherorg/postgres-otherrepo-dev:16-0.102.0-ferretdb-2.0.0-rc2", - "ghcr.io/otherorg/postgres-otherrepo-dev:16.7-0.102.0-ferretdb-2.0.0-rc2", - "ghcr.io/otherorg/postgres-otherrepo-dev:latest", - }, - productionImages: []string{ - "ghcr.io/otherorg/postgres-otherrepo:16-0.102.0-ferretdb-2.0.0-rc2", - "ghcr.io/otherorg/postgres-otherrepo:16.7-0.102.0-ferretdb-2.0.0-rc2", - "ghcr.io/otherorg/postgres-otherrepo:latest", - }, - }, - }, - - "push/tag/release// missing minor": { - env: map[string]string{ - "GITHUB_BASE_REF": "", - "GITHUB_EVENT_NAME": "push", - "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "v0.102.0-ferretdb-2.0.0-rc2", - "GITHUB_REF_TYPE": "tag", - "GITHUB_REPOSITORY": "FerretDB/documentdb", - "INPUT_PG_VERSION": "16", // missing minor - }, - }, - "push/tag/release-missing-minor-other": { - env: map[string]string{ - "GITHUB_BASE_REF": "", - "GITHUB_EVENT_NAME": "push", - "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "v0.102.0-ferretdb-2.0.0-rc2", - "GITHUB_REF_TYPE": "tag", - "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", - "INPUT_PG_VERSION": "16", // missing minor - }, - }, - - "push/tag/wrong": { - env: map[string]string{ - "GITHUB_BASE_REF": "", - "GITHUB_EVENT_NAME": "push", - "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "0.102.0-ferretdb-2.0.0-rc2", // no leading v - "GITHUB_REF_TYPE": "tag", - "GITHUB_REPOSITORY": "FerretDB/documentdb", - "INPUT_PG_VERSION": "16", - }, - }, - "push/tag/wrong-other": { - env: map[string]string{ - "GITHUB_BASE_REF": "", - "GITHUB_EVENT_NAME": "push", - "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "0.102.0-ferretdb-2.0.0-rc2", // no leading v - "GITHUB_REF_TYPE": "tag", - "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", - "INPUT_PG_VERSION": "16", - }, - }, - - "schedule": { - env: map[string]string{ - "GITHUB_BASE_REF": "", - "GITHUB_EVENT_NAME": "schedule", - "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "ferretdb", - "GITHUB_REF_TYPE": "branch", - "GITHUB_REPOSITORY": "FerretDB/documentdb", - "INPUT_PG_VERSION": "16", - }, - expected: &images{ - developmentImages: []string{ - //"ferretdb/postgres-documentdb-dev:ferretdb", - "ghcr.io/ferretdb/postgres-documentdb-dev:ferretdb", - //"quay.io/ferretdb/postgres-documentdb-dev:ferretdb", - }, - }, - }, - "schedule-other": { - env: map[string]string{ - "GITHUB_BASE_REF": "", - "GITHUB_EVENT_NAME": "schedule", - "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "ferretdb", - "GITHUB_REF_TYPE": "branch", - "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", - "INPUT_PG_VERSION": "16", - }, - expected: &images{ - developmentImages: []string{ - "ghcr.io/otherorg/postgres-otherrepo-dev:ferretdb", - }, - }, - }, - - "workflow_run": { - env: map[string]string{ - "GITHUB_BASE_REF": "", - "GITHUB_EVENT_NAME": "workflow_run", - "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "ferretdb", - "GITHUB_REF_TYPE": "branch", - "GITHUB_REPOSITORY": "FerretDB/documentdb", - "INPUT_PG_VERSION": "16", - }, - expected: &images{ - developmentImages: []string{ - //"ferretdb/postgres-documentdb-dev:ferretdb", - "ghcr.io/ferretdb/postgres-documentdb-dev:ferretdb", - //"quay.io/ferretdb/postgres-documentdb-dev:ferretdb", - }, - }, - }, - "workflow_run-other": { - env: map[string]string{ - "GITHUB_BASE_REF": "", - "GITHUB_EVENT_NAME": "workflow_run", - "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "ferretdb", - "GITHUB_REF_TYPE": "branch", - "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", - "INPUT_PG_VERSION": "16", - }, - expected: &images{ - developmentImages: []string{ - "ghcr.io/otherorg/postgres-otherrepo-dev:ferretdb", - }, - }, - }, - } { - t.Run(name, func(t *testing.T) { - actual, err := defineDockerTags(getEnvFunc(t, tc.env)) - if tc.expected == nil { - require.Error(t, err) - return - } - - require.NoError(t, err) - assert.Equal(t, tc.expected, actual) - }) - } -} - -func TestImageURL(t *testing.T) { - // expected URLs should work - assert.Equal( - t, - "https://ghcr.io/ferretdb/postgres-documentdb-dev:pr-docker-tag", - imageURL("ghcr.io/ferretdb/postgres-documentdb-dev:pr-docker-tag"), - ) - assert.Equal( - t, - "https://quay.io/ferretdb/postgres-documentdb-dev:pr-docker-tag", - imageURL("quay.io/ferretdb/postgres-documentdb-dev:pr-docker-tag"), - ) - assert.Equal( - t, - "https://hub.docker.com/r/ferretdb/postgres-documentdb-dev/tags", - imageURL("ferretdb/postgres-documentdb-dev:pr-docker-tag"), - ) -} - -func TestDockerTagsResults(t *testing.T) { - dir := t.TempDir() - - summaryF, err := os.CreateTemp(dir, "summary") - require.NoError(t, err) - defer summaryF.Close() - - outputF, err := os.CreateTemp(dir, "output") - require.NoError(t, err) - defer outputF.Close() - - var stdout bytes.Buffer - getenv := getEnvFunc(t, map[string]string{ - "GITHUB_STEP_SUMMARY": summaryF.Name(), - "GITHUB_OUTPUT": outputF.Name(), - }) - action := githubactions.New(githubactions.WithGetenv(getenv), githubactions.WithWriter(&stdout)) - - result := &images{ - developmentImages: []string{ - "ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb", - "ghcr.io/ferretdb/postgres-documentdb-dev:latest", - }, - productionImages: []string{ - "quay.io/ferretdb/postgres-documentdb:latest", - }, - } - - setDockerTagsResults(action, result) - - expectedStdout := strings.ReplaceAll(` - |Type |Image | - |---- |----- | - |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb) | - |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:latest'](https://ghcr.io/ferretdb/postgres-documentdb-dev:latest) | - |Production |['quay.io/ferretdb/postgres-documentdb:latest'](https://quay.io/ferretdb/postgres-documentdb:latest) | - -`[1:], "'", "`", - ) - assert.Equal(t, expectedStdout, stdout.String(), "stdout does not match") - - expectedSummary := strings.ReplaceAll(` - |Type |Image | - |---- |----- | - |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb) | - |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:latest'](https://ghcr.io/ferretdb/postgres-documentdb-dev:latest) | - |Production |['quay.io/ferretdb/postgres-documentdb:latest'](https://quay.io/ferretdb/postgres-documentdb:latest) | - -`[1:], "'", "`", - ) - b, err := io.ReadAll(summaryF) - require.NoError(t, err) - assert.Equal(t, expectedSummary, string(b), "summary does not match") - - expectedOutput := ` -development_images<<_GitHubActionsFileCommandDelimeter_ -ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb,ghcr.io/ferretdb/postgres-documentdb-dev:latest -_GitHubActionsFileCommandDelimeter_ -production_images<<_GitHubActionsFileCommandDelimeter_ -quay.io/ferretdb/postgres-documentdb:latest -_GitHubActionsFileCommandDelimeter_ -`[1:] - b, err = io.ReadAll(outputF) - require.NoError(t, err) - assert.Equal(t, expectedOutput, string(b), "output parameters does not match") -} diff --git a/ferretdb_packaging/defineversion/docker_test.go b/ferretdb_packaging/defineversion/docker_test.go new file mode 100644 index 000000000..e5b6e7ea8 --- /dev/null +++ b/ferretdb_packaging/defineversion/docker_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "bytes" + "io" + "os" + "strings" + "testing" + + "github.com/sethvargo/go-githubactions" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestImageURL(t *testing.T) { + // expected URLs should work + assert.Equal( + t, + "https://ghcr.io/ferretdb/postgres-documentdb-dev:pr-docker-tag", + dockerImageURL("ghcr.io/ferretdb/postgres-documentdb-dev:pr-docker-tag"), + ) + assert.Equal( + t, + "https://quay.io/ferretdb/postgres-documentdb-dev:pr-docker-tag", + dockerImageURL("quay.io/ferretdb/postgres-documentdb-dev:pr-docker-tag"), + ) + assert.Equal( + t, + "https://hub.docker.com/r/ferretdb/postgres-documentdb-dev/tags", + dockerImageURL("ferretdb/postgres-documentdb-dev:pr-docker-tag"), + ) +} + +func TestDockerSummary(t *testing.T) { + dir := t.TempDir() + + summaryF, err := os.CreateTemp(dir, "summary") + require.NoError(t, err) + defer summaryF.Close() + + outputF, err := os.CreateTemp(dir, "output") + require.NoError(t, err) + defer outputF.Close() + + var stdout bytes.Buffer + getenv := getEnvFunc(t, map[string]string{ + "GITHUB_STEP_SUMMARY": summaryF.Name(), + "GITHUB_OUTPUT": outputF.Name(), + }) + action := githubactions.New(githubactions.WithGetenv(getenv), githubactions.WithWriter(&stdout)) + + result := &images{ + developmentImages: []string{ + "ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:latest", + }, + productionImages: []string{ + "quay.io/ferretdb/postgres-documentdb:latest", + }, + } + + dockerSummary(action, result) + + expectedStdout := strings.ReplaceAll(` + |Type |Image | + |---- |----- | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb) | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:latest'](https://ghcr.io/ferretdb/postgres-documentdb-dev:latest) | + |Production |['quay.io/ferretdb/postgres-documentdb:latest'](https://quay.io/ferretdb/postgres-documentdb:latest) | + +`[1:], "'", "`", + ) + assert.Equal(t, expectedStdout, stdout.String(), "stdout does not match") + + expectedSummary := strings.ReplaceAll(` + |Type |Image | + |---- |----- | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb) | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:latest'](https://ghcr.io/ferretdb/postgres-documentdb-dev:latest) | + |Production |['quay.io/ferretdb/postgres-documentdb:latest'](https://quay.io/ferretdb/postgres-documentdb:latest) | + +`[1:], "'", "`", + ) + b, err := io.ReadAll(summaryF) + require.NoError(t, err) + assert.Equal(t, expectedSummary, string(b), "summary does not match") +} diff --git a/ferretdb_packaging/defineversion/main.go b/ferretdb_packaging/defineversion/main.go index d3976c370..c131d9e34 100644 --- a/ferretdb_packaging/defineversion/main.go +++ b/ferretdb_packaging/defineversion/main.go @@ -1,18 +1,4 @@ -// Copyright 2021 FerretDB Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package main defines debian version number of DocumentDB for CI builds. +// Package main defines version numbers for DocumentDB. package main import ( @@ -21,66 +7,52 @@ import ( "os" "regexp" "slices" + "strconv" "strings" "github.com/sethvargo/go-githubactions" ) -func main() { - commandF := flag.String("command", "", "command to run, possible values: [deb-version, docker-tags]") - - controlFileF := flag.String("control-file", "../pg_documentdb/documentdb.control", "pg_documentdb/documentdb.control file path") - - flag.Parse() - - action := githubactions.New() - - debugEnv(action) +// semVerTag is a https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string, +// but with a leading `v`. +var semVerTag = regexp.MustCompile(`^v(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`) - if *commandF == "" { - action.Fatalf("-command flag is empty.") +// parseGitTag parses git tag in specific format and returns SemVer components. +// +// Expected format is v0.100.0-ferretdb-2.0.0-rc.2, +// where v0.100.0 is a DocumentDB version (0.100-0 -> 0.100.0), +// and ferretdb-2.0.0-rc.2 is a compatible FerretDB version. +func parseGitTag(tag string) (major, minor, patch int, prerelease string, err error) { + match := semVerTag.FindStringSubmatch(tag) + if match == nil || len(match) != semVerTag.NumSubexp()+1 { + err = fmt.Errorf("unexpected git tag format %q", tag) + return } - switch *commandF { - case "deb-version": - if *controlFileF == "" { - action.Fatalf("%s", "-control-file flag is empty.") - } - - controlDefaultVersion, err := getControlDefaultVersion(*controlFileF) - if err != nil { - action.Fatalf("%s", err) - } - - packageVersion, err := definePackageVersion(controlDefaultVersion, action.Getenv) - if err != nil { - action.Fatalf("%s", err) - } - - setDebianVersionResults(action, packageVersion) - case "docker-tags": - res, err := defineDockerTags(action.Getenv) - if err != nil { - action.Fatalf("%s", err) - } - - setDockerTagsResults(action, res) - default: - action.Fatalf("unhandled command %q", *commandF) + if major, err = strconv.Atoi(match[semVerTag.SubexpIndex("major")]); err != nil { + return } -} + if minor, err = strconv.Atoi(match[semVerTag.SubexpIndex("minor")]); err != nil { + return + } + if patch, err = strconv.Atoi(match[semVerTag.SubexpIndex("patch")]); err != nil { + return + } + prerelease = match[semVerTag.SubexpIndex("prerelease")] + buildmetadata := match[semVerTag.SubexpIndex("buildmetadata")] -// controlDefaultVer matches major, minor and "patch" from default_version field in control file, -// see pg_documentdb_core/documentdb_core.control. -var controlDefaultVer = regexp.MustCompile(`(?m)^default_version = '(?P[0-9]+)\.(?P[0-9]+)-(?P[0-9]+)'$`) + if !strings.HasPrefix(prerelease, "ferretdb-") { + err = fmt.Errorf("prerelease %q should start with 'ferretdb-'", prerelease) + return + } -// semVerTag is a https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string, -// but with a leading `v`. -var semVerTag = regexp.MustCompile(`^v(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`) + if buildmetadata != "" { + err = fmt.Errorf("buildmetadata %q is present", buildmetadata) + return + } -// disallowedVer matches disallowed characters of Debian `upstream_version` when used without `debian_revision`. -// See https://www.debian.org/doc/debian-policy/ch-controlfields.html#version. -var disallowedVer = regexp.MustCompile(`[^A-Za-z0-9~.+]`) + return +} // debugEnv logs all environment variables that start with `GITHUB_` or `INPUT_` // in debug level. @@ -102,139 +74,66 @@ func debugEnv(action *githubactions.Action) { } } -// getControlDefaultVersion returns the default_version field from the control file -// in SemVer format (0.100-0 -> 0.100.0). -func getControlDefaultVersion(f string) (string, error) { - b, err := os.ReadFile(f) +// defineVersion returns Debian package version and Docker tags. +func defineVersion(controlDefaultVersion, pgVersion string, getenv githubactions.GetenvFunc) (string, *images, error) { + debian, err := defineDebianVersion(controlDefaultVersion, pgVersion, getenv) if err != nil { - return "", err - } - - match := controlDefaultVer.FindSubmatch(b) - if match == nil || len(match) != controlDefaultVer.NumSubexp()+1 { - return "", fmt.Errorf("control file did not find default_version: %s", f) - } - - major := match[controlDefaultVer.SubexpIndex("major")] - minor := match[controlDefaultVer.SubexpIndex("minor")] - patch := match[controlDefaultVer.SubexpIndex("patch")] - - return fmt.Sprintf("%s.%s.%s", major, minor, patch), nil -} - -// definePackageVersion returns valid Debian package version, -// based on `default_version` in the control file and environment variables set by GitHub Actions. -// -// See https://www.debian.org/doc/debian-policy/ch-controlfields.html#version. -// We use `upstream_version` only. -// For that reason, we can't use `-`, so we replace it with `~`. -func definePackageVersion(controlDefaultVersion string, getenv githubactions.GetenvFunc) (string, error) { - var packageVersion string - var err error - - switch event := getenv("GITHUB_EVENT_NAME"); event { - case "pull_request", "pull_request_target": - branch := strings.ToLower(getenv("GITHUB_HEAD_REF")) - packageVersion = definePackageVersionForPR(controlDefaultVersion, branch) - - case "push", "schedule", "workflow_run": - refName := strings.ToLower(getenv("GITHUB_REF_NAME")) - - switch refType := strings.ToLower(getenv("GITHUB_REF_TYPE")); refType { - case "branch": - packageVersion, err = definePackageVersionForBranch(controlDefaultVersion, refName) - - case "tag": - packageVersion, err = definePackagerVersionForTag(refName) - - default: - err = fmt.Errorf("unhandled ref type %q for event %q", refType, event) - } - - default: - err = fmt.Errorf("unhandled event type %q", event) + return "", nil, err } + docker, err := defineDockerVersion(controlDefaultVersion, pgVersion, getenv) if err != nil { - return "", err + return "", nil, err } - if packageVersion == "" { - return "", fmt.Errorf("both packageVersion and err are nil") - } - - return packageVersion, nil + return debian, docker, nil } -// definePackageVersionForPR returns valid Debian package version for PR. -// See [definePackageVersion]. -func definePackageVersionForPR(controlDefaultVersion, branch string) string { - // for branches like "dependabot/submodules/XXX" - parts := strings.Split(branch, "/") - branch = parts[len(parts)-1] - res := fmt.Sprintf("%s-pr-%s", controlDefaultVersion, branch) - - return disallowedVer.ReplaceAllString(res, "~") -} +func main() { + controlFileF := flag.String("control-file", "../../pg_documentdb/documentdb.control", "pg_documentdb/documentdb.control file path") + pgVersionF := flag.String("pg-version", "17", "Major PostgreSQL version") -// definePackageVersionForBranch returns valid Debian package version for branch. -// See [definePackageVersion]. -func definePackageVersionForBranch(controlDefaultVersion, branch string) (string, error) { - switch branch { - case "ferretdb": - return fmt.Sprintf("%s~branch~%s", controlDefaultVersion, branch), nil - default: - return "", fmt.Errorf("unhandled branch %q", branch) - } -} + flag.Parse() -// semVar parses tag and returns version components. -// -// It returns error for invalid tag syntax, prerelease is missing `ferretdb` or if it has buildmetadata. -func semVar(tag string) (major, minor, patch, prerelease string, err error) { - match := semVerTag.FindStringSubmatch(tag) - if match == nil || len(match) != semVerTag.NumSubexp()+1 { - return "", "", "", "", fmt.Errorf("unexpected tag syntax %q", tag) - } + action := githubactions.New() - major = match[semVerTag.SubexpIndex("major")] - minor = match[semVerTag.SubexpIndex("minor")] - patch = match[semVerTag.SubexpIndex("patch")] - prerelease = match[semVerTag.SubexpIndex("prerelease")] - buildmetadata := match[semVerTag.SubexpIndex("buildmetadata")] + debugEnv(action) - if prerelease == "" { - return "", "", "", "", fmt.Errorf("prerelease is empty") + if *controlFileF == "" { + action.Fatalf("%s", "-control-file flag is empty.") } - if !strings.Contains(prerelease, "ferretdb") { - return "", "", "", "", fmt.Errorf("prerelease %q should include `ferretdb`", prerelease) + switch *pgVersionF { + case "15", "16", "17": + // nothing + default: + action.Fatalf("%s", fmt.Sprintf("Invalid PostgreSQL version %q.", *pgVersionF)) } - if buildmetadata != "" { - return "", "", "", "", fmt.Errorf("buildmetadata %q is present", buildmetadata) + controlDefaultVersion, err := getControlDefaultVersion(*controlFileF) + if err != nil { + action.Fatalf("%s", err) } - return -} - -// definePackagerVersionForTag returns valid Debian package version for tag. -// See [definePackageVersion]. -func definePackagerVersionForTag(tag string) (string, error) { - major, minor, patch, prerelease, err := semVar(tag) + debian, docker, err := defineVersion(controlDefaultVersion, *pgVersionF, action.Getenv) if err != nil { - return "", err + action.Fatalf("%s", err) } - res := fmt.Sprintf("%s.%s.%s-%s", major, minor, patch, prerelease) - return disallowedVer.ReplaceAllString(res, "~"), nil -} + debianSummary(action, debian) + action.SetOutput("debian_version", debian) + + dockerSummary(action, docker) -// setDebianVersionResults sets action output parameters, summary, etc. -func setDebianVersionResults(action *githubactions.Action, res string) { - output := fmt.Sprintf("version: `%s`", res) + developmentTagFlags := make([]string, len(docker.developmentImages)) + for i, image := range docker.developmentImages { + developmentTagFlags[i] = fmt.Sprintf("--tag %s", image) + } + action.SetOutput("docker_development_tag_flags", strings.Join(developmentTagFlags, " ")) - action.AddStepSummary(output) - action.Infof("%s", output) - action.SetOutput("version", res) + productionTagFlags := make([]string, len(docker.productionImages)) + for i, image := range docker.productionImages { + productionTagFlags[i] = fmt.Sprintf("--tag %s", image) + } + action.SetOutput("docker_production_tag_flags", strings.Join(productionTagFlags, " ")) } diff --git a/ferretdb_packaging/defineversion/main_test.go b/ferretdb_packaging/defineversion/main_test.go index 71eea4877..e616b5257 100644 --- a/ferretdb_packaging/defineversion/main_test.go +++ b/ferretdb_packaging/defineversion/main_test.go @@ -1,26 +1,9 @@ -// Copyright 2021 FerretDB Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package main import ( - "bytes" - "io" - "os" + "fmt" "testing" - "github.com/sethvargo/go-githubactions" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -37,248 +20,319 @@ func getEnvFunc(t *testing.T, env map[string]string) func(string) string { } } -func TestDefine(t *testing.T) { +func TestParseGitTag(t *testing.T) { + tests := map[string]struct { + major int + minor int + patch int + prerelease string + err string + }{ + "v0.100.0-ferretdb-2.0.0": { + major: 0, + minor: 100, + patch: 0, + prerelease: "ferretdb-2.0.0", + }, + "0.100.0-ferretdb-2.0.0": { + err: `unexpected git tag format "0.100.0-ferretdb-2.0.0"`, + }, + "v0.100.0-ferretdb": { + err: `prerelease "ferretdb" should start with 'ferretdb-'`, + }, + "v0.100.0": { + err: `prerelease "" should start with 'ferretdb-'`, + }, + } + + for tag, tc := range tests { + t.Run(tag, func(t *testing.T) { + major, minor, patch, prerelease, err := parseGitTag(tag) + if tc.err != "" { + require.EqualError(t, err, tc.err) + return + } + + require.NoError(t, err) + + assert.Equal(t, tc.major, major) + assert.Equal(t, tc.minor, minor) + assert.Equal(t, tc.patch, patch) + assert.Equal(t, tc.prerelease, prerelease) + }) + } +} + +func TestDefineVersion(t *testing.T) { + const ( + controlDefaultVersion = "0.100.0" + pgVersion = "17" + ) + for name, tc := range map[string]struct { - env map[string]string - controlDefaultVersion string - expected string + env map[string]string + expectedDebian string + expectedDocker *images + expectedErr error }{ "pull_request": { env: map[string]string{ + "GITHUB_BASE_REF": "ferretdb", "GITHUB_EVENT_NAME": "pull_request", "GITHUB_HEAD_REF": "define-version", "GITHUB_REF_NAME": "1/merge", "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + }, + expectedDebian: "0.100.0~pr~define~version", + expectedDocker: &images{ + developmentImages: []string{ + "ghcr.io/ferretdb/postgres-documentdb-dev:pr-define-version", + }, + }, + }, + "pull_request-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "ferretdb", + "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_HEAD_REF": "define-version", + "GITHUB_REF_NAME": "1/merge", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + }, + expectedDebian: "0.100.0~pr~define~version", + expectedDocker: &images{ + developmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:pr-define-version", + }, }, - controlDefaultVersion: "0.100.0", - expected: "0.100.0~pr~define~version", }, "pull_request_target": { env: map[string]string{ + "GITHUB_BASE_REF": "ferretdb", + "GITHUB_EVENT_NAME": "pull_request_target", + "GITHUB_HEAD_REF": "define-version", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + }, + expectedDebian: "0.100.0~pr~define~version", + expectedDocker: &images{ + developmentImages: []string{ + "ghcr.io/ferretdb/postgres-documentdb-dev:pr-define-version", + }, + }, + }, + "pull_request_target-other": { + env: map[string]string{ + "GITHUB_BASE_REF": "ferretdb", "GITHUB_EVENT_NAME": "pull_request_target", "GITHUB_HEAD_REF": "define-version", "GITHUB_REF_NAME": "ferretdb", "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + }, + expectedDebian: "0.100.0~pr~define~version", + expectedDocker: &images{ + developmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:pr-define-version", + }, }, - controlDefaultVersion: "0.100.0", - expected: "0.100.0~pr~define~version", }, "push/ferretdb": { env: map[string]string{ + "GITHUB_BASE_REF": "", "GITHUB_EVENT_NAME": "push", "GITHUB_HEAD_REF": "", "GITHUB_REF_NAME": "ferretdb", "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + }, + expectedDebian: "0.100.0~branch~ferretdb", + expectedDocker: &images{ + developmentImages: []string{ + //"ferretdb/postgres-documentdb-dev:ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:ferretdb", + //"quay.io/ferretdb/postgres-documentdb-dev:ferretdb", + }, }, - controlDefaultVersion: "0.100.0", - expected: "0.100.0~branch~ferretdb", }, - "push/other": { + "push/ferretdb-other": { env: map[string]string{ + "GITHUB_BASE_REF": "", "GITHUB_EVENT_NAME": "push", "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "releases", - "GITHUB_REF_TYPE": "other", // not ferretdb branch + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + }, + expectedDebian: "0.100.0~branch~ferretdb", + expectedDocker: &images{ + developmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:ferretdb", + }, }, }, - "push/tag/v0.100.0-ferretdb": { + "push/main": { env: map[string]string{ + "GITHUB_BASE_REF": "", "GITHUB_EVENT_NAME": "push", "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "v0.100.0-ferretdb", - "GITHUB_REF_TYPE": "tag", + "GITHUB_REF_NAME": "main", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", }, - expected: "0.100.0~ferretdb", + expectedErr: fmt.Errorf(`unhandled branch "main"`), }, - "push/tag/v0.100.0-ferretdb-2.0.1": { + "push/main-other": { env: map[string]string{ + "GITHUB_BASE_REF": "", "GITHUB_EVENT_NAME": "push", "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "v0.100.0-ferretdb-2.0.1", - "GITHUB_REF_TYPE": "tag", + "GITHUB_REF_NAME": "main", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, - expected: "0.100.0~ferretdb~2.0.1", + expectedErr: fmt.Errorf(`unhandled branch "main"`), }, - "push/tag/missing-prerelease": { + "push/tag/release": { env: map[string]string{ + "GITHUB_BASE_REF": "", "GITHUB_EVENT_NAME": "push", "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "v0.100.0", // missing prerelease + "GITHUB_REF_NAME": "v0.100.0-ferretdb-2.0.0", "GITHUB_REF_TYPE": "tag", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + }, + expectedDebian: "0.100.0~ferretdb~2.0.0", + expectedDocker: &images{ + developmentImages: []string{ + //"ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb", + //"ferretdb/postgres-documentdb-dev:latest", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb-2.0.0", + "ghcr.io/ferretdb/postgres-documentdb-dev:latest", + //"quay.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb", + //"quay.io/ferretdb/postgres-documentdb-dev:latest", + }, + productionImages: []string{ + //"ferretdb/postgres-documentdb:17-0.100.0-ferretdb", + //"ferretdb/postgres-documentdb:latest", + "ghcr.io/ferretdb/postgres-documentdb:17-0.100.0-ferretdb-2.0.0", + "ghcr.io/ferretdb/postgres-documentdb:latest", + //"quay.io/ferretdb/postgres-documentdb:17-0.100.0-ferretdb", + //"quay.io/ferretdb/postgres-documentdb:latest", + }, }, }, - "push/tag/not-ferretdb-prerelease": { + "push/tag/release-other": { env: map[string]string{ + "GITHUB_BASE_REF": "", "GITHUB_EVENT_NAME": "push", "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "v0.100.0-other", // missing ferretdb in prerelease + "GITHUB_REF_NAME": "v0.100.0-ferretdb-2.0.0", "GITHUB_REF_TYPE": "tag", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + }, + expectedDebian: "0.100.0~ferretdb~2.0.0", + expectedDocker: &images{ + developmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17-0.100.0-ferretdb-2.0.0", + "ghcr.io/otherorg/postgres-otherrepo-dev:latest", + }, + productionImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo:17-0.100.0-ferretdb-2.0.0", + "ghcr.io/otherorg/postgres-otherrepo:latest", + }, }, }, - "push/tag/missing-v": { + + "schedule": { env: map[string]string{ - "GITHUB_EVENT_NAME": "push", + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "schedule", "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "0.100.0-ferretdb", - "GITHUB_REF_TYPE": "tag", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + }, + expectedDebian: "0.100.0~branch~ferretdb", + expectedDocker: &images{ + developmentImages: []string{ + //"ferretdb/postgres-documentdb-dev:ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:ferretdb", + //"quay.io/ferretdb/postgres-documentdb-dev:ferretdb", + }, }, }, - "push/tag/not-semvar": { + "schedule-other": { env: map[string]string{ - "GITHUB_EVENT_NAME": "push", + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "schedule", "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "v0.100-0-ferretdb", - "GITHUB_REF_TYPE": "tag", + "GITHUB_REF_NAME": "ferretdb", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + }, + expectedDebian: "0.100.0~branch~ferretdb", + expectedDocker: &images{ + developmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:ferretdb", + }, }, }, - "schedule": { + "workflow_run": { env: map[string]string{ - "GITHUB_EVENT_NAME": "schedule", + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "workflow_run", "GITHUB_HEAD_REF": "", "GITHUB_REF_NAME": "ferretdb", "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + }, + expectedDebian: "0.100.0~branch~ferretdb", + expectedDocker: &images{ + developmentImages: []string{ + //"ferretdb/postgres-documentdb-dev:ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:ferretdb", + //"quay.io/ferretdb/postgres-documentdb-dev:ferretdb", + }, }, - controlDefaultVersion: "0.100.0", - expected: "0.100.0~branch~ferretdb", }, - - "workflow_run": { + "workflow_run-other": { env: map[string]string{ + "GITHUB_BASE_REF": "", "GITHUB_EVENT_NAME": "workflow_run", "GITHUB_HEAD_REF": "", "GITHUB_REF_NAME": "ferretdb", "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", + }, + expectedDebian: "0.100.0~branch~ferretdb", + expectedDocker: &images{ + developmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:ferretdb", + }, }, - controlDefaultVersion: "0.100.0", - expected: "0.100.0~branch~ferretdb", }, } { t.Run(name, func(t *testing.T) { - actual, err := definePackageVersion(tc.controlDefaultVersion, getEnvFunc(t, tc.env)) - if tc.expected == "" { - require.Error(t, err) + debian, docker, err := defineVersion(controlDefaultVersion, pgVersion, getEnvFunc(t, tc.env)) + if tc.expectedDebian == "" && tc.expectedDocker == nil { + require.Error(t, tc.expectedErr) + require.Equal(t, err, tc.expectedErr) return } + require.NoError(t, tc.expectedErr) require.NoError(t, err) - assert.Equal(t, tc.expected, actual) - }) - } -} - -func TestResults(t *testing.T) { - dir := t.TempDir() - - summaryF, err := os.CreateTemp(dir, "summary") - require.NoError(t, err) - defer summaryF.Close() //nolint:errcheck // temporary file for testing - - outputF, err := os.CreateTemp(dir, "output") - require.NoError(t, err) - defer outputF.Close() //nolint:errcheck // temporary file for testing - - var stdout bytes.Buffer - getenv := getEnvFunc(t, map[string]string{ - "GITHUB_STEP_SUMMARY": summaryF.Name(), - "GITHUB_OUTPUT": outputF.Name(), - }) - action := githubactions.New(githubactions.WithGetenv(getenv), githubactions.WithWriter(&stdout)) - - version := "0.100.0~ferretdb" - - setDebianVersionResults(action, version) - - expected := "version: `0.100.0~ferretdb`\n" - assert.Equal(t, expected, stdout.String(), "stdout does not match") - - b, err := io.ReadAll(summaryF) - require.NoError(t, err) - assert.Equal(t, expected, string(b), "summary does not match") - - expectedOutput := ` -version<<_GitHubActionsFileCommandDelimeter_ -0.100.0~ferretdb -_GitHubActionsFileCommandDelimeter_ -`[1:] - b, err = io.ReadAll(outputF) - require.NoError(t, err) - assert.Equal(t, expectedOutput, string(b), "output parameters does not match") -} - -func TestReadControlDefaultVersion(t *testing.T) { - controlF, err := os.CreateTemp(t.TempDir(), "test.control") - require.NoError(t, err) - - defer controlF.Close() //nolint:errcheck // temporary file for testing - - buf := `comment = 'API surface for DocumentDB for PostgreSQL' -default_version = '0.100-0' -module_pathname = '$libdir/pg_documentdb' -relocatable = false -superuser = true -requires = 'documentdb_core, pg_cron, tsm_system_rows, vector, postgis, rum'` - _, err = io.WriteString(controlF, buf) - require.NoError(t, err) - - controlDefaultVersion, err := getControlDefaultVersion(controlF.Name()) - require.NoError(t, err) - - require.Equal(t, "0.100.0", controlDefaultVersion) -} - -func TestSemVar(t *testing.T) { - t.Parallel() - - tests := map[string]struct { - tag string - major string - minor string - patch string - prerelease string - err string - }{ - "Valid": { - tag: "v1.100.0-ferretdb", - major: "1", - minor: "100", - patch: "0", - prerelease: "ferretdb", - }, - "SpecificVersion": { - tag: "v1.100.0-ferretdb-2.0.1", - major: "1", - minor: "100", - patch: "0", - prerelease: "ferretdb-2.0.1", - }, - "MissingV": { - tag: "0.100.0-ferretdb", - err: `unexpected tag syntax "0.100.0-ferretdb"`, - }, - "MissingFerretDB": { - tag: "v0.100.0", - err: "prerelease is empty", - }, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - major, minor, patch, prerelease, err := semVar(tc.tag) - - if tc.err != "" { - require.EqualError(t, err, tc.err) - return - } - require.Equal(t, tc.major, major) - require.Equal(t, tc.minor, minor) - require.Equal(t, tc.patch, patch) - require.Equal(t, tc.prerelease, prerelease) + assert.Equal(t, tc.expectedDebian, debian) + assert.Equal(t, tc.expectedDocker, docker) }) } } From 8d0e80853a148f938a48d4e00204e608b940a223 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Mon, 3 Mar 2025 23:07:52 +0400 Subject: [PATCH 17/59] Update workflow (#24) --- .github/workflows/ferretdb_packages.yml | 33 ++++++++++++++----------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index 227512acc..44efe77eb 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -60,6 +60,7 @@ jobs: if: > github.event_name != 'pull_request_target' || ( + contains(github.event.pull_request.labels.*.name, 'trust') && !contains(github.event.pull_request.labels.*.name, 'not ready') && contains(github.event.pull_request.labels.*.name, 'packages') ) @@ -102,22 +103,22 @@ jobs: - name: Setup Go uses: FerretDB/github-actions/setup-go@main - - name: Define Debian package version + - name: Define version id: version run: | cd ferretdb_packaging go mod tidy go mod verify - go run ./defineversion --command deb-version --control-file ../pg_documentdb_core/documentdb_core.control + go run ./defineversion -pg-version ${{ matrix.pg }} - - name: Build ${{ steps.version.outputs.version }} - if: steps.version.outputs.version != '' - run: ./ferretdb_packaging/build_packages.sh --os ${{ matrix.os }} --pg ${{ matrix.pg }} --version ${{ steps.version.outputs.version }} --test-clean-install + - name: Build ${{ steps.version.outputs.debian_version }} + if: steps.version.outputs.debian_version != '' + run: ./ferretdb_packaging/build_packages.sh --os ${{ matrix.os }} --pg ${{ matrix.pg }} --version ${{ steps.version.outputs.debian_version }} --test-clean-install - - name: Upload + - name: Upload .deb packages uses: actions/upload-artifact@v4 with: - name: ${{ matrix.os }}-${{ matrix.pg }}-${{ steps.version.outputs.version }} + name: ${{ matrix.os }}-${{ matrix.pg }}-${{ steps.version.outputs.debian_version }} path: packaging/*.deb retention-days: 1 if-no-files-found: error @@ -134,6 +135,8 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 40 + needs: deb + if: > github.event_name != 'pull_request_target' || ( @@ -148,7 +151,7 @@ jobs: strategy: fail-fast: false matrix: - pg: [15.12, 16.8, 17.4] + pg: [15, 16, 17] steps: # TODO https://github.com/FerretDB/github-actions/issues/211 @@ -180,17 +183,19 @@ jobs: - name: Setup Go uses: FerretDB/github-actions/setup-go@main - - name: Define Docker tags - id: tag + - name: Define version + id: version run: | cd ferretdb_packaging go mod tidy go mod verify - go run ./defineversion --command docker-tags - env: - INPUT_PG_VERSION: ${{ matrix.pg }} + go run ./defineversion -pg-version ${{ matrix.pg }} - # Build and push Docker images + - name: Download .deb package + uses: actions/download-artifact@v4 + with: + name: deb12-${{ matrix.pg }}-${{ steps.version.outputs.debian_version }} + path: packaging - name: Check dirty run: | From 92eb37214c5b82e4ab3aa63aeb530becba89420f Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Mon, 3 Mar 2025 23:25:25 +0400 Subject: [PATCH 18/59] Prepare Docker images builds (#25) Co-authored-by: Chi Fujii --- .github/workflows/ferretdb_packages.yml | 67 ++++++++++++++++++- ferretdb_packaging/10-preload.sh | 12 ++++ ferretdb_packaging/20-install.sql | 1 + ferretdb_packaging/90-install-development.sql | 1 + .../Dockerfile_build_deb_packages | 2 +- ferretdb_packaging/Makefile | 37 ++++++++++ ferretdb_packaging/buildkitd.toml | 5 ++ ferretdb_packaging/defineversion/main.go | 2 +- ferretdb_packaging/development.Dockerfile | 61 +++++++++++++++++ ferretdb_packaging/production.Dockerfile | 48 +++++++++++++ 10 files changed, 232 insertions(+), 4 deletions(-) create mode 100755 ferretdb_packaging/10-preload.sh create mode 100644 ferretdb_packaging/20-install.sql create mode 100644 ferretdb_packaging/90-install-development.sql create mode 100644 ferretdb_packaging/Makefile create mode 100644 ferretdb_packaging/buildkitd.toml create mode 100644 ferretdb_packaging/development.Dockerfile create mode 100644 ferretdb_packaging/production.Dockerfile diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index 44efe77eb..56910e64e 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -109,7 +109,7 @@ jobs: cd ferretdb_packaging go mod tidy go mod verify - go run ./defineversion -pg-version ${{ matrix.pg }} + go run ./defineversion -control-file ../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} - name: Build ${{ steps.version.outputs.debian_version }} if: steps.version.outputs.debian_version != '' @@ -189,7 +189,7 @@ jobs: cd ferretdb_packaging go mod tidy go mod verify - go run ./defineversion -pg-version ${{ matrix.pg }} + go run ./defineversion -control-file ../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} - name: Download .deb package uses: actions/download-artifact@v4 @@ -197,6 +197,69 @@ jobs: name: deb12-${{ matrix.pg }}-${{ steps.version.outputs.debian_version }} path: packaging + - name: Initialize Docker builder + run: make -C ferretdb_packaging docker-init + + - name: Build local development Docker image + if: steps.version.outputs.docker_development_tag_flags != '' + run: > + make -C ferretdb_packaging docker-build + POSTGRES_VERSION=${{ matrix.pg }} + DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} + FILE=development + OUTPUT='type=docker' + TAGS=${{ steps.version.outputs.docker_development_tag_flags }} + + - name: Build local production Docker image + if: steps.version.outputs.docker_production_tag_flags != '' + run: > + make -C ferretdb_packaging docker-build + POSTGRES_VERSION=${{ matrix.pg }} + DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} + FILE=production + OUTPUT='type=docker' + TAGS=${{ steps.version.outputs.docker_production_tag_flags }} + + # - name: Login to Docker Hub + # uses: docker/login-action@v3 + # with: + # username: ferretdbbot + # password: ${{ secrets.DOCKER_HUB_TOKEN }} + + # - name: Login to GitHub Container Registry + # uses: docker/login-action@v3 + # with: + # registry: ghcr.io + # username: ${{ github.actor }} + # password: ${{ secrets.GITHUB_TOKEN }} + + # - name: Login to Quay.io + # uses: docker/login-action@v3 + # with: + # registry: quay.io + # username: ferretdbbot + # password: ${{ secrets.QUAY_TOKEN }} + + - name: Build and push development Docker images + if: steps.version.outputs.docker_development_tag_flags != '' + run: > + make -C ferretdb_packaging docker-build + POSTGRES_VERSION=${{ matrix.pg }} + DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} + FILE=development + OUTPUT='type=image,push=true' + TAGS=${{ steps.version.outputs.docker_development_tag_flags }} + + - name: Build and push production Docker images + if: steps.version.outputs.docker_production_tag_flags != '' + run: > + make -C ferretdb_packaging docker-build + POSTGRES_VERSION=${{ matrix.pg }} + DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} + FILE=production + OUTPUT='type=image,push=true' + TAGS=${{ steps.version.outputs.docker_production_tag_flags }} + - name: Check dirty run: | git status diff --git a/ferretdb_packaging/10-preload.sh b/ferretdb_packaging/10-preload.sh new file mode 100755 index 000000000..f1cff5645 --- /dev/null +++ b/ferretdb_packaging/10-preload.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +echo "shared_preload_libraries = 'pg_cron,pg_documentdb_core,pg_documentdb'" >> $PGDATA/postgresql.conf +echo "cron.database_name = 'postgres'" >> $PGDATA/postgresql.conf + +source /usr/local/bin/docker-entrypoint.sh + +# restarting is needed to create extension +docker_temp_server_stop +docker_temp_server_start "$@" diff --git a/ferretdb_packaging/20-install.sql b/ferretdb_packaging/20-install.sql new file mode 100644 index 000000000..34c7b1140 --- /dev/null +++ b/ferretdb_packaging/20-install.sql @@ -0,0 +1 @@ +CREATE EXTENSION documentdb CASCADE; diff --git a/ferretdb_packaging/90-install-development.sql b/ferretdb_packaging/90-install-development.sql new file mode 100644 index 000000000..070b769a5 --- /dev/null +++ b/ferretdb_packaging/90-install-development.sql @@ -0,0 +1 @@ +CREATE EXTENSION pgtap; diff --git a/ferretdb_packaging/Dockerfile_build_deb_packages b/ferretdb_packaging/Dockerfile_build_deb_packages index d6caf9d7d..b55b1a93b 100644 --- a/ferretdb_packaging/Dockerfile_build_deb_packages +++ b/ferretdb_packaging/Dockerfile_build_deb_packages @@ -2,7 +2,7 @@ ARG BASE_IMAGE=debian:bookworm FROM --platform=linux/amd64 ${BASE_IMAGE} ARG DEBIAN_FRONTEND=noninteractive -ARG POSTGRES_VERSION=16 +ARG POSTGRES_VERSION ARG DOCUMENTDB_VERSION RUN test -n "$DOCUMENTDB_VERSION" || (echo "DOCUMENTDB_VERSION not set" && false) diff --git a/ferretdb_packaging/Makefile b/ferretdb_packaging/Makefile new file mode 100644 index 000000000..ea9b68f31 --- /dev/null +++ b/ferretdb_packaging/Makefile @@ -0,0 +1,37 @@ +.PHONY: docker-init docker-build docker-cleanup + +docker-init: + docker buildx create \ + --driver=docker-container \ + --name=ferretdb \ + --bootstrap=true \ + --use=false \ + --config=./buildkitd.toml \ + --driver-opt network=host \ + --driver-opt env.JAEGER_TRACE=127.0.0.1:6831 \ + --driver-opt env.BUILDKIT_STEP_LOG_MAX_SIZE=-1 \ + --driver-opt env.BUILDKIT_STEP_LOG_MAX_SPEED=-1 \ + || true + docker buildx ls + +docker-build: + test -n "$(POSTGRES_VERSION)" || (echo "POSTGRES_VERSION not set" && false) + test -n "$(DOCUMENTDB_VERSION)" || (echo "DOCUMENTDB_VERSION not set" && false) + test -n "$(FILE)" || (echo "FILE not set" && false) + test -n "$(OUTPUT)" || (echo "OUTPUT not set" && false) + test -n "$(TAGS)" || (echo "TAGS not set" && false) + docker buildx build --builder=documentdb \ + --file=ferretdb_packaging/$(TARGET).Dockerfile \ + --build-arg=POSTGRES_VERSION=$(POSTGRES_VERSION) \ + --build-arg=DOCUMENTDB_VERSION=$(DOCUMENTDB_VERSION) \ + --platform=linux/amd64 \ + $(TAGS) \ + --output=$(OUTPUT) \ + .. + +docker-cleanup: + docker system df + docker buildx --builder=documentdb du || true + docker buildx --builder=documentdb rm --force || true + docker system prune --force + docker system df diff --git a/ferretdb_packaging/buildkitd.toml b/ferretdb_packaging/buildkitd.toml new file mode 100644 index 000000000..1add01ddb --- /dev/null +++ b/ferretdb_packaging/buildkitd.toml @@ -0,0 +1,5 @@ +[worker.oci] + gc = false + +[worker.containerd] + gc = false diff --git a/ferretdb_packaging/defineversion/main.go b/ferretdb_packaging/defineversion/main.go index c131d9e34..67c48e294 100644 --- a/ferretdb_packaging/defineversion/main.go +++ b/ferretdb_packaging/defineversion/main.go @@ -90,7 +90,7 @@ func defineVersion(controlDefaultVersion, pgVersion string, getenv githubactions } func main() { - controlFileF := flag.String("control-file", "../../pg_documentdb/documentdb.control", "pg_documentdb/documentdb.control file path") + controlFileF := flag.String("control-file", "../pg_documentdb/documentdb.control", "pg_documentdb/documentdb.control file path") pgVersionF := flag.String("pg-version", "17", "Major PostgreSQL version") flag.Parse() diff --git a/ferretdb_packaging/development.Dockerfile b/ferretdb_packaging/development.Dockerfile new file mode 100644 index 000000000..702b54783 --- /dev/null +++ b/ferretdb_packaging/development.Dockerfile @@ -0,0 +1,61 @@ +# syntax=docker/dockerfile:1 + +ARG POSTGRES_VERSION +ARG DOCUMENTDB_VERSION + +FROM postgres:${POSTGRES_VERSION} AS development + +RUN --mount=type=cache,sharing=locked,target=/var/cache/apt < Date: Tue, 4 Mar 2025 00:00:19 +0400 Subject: [PATCH 19/59] Minor tweaks (#26) --- .github/workflows/ferretdb_packages.yml | 12 ++++----- ferretdb_packaging/Makefile | 8 +++--- ferretdb_packaging/defineversion/docker.go | 4 +-- .../defineversion/docker_test.go | 14 +++++----- ferretdb_packaging/defineversion/main.go | 26 ++++++++++++------- 5 files changed, 35 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index 56910e64e..02329a222 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -109,7 +109,7 @@ jobs: cd ferretdb_packaging go mod tidy go mod verify - go run ./defineversion -control-file ../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} + go run ./defineversion -control-file ../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} -debian-only - name: Build ${{ steps.version.outputs.debian_version }} if: steps.version.outputs.debian_version != '' @@ -191,7 +191,7 @@ jobs: go mod verify go run ./defineversion -control-file ../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} - - name: Download .deb package + - name: Download deb12-${{ matrix.pg }}-${{ steps.version.outputs.debian_version }} uses: actions/download-artifact@v4 with: name: deb12-${{ matrix.pg }}-${{ steps.version.outputs.debian_version }} @@ -208,7 +208,7 @@ jobs: DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} FILE=development OUTPUT='type=docker' - TAGS=${{ steps.version.outputs.docker_development_tag_flags }} + TAGS='${{ steps.version.outputs.docker_development_tag_flags }}' - name: Build local production Docker image if: steps.version.outputs.docker_production_tag_flags != '' @@ -218,7 +218,7 @@ jobs: DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} FILE=production OUTPUT='type=docker' - TAGS=${{ steps.version.outputs.docker_production_tag_flags }} + TAGS='${{ steps.version.outputs.docker_production_tag_flags }}' # - name: Login to Docker Hub # uses: docker/login-action@v3 @@ -248,7 +248,7 @@ jobs: DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} FILE=development OUTPUT='type=image,push=true' - TAGS=${{ steps.version.outputs.docker_development_tag_flags }} + TAGS='${{ steps.version.outputs.docker_development_tag_flags }}' - name: Build and push production Docker images if: steps.version.outputs.docker_production_tag_flags != '' @@ -258,7 +258,7 @@ jobs: DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} FILE=production OUTPUT='type=image,push=true' - TAGS=${{ steps.version.outputs.docker_production_tag_flags }} + TAGS='${{ steps.version.outputs.docker_production_tag_flags }}' - name: Check dirty run: | diff --git a/ferretdb_packaging/Makefile b/ferretdb_packaging/Makefile index ea9b68f31..ee53b7879 100644 --- a/ferretdb_packaging/Makefile +++ b/ferretdb_packaging/Makefile @@ -3,7 +3,7 @@ docker-init: docker buildx create \ --driver=docker-container \ - --name=ferretdb \ + --name=documentdb \ --bootstrap=true \ --use=false \ --config=./buildkitd.toml \ @@ -22,11 +22,11 @@ docker-build: test -n "$(TAGS)" || (echo "TAGS not set" && false) docker buildx build --builder=documentdb \ --file=ferretdb_packaging/$(TARGET).Dockerfile \ - --build-arg=POSTGRES_VERSION=$(POSTGRES_VERSION) \ - --build-arg=DOCUMENTDB_VERSION=$(DOCUMENTDB_VERSION) \ + --build-arg='POSTGRES_VERSION=$(POSTGRES_VERSION)' \ + --build-arg='DOCUMENTDB_VERSION=$(DOCUMENTDB_VERSION)' \ --platform=linux/amd64 \ + --output='$(OUTPUT)' \ $(TAGS) \ - --output=$(OUTPUT) \ .. docker-cleanup: diff --git a/ferretdb_packaging/defineversion/docker.go b/ferretdb_packaging/defineversion/docker.go index ffb9db561..e80b7a7fa 100644 --- a/ferretdb_packaging/defineversion/docker.go +++ b/ferretdb_packaging/defineversion/docker.go @@ -172,8 +172,8 @@ func dockerImageURL(name string) string { func dockerSummary(action *githubactions.Action, version *images) { var buf strings.Builder w := tabwriter.NewWriter(&buf, 1, 1, 1, ' ', tabwriter.Debug) - fmt.Fprintf(w, "\tType\tImage\t\n") - fmt.Fprintf(w, "\t----\t-----\t\n") + fmt.Fprintf(w, "\tType\tDocker image\t\n") + fmt.Fprintf(w, "\t----\t------------\t\n") for _, image := range version.developmentImages { u := dockerImageURL(image) diff --git a/ferretdb_packaging/defineversion/docker_test.go b/ferretdb_packaging/defineversion/docker_test.go index e5b6e7ea8..aaf1affd9 100644 --- a/ferretdb_packaging/defineversion/docker_test.go +++ b/ferretdb_packaging/defineversion/docker_test.go @@ -51,7 +51,7 @@ func TestDockerSummary(t *testing.T) { result := &images{ developmentImages: []string{ - "ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb", "ghcr.io/ferretdb/postgres-documentdb-dev:latest", }, productionImages: []string{ @@ -62,9 +62,9 @@ func TestDockerSummary(t *testing.T) { dockerSummary(action, result) expectedStdout := strings.ReplaceAll(` - |Type |Image | - |---- |----- | - |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb) | + |Type |Docker image | + |---- |------------ | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb) | |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:latest'](https://ghcr.io/ferretdb/postgres-documentdb-dev:latest) | |Production |['quay.io/ferretdb/postgres-documentdb:latest'](https://quay.io/ferretdb/postgres-documentdb:latest) | @@ -73,9 +73,9 @@ func TestDockerSummary(t *testing.T) { assert.Equal(t, expectedStdout, stdout.String(), "stdout does not match") expectedSummary := strings.ReplaceAll(` - |Type |Image | - |---- |----- | - |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:16-0.102.0-ferretdb) | + |Type |Docker image | + |---- |------------ | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb) | |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:latest'](https://ghcr.io/ferretdb/postgres-documentdb-dev:latest) | |Production |['quay.io/ferretdb/postgres-documentdb:latest'](https://quay.io/ferretdb/postgres-documentdb:latest) | diff --git a/ferretdb_packaging/defineversion/main.go b/ferretdb_packaging/defineversion/main.go index 67c48e294..b171a3c32 100644 --- a/ferretdb_packaging/defineversion/main.go +++ b/ferretdb_packaging/defineversion/main.go @@ -92,6 +92,7 @@ func defineVersion(controlDefaultVersion, pgVersion string, getenv githubactions func main() { controlFileF := flag.String("control-file", "../pg_documentdb/documentdb.control", "pg_documentdb/documentdb.control file path") pgVersionF := flag.String("pg-version", "17", "Major PostgreSQL version") + debianOnlyF := flag.Bool("debian-only", false, "Only set output for Debian package version") flag.Parse() @@ -123,17 +124,22 @@ func main() { debianSummary(action, debian) action.SetOutput("debian_version", debian) - dockerSummary(action, docker) + if !*debianOnlyF { + dockerSummary(action, docker) - developmentTagFlags := make([]string, len(docker.developmentImages)) - for i, image := range docker.developmentImages { - developmentTagFlags[i] = fmt.Sprintf("--tag %s", image) - } - action.SetOutput("docker_development_tag_flags", strings.Join(developmentTagFlags, " ")) + action.SetOutput("docker_development_images", strings.Join(docker.developmentImages, ",")) + action.SetOutput("docker_production_images", strings.Join(docker.productionImages, ",")) + + developmentTagFlags := make([]string, len(docker.developmentImages)) + for i, image := range docker.developmentImages { + developmentTagFlags[i] = fmt.Sprintf("--tag %s", image) + } + action.SetOutput("docker_development_tag_flags", strings.Join(developmentTagFlags, " ")) - productionTagFlags := make([]string, len(docker.productionImages)) - for i, image := range docker.productionImages { - productionTagFlags[i] = fmt.Sprintf("--tag %s", image) + productionTagFlags := make([]string, len(docker.productionImages)) + for i, image := range docker.productionImages { + productionTagFlags[i] = fmt.Sprintf("--tag %s", image) + } + action.SetOutput("docker_production_tag_flags", strings.Join(productionTagFlags, " ")) } - action.SetOutput("docker_production_tag_flags", strings.Join(productionTagFlags, " ")) } From 6b1b7c3135f4d306e71ae21e9c76b8e21661225e Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 4 Mar 2025 00:55:33 +0400 Subject: [PATCH 20/59] Fix Docker images building (#27) --- .github/workflows/ferretdb_packages.yml | 38 +++++++++++------------ ferretdb_packaging/Makefile | 2 +- ferretdb_packaging/development.Dockerfile | 4 ++- ferretdb_packaging/production.Dockerfile | 4 ++- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index 02329a222..a14fe058a 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -220,25 +220,25 @@ jobs: OUTPUT='type=docker' TAGS='${{ steps.version.outputs.docker_production_tag_flags }}' - # - name: Login to Docker Hub - # uses: docker/login-action@v3 - # with: - # username: ferretdbbot - # password: ${{ secrets.DOCKER_HUB_TOKEN }} - - # - name: Login to GitHub Container Registry - # uses: docker/login-action@v3 - # with: - # registry: ghcr.io - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} - - # - name: Login to Quay.io - # uses: docker/login-action@v3 - # with: - # registry: quay.io - # username: ferretdbbot - # password: ${{ secrets.QUAY_TOKEN }} + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ferretdbbot + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to Quay.io + uses: docker/login-action@v3 + with: + registry: quay.io + username: ferretdbbot + password: ${{ secrets.QUAY_TOKEN }} - name: Build and push development Docker images if: steps.version.outputs.docker_development_tag_flags != '' diff --git a/ferretdb_packaging/Makefile b/ferretdb_packaging/Makefile index ee53b7879..a304d0940 100644 --- a/ferretdb_packaging/Makefile +++ b/ferretdb_packaging/Makefile @@ -21,7 +21,7 @@ docker-build: test -n "$(OUTPUT)" || (echo "OUTPUT not set" && false) test -n "$(TAGS)" || (echo "TAGS not set" && false) docker buildx build --builder=documentdb \ - --file=ferretdb_packaging/$(TARGET).Dockerfile \ + --file=$(FILE).Dockerfile \ --build-arg='POSTGRES_VERSION=$(POSTGRES_VERSION)' \ --build-arg='DOCUMENTDB_VERSION=$(DOCUMENTDB_VERSION)' \ --platform=linux/amd64 \ diff --git a/ferretdb_packaging/development.Dockerfile b/ferretdb_packaging/development.Dockerfile index 702b54783..7d63aa8c4 100644 --- a/ferretdb_packaging/development.Dockerfile +++ b/ferretdb_packaging/development.Dockerfile @@ -1,10 +1,12 @@ # syntax=docker/dockerfile:1 ARG POSTGRES_VERSION -ARG DOCUMENTDB_VERSION FROM postgres:${POSTGRES_VERSION} AS development +ARG POSTGRES_VERSION +ARG DOCUMENTDB_VERSION + RUN --mount=type=cache,sharing=locked,target=/var/cache/apt < Date: Tue, 4 Mar 2025 01:05:18 +0400 Subject: [PATCH 21/59] Fix quay.io login --- .github/workflows/ferretdb_packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index a14fe058a..0c696994e 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -237,7 +237,7 @@ jobs: uses: docker/login-action@v3 with: registry: quay.io - username: ferretdbbot + username: ferretdb+ferretdbbot password: ${{ secrets.QUAY_TOKEN }} - name: Build and push development Docker images From bf0ebfa0a8cb21e3649848da3c5e9127c2b00eef Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 4 Mar 2025 01:46:04 +0400 Subject: [PATCH 22/59] Enable other registries (#28) --- ferretdb_packaging/defineversion/docker.go | 18 ++++++------ ferretdb_packaging/defineversion/main_test.go | 28 +++++++++---------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/ferretdb_packaging/defineversion/docker.go b/ferretdb_packaging/defineversion/docker.go index e80b7a7fa..b97de0bc4 100644 --- a/ferretdb_packaging/defineversion/docker.go +++ b/ferretdb_packaging/defineversion/docker.go @@ -117,8 +117,8 @@ func defineDockerVersionForBranch(owner, repo, branch string) (*images, error) { return res, nil } - // res.developmentImages = append(res.developmentImages, "quay.io/ferretdb/postgres-documentdb-dev:ferretdb") - // res.developmentImages = append(res.developmentImages, "ferretdb/postgres-documentdb-dev:ferretdb") + res.developmentImages = append(res.developmentImages, "quay.io/ferretdb/postgres-documentdb-dev:ferretdb") + res.developmentImages = append(res.developmentImages, "ferretdb/postgres-documentdb-dev:ferretdb") return res, nil } @@ -142,13 +142,13 @@ func defineDockerVersionForTag(owner, repo string, tags []string) *images { return res } - //for _, t := range tags { - // res.developmentImages = append(res.developmentImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb-dev:%s", t)) - // res.productionImages = append(res.productionImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb:%s", t)) - // - // res.developmentImages = append(res.developmentImages, fmt.Sprintf("ferretdb/postgres-documentdb-dev:%s", t)) - // res.productionImages = append(res.productionImages, fmt.Sprintf("ferretdb/postgres-documentdb:%s", t)) - //} + for _, t := range tags { + res.developmentImages = append(res.developmentImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb-dev:%s", t)) + res.productionImages = append(res.productionImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb:%s", t)) + + res.developmentImages = append(res.developmentImages, fmt.Sprintf("ferretdb/postgres-documentdb-dev:%s", t)) + res.productionImages = append(res.productionImages, fmt.Sprintf("ferretdb/postgres-documentdb:%s", t)) + } return res } diff --git a/ferretdb_packaging/defineversion/main_test.go b/ferretdb_packaging/defineversion/main_test.go index e616b5257..00862a2e4 100644 --- a/ferretdb_packaging/defineversion/main_test.go +++ b/ferretdb_packaging/defineversion/main_test.go @@ -153,9 +153,9 @@ func TestDefineVersion(t *testing.T) { expectedDebian: "0.100.0~branch~ferretdb", expectedDocker: &images{ developmentImages: []string{ - //"ferretdb/postgres-documentdb-dev:ferretdb", + "ferretdb/postgres-documentdb-dev:ferretdb", "ghcr.io/ferretdb/postgres-documentdb-dev:ferretdb", - //"quay.io/ferretdb/postgres-documentdb-dev:ferretdb", + "quay.io/ferretdb/postgres-documentdb-dev:ferretdb", }, }, }, @@ -211,20 +211,20 @@ func TestDefineVersion(t *testing.T) { expectedDebian: "0.100.0~ferretdb~2.0.0", expectedDocker: &images{ developmentImages: []string{ - //"ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb", - //"ferretdb/postgres-documentdb-dev:latest", + "ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb-2.0.0", + "ferretdb/postgres-documentdb-dev:latest", "ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb-2.0.0", "ghcr.io/ferretdb/postgres-documentdb-dev:latest", - //"quay.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb", - //"quay.io/ferretdb/postgres-documentdb-dev:latest", + "quay.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb-2.0.0", + "quay.io/ferretdb/postgres-documentdb-dev:latest", }, productionImages: []string{ - //"ferretdb/postgres-documentdb:17-0.100.0-ferretdb", - //"ferretdb/postgres-documentdb:latest", + "ferretdb/postgres-documentdb:17-0.100.0-ferretdb-2.0.0", + "ferretdb/postgres-documentdb:latest", "ghcr.io/ferretdb/postgres-documentdb:17-0.100.0-ferretdb-2.0.0", "ghcr.io/ferretdb/postgres-documentdb:latest", - //"quay.io/ferretdb/postgres-documentdb:17-0.100.0-ferretdb", - //"quay.io/ferretdb/postgres-documentdb:latest", + "quay.io/ferretdb/postgres-documentdb:17-0.100.0-ferretdb-2.0.0", + "quay.io/ferretdb/postgres-documentdb:latest", }, }, }, @@ -262,9 +262,9 @@ func TestDefineVersion(t *testing.T) { expectedDebian: "0.100.0~branch~ferretdb", expectedDocker: &images{ developmentImages: []string{ - //"ferretdb/postgres-documentdb-dev:ferretdb", + "ferretdb/postgres-documentdb-dev:ferretdb", "ghcr.io/ferretdb/postgres-documentdb-dev:ferretdb", - //"quay.io/ferretdb/postgres-documentdb-dev:ferretdb", + "quay.io/ferretdb/postgres-documentdb-dev:ferretdb", }, }, }, @@ -297,9 +297,9 @@ func TestDefineVersion(t *testing.T) { expectedDebian: "0.100.0~branch~ferretdb", expectedDocker: &images{ developmentImages: []string{ - //"ferretdb/postgres-documentdb-dev:ferretdb", + "ferretdb/postgres-documentdb-dev:ferretdb", "ghcr.io/ferretdb/postgres-documentdb-dev:ferretdb", - //"quay.io/ferretdb/postgres-documentdb-dev:ferretdb", + "quay.io/ferretdb/postgres-documentdb-dev:ferretdb", }, }, }, From aa5e3104291e6b44720d5e8d8e4eb850d0301b99 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 4 Mar 2025 09:26:04 +0400 Subject: [PATCH 23/59] Add more Docker tags (#30) Co-authored-by: Chi Fujii --- ferretdb_packaging/defineversion/debian.go | 5 +- ferretdb_packaging/defineversion/docker.go | 60 ++++++++++--------- ferretdb_packaging/defineversion/main_test.go | 48 ++++++++++----- 3 files changed, 68 insertions(+), 45 deletions(-) diff --git a/ferretdb_packaging/defineversion/debian.go b/ferretdb_packaging/defineversion/debian.go index 0bde19a79..dc49c5856 100644 --- a/ferretdb_packaging/defineversion/debian.go +++ b/ferretdb_packaging/defineversion/debian.go @@ -119,6 +119,9 @@ func defineDebianVersionForTag(tag string) (string, error) { func debianSummary(action *githubactions.Action, version string) { output := fmt.Sprintf("Debian package version (`upstream_version` only): `%s`", version) - action.AddStepSummary(output) + // Only 3 summaries are shown in the GitHub Actions UI by default, + // and Docker summaries are more important (and include Debian version anyway). + // action.AddStepSummary(output) + action.Infof("%s", output) } diff --git a/ferretdb_packaging/defineversion/docker.go b/ferretdb_packaging/defineversion/docker.go index b97de0bc4..2d7b264a4 100644 --- a/ferretdb_packaging/defineversion/docker.go +++ b/ferretdb_packaging/defineversion/docker.go @@ -33,28 +33,17 @@ func defineDockerVersion(controlDefaultVersion, pgVersion string, getenv githuba switch event := getenv("GITHUB_EVENT_NAME"); event { case "pull_request", "pull_request_target": branch := strings.ToLower(getenv("GITHUB_HEAD_REF")) - res = defineDockerVersionForPR(owner, repo, branch) + res = defineDockerVersionForPR(pgVersion, owner, repo, branch) case "push", "schedule", "workflow_run": refName := strings.ToLower(getenv("GITHUB_REF_NAME")) switch refType := strings.ToLower(getenv("GITHUB_REF_TYPE")); refType { case "branch": - res, err = defineDockerVersionForBranch(owner, repo, refName) + res, err = defineDockerVersionForBranch(pgVersion, owner, repo, refName) case "tag": - var major, minor, patch int - var prerelease string - if major, minor, patch, prerelease, err = parseGitTag(refName); err != nil { - return nil, err - } - - tags := []string{ - fmt.Sprintf("%s-%d.%d.%d-%s", pgVersion, major, minor, patch, prerelease), - "latest", - } - - res = defineDockerVersionForTag(owner, repo, tags) + res, err = defineDockerVersionForTag(pgVersion, owner, repo, refName) default: err = fmt.Errorf("unhandled ref type %q for event %q", refType, event) @@ -78,15 +67,15 @@ func defineDockerVersion(controlDefaultVersion, pgVersion string, getenv githuba return res, nil } -// defineDockerVersionForPR defines Docker image names and tags for pull requests. -func defineDockerVersionForPR(owner, repo, branch string) *images { +// defineDockerVersionForPR defines Docker image names and tags for PR. +func defineDockerVersionForPR(pgVersion, owner, repo, branch string) *images { // for branches like "dependabot/submodules/XXX" parts := strings.Split(branch, "/") branch = parts[len(parts)-1] res := &images{ developmentImages: []string{ - fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:pr-%s", owner, repo, branch), + fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-pr-%s", owner, repo, pgVersion, branch), }, } @@ -95,15 +84,15 @@ func defineDockerVersionForPR(owner, repo, branch string) *images { return res } -// defineDockerVersionForBranch defines Docker image names and tags for branch builds. -func defineDockerVersionForBranch(owner, repo, branch string) (*images, error) { +// defineDockerVersionForBranch defines Docker image names and tags for branch. +func defineDockerVersionForBranch(pgVersion, owner, repo, branch string) (*images, error) { if branch != "ferretdb" { return nil, fmt.Errorf("unhandled branch %q", branch) } res := &images{ developmentImages: []string{ - fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:ferretdb", owner, repo), + fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-ferretdb", owner, repo, pgVersion), }, } @@ -117,15 +106,30 @@ func defineDockerVersionForBranch(owner, repo, branch string) (*images, error) { return res, nil } - res.developmentImages = append(res.developmentImages, "quay.io/ferretdb/postgres-documentdb-dev:ferretdb") - res.developmentImages = append(res.developmentImages, "ferretdb/postgres-documentdb-dev:ferretdb") + res.developmentImages = append(res.developmentImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb-dev:%s-ferretdb", pgVersion)) + res.developmentImages = append(res.developmentImages, fmt.Sprintf("ferretdb/postgres-documentdb-dev:%s-ferretdb", pgVersion)) return res, nil } -// defineDockerVersionForTag defines Docker image names and tags for prerelease tag builds. -func defineDockerVersionForTag(owner, repo string, tags []string) *images { - res := new(images) +// defineDockerVersionForBranch defines Docker image names and tags for tag. +func defineDockerVersionForTag(pgVersion, owner, repo, tag string) (*images, error) { + major, minor, patch, prerelease, err := parseGitTag(tag) + if err != nil { + return nil, err + } + + tags := []string{ + fmt.Sprintf("%s-%d.%d.%d-%s", pgVersion, major, minor, patch, prerelease), + fmt.Sprintf("%s-%d.%d.%d", pgVersion, major, minor, patch), + fmt.Sprintf("%s", pgVersion), + } + + if pgVersion == "17" { + tags = append(tags, "latest") + } + + var res images for _, t := range tags { res.developmentImages = append(res.developmentImages, fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s", owner, repo, t)) @@ -134,12 +138,12 @@ func defineDockerVersionForTag(owner, repo string, tags []string) *images { // forks don't have Quay.io and Docker Hub orgs if owner != "ferretdb" { - return res + return &res, nil } // we don't have Quay.io and Docker Hub repos for other GitHub repos if repo != "documentdb" { - return res + return &res, nil } for _, t := range tags { @@ -150,7 +154,7 @@ func defineDockerVersionForTag(owner, repo string, tags []string) *images { res.productionImages = append(res.productionImages, fmt.Sprintf("ferretdb/postgres-documentdb:%s", t)) } - return res + return &res, nil } // dockerImageURL returns HTML page URL for the given image name and tag. diff --git a/ferretdb_packaging/defineversion/main_test.go b/ferretdb_packaging/defineversion/main_test.go index 00862a2e4..6a1ef0c6b 100644 --- a/ferretdb_packaging/defineversion/main_test.go +++ b/ferretdb_packaging/defineversion/main_test.go @@ -87,7 +87,7 @@ func TestDefineVersion(t *testing.T) { expectedDebian: "0.100.0~pr~define~version", expectedDocker: &images{ developmentImages: []string{ - "ghcr.io/ferretdb/postgres-documentdb-dev:pr-define-version", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-pr-define-version", }, }, }, @@ -103,7 +103,7 @@ func TestDefineVersion(t *testing.T) { expectedDebian: "0.100.0~pr~define~version", expectedDocker: &images{ developmentImages: []string{ - "ghcr.io/otherorg/postgres-otherrepo-dev:pr-define-version", + "ghcr.io/otherorg/postgres-otherrepo-dev:17-pr-define-version", }, }, }, @@ -120,7 +120,7 @@ func TestDefineVersion(t *testing.T) { expectedDebian: "0.100.0~pr~define~version", expectedDocker: &images{ developmentImages: []string{ - "ghcr.io/ferretdb/postgres-documentdb-dev:pr-define-version", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-pr-define-version", }, }, }, @@ -136,7 +136,7 @@ func TestDefineVersion(t *testing.T) { expectedDebian: "0.100.0~pr~define~version", expectedDocker: &images{ developmentImages: []string{ - "ghcr.io/otherorg/postgres-otherrepo-dev:pr-define-version", + "ghcr.io/otherorg/postgres-otherrepo-dev:17-pr-define-version", }, }, }, @@ -153,9 +153,9 @@ func TestDefineVersion(t *testing.T) { expectedDebian: "0.100.0~branch~ferretdb", expectedDocker: &images{ developmentImages: []string{ - "ferretdb/postgres-documentdb-dev:ferretdb", - "ghcr.io/ferretdb/postgres-documentdb-dev:ferretdb", - "quay.io/ferretdb/postgres-documentdb-dev:ferretdb", + "ferretdb/postgres-documentdb-dev:17-ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb", + "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb", }, }, }, @@ -171,7 +171,7 @@ func TestDefineVersion(t *testing.T) { expectedDebian: "0.100.0~branch~ferretdb", expectedDocker: &images{ developmentImages: []string{ - "ghcr.io/otherorg/postgres-otherrepo-dev:ferretdb", + "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", }, }, }, @@ -211,18 +211,30 @@ func TestDefineVersion(t *testing.T) { expectedDebian: "0.100.0~ferretdb~2.0.0", expectedDocker: &images{ developmentImages: []string{ + "ferretdb/postgres-documentdb-dev:17", + "ferretdb/postgres-documentdb-dev:17-0.100.0", "ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb-2.0.0", "ferretdb/postgres-documentdb-dev:latest", + "ghcr.io/ferretdb/postgres-documentdb-dev:17", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0", "ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb-2.0.0", "ghcr.io/ferretdb/postgres-documentdb-dev:latest", + "quay.io/ferretdb/postgres-documentdb-dev:17", + "quay.io/ferretdb/postgres-documentdb-dev:17-0.100.0", "quay.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb-2.0.0", "quay.io/ferretdb/postgres-documentdb-dev:latest", }, productionImages: []string{ + "ferretdb/postgres-documentdb:17", + "ferretdb/postgres-documentdb:17-0.100.0", "ferretdb/postgres-documentdb:17-0.100.0-ferretdb-2.0.0", "ferretdb/postgres-documentdb:latest", + "ghcr.io/ferretdb/postgres-documentdb:17", + "ghcr.io/ferretdb/postgres-documentdb:17-0.100.0", "ghcr.io/ferretdb/postgres-documentdb:17-0.100.0-ferretdb-2.0.0", "ghcr.io/ferretdb/postgres-documentdb:latest", + "quay.io/ferretdb/postgres-documentdb:17", + "quay.io/ferretdb/postgres-documentdb:17-0.100.0", "quay.io/ferretdb/postgres-documentdb:17-0.100.0-ferretdb-2.0.0", "quay.io/ferretdb/postgres-documentdb:latest", }, @@ -240,10 +252,14 @@ func TestDefineVersion(t *testing.T) { expectedDebian: "0.100.0~ferretdb~2.0.0", expectedDocker: &images{ developmentImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17", + "ghcr.io/otherorg/postgres-otherrepo-dev:17-0.100.0", "ghcr.io/otherorg/postgres-otherrepo-dev:17-0.100.0-ferretdb-2.0.0", "ghcr.io/otherorg/postgres-otherrepo-dev:latest", }, productionImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo:17", + "ghcr.io/otherorg/postgres-otherrepo:17-0.100.0", "ghcr.io/otherorg/postgres-otherrepo:17-0.100.0-ferretdb-2.0.0", "ghcr.io/otherorg/postgres-otherrepo:latest", }, @@ -262,9 +278,9 @@ func TestDefineVersion(t *testing.T) { expectedDebian: "0.100.0~branch~ferretdb", expectedDocker: &images{ developmentImages: []string{ - "ferretdb/postgres-documentdb-dev:ferretdb", - "ghcr.io/ferretdb/postgres-documentdb-dev:ferretdb", - "quay.io/ferretdb/postgres-documentdb-dev:ferretdb", + "ferretdb/postgres-documentdb-dev:17-ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb", + "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb", }, }, }, @@ -280,7 +296,7 @@ func TestDefineVersion(t *testing.T) { expectedDebian: "0.100.0~branch~ferretdb", expectedDocker: &images{ developmentImages: []string{ - "ghcr.io/otherorg/postgres-otherrepo-dev:ferretdb", + "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", }, }, }, @@ -297,9 +313,9 @@ func TestDefineVersion(t *testing.T) { expectedDebian: "0.100.0~branch~ferretdb", expectedDocker: &images{ developmentImages: []string{ - "ferretdb/postgres-documentdb-dev:ferretdb", - "ghcr.io/ferretdb/postgres-documentdb-dev:ferretdb", - "quay.io/ferretdb/postgres-documentdb-dev:ferretdb", + "ferretdb/postgres-documentdb-dev:17-ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb", + "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb", }, }, }, @@ -315,7 +331,7 @@ func TestDefineVersion(t *testing.T) { expectedDebian: "0.100.0~branch~ferretdb", expectedDocker: &images{ developmentImages: []string{ - "ghcr.io/otherorg/postgres-otherrepo-dev:ferretdb", + "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", }, }, }, From 6092471c767694d2e88060d5bcc1339c8e6b5652 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 4 Mar 2025 10:04:28 +0400 Subject: [PATCH 24/59] Remove `branch` from Debian package versions (#32) --- ferretdb_packaging/defineversion/debian.go | 18 ++-------- ferretdb_packaging/defineversion/docker.go | 7 ++-- .../defineversion/docker_test.go | 6 +++- ferretdb_packaging/defineversion/main.go | 35 +++++++++++-------- ferretdb_packaging/defineversion/main_test.go | 12 +++---- 5 files changed, 39 insertions(+), 39 deletions(-) diff --git a/ferretdb_packaging/defineversion/debian.go b/ferretdb_packaging/defineversion/debian.go index dc49c5856..255afb1ec 100644 --- a/ferretdb_packaging/defineversion/debian.go +++ b/ferretdb_packaging/defineversion/debian.go @@ -95,12 +95,11 @@ func defineDebianVersionForPR(controlDefaultVersion, branch string) string { // defineDebianVersionForBranch returns valid Debian package version for branch. // See [defineDebianVersion]. func defineDebianVersionForBranch(controlDefaultVersion, branch string) (string, error) { - switch branch { - case "ferretdb": - return fmt.Sprintf("%s~branch~%s", controlDefaultVersion, branch), nil - default: + if branch != "ferretdb" { return "", fmt.Errorf("unhandled branch %q", branch) } + + return fmt.Sprintf("%s~ferretdb", controlDefaultVersion), nil } // defineDebianVersionForTag returns valid Debian package version for tag. @@ -114,14 +113,3 @@ func defineDebianVersionForTag(tag string) (string, error) { res := fmt.Sprintf("%d.%d.%d-%s", major, minor, patch, prerelease) return disallowedDebian.ReplaceAllString(res, "~"), nil } - -// debianSummary sets action summary. -func debianSummary(action *githubactions.Action, version string) { - output := fmt.Sprintf("Debian package version (`upstream_version` only): `%s`", version) - - // Only 3 summaries are shown in the GitHub Actions UI by default, - // and Docker summaries are more important (and include Debian version anyway). - // action.AddStepSummary(output) - - action.Infof("%s", output) -} diff --git a/ferretdb_packaging/defineversion/docker.go b/ferretdb_packaging/defineversion/docker.go index 2d7b264a4..5c59ffd4f 100644 --- a/ferretdb_packaging/defineversion/docker.go +++ b/ferretdb_packaging/defineversion/docker.go @@ -172,9 +172,12 @@ func dockerImageURL(name string) string { return fmt.Sprintf("https://hub.docker.com/r/%s/tags", name) } -// dockerSummary sets action summary. -func dockerSummary(action *githubactions.Action, version *images) { +// setSummary sets action summary. +func setSummary(action *githubactions.Action, debian string, version *images) { var buf strings.Builder + + fmt.Fprintf(&buf, "Debian package version (`upstream_version` only): `%s`\n\n", debian) + w := tabwriter.NewWriter(&buf, 1, 1, 1, ' ', tabwriter.Debug) fmt.Fprintf(w, "\tType\tDocker image\t\n") fmt.Fprintf(w, "\t----\t------------\t\n") diff --git a/ferretdb_packaging/defineversion/docker_test.go b/ferretdb_packaging/defineversion/docker_test.go index aaf1affd9..eb459de05 100644 --- a/ferretdb_packaging/defineversion/docker_test.go +++ b/ferretdb_packaging/defineversion/docker_test.go @@ -59,9 +59,11 @@ func TestDockerSummary(t *testing.T) { }, } - dockerSummary(action, result) + setSummary(action, "0.100.0~ferretdb", result) expectedStdout := strings.ReplaceAll(` +Debian package version ('upstream_version' only): '0.100.0~ferretdb' + |Type |Docker image | |---- |------------ | |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb) | @@ -73,6 +75,8 @@ func TestDockerSummary(t *testing.T) { assert.Equal(t, expectedStdout, stdout.String(), "stdout does not match") expectedSummary := strings.ReplaceAll(` +Debian package version ('upstream_version' only): '0.100.0~ferretdb' + |Type |Docker image | |---- |------------ | |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb) | diff --git a/ferretdb_packaging/defineversion/main.go b/ferretdb_packaging/defineversion/main.go index b171a3c32..db644cf5b 100644 --- a/ferretdb_packaging/defineversion/main.go +++ b/ferretdb_packaging/defineversion/main.go @@ -121,25 +121,30 @@ func main() { action.Fatalf("%s", err) } - debianSummary(action, debian) action.SetOutput("debian_version", debian) - if !*debianOnlyF { - dockerSummary(action, docker) + if *debianOnlyF { + // Only 3 summaries are shown in the GitHub Actions UI by default, + // and Docker summaries are more important (and include Debian version anyway). + output := fmt.Sprintf("Debian package version (`upstream_version` only): `%s`", debian) + action.Infof("%s", output) + return + } - action.SetOutput("docker_development_images", strings.Join(docker.developmentImages, ",")) - action.SetOutput("docker_production_images", strings.Join(docker.productionImages, ",")) + setSummary(action, debian, docker) - developmentTagFlags := make([]string, len(docker.developmentImages)) - for i, image := range docker.developmentImages { - developmentTagFlags[i] = fmt.Sprintf("--tag %s", image) - } - action.SetOutput("docker_development_tag_flags", strings.Join(developmentTagFlags, " ")) + action.SetOutput("docker_development_images", strings.Join(docker.developmentImages, ",")) + action.SetOutput("docker_production_images", strings.Join(docker.productionImages, ",")) - productionTagFlags := make([]string, len(docker.productionImages)) - for i, image := range docker.productionImages { - productionTagFlags[i] = fmt.Sprintf("--tag %s", image) - } - action.SetOutput("docker_production_tag_flags", strings.Join(productionTagFlags, " ")) + developmentTagFlags := make([]string, len(docker.developmentImages)) + for i, image := range docker.developmentImages { + developmentTagFlags[i] = fmt.Sprintf("--tag %s", image) + } + action.SetOutput("docker_development_tag_flags", strings.Join(developmentTagFlags, " ")) + + productionTagFlags := make([]string, len(docker.productionImages)) + for i, image := range docker.productionImages { + productionTagFlags[i] = fmt.Sprintf("--tag %s", image) } + action.SetOutput("docker_production_tag_flags", strings.Join(productionTagFlags, " ")) } diff --git a/ferretdb_packaging/defineversion/main_test.go b/ferretdb_packaging/defineversion/main_test.go index 6a1ef0c6b..b0d5af32f 100644 --- a/ferretdb_packaging/defineversion/main_test.go +++ b/ferretdb_packaging/defineversion/main_test.go @@ -150,7 +150,7 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, - expectedDebian: "0.100.0~branch~ferretdb", + expectedDebian: "0.100.0~ferretdb", expectedDocker: &images{ developmentImages: []string{ "ferretdb/postgres-documentdb-dev:17-ferretdb", @@ -168,7 +168,7 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, - expectedDebian: "0.100.0~branch~ferretdb", + expectedDebian: "0.100.0~ferretdb", expectedDocker: &images{ developmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", @@ -275,7 +275,7 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, - expectedDebian: "0.100.0~branch~ferretdb", + expectedDebian: "0.100.0~ferretdb", expectedDocker: &images{ developmentImages: []string{ "ferretdb/postgres-documentdb-dev:17-ferretdb", @@ -293,7 +293,7 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, - expectedDebian: "0.100.0~branch~ferretdb", + expectedDebian: "0.100.0~ferretdb", expectedDocker: &images{ developmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", @@ -310,7 +310,7 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, - expectedDebian: "0.100.0~branch~ferretdb", + expectedDebian: "0.100.0~ferretdb", expectedDocker: &images{ developmentImages: []string{ "ferretdb/postgres-documentdb-dev:17-ferretdb", @@ -328,7 +328,7 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, - expectedDebian: "0.100.0~branch~ferretdb", + expectedDebian: "0.100.0~ferretdb", expectedDocker: &images{ developmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", From 696c433617b9103d5577143404adf219350c3f22 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 4 Mar 2025 10:31:01 +0400 Subject: [PATCH 25/59] Prepare release (#33) --- .github/RELEASE_CHECKLIST.md | 15 ++++++++------- CHANGELOG.md | 10 ++++++++++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/RELEASE_CHECKLIST.md b/.github/RELEASE_CHECKLIST.md index 4f1cc394a..781443631 100644 --- a/.github/RELEASE_CHECKLIST.md +++ b/.github/RELEASE_CHECKLIST.md @@ -3,14 +3,14 @@ ## Preparation 1. Create draft release on GitHub to see a list of merged PRs. -2. Update CHANGELOG.md manually. -3. Push changes. +2. Update CHANGELOG.md manually. It will point to versions of DocumentDB and FerretDB that are not released yet. +3. Send PR with changes, merge it. ## Git tag -1. Make a signed tag `vX.Y.Z-ferretdb-A.B.C(-p)` (like `v0.102.0-ferretdb-2.0.0-rc.2`), +1. Make a signed tag `vX.Y.Z-ferretdb-A.B.C(-p)` (like `v0.102.0-ferretdb-2.0.0-rc.5`), where `X.Y.Z` is the SemVar formatted version of DocumentDB (like `0.102.0`), - and `A.B.C(-p)` is the compatible FerretDB version (like `2.0.0-rc.2`). + and `A.B.C(-p)` is the compatible FerretDB version (like `2.0.0-rc.5`). 2. Check `git status` output. 3. Push it! @@ -18,6 +18,7 @@ 1. Find [Packages CI build](https://github.com/FerretDB/documentdb/actions/workflows/ferretdb_packages.yml?query=event%3Apush) for the tag to release. -2. Upload `.deb` packages to the draft release. -3. Update release notes with the list of changes from CHANGELOG.md. -4. Publish release on GitHub. +2. Check Docker images. +3. Upload `.deb` packages to the draft release. +4. Update release notes with the content of CHANGELOG.md. +5. Publish release on GitHub. diff --git a/CHANGELOG.md b/CHANGELOG.md index bb7880c52..bc167680e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +### DocumentDB v0.102.0-ferretdb-2.0.0-rc.5 (March 4, 2025) ### + +This version works best with [FerretDB v2.0.0-rc.5](https://github.com/FerretDB/FerretDB/releases/tag/v2.0.0-rc.5). + +Debian and Ubuntu `.deb` packages are provided [here](https://github.com/FerretDB/documentdb/releases/tag/v0.102.0-ferretdb-2.0.0-rc.5). +Docker images are available [here](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). + +Please note that naming schemes for both `.deb` packages and Docker image tags have been changed. +We always recommend specifying the full version (e.g., `:17-0.102.0-ferretdb-2.0.0-rc.5`) to avoid unexpected updates. + ### DocumentDB v0.102.0-ferretdb-2.0.0-rc.2 (February 24, 2025) ### This version works best with [FerretDB v2.0.0-rc.2](https://github.com/FerretDB/FerretDB/releases/tag/v2.0.0-rc.2). From f7c318ca521b9ee94d255f8df77a54cad7c56c0b Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 5 Mar 2025 11:51:19 +0400 Subject: [PATCH 26/59] Prepare v0.102.0-ferretdb-2.0.0 (GA) release (#37) --- CHANGELOG.md | 8 ++++++++ ferretdb_packaging/go.mod | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc167680e..10444861b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +### DocumentDB v0.102.0-ferretdb-2.0.0 (GA) (March 5, 2025) ### + +This version works best with [FerretDB v2.0.0 (GA)](https://github.com/FerretDB/FerretDB/releases/tag/v2.0.0). + +Debian and Ubuntu `.deb` packages are provided [here](https://github.com/FerretDB/documentdb/releases/tag/v0.102.0-ferretdb-2.0.0). +Docker images are available [here](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). +We always recommend specifying the full version (e.g., `:17-0.102.0-ferretdb-2.0.0`) to avoid unexpected updates. + ### DocumentDB v0.102.0-ferretdb-2.0.0-rc.5 (March 4, 2025) ### This version works best with [FerretDB v2.0.0-rc.5](https://github.com/FerretDB/FerretDB/releases/tag/v2.0.0-rc.5). diff --git a/ferretdb_packaging/go.mod b/ferretdb_packaging/go.mod index 50a865286..c5aa597fa 100644 --- a/ferretdb_packaging/go.mod +++ b/ferretdb_packaging/go.mod @@ -2,7 +2,7 @@ module github.com/FerretDB/documentdb/ferretdb_packaging go 1.24 -toolchain go1.24.0 +toolchain go1.24.1 require ( github.com/sethvargo/go-githubactions v1.3.0 From 83660738424bd87cda26a0967d36e03b302a24c4 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Thu, 6 Mar 2025 20:31:29 +0900 Subject: [PATCH 27/59] Merge debian and docker version code (#35) Closes FerretDB/FerretDB#4725. --- ferretdb_packaging/defineversion/debian.go | 80 ------- ferretdb_packaging/defineversion/docker.go | 178 --------------- .../defineversion/docker_test.go | 65 ------ ferretdb_packaging/defineversion/main.go | 207 ++++++++++++++++-- ferretdb_packaging/defineversion/main_test.go | 155 +++++++++---- 5 files changed, 299 insertions(+), 386 deletions(-) diff --git a/ferretdb_packaging/defineversion/debian.go b/ferretdb_packaging/defineversion/debian.go index 255afb1ec..2b77396ea 100644 --- a/ferretdb_packaging/defineversion/debian.go +++ b/ferretdb_packaging/defineversion/debian.go @@ -4,9 +4,6 @@ import ( "fmt" "os" "regexp" - "strings" - - "github.com/sethvargo/go-githubactions" ) // controlDefaultVer matches major, minor and "patch" from `default_version` field in control file, @@ -36,80 +33,3 @@ func getControlDefaultVersion(f string) (string, error) { return fmt.Sprintf("%s.%s.%s", major, minor, patch), nil } - -// defineDebianVersion returns valid Debian package version, -// based on `default_version` in the control file and environment variables set by GitHub Actions. -// -// See https://www.debian.org/doc/debian-policy/ch-controlfields.html#version. -// We use `upstream_version` only. -// For that reason, we can't use `-`, so we replace it with `~`. -func defineDebianVersion(controlDefaultVersion, pgVersion string, getenv githubactions.GetenvFunc) (string, error) { - var res string - var err error - - switch event := getenv("GITHUB_EVENT_NAME"); event { - case "pull_request", "pull_request_target": - branch := strings.ToLower(getenv("GITHUB_HEAD_REF")) - res = defineDebianVersionForPR(controlDefaultVersion, branch) - - case "push", "schedule", "workflow_run": - refName := strings.ToLower(getenv("GITHUB_REF_NAME")) - - switch refType := strings.ToLower(getenv("GITHUB_REF_TYPE")); refType { - case "branch": - res, err = defineDebianVersionForBranch(controlDefaultVersion, refName) - - case "tag": - res, err = defineDebianVersionForTag(refName) - - default: - err = fmt.Errorf("unhandled ref type %q for event %q", refType, event) - } - - default: - err = fmt.Errorf("unhandled event type %q", event) - } - - if err != nil { - return "", err - } - - if res == "" { - return "", fmt.Errorf("both packageVersion and err are nil") - } - - return res, nil -} - -// defineDebianVersionForPR returns valid Debian package version for PR. -// See [defineDebianVersion]. -func defineDebianVersionForPR(controlDefaultVersion, branch string) string { - // for branches like "dependabot/submodules/XXX" - parts := strings.Split(branch, "/") - branch = parts[len(parts)-1] - res := fmt.Sprintf("%s-pr-%s", controlDefaultVersion, branch) - - return disallowedDebian.ReplaceAllString(res, "~") -} - -// defineDebianVersionForBranch returns valid Debian package version for branch. -// See [defineDebianVersion]. -func defineDebianVersionForBranch(controlDefaultVersion, branch string) (string, error) { - if branch != "ferretdb" { - return "", fmt.Errorf("unhandled branch %q", branch) - } - - return fmt.Sprintf("%s~ferretdb", controlDefaultVersion), nil -} - -// defineDebianVersionForTag returns valid Debian package version for tag. -// See [defineDebianVersion]. -func defineDebianVersionForTag(tag string) (string, error) { - major, minor, patch, prerelease, err := parseGitTag(tag) - if err != nil { - return "", err - } - - res := fmt.Sprintf("%d.%d.%d-%s", major, minor, patch, prerelease) - return disallowedDebian.ReplaceAllString(res, "~"), nil -} diff --git a/ferretdb_packaging/defineversion/docker.go b/ferretdb_packaging/defineversion/docker.go index 5c59ffd4f..ebe2c47e8 100644 --- a/ferretdb_packaging/defineversion/docker.go +++ b/ferretdb_packaging/defineversion/docker.go @@ -2,161 +2,9 @@ package main import ( "fmt" - "slices" "strings" - "text/tabwriter" - - "github.com/sethvargo/go-githubactions" ) -// images represents Docker image names and tags extracted from the environment. -type images struct { - developmentImages []string - productionImages []string -} - -// defineDockerVersion extracts Docker image names and tags from the environment variables defined by GitHub Actions. -func defineDockerVersion(controlDefaultVersion, pgVersion string, getenv githubactions.GetenvFunc) (*images, error) { - repo := getenv("GITHUB_REPOSITORY") - - // to support GitHub forks - parts := strings.Split(strings.ToLower(repo), "/") - if len(parts) != 2 { - return nil, fmt.Errorf("failed to split %q into owner and name", repo) - } - owner := parts[0] - repo = parts[1] - - var res *images - var err error - - switch event := getenv("GITHUB_EVENT_NAME"); event { - case "pull_request", "pull_request_target": - branch := strings.ToLower(getenv("GITHUB_HEAD_REF")) - res = defineDockerVersionForPR(pgVersion, owner, repo, branch) - - case "push", "schedule", "workflow_run": - refName := strings.ToLower(getenv("GITHUB_REF_NAME")) - - switch refType := strings.ToLower(getenv("GITHUB_REF_TYPE")); refType { - case "branch": - res, err = defineDockerVersionForBranch(pgVersion, owner, repo, refName) - - case "tag": - res, err = defineDockerVersionForTag(pgVersion, owner, repo, refName) - - default: - err = fmt.Errorf("unhandled ref type %q for event %q", refType, event) - } - - default: - err = fmt.Errorf("unhandled event type %q", event) - } - - if err != nil { - return nil, err - } - - if res == nil { - return nil, fmt.Errorf("both res and err are nil") - } - - slices.Sort(res.developmentImages) - slices.Sort(res.productionImages) - - return res, nil -} - -// defineDockerVersionForPR defines Docker image names and tags for PR. -func defineDockerVersionForPR(pgVersion, owner, repo, branch string) *images { - // for branches like "dependabot/submodules/XXX" - parts := strings.Split(branch, "/") - branch = parts[len(parts)-1] - - res := &images{ - developmentImages: []string{ - fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-pr-%s", owner, repo, pgVersion, branch), - }, - } - - // PRs are only for testing; no Quay.io and Docker Hub repos - - return res -} - -// defineDockerVersionForBranch defines Docker image names and tags for branch. -func defineDockerVersionForBranch(pgVersion, owner, repo, branch string) (*images, error) { - if branch != "ferretdb" { - return nil, fmt.Errorf("unhandled branch %q", branch) - } - - res := &images{ - developmentImages: []string{ - fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-ferretdb", owner, repo, pgVersion), - }, - } - - // forks don't have Quay.io and Docker Hub orgs - if owner != "ferretdb" { - return res, nil - } - - // we don't have Quay.io and Docker Hub repos for other GitHub repos - if repo != "documentdb" { - return res, nil - } - - res.developmentImages = append(res.developmentImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb-dev:%s-ferretdb", pgVersion)) - res.developmentImages = append(res.developmentImages, fmt.Sprintf("ferretdb/postgres-documentdb-dev:%s-ferretdb", pgVersion)) - - return res, nil -} - -// defineDockerVersionForBranch defines Docker image names and tags for tag. -func defineDockerVersionForTag(pgVersion, owner, repo, tag string) (*images, error) { - major, minor, patch, prerelease, err := parseGitTag(tag) - if err != nil { - return nil, err - } - - tags := []string{ - fmt.Sprintf("%s-%d.%d.%d-%s", pgVersion, major, minor, patch, prerelease), - fmt.Sprintf("%s-%d.%d.%d", pgVersion, major, minor, patch), - fmt.Sprintf("%s", pgVersion), - } - - if pgVersion == "17" { - tags = append(tags, "latest") - } - - var res images - - for _, t := range tags { - res.developmentImages = append(res.developmentImages, fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s", owner, repo, t)) - res.productionImages = append(res.productionImages, fmt.Sprintf("ghcr.io/%s/postgres-%s:%s", owner, repo, t)) - } - - // forks don't have Quay.io and Docker Hub orgs - if owner != "ferretdb" { - return &res, nil - } - - // we don't have Quay.io and Docker Hub repos for other GitHub repos - if repo != "documentdb" { - return &res, nil - } - - for _, t := range tags { - res.developmentImages = append(res.developmentImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb-dev:%s", t)) - res.productionImages = append(res.productionImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb:%s", t)) - - res.developmentImages = append(res.developmentImages, fmt.Sprintf("ferretdb/postgres-documentdb-dev:%s", t)) - res.productionImages = append(res.productionImages, fmt.Sprintf("ferretdb/postgres-documentdb:%s", t)) - } - - return &res, nil -} - // dockerImageURL returns HTML page URL for the given image name and tag. func dockerImageURL(name string) string { switch { @@ -171,29 +19,3 @@ func dockerImageURL(name string) string { // there is no easy way to get Docker Hub URL for the given tag return fmt.Sprintf("https://hub.docker.com/r/%s/tags", name) } - -// setSummary sets action summary. -func setSummary(action *githubactions.Action, debian string, version *images) { - var buf strings.Builder - - fmt.Fprintf(&buf, "Debian package version (`upstream_version` only): `%s`\n\n", debian) - - w := tabwriter.NewWriter(&buf, 1, 1, 1, ' ', tabwriter.Debug) - fmt.Fprintf(w, "\tType\tDocker image\t\n") - fmt.Fprintf(w, "\t----\t------------\t\n") - - for _, image := range version.developmentImages { - u := dockerImageURL(image) - _, _ = fmt.Fprintf(w, "\tDevelopment\t[`%s`](%s)\t\n", image, u) - } - - for _, image := range version.productionImages { - u := dockerImageURL(image) - _, _ = fmt.Fprintf(w, "\tProduction\t[`%s`](%s)\t\n", image, u) - } - - _ = w.Flush() - - action.AddStepSummary(buf.String()) - action.Infof("%s", buf.String()) -} diff --git a/ferretdb_packaging/defineversion/docker_test.go b/ferretdb_packaging/defineversion/docker_test.go index eb459de05..9911ba38b 100644 --- a/ferretdb_packaging/defineversion/docker_test.go +++ b/ferretdb_packaging/defineversion/docker_test.go @@ -1,15 +1,9 @@ package main import ( - "bytes" - "io" - "os" - "strings" "testing" - "github.com/sethvargo/go-githubactions" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestImageURL(t *testing.T) { @@ -30,62 +24,3 @@ func TestImageURL(t *testing.T) { dockerImageURL("ferretdb/postgres-documentdb-dev:pr-docker-tag"), ) } - -func TestDockerSummary(t *testing.T) { - dir := t.TempDir() - - summaryF, err := os.CreateTemp(dir, "summary") - require.NoError(t, err) - defer summaryF.Close() - - outputF, err := os.CreateTemp(dir, "output") - require.NoError(t, err) - defer outputF.Close() - - var stdout bytes.Buffer - getenv := getEnvFunc(t, map[string]string{ - "GITHUB_STEP_SUMMARY": summaryF.Name(), - "GITHUB_OUTPUT": outputF.Name(), - }) - action := githubactions.New(githubactions.WithGetenv(getenv), githubactions.WithWriter(&stdout)) - - result := &images{ - developmentImages: []string{ - "ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb", - "ghcr.io/ferretdb/postgres-documentdb-dev:latest", - }, - productionImages: []string{ - "quay.io/ferretdb/postgres-documentdb:latest", - }, - } - - setSummary(action, "0.100.0~ferretdb", result) - - expectedStdout := strings.ReplaceAll(` -Debian package version ('upstream_version' only): '0.100.0~ferretdb' - - |Type |Docker image | - |---- |------------ | - |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb) | - |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:latest'](https://ghcr.io/ferretdb/postgres-documentdb-dev:latest) | - |Production |['quay.io/ferretdb/postgres-documentdb:latest'](https://quay.io/ferretdb/postgres-documentdb:latest) | - -`[1:], "'", "`", - ) - assert.Equal(t, expectedStdout, stdout.String(), "stdout does not match") - - expectedSummary := strings.ReplaceAll(` -Debian package version ('upstream_version' only): '0.100.0~ferretdb' - - |Type |Docker image | - |---- |------------ | - |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb) | - |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:latest'](https://ghcr.io/ferretdb/postgres-documentdb-dev:latest) | - |Production |['quay.io/ferretdb/postgres-documentdb:latest'](https://quay.io/ferretdb/postgres-documentdb:latest) | - -`[1:], "'", "`", - ) - b, err := io.ReadAll(summaryF) - require.NoError(t, err) - assert.Equal(t, expectedSummary, string(b), "summary does not match") -} diff --git a/ferretdb_packaging/defineversion/main.go b/ferretdb_packaging/defineversion/main.go index db644cf5b..714111ccb 100644 --- a/ferretdb_packaging/defineversion/main.go +++ b/ferretdb_packaging/defineversion/main.go @@ -9,6 +9,7 @@ import ( "slices" "strconv" "strings" + "text/tabwriter" "github.com/sethvargo/go-githubactions" ) @@ -17,6 +18,13 @@ import ( // but with a leading `v`. var semVerTag = regexp.MustCompile(`^v(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`) +// versions represents Docker image names and tags, and Debian package version. +type versions struct { + dockerDevelopmentImages []string + dockerProductionImages []string + debian string +} + // parseGitTag parses git tag in specific format and returns SemVer components. // // Expected format is v0.100.0-ferretdb-2.0.0-rc.2, @@ -74,19 +82,184 @@ func debugEnv(action *githubactions.Action) { } } -// defineVersion returns Debian package version and Docker tags. -func defineVersion(controlDefaultVersion, pgVersion string, getenv githubactions.GetenvFunc) (string, *images, error) { - debian, err := defineDebianVersion(controlDefaultVersion, pgVersion, getenv) +// defineVersion extracts Docker image names and tags, and Debian package version using the environment variables defined by GitHub Actions. +// +// The Debian package version is based on `default_version` in the control file. +// See https://www.debian.org/doc/debian-policy/ch-controlfields.html#version. +// We use `upstream_version` only. +// For that reason, we can't use `-`, so we replace it with `~`. +func defineVersion(controlDefaultVersion, pgVersion string, getenv githubactions.GetenvFunc) (*versions, error) { + repo := getenv("GITHUB_REPOSITORY") + + // to support GitHub forks + parts := strings.Split(strings.ToLower(repo), "/") + if len(parts) != 2 { + return nil, fmt.Errorf("failed to split %q into owner and name", repo) + } + owner := parts[0] + repo = parts[1] + + var res *versions + var err error + + switch event := getenv("GITHUB_EVENT_NAME"); event { + case "pull_request", "pull_request_target": + branch := strings.ToLower(getenv("GITHUB_HEAD_REF")) + res = defineVersionForPR(controlDefaultVersion, pgVersion, owner, repo, branch) + + case "push", "schedule", "workflow_run": + refName := strings.ToLower(getenv("GITHUB_REF_NAME")) + + switch refType := strings.ToLower(getenv("GITHUB_REF_TYPE")); refType { + case "branch": + res, err = defineVersionForBranch(controlDefaultVersion, pgVersion, owner, repo, refName) + + case "tag": + res, err = defineVersionForTag(pgVersion, owner, repo, refName) + + default: + err = fmt.Errorf("unhandled ref type %q for event %q", refType, event) + } + + default: + err = fmt.Errorf("unhandled event type %q", event) + } + if err != nil { - return "", nil, err + return nil, err } - docker, err := defineDockerVersion(controlDefaultVersion, pgVersion, getenv) + if res == nil { + return nil, fmt.Errorf("both res and err are nil") + } + + slices.Sort(res.dockerDevelopmentImages) + slices.Sort(res.dockerProductionImages) + + return res, nil +} + +// defineVersionForPR defines Docker image names and tags, and Debian package version for PR. +// See [defineVersion]. +func defineVersionForPR(controlDefaultVersion, pgVersion, owner, repo, branch string) *versions { + // for branches like "dependabot/submodules/XXX" + parts := strings.Split(branch, "/") + branch = parts[len(parts)-1] + + res := &versions{ + dockerDevelopmentImages: []string{ + fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-pr-%s", owner, repo, pgVersion, branch), + }, + debian: disallowedDebian.ReplaceAllString(fmt.Sprintf("%s-pr-%s", controlDefaultVersion, branch), "~"), + } + + // PRs are only for testing; no Quay.io and Docker Hub repos + + return res +} + +// defineVersionForBranch defines Docker image names and tags, and Debian package version for branch. +// See [defineVersion]. +func defineVersionForBranch(controlDefaultVersion, pgVersion, owner, repo, branch string) (*versions, error) { + if branch != "ferretdb" { + return nil, fmt.Errorf("unhandled branch %q", branch) + } + + res := &versions{ + dockerDevelopmentImages: []string{ + fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-ferretdb", owner, repo, pgVersion), + }, + debian: fmt.Sprintf("%s~ferretdb", controlDefaultVersion), + } + + // forks don't have Quay.io and Docker Hub orgs + if owner != "ferretdb" { + return res, nil + } + + // we don't have Quay.io and Docker Hub repos for other GitHub repos + if repo != "documentdb" { + return res, nil + } + + res.dockerDevelopmentImages = append(res.dockerDevelopmentImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb-dev:%s-ferretdb", pgVersion)) + res.dockerDevelopmentImages = append(res.dockerDevelopmentImages, fmt.Sprintf("ferretdb/postgres-documentdb-dev:%s-ferretdb", pgVersion)) + + return res, nil +} + +// defineVersionForTag defines Docker image names and tags, and Debian package version for tag. +// See [defineVersion]. +func defineVersionForTag(pgVersion, owner, repo, tag string) (*versions, error) { + major, minor, patch, prerelease, err := parseGitTag(tag) if err != nil { - return "", nil, err + return nil, err + } + + tags := []string{ + fmt.Sprintf("%s-%d.%d.%d-%s", pgVersion, major, minor, patch, prerelease), + fmt.Sprintf("%s-%d.%d.%d", pgVersion, major, minor, patch), + fmt.Sprintf("%s", pgVersion), + } + + if pgVersion == "17" { + tags = append(tags, "latest") + } + + res := versions{ + debian: disallowedDebian.ReplaceAllString(fmt.Sprintf("%d.%d.%d-%s", major, minor, patch, prerelease), "~"), + } + + for _, t := range tags { + res.dockerDevelopmentImages = append(res.dockerDevelopmentImages, fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s", owner, repo, t)) + res.dockerProductionImages = append(res.dockerProductionImages, fmt.Sprintf("ghcr.io/%s/postgres-%s:%s", owner, repo, t)) + } + + // forks don't have Quay.io and Docker Hub orgs + if owner != "ferretdb" { + return &res, nil } - return debian, docker, nil + // we don't have Quay.io and Docker Hub repos for other GitHub repos + if repo != "documentdb" { + return &res, nil + } + + for _, t := range tags { + res.dockerDevelopmentImages = append(res.dockerDevelopmentImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb-dev:%s", t)) + res.dockerProductionImages = append(res.dockerProductionImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb:%s", t)) + + res.dockerDevelopmentImages = append(res.dockerDevelopmentImages, fmt.Sprintf("ferretdb/postgres-documentdb-dev:%s", t)) + res.dockerProductionImages = append(res.dockerProductionImages, fmt.Sprintf("ferretdb/postgres-documentdb:%s", t)) + } + + return &res, nil +} + +// setSummary sets action summary. +func setSummary(action *githubactions.Action, version *versions) { + var buf strings.Builder + + fmt.Fprintf(&buf, "Debian package version (`upstream_version` only): `%s`\n\n", version.debian) + + w := tabwriter.NewWriter(&buf, 1, 1, 1, ' ', tabwriter.Debug) + fmt.Fprintf(w, "\tType\tDocker image\t\n") + fmt.Fprintf(w, "\t----\t------------\t\n") + + for _, image := range version.dockerDevelopmentImages { + u := dockerImageURL(image) + _, _ = fmt.Fprintf(w, "\tDevelopment\t[`%s`](%s)\t\n", image, u) + } + + for _, image := range version.dockerProductionImages { + u := dockerImageURL(image) + _, _ = fmt.Fprintf(w, "\tProduction\t[`%s`](%s)\t\n", image, u) + } + + _ = w.Flush() + + action.AddStepSummary(buf.String()) + action.Infof("%s", buf.String()) } func main() { @@ -116,34 +289,34 @@ func main() { action.Fatalf("%s", err) } - debian, docker, err := defineVersion(controlDefaultVersion, *pgVersionF, action.Getenv) + res, err := defineVersion(controlDefaultVersion, *pgVersionF, action.Getenv) if err != nil { action.Fatalf("%s", err) } - action.SetOutput("debian_version", debian) + action.SetOutput("debian_version", res.debian) if *debianOnlyF { // Only 3 summaries are shown in the GitHub Actions UI by default, // and Docker summaries are more important (and include Debian version anyway). - output := fmt.Sprintf("Debian package version (`upstream_version` only): `%s`", debian) + output := fmt.Sprintf("Debian package version (`upstream_version` only): `%s`", res.debian) action.Infof("%s", output) return } - setSummary(action, debian, docker) + setSummary(action, res) - action.SetOutput("docker_development_images", strings.Join(docker.developmentImages, ",")) - action.SetOutput("docker_production_images", strings.Join(docker.productionImages, ",")) + action.SetOutput("docker_development_images", strings.Join(res.dockerDevelopmentImages, ",")) + action.SetOutput("docker_production_images", strings.Join(res.dockerProductionImages, ",")) - developmentTagFlags := make([]string, len(docker.developmentImages)) - for i, image := range docker.developmentImages { + developmentTagFlags := make([]string, len(res.dockerDevelopmentImages)) + for i, image := range res.dockerDevelopmentImages { developmentTagFlags[i] = fmt.Sprintf("--tag %s", image) } action.SetOutput("docker_development_tag_flags", strings.Join(developmentTagFlags, " ")) - productionTagFlags := make([]string, len(docker.productionImages)) - for i, image := range docker.productionImages { + productionTagFlags := make([]string, len(res.dockerProductionImages)) + for i, image := range res.dockerProductionImages { productionTagFlags[i] = fmt.Sprintf("--tag %s", image) } action.SetOutput("docker_production_tag_flags", strings.Join(productionTagFlags, " ")) diff --git a/ferretdb_packaging/defineversion/main_test.go b/ferretdb_packaging/defineversion/main_test.go index b0d5af32f..03564af97 100644 --- a/ferretdb_packaging/defineversion/main_test.go +++ b/ferretdb_packaging/defineversion/main_test.go @@ -1,9 +1,14 @@ package main import ( + "bytes" "fmt" + "io" + "os" + "strings" "testing" + "github.com/sethvargo/go-githubactions" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -70,10 +75,9 @@ func TestDefineVersion(t *testing.T) { ) for name, tc := range map[string]struct { - env map[string]string - expectedDebian string - expectedDocker *images - expectedErr error + env map[string]string + expected *versions + expectedErr error }{ "pull_request": { env: map[string]string{ @@ -84,11 +88,11 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, - expectedDebian: "0.100.0~pr~define~version", - expectedDocker: &images{ - developmentImages: []string{ + expected: &versions{ + dockerDevelopmentImages: []string{ "ghcr.io/ferretdb/postgres-documentdb-dev:17-pr-define-version", }, + debian: "0.100.0~pr~define~version", }, }, "pull_request-other": { @@ -100,11 +104,11 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, - expectedDebian: "0.100.0~pr~define~version", - expectedDocker: &images{ - developmentImages: []string{ + expected: &versions{ + dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-pr-define-version", }, + debian: "0.100.0~pr~define~version", }, }, @@ -117,11 +121,11 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, - expectedDebian: "0.100.0~pr~define~version", - expectedDocker: &images{ - developmentImages: []string{ + expected: &versions{ + dockerDevelopmentImages: []string{ "ghcr.io/ferretdb/postgres-documentdb-dev:17-pr-define-version", }, + debian: "0.100.0~pr~define~version", }, }, "pull_request_target-other": { @@ -133,11 +137,11 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, - expectedDebian: "0.100.0~pr~define~version", - expectedDocker: &images{ - developmentImages: []string{ + expected: &versions{ + dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-pr-define-version", }, + debian: "0.100.0~pr~define~version", }, }, @@ -150,13 +154,13 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, - expectedDebian: "0.100.0~ferretdb", - expectedDocker: &images{ - developmentImages: []string{ + expected: &versions{ + dockerDevelopmentImages: []string{ "ferretdb/postgres-documentdb-dev:17-ferretdb", "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb", "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb", }, + debian: "0.100.0~ferretdb", }, }, "push/ferretdb-other": { @@ -168,11 +172,11 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, - expectedDebian: "0.100.0~ferretdb", - expectedDocker: &images{ - developmentImages: []string{ + expected: &versions{ + dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", }, + debian: "0.100.0~ferretdb", }, }, @@ -208,9 +212,8 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "tag", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, - expectedDebian: "0.100.0~ferretdb~2.0.0", - expectedDocker: &images{ - developmentImages: []string{ + expected: &versions{ + dockerDevelopmentImages: []string{ "ferretdb/postgres-documentdb-dev:17", "ferretdb/postgres-documentdb-dev:17-0.100.0", "ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb-2.0.0", @@ -224,7 +227,7 @@ func TestDefineVersion(t *testing.T) { "quay.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb-2.0.0", "quay.io/ferretdb/postgres-documentdb-dev:latest", }, - productionImages: []string{ + dockerProductionImages: []string{ "ferretdb/postgres-documentdb:17", "ferretdb/postgres-documentdb:17-0.100.0", "ferretdb/postgres-documentdb:17-0.100.0-ferretdb-2.0.0", @@ -238,6 +241,7 @@ func TestDefineVersion(t *testing.T) { "quay.io/ferretdb/postgres-documentdb:17-0.100.0-ferretdb-2.0.0", "quay.io/ferretdb/postgres-documentdb:latest", }, + debian: "0.100.0~ferretdb~2.0.0", }, }, "push/tag/release-other": { @@ -249,20 +253,20 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "tag", "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, - expectedDebian: "0.100.0~ferretdb~2.0.0", - expectedDocker: &images{ - developmentImages: []string{ + expected: &versions{ + dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17", "ghcr.io/otherorg/postgres-otherrepo-dev:17-0.100.0", "ghcr.io/otherorg/postgres-otherrepo-dev:17-0.100.0-ferretdb-2.0.0", "ghcr.io/otherorg/postgres-otherrepo-dev:latest", }, - productionImages: []string{ + dockerProductionImages: []string{ "ghcr.io/otherorg/postgres-otherrepo:17", "ghcr.io/otherorg/postgres-otherrepo:17-0.100.0", "ghcr.io/otherorg/postgres-otherrepo:17-0.100.0-ferretdb-2.0.0", "ghcr.io/otherorg/postgres-otherrepo:latest", }, + debian: "0.100.0~ferretdb~2.0.0", }, }, @@ -275,13 +279,13 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, - expectedDebian: "0.100.0~ferretdb", - expectedDocker: &images{ - developmentImages: []string{ + expected: &versions{ + dockerDevelopmentImages: []string{ "ferretdb/postgres-documentdb-dev:17-ferretdb", "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb", "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb", }, + debian: "0.100.0~ferretdb", }, }, "schedule-other": { @@ -293,11 +297,11 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, - expectedDebian: "0.100.0~ferretdb", - expectedDocker: &images{ - developmentImages: []string{ + expected: &versions{ + dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", }, + debian: "0.100.0~ferretdb", }, }, @@ -310,13 +314,13 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, - expectedDebian: "0.100.0~ferretdb", - expectedDocker: &images{ - developmentImages: []string{ + expected: &versions{ + dockerDevelopmentImages: []string{ "ferretdb/postgres-documentdb-dev:17-ferretdb", "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb", "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb", }, + debian: "0.100.0~ferretdb", }, }, "workflow_run-other": { @@ -328,17 +332,17 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, - expectedDebian: "0.100.0~ferretdb", - expectedDocker: &images{ - developmentImages: []string{ + expected: &versions{ + dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", }, + debian: "0.100.0~ferretdb", }, }, } { t.Run(name, func(t *testing.T) { - debian, docker, err := defineVersion(controlDefaultVersion, pgVersion, getEnvFunc(t, tc.env)) - if tc.expectedDebian == "" && tc.expectedDocker == nil { + docker, err := defineVersion(controlDefaultVersion, pgVersion, getEnvFunc(t, tc.env)) + if tc.expected == nil { require.Error(t, tc.expectedErr) require.Equal(t, err, tc.expectedErr) return @@ -347,8 +351,67 @@ func TestDefineVersion(t *testing.T) { require.NoError(t, tc.expectedErr) require.NoError(t, err) - assert.Equal(t, tc.expectedDebian, debian) - assert.Equal(t, tc.expectedDocker, docker) + assert.Equal(t, tc.expected, docker) }) } } + +func TestSummary(t *testing.T) { + dir := t.TempDir() + + summaryF, err := os.CreateTemp(dir, "summary") + require.NoError(t, err) + defer summaryF.Close() + + outputF, err := os.CreateTemp(dir, "output") + require.NoError(t, err) + defer outputF.Close() + + var stdout bytes.Buffer + getenv := getEnvFunc(t, map[string]string{ + "GITHUB_STEP_SUMMARY": summaryF.Name(), + "GITHUB_OUTPUT": outputF.Name(), + }) + action := githubactions.New(githubactions.WithGetenv(getenv), githubactions.WithWriter(&stdout)) + + result := &versions{ + dockerDevelopmentImages: []string{ + "ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:latest", + }, + dockerProductionImages: []string{ + "quay.io/ferretdb/postgres-documentdb:latest", + }, + debian: "0.100.0~ferretdb", + } + + setSummary(action, result) + + expectedStdout := strings.ReplaceAll(` +Debian package version ('upstream_version' only): '0.100.0~ferretdb' + + |Type |Docker image | + |---- |------------ | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb) | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:latest'](https://ghcr.io/ferretdb/postgres-documentdb-dev:latest) | + |Production |['quay.io/ferretdb/postgres-documentdb:latest'](https://quay.io/ferretdb/postgres-documentdb:latest) | + +`[1:], "'", "`", + ) + assert.Equal(t, expectedStdout, stdout.String(), "stdout does not match") + + expectedSummary := strings.ReplaceAll(` +Debian package version ('upstream_version' only): '0.100.0~ferretdb' + + |Type |Docker image | + |---- |------------ | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb) | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:latest'](https://ghcr.io/ferretdb/postgres-documentdb-dev:latest) | + |Production |['quay.io/ferretdb/postgres-documentdb:latest'](https://quay.io/ferretdb/postgres-documentdb:latest) | + +`[1:], "'", "`", + ) + b, err := io.ReadAll(summaryF) + require.NoError(t, err) + assert.Equal(t, expectedSummary, string(b), "summary does not match") +} From b5ed8920f488533fa862d5569346e069d1eb5313 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Fri, 7 Mar 2025 17:31:41 +0900 Subject: [PATCH 28/59] Check control file default version (#36) Closes FerretDB/FerretDB#4725. --- ferretdb_packaging/defineversion/main.go | 15 ++++++---- ferretdb_packaging/defineversion/main_test.go | 30 ++++++++++++++----- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/ferretdb_packaging/defineversion/main.go b/ferretdb_packaging/defineversion/main.go index 714111ccb..022f95ca7 100644 --- a/ferretdb_packaging/defineversion/main.go +++ b/ferretdb_packaging/defineversion/main.go @@ -115,7 +115,7 @@ func defineVersion(controlDefaultVersion, pgVersion string, getenv githubactions res, err = defineVersionForBranch(controlDefaultVersion, pgVersion, owner, repo, refName) case "tag": - res, err = defineVersionForTag(pgVersion, owner, repo, refName) + res, err = defineVersionForTag(controlDefaultVersion, pgVersion, owner, repo, refName) default: err = fmt.Errorf("unhandled ref type %q for event %q", refType, event) @@ -190,15 +190,20 @@ func defineVersionForBranch(controlDefaultVersion, pgVersion, owner, repo, branc // defineVersionForTag defines Docker image names and tags, and Debian package version for tag. // See [defineVersion]. -func defineVersionForTag(pgVersion, owner, repo, tag string) (*versions, error) { +func defineVersionForTag(controlDefaultVersion, pgVersion, owner, repo, tag string) (*versions, error) { major, minor, patch, prerelease, err := parseGitTag(tag) if err != nil { return nil, err } + tagVersion := fmt.Sprintf("%d.%d.%d", major, minor, patch) + if tagVersion != controlDefaultVersion { + return nil, fmt.Errorf("git tag version %q does not match the control file default version %q", tagVersion, controlDefaultVersion) + } + tags := []string{ - fmt.Sprintf("%s-%d.%d.%d-%s", pgVersion, major, minor, patch, prerelease), - fmt.Sprintf("%s-%d.%d.%d", pgVersion, major, minor, patch), + fmt.Sprintf("%s-%s-%s", pgVersion, tagVersion, prerelease), + fmt.Sprintf("%s-%s", pgVersion, tagVersion), fmt.Sprintf("%s", pgVersion), } @@ -207,7 +212,7 @@ func defineVersionForTag(pgVersion, owner, repo, tag string) (*versions, error) } res := versions{ - debian: disallowedDebian.ReplaceAllString(fmt.Sprintf("%d.%d.%d-%s", major, minor, patch, prerelease), "~"), + debian: disallowedDebian.ReplaceAllString(fmt.Sprintf("%s-%s", tagVersion, prerelease), "~"), } for _, t := range tags { diff --git a/ferretdb_packaging/defineversion/main_test.go b/ferretdb_packaging/defineversion/main_test.go index 03564af97..d14c42ff4 100644 --- a/ferretdb_packaging/defineversion/main_test.go +++ b/ferretdb_packaging/defineversion/main_test.go @@ -69,15 +69,13 @@ func TestParseGitTag(t *testing.T) { } func TestDefineVersion(t *testing.T) { - const ( - controlDefaultVersion = "0.100.0" - pgVersion = "17" - ) + const pgVersion = "17" for name, tc := range map[string]struct { - env map[string]string - expected *versions - expectedErr error + env map[string]string + controlDefaultVersion string // defaults to "0.100.0" + expected *versions + expectedErr error }{ "pull_request": { env: map[string]string{ @@ -270,6 +268,19 @@ func TestDefineVersion(t *testing.T) { }, }, + "push/tag/release-wrong-control-default-version": { + env: map[string]string{ + "GITHUB_BASE_REF": "", + "GITHUB_EVENT_NAME": "push", + "GITHUB_HEAD_REF": "", + "GITHUB_REF_NAME": "v0.100.0-ferretdb-2.0.0", + "GITHUB_REF_TYPE": "tag", + "GITHUB_REPOSITORY": "FerretDB/documentdb", + }, + controlDefaultVersion: "0.101.0", + expectedErr: fmt.Errorf(`git tag version "0.100.0" does not match the control file default version "0.101.0"`), + }, + "schedule": { env: map[string]string{ "GITHUB_BASE_REF": "", @@ -341,6 +352,11 @@ func TestDefineVersion(t *testing.T) { }, } { t.Run(name, func(t *testing.T) { + controlDefaultVersion := tc.controlDefaultVersion + if controlDefaultVersion == "" { + controlDefaultVersion = "0.100.0" + } + docker, err := defineVersion(controlDefaultVersion, pgVersion, getEnvFunc(t, tc.env)) if tc.expected == nil { require.Error(t, tc.expectedErr) From 9129ad566167b05f9096923dba9372eb751a650f Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 25 Mar 2025 12:49:28 +0400 Subject: [PATCH 29/59] Fix handling of explicit `maxTimeMS` zero values (#41) --- pg_documentdb/src/commands/commands_common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pg_documentdb/src/commands/commands_common.c b/pg_documentdb/src/commands/commands_common.c index 9a3cd631c..7e8171c2f 100644 --- a/pg_documentdb/src/commands/commands_common.c +++ b/pg_documentdb/src/commands/commands_common.c @@ -173,7 +173,7 @@ IsCommonSpecIgnoredField(const char *fieldName) void SetExplicitStatementTimeout(int timeoutMilliseconds) { - if (!EnableBackendStatementTimeout) + if (!EnableBackendStatementTimeout || timeoutMilliseconds <= 0) { return; } From 9c521ebb2506532593f4ee172da287b00ce4fbae Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 26 Mar 2025 20:13:32 +0400 Subject: [PATCH 30/59] Fix `listDatabases` and related commands Closes FerretDB/FerretDB#4983. --- pg_documentdb/src/commands/db_stats.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pg_documentdb/src/commands/db_stats.c b/pg_documentdb/src/commands/db_stats.c index c54e8f480..6d6694497 100644 --- a/pg_documentdb/src/commands/db_stats.c +++ b/pg_documentdb/src/commands/db_stats.c @@ -178,7 +178,7 @@ command_list_databases(PG_FUNCTION_ARGS) { sizeOnDiskSelector = ", 0::int8 AS \"sizeOnDisk\", false AS empty"; totalSizeSelector = - "pg_catalog.pg_database_size(pg_catalog.current_database())::int4 AS \"totalSize\", "; + "pg_catalog.pg_database_size(pg_catalog.current_database())::int8 AS \"totalSize\", "; } if (filter != NULL) From 80462f5ce84701d281d5081d11bc54079c0d82c5 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 2 Apr 2025 17:43:05 +0400 Subject: [PATCH 31/59] Prepare v0.102.0-ferretdb-2.1.0 release (#44) --- .github/workflows/regress_tests.yml | 1 + CHANGELOG.md | 44 ++++++++++++++++------------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/.github/workflows/regress_tests.yml b/.github/workflows/regress_tests.yml index c8f8e70ba..04a27d8c4 100644 --- a/.github/workflows/regress_tests.yml +++ b/.github/workflows/regress_tests.yml @@ -9,6 +9,7 @@ on: push: branches: - 'main' + - 'ferretdb' paths-ignore: - 'docs/**' - '.devcontainer/**' diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c92a16b9..c06830b08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,29 +1,35 @@ -### DocumentDB v0.102.0-ferretdb-2.0.0 (GA) (March 5, 2025) ### - -This version works best with [FerretDB v2.0.0 (GA)](https://github.com/FerretDB/FerretDB/releases/tag/v2.0.0). +### DocumentDB v0.102.0-ferretdb-2.1.0 (April 2, 2025) ### -Debian and Ubuntu `.deb` packages are provided [here](https://github.com/FerretDB/documentdb/releases/tag/v0.102.0-ferretdb-2.0.0). -Docker images are available [here](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). -We always recommend specifying the full version (e.g., `:17-0.102.0-ferretdb-2.0.0`) to avoid unexpected updates. +> [!CAUTION] +> Please note that due to incompatibilities in our previous releases, they can't be updated in place, +> even with a manual `ALTER EXTENSION UPDATE` query or other means. +> A new clean installation into an empty data directory/volume is required. +> All data should be backed up with `mongodump`/`mongoexport` before +> and restored with `mongorestore`/`mongoimport` after. +> +> We expect future updates to be much smoother. -### DocumentDB v0.102.0-ferretdb-2.0.0-rc.5 (March 4, 2025) ### +This version works best with the upcoming FerretDB v2.1.0. -This version works best with [FerretDB v2.0.0-rc.5](https://github.com/FerretDB/FerretDB/releases/tag/v2.0.0-rc.5). +Debian and Ubuntu `.deb` packages are provided [on the release page](https://github.com/FerretDB/documentdb/releases/tag/v0.102.0-ferretdb-2.1.0). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/deb/). -Debian and Ubuntu `.deb` packages are provided [here](https://github.com/FerretDB/documentdb/releases/tag/v0.102.0-ferretdb-2.0.0-rc.5). -Docker images are available [here](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). +Docker images are available [in the registry](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/docker/). +We always recommend specifying the full image tag (e.g., `17-0.102.0-ferretdb-2.1.0`, not just `17` or `17-0.102.0`) to avoid unexpected updates. -Please note that naming schemes for both `.deb` packages and Docker image tags have been changed. -We always recommend specifying the full version (e.g., `:17-0.102.0-ferretdb-2.0.0-rc.5`) to avoid unexpected updates. +### DocumentDB v0.102.0-ferretdb-2.0.0 (GA) (March 5, 2025) ### -### DocumentDB v0.102.0-ferretdb-2.0.0-rc.2 (February 24, 2025) ### +This version works best with [FerretDB v2.0.0 (GA)](https://github.com/FerretDB/FerretDB/releases/tag/v2.0.0). -This version works best with [FerretDB v2.0.0-rc.2](https://github.com/FerretDB/FerretDB/releases/tag/v2.0.0-rc.2). +Debian and Ubuntu `.deb` packages are provided [on the release page](https://github.com/FerretDB/documentdb/releases/tag/v0.102.0-ferretdb-2.0.0). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/deb/). -Debian and Ubuntu `.deb` packages are now [provided](https://github.com/FerretDB/documentdb/releases/tag/v0.102.0-ferretdb-2.0.0-rc.2). -Docker images are available [here](https://github.com/FerretDB/FerretDB/pkgs/container/postgres-documentdb). +Docker images are available [in the registry](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/docker/). +We always recommend specifying the full image tag (e.g., `17-0.102.0-ferretdb-2.0.0`, not just `17` or `17-0.102.0`) to avoid unexpected updates. -### documentdb v0.102-0 (Unreleased) ### +### documentdb v0.102-0 (March 26, 2025) ### * Support index pushdown for vector search queries *[Bugfix]* * Support exact search for vector search queries *[Feature]* * Inline $match with let in $lookup pipelines as JOIN Filter *[Perf]* @@ -34,11 +40,11 @@ Docker images are available [here](https://github.com/FerretDB/FerretDB/pkgs/con * Disable analyze statistics for unique index uuid columns which improves resource usage *[Perf]* * Support collation with `$expr`, `$in`, `$cmp`, `$eq`, `$ne`, `$lt`, `$lte`, `$gt`, `$gte` comparison operators (Opt-in) *[Feature]* * Support collation in `find`, aggregation `$project`, `$redact`, `$set`, `$addFields`, `$replaceRoot` stages (Opt-in) *[Feature]* -* Support collation with `$setEquals`, `$setUnion`, `$setIntersection`, `$setDifference`, `$setIsSubet` in the aggregation pipeline (Opt-in) *[Feature]* +* Support collation with `$setEquals`, `$setUnion`, `$setIntersection`, `$setDifference`, `$setIsSubset` in the aggregation pipeline (Opt-in) *[Feature]* * Support unique index truncation by default with new operator class *[Feature]* * Top level aggregate command `let` variables support for `$geoNear` stage *[Feature]* * Enable Backend Command support for Statement Timeout *[Feature]* -* Support type aggregation operator `$toUUID`. *[Feature]* +* Support type aggregation operator `$toUUID`. *[Feature]* * Support Partial filter pushdown for `$in` predicates *[Perf]* * Support the $dateFromString operator with full functionality *[Feature]* * Support extended syntax for `$getField` aggregation operator. Now the value of 'field' could be an expression that resolves to a string. *[Feature]* From 6106bc9930e0ae1b02ebbb439c909e8e6ea3d407 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Mon, 7 Apr 2025 08:57:29 +0400 Subject: [PATCH 32/59] Install `sudo` --- .../Dockerfile_build_deb_packages | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/ferretdb_packaging/Dockerfile_build_deb_packages b/ferretdb_packaging/Dockerfile_build_deb_packages index b55b1a93b..1705e0d84 100644 --- a/ferretdb_packaging/Dockerfile_build_deb_packages +++ b/ferretdb_packaging/Dockerfile_build_deb_packages @@ -10,11 +10,11 @@ RUN test -n "$DOCUMENTDB_VERSION" || (echo "DOCUMENTDB_VERSION not set" && false RUN apt-get update RUN apt-get install -y --no-install-recommends \ - wget \ + ca-certificates \ gnupg2 \ + locales \ lsb-release \ - ca-certificates \ - locales + wget RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ locale-gen en_US.UTF-8 @@ -29,24 +29,25 @@ RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg ma wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - RUN apt-get update && apt-get install -y --no-install-recommends \ - curl \ - tzdata \ build-essential \ - pkg-config \ cmake \ + curl \ + debhelper \ + devscripts \ + dpkg-dev \ git \ - postgresql-${POSTGRES_VERSION} \ - postgresql-server-dev-${POSTGRES_VERSION} \ - libpq-dev \ libicu-dev \ libkrb5-dev \ + libpq-dev \ + pkg-config \ + postgresql-${POSTGRES_VERSION} \ postgresql-${POSTGRES_VERSION}-cron \ postgresql-${POSTGRES_VERSION}-pgvector \ postgresql-${POSTGRES_VERSION}-postgis-3 \ postgresql-${POSTGRES_VERSION}-rum \ - devscripts \ - debhelper \ - dpkg-dev \ + postgresql-server-dev-${POSTGRES_VERSION} \ + sudo \ + tzdata \ && rm -rf /var/lib/apt/lists/* COPY scripts /tmp/install_setup From 7486bb97ed0ea7921d4b545cdd07fb991de22a68 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 24 Apr 2025 18:46:07 +0400 Subject: [PATCH 33/59] Enable more feature toggles (#51) --- .github/RELEASE_CHECKLIST.md | 6 +- ferretdb_packaging/10-preload.sh | 18 ++- ferretdb_packaging/defineversion/debian.go | 2 +- .../defineversion/debian_test.go | 4 +- ferretdb_packaging/defineversion/main.go | 6 +- ferretdb_packaging/defineversion/main_test.go | 119 ++++++++++-------- ferretdb_packaging/go.mod | 2 +- 7 files changed, 90 insertions(+), 67 deletions(-) diff --git a/.github/RELEASE_CHECKLIST.md b/.github/RELEASE_CHECKLIST.md index 781443631..c656d50b5 100644 --- a/.github/RELEASE_CHECKLIST.md +++ b/.github/RELEASE_CHECKLIST.md @@ -8,9 +8,9 @@ ## Git tag -1. Make a signed tag `vX.Y.Z-ferretdb-A.B.C(-p)` (like `v0.102.0-ferretdb-2.0.0-rc.5`), - where `X.Y.Z` is the SemVar formatted version of DocumentDB (like `0.102.0`), - and `A.B.C(-p)` is the compatible FerretDB version (like `2.0.0-rc.5`). +1. Make a signed tag with `git tag -s --cleanup=verbatim vX.Y.Z-ferretdb-A.B.C(-p)` (like `v0.103.0-ferretdb-2.2.0-beta.1`), + where `X.Y.Z` is the SemVar formatted version of DocumentDB (like `0.103.0`), + and `A.B.C(-p)` is the compatible FerretDB version (like `2.2.0-beta.1`). 2. Check `git status` output. 3. Push it! diff --git a/ferretdb_packaging/10-preload.sh b/ferretdb_packaging/10-preload.sh index f1cff5645..7e1e9eba6 100755 --- a/ferretdb_packaging/10-preload.sh +++ b/ferretdb_packaging/10-preload.sh @@ -2,8 +2,22 @@ set -e -echo "shared_preload_libraries = 'pg_cron,pg_documentdb_core,pg_documentdb'" >> $PGDATA/postgresql.conf -echo "cron.database_name = 'postgres'" >> $PGDATA/postgresql.conf +# https://github.com/microsoft/documentdb/tree/main/pg_documentdb/src/configs + +cat <> $PGDATA/postgresql.conf +shared_preload_libraries = 'pg_cron,pg_documentdb_core,pg_documentdb' +cron.database_name = 'postgres' + +documentdb.enableLetAndCollationForQueryMatch = true +documentdb.enableNowSystemVariable = true +documentdb.enableSortbyIdPushDownToPrimaryKey = true + +documentdb.enableSchemaValidation = true +documentdb.enableBypassDocumentValidation = true + +documentdb.enableUserCrud = true +documentdb.maxUserLimit = 100 +EOT source /usr/local/bin/docker-entrypoint.sh diff --git a/ferretdb_packaging/defineversion/debian.go b/ferretdb_packaging/defineversion/debian.go index 2b77396ea..6cb0191f8 100644 --- a/ferretdb_packaging/defineversion/debian.go +++ b/ferretdb_packaging/defineversion/debian.go @@ -15,7 +15,7 @@ var controlDefaultVer = regexp.MustCompile(`(?m)^default_version = '(?P[0 var disallowedDebian = regexp.MustCompile(`[^A-Za-z0-9\.+~]`) // getControlDefaultVersion returns the default_version field from the control file -// in SemVer format (0.100-0 -> 0.100.0). +// in SemVer format (0.103-0 -> 0.103.0). func getControlDefaultVersion(f string) (string, error) { b, err := os.ReadFile(f) if err != nil { diff --git a/ferretdb_packaging/defineversion/debian_test.go b/ferretdb_packaging/defineversion/debian_test.go index f8f2cd855..c98e6f1bc 100644 --- a/ferretdb_packaging/defineversion/debian_test.go +++ b/ferretdb_packaging/defineversion/debian_test.go @@ -15,7 +15,7 @@ func TestReadControlDefaultVersion(t *testing.T) { defer controlF.Close() //nolint:errcheck // temporary file for testing buf := `comment = 'API surface for DocumentDB for PostgreSQL' -default_version = '0.100-0' +default_version = '0.103-0' module_pathname = '$libdir/pg_documentdb' relocatable = false superuser = true @@ -26,5 +26,5 @@ requires = 'documentdb_core, pg_cron, tsm_system_rows, vector, postgis, rum'` controlDefaultVersion, err := getControlDefaultVersion(controlF.Name()) require.NoError(t, err) - require.Equal(t, "0.100.0", controlDefaultVersion) + require.Equal(t, "0.103.0", controlDefaultVersion) } diff --git a/ferretdb_packaging/defineversion/main.go b/ferretdb_packaging/defineversion/main.go index 022f95ca7..fd7a8b4e4 100644 --- a/ferretdb_packaging/defineversion/main.go +++ b/ferretdb_packaging/defineversion/main.go @@ -27,9 +27,9 @@ type versions struct { // parseGitTag parses git tag in specific format and returns SemVer components. // -// Expected format is v0.100.0-ferretdb-2.0.0-rc.2, -// where v0.100.0 is a DocumentDB version (0.100-0 -> 0.100.0), -// and ferretdb-2.0.0-rc.2 is a compatible FerretDB version. +// Expected format is v0.103.0-ferretdb-2.2.0-beta.1, +// where v0.103.0 is a DocumentDB version (0.103-0 -> 0.103.0), +// and ferretdb-2.2.0-beta.1 is a compatible FerretDB version. func parseGitTag(tag string) (major, minor, patch int, prerelease string, err error) { match := semVerTag.FindStringSubmatch(tag) if match == nil || len(match) != semVerTag.NumSubexp()+1 { diff --git a/ferretdb_packaging/defineversion/main_test.go b/ferretdb_packaging/defineversion/main_test.go index d14c42ff4..6dba699af 100644 --- a/ferretdb_packaging/defineversion/main_test.go +++ b/ferretdb_packaging/defineversion/main_test.go @@ -33,19 +33,19 @@ func TestParseGitTag(t *testing.T) { prerelease string err string }{ - "v0.100.0-ferretdb-2.0.0": { + "v0.103.0-ferretdb-2.2.0-beta.1": { major: 0, - minor: 100, + minor: 103, patch: 0, - prerelease: "ferretdb-2.0.0", + prerelease: "ferretdb-2.2.0-beta.1", }, - "0.100.0-ferretdb-2.0.0": { - err: `unexpected git tag format "0.100.0-ferretdb-2.0.0"`, + "0.103.0-ferretdb-2.2.0-beta.1": { + err: `unexpected git tag format "0.103.0-ferretdb-2.2.0-beta.1"`, }, - "v0.100.0-ferretdb": { + "v0.103.0-ferretdb": { err: `prerelease "ferretdb" should start with 'ferretdb-'`, }, - "v0.100.0": { + "v0.103.0": { err: `prerelease "" should start with 'ferretdb-'`, }, } @@ -73,7 +73,7 @@ func TestDefineVersion(t *testing.T) { for name, tc := range map[string]struct { env map[string]string - controlDefaultVersion string // defaults to "0.100.0" + controlDefaultVersion string expected *versions expectedErr error }{ @@ -86,11 +86,12 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, + controlDefaultVersion: "0.103.0", expected: &versions{ dockerDevelopmentImages: []string{ "ghcr.io/ferretdb/postgres-documentdb-dev:17-pr-define-version", }, - debian: "0.100.0~pr~define~version", + debian: "0.103.0~pr~define~version", }, }, "pull_request-other": { @@ -102,11 +103,12 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, + controlDefaultVersion: "0.103.0", expected: &versions{ dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-pr-define-version", }, - debian: "0.100.0~pr~define~version", + debian: "0.103.0~pr~define~version", }, }, @@ -119,11 +121,12 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, + controlDefaultVersion: "0.103.0", expected: &versions{ dockerDevelopmentImages: []string{ "ghcr.io/ferretdb/postgres-documentdb-dev:17-pr-define-version", }, - debian: "0.100.0~pr~define~version", + debian: "0.103.0~pr~define~version", }, }, "pull_request_target-other": { @@ -135,11 +138,12 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, + controlDefaultVersion: "0.103.0", expected: &versions{ dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-pr-define-version", }, - debian: "0.100.0~pr~define~version", + debian: "0.103.0~pr~define~version", }, }, @@ -152,13 +156,14 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, + controlDefaultVersion: "0.103.0", expected: &versions{ dockerDevelopmentImages: []string{ "ferretdb/postgres-documentdb-dev:17-ferretdb", "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb", "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb", }, - debian: "0.100.0~ferretdb", + debian: "0.103.0~ferretdb", }, }, "push/ferretdb-other": { @@ -170,11 +175,12 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, + controlDefaultVersion: "0.103.0", expected: &versions{ dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", }, - debian: "0.100.0~ferretdb", + debian: "0.103.0~ferretdb", }, }, @@ -187,7 +193,8 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, - expectedErr: fmt.Errorf(`unhandled branch "main"`), + controlDefaultVersion: "0.103.0", + expectedErr: fmt.Errorf(`unhandled branch "main"`), }, "push/main-other": { env: map[string]string{ @@ -198,7 +205,8 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, - expectedErr: fmt.Errorf(`unhandled branch "main"`), + controlDefaultVersion: "0.103.0", + expectedErr: fmt.Errorf(`unhandled branch "main"`), }, "push/tag/release": { @@ -206,40 +214,41 @@ func TestDefineVersion(t *testing.T) { "GITHUB_BASE_REF": "", "GITHUB_EVENT_NAME": "push", "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "v0.100.0-ferretdb-2.0.0", + "GITHUB_REF_NAME": "v0.103.0-ferretdb-2.2.0-beta.1", "GITHUB_REF_TYPE": "tag", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, + controlDefaultVersion: "0.103.0", expected: &versions{ dockerDevelopmentImages: []string{ "ferretdb/postgres-documentdb-dev:17", - "ferretdb/postgres-documentdb-dev:17-0.100.0", - "ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb-2.0.0", + "ferretdb/postgres-documentdb-dev:17-0.103.0", + "ferretdb/postgres-documentdb-dev:17-0.103.0-ferretdb-2.2.0-beta.1", "ferretdb/postgres-documentdb-dev:latest", "ghcr.io/ferretdb/postgres-documentdb-dev:17", - "ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0", - "ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb-2.0.0", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-0.103.0", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-0.103.0-ferretdb-2.2.0-beta.1", "ghcr.io/ferretdb/postgres-documentdb-dev:latest", "quay.io/ferretdb/postgres-documentdb-dev:17", - "quay.io/ferretdb/postgres-documentdb-dev:17-0.100.0", - "quay.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb-2.0.0", + "quay.io/ferretdb/postgres-documentdb-dev:17-0.103.0", + "quay.io/ferretdb/postgres-documentdb-dev:17-0.103.0-ferretdb-2.2.0-beta.1", "quay.io/ferretdb/postgres-documentdb-dev:latest", }, dockerProductionImages: []string{ "ferretdb/postgres-documentdb:17", - "ferretdb/postgres-documentdb:17-0.100.0", - "ferretdb/postgres-documentdb:17-0.100.0-ferretdb-2.0.0", + "ferretdb/postgres-documentdb:17-0.103.0", + "ferretdb/postgres-documentdb:17-0.103.0-ferretdb-2.2.0-beta.1", "ferretdb/postgres-documentdb:latest", "ghcr.io/ferretdb/postgres-documentdb:17", - "ghcr.io/ferretdb/postgres-documentdb:17-0.100.0", - "ghcr.io/ferretdb/postgres-documentdb:17-0.100.0-ferretdb-2.0.0", + "ghcr.io/ferretdb/postgres-documentdb:17-0.103.0", + "ghcr.io/ferretdb/postgres-documentdb:17-0.103.0-ferretdb-2.2.0-beta.1", "ghcr.io/ferretdb/postgres-documentdb:latest", "quay.io/ferretdb/postgres-documentdb:17", - "quay.io/ferretdb/postgres-documentdb:17-0.100.0", - "quay.io/ferretdb/postgres-documentdb:17-0.100.0-ferretdb-2.0.0", + "quay.io/ferretdb/postgres-documentdb:17-0.103.0", + "quay.io/ferretdb/postgres-documentdb:17-0.103.0-ferretdb-2.2.0-beta.1", "quay.io/ferretdb/postgres-documentdb:latest", }, - debian: "0.100.0~ferretdb~2.0.0", + debian: "0.103.0~ferretdb~2.2.0~beta.1", }, }, "push/tag/release-other": { @@ -247,24 +256,25 @@ func TestDefineVersion(t *testing.T) { "GITHUB_BASE_REF": "", "GITHUB_EVENT_NAME": "push", "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "v0.100.0-ferretdb-2.0.0", + "GITHUB_REF_NAME": "v0.103.0-ferretdb-2.2.0-beta.1", "GITHUB_REF_TYPE": "tag", "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, + controlDefaultVersion: "0.103.0", expected: &versions{ dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17", - "ghcr.io/otherorg/postgres-otherrepo-dev:17-0.100.0", - "ghcr.io/otherorg/postgres-otherrepo-dev:17-0.100.0-ferretdb-2.0.0", + "ghcr.io/otherorg/postgres-otherrepo-dev:17-0.103.0", + "ghcr.io/otherorg/postgres-otherrepo-dev:17-0.103.0-ferretdb-2.2.0-beta.1", "ghcr.io/otherorg/postgres-otherrepo-dev:latest", }, dockerProductionImages: []string{ "ghcr.io/otherorg/postgres-otherrepo:17", - "ghcr.io/otherorg/postgres-otherrepo:17-0.100.0", - "ghcr.io/otherorg/postgres-otherrepo:17-0.100.0-ferretdb-2.0.0", + "ghcr.io/otherorg/postgres-otherrepo:17-0.103.0", + "ghcr.io/otherorg/postgres-otherrepo:17-0.103.0-ferretdb-2.2.0-beta.1", "ghcr.io/otherorg/postgres-otherrepo:latest", }, - debian: "0.100.0~ferretdb~2.0.0", + debian: "0.103.0~ferretdb~2.2.0~beta.1", }, }, @@ -273,12 +283,12 @@ func TestDefineVersion(t *testing.T) { "GITHUB_BASE_REF": "", "GITHUB_EVENT_NAME": "push", "GITHUB_HEAD_REF": "", - "GITHUB_REF_NAME": "v0.100.0-ferretdb-2.0.0", + "GITHUB_REF_NAME": "v0.103.0-ferretdb-2.2.0-beta.1", "GITHUB_REF_TYPE": "tag", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, - controlDefaultVersion: "0.101.0", - expectedErr: fmt.Errorf(`git tag version "0.100.0" does not match the control file default version "0.101.0"`), + controlDefaultVersion: "0.104.0", + expectedErr: fmt.Errorf(`git tag version "0.103.0" does not match the control file default version "0.104.0"`), }, "schedule": { @@ -290,13 +300,14 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, + controlDefaultVersion: "0.103.0", expected: &versions{ dockerDevelopmentImages: []string{ "ferretdb/postgres-documentdb-dev:17-ferretdb", "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb", "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb", }, - debian: "0.100.0~ferretdb", + debian: "0.103.0~ferretdb", }, }, "schedule-other": { @@ -308,11 +319,12 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, + controlDefaultVersion: "0.103.0", expected: &versions{ dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", }, - debian: "0.100.0~ferretdb", + debian: "0.103.0~ferretdb", }, }, @@ -325,13 +337,14 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "FerretDB/documentdb", }, + controlDefaultVersion: "0.103.0", expected: &versions{ dockerDevelopmentImages: []string{ "ferretdb/postgres-documentdb-dev:17-ferretdb", "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb", "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb", }, - debian: "0.100.0~ferretdb", + debian: "0.103.0~ferretdb", }, }, "workflow_run-other": { @@ -343,21 +356,17 @@ func TestDefineVersion(t *testing.T) { "GITHUB_REF_TYPE": "branch", "GITHUB_REPOSITORY": "OtherOrg/OtherRepo", }, + controlDefaultVersion: "0.103.0", expected: &versions{ dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", }, - debian: "0.100.0~ferretdb", + debian: "0.103.0~ferretdb", }, }, } { t.Run(name, func(t *testing.T) { - controlDefaultVersion := tc.controlDefaultVersion - if controlDefaultVersion == "" { - controlDefaultVersion = "0.100.0" - } - - docker, err := defineVersion(controlDefaultVersion, pgVersion, getEnvFunc(t, tc.env)) + docker, err := defineVersion(tc.controlDefaultVersion, pgVersion, getEnvFunc(t, tc.env)) if tc.expected == nil { require.Error(t, tc.expectedErr) require.Equal(t, err, tc.expectedErr) @@ -392,23 +401,23 @@ func TestSummary(t *testing.T) { result := &versions{ dockerDevelopmentImages: []string{ - "ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-0.103.0-ferretdb", "ghcr.io/ferretdb/postgres-documentdb-dev:latest", }, dockerProductionImages: []string{ "quay.io/ferretdb/postgres-documentdb:latest", }, - debian: "0.100.0~ferretdb", + debian: "0.103.0~ferretdb", } setSummary(action, result) expectedStdout := strings.ReplaceAll(` -Debian package version ('upstream_version' only): '0.100.0~ferretdb' +Debian package version ('upstream_version' only): '0.103.0~ferretdb' |Type |Docker image | |---- |------------ | - |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb) | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:17-0.103.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:17-0.103.0-ferretdb) | |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:latest'](https://ghcr.io/ferretdb/postgres-documentdb-dev:latest) | |Production |['quay.io/ferretdb/postgres-documentdb:latest'](https://quay.io/ferretdb/postgres-documentdb:latest) | @@ -417,11 +426,11 @@ Debian package version ('upstream_version' only): '0.100.0~ferretdb' assert.Equal(t, expectedStdout, stdout.String(), "stdout does not match") expectedSummary := strings.ReplaceAll(` -Debian package version ('upstream_version' only): '0.100.0~ferretdb' +Debian package version ('upstream_version' only): '0.103.0~ferretdb' |Type |Docker image | |---- |------------ | - |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:17-0.100.0-ferretdb) | + |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:17-0.103.0-ferretdb'](https://ghcr.io/ferretdb/postgres-documentdb-dev:17-0.103.0-ferretdb) | |Development |['ghcr.io/ferretdb/postgres-documentdb-dev:latest'](https://ghcr.io/ferretdb/postgres-documentdb-dev:latest) | |Production |['quay.io/ferretdb/postgres-documentdb:latest'](https://quay.io/ferretdb/postgres-documentdb:latest) | diff --git a/ferretdb_packaging/go.mod b/ferretdb_packaging/go.mod index c5aa597fa..e6b36d58c 100644 --- a/ferretdb_packaging/go.mod +++ b/ferretdb_packaging/go.mod @@ -2,7 +2,7 @@ module github.com/FerretDB/documentdb/ferretdb_packaging go 1.24 -toolchain go1.24.1 +toolchain go1.24.2 require ( github.com/sethvargo/go-githubactions v1.3.0 From 73c2395a3cafd9731c087fcf1ce4db862af1f5b4 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 24 Apr 2025 18:49:30 +0400 Subject: [PATCH 34/59] Remove label that does nothing --- .github/settings.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/settings.yml b/.github/settings.yml index 974cae3f9..9edfd2909 100644 --- a/.github/settings.yml +++ b/.github/settings.yml @@ -21,10 +21,6 @@ labels: color: "#D4C5F9" description: PRs that update dependencies - - name: do not merge - color: "#0052CC" - description: PRs that should not be merged - - name: not ready color: "#000000" description: Issues that are not ready to be worked on; PRs that should skip CI From d89f70b8a808f0a7ffae89d9156e5f61e673c115 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Mon, 28 Apr 2025 20:19:55 +0400 Subject: [PATCH 35/59] Normalize whitespace in build scripts --- packaging/Dockerfile_build_deb_packages | 2 +- packaging/README.md | 2 +- packaging/build_packages.sh | 2 +- packaging/debian_files/changelog | 2 +- packaging/debian_files/compat | 2 +- packaging/debian_files/control | 2 +- packaging/debian_files/rules | 2 +- packaging/packaging-entrypoint.sh | 2 +- packaging/test_packages/Dockerfile_test_install_deb_packages | 2 +- packaging/test_packages/test-install-entrypoint.sh | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packaging/Dockerfile_build_deb_packages b/packaging/Dockerfile_build_deb_packages index 8e5cdbe1e..505f7d93f 100755 --- a/packaging/Dockerfile_build_deb_packages +++ b/packaging/Dockerfile_build_deb_packages @@ -77,4 +77,4 @@ RUN sed -i "s/DOCUMENTDB_VERSION/${DOCUMENTDB_VERSION}/g" /build/debian/changelo COPY packaging/packaging-entrypoint.sh /usr/local/bin/packaging-entrypoint.sh # Set the entrypoint -ENTRYPOINT ["packaging-entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["packaging-entrypoint.sh"] diff --git a/packaging/README.md b/packaging/README.md index 9d94009e6..8bdb04021 100755 --- a/packaging/README.md +++ b/packaging/README.md @@ -9,4 +9,4 @@ E.g. to build for Debian 12 and PostgreSQL 16, run: Packages can be found at the `packages` directory by default, but it can be configured with the `--output-dir` option. -**Note:** The packages do not include pg_documentdb_distributed in the `internal` directory. \ No newline at end of file +**Note:** The packages do not include pg_documentdb_distributed in the `internal` directory. diff --git a/packaging/build_packages.sh b/packaging/build_packages.sh index 72f266a37..b52a53ae0 100755 --- a/packaging/build_packages.sh +++ b/packaging/build_packages.sh @@ -156,4 +156,4 @@ if [[ $TEST_CLEAN_INSTALL == true ]]; then echo "Clean installation test successful!!" fi -echo "Packages are available in $abs_output_dir" \ No newline at end of file +echo "Packages are available in $abs_output_dir" diff --git a/packaging/debian_files/changelog b/packaging/debian_files/changelog index 56a7c8779..23203bb3e 100755 --- a/packaging/debian_files/changelog +++ b/packaging/debian_files/changelog @@ -44,4 +44,4 @@ documentdb (0.100-0) unstable; urgency=medium * Initial release - -- Shuai Tian Thu, 23 Jan 2025 10:00:00 +0000 \ No newline at end of file + -- Shuai Tian Thu, 23 Jan 2025 10:00:00 +0000 diff --git a/packaging/debian_files/compat b/packaging/debian_files/compat index 3cacc0b93..48082f72f 100755 --- a/packaging/debian_files/compat +++ b/packaging/debian_files/compat @@ -1 +1 @@ -12 \ No newline at end of file +12 diff --git a/packaging/debian_files/control b/packaging/debian_files/control index 959ba2e7e..087917b9b 100755 --- a/packaging/debian_files/control +++ b/packaging/debian_files/control @@ -7,4 +7,4 @@ Maintainer: Shuai Tian Package: postgresql-POSTGRES_VERSION-documentdb Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends}, postgresql-POSTGRES_VERSION, postgresql-POSTGRES_VERSION-cron, postgresql-POSTGRES_VERSION-pgvector, postgresql-POSTGRES_VERSION-postgis-3, postgresql-POSTGRES_VERSION-rum -Description: DocumentDB is the open-source engine powering vCore-based Azure Cosmos DB for MongoDB. It offers a native implementation of document-oriented NoSQL database, enabling seamless CRUD operations on BSON data types within a PostgreSQL framework. \ No newline at end of file +Description: DocumentDB is the open-source engine powering vCore-based Azure Cosmos DB for MongoDB. It offers a native implementation of document-oriented NoSQL database, enabling seamless CRUD operations on BSON data types within a PostgreSQL framework. diff --git a/packaging/debian_files/rules b/packaging/debian_files/rules index 3478d0506..363dd034d 100755 --- a/packaging/debian_files/rules +++ b/packaging/debian_files/rules @@ -7,4 +7,4 @@ override_dh_auto_test: make install adduser --disabled-password --gecos "" documentdb chown -R documentdb:documentdb . - su documentdb -c "make check" \ No newline at end of file + su documentdb -c "make check" diff --git a/packaging/packaging-entrypoint.sh b/packaging/packaging-entrypoint.sh index 2b0b5fb1e..0bd083629 100755 --- a/packaging/packaging-entrypoint.sh +++ b/packaging/packaging-entrypoint.sh @@ -27,4 +27,4 @@ mkdir -p /output cp *.deb /output/ # Change ownership of the output files to match the host user's UID and GID -chown -R $(stat -c "%u:%g" /output) /output \ No newline at end of file +chown -R $(stat -c "%u:%g" /output) /output diff --git a/packaging/test_packages/Dockerfile_test_install_deb_packages b/packaging/test_packages/Dockerfile_test_install_deb_packages index e52cd6543..c0792fc4b 100755 --- a/packaging/test_packages/Dockerfile_test_install_deb_packages +++ b/packaging/test_packages/Dockerfile_test_install_deb_packages @@ -44,4 +44,4 @@ RUN dpkg -i ${DEB_PACKAGE_REL_PATH} COPY packaging/test_packages/test-install-entrypoint.sh /usr/local/bin/test-install-entrypoint.sh -ENTRYPOINT ["test-install-entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["test-install-entrypoint.sh"] diff --git a/packaging/test_packages/test-install-entrypoint.sh b/packaging/test_packages/test-install-entrypoint.sh index a894e6e28..fd22dfc86 100755 --- a/packaging/test_packages/test-install-entrypoint.sh +++ b/packaging/test_packages/test-install-entrypoint.sh @@ -10,4 +10,4 @@ sed -i '/internal/d' Makefile # Run the test adduser --disabled-password --gecos "" documentdb chown -R documentdb:documentdb . -su documentdb -c "make check" \ No newline at end of file +su documentdb -c "make check" From b66a82ba7696bfb3b6bcab251f383762e639016d Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 29 Apr 2025 07:17:25 +0400 Subject: [PATCH 36/59] Unify `debian_files` --- .github/RELEASE_CHECKLIST.md | 3 ++- ferretdb_packaging/debian_files/changelog | 5 ----- ferretdb_packaging/debian_files/compat | 1 - ferretdb_packaging/debian_files/control | 10 ---------- ferretdb_packaging/debian_files/rules | 10 ---------- packaging/debian_files/changelog | 8 ++++---- packaging/debian_files/control | 2 +- 7 files changed, 7 insertions(+), 32 deletions(-) delete mode 100644 ferretdb_packaging/debian_files/changelog delete mode 100644 ferretdb_packaging/debian_files/compat delete mode 100644 ferretdb_packaging/debian_files/control delete mode 100755 ferretdb_packaging/debian_files/rules diff --git a/.github/RELEASE_CHECKLIST.md b/.github/RELEASE_CHECKLIST.md index c656d50b5..250b8a667 100644 --- a/.github/RELEASE_CHECKLIST.md +++ b/.github/RELEASE_CHECKLIST.md @@ -4,7 +4,8 @@ 1. Create draft release on GitHub to see a list of merged PRs. 2. Update CHANGELOG.md manually. It will point to versions of DocumentDB and FerretDB that are not released yet. -3. Send PR with changes, merge it. +3. Update `packaging/debian_files/changelog`. +4. Send PR with changes, merge it. ## Git tag diff --git a/ferretdb_packaging/debian_files/changelog b/ferretdb_packaging/debian_files/changelog deleted file mode 100644 index e8710acec..000000000 --- a/ferretdb_packaging/debian_files/changelog +++ /dev/null @@ -1,5 +0,0 @@ -documentdb (DOCUMENTDB_VERSION) unstable; urgency=low - - * Initial release. - - -- FerretDB Packages Fri, 24 Jan 2025 12:00:00 +0000 diff --git a/ferretdb_packaging/debian_files/compat b/ferretdb_packaging/debian_files/compat deleted file mode 100644 index 48082f72f..000000000 --- a/ferretdb_packaging/debian_files/compat +++ /dev/null @@ -1 +0,0 @@ -12 diff --git a/ferretdb_packaging/debian_files/control b/ferretdb_packaging/debian_files/control deleted file mode 100644 index 4574d1cb7..000000000 --- a/ferretdb_packaging/debian_files/control +++ /dev/null @@ -1,10 +0,0 @@ -Source: documentdb -Section: database -Priority: optional -Build-Depends: debhelper (>= 11), postgresql-server-dev-POSTGRES_VERSION -Maintainer: FerretDB Packages - -Package: postgresql-POSTGRES_VERSION-documentdb -Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends}, postgresql-POSTGRES_VERSION, postgresql-POSTGRES_VERSION-cron, postgresql-POSTGRES_VERSION-pgvector, postgresql-POSTGRES_VERSION-postgis-3, postgresql-POSTGRES_VERSION-rum -Description: DOCUMENTDB diff --git a/ferretdb_packaging/debian_files/rules b/ferretdb_packaging/debian_files/rules deleted file mode 100755 index 363dd034d..000000000 --- a/ferretdb_packaging/debian_files/rules +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/make -f - -%: - dh $@ - -override_dh_auto_test: - make install - adduser --disabled-password --gecos "" documentdb - chown -R documentdb:documentdb . - su documentdb -c "make check" diff --git a/packaging/debian_files/changelog b/packaging/debian_files/changelog index 23203bb3e..044acf137 100755 --- a/packaging/debian_files/changelog +++ b/packaging/debian_files/changelog @@ -4,7 +4,7 @@ documentdb (DOCUMENTDB_VERSION) Unreleased; urgency=low * Fix list_databases for databases with size > 2 GB *[Bugfix]* (#119) * Support ARM64 architecture when building docker container *[Preview]* - -- Shuai Tian Wed, 09 Apr 2026 12:00:00 +0000 + -- FerretDB Packages Wed, 09 Apr 2026 12:00:00 +0000 documentdb (0.102-0) unstable; urgency=medium @@ -27,7 +27,7 @@ documentdb (0.102-0) unstable; urgency=medium * Support the $dateFromString operator with full functionality [Feature] * Support extended syntax for $getField aggregation operator (field as expression) [Feature] - -- Shuai Tian Wed, 26 Mar 2025 10:00:00 +0000 + -- FerretDB Packages Wed, 26 Mar 2025 10:00:00 +0000 documentdb (0.101-0) unstable; urgency=medium @@ -38,10 +38,10 @@ documentdb (0.101-0) unstable; urgency=medium * Skip loading documents if group expression is constant [Perf] * Fix Merge stage not outputting to target collection [Bugfix] (#20) - -- Shuai Tian Tue, 12 Feb 2025 10:00:00 +0000 + -- FerretDB Packages Tue, 12 Feb 2025 10:00:00 +0000 documentdb (0.100-0) unstable; urgency=medium * Initial release - -- Shuai Tian Thu, 23 Jan 2025 10:00:00 +0000 + -- FerretDB Packages Thu, 23 Jan 2025 10:00:00 +0000 diff --git a/packaging/debian_files/control b/packaging/debian_files/control index 087917b9b..c7719d2eb 100755 --- a/packaging/debian_files/control +++ b/packaging/debian_files/control @@ -2,7 +2,7 @@ Source: documentdb Section: database Priority: optional Build-Depends: debhelper (>= 11), postgresql-server-dev-POSTGRES_VERSION -Maintainer: Shuai Tian +Maintainer: FerretDB Packages Package: postgresql-POSTGRES_VERSION-documentdb Architecture: any From bfc0b310edfd3bb06c3ef4a703855cd2d4b81e6c Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 29 Apr 2025 07:25:32 +0400 Subject: [PATCH 37/59] Unify test packages --- .../Dockerfile_test_install_deb_packages | 47 ------------------- .../test_packages/test-install-entrypoint.sh | 13 ----- packaging/Dockerfile_build_deb_packages | 0 packaging/README.md | 0 packaging/debian_files/changelog | 0 packaging/debian_files/compat | 0 packaging/debian_files/control | 0 packaging/debian_files/rules | 0 .../Dockerfile_test_install_deb_packages | 0 9 files changed, 60 deletions(-) delete mode 100644 ferretdb_packaging/test_packages/Dockerfile_test_install_deb_packages delete mode 100755 ferretdb_packaging/test_packages/test-install-entrypoint.sh mode change 100755 => 100644 packaging/Dockerfile_build_deb_packages mode change 100755 => 100644 packaging/README.md mode change 100755 => 100644 packaging/debian_files/changelog mode change 100755 => 100644 packaging/debian_files/compat mode change 100755 => 100644 packaging/debian_files/control mode change 100755 => 100644 packaging/debian_files/rules mode change 100755 => 100644 packaging/test_packages/Dockerfile_test_install_deb_packages diff --git a/ferretdb_packaging/test_packages/Dockerfile_test_install_deb_packages b/ferretdb_packaging/test_packages/Dockerfile_test_install_deb_packages deleted file mode 100644 index 89f918e54..000000000 --- a/ferretdb_packaging/test_packages/Dockerfile_test_install_deb_packages +++ /dev/null @@ -1,47 +0,0 @@ -ARG BASE_IMAGE=debian:bookworm -FROM --platform=linux/amd64 ${BASE_IMAGE} -ARG DEBIAN_FRONTEND=noninteractive - -ARG POSTGRES_VERSION=16 -ARG DEB_PACKAGE_REL_PATH=packages/postgresql-16-documentdb-1_1.0.0_amd64.deb - -RUN apt-get update - -RUN apt-get install -y --no-install-recommends \ - make \ - wget \ - gnupg2 \ - lsb-release \ - ca-certificates \ - locales \ - python3 - -RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ - locale-gen en_US.UTF-8 - -ENV LC_ALL=en_US.UTF-8 -ENV LANGUAGE=en_US -ENV LC_COLLATE=en_US.UTF-8 -ENV LC_CTYPE=en_US.UTF-8 -ENV LANG=en_US.UTF-8 - -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main ${POSTGRES_VERSION}" > /etc/apt/sources.list.d/pgdg.list && \ - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - -# actual dependencies of the package -RUN apt-get update && apt-get install -y \ - postgresql-${POSTGRES_VERSION} \ - postgresql-${POSTGRES_VERSION}-cron \ - postgresql-${POSTGRES_VERSION}-pgvector \ - postgresql-${POSTGRES_VERSION}-postgis-3 \ - postgresql-${POSTGRES_VERSION}-rum - -WORKDIR /test-install - -COPY . /test-install - -RUN dpkg -i ${DEB_PACKAGE_REL_PATH} - -COPY ferretdb_packaging/test_packages/test-install-entrypoint.sh /usr/local/bin/test-install-entrypoint.sh - -ENTRYPOINT ["test-install-entrypoint.sh"] diff --git a/ferretdb_packaging/test_packages/test-install-entrypoint.sh b/ferretdb_packaging/test_packages/test-install-entrypoint.sh deleted file mode 100755 index fd22dfc86..000000000 --- a/ferretdb_packaging/test_packages/test-install-entrypoint.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# Change to the test directory -cd /test-install - -# Keep the internal directory out of the testing -sed -i '/internal/d' Makefile - -# Run the test -adduser --disabled-password --gecos "" documentdb -chown -R documentdb:documentdb . -su documentdb -c "make check" diff --git a/packaging/Dockerfile_build_deb_packages b/packaging/Dockerfile_build_deb_packages old mode 100755 new mode 100644 diff --git a/packaging/README.md b/packaging/README.md old mode 100755 new mode 100644 diff --git a/packaging/debian_files/changelog b/packaging/debian_files/changelog old mode 100755 new mode 100644 diff --git a/packaging/debian_files/compat b/packaging/debian_files/compat old mode 100755 new mode 100644 diff --git a/packaging/debian_files/control b/packaging/debian_files/control old mode 100755 new mode 100644 diff --git a/packaging/debian_files/rules b/packaging/debian_files/rules old mode 100755 new mode 100644 diff --git a/packaging/test_packages/Dockerfile_test_install_deb_packages b/packaging/test_packages/Dockerfile_test_install_deb_packages old mode 100755 new mode 100644 From 80c886172c3898bbed4c6a0f1c0d0fba12f3435b Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 29 Apr 2025 07:35:49 +0400 Subject: [PATCH 38/59] Unify more packaging files --- .../Dockerfile_build_deb_packages | 80 ------------------- ferretdb_packaging/README.md | 13 --- ferretdb_packaging/packaging-entrypoint.sh | 30 ------- packaging/Dockerfile_build_deb_packages | 2 +- packaging/README.md | 4 +- 5 files changed, 3 insertions(+), 126 deletions(-) delete mode 100644 ferretdb_packaging/Dockerfile_build_deb_packages delete mode 100644 ferretdb_packaging/README.md delete mode 100755 ferretdb_packaging/packaging-entrypoint.sh diff --git a/ferretdb_packaging/Dockerfile_build_deb_packages b/ferretdb_packaging/Dockerfile_build_deb_packages deleted file mode 100644 index 1705e0d84..000000000 --- a/ferretdb_packaging/Dockerfile_build_deb_packages +++ /dev/null @@ -1,80 +0,0 @@ -ARG BASE_IMAGE=debian:bookworm -FROM --platform=linux/amd64 ${BASE_IMAGE} -ARG DEBIAN_FRONTEND=noninteractive - -ARG POSTGRES_VERSION -ARG DOCUMENTDB_VERSION - -RUN test -n "$DOCUMENTDB_VERSION" || (echo "DOCUMENTDB_VERSION not set" && false) - -RUN apt-get update - -RUN apt-get install -y --no-install-recommends \ - ca-certificates \ - gnupg2 \ - locales \ - lsb-release \ - wget - -RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ - locale-gen en_US.UTF-8 - -ENV LC_ALL=en_US.UTF-8 -ENV LANGUAGE=en_US -ENV LC_COLLATE=en_US.UTF-8 -ENV LC_CTYPE=en_US.UTF-8 -ENV LANG=en_US.UTF-8 - -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main ${POSTGRES_VERSION}" > /etc/apt/sources.list.d/pgdg.list && \ - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cmake \ - curl \ - debhelper \ - devscripts \ - dpkg-dev \ - git \ - libicu-dev \ - libkrb5-dev \ - libpq-dev \ - pkg-config \ - postgresql-${POSTGRES_VERSION} \ - postgresql-${POSTGRES_VERSION}-cron \ - postgresql-${POSTGRES_VERSION}-pgvector \ - postgresql-${POSTGRES_VERSION}-postgis-3 \ - postgresql-${POSTGRES_VERSION}-rum \ - postgresql-server-dev-${POSTGRES_VERSION} \ - sudo \ - tzdata \ - && rm -rf /var/lib/apt/lists/* - -COPY scripts /tmp/install_setup - -RUN CLEAN_SETUP=1 INSTALL_DEPENDENCIES_ROOT=/tmp/install_setup /tmp/install_setup/install_setup_pcre2.sh -RUN CLEAN_SETUP=1 INSTALL_DEPENDENCIES_ROOT=/tmp/install_setup /tmp/install_setup/install_setup_intel_decimal_math_lib.sh -RUN CLEAN_SETUP=1 INSTALL_DEPENDENCIES_ROOT=/tmp/install_setup /tmp/install_setup/install_citus_indent.sh - -RUN export CLEAN_SETUP=1 && \ - export INSTALL_DEPENDENCIES_ROOT=/tmp/install_setup && \ - MAKE_PROGRAM=cmake /tmp/install_setup/install_setup_libbson.sh && \ - /tmp/install_setup/install_setup_pcre2.sh && \ - /tmp/install_setup/install_setup_intel_decimal_math_lib.sh && \ - /tmp/install_setup/install_citus_indent.sh - -# Set the working directory inside the container -WORKDIR /build - -# Copy the source code into the container -COPY . /build - -# Setup the debian packaging -COPY ferretdb_packaging/debian_files /build/debian -RUN sed -i "s/POSTGRES_VERSION/${POSTGRES_VERSION}/g" /build/debian/control -RUN sed -i "s/DOCUMENTDB_VERSION/${DOCUMENTDB_VERSION}/g" /build/debian/changelog - -COPY ferretdb_packaging/packaging-entrypoint.sh /usr/local/bin/packaging-entrypoint.sh - -# Set the entrypoint -ENTRYPOINT ["packaging-entrypoint.sh"] diff --git a/ferretdb_packaging/README.md b/ferretdb_packaging/README.md deleted file mode 100644 index 18f9dbe53..000000000 --- a/ferretdb_packaging/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# To Build Your Own Debian Packages With Docker - -Run `./ferretdb_packaging/build_packages.sh -h` and follow the instructions. -E.g. to build for Debian 12 and PostgreSQL 16 for `0.102.0~ferretdb~2.0.0~rc.2` version, run: - -```sh -./ferretdb_packaging/build_packages.sh --os deb12 --pg 16 --version 0.102.0~ferretdb~2.0.0~rc.2 -``` - -Packages can be found at the `packages` directory by default, -but that can be configured with the `--output-dir` option. - -**Note:** The packages do not include pg_documentdb_distributed in the `internal` directory. diff --git a/ferretdb_packaging/packaging-entrypoint.sh b/ferretdb_packaging/packaging-entrypoint.sh deleted file mode 100755 index 0bd083629..000000000 --- a/ferretdb_packaging/packaging-entrypoint.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -e - -test -n "$OS" || (echo "OS not set" && false) - -# Change to the build directory -cd /build - -# Keep the internal directory out of the Debian package -sed -i '/internal/d' Makefile - -# Build the Debian package -debuild -us -uc - -# Change to the root to make file renaming expression simpler -cd / - -# Rename .deb files to include the OS name prefix -for f in *.deb; do - mv $f $OS-$f; -done - -# Create the output directory if it doesn't exist -mkdir -p /output - -# Copy the built packages to the output directory -cp *.deb /output/ - -# Change ownership of the output files to match the host user's UID and GID -chown -R $(stat -c "%u:%g" /output) /output diff --git a/packaging/Dockerfile_build_deb_packages b/packaging/Dockerfile_build_deb_packages index 505f7d93f..aea511693 100644 --- a/packaging/Dockerfile_build_deb_packages +++ b/packaging/Dockerfile_build_deb_packages @@ -2,7 +2,7 @@ ARG BASE_IMAGE=debian:bookworm FROM ${BASE_IMAGE} ARG DEBIAN_FRONTEND=noninteractive -ARG POSTGRES_VERSION=16 +ARG POSTGRES_VERSION ARG DOCUMENTDB_VERSION RUN test -n "$DOCUMENTDB_VERSION" || (echo "DOCUMENTDB_VERSION not set" && false) diff --git a/packaging/README.md b/packaging/README.md index 8bdb04021..b2c852845 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -1,10 +1,10 @@ # To Build Your Own Debian Packages With Docker Run `./packaging/build_packages.sh -h` and follow the instructions. -E.g. to build for Debian 12 and PostgreSQL 16, run: +E.g. to build for Debian 12 and PostgreSQL 16 for `0.102.0~ferretdb~2.0.0~rc.2` version, run: ```sh -./packaging/build_packages.sh --os deb12 --pg 16 +./packaging/build_packages.sh --os deb12 --pg 16 --version 0.102.0~ferretdb~2.0.0~rc.2 ``` Packages can be found at the `packages` directory by default, but it can be configured with the `--output-dir` option. From 0e6485398ae7ef603180713a1d0f2ef279d2c973 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 29 Apr 2025 14:58:16 +0400 Subject: [PATCH 39/59] Unify build script --- ferretdb_packaging/build_packages.sh | 151 --------------------------- packaging/build_packages.sh | 18 +--- 2 files changed, 5 insertions(+), 164 deletions(-) delete mode 100755 ferretdb_packaging/build_packages.sh diff --git a/ferretdb_packaging/build_packages.sh b/ferretdb_packaging/build_packages.sh deleted file mode 100755 index 86b451b00..000000000 --- a/ferretdb_packaging/build_packages.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Function to display help message -function show_help { - echo "Usage: $0 --os --pg [--test-clean-install] [--output-dir ] [-h|--help]" - echo "" - echo "Description:" - echo " This script builds extension packages using Docker." - echo "" - echo "Mandatory Arguments:" - echo " --os OS to build packages for. Possible values: [deb11, deb12, ubuntu22.04, ubuntu24.04]" - echo " --pg PG version to build packages for. Possible values: [15, 16, 17]" - echo " --version The debian conformed version of documentdb to build. Example: 0.102.0~ferretdb~2.0.0~rc.2" - echo "" - echo "Optional Arguments:" - echo " --test-clean-install Test installing the packages in a clean Docker container." - echo " --output-dir Relative path from the repo root of the directory where to drop the packages. The directory will be created if it doesn't exist. Default: packaging" - echo " -h, --help Display this help message." - exit 0 -} - -# Initialize variables -OS="" -PG="" -DOCUMENTDB_VERSION="" -TEST_CLEAN_INSTALL=false -OUTPUT_DIR="packaging" # Default value for output directory - -# Process arguments to convert long options to short ones -while [[ $# -gt 0 ]]; do - case "$1" in - --os) - shift - case $1 in - deb11|deb12|ubuntu22.04|ubuntu24.04) - OS=$1 - ;; - *) - error_exit "Invalid --os value. Allowed values are [deb11, deb12, ubuntu22.04, ubuntu24.04]" - ;; - esac - ;; - --pg) - shift - case $1 in - 15|16|17) - PG=$1 - ;; - *) - error_exit "Invalid --pg value. Allowed values are [15, 16, 17]" - ;; - esac - ;; - --version) - shift - DOCUMENTDB_VERSION=$1 - ;; - --test-clean-install) - TEST_CLEAN_INSTALL=true - ;; - --output-dir) - shift - OUTPUT_DIR=$1 - ;; - -h|--help) - show_help - ;; - *) - echo "Unknown argument: $1" - show_help - exit 1 - ;; - esac - shift -done - -# Check mandatory arguments -if [[ -z "$OS" ]]; then - echo "Error: --os is required." - show_help - exit 1 -fi - -if [[ -z "$PG" ]]; then - echo "Error: --pg is required." - show_help - exit 1 -fi - -if [[ -z "$DOCUMENTDB_VERSION" ]]; then - echo "Error: --version is required." - show_help - exit 1 -fi - -# Set the appropriate Docker image based on the OS -case $OS in - deb11) - DOCKER_IMAGE="debian:bullseye" - ;; - deb12) - DOCKER_IMAGE="debian:bookworm" - ;; - ubuntu22.04) - DOCKER_IMAGE="ubuntu:22.04" - ;; - ubuntu24.04) - DOCKER_IMAGE="ubuntu:24.04" - ;; -esac - -repo_root=$(git rev-parse --show-toplevel) -abs_output_dir="$repo_root/$OUTPUT_DIR" -cd $repo_root - -echo "Building packages for OS: $OS, PostgreSQL version: $PG, DOCUMENTDB version: $DOCUMENTDB_VERSION" -echo "Output directory: $abs_output_dir" - -# Create the output directory if it doesn't exist -mkdir -p $abs_output_dir - -# Build the Docker image while showing the output to the console -docker build --platform linux/amd64 -t documentdb-build-packages:latest -f ferretdb_packaging/Dockerfile_build_deb_packages \ - --build-arg BASE_IMAGE=$DOCKER_IMAGE --build-arg POSTGRES_VERSION=$PG --build-arg DOCUMENTDB_VERSION=$DOCUMENTDB_VERSION . - -# Run the Docker container to build the packages -docker run --platform linux/amd64 --rm --env OS=$OS -v $abs_output_dir:/output documentdb-build-packages:latest - -echo "Packages built successfully!!" - -if [[ $TEST_CLEAN_INSTALL == true ]]; then - echo "Testing clean installation in a Docker container..." - - deb_package_name=$(ls $abs_output_dir | grep -E "${OS}-postgresql-${PG}-documentdb_${DOCUMENTDB_VERSION}_amd64.deb" | grep -v "dbg" | head -n 1) - deb_package_rel_path="$OUTPUT_DIR/$deb_package_name" - - echo "Debian package path: $deb_package_rel_path" - - # Build the Docker image while showing the output to the console - docker build --platform linux/amd64 -t documentdb-test-packages:latest -f ferretdb_packaging/test_packages/Dockerfile_test_install_deb_packages \ - --build-arg BASE_IMAGE=$DOCKER_IMAGE --build-arg POSTGRES_VERSION=$PG --build-arg DEB_PACKAGE_REL_PATH=$deb_package_rel_path . - - # Run the Docker container to test the packages - docker run --platform linux/amd64 --rm documentdb-test-packages:latest - - echo "Clean installation test successful!!" -fi - -echo "Packages are available in $abs_output_dir" diff --git a/packaging/build_packages.sh b/packaging/build_packages.sh index b52a53ae0..a88294b4a 100755 --- a/packaging/build_packages.sh +++ b/packaging/build_packages.sh @@ -12,9 +12,9 @@ function show_help { echo "Mandatory Arguments:" echo " --os OS to build packages for. Possible values: [deb11, deb12, ubuntu22.04, ubuntu24.04]" echo " --pg PG version to build packages for. Possible values: [15, 16, 17]" + echo " --version The version of documentdb to build. Examples: [0.100.0, 0.101.0]" echo "" echo "Optional Arguments:" - echo " --version The version of documentdb to build. Examples: [0.100.0, 0.101.0]" echo " --test-clean-install Test installing the packages in a clean Docker container." echo " --output-dir Relative path from the repo root of the directory where to drop the packages. The directory will be created if it doesn't exist. Default: packaging" echo " -h, --help Display this help message." @@ -89,16 +89,10 @@ if [[ -z "$PG" ]]; then exit 1 fi -# get the version from control file if [[ -z "$DOCUMENTDB_VERSION" ]]; then - DOCUMENTDB_VERSION=$(grep -E "^default_version" pg_documentdb_core/documentdb_core.control | sed -E "s/.*'([0-9]+\.[0-9]+-[0-9]+)'.*/\1/") - DOCUMENTDB_VERSION=$(echo $DOCUMENTDB_VERSION | sed "s/-/./g") - echo "DOCUMENTDB_VERSION extracted from control file: $DOCUMENTDB_VERSION" - if [[ -z "$DOCUMENTDB_VERSION" ]]; then - echo "Error: --version is required and could not be found in the control file." - show_help - exit 1 - fi + echo "Error: --version is required." + show_help + exit 1 fi # Set the appropriate Docker image based on the OS @@ -117,8 +111,6 @@ case $OS in ;; esac -TAG=documentdb-build-packages-$OS-pg$PG:latest - repo_root=$(git rev-parse --show-toplevel) abs_output_dir="$repo_root/$OUTPUT_DIR" cd $repo_root @@ -141,7 +133,7 @@ echo "Packages built successfully!!" if [[ $TEST_CLEAN_INSTALL == true ]]; then echo "Testing clean installation in a Docker container..." - deb_package_name=$(ls $abs_output_dir | grep -E "${OS}-postgresql-$PG-documentdb_${DOCUMENTDB_VERSION}.*\.deb" | grep -v "dbg" | head -n 1) + deb_package_name=$(ls $abs_output_dir | grep -E "${OS}-postgresql-${PG}-documentdb_${DOCUMENTDB_VERSION}.*\.deb" | grep -v "dbg" | head -n 1) deb_package_rel_path="$OUTPUT_DIR/$deb_package_name" echo "Debian package path: $deb_package_rel_path" From ab79acf604b9b11cf521a29b3b5bfe47b3a2fc20 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 29 Apr 2025 15:29:49 +0400 Subject: [PATCH 40/59] Tweak help --- packaging/build_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/build_packages.sh b/packaging/build_packages.sh index a88294b4a..6c153abe8 100755 --- a/packaging/build_packages.sh +++ b/packaging/build_packages.sh @@ -4,7 +4,7 @@ set -euo pipefail # Function to display help message function show_help { - echo "Usage: $0 --os --pg [--test-clean-install] [--output-dir ] [-h|--help]" + echo "Usage: $0 --os --pg --version [--test-clean-install] [--output-dir ] [-h|--help]" echo "" echo "Description:" echo " This script builds extension packages using Docker." From d4de5a31af722adadb285e7dabf06d548ee15cf5 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Fri, 2 May 2025 15:18:20 +0400 Subject: [PATCH 41/59] Move packaging files (#64) --- .github/workflows/ferretdb_go_tests.yml | 2 +- .github/workflows/ferretdb_packages.yml | 20 +++++++++---------- .gitignore | 4 +--- ferretdb_packaging/.gitignore | 2 -- packaging/.gitignore | 2 ++ .../10-preload.sh | 0 .../20-install.sql | 0 .../90-install-development.sql | 0 {ferretdb_packaging => packaging}/Makefile | 0 .../buildkitd.toml | 0 .../defineversion/debian.go | 0 .../defineversion/debian_test.go | 0 .../defineversion/docker.go | 0 .../defineversion/docker_test.go | 0 .../defineversion}/go.mod | 4 ++-- .../defineversion}/go.sum | 4 ++-- .../defineversion/main.go | 0 .../defineversion/main_test.go | 0 .../development.Dockerfile | 4 ++-- .../production.Dockerfile | 2 +- 20 files changed, 21 insertions(+), 23 deletions(-) delete mode 100644 ferretdb_packaging/.gitignore create mode 100644 packaging/.gitignore rename {ferretdb_packaging => packaging}/10-preload.sh (100%) rename {ferretdb_packaging => packaging}/20-install.sql (100%) rename {ferretdb_packaging => packaging}/90-install-development.sql (100%) rename {ferretdb_packaging => packaging}/Makefile (100%) rename {ferretdb_packaging => packaging}/buildkitd.toml (100%) rename {ferretdb_packaging => packaging}/defineversion/debian.go (100%) rename {ferretdb_packaging => packaging}/defineversion/debian_test.go (100%) rename {ferretdb_packaging => packaging}/defineversion/docker.go (100%) rename {ferretdb_packaging => packaging}/defineversion/docker_test.go (100%) rename {ferretdb_packaging => packaging/defineversion}/go.mod (67%) rename {ferretdb_packaging => packaging/defineversion}/go.sum (85%) rename {ferretdb_packaging => packaging}/defineversion/main.go (100%) rename {ferretdb_packaging => packaging}/defineversion/main_test.go (100%) rename {ferretdb_packaging => packaging}/development.Dockerfile (89%) rename {ferretdb_packaging => packaging}/production.Dockerfile (92%) diff --git a/.github/workflows/ferretdb_go_tests.yml b/.github/workflows/ferretdb_go_tests.yml index 8a36fd108..8015b0189 100644 --- a/.github/workflows/ferretdb_go_tests.yml +++ b/.github/workflows/ferretdb_go_tests.yml @@ -46,7 +46,7 @@ jobs: - name: Run tests run: | - cd ferretdb_packaging + cd packaging go mod tidy go mod verify go test ./... diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index 0c696994e..868a635bc 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -106,14 +106,14 @@ jobs: - name: Define version id: version run: | - cd ferretdb_packaging + cd packaging/defineversion go mod tidy go mod verify - go run ./defineversion -control-file ../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} -debian-only + go run . -control-file ../../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} -debian-only - name: Build ${{ steps.version.outputs.debian_version }} if: steps.version.outputs.debian_version != '' - run: ./ferretdb_packaging/build_packages.sh --os ${{ matrix.os }} --pg ${{ matrix.pg }} --version ${{ steps.version.outputs.debian_version }} --test-clean-install + run: ./packaging/build_packages.sh --os ${{ matrix.os }} --pg ${{ matrix.pg }} --version ${{ steps.version.outputs.debian_version }} --test-clean-install - name: Upload .deb packages uses: actions/upload-artifact@v4 @@ -186,10 +186,10 @@ jobs: - name: Define version id: version run: | - cd ferretdb_packaging + cd packaging/defineversion go mod tidy go mod verify - go run ./defineversion -control-file ../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} + go run . -control-file ../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} - name: Download deb12-${{ matrix.pg }}-${{ steps.version.outputs.debian_version }} uses: actions/download-artifact@v4 @@ -198,12 +198,12 @@ jobs: path: packaging - name: Initialize Docker builder - run: make -C ferretdb_packaging docker-init + run: make -C packaging docker-init - name: Build local development Docker image if: steps.version.outputs.docker_development_tag_flags != '' run: > - make -C ferretdb_packaging docker-build + make -C packaging docker-build POSTGRES_VERSION=${{ matrix.pg }} DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} FILE=development @@ -213,7 +213,7 @@ jobs: - name: Build local production Docker image if: steps.version.outputs.docker_production_tag_flags != '' run: > - make -C ferretdb_packaging docker-build + make -C packaging docker-build POSTGRES_VERSION=${{ matrix.pg }} DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} FILE=production @@ -243,7 +243,7 @@ jobs: - name: Build and push development Docker images if: steps.version.outputs.docker_development_tag_flags != '' run: > - make -C ferretdb_packaging docker-build + make -C packaging docker-build POSTGRES_VERSION=${{ matrix.pg }} DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} FILE=development @@ -253,7 +253,7 @@ jobs: - name: Build and push production Docker images if: steps.version.outputs.docker_production_tag_flags != '' run: > - make -C ferretdb_packaging docker-build + make -C packaging docker-build POSTGRES_VERSION=${{ matrix.pg }} DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} FILE=production diff --git a/.gitignore b/.gitignore index 1a39a86b9..760bb8269 100755 --- a/.gitignore +++ b/.gitignore @@ -43,8 +43,6 @@ *.idb *.pdb -# FerretDB packaging -*.deb # Kernel Module Compile Results *.mod* @@ -74,4 +72,4 @@ build/ *.tmp # deb packages -*.deb \ No newline at end of file +*.deb diff --git a/ferretdb_packaging/.gitignore b/ferretdb_packaging/.gitignore deleted file mode 100644 index 353ce6cb8..000000000 --- a/ferretdb_packaging/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# FerretDB packaging -!*go.mod diff --git a/packaging/.gitignore b/packaging/.gitignore new file mode 100644 index 000000000..6216440af --- /dev/null +++ b/packaging/.gitignore @@ -0,0 +1,2 @@ +# defineversion's +!go.mod diff --git a/ferretdb_packaging/10-preload.sh b/packaging/10-preload.sh similarity index 100% rename from ferretdb_packaging/10-preload.sh rename to packaging/10-preload.sh diff --git a/ferretdb_packaging/20-install.sql b/packaging/20-install.sql similarity index 100% rename from ferretdb_packaging/20-install.sql rename to packaging/20-install.sql diff --git a/ferretdb_packaging/90-install-development.sql b/packaging/90-install-development.sql similarity index 100% rename from ferretdb_packaging/90-install-development.sql rename to packaging/90-install-development.sql diff --git a/ferretdb_packaging/Makefile b/packaging/Makefile similarity index 100% rename from ferretdb_packaging/Makefile rename to packaging/Makefile diff --git a/ferretdb_packaging/buildkitd.toml b/packaging/buildkitd.toml similarity index 100% rename from ferretdb_packaging/buildkitd.toml rename to packaging/buildkitd.toml diff --git a/ferretdb_packaging/defineversion/debian.go b/packaging/defineversion/debian.go similarity index 100% rename from ferretdb_packaging/defineversion/debian.go rename to packaging/defineversion/debian.go diff --git a/ferretdb_packaging/defineversion/debian_test.go b/packaging/defineversion/debian_test.go similarity index 100% rename from ferretdb_packaging/defineversion/debian_test.go rename to packaging/defineversion/debian_test.go diff --git a/ferretdb_packaging/defineversion/docker.go b/packaging/defineversion/docker.go similarity index 100% rename from ferretdb_packaging/defineversion/docker.go rename to packaging/defineversion/docker.go diff --git a/ferretdb_packaging/defineversion/docker_test.go b/packaging/defineversion/docker_test.go similarity index 100% rename from ferretdb_packaging/defineversion/docker_test.go rename to packaging/defineversion/docker_test.go diff --git a/ferretdb_packaging/go.mod b/packaging/defineversion/go.mod similarity index 67% rename from ferretdb_packaging/go.mod rename to packaging/defineversion/go.mod index e6b36d58c..6f3df772e 100644 --- a/ferretdb_packaging/go.mod +++ b/packaging/defineversion/go.mod @@ -1,11 +1,11 @@ -module github.com/FerretDB/documentdb/ferretdb_packaging +module github.com/FerretDB/documentdb/packaging/defineversion go 1.24 toolchain go1.24.2 require ( - github.com/sethvargo/go-githubactions v1.3.0 + github.com/sethvargo/go-githubactions v1.3.1 github.com/stretchr/testify v1.10.0 ) diff --git a/ferretdb_packaging/go.sum b/packaging/defineversion/go.sum similarity index 85% rename from ferretdb_packaging/go.sum rename to packaging/defineversion/go.sum index 0051d23f1..882f7c0cb 100644 --- a/ferretdb_packaging/go.sum +++ b/packaging/defineversion/go.sum @@ -2,8 +2,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sethvargo/go-githubactions v1.3.0 h1:Kg633LIUV2IrJsqy2MfveiED/Ouo+H2P0itWS0eLh8A= -github.com/sethvargo/go-githubactions v1.3.0/go.mod h1:7/4WeHgYfSz9U5vwuToCK9KPnELVHAhGtRwLREOQV80= +github.com/sethvargo/go-githubactions v1.3.1 h1:rlwwLRUaunWLQ1aN2o5Y+3s0xhaTC30YObCnilRx448= +github.com/sethvargo/go-githubactions v1.3.1/go.mod h1:7/4WeHgYfSz9U5vwuToCK9KPnELVHAhGtRwLREOQV80= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/ferretdb_packaging/defineversion/main.go b/packaging/defineversion/main.go similarity index 100% rename from ferretdb_packaging/defineversion/main.go rename to packaging/defineversion/main.go diff --git a/ferretdb_packaging/defineversion/main_test.go b/packaging/defineversion/main_test.go similarity index 100% rename from ferretdb_packaging/defineversion/main_test.go rename to packaging/defineversion/main_test.go diff --git a/ferretdb_packaging/development.Dockerfile b/packaging/development.Dockerfile similarity index 89% rename from ferretdb_packaging/development.Dockerfile rename to packaging/development.Dockerfile index 7d63aa8c4..6ec061395 100644 --- a/ferretdb_packaging/development.Dockerfile +++ b/packaging/development.Dockerfile @@ -49,8 +49,8 @@ set -ex cd /src -cp ferretdb_packaging/10-preload.sh ferretdb_packaging/20-install.sql /docker-entrypoint-initdb.d/ -cp ferretdb_packaging/90-install-development.sql /docker-entrypoint-initdb.d/ +cp packaging/10-preload.sh packaging/20-install.sql /docker-entrypoint-initdb.d/ +cp packaging/90-install-development.sql /docker-entrypoint-initdb.d/ EOF diff --git a/ferretdb_packaging/production.Dockerfile b/packaging/production.Dockerfile similarity index 92% rename from ferretdb_packaging/production.Dockerfile rename to packaging/production.Dockerfile index bbc40a0ca..f8cb37b0e 100644 --- a/ferretdb_packaging/production.Dockerfile +++ b/packaging/production.Dockerfile @@ -37,7 +37,7 @@ set -ex cd /src -cp ferretdb_packaging/10-preload.sh ferretdb_packaging/20-install.sql /docker-entrypoint-initdb.d/ +cp packaging/10-preload.sh packaging/20-install.sql /docker-entrypoint-initdb.d/ EOF From 2b0e6f940cd3cac831f65518fd1401602cf7cb39 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Fri, 2 May 2025 16:58:18 +0400 Subject: [PATCH 42/59] Build arm64 .deb packages (#67) --- .github/workflows/ferretdb_packages.yml | 39 +++++++++++++++---------- .gitignore | 1 - CODEOWNERS | 12 -------- packaging/Makefile | 2 +- packaging/build_packages.sh | 2 ++ 5 files changed, 26 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index 868a635bc..23f62bd58 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -53,8 +53,19 @@ concurrency: jobs: deb: - name: Build .debs (${{ matrix.os }}, Pg${{ matrix.pg }}) - runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + os: [deb11, deb12, ubuntu22.04, ubuntu24.04] + pg: [15, 16, 17] + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + + name: Build .debs (${{ matrix.os }}, ${{ matrix.arch }}, Pg${{ matrix.pg }}) + runs-on: ${{ matrix.runner }} timeout-minutes: 40 if: > @@ -67,12 +78,6 @@ jobs: permissions: {} - strategy: - fail-fast: false - matrix: - os: [deb11, deb12, ubuntu22.04, ubuntu24.04] - pg: [15, 16, 17] - steps: # TODO https://github.com/FerretDB/github-actions/issues/211 - name: Checkout code @@ -118,7 +123,7 @@ jobs: - name: Upload .deb packages uses: actions/upload-artifact@v4 with: - name: ${{ matrix.os }}-${{ matrix.pg }}-${{ steps.version.outputs.debian_version }} + name: ${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.pg }}-${{ steps.version.outputs.debian_version }} path: packaging/*.deb retention-days: 1 if-no-files-found: error @@ -137,13 +142,15 @@ jobs: needs: deb - if: > - github.event_name != 'pull_request_target' || - ( - contains(github.event.pull_request.labels.*.name, 'trust') && - !contains(github.event.pull_request.labels.*.name, 'not ready') && - contains(github.event.pull_request.labels.*.name, 'packages') - ) + if: false + + # if: > + # github.event_name != 'pull_request_target' || + # ( + # contains(github.event.pull_request.labels.*.name, 'trust') && + # !contains(github.event.pull_request.labels.*.name, 'not ready') && + # contains(github.event.pull_request.labels.*.name, 'packages') + # ) permissions: packages: write diff --git a/.gitignore b/.gitignore index 760bb8269..9e7f1052c 100755 --- a/.gitignore +++ b/.gitignore @@ -43,7 +43,6 @@ *.idb *.pdb - # Kernel Module Compile Results *.mod* *.cmd diff --git a/CODEOWNERS b/CODEOWNERS index 9a9db1207..8d47a1941 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,13 +1 @@ -* @microsoft/documentdb-contributors - -/.pipelines/ @safern @visridha @shuaitian-git @lichoil -/.github/workflows/ @safern @visridha @shuaitian-git @lichoil - -*.csv @microsoft/documentdb-engine-reviewers -*.sql @microsoft/documentdb-engine-reviewers -Make* @microsoft/documentdb-engine-reviewers -/internal/ @microsoft/documentdb-engine-reviewers -/pg_documentdb/ @microsoft/documentdb-engine-reviewers -/pg_documentdb_core/ @microsoft/documentdb-engine-reviewers - * @AlekSi diff --git a/packaging/Makefile b/packaging/Makefile index a304d0940..025e04c04 100644 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -24,7 +24,7 @@ docker-build: --file=$(FILE).Dockerfile \ --build-arg='POSTGRES_VERSION=$(POSTGRES_VERSION)' \ --build-arg='DOCUMENTDB_VERSION=$(DOCUMENTDB_VERSION)' \ - --platform=linux/amd64 \ + --platform=linux/amd64,linux/arm64 \ --output='$(OUTPUT)' \ $(TAGS) \ .. diff --git a/packaging/build_packages.sh b/packaging/build_packages.sh index 6c153abe8..628d680b9 100755 --- a/packaging/build_packages.sh +++ b/packaging/build_packages.sh @@ -111,6 +111,8 @@ case $OS in ;; esac +TAG=documentdb-build-packages-$OS-pg$PG:latest + repo_root=$(git rev-parse --show-toplevel) abs_output_dir="$repo_root/$OUTPUT_DIR" cd $repo_root From 949277828d0c8b867bad67e82508635d60bfeaa4 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 6 May 2025 17:33:09 +0400 Subject: [PATCH 43/59] Fix arm64 builds, part 1 (#68) Co-authored-by: Chi Fujii --- .github/workflows/ferretdb_go_tests.yml | 2 +- .github/workflows/ferretdb_packages.yml | 25 +++++++++++++------------ .github/workflows/regress_tests.yml | 2 +- packaging/development.Dockerfile | 2 ++ packaging/production.Dockerfile | 1 + 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ferretdb_go_tests.yml b/.github/workflows/ferretdb_go_tests.yml index 8015b0189..dc5230084 100644 --- a/.github/workflows/ferretdb_go_tests.yml +++ b/.github/workflows/ferretdb_go_tests.yml @@ -46,7 +46,7 @@ jobs: - name: Run tests run: | - cd packaging + cd packaging/defineversion go mod tidy go mod verify go test ./... diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index 23f62bd58..a5981c22a 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -23,7 +23,9 @@ name: Packages on: - pull_request_target: + # FIXME do not merge this, it's for testing only + # pull_request_target: + pull_request: types: - unlabeled # if GitHub Actions stuck, add and remove "not ready" label to force rebuild - opened @@ -57,6 +59,7 @@ jobs: fail-fast: false matrix: os: [deb11, deb12, ubuntu22.04, ubuntu24.04] + arch: [amd64, arm64] pg: [15, 16, 17] include: - arch: amd64 @@ -64,7 +67,7 @@ jobs: - arch: arm64 runner: ubuntu-24.04-arm - name: Build .debs (${{ matrix.os }}, ${{ matrix.arch }}, Pg${{ matrix.pg }}) + name: .debs (${{ matrix.os }}, ${{ matrix.arch }}, Pg${{ matrix.pg }}) runs-on: ${{ matrix.runner }} timeout-minutes: 40 @@ -136,21 +139,19 @@ jobs: git diff --exit-code docker: - name: Build Docker (Pg${{ matrix.pg }}) + name: Docker (Pg${{ matrix.pg }}) runs-on: ubuntu-24.04 timeout-minutes: 40 needs: deb - if: false - - # if: > - # github.event_name != 'pull_request_target' || - # ( - # contains(github.event.pull_request.labels.*.name, 'trust') && - # !contains(github.event.pull_request.labels.*.name, 'not ready') && - # contains(github.event.pull_request.labels.*.name, 'packages') - # ) + if: > + github.event_name != 'pull_request_target' || + ( + contains(github.event.pull_request.labels.*.name, 'trust') && + !contains(github.event.pull_request.labels.*.name, 'not ready') && + contains(github.event.pull_request.labels.*.name, 'packages') + ) permissions: packages: write diff --git a/.github/workflows/regress_tests.yml b/.github/workflows/regress_tests.yml index f406f00a5..2e71c0205 100644 --- a/.github/workflows/regress_tests.yml +++ b/.github/workflows/regress_tests.yml @@ -161,6 +161,6 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: logs_$${{ matrix.runner }}_PG${{ matrix.pg_version }} + name: logs_${{ matrix.runner }}_PG${{ matrix.pg_version }} overwrite: true path: "**/*.log" diff --git a/packaging/development.Dockerfile b/packaging/development.Dockerfile index 6ec061395..2cd469be5 100644 --- a/packaging/development.Dockerfile +++ b/packaging/development.Dockerfile @@ -26,10 +26,12 @@ set -ex cd /src +# FIXME cp packaging/deb12-postgresql-${POSTGRES_VERSION}-documentdb_${DOCUMENTDB_VERSION}_amd64.deb /tmp/documentdb.deb dpkg -i /tmp/documentdb.deb rm /tmp/documentdb.deb +# FIXME cp packaging/deb12-postgresql-${POSTGRES_VERSION}-documentdb-dbgsym_${DOCUMENTDB_VERSION}_amd64.deb /tmp/documentdb-dbgsym.deb dpkg -i /tmp/documentdb-dbgsym.deb rm /tmp/documentdb-dbgsym.deb diff --git a/packaging/production.Dockerfile b/packaging/production.Dockerfile index f8cb37b0e..2b2652566 100644 --- a/packaging/production.Dockerfile +++ b/packaging/production.Dockerfile @@ -26,6 +26,7 @@ set -ex cd /src +# FIXME cp packaging/deb12-postgresql-${POSTGRES_VERSION}-documentdb_${DOCUMENTDB_VERSION}_amd64.deb /tmp/documentdb.deb dpkg -i /tmp/documentdb.deb rm /tmp/documentdb.deb From d321a1cb7b603bf5a10de4d7ab2393c3e1a26e34 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 6 May 2025 23:12:35 +0400 Subject: [PATCH 44/59] Fix arm64 builds, part 2 (#69) Co-authored-by: Chi Fujii --- .github/workflows/ferretdb_packages.yml | 23 ++++++++++++++--------- packaging/development.Dockerfile | 7 +++---- packaging/production.Dockerfile | 4 ++-- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index a5981c22a..553c4190e 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -23,9 +23,7 @@ name: Packages on: - # FIXME do not merge this, it's for testing only - # pull_request_target: - pull_request: + pull_request_target: types: - unlabeled # if GitHub Actions stuck, add and remove "not ready" label to force rebuild - opened @@ -197,35 +195,42 @@ jobs: cd packaging/defineversion go mod tidy go mod verify - go run . -control-file ../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} + go run . -control-file ../../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} - name: Download deb12-${{ matrix.pg }}-${{ steps.version.outputs.debian_version }} uses: actions/download-artifact@v4 with: - name: deb12-${{ matrix.pg }}-${{ steps.version.outputs.debian_version }} + pattern: deb12-*-${{ matrix.pg }}-${{ steps.version.outputs.debian_version }} path: packaging + merge-multiple: true + + - name: List files + run: ls -l packaging + + - name: Setup QEMU + uses: docker/setup-qemu-action@v3 - name: Initialize Docker builder run: make -C packaging docker-init - - name: Build local development Docker image + - name: Build local development Docker images if: steps.version.outputs.docker_development_tag_flags != '' run: > make -C packaging docker-build POSTGRES_VERSION=${{ matrix.pg }} DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} FILE=development - OUTPUT='type=docker' + OUTPUT='type=image' TAGS='${{ steps.version.outputs.docker_development_tag_flags }}' - - name: Build local production Docker image + - name: Build local production Docker images if: steps.version.outputs.docker_production_tag_flags != '' run: > make -C packaging docker-build POSTGRES_VERSION=${{ matrix.pg }} DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} FILE=production - OUTPUT='type=docker' + OUTPUT='type=image' TAGS='${{ steps.version.outputs.docker_production_tag_flags }}' - name: Login to Docker Hub diff --git a/packaging/development.Dockerfile b/packaging/development.Dockerfile index 2cd469be5..1d6349ddc 100644 --- a/packaging/development.Dockerfile +++ b/packaging/development.Dockerfile @@ -4,6 +4,7 @@ ARG POSTGRES_VERSION FROM postgres:${POSTGRES_VERSION} AS development +ARG TARGETARCH ARG POSTGRES_VERSION ARG DOCUMENTDB_VERSION @@ -26,13 +27,11 @@ set -ex cd /src -# FIXME -cp packaging/deb12-postgresql-${POSTGRES_VERSION}-documentdb_${DOCUMENTDB_VERSION}_amd64.deb /tmp/documentdb.deb +cp packaging/deb12-postgresql-${POSTGRES_VERSION}-documentdb_${DOCUMENTDB_VERSION}_${TARGETARCH}.deb /tmp/documentdb.deb dpkg -i /tmp/documentdb.deb rm /tmp/documentdb.deb -# FIXME -cp packaging/deb12-postgresql-${POSTGRES_VERSION}-documentdb-dbgsym_${DOCUMENTDB_VERSION}_amd64.deb /tmp/documentdb-dbgsym.deb +cp packaging/deb12-postgresql-${POSTGRES_VERSION}-documentdb-dbgsym_${DOCUMENTDB_VERSION}_${TARGETARCH}.deb /tmp/documentdb-dbgsym.deb dpkg -i /tmp/documentdb-dbgsym.deb rm /tmp/documentdb-dbgsym.deb diff --git a/packaging/production.Dockerfile b/packaging/production.Dockerfile index 2b2652566..5bf45bf5d 100644 --- a/packaging/production.Dockerfile +++ b/packaging/production.Dockerfile @@ -4,6 +4,7 @@ ARG POSTGRES_VERSION FROM postgres:${POSTGRES_VERSION} AS production +ARG TARGETARCH ARG POSTGRES_VERSION ARG DOCUMENTDB_VERSION @@ -26,8 +27,7 @@ set -ex cd /src -# FIXME -cp packaging/deb12-postgresql-${POSTGRES_VERSION}-documentdb_${DOCUMENTDB_VERSION}_amd64.deb /tmp/documentdb.deb +cp packaging/deb12-postgresql-${POSTGRES_VERSION}-documentdb_${DOCUMENTDB_VERSION}_${TARGETARCH}.deb /tmp/documentdb.deb dpkg -i /tmp/documentdb.deb rm /tmp/documentdb.deb From fa129dcf70050854501f8fe21a88d66a1e42b222 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Fri, 9 May 2025 12:58:53 +0400 Subject: [PATCH 45/59] Bump Go (#70) --- packaging/defineversion/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/defineversion/go.mod b/packaging/defineversion/go.mod index 6f3df772e..44d1e78ab 100644 --- a/packaging/defineversion/go.mod +++ b/packaging/defineversion/go.mod @@ -2,7 +2,7 @@ module github.com/FerretDB/documentdb/packaging/defineversion go 1.24 -toolchain go1.24.2 +toolchain go1.24.3 require ( github.com/sethvargo/go-githubactions v1.3.1 From 751423287693ee2c1ab15f5bc42c7c83ce18cf13 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Fri, 9 May 2025 18:08:50 +0400 Subject: [PATCH 46/59] Prepare v0.103.0-ferretdb-2.2.0 release (#73) --- CHANGELOG.md | 18 ++++++++++++++++-- packaging/debian_files/changelog | 7 +++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b9826848..75e5f0b42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,24 @@ -### documentdb v0.103-0 (Unreleased) ### +### DocumentDB v0.103.0-ferretdb-2.2.0 (May 9, 2025) ### + +This version works best with FerretDB v2.2.0. + +Debian and Ubuntu `.deb` packages are provided +[on the release page](https://github.com/FerretDB/documentdb/releases/tag/v0.103.0-ferretdb-2.2.0). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/deb/). + +Docker images are available +[in the registry](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/docker/). +We always recommend specifying the full image tag (e.g., `17-0.103.0-ferretdb-2.2.0`, not just `17` or `17-0.103.0`) +to avoid unexpected updates. + +### documentdb v0.103-0 (May 9, 2025) ### * Support collation with aggregation and find on sharded collections *[Feature]* * Support `$convert` on `binData` to `binData`, `string` to `binData` and `binData` to `string` (except with `format: auto`) *[Feature]* * Fix list_databases for databases with size > 2 GB *[Bugfix]* (#119) * Support half-precision vector indexing, vectors can have up to 4,000 dimensions *[Feature]* * Support ARM64 architecture when building docker container *[Preview]* -* Support collation with `$documents` and `$replceWith` stage of the aggregation pipeline *[Feature]* +* Support collation with `$documents` and `$replaceWith` stage of the aggregation pipeline *[Feature]* ### DocumentDB v0.102.0-ferretdb-2.1.0 (April 2, 2025) ### diff --git a/packaging/debian_files/changelog b/packaging/debian_files/changelog index 044acf137..ec6fdaea9 100644 --- a/packaging/debian_files/changelog +++ b/packaging/debian_files/changelog @@ -1,10 +1,13 @@ -documentdb (DOCUMENTDB_VERSION) Unreleased; urgency=low +documentdb (DOCUMENTDB_VERSION) unstable; urgency=medium + * Support collation with aggregation and find on sharded collections *[Feature]* * Support `$convert` on `binData` to `binData`, `string` to `binData` and `binData` to `string` (except with `format: auto`) *[Feature]* * Fix list_databases for databases with size > 2 GB *[Bugfix]* (#119) + * Support half-precision vector indexing, vectors can have up to 4,000 dimensions *[Feature]* * Support ARM64 architecture when building docker container *[Preview]* + * Support collation with `$documents` and `$replaceWith` stage of the aggregation pipeline *[Feature]* - -- FerretDB Packages Wed, 09 Apr 2026 12:00:00 +0000 + -- FerretDB Packages Fri, 09 May 2025 12:00:00 +0000 documentdb (0.102-0) unstable; urgency=medium From aa58f474d9a0b530e9db36681904abf7813c617d Mon Sep 17 00:00:00 2001 From: Joachim Hill-Grannec Date: Wed, 28 May 2025 06:15:54 -0400 Subject: [PATCH 47/59] Add `barman-cli-cloud` for backups (#75) --- packaging/development.Dockerfile | 3 ++- packaging/production.Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packaging/development.Dockerfile b/packaging/development.Dockerfile index 1d6349ddc..48a9229ae 100644 --- a/packaging/development.Dockerfile +++ b/packaging/development.Dockerfile @@ -19,7 +19,8 @@ apt install -y \ postgresql-${POSTGRES_VERSION}-pgvector \ postgresql-${POSTGRES_VERSION}-postgis-3 \ postgresql-${POSTGRES_VERSION}-rum \ - postgresql-server-dev-${POSTGRES_VERSION} + postgresql-server-dev-${POSTGRES_VERSION} \ + barman-cli-cloud EOF RUN --mount=target=/src,rw < Date: Tue, 10 Jun 2025 08:47:22 +0400 Subject: [PATCH 48/59] Prepare v0.104.0-ferretdb-2.3.0 release (#78) --- CHANGELOG.md | 28 +++++++++++++++++++++------- packaging/10-preload.sh | 2 ++ packaging/debian_files/changelog | 29 +++++++++++++++++++++++------ 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e815984e..bc1ef3d2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,21 @@ -### documentdb v0.104-0 (Unreleased) ### +### DocumentDB v0.104.0-ferretdb-2.3.0 (Jun 10, 2025) ### + +This version works best with FerretDB v2.3.0. + +Debian and Ubuntu `.deb` packages are provided +[on the release page](https://github.com/FerretDB/documentdb/releases/tag/v0.104.0-ferretdb-2.3.0). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/deb/). + +Docker images are available +[in the registry](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/docker/). +We always recommend specifying the full image tag (e.g., `17-0.104.0-ferretdb-2.3.0`, not just `17` or `17-0.104.0`) +to avoid unexpected updates. + +### documentdb v0.104-0 (Jun 10, 2025) ### * Add string case support for `$toDate` operator -* Support `sort` with collation in runtime*[Feature]* -* Support collation with `$indexOfArray` aggregation operator. *[Feature]* +* Support `sort` with collation in runtime *[Feature]* +* Support collation with `$indexOfArray` aggregation operator *[Feature]* * Support collation with arrays and objects comparisons *[Feature]* * Support background index builds *[Bugfix]* (#36) * Enable user CRUD by default *[Feature]* @@ -9,7 +23,7 @@ * Enable rum_enable_index_scan as default on *[Perf]* * Add public `documentdb-local` Docker image with gateway to GHCR * Support `compact` command *[Feature]*. Requires `documentdb.enablecompact` GUC to be `on`. -* Enable role privileges for `usersInfo` command *[Feature]* +* Enable role privileges for `usersInfo` command *[Feature]* ### DocumentDB v0.103.0-ferretdb-2.2.0 (May 9, 2025) ### @@ -80,10 +94,10 @@ We always recommend specifying the full image tag (e.g., `17-0.102.0-ferretdb-2. * Support unique index truncation by default with new operator class *[Feature]* * Top level aggregate command `let` variables support for `$geoNear` stage *[Feature]* * Enable Backend Command support for Statement Timeout *[Feature]* -* Support type aggregation operator `$toUUID`. *[Feature]* +* Support type aggregation operator `$toUUID` *[Feature]* * Support Partial filter pushdown for `$in` predicates *[Perf]* * Support the $dateFromString operator with full functionality *[Feature]* -* Support extended syntax for `$getField` aggregation operator. Now the value of 'field' could be an expression that resolves to a string. *[Feature]* +* Support extended syntax for `$getField` aggregation operator (field as expression) *[Feature]* ### documentdb v0.101-0 (February 12, 2025) ### * Push $graphlookup recursive CTE JOIN filters to index *[Perf]* @@ -91,7 +105,7 @@ We always recommend specifying the full image tag (e.g., `17-0.102.0-ferretdb-2. * Enable support of currentOp aggregation stage, along with collstats, dbstats, and indexStats *[Commands]* (#52) * Allow inlining $unwind with $lookup with `preserveNullAndEmptyArrays` *[Perf]* * Skip loading documents if group expression is constant *[Perf]* -* Fix Merge stage not outputing to target collection *[Bugfix]* (#20) +* Fix Merge stage not outputting to target collection *[Bugfix]* (#20) ### documentdb v0.100-0 (January 23rd, 2025) ### Initial Release diff --git a/packaging/10-preload.sh b/packaging/10-preload.sh index 7e1e9eba6..1e156df7c 100755 --- a/packaging/10-preload.sh +++ b/packaging/10-preload.sh @@ -8,6 +8,8 @@ cat <> $PGDATA/postgresql.conf shared_preload_libraries = 'pg_cron,pg_documentdb_core,pg_documentdb' cron.database_name = 'postgres' +documentdb.enableCompact = true + documentdb.enableLetAndCollationForQueryMatch = true documentdb.enableNowSystemVariable = true documentdb.enableSortbyIdPushDownToPrimaryKey = true diff --git a/packaging/debian_files/changelog b/packaging/debian_files/changelog index ec6fdaea9..9ff1d233e 100644 --- a/packaging/debian_files/changelog +++ b/packaging/debian_files/changelog @@ -1,11 +1,28 @@ documentdb (DOCUMENTDB_VERSION) unstable; urgency=medium - * Support collation with aggregation and find on sharded collections *[Feature]* - * Support `$convert` on `binData` to `binData`, `string` to `binData` and `binData` to `string` (except with `format: auto`) *[Feature]* - * Fix list_databases for databases with size > 2 GB *[Bugfix]* (#119) - * Support half-precision vector indexing, vectors can have up to 4,000 dimensions *[Feature]* - * Support ARM64 architecture when building docker container *[Preview]* - * Support collation with `$documents` and `$replaceWith` stage of the aggregation pipeline *[Feature]* + * Add string case support for $toDate operator + * Support sort with collation in runtime [Feature] + * Support collation with $indexOfArray aggregation operator [Feature] + * Support collation with arrays and objects comparisons [Feature] + * Support background index builds [Bugfix] (#36) + * Enable user CRUD by default [Feature] + * Enable let support for delete queries [Feature]. Requires EnableVariablesSupportForWriteCommands to be on. + * Enable rum_enable_index_scan as default on [Perf] + * Add public documentdb-local Docker image with gateway to GHCR + * Support compact command [Feature]. Requires documentdb.enablecompact GUC to be on. + * Enable role privileges for usersInfo command [Feature] + + -- FerretDB Packages Tue, 10 Jun 2025 12:00:00 +0000 + +documentdb (0.103-0) unstable; urgency=medium + + * Support collation with aggregation and find on sharded collections [Feature] + * Support $convert on binData to binData, string to binData and binData to string (except with format: auto) [Feature] + * Fix list_databases for databases with size > 2 GB [Bugfix] (#119) + * Support half-precision vector indexing, vectors can have up to 4,000 dimensions [Feature] + * Support ARM64 architecture when building docker container [Preview] + * Support collation with $documents and $replaceWith stage of the aggregation pipeline [Feature] + * Push pg_documentdb_gw for documentdb connections [Feature] -- FerretDB Packages Fri, 09 May 2025 12:00:00 +0000 From 1190e102760b1e9a8f2cc539dd3fb00c916a609e Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Wed, 11 Jun 2025 17:40:15 +0900 Subject: [PATCH 49/59] Provide production build docker images for branch builds (#79) Closes FerretDB/FerretDB#5227. --- packaging/defineversion/main.go | 8 +++++++ packaging/defineversion/main_test.go | 36 ++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/packaging/defineversion/main.go b/packaging/defineversion/main.go index fd7a8b4e4..e6fd89f86 100644 --- a/packaging/defineversion/main.go +++ b/packaging/defineversion/main.go @@ -150,6 +150,9 @@ func defineVersionForPR(controlDefaultVersion, pgVersion, owner, repo, branch st dockerDevelopmentImages: []string{ fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-pr-%s", owner, repo, pgVersion, branch), }, + dockerProductionImages: []string{ + fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-pr-%s-prod", owner, repo, pgVersion, branch), + }, debian: disallowedDebian.ReplaceAllString(fmt.Sprintf("%s-pr-%s", controlDefaultVersion, branch), "~"), } @@ -169,6 +172,9 @@ func defineVersionForBranch(controlDefaultVersion, pgVersion, owner, repo, branc dockerDevelopmentImages: []string{ fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-ferretdb", owner, repo, pgVersion), }, + dockerProductionImages: []string{ + fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-ferretdb-prod", owner, repo, pgVersion), + }, debian: fmt.Sprintf("%s~ferretdb", controlDefaultVersion), } @@ -183,7 +189,9 @@ func defineVersionForBranch(controlDefaultVersion, pgVersion, owner, repo, branc } res.dockerDevelopmentImages = append(res.dockerDevelopmentImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb-dev:%s-ferretdb", pgVersion)) + res.dockerProductionImages = append(res.dockerProductionImages, fmt.Sprintf("quay.io/ferretdb/postgres-documentdb-dev:%s-ferretdb-prod", pgVersion)) res.dockerDevelopmentImages = append(res.dockerDevelopmentImages, fmt.Sprintf("ferretdb/postgres-documentdb-dev:%s-ferretdb", pgVersion)) + res.dockerProductionImages = append(res.dockerProductionImages, fmt.Sprintf("ferretdb/postgres-documentdb-dev:%s-ferretdb-prod", pgVersion)) return res, nil } diff --git a/packaging/defineversion/main_test.go b/packaging/defineversion/main_test.go index 6dba699af..4b4fa4451 100644 --- a/packaging/defineversion/main_test.go +++ b/packaging/defineversion/main_test.go @@ -91,6 +91,9 @@ func TestDefineVersion(t *testing.T) { dockerDevelopmentImages: []string{ "ghcr.io/ferretdb/postgres-documentdb-dev:17-pr-define-version", }, + dockerProductionImages: []string{ + "ghcr.io/ferretdb/postgres-documentdb-dev:17-pr-define-version-prod", + }, debian: "0.103.0~pr~define~version", }, }, @@ -108,6 +111,9 @@ func TestDefineVersion(t *testing.T) { dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-pr-define-version", }, + dockerProductionImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17-pr-define-version-prod", + }, debian: "0.103.0~pr~define~version", }, }, @@ -126,6 +132,9 @@ func TestDefineVersion(t *testing.T) { dockerDevelopmentImages: []string{ "ghcr.io/ferretdb/postgres-documentdb-dev:17-pr-define-version", }, + dockerProductionImages: []string{ + "ghcr.io/ferretdb/postgres-documentdb-dev:17-pr-define-version-prod", + }, debian: "0.103.0~pr~define~version", }, }, @@ -143,6 +152,9 @@ func TestDefineVersion(t *testing.T) { dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-pr-define-version", }, + dockerProductionImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17-pr-define-version-prod", + }, debian: "0.103.0~pr~define~version", }, }, @@ -163,6 +175,11 @@ func TestDefineVersion(t *testing.T) { "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb", "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb", }, + dockerProductionImages: []string{ + "ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + }, debian: "0.103.0~ferretdb", }, }, @@ -180,6 +197,9 @@ func TestDefineVersion(t *testing.T) { dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", }, + dockerProductionImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb-prod", + }, debian: "0.103.0~ferretdb", }, }, @@ -307,6 +327,11 @@ func TestDefineVersion(t *testing.T) { "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb", "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb", }, + dockerProductionImages: []string{ + "ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + }, debian: "0.103.0~ferretdb", }, }, @@ -324,6 +349,9 @@ func TestDefineVersion(t *testing.T) { dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", }, + dockerProductionImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb-prod", + }, debian: "0.103.0~ferretdb", }, }, @@ -344,6 +372,11 @@ func TestDefineVersion(t *testing.T) { "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb", "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb", }, + dockerProductionImages: []string{ + "ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", + }, debian: "0.103.0~ferretdb", }, }, @@ -361,6 +394,9 @@ func TestDefineVersion(t *testing.T) { dockerDevelopmentImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb", }, + dockerProductionImages: []string{ + "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb-prod", + }, debian: "0.103.0~ferretdb", }, }, From 21016ffdd7254c1cc6d9eba767a7f50735a600c0 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 11 Jun 2025 18:47:53 +0400 Subject: [PATCH 50/59] Unify production and development Dockerfiles To reuse build cache. --- packaging/development.Dockerfile | 17 +++++++++++------ packaging/production.Dockerfile | 14 +++++--------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/packaging/development.Dockerfile b/packaging/development.Dockerfile index 48a9229ae..3fffd7036 100644 --- a/packaging/development.Dockerfile +++ b/packaging/development.Dockerfile @@ -2,12 +2,14 @@ ARG POSTGRES_VERSION -FROM postgres:${POSTGRES_VERSION} AS development +FROM postgres:${POSTGRES_VERSION} ARG TARGETARCH ARG POSTGRES_VERSION ARG DOCUMENTDB_VERSION +# common steps for production and development + RUN --mount=type=cache,sharing=locked,target=/var/cache/apt < Date: Thu, 12 Jun 2025 13:40:26 +0400 Subject: [PATCH 51/59] Add version annotation to Docker images (#82) --- CHANGELOG.md | 6 +++++- packaging/development.Dockerfile | 1 + packaging/production.Dockerfile | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc1ef3d2f..a4dcc6b7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ ### DocumentDB v0.104.0-ferretdb-2.3.0 (Jun 10, 2025) ### -This version works best with FerretDB v2.3.0. +This version works best with FerretDB v2.3.0 and v2.3.1. + +> [!NOTE] +> Docker tags `XX-0.104.0-ferretdb-2.3.0` and `XX-0.104.0-ferretdb-2.3.1` point to the same images. +> Additional tags were added to accommodate FerretDB hotfix release 2.3.1. Debian and Ubuntu `.deb` packages are provided [on the release page](https://github.com/FerretDB/documentdb/releases/tag/v0.104.0-ferretdb-2.3.0). diff --git a/packaging/development.Dockerfile b/packaging/development.Dockerfile index 3fffd7036..169989679 100644 --- a/packaging/development.Dockerfile +++ b/packaging/development.Dockerfile @@ -68,3 +68,4 @@ LABEL org.opencontainers.image.description="PostgreSQL with DocumentDB extension LABEL org.opencontainers.image.source="https://github.com/FerretDB/documentdb" LABEL org.opencontainers.image.url="https://www.ferretdb.com/" LABEL org.opencontainers.image.vendor="FerretDB Inc." +LABEL org.opencontainers.image.version="${DOCUMENTDB_VERSION}" diff --git a/packaging/production.Dockerfile b/packaging/production.Dockerfile index 8dda6888c..9d4a254d6 100644 --- a/packaging/production.Dockerfile +++ b/packaging/production.Dockerfile @@ -46,3 +46,4 @@ LABEL org.opencontainers.image.description="PostgreSQL with DocumentDB extension LABEL org.opencontainers.image.source="https://github.com/FerretDB/documentdb" LABEL org.opencontainers.image.url="https://www.ferretdb.com/" LABEL org.opencontainers.image.vendor="FerretDB Inc." +LABEL org.opencontainers.image.version="${DOCUMENTDB_VERSION}" From ec5b9cc495975d7d20bfa87f9bad4e3f6da47f23 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 15 Jul 2025 16:03:37 +0400 Subject: [PATCH 52/59] Prepare v0.105.0-ferretdb-2.4.0 release --- CHANGELOG.md | 20 +++++++++++++++++--- packaging/debian_files/changelog | 7 +++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bad519eb0..ba18c3f28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,22 @@ -### documentdb v0.105-0 (Unreleased) ### +### DocumentDB v0.105.0-ferretdb-2.4.0 (July 15, 2025) ### + +This version works best with FerretDB v2.4.0. + +Debian and Ubuntu `.deb` packages are provided +[on the release page](https://github.com/FerretDB/documentdb/releases/tag/v0.105.0-ferretdb-2.4.0). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/deb/). + +Docker images are available +[in the registry](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/docker/). +We always recommend specifying the full image tag (e.g., `17-0.105.0-ferretdb-2.4.0`, not just `17` or `17-0.105.0`) +to avoid unexpected updates. + +### documentdb v0.105-0 (July 09, 2025) ### * Support `$bucketAuto` aggregation stage, with granularity types: `POWERSOF2`, `1-2-5`, `R5`, `R10`, `R20`, `R40`, `R80`, `E6`, `E12`, `E24`, `E48`, `E96`, `E192` *[Feature]* * Support `conectionStatus` command *[Feature]*. -### DocumentDB v0.104.0-ferretdb-2.3.0 (Jun 10, 2025) ### +### DocumentDB v0.104.0-ferretdb-2.3.0 (June 10, 2025) ### This version works best with FerretDB v2.3.0 and v2.3.1. @@ -33,7 +47,7 @@ to avoid unexpected updates. * Support `compact` command *[Feature]*. Requires `documentdb.enablecompact` GUC to be `on`. * Enable role privileges for `usersInfo` command *[Feature]* -### DocumentDB v0.103.0-ferretdb-2.2.0 (May 9, 2025) ### +### DocumentDB v0.103.0-ferretdb-2.2.0 (May 09, 2025) ### This version works best with FerretDB v2.2.0. diff --git a/packaging/debian_files/changelog b/packaging/debian_files/changelog index 9ff1d233e..16a25f6ce 100644 --- a/packaging/debian_files/changelog +++ b/packaging/debian_files/changelog @@ -1,5 +1,12 @@ documentdb (DOCUMENTDB_VERSION) unstable; urgency=medium + * Support `$bucketAuto` aggregation stage, with granularity types: `POWERSOF2`, `1-2-5`, `R5`, `R10`, `R20`, `R40`, `R80`, `E6`, `E12`, `E24`, `E48`, `E96`, `E192` [Feature] + * Support `conectionStatus` command [Feature] + +-- FerretDB Packages Tue, 09 Jul 2025 12:00:00 +0000 + +documentdb (0.104-0) unstable; urgency=medium + * Add string case support for $toDate operator * Support sort with collation in runtime [Feature] * Support collation with $indexOfArray aggregation operator [Feature] From 09ef0598a8f473b85cac26eeabb4ef1bc7f6eaa6 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Mon, 28 Jul 2025 09:56:29 +0400 Subject: [PATCH 53/59] Merge CodeQL changes from upstream --- .github/workflows/codeql.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index db18a1bb7..c7db78a99 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -51,10 +51,10 @@ jobs: libpq-dev \ libicu-dev \ libkrb5-dev \ - postgresql-15-cron \ - postgresql-15-pgvector \ - postgresql-15-postgis-3 \ - postgresql-15-rum + postgresql-16-cron \ + postgresql-16-pgvector \ + postgresql-16-postgis-3 \ + postgresql-16-rum export CLEAN_SETUP=1 export INSTALL_DEPENDENCIES_ROOT=/tmp/install_setup mkdir -p /tmp/install_setup From 61e3e55d60f32b444cbbd53fd2abd00913349f32 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 30 Jul 2025 14:00:08 +0400 Subject: [PATCH 54/59] Abbreviate SHA hashes to at least 10 characters That solves a problem with builds of older tags when new commits with the same prefix are added: ``` 0.104.0 gitref: HEAD sha:2045d0e buildId:0 0.104.0 gitref: HEAD sha:2045d0e0 buildId:0 ``` --- scripts/generate_extension_version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_extension_version.sh b/scripts/generate_extension_version.sh index b50b3e71d..469941629 100755 --- a/scripts/generate_extension_version.sh +++ b/scripts/generate_extension_version.sh @@ -33,7 +33,7 @@ if [ -f $gitIndexDir ]; then else # set GIT_VERSION with current branch's name and the short sha of the HEAD GIT_VERSION=$(git rev-parse --abbrev-ref HEAD) - GIT_SHA=$(git rev-parse --short HEAD) + GIT_SHA=$(git rev-parse --short=10 HEAD) fi if [[ "$GIT_VERSION" == "" ]] || [[ "$GIT_SHA" == "" ]]; then From 04a0257b462c68d7e27d2683041b7699efb4f2d8 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Fri, 1 Aug 2025 17:39:16 +0400 Subject: [PATCH 55/59] Build RPM packages (#96) Closes FerretDB/FerretDB#5379. Co-authored-by: Chi Fujii --- .github/workflows/ferretdb_packages.yml | 51 ++++++++++++------- .gitignore | 3 +- packaging/build_packages.sh | 2 +- packaging/defineversion/main.go | 48 ++++++++++------- packaging/defineversion/main_test.go | 30 +++++------ .../{debian.go => package_version.go} | 4 ++ ...debian_test.go => package_version_test.go} | 0 7 files changed, 84 insertions(+), 54 deletions(-) rename packaging/defineversion/{debian.go => package_version.go} (84%) rename packaging/defineversion/{debian_test.go => package_version_test.go} (100%) diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index 553c4190e..d9a52be0c 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -52,11 +52,11 @@ concurrency: cancel-in-progress: false jobs: - deb: + packages: strategy: fail-fast: false matrix: - os: [deb11, deb12, ubuntu22.04, ubuntu24.04] + os: [deb11, deb12, ubuntu22.04, ubuntu24.04, rhel8, rhel9] arch: [amd64, arm64] pg: [15, 16, 17] include: @@ -64,8 +64,19 @@ jobs: runner: ubuntu-24.04 - arch: arm64 runner: ubuntu-24.04-arm + exclude: + # No PostgreSQL 15 for RHEL. + - os: rhel8 + pg: 15 + - os: rhel9 + pg: 15 + # TODO https://github.com/microsoft/documentdb/issues/259 + - arch: arm64 + os: rhel8 + - arch: arm64 + os: rhel9 - name: .debs (${{ matrix.os }}, ${{ matrix.arch }}, Pg${{ matrix.pg }}) + name: ${{ matrix.os }}, ${{ matrix.arch }}, Pg${{ matrix.pg }} runs-on: ${{ matrix.runner }} timeout-minutes: 40 @@ -115,17 +126,19 @@ jobs: cd packaging/defineversion go mod tidy go mod verify - go run . -control-file ../../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} -debian-only + go run . -control-file ../../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} -package-version-only - - name: Build ${{ steps.version.outputs.debian_version }} - if: steps.version.outputs.debian_version != '' - run: ./packaging/build_packages.sh --os ${{ matrix.os }} --pg ${{ matrix.pg }} --version ${{ steps.version.outputs.debian_version }} --test-clean-install + - name: Build ${{ steps.version.outputs.package_version }} + if: steps.version.outputs.package_version != '' + run: ./packaging/build_packages.sh --os ${{ matrix.os }} --pg ${{ matrix.pg }} --version ${{ steps.version.outputs.package_version }} --test-clean-install - - name: Upload .deb packages + - name: Upload packages uses: actions/upload-artifact@v4 with: - name: ${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.pg }}-${{ steps.version.outputs.debian_version }} - path: packaging/*.deb + name: ${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.pg }}-${{ steps.version.outputs.package_version }} + path: | + packaging/*.rpm + packaging/*.deb retention-days: 1 if-no-files-found: error compression-level: 0 @@ -137,11 +150,11 @@ jobs: git diff --exit-code docker: - name: Docker (Pg${{ matrix.pg }}) + name: Docker Pg${{ matrix.pg }} runs-on: ubuntu-24.04 timeout-minutes: 40 - needs: deb + needs: packages if: > github.event_name != 'pull_request_target' || @@ -197,10 +210,12 @@ jobs: go mod verify go run . -control-file ../../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} - - name: Download deb12-${{ matrix.pg }}-${{ steps.version.outputs.debian_version }} + # Official Docker images are built on top of Debian 12: + # docker run --rm postgres:15 cat /etc/os-release + - name: Download deb12-${{ matrix.pg }}-${{ steps.version.outputs.package_version }} uses: actions/download-artifact@v4 with: - pattern: deb12-*-${{ matrix.pg }}-${{ steps.version.outputs.debian_version }} + pattern: deb12-*-${{ matrix.pg }}-${{ steps.version.outputs.package_version }} path: packaging merge-multiple: true @@ -218,7 +233,7 @@ jobs: run: > make -C packaging docker-build POSTGRES_VERSION=${{ matrix.pg }} - DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} + DOCUMENTDB_VERSION=${{ steps.version.outputs.package_version }} FILE=development OUTPUT='type=image' TAGS='${{ steps.version.outputs.docker_development_tag_flags }}' @@ -228,7 +243,7 @@ jobs: run: > make -C packaging docker-build POSTGRES_VERSION=${{ matrix.pg }} - DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} + DOCUMENTDB_VERSION=${{ steps.version.outputs.package_version }} FILE=production OUTPUT='type=image' TAGS='${{ steps.version.outputs.docker_production_tag_flags }}' @@ -258,7 +273,7 @@ jobs: run: > make -C packaging docker-build POSTGRES_VERSION=${{ matrix.pg }} - DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} + DOCUMENTDB_VERSION=${{ steps.version.outputs.package_version }} FILE=development OUTPUT='type=image,push=true' TAGS='${{ steps.version.outputs.docker_development_tag_flags }}' @@ -268,7 +283,7 @@ jobs: run: > make -C packaging docker-build POSTGRES_VERSION=${{ matrix.pg }} - DOCUMENTDB_VERSION=${{ steps.version.outputs.debian_version }} + DOCUMENTDB_VERSION=${{ steps.version.outputs.package_version }} FILE=production OUTPUT='type=image,push=true' TAGS='${{ steps.version.outputs.docker_production_tag_flags }}' diff --git a/.gitignore b/.gitignore index 8e9d653dc..e6f66eb57 100755 --- a/.gitignore +++ b/.gitignore @@ -70,7 +70,8 @@ build/ # temp schedules *.tmp -# deb packages +# packages *.deb +*.rpm site/ diff --git a/packaging/build_packages.sh b/packaging/build_packages.sh index eba05a0e6..8449198ed 100755 --- a/packaging/build_packages.sh +++ b/packaging/build_packages.sh @@ -192,7 +192,7 @@ if [[ $TEST_CLEAN_INSTALL == true ]]; then docker run --rm documentdb-test-packages:latest elif [[ "$PACKAGE_TYPE" == "rpm" ]]; then - rpm_package_name=$(ls "$abs_output_dir" | grep -E "${OS}-postgresql${PG}-documentdb-${DOCUMENTDB_VERSION}.*\.x86_64\.rpm" | head -n 1) + rpm_package_name=$(ls "$abs_output_dir" | grep -E "${OS}-postgresql${PG}-documentdb-${DOCUMENTDB_VERSION}.*\.rpm" | head -n 1) if [[ -z "$rpm_package_name" ]]; then echo "Error: Could not find the built RPM package in $abs_output_dir for testing." exit 1 diff --git a/packaging/defineversion/main.go b/packaging/defineversion/main.go index e6fd89f86..ef2501117 100644 --- a/packaging/defineversion/main.go +++ b/packaging/defineversion/main.go @@ -18,11 +18,11 @@ import ( // but with a leading `v`. var semVerTag = regexp.MustCompile(`^v(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`) -// versions represents Docker image names and tags, and Debian package version. +// versions represents Docker image names and tags, and package versions used by Debian and RPM. type versions struct { dockerDevelopmentImages []string dockerProductionImages []string - debian string + packageVersion string } // parseGitTag parses git tag in specific format and returns SemVer components. @@ -82,11 +82,13 @@ func debugEnv(action *githubactions.Action) { } } -// defineVersion extracts Docker image names and tags, and Debian package version using the environment variables defined by GitHub Actions. +// defineVersion extracts Docker image names and tags, and package versions used by Debian and RPM +// using the environment variables defined by GitHub Actions. // -// The Debian package version is based on `default_version` in the control file. -// See https://www.debian.org/doc/debian-policy/ch-controlfields.html#version. -// We use `upstream_version` only. +// The Debian and RPM package versions are based on `default_version` in the control file. +// See https://www.debian.org/doc/debian-policy/ch-controlfields.html#version, +// and https://fedoraproject.org/wiki/PackagingDrafts/TildeVersioning#Basic_versioning_rules. +// The Debian uses `upstream_version` only. // For that reason, we can't use `-`, so we replace it with `~`. func defineVersion(controlDefaultVersion, pgVersion string, getenv githubactions.GetenvFunc) (*versions, error) { repo := getenv("GITHUB_REPOSITORY") @@ -139,13 +141,17 @@ func defineVersion(controlDefaultVersion, pgVersion string, getenv githubactions return res, nil } -// defineVersionForPR defines Docker image names and tags, and Debian package version for PR. +// defineVersionForPR defines Docker image names and tags, and package version for PR. // See [defineVersion]. func defineVersionForPR(controlDefaultVersion, pgVersion, owner, repo, branch string) *versions { // for branches like "dependabot/submodules/XXX" parts := strings.Split(branch, "/") branch = parts[len(parts)-1] + packageVersion := fmt.Sprintf("%s-pr-%s", controlDefaultVersion, branch) + packageVersion = disallowedDebian.ReplaceAllString(packageVersion, "~") + packageVersion = disallowedRPM.ReplaceAllString(packageVersion, "~") + res := &versions{ dockerDevelopmentImages: []string{ fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-pr-%s", owner, repo, pgVersion, branch), @@ -153,7 +159,7 @@ func defineVersionForPR(controlDefaultVersion, pgVersion, owner, repo, branch st dockerProductionImages: []string{ fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-pr-%s-prod", owner, repo, pgVersion, branch), }, - debian: disallowedDebian.ReplaceAllString(fmt.Sprintf("%s-pr-%s", controlDefaultVersion, branch), "~"), + packageVersion: packageVersion, } // PRs are only for testing; no Quay.io and Docker Hub repos @@ -161,7 +167,7 @@ func defineVersionForPR(controlDefaultVersion, pgVersion, owner, repo, branch st return res } -// defineVersionForBranch defines Docker image names and tags, and Debian package version for branch. +// defineVersionForBranch defines Docker image names and tags, and package version for branch. // See [defineVersion]. func defineVersionForBranch(controlDefaultVersion, pgVersion, owner, repo, branch string) (*versions, error) { if branch != "ferretdb" { @@ -175,7 +181,7 @@ func defineVersionForBranch(controlDefaultVersion, pgVersion, owner, repo, branc dockerProductionImages: []string{ fmt.Sprintf("ghcr.io/%s/postgres-%s-dev:%s-ferretdb-prod", owner, repo, pgVersion), }, - debian: fmt.Sprintf("%s~ferretdb", controlDefaultVersion), + packageVersion: fmt.Sprintf("%s~ferretdb", controlDefaultVersion), } // forks don't have Quay.io and Docker Hub orgs @@ -196,7 +202,7 @@ func defineVersionForBranch(controlDefaultVersion, pgVersion, owner, repo, branc return res, nil } -// defineVersionForTag defines Docker image names and tags, and Debian package version for tag. +// defineVersionForTag defines Docker image names and tags, and package version for tag. // See [defineVersion]. func defineVersionForTag(controlDefaultVersion, pgVersion, owner, repo, tag string) (*versions, error) { major, minor, patch, prerelease, err := parseGitTag(tag) @@ -219,8 +225,12 @@ func defineVersionForTag(controlDefaultVersion, pgVersion, owner, repo, tag stri tags = append(tags, "latest") } + packageVersion := fmt.Sprintf("%s-%s", tagVersion, prerelease) + packageVersion = disallowedDebian.ReplaceAllString(packageVersion, "~") + packageVersion = disallowedRPM.ReplaceAllString(packageVersion, "~") + res := versions{ - debian: disallowedDebian.ReplaceAllString(fmt.Sprintf("%s-%s", tagVersion, prerelease), "~"), + packageVersion: packageVersion, } for _, t := range tags { @@ -253,7 +263,7 @@ func defineVersionForTag(controlDefaultVersion, pgVersion, owner, repo, tag stri func setSummary(action *githubactions.Action, version *versions) { var buf strings.Builder - fmt.Fprintf(&buf, "Debian package version (`upstream_version` only): `%s`\n\n", version.debian) + fmt.Fprintf(&buf, "Package version (Debian with `upstream_version` only, or RPM): `%s`\n\n", version.packageVersion) w := tabwriter.NewWriter(&buf, 1, 1, 1, ' ', tabwriter.Debug) fmt.Fprintf(w, "\tType\tDocker image\t\n") @@ -278,7 +288,7 @@ func setSummary(action *githubactions.Action, version *versions) { func main() { controlFileF := flag.String("control-file", "../pg_documentdb/documentdb.control", "pg_documentdb/documentdb.control file path") pgVersionF := flag.String("pg-version", "17", "Major PostgreSQL version") - debianOnlyF := flag.Bool("debian-only", false, "Only set output for Debian package version") + packageVersionOnlyF := flag.Bool("package-version-only", false, "Only set output for package version") flag.Parse() @@ -307,13 +317,13 @@ func main() { action.Fatalf("%s", err) } - action.SetOutput("debian_version", res.debian) + action.SetOutput("package_version", res.packageVersion) - if *debianOnlyF { + if *packageVersionOnlyF { // Only 3 summaries are shown in the GitHub Actions UI by default, - // and Docker summaries are more important (and include Debian version anyway). - output := fmt.Sprintf("Debian package version (`upstream_version` only): `%s`", res.debian) - action.Infof("%s", output) + // and Docker summaries are more important (and include package version anyway). + action.Infof("package version (Debian with `upstream_version` only, or RPM): `%s`", res.packageVersion) + return } diff --git a/packaging/defineversion/main_test.go b/packaging/defineversion/main_test.go index 4b4fa4451..b56b0631c 100644 --- a/packaging/defineversion/main_test.go +++ b/packaging/defineversion/main_test.go @@ -94,7 +94,7 @@ func TestDefineVersion(t *testing.T) { dockerProductionImages: []string{ "ghcr.io/ferretdb/postgres-documentdb-dev:17-pr-define-version-prod", }, - debian: "0.103.0~pr~define~version", + packageVersion: "0.103.0~pr~define~version", }, }, "pull_request-other": { @@ -114,7 +114,7 @@ func TestDefineVersion(t *testing.T) { dockerProductionImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-pr-define-version-prod", }, - debian: "0.103.0~pr~define~version", + packageVersion: "0.103.0~pr~define~version", }, }, @@ -135,7 +135,7 @@ func TestDefineVersion(t *testing.T) { dockerProductionImages: []string{ "ghcr.io/ferretdb/postgres-documentdb-dev:17-pr-define-version-prod", }, - debian: "0.103.0~pr~define~version", + packageVersion: "0.103.0~pr~define~version", }, }, "pull_request_target-other": { @@ -155,7 +155,7 @@ func TestDefineVersion(t *testing.T) { dockerProductionImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-pr-define-version-prod", }, - debian: "0.103.0~pr~define~version", + packageVersion: "0.103.0~pr~define~version", }, }, @@ -180,7 +180,7 @@ func TestDefineVersion(t *testing.T) { "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", }, - debian: "0.103.0~ferretdb", + packageVersion: "0.103.0~ferretdb", }, }, "push/ferretdb-other": { @@ -200,7 +200,7 @@ func TestDefineVersion(t *testing.T) { dockerProductionImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb-prod", }, - debian: "0.103.0~ferretdb", + packageVersion: "0.103.0~ferretdb", }, }, @@ -268,7 +268,7 @@ func TestDefineVersion(t *testing.T) { "quay.io/ferretdb/postgres-documentdb:17-0.103.0-ferretdb-2.2.0-beta.1", "quay.io/ferretdb/postgres-documentdb:latest", }, - debian: "0.103.0~ferretdb~2.2.0~beta.1", + packageVersion: "0.103.0~ferretdb~2.2.0~beta.1", }, }, "push/tag/release-other": { @@ -294,7 +294,7 @@ func TestDefineVersion(t *testing.T) { "ghcr.io/otherorg/postgres-otherrepo:17-0.103.0-ferretdb-2.2.0-beta.1", "ghcr.io/otherorg/postgres-otherrepo:latest", }, - debian: "0.103.0~ferretdb~2.2.0~beta.1", + packageVersion: "0.103.0~ferretdb~2.2.0~beta.1", }, }, @@ -332,7 +332,7 @@ func TestDefineVersion(t *testing.T) { "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", }, - debian: "0.103.0~ferretdb", + packageVersion: "0.103.0~ferretdb", }, }, "schedule-other": { @@ -352,7 +352,7 @@ func TestDefineVersion(t *testing.T) { dockerProductionImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb-prod", }, - debian: "0.103.0~ferretdb", + packageVersion: "0.103.0~ferretdb", }, }, @@ -377,7 +377,7 @@ func TestDefineVersion(t *testing.T) { "ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", "quay.io/ferretdb/postgres-documentdb-dev:17-ferretdb-prod", }, - debian: "0.103.0~ferretdb", + packageVersion: "0.103.0~ferretdb", }, }, "workflow_run-other": { @@ -397,7 +397,7 @@ func TestDefineVersion(t *testing.T) { dockerProductionImages: []string{ "ghcr.io/otherorg/postgres-otherrepo-dev:17-ferretdb-prod", }, - debian: "0.103.0~ferretdb", + packageVersion: "0.103.0~ferretdb", }, }, } { @@ -443,13 +443,13 @@ func TestSummary(t *testing.T) { dockerProductionImages: []string{ "quay.io/ferretdb/postgres-documentdb:latest", }, - debian: "0.103.0~ferretdb", + packageVersion: "0.103.0~ferretdb", } setSummary(action, result) expectedStdout := strings.ReplaceAll(` -Debian package version ('upstream_version' only): '0.103.0~ferretdb' +Package version (Debian with 'upstream_version' only, or RPM): '0.103.0~ferretdb' |Type |Docker image | |---- |------------ | @@ -462,7 +462,7 @@ Debian package version ('upstream_version' only): '0.103.0~ferretdb' assert.Equal(t, expectedStdout, stdout.String(), "stdout does not match") expectedSummary := strings.ReplaceAll(` -Debian package version ('upstream_version' only): '0.103.0~ferretdb' +Package version (Debian with 'upstream_version' only, or RPM): '0.103.0~ferretdb' |Type |Docker image | |---- |------------ | diff --git a/packaging/defineversion/debian.go b/packaging/defineversion/package_version.go similarity index 84% rename from packaging/defineversion/debian.go rename to packaging/defineversion/package_version.go index 6cb0191f8..636b815ea 100644 --- a/packaging/defineversion/debian.go +++ b/packaging/defineversion/package_version.go @@ -14,6 +14,10 @@ var controlDefaultVer = regexp.MustCompile(`(?m)^default_version = '(?P[0 // See https://www.debian.org/doc/debian-policy/ch-controlfields.html#version. var disallowedDebian = regexp.MustCompile(`[^A-Za-z0-9\.+~]`) +// disallowedRPM matches disallowed characters of pre-release string. +// See https://fedoraproject.org/wiki/PackagingDrafts/TildeVersioning#Basic_versioning_rules. +var disallowedRPM = regexp.MustCompile(`[^A-Za-z0-9\._+~]`) + // getControlDefaultVersion returns the default_version field from the control file // in SemVer format (0.103-0 -> 0.103.0). func getControlDefaultVersion(f string) (string, error) { diff --git a/packaging/defineversion/debian_test.go b/packaging/defineversion/package_version_test.go similarity index 100% rename from packaging/defineversion/debian_test.go rename to packaging/defineversion/package_version_test.go From beb9d25d9845477c85559d3293cd1f929cdf2b9c Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 12 Aug 2025 08:55:38 +0400 Subject: [PATCH 56/59] Prepare v0.106.0-ferretdb-2.5.0 release (#99) --- .github/RELEASE_CHECKLIST.md | 4 ++-- CHANGELOG.md | 20 ++++++++++++---- packaging/debian_files/changelog | 22 +++++++++++++---- packaging/defineversion/go.mod | 2 +- packaging/rpm_files/documentdb.spec | 37 +++++++++++++++++++++-------- 5 files changed, 63 insertions(+), 22 deletions(-) diff --git a/.github/RELEASE_CHECKLIST.md b/.github/RELEASE_CHECKLIST.md index 250b8a667..613bb0a46 100644 --- a/.github/RELEASE_CHECKLIST.md +++ b/.github/RELEASE_CHECKLIST.md @@ -4,7 +4,7 @@ 1. Create draft release on GitHub to see a list of merged PRs. 2. Update CHANGELOG.md manually. It will point to versions of DocumentDB and FerretDB that are not released yet. -3. Update `packaging/debian_files/changelog`. +3. Update `packaging/debian_files/changelog` and `packaging/rpm_files/documentdb.spec`. 4. Send PR with changes, merge it. ## Git tag @@ -20,6 +20,6 @@ 1. Find [Packages CI build](https://github.com/FerretDB/documentdb/actions/workflows/ferretdb_packages.yml?query=event%3Apush) for the tag to release. 2. Check Docker images. -3. Upload `.deb` packages to the draft release. +3. Upload `.deb` and `.rpm` packages to the draft release. 4. Update release notes with the content of CHANGELOG.md. 5. Publish release on GitHub. diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ca66a0f7..47481221f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,19 @@ -### documentdb v1.106-0 (Unreleased) ### -* Add internal extension that provides extensions to the `rum` index. *[Feature]* +### DocumentDB v0.106.0-ferretdb-2.5.0 (August 12, 2025) ### + +This version works best with FerretDB v2.5.0. + +Docker images are available +[in the registry](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). +`.deb` packages for Debian and Ubuntu, and `.rpm` packages for Red Hat Enterprise Linux (RHEL) are provided +[on the release page](https://github.com/FerretDB/documentdb/releases/tag/v0.106.0-ferretdb-2.5.0). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/). + +### documentdb v1.106-0 (August 07, 2025) ### +* Add internal extension that provides extensions to the `rum` index *[Feature]* * Enable let support for update queries *[Feature]*. Requires `EnableVariablesSupportForWriteCommands` to be `on`. * Enable let support for findAndModify queries *[Feature]*. Requires `EnableVariablesSupportForWriteCommands` to be `on`. -* Add internal extension that provides extensions to the `rum` index. *[Feature]* -* Optimized query for `usersInfo` command. +* Add internal extension that provides extensions to the `rum` index *[Feature]* +* Optimized query for `usersInfo` command * Support collation with `delete` *[Feature]*. Requires `EnableCollation` to be `on`. * Support for index hints for find/aggregate/count/distinct *[Feature]* * Support `createRole` command *[Feature]* @@ -26,7 +36,7 @@ to avoid unexpected updates. ### documentdb v0.105-0 (July 09, 2025) ### * Support `$bucketAuto` aggregation stage, with granularity types: `POWERSOF2`, `1-2-5`, `R5`, `R10`, `R20`, `R40`, `R80`, `E6`, `E12`, `E24`, `E48`, `E96`, `E192` *[Feature]* -* Support `conectionStatus` command *[Feature]*. +* Support `connectionStatus` command *[Feature]*. ### DocumentDB v0.104.0-ferretdb-2.3.0 (June 10, 2025) ### diff --git a/packaging/debian_files/changelog b/packaging/debian_files/changelog index 16a25f6ce..572056329 100644 --- a/packaging/debian_files/changelog +++ b/packaging/debian_files/changelog @@ -1,9 +1,23 @@ documentdb (DOCUMENTDB_VERSION) unstable; urgency=medium - * Support `$bucketAuto` aggregation stage, with granularity types: `POWERSOF2`, `1-2-5`, `R5`, `R10`, `R20`, `R40`, `R80`, `E6`, `E12`, `E24`, `E48`, `E96`, `E192` [Feature] - * Support `conectionStatus` command [Feature] + * Add internal extension that provides extensions to the rum index [Feature] + * Enable let support for update queries [Feature]. Requires EnableVariablesSupportForWriteCommands to be on. + * Enable let support for findAndModify queries [Feature]. Requires EnableVariablesSupportForWriteCommands to be on. + * Optimized query for usersInfo command + * Support collation with delete [Feature]. Requires EnableCollation to be on. + * Support for index hints for find/aggregate/count/distinct [Feature] + * Support createRole command [Feature] + * Add schema changes for Role CRUD APIs [Feature] + * Add support for using EntraId tokens via Plain Auth --- FerretDB Packages Tue, 09 Jul 2025 12:00:00 +0000 + -- FerretDB Packages Thu, 07 Aug 2025 12:00:00 +0000 + +documentdb (0.105-0) unstable; urgency=medium + + * Support $bucketAuto aggregation stage, with granularity types: POWERSOF2, 1-2-5, R5, R10, R20, R40, R80, E6, E12, E24, E48, E96, E192 [Feature] + * Support connectionStatus command [Feature] + + -- FerretDB Packages Wed, 09 Jul 2025 12:00:00 +0000 documentdb (0.104-0) unstable; urgency=medium @@ -19,7 +33,7 @@ documentdb (0.104-0) unstable; urgency=medium * Support compact command [Feature]. Requires documentdb.enablecompact GUC to be on. * Enable role privileges for usersInfo command [Feature] - -- FerretDB Packages Tue, 10 Jun 2025 12:00:00 +0000 + -- FerretDB Packages Mon, 09 Jun 2025 12:00:00 +0000 documentdb (0.103-0) unstable; urgency=medium diff --git a/packaging/defineversion/go.mod b/packaging/defineversion/go.mod index 44d1e78ab..5889ffcba 100644 --- a/packaging/defineversion/go.mod +++ b/packaging/defineversion/go.mod @@ -2,7 +2,7 @@ module github.com/FerretDB/documentdb/packaging/defineversion go 1.24 -toolchain go1.24.3 +toolchain go1.24.6 require ( github.com/sethvargo/go-githubactions v1.3.1 diff --git a/packaging/rpm_files/documentdb.spec b/packaging/rpm_files/documentdb.spec index 11887f40f..8e5d2ac7c 100644 --- a/packaging/rpm_files/documentdb.spec +++ b/packaging/rpm_files/documentdb.spec @@ -36,8 +36,8 @@ Requires: rum_%{pg_version} # libbid.a is bundled. %description -DocumentDB is the open-source engine powering vCore-based Azure Cosmos DB for MongoDB. -It offers a native implementation of document-oriented NoSQL database, enabling seamless +DocumentDB is the open-source engine powering vCore-based Azure Cosmos DB for MongoDB. +It offers a native implementation of document-oriented NoSQL database, enabling seamless CRUD operations on BSON data types within a PostgreSQL framework. %prep @@ -103,18 +103,35 @@ rm -rf %{buildroot}/usr/src/documentdb/build %{_libdir}/pkgconfig/libbson-static-1.0.pc %changelog -* Mon Jun 09 2025 Shuai Tian - 0.104-0-1 +* Thu Aug 07 2025 FerretDB Packages - 0.106-0-1 +- Add internal extension that provides extensions to the rum index *[Feature]* +- Enable let support for update queries *[Feature]*. Requires EnableVariablesSupportForWriteCommands to be on. +- Enable let support for findAndModify queries *[Feature]*. Requires EnableVariablesSupportForWriteCommands to be on. +- Optimized query for usersInfo command +- Support collation with delete *[Feature]*. Requires EnableCollation to be on. +- Support for index hints for find/aggregate/count/distinct *[Feature]* +- Support createRole command *[Feature]* +- Add schema changes for Role CRUD APIs *[Feature]* +- Add support for using EntraId tokens via Plain Auth + +* Wed Jul 09 2025 FerretDB Packages - 0.105-0-1 +- Support $bucketAuto aggregation stage, with granularity types: POWERSOF2, 1-2-5, R5, R10, R20, R40, R80, E6, E12, E24, E48, E96, E192 *[Feature]* +- Support connectionStatus command *[Feature]*. + +* Mon Jun 09 2025 FerretDB Packages - 0.104-0-1 - Add string case support for $toDate operator -- Support sort with collation in runtime*[Feature]* -- Support collation with $indexOfArray aggregation operator. *[Feature]* +- Support sort with collation in runtime *[Feature]* +- Support collation with $indexOfArray aggregation operator *[Feature]* - Support collation with arrays and objects comparisons *[Feature]* - Support background index builds *[Bugfix]* (#36) - Enable user CRUD by default *[Feature]* - Enable let support for delete queries *[Feature]*. Requires EnableVariablesSupportForWriteCommands to be on. - Enable rum_enable_index_scan as default on *[Perf]* - Add public documentdb-local Docker image with gateway to GHCR +- Support compact command *[Feature]*. Requires documentdb.enablecompact GUC to be on. +- Enable role privileges for usersInfo command *[Feature]* -* Fri May 09 2025 Shuai Tian - 0.103-0-1 +* Fri May 09 2025 FerretDB Packages - 0.103-0-1 - Support collation with aggregation and find on sharded collections *[Feature]* - Support $convert on binData to binData, string to binData and binData to string (except with format: auto) *[Feature]* - Fix list_databases for databases with size > 2 GB *[Bugfix]* (#119) @@ -123,7 +140,7 @@ rm -rf %{buildroot}/usr/src/documentdb/build - Support collation with $documents and $replaceWith stage of the aggregation pipeline *[Feature]* - Push pg_documentdb_gw for documentdb connections *[Feature]* -* Wed Mar 26 2025 Shuai Tian - 0.102-0-1 +* Wed Mar 26 2025 FerretDB Packages - 0.102-0-1 - Support index pushdown for vector search queries *[Bugfix]* - Support exact search for vector search queries *[Feature]* - Inline $match with let in $lookup pipelines as JOIN Filter *[Perf]* @@ -143,7 +160,7 @@ rm -rf %{buildroot}/usr/src/documentdb/build - Support the $dateFromString operator with full functionality *[Feature]* - Support extended syntax for $getField aggregation operator. Now the value of 'field' could be an expression that resolves to a string. *[Feature]* -* Wed Feb 12 2025 Shuai Tian - 0.101-0-1 +* Wed Feb 12 2025 FerretDB Packages - 0.101-0-1 - Push $graphlookup recursive CTE JOIN filters to index *[Perf]* - Build pg_documentdb for PostgreSQL 17 *[Infra]* (#13) - Enable support of currentOp aggregation stage, along with collstats, dbstats, and indexStats *[Commands]* (#52) @@ -151,5 +168,5 @@ rm -rf %{buildroot}/usr/src/documentdb/build - Skip loading documents if group expression is constant *[Perf]* - Fix Merge stage not outputing to target collection *[Bugfix]* (#20) -* Thu Jan 23 2025 Shuai Tian - 0.100-0-1 -- Initial Release \ No newline at end of file +* Thu Jan 23 2025 FerretDB Packages - 0.100-0-1 +- Initial Release From 27cb619d9a9b851ad5d33c1f706ffb074bfb7b52 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 14 Aug 2025 14:08:53 +0400 Subject: [PATCH 57/59] Use Debian 12 Bookwork as base image --- .github/workflows/ferretdb_packages.yml | 3 ++- packaging/development.Dockerfile | 2 +- packaging/production.Dockerfile | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index d9a52be0c..7e2e45940 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -210,8 +210,9 @@ jobs: go mod verify go run . -control-file ../../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} - # Official Docker images are built on top of Debian 12: + # Official Docker images are built on top of Debian 13 Trixie; use postgres:xx-bookworm for now: # docker run --rm postgres:15 cat /etc/os-release + # docker run --rm postgres:15-bookworm cat /etc/os-release - name: Download deb12-${{ matrix.pg }}-${{ steps.version.outputs.package_version }} uses: actions/download-artifact@v4 with: diff --git a/packaging/development.Dockerfile b/packaging/development.Dockerfile index 169989679..ad819a154 100644 --- a/packaging/development.Dockerfile +++ b/packaging/development.Dockerfile @@ -2,7 +2,7 @@ ARG POSTGRES_VERSION -FROM postgres:${POSTGRES_VERSION} +FROM postgres:${POSTGRES_VERSION}-bookworm ARG TARGETARCH ARG POSTGRES_VERSION diff --git a/packaging/production.Dockerfile b/packaging/production.Dockerfile index 9d4a254d6..6b5f6dedc 100644 --- a/packaging/production.Dockerfile +++ b/packaging/production.Dockerfile @@ -2,7 +2,7 @@ ARG POSTGRES_VERSION -FROM postgres:${POSTGRES_VERSION} +FROM postgres:${POSTGRES_VERSION}-bookworm ARG TARGETARCH ARG POSTGRES_VERSION From 3f80da90a9e0b04f214183aa656c4fb0b4bc31e4 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 21 Aug 2025 11:30:28 +0400 Subject: [PATCH 58/59] Add TODO comment about Debian 13 Trixie Refs FerretDB/FerretDB#5449. --- .github/workflows/ferretdb_packages.yml | 5 ++--- packaging/development.Dockerfile | 1 + packaging/production.Dockerfile | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ferretdb_packages.yml b/.github/workflows/ferretdb_packages.yml index 7e2e45940..f18307fdc 100644 --- a/.github/workflows/ferretdb_packages.yml +++ b/.github/workflows/ferretdb_packages.yml @@ -210,9 +210,8 @@ jobs: go mod verify go run . -control-file ../../pg_documentdb/documentdb.control -pg-version ${{ matrix.pg }} - # Official Docker images are built on top of Debian 13 Trixie; use postgres:xx-bookworm for now: - # docker run --rm postgres:15 cat /etc/os-release - # docker run --rm postgres:15-bookworm cat /etc/os-release + # We should use deb13 once we switch to Debian 13 Trixie. + # TODO https://github.com/FerretDB/FerretDB/issues/5449 - name: Download deb12-${{ matrix.pg }}-${{ steps.version.outputs.package_version }} uses: actions/download-artifact@v4 with: diff --git a/packaging/development.Dockerfile b/packaging/development.Dockerfile index ad819a154..b6f6e1a4a 100644 --- a/packaging/development.Dockerfile +++ b/packaging/development.Dockerfile @@ -2,6 +2,7 @@ ARG POSTGRES_VERSION +# TODO https://github.com/FerretDB/FerretDB/issues/5449 FROM postgres:${POSTGRES_VERSION}-bookworm ARG TARGETARCH diff --git a/packaging/production.Dockerfile b/packaging/production.Dockerfile index 6b5f6dedc..6845f2974 100644 --- a/packaging/production.Dockerfile +++ b/packaging/production.Dockerfile @@ -2,6 +2,7 @@ ARG POSTGRES_VERSION +# TODO https://github.com/FerretDB/FerretDB/issues/5449 FROM postgres:${POSTGRES_VERSION}-bookworm ARG TARGETARCH From e63835403de6ee285cea32410a765f26f1572727 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Sun, 9 Nov 2025 20:47:03 +0400 Subject: [PATCH 59/59] Prepare v0.107.0-ferretdb-2.7.0 release --- CHANGELOG.md | 13 ++++++++++++- packaging/defineversion/go.mod | 6 +++--- packaging/defineversion/go.sum | 4 ++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9c9c6b4b..d3928ccee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,15 @@ -### documentdb v1.107-0 (Unreleased) ### +### DocumentDB v0.107.0-ferretdb-2.7.0 (November 9, 2025) ### + +This version works best with FerretDB v2.7.0. +(We skipped v2.6.0 to align DocumentDB and FerretDB version numbers.) + +Docker images are available +[in the registry](https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb). +`.deb` packages for Debian and Ubuntu, and `.rpm` packages for Red Hat Enterprise Linux (RHEL) are provided +[on the release page](https://github.com/FerretDB/documentdb/releases/tag/v0.107.0-ferretdb-2.7.0). +See installation instructions [in our documentation](https://docs.ferretdb.io/installation/documentdb/). + +### documentdb v1.107-0 (August 20, 2025) ### - Support sort by _id against the _id index using the enableIndexOrderbyPushdown flag *[Feature]*. - Improvements to explain for various scan types *[Feature]*. diff --git a/packaging/defineversion/go.mod b/packaging/defineversion/go.mod index 5889ffcba..4aae104ae 100644 --- a/packaging/defineversion/go.mod +++ b/packaging/defineversion/go.mod @@ -1,12 +1,12 @@ module github.com/FerretDB/documentdb/packaging/defineversion -go 1.24 +go 1.25 -toolchain go1.24.6 +toolchain go1.25.4 require ( github.com/sethvargo/go-githubactions v1.3.1 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 ) require ( diff --git a/packaging/defineversion/go.sum b/packaging/defineversion/go.sum index 882f7c0cb..2402ca0e4 100644 --- a/packaging/defineversion/go.sum +++ b/packaging/defineversion/go.sum @@ -4,8 +4,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sethvargo/go-githubactions v1.3.1 h1:rlwwLRUaunWLQ1aN2o5Y+3s0xhaTC30YObCnilRx448= github.com/sethvargo/go-githubactions v1.3.1/go.mod h1:7/4WeHgYfSz9U5vwuToCK9KPnELVHAhGtRwLREOQV80= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=