From 7e7f65c7761877bddce4cc39fd4e078576372d88 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 13 Nov 2025 08:46:48 +0700 Subject: [PATCH 01/13] feat: preliminary podman tests Work was done based on https://github.com/getsentry/self-hosted/compare/master...doc-sheet:self-hosted:podman-tests Co-authored-by: Nikita Korolev <66738864+doc-sheet@users.noreply.github.com> --- .github/workflows/test.yml | 3 ++- action.yaml | 54 +++++++++++++++++++++++++++++++------- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 49298277c4f..4e78a51a22e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,7 +47,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-24.04, ubuntu-24.04-arm] - container_engine: ['docker'] # TODO: add 'podman' into the list + container_engine: ['docker', 'podman'] compose_profiles: ['feature-complete', 'errors-only'] name: ${{ format('integration test{0}{1}{2}', matrix.os == 'ubuntu-24.04-arm' && ' (arm64)' || '', matrix.container_engine == 'podman' && ' (podman)' || '', matrix.compose_profiles == 'errors-only' && ' (errors-only)' || '') }} env: @@ -73,4 +73,5 @@ jobs: uses: './' with: compose_profiles: ${{ matrix.compose_profiles }} + container_engine: ${{ matrix.container_engine }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/action.yaml b/action.yaml index 7f7a98f84fc..4fecaedb336 100644 --- a/action.yaml +++ b/action.yaml @@ -9,6 +9,9 @@ inputs: compose_profiles: required: false description: "Docker Compose profile to use. Defaults to feature-complete." + container_engine: + required: false + default: docker CODECOV_TOKEN: required: false description: "The Codecov token to upload coverage." @@ -67,9 +70,11 @@ runs: fi - name: Get Compose + if: ${{ inputs.container_engine == 'docker' }} uses: getsentry/self-hosted/get-compose-action@master - name: Compute Docker Volume Cache Keys + if: ${{ inputs.container_engine == 'docker' }} id: cache_key shell: bash run: | @@ -83,6 +88,7 @@ runs: echo "ARCH=$(uname -m)" >> $GITHUB_OUTPUT - name: Restore Sentry Volume Cache + if: ${{ inputs.container_engine == 'docker' }} id: restore_cache_sentry uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd with: @@ -93,6 +99,7 @@ runs: sentry-postgres - name: Restore Snuba Volume Cache + if: ${{ inputs.container_engine == 'docker' }} id: restore_cache_snuba uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd with: @@ -103,6 +110,7 @@ runs: sentry-clickhouse - name: Restore Kafka Volume Cache + if: ${{ inputs.container_engine == 'docker' }} id: restore_cache_kafka uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd with: @@ -121,6 +129,8 @@ runs: # changed. Heats up your head a bit but if you think about it, it makes sense. SKIP_SENTRY_MIGRATIONS: ${{ steps.restore_cache_sentry.outputs.cache-hit == 'true' && '1' || '' }} SKIP_SNUBA_MIGRATIONS: ${{ steps.restore_cache_snuba.outputs.cache-hit == 'true' && '1' || '' }} + CONTAINER_ENGINE: ${{ inputs.container_engine }} + DEBUG: ${{ env.ACTIONS_STEP_DEBUG == 'true' }} shell: bash run: | cd ${{ github.action_path }} @@ -128,32 +138,40 @@ runs: cat <> sentry/enhance-image.sh #!/bin/bash touch /created-by-enhance-image + echo 'debconf debconf/frontend select noninteractive' | debconf-set-selections apt-get update - apt-get install -y gcc libsasl2-dev python-dev-is-python3 libldap2-dev libssl-dev + apt-get install --yes --no-install-recommends gcc libsasl2-dev python-dev-is-python3 libldap2-dev libssl-dev EOT chmod 755 sentry/enhance-image.sh echo "python-ldap" > sentry/requirements.txt ./install.sh --no-report-self-hosted-issues --skip-commit-check - - name: Save Sentry Volume Cache - if: steps.restore_cache_sentry.outputs.cache-hit != 'true' + - name: Save Sentry Docker Volume Cache + if: ${{ steps.restore_cache_sentry.outputs.cache-hit != 'true' && inputs.container_engine == 'docker' }} uses: BYK/docker-volume-cache-action/save@be89365902126f508dcae387a32ec3712df6b1cd with: key: ${{ steps.restore_cache_sentry.outputs.cache-primary-key }} volumes: | sentry-postgres - - name: Save Snuba Volume Cache - if: steps.restore_cache_snuba.outputs.cache-hit != 'true' + - name: Save Sentry Podman Volume Cache + if: ${{ steps.restore_cache_sentry.outputs.cache-hit != 'true' && inputs.container_engine == 'podman' }} + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 + with: + key: ${{ steps.restore_cache_sentry.outputs.cache-primary-key }}-podman + path: ${{ runner.temp }}/sentry-postgres.tar + + - name: Save Snuba Docker Volume Cache + if: ${{ steps.restore_cache_snuba.outputs.cache-hit != 'true' && inputs.container_engine == 'docker' }} uses: BYK/docker-volume-cache-action/save@be89365902126f508dcae387a32ec3712df6b1cd with: key: ${{ steps.restore_cache_snuba.outputs.cache-primary-key }} volumes: | sentry-clickhouse - - name: Save Kafka Volume Cache - if: steps.restore_cache_kafka.outputs.cache-hit != 'true' + - name: Save Kafka Docker Volume Cache + if: ${{ steps.restore_cache_kafka.outputs.cache-hit != 'true' && inputs.container_engine == 'docker' }} uses: BYK/docker-volume-cache-action/save@be89365902126f508dcae387a32ec3712df6b1cd with: key: ${{ steps.restore_cache_kafka.outputs.cache-primary-key }} @@ -171,10 +189,13 @@ runs: sudo swapon --show free -h - - name: Integration Test - shell: bash + - name: Integration Test with docker + if: ${{ inputs.container_engine == 'docker' }} env: + CONTAINER_ENGINE: ${{ inputs.container_engine }} + CONTAINER_ENGINE_PODMAN: ${{ inputs.container_engine == 'podman' && '1' || '0' }} COMPOSE_PROFILES: ${{ inputs.compose_profiles }} + shell: bash run: | sudo chown root /usr/bin/rsync && sudo chmod u+s /usr/bin/rsync rsync -aW --super --numeric-ids --no-compress --mkpath \ @@ -185,6 +206,21 @@ runs: cd ${{ github.action_path }} pytest -x --cov --junitxml=junit.xml _integration-test/ + - name: Integration Test with podman + if: ${{ inputs.container_engine == 'podman' }} + env: + CONTAINER_ENGINE: ${{ inputs.container_engine }} + CONTAINER_ENGINE_PODMAN: ${{ inputs.container_engine == 'podman' && '1' || '0' }} + COMPOSE_PROFILES: ${{ inputs.compose_profiles }} + shell: bash + run: | + set -x + for volume in sentry-postgres sentry-clickhouse sentry-kafka; do + podman volume export -o "${RUNNER_TEMP}/volumes/${volume}.tar" "$volume" + done + cd ${{ github.action_path }} + pytest -s -v -x --cov --junitxml=junit.xml _integration-test/ + - name: Upload coverage to Codecov if: inputs.CODECOV_TOKEN continue-on-error: true From 994b555e740e772f9467478467c8765a6ef9d14c Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 16 Nov 2025 15:49:37 +0700 Subject: [PATCH 02/13] fix: use start_service_and_wait_ready for setup s3 nodestore --- install/bootstrap-s3-nodestore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/bootstrap-s3-nodestore.sh b/install/bootstrap-s3-nodestore.sh index c2cf91669f5..a1e273a00c2 100644 --- a/install/bootstrap-s3-nodestore.sh +++ b/install/bootstrap-s3-nodestore.sh @@ -1,6 +1,6 @@ echo "${_group}Bootstrapping seaweedfs (node store)..." -$dc up --wait seaweedfs postgres +start_service_and_wait_ready seaweedfs postgres $dc exec -e "HTTP_PROXY=${HTTP_PROXY:-}" -e "HTTPS_PROXY=${HTTPS_PROXY:-}" -e "NO_PROXY=${NO_PROXY:-}" -e "http_proxy=${http_proxy:-}" -e "https_proxy=${https_proxy:-}" -e "no_proxy=${no_proxy:-}" seaweedfs apk add --no-cache s3cmd $dc exec seaweedfs mkdir -p /data/idx/ s3cmd="$dc exec seaweedfs s3cmd" From 3891f1dfa4918a514b5a9b05592782e0b9348dd3 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 16 Nov 2025 15:53:14 +0700 Subject: [PATCH 03/13] fix: add 'arm64' into supported docker arch --- install/detect-platform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/detect-platform.sh b/install/detect-platform.sh index 9009f79b63d..4bebae7cba3 100644 --- a/install/detect-platform.sh +++ b/install/detect-platform.sh @@ -22,7 +22,7 @@ fi export DOCKER_ARCH=$($CONTAINER_ENGINE info --format "$FORMAT") if [[ "$DOCKER_ARCH" = "x86_64" || "$DOCKER_ARCH" = "amd64" ]]; then export DOCKER_PLATFORM="linux/amd64" -elif [[ "$DOCKER_ARCH" = "aarch64" ]]; then +elif [[ "$DOCKER_ARCH" = "aarch64" || "$DOCKER_ARCH" = "arm64" ]]; then export DOCKER_PLATFORM="linux/arm64" else echo "FAIL: Unsupported docker architecture $DOCKER_ARCH." From 98e0533cbbe5eda8a03a85a01bf49da54b1f36e3 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 16 Nov 2025 15:54:45 +0700 Subject: [PATCH 04/13] chore: put note on container engine output info --- install/detect-platform.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install/detect-platform.sh b/install/detect-platform.sh index 4bebae7cba3..a69eddfd9ca 100644 --- a/install/detect-platform.sh +++ b/install/detect-platform.sh @@ -22,6 +22,7 @@ fi export DOCKER_ARCH=$($CONTAINER_ENGINE info --format "$FORMAT") if [[ "$DOCKER_ARCH" = "x86_64" || "$DOCKER_ARCH" = "amd64" ]]; then export DOCKER_PLATFORM="linux/amd64" +# NOTE(aldy505): Docker outputs `aarch64`, but Podman outputs `arm64`. elif [[ "$DOCKER_ARCH" = "aarch64" || "$DOCKER_ARCH" = "arm64" ]]; then export DOCKER_PLATFORM="linux/arm64" else From 7db0d7ed0d05a51e49dfa7e6e8356e283e7ed2ef Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 16 Nov 2025 16:01:04 +0700 Subject: [PATCH 05/13] debug: check ulimits value --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4e78a51a22e..35077103edb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,6 +68,9 @@ jobs: pip3 install --user https://github.com/containers/podman-compose/archive/main.tar.gz echo "PODMAN_COMPOSE_PROVIDER=podman-compose" >> $GITHUB_ENV echo "PODMAN_COMPOSE_WARNING_LOGS=false" >> $GITHUB_ENV + + # DEBUG: Check ulimits + ulimit -a - name: Use action from local checkout uses: './' From a2f6744fe8e6028be13c11cc020a4f6f7381bd51 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 16 Nov 2025 16:09:32 +0700 Subject: [PATCH 06/13] fix: increase ulimit for podman during bootstrap-snuba --- .github/workflows/test.yml | 3 --- install/bootstrap-snuba.sh | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 35077103edb..4e78a51a22e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,9 +68,6 @@ jobs: pip3 install --user https://github.com/containers/podman-compose/archive/main.tar.gz echo "PODMAN_COMPOSE_PROVIDER=podman-compose" >> $GITHUB_ENV echo "PODMAN_COMPOSE_WARNING_LOGS=false" >> $GITHUB_ENV - - # DEBUG: Check ulimits - ulimit -a - name: Use action from local checkout uses: './' diff --git a/install/bootstrap-snuba.sh b/install/bootstrap-snuba.sh index 496becd6c57..c1a1306515c 100644 --- a/install/bootstrap-snuba.sh +++ b/install/bootstrap-snuba.sh @@ -1,6 +1,11 @@ echo "${_group}Bootstrapping and migrating Snuba ..." if [[ -z "${SKIP_SNUBA_MIGRATIONS:-}" ]]; then + # NOTE(aldy505): Temporarily increase the number of open files to avoid errors + # Otherwise, we'll get "crun: setrlimit `RLIMIT_NOFILE`: Operation not permitted: OCI permission denied" + if [[ "$CONTAINER_ENGINE" == "podman" ]]; then + ulimit -u 100000 + fi $dcr snuba-api bootstrap --force else echo "Skipped DB migrations due to SKIP_SNUBA_MIGRATIONS=$SKIP_SNUBA_MIGRATIONS" From 1c82856d2473576577bf2810c9046f93cf8286a0 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 16 Nov 2025 19:31:30 +0700 Subject: [PATCH 07/13] fix: use host's ulimit --- install/bootstrap-snuba.sh | 5 ----- install/dc-detect-version.sh | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/install/bootstrap-snuba.sh b/install/bootstrap-snuba.sh index c1a1306515c..496becd6c57 100644 --- a/install/bootstrap-snuba.sh +++ b/install/bootstrap-snuba.sh @@ -1,11 +1,6 @@ echo "${_group}Bootstrapping and migrating Snuba ..." if [[ -z "${SKIP_SNUBA_MIGRATIONS:-}" ]]; then - # NOTE(aldy505): Temporarily increase the number of open files to avoid errors - # Otherwise, we'll get "crun: setrlimit `RLIMIT_NOFILE`: Operation not permitted: OCI permission denied" - if [[ "$CONTAINER_ENGINE" == "podman" ]]; then - ulimit -u 100000 - fi $dcr snuba-api bootstrap --force else echo "Skipped DB migrations due to SKIP_SNUBA_MIGRATIONS=$SKIP_SNUBA_MIGRATIONS" diff --git a/install/dc-detect-version.sh b/install/dc-detect-version.sh index f7a4cbdda97..9bc3c448700 100644 --- a/install/dc-detect-version.sh +++ b/install/dc-detect-version.sh @@ -53,7 +53,7 @@ if [[ "$CONTAINER_ENGINE" == "podman" ]]; then # Disable pod creation as these are one-off commands and creating a pod # prints its pod id to stdout which is messing with the output that we # rely on various places such as configuration generation - dcr="$dc --profile=feature-complete --in-pod=false run --rm" + dcr="$dc --profile=feature-complete --in-pod=false run --rm --ulimit host" else proxy_args_dc=$proxy_args dcr="$dc run --pull=never --rm" From c4d807b3fe48fdd7364e262a7a081845ab43d9cc Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 16 Nov 2025 19:37:22 +0700 Subject: [PATCH 08/13] fix: use --podman-run-args --- install/dc-detect-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/dc-detect-version.sh b/install/dc-detect-version.sh index 9bc3c448700..73a111eac13 100644 --- a/install/dc-detect-version.sh +++ b/install/dc-detect-version.sh @@ -53,7 +53,7 @@ if [[ "$CONTAINER_ENGINE" == "podman" ]]; then # Disable pod creation as these are one-off commands and creating a pod # prints its pod id to stdout which is messing with the output that we # rely on various places such as configuration generation - dcr="$dc --profile=feature-complete --in-pod=false run --rm --ulimit host" + dcr="$dc --profile=feature-complete --in-pod=false run --rm --podman-run-args=\"--ulimit host\"" else proxy_args_dc=$proxy_args dcr="$dc run --pull=never --rm" From 4856c52392003be87a631a4b593550f552b4c060 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 16 Nov 2025 19:40:19 +0700 Subject: [PATCH 09/13] fix: try using single quotes --- install/dc-detect-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/dc-detect-version.sh b/install/dc-detect-version.sh index 73a111eac13..68bb5e6b42a 100644 --- a/install/dc-detect-version.sh +++ b/install/dc-detect-version.sh @@ -53,7 +53,7 @@ if [[ "$CONTAINER_ENGINE" == "podman" ]]; then # Disable pod creation as these are one-off commands and creating a pod # prints its pod id to stdout which is messing with the output that we # rely on various places such as configuration generation - dcr="$dc --profile=feature-complete --in-pod=false run --rm --podman-run-args=\"--ulimit host\"" + dcr="$dc --profile=feature-complete --in-pod=false run --rm --podman-run-args='--ulimit host'" else proxy_args_dc=$proxy_args dcr="$dc run --pull=never --rm" From f7c47bd76cb7c07b719dec17c8afc5c8b58c7e2d Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 16 Nov 2025 19:42:44 +0700 Subject: [PATCH 10/13] fix: don't use equal sign --- install/dc-detect-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/dc-detect-version.sh b/install/dc-detect-version.sh index 68bb5e6b42a..3ffd2ff44b5 100644 --- a/install/dc-detect-version.sh +++ b/install/dc-detect-version.sh @@ -53,7 +53,7 @@ if [[ "$CONTAINER_ENGINE" == "podman" ]]; then # Disable pod creation as these are one-off commands and creating a pod # prints its pod id to stdout which is messing with the output that we # rely on various places such as configuration generation - dcr="$dc --profile=feature-complete --in-pod=false run --rm --podman-run-args='--ulimit host'" + dcr="$dc --profile=feature-complete --in-pod=false run --rm --podman-run-args '--ulimit host'" else proxy_args_dc=$proxy_args dcr="$dc run --pull=never --rm" From 1fc99c2f16892e1556c8a7af13f2062aba6d90e8 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 16 Nov 2025 19:47:16 +0700 Subject: [PATCH 11/13] fix: it should be before 'run' --- install/dc-detect-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/dc-detect-version.sh b/install/dc-detect-version.sh index 3ffd2ff44b5..f77a46a182a 100644 --- a/install/dc-detect-version.sh +++ b/install/dc-detect-version.sh @@ -53,7 +53,7 @@ if [[ "$CONTAINER_ENGINE" == "podman" ]]; then # Disable pod creation as these are one-off commands and creating a pod # prints its pod id to stdout which is messing with the output that we # rely on various places such as configuration generation - dcr="$dc --profile=feature-complete --in-pod=false run --rm --podman-run-args '--ulimit host'" + dcr="$dc --profile=feature-complete --in-pod=false --podman-run-args \"--ulimit host\" run --rm" else proxy_args_dc=$proxy_args dcr="$dc run --pull=never --rm" From db2e44a5498795c5cbc92b3a1c8566ee759eba19 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 16 Nov 2025 19:50:03 +0700 Subject: [PATCH 12/13] fix: continue to set ulimit on GHA --- .github/workflows/test.yml | 2 ++ install/dc-detect-version.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4e78a51a22e..ddd11b4527a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,6 +68,8 @@ jobs: pip3 install --user https://github.com/containers/podman-compose/archive/main.tar.gz echo "PODMAN_COMPOSE_PROVIDER=podman-compose" >> $GITHUB_ENV echo "PODMAN_COMPOSE_WARNING_LOGS=false" >> $GITHUB_ENV + sudo ulimit -u 100000 + ulimit -u 100000 - name: Use action from local checkout uses: './' diff --git a/install/dc-detect-version.sh b/install/dc-detect-version.sh index f77a46a182a..f7a4cbdda97 100644 --- a/install/dc-detect-version.sh +++ b/install/dc-detect-version.sh @@ -53,7 +53,7 @@ if [[ "$CONTAINER_ENGINE" == "podman" ]]; then # Disable pod creation as these are one-off commands and creating a pod # prints its pod id to stdout which is messing with the output that we # rely on various places such as configuration generation - dcr="$dc --profile=feature-complete --in-pod=false --podman-run-args \"--ulimit host\" run --rm" + dcr="$dc --profile=feature-complete --in-pod=false run --rm" else proxy_args_dc=$proxy_args dcr="$dc run --pull=never --rm" From 434641579c76f4dc73e90c285391be2c3afea830 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 16 Nov 2025 19:52:23 +0700 Subject: [PATCH 13/13] I give up for today --- .github/workflows/test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ddd11b4527a..4e78a51a22e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,8 +68,6 @@ jobs: pip3 install --user https://github.com/containers/podman-compose/archive/main.tar.gz echo "PODMAN_COMPOSE_PROVIDER=podman-compose" >> $GITHUB_ENV echo "PODMAN_COMPOSE_WARNING_LOGS=false" >> $GITHUB_ENV - sudo ulimit -u 100000 - ulimit -u 100000 - name: Use action from local checkout uses: './'