CLOUDP-339241 - re-enable multi arch smoke tests (#539)

# Summary

Key changes include:

- Added support for IBM Power and IBM Z architectures with podman
builder configuration
- Updated repository URLs from
`quay.io/mongodb/mongodb-kubernetes-tests` to
`268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests`
for release scenarios. This will prevent `meko-test` image to be
available also for our customers
- Enhanced build pipeline to support both Docker and Podman builders for
different architectures

## Proof of Work

Staging job is successful ->
https://spruce.mongodb.com/version/69033a1e9701b900075d742f (ignore ibm
z which is disabled in updated code)

Release job is successful ->
https://spruce.mongodb.com/version/69036de79701b900075df4de (ignore GKE
code snippets tasks, they need different fix.
[create_chart_release_pr](https://spruce.mongodb.com/task/mongodb_kubernetes_create_chart_release_pr_create_chart_release_pr_patch_b47894435713aeb2e8c26f99034205150cfad247_69036de79701b900075df4de_25_10_30_13_53_44?execution=1)
will fail due to missing tag)

We are not ready yet to enable smoke tests for IBM Z, because our
evergreen machines are unstable ->
https://jira.mongodb.org/browse/DEVPROD-23283

## Checklist

- [ ] Have you linked a jira ticket and/or is the ticket in the title?
- [ ] Have you checked whether your jira ticket required DOCSP changes?
- [x] Have you added changelog file?
    - use `skip-changelog` label if not needed
- refer to [Changelog files and Release
Notes](https://github.com/mongodb/mongodb-kubernetes/blob/master/CONTRIBUTING.md#changelog-files-and-release-notes)
section in CONTRIBUTING.md for more details

Author: MaciejKaras

.evergreen-functions.yml

@@ -294,6 +294,9 @@ functions:
     - command: subprocess.exec
       type: setup
       params:
+        env:
+          SKIP_MINIKUBE_SETUP: ${skip_minikube_setup!|false}
+          SKIP_INSTALL_REQUIREMENTS: ${skip_install_python_requirements!|true}
         working_dir: src/github.com/mongodb/mongodb-kubernetes
         add_to_path:
           - ${workdir}/bin
@@ -523,15 +526,6 @@ functions:
         working_dir: src/github.com/mongodb/mongodb-kubernetes
         binary: scripts/release/kubectl_mongodb/download_kubectl_plugin.sh
 
-  build_test_image_ibm:
-    - command: subprocess.exec
-      params:
-        shell: bash
-        working_dir: src/github.com/mongodb/mongodb-kubernetes
-        add_to_path:
-          - ${workdir}/bin
-        binary: scripts/evergreen/e2e/build_tests_image_ibm.sh
-
   pipeline:
     - command: subprocess.exec
       retry_on_failure: true
@@ -540,6 +534,8 @@ functions:
         shell: bash
         <<: *e2e_include_expansions_in_env
         working_dir: src/github.com/mongodb/mongodb-kubernetes
+        add_to_path:
+          - ${workdir}/bin
         binary: scripts/release/pipeline.sh
         env:
           IMAGE_NAME: ${image_name}
@@ -848,6 +844,8 @@ functions:
           - workdir
           - triggered_by_git_tag
           - OPERATOR_VERSION
+          - STAGING_VERSION_OVERRIDE
+          - SKIP_GITHUB_RELEASE_UPLOAD
         env:
           MACOS_NOTARY_KEY: ${macos_notary_keyid}
           MACOS_NOTARY_SECRET: ${macos_notary_secret}

.evergreen-release.yml

@@ -2,18 +2,6 @@ include:
   - filename: .evergreen-functions.yml
 
 tasks:
-  - name: build_test_image_for_smoke_tests
-    display_name: build_test_image_for_smoke_tests
-    tags: [ "image_release" ]
-    allowed_requesters: [ "patch", "github_tag" ]
-    commands:
-      - func: clone
-      - func: setup_building_host
-      - func: download_multi_cluster_binary
-      - func: pipeline
-        vars:
-          image_name: meko-tests
-
   - name: release_operator
     tags: [ "image_release" ]
     allowed_requesters: [ "patch", "github_tag" ]
@@ -119,8 +107,8 @@ tasks:
           task: prepare_and_upload_openshift_bundles
 
   - name: release_kubectl_mongodb_plugin
-    allowed_requesters: [ "patch", "github_tag" ]
     tags: [ "binary_release" ]
+    allowed_requesters: [ "patch", "github_tag" ]
     commands:
       - func: clone
       - func: python_venv
@@ -150,10 +138,10 @@ buildvariants:
   - name: release_images
     display_name: release_images
     tags: [ "release" ]
-    allowed_requesters: [ "patch", "github_tag" ]
-    max_hosts: -1
     run_on:
       - release-ubuntu2404-small # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780
+    allowed_requesters: [ "patch", "github_tag" ]
+    max_hosts: -1
     tasks:
       - name: release_operator
       - name: release_init_appdb
@@ -166,12 +154,12 @@ buildvariants:
   - name: preflight_release_images
     display_name: preflight_release_images
     tags: [ "release" ]
+    run_on:
+      - rhel90-large
     allowed_requesters: [ "patch", "github_tag" ]
     depends_on:
       - name: "*"
         variant: release_images
-    run_on:
-      - rhel90-large
     expansions:
       preflight_submit: true
     tasks:
@@ -180,56 +168,100 @@ buildvariants:
   - name: prepare_openshift_bundles
     display_name: prepare_openshift_bundles
     tags: [ "release" ]
+    run_on:
+      - ubuntu2404-small
     allowed_requesters: [ "patch", "github_tag" ]
     depends_on:
       - name: "*"
         variant: release_images
       - name: "*"
         variant: preflight_release_images
-    run_on:
-      - ubuntu2404-small
     tasks:
       - name: run_conditionally_prepare_and_upload_openshift_bundles
 
   - name: prerelease_kind_code_snippets
     display_name: prerelease_kind_code_snippets
     tags: [ "release", "prerelease_code_snippets" ]
+    run_on:
+      - ubuntu2404-large
     allowed_requesters: [ "patch", "github_tag" ]
     depends_on:
       - name: "*"
         variant: release_images
         patch_optional: true
-    run_on:
-      - ubuntu2404-large
     tasks:
       - name: kind_code_snippets_task_group
 
   - name: prerelease_gke_code_snippets
     display_name: prerelease_gke_code_snippets
     tags: [ "release", "prerelease_code_snippets" ]
+    run_on:
+      - ubuntu2404-small
     allowed_requesters: ["patch", "github_tag"]
     depends_on:
       - variant: release_images
         name: '*'
         patch_optional: true
-    run_on:
-      - ubuntu2404-small
     tasks:
       - name: gke_code_snippets_task_group
 
-  - name: init_smoke_tests
-    display_name: init_smoke_tests
+  - name: init_test_run_release
+    display_name: init_test_run
+    tags: [ "release", "e2e_smoke_release_test_suite" ]
+    run_on:
+      - release-ubuntu2404-small # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780
+    allowed_requesters: [ "patch", "github_tag" ]
+    depends_on:
+      - name: release_kubectl_mongodb_plugin
+        variant: release_kubectl_mongodb_plugin
+    tasks:
+      - name: build_test_image
+
+  - name: init_smoke_tests_ibm_power_release
+    display_name: init_smoke_tests_ibm_power
+    max_hosts: -1
     tags: [ "release", "e2e_smoke_release_test_suite" ]
+    run_on:
+      - release-rhel9-power-small # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780
+      - release-rhel9-power-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780
     allowed_requesters: [ "patch", "github_tag" ]
+    depends_on:
+      - name: release_kubectl_mongodb_plugin
+        variant: release_kubectl_mongodb_plugin
+    tasks:
+      - name: build_test_image_ibm_power
+
+  - name: init_smoke_tests_ibm_z_release
+    display_name: init_smoke_tests_ibm_z
+    max_hosts: -1
+    tags: [ "release", "e2e_smoke_release_test_suite" ]
+    # TODO: Re-enable when ibm_z series is stable
+    # https://jira.mongodb.org/browse/DEVPROD-23283
+    disable: true
+    run_on:
+      - release-rhel9-zseries-small # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780
+      - release-rhel9-zseries-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780
+    allowed_requesters: [ "patch", "github_tag" ]
+    depends_on:
+      - name: release_kubectl_mongodb_plugin
+        variant: release_kubectl_mongodb_plugin
+    tasks:
+      - name: build_test_image_ibm_z
+
+  - name: init_smoke_tests_arm_release
+    display_name: init_smoke_tests_arm
+    max_hosts: -1
+    tags: [ "release", "e2e_smoke_release_test_suite" ]
     run_on:
       - release-ubuntu2404-small # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780
+    allowed_requesters: [ "patch", "github_tag" ]
     depends_on:
       - name: release_kubectl_mongodb_plugin
         variant: release_kubectl_mongodb_plugin
     tasks:
-      - name: build_test_image_for_smoke_tests
+      - name: build_test_image_arm
 
-  - name: e2e_smoke
+  - name: e2e_smoke_release
     display_name: e2e_smoke
     tags: [ "release", "e2e_smoke_release_test_suite" ]
     run_on:
@@ -239,24 +271,118 @@ buildvariants:
       - name: "*"
         variant: release_images
       - name: "*"
-        variant: init_smoke_tests
+        variant: init_test_run_release
     tasks:
       - name: e2e_smoke_task_group
 
-  - name: e2e_static_smoke
+  - name: e2e_static_smoke_release
     display_name: e2e_static_smoke
-    tags: [ "release", "e2e_smoke_release_test_suite" ]
+    tags: [ "release", "e2e_smoke_release_test_suite", "static" ]
     run_on:
       - ubuntu2404-large
     allowed_requesters: [ "patch", "github_tag" ]
     depends_on:
       - name: "*"
         variant: release_images
       - name: "*"
-        variant: init_smoke_tests
+        variant: init_test_run_release
     tasks:
       - name: e2e_smoke_task_group
 
+  - name: e2e_smoke_ibm_power_release
+    display_name: e2e_smoke_ibm_power
+    tags: [ "release", "e2e_smoke_release_test_suite" ]
+    run_on:
+      - rhel9-power-small
+      - rhel9-power-large
+    allowed_requesters: [ "patch", "github_tag" ]
+    depends_on:
+      - name: "*"
+        variant: release_images
+      - name: "*"
+        variant: init_smoke_tests_ibm_power_release
+    tasks:
+      - name: e2e_smoke_ibm_task_group
+
+  - name: e2e_static_smoke_ibm_power_release
+    display_name: e2e_static_smoke_ibm_power
+    tags: [ "release", "e2e_smoke_release_test_suite", "static" ]
+    run_on:
+      - rhel9-power-small
+      - rhel9-power-large
+    allowed_requesters: [ "patch", "github_tag" ]
+    depends_on:
+      - name: "*"
+        variant: release_images
+      - name: "*"
+        variant: init_smoke_tests_ibm_power_release
+    tasks:
+      - name: e2e_smoke_ibm_task_group
+
+  - name: e2e_smoke_ibm_z_release
+    display_name: e2e_smoke_ibm_z
+    tags: [ "release", "e2e_smoke_release_test_suite" ]
+    # TODO: Re-enable when ibm_z series is stable
+    # https://jira.mongodb.org/browse/DEVPROD-23283
+    disable: true
+    run_on:
+      - rhel9-zseries-small
+      - rhel9-zseries-large
+    allowed_requesters: [ "patch", "github_tag" ]
+    depends_on:
+      - name: "*"
+        variant: release_images
+      - name: "*"
+        variant: init_smoke_tests_ibm_z_release
+    tasks:
+      - name: e2e_smoke_ibm_task_group
+
+  - name: e2e_static_smoke_ibm_z_release
+    display_name: e2e_static_smoke_ibm_z
+    tags: [ "release", "e2e_smoke_release_test_suite", "static" ]
+    # TODO: Re-enable when ibm_z series is stable
+    # https://jira.mongodb.org/browse/DEVPROD-23283
+    disable: true
+    run_on:
+      - rhel9-zseries-small
+      - rhel9-zseries-large
+    allowed_requesters: [ "patch", "github_tag" ]
+    depends_on:
+      - name: "*"
+        variant: release_images
+      - name: "*"
+        variant: init_smoke_tests_ibm_z_release
+    tasks:
+      - name: e2e_smoke_ibm_task_group
+
+  - name: e2e_smoke_arm_release
+    display_name: e2e_smoke_arm
+    tags: [ "release", "e2e_smoke_release_test_suite" ]
+    run_on:
+      - ubuntu2404-arm64-large
+    allowed_requesters: [ "patch", "github_tag" ]
+    depends_on:
+      - name: "*"
+        variant: release_images
+      - name: "*"
+        variant: init_smoke_tests_arm_release
+    tasks:
+      - name: e2e_smoke_arm_task_group
+
+  - name: e2e_static_smoke_arm_release
+    display_name: e2e_static_smoke_arm
+    tags: [ "release", "e2e_smoke_release_test_suite", "static" ]
+    run_on:
+      - ubuntu2404-arm64-large
+    allowed_requesters: [ "patch", "github_tag" ]
+    depends_on:
+      - name: "*"
+        variant: release_images
+      - name: "*"
+        variant: init_smoke_tests_arm_release
+    tasks:
+      - name: e2e_smoke_arm_task_group
+
   - name: release_kubectl_mongodb_plugin
     display_name: release_kubectl_mongodb_plugin
     tags: [ "release" ]