Resolved conflicts

Author: MSeal

.ducktape/metadata/session_id

@@ -34,4 +34,5 @@ venv_examples
 .coverage
 **/coverage.xml
 **/test-report.xml
-*.ducktape
+.ducktape
+results

.gitignore

@@ -8,13 +8,13 @@ execution_time_limit:
 global_job_config:
   env_vars:
     - name: LIBRDKAFKA_VERSION
-      value: v2.11.1
+      value: v2.12.1
   prologue:
     commands:
       - checkout
       - mkdir artifacts
 blocks:
-  - name: "Wheels: OSX x64 - Python 3.7-3.12"
+  - name: "Wheels: OSX x64 - Python 3.8-3.12"
     run:
       when: "tag =~ '.*'"
     dependencies: []
@@ -34,7 +34,7 @@ blocks:
             - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse 2.16.2
             - tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse
             - artifact push workflow wheelhouse-macOS-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
-  - name: "Wheels: OSX x64 - Python 3.13"
+  - name: "Wheels: OSX x64 - Python 3.13-3.14"
     run:
       when: "tag =~ '.*'"
     dependencies: []
@@ -54,10 +54,10 @@ blocks:
       jobs:
         - name: Build
           commands:
-            - sem-version python 3.11
+            - sem-version python 3.13
             - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
-            - tar -czf wheelhouse-macOS-${ARCH}-py313.tgz wheelhouse
-            - artifact push workflow wheelhouse-macOS-${ARCH}-py313.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}-py313.tgz/
+            - tar -czf wheelhouse-macOS-${ARCH}-py313-plus.tgz wheelhouse
+            - artifact push workflow wheelhouse-macOS-${ARCH}-py313-plus.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}-py313-plus.tgz/
   - name: "Wheels: OSX arm64 - Python 3.8-3.12"
     run:
       when: "tag =~ '.*'"
@@ -80,7 +80,7 @@ blocks:
             - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse 2.16.2
             - tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse
             - artifact push workflow wheelhouse-macOS-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
-  - name: "Wheels: OSX arm64 - Python 3.13"
+  - name: "Wheels: OSX arm64 - Python 3.13-3.14"
     run:
       when: "tag =~ '.*'"
     dependencies: []
@@ -102,10 +102,10 @@ blocks:
       jobs:
         - name: Build
           commands:
-            - sem-version python 3.11
+            - sem-version python 3.13
             - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
-            - tar -czf wheelhouse-macOS-${ARCH}-py313.tgz wheelhouse
-            - artifact push workflow wheelhouse-macOS-${ARCH}-py313.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}-py313.tgz/
+            - tar -czf wheelhouse-macOS-${ARCH}-py313-plus.tgz wheelhouse
+            - artifact push workflow wheelhouse-macOS-${ARCH}-py313-plus.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}-py313-plus.tgz/
   - name: "Wheels: Linux arm64"
     run:
       when: "tag =~ '.*'"
@@ -122,7 +122,8 @@ blocks:
       jobs:
         - name: Build
           commands:
-            - ./tools/build-manylinux.sh "${LIBRDKAFKA_VERSION#v}"
+            - sem-version python 3.13
+            - ./tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
             - tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse
             - artifact push workflow wheelhouse-linux-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
   - name: "Wheels: Linux x64"
@@ -141,6 +142,7 @@ blocks:
       jobs:
         - name: Build
           commands:
+            - sem-version python 3.11
             - ./tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
             - tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse
             - artifact push workflow wheelhouse-linux-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/
@@ -170,6 +172,8 @@ blocks:
             - name: MSYSTEM
               value: UCRT64
           commands:
+            - pyenv install 3.11.9
+            - pyenv global 3.11.9
             - bash tools/mingw-w64/semaphore_commands.sh
             - bash tools/wheels/install-librdkafka.sh $env:LIBRDKAFKA_VERSION.TrimStart("v") dest
             - tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse
@@ -275,14 +279,114 @@ blocks:
             - python3 -m venv _venv && source _venv/bin/activate
             - chmod u+r+x tools/source-package-verification.sh
             - tools/source-package-verification.sh
+  - name: "Ducktape Performance Tests (Linux x64)"
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-ubuntu24-04-amd64-3
+      env_vars:
+        - name: OS_NAME
+          value: linux
+        - name: ARCH
+          value: x64
+        - name: BENCHMARK_BOUNDS_CONFIG
+          value: tests/ducktape/producer_benchmark_bounds.json
+        - name: BENCHMARK_ENVIRONMENT
+          value: ci
+      prologue:
+        commands:
+          - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY'
+      jobs:
+        - name: Build and Tests
+          commands:
+            # Setup Python environment
+            - sem-version python 3.9
+            - python3 -m venv _venv && source _venv/bin/activate
+
+            # Install ducktape framework and additional dependencies
+            - pip install ducktape psutil
+
+            # Install existing test requirements
+            - pip install -r requirements/requirements-tests-install.txt
+
+            # Build and install confluent-kafka from source
+            - lib_dir=dest/runtimes/$OS_NAME-$ARCH/native
+            - tools/wheels/install-librdkafka.sh "${LIBRDKAFKA_VERSION#v}" dest
+            - export CFLAGS="$CFLAGS -I${PWD}/dest/build/native/include"
+            - export LDFLAGS="$LDFLAGS -L${PWD}/${lib_dir}"
+            - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/$lib_dir"
+            - python3 -m pip install -e .
+
+            # Store project root for reliable navigation
+            - PROJECT_ROOT="${PWD}"
+
+            # Start Kafka cluster and Schema Registry using dedicated ducktape compose file (KRaft mode)
+            - cd "${PROJECT_ROOT}/tests/docker"
+            - docker-compose -f docker-compose.ducktape.yml up -d kafka schema-registry
+
+            # Debug: Check container status and logs
+            - echo "=== Container Status ==="
+            - docker-compose -f docker-compose.ducktape.yml ps
+            - echo "=== Kafka Logs ==="
+            - docker-compose -f docker-compose.ducktape.yml logs kafka | tail -50
+
+            # Wait for Kafka to be ready (using PLAINTEXT listener for external access)
+            - |
+              timeout 1800 bash -c '
+                counter=0
+                until docker-compose -f docker-compose.ducktape.yml exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; do
+                  echo "Waiting for Kafka... (attempt $((counter+1)))"
+
+                  # Show logs every 4th attempt (every 20 seconds)
+                  if [ $((counter % 4)) -eq 0 ] && [ $counter -gt 0 ]; then
+                    echo "=== Recent Kafka Logs ==="
+                    docker-compose -f docker-compose.ducktape.yml logs --tail=10 kafka
+                    echo "=== Container Status ==="
+                    docker-compose -f docker-compose.ducktape.yml ps kafka
+                  fi
+
+                  counter=$((counter+1))
+                  sleep 5
+                done
+              '
+            - echo "Kafka cluster is ready!"
+
+            # Wait for Schema Registry to be ready
+            - echo "=== Waiting for Schema Registry ==="
+            - |
+              timeout 300 bash -c '
+                counter=0
+                until curl -f http://localhost:8081/subjects >/dev/null 2>&1; do
+                  echo "Waiting for Schema Registry... (attempt $((counter+1)))"
+
+                  # Show logs every 3rd attempt (every 15 seconds)
+                  if [ $((counter % 3)) -eq 0 ] && [ $counter -gt 0 ]; then
+                    echo "=== Recent Schema Registry Logs ==="
+                    docker-compose -f docker-compose.ducktape.yml logs --tail=10 schema-registry
+                    echo "=== Schema Registry Container Status ==="
+                    docker-compose -f docker-compose.ducktape.yml ps schema-registry
+                  fi
+
+                  counter=$((counter+1))
+                  sleep 5
+                done
+              '
+            - echo "Schema Registry is ready!"
+
+            # Run standard ducktape tests with CI bounds
+            - cd "${PROJECT_ROOT}" && PYTHONPATH="${PROJECT_ROOT}" python tests/ducktape/run_ducktape_test.py
+
+            # Cleanup
+            - cd "${PROJECT_ROOT}/tests/docker" && docker-compose -f docker-compose.ducktape.yml down -v || true
   - name: "Packaging"
     run:
       when: "tag =~ '.*'"
     dependencies:
-      - "Wheels: OSX x64 - Python 3.7-3.12"
-      - "Wheels: OSX x64 - Python 3.13"
+      - "Wheels: OSX x64 - Python 3.8-3.12"
+      - "Wheels: OSX x64 - Python 3.13-3.14"
       - "Wheels: OSX arm64 - Python 3.8-3.12"
-      - "Wheels: OSX arm64 - Python 3.13"
+      - "Wheels: OSX arm64 - Python 3.13-3.14"
       - "Wheels: Linux arm64"
       - "Wheels: Linux x64"
       - "Wheels: Windows"
@@ -312,6 +416,6 @@ after_pipeline:
       - name: SonarQube
         commands:
           - checkout
-          - sem-version java 11
+          - sem-version java 17
           - artifact pull workflow coverage.xml
-          - emit-sonarqube-data --run_only_sonar_scan
+          - emit-sonarqube-data --enable_sonarqube_enterprise --run_only_sonar_scan