From 324cf310ca27103c5035d220aefe63c3f40d300d Mon Sep 17 00:00:00 2001 From: ptiurin Date: Fri, 25 Jul 2025 09:31:17 +0100 Subject: [PATCH 1/5] ci(NoTicket): use core setup action in tests --- .github/workflows/integration-tests-core.yml | 88 ++------------------ 1 file changed, 8 insertions(+), 80 deletions(-) diff --git a/.github/workflows/integration-tests-core.yml b/.github/workflows/integration-tests-core.yml index cf9deca010f..6d3ad2a7e50 100644 --- a/.github/workflows/integration-tests-core.yml +++ b/.github/workflows/integration-tests-core.yml @@ -41,18 +41,16 @@ on: secrets: SLACK_BOT_TOKEN: required: false -env: - DEFAULT_IMAGE_TAG: ${{ vars.DEFAULT_CORE_IMAGE_TAG }} jobs: run-core-integration-tests: runs-on: ${{ inputs.os_name }} - env: - DOCKER_COMPOSE_FILE: ${{ github.workspace }}/.github/resources/core/docker-compose.yml - SERVICE_PORT: 3473 - SERVICE_URL: http://localhost:3473 - MAX_RETRIES: 30 - RETRY_INTERVAL: 2 steps: + - name: Setup Firebolt Core + id: setup-core + uses: firebolt-db/action-setup-core@main + with: + tag_version: ${{ inputs.tag_version || vars.DEFAULT_CORE_IMAGE_TAG }} + - name: Check out code uses: actions/checkout@v4 with: @@ -68,76 +66,6 @@ jobs: python -m pip install --upgrade pip pip install ".[dev]" - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Write certificate and certificate key to file - run: | - mkdir "${{ github.workspace }}/.github/resources/core/certs" - pip install trustme - # Generate a self-signed certificate for localhost - python3 -m trustme -d "${{ github.workspace }}/.github/resources/core/certs/" - - - name: Install certs to keystore - run: | - sudo cp ${GITHUB_WORKSPACE}/.github/resources/core/certs/client.pem /usr/local/share/ca-certificates/client.crt - sudo update-ca-certificates - - # if no image tag was passed in, then use the image tag from the defaults - - name: Set image tag - id: set-tag - run: | - IMAGE_TAG="${{ inputs.tag_version }}" - if [ -z "$IMAGE_TAG" ]; then - IMAGE_TAG="$DEFAULT_IMAGE_TAG" - fi - echo "tag=$IMAGE_TAG" >> $GITHUB_OUTPUT - - - name: Prepare docker-compose.yml - run: | - if [ ! -f "$DOCKER_COMPOSE_FILE" ]; then - echo "Error: Docker compose file not found at $DOCKER_COMPOSE_FILE" - exit 1 - fi - sed -i "s|\${IMAGE_TAG}|${{ steps.set-tag.outputs.tag }}|g" "$DOCKER_COMPOSE_FILE" - sed -i "s|\${BASE_DIR}|${{ github.workspace }}|g" "$DOCKER_COMPOSE_FILE" - echo "Docker compose file prepared:" - cat "$DOCKER_COMPOSE_FILE" - - - name: Start service container - run: | - docker compose -f "$DOCKER_COMPOSE_FILE" up -d - docker compose -f "$DOCKER_COMPOSE_FILE" ps - - - name: Wait for service to be ready - run: | - for i in $(seq 1 $MAX_RETRIES); do - if curl --silent --fail "$SERVICE_URL" --data-binary "SELECT 1" | grep -q "1"; then - echo "Service is up and responding!" - exit 0 - fi - echo "Waiting for service... ($i/$MAX_RETRIES)" - sleep $RETRY_INTERVAL - done - echo "Error: Service failed to start within timeout" - docker compose -f "$DOCKER_COMPOSE_FILE" logs - exit 1 - - name: Run integration tests HTTP env: SERVICE_ID: ${{ secrets.FIREBOLT_CLIENT_ID_STG_NEW_IDN }} @@ -147,7 +75,7 @@ jobs: STOPPED_ENGINE_NAME: "" API_ENDPOINT: "" ACCOUNT_NAME: "" - CORE_URL: "http://localhost:3473" + CORE_URL: ${{ steps.setup-core.outputs.service_url }} run: | pytest -o log_cli=true -o log_cli_level=WARNING tests/integration -k "core" --alluredir=allure-results/ @@ -160,7 +88,7 @@ jobs: STOPPED_ENGINE_NAME: "" API_ENDPOINT: "" ACCOUNT_NAME: "" - CORE_URL: "https://localhost:443" + CORE_URL: ${{ steps.setup-core.outputs.service_https_url }} run: | pytest -o log_cli=true -o log_cli_level=WARNING tests/integration -k "core" --alluredir=allure-results-https/ From 2cccce0cdda09f346b40ab9efa3fd541079102ef Mon Sep 17 00:00:00 2001 From: ptiurin Date: Fri, 25 Jul 2025 10:39:19 +0100 Subject: [PATCH 2/5] swap around --- .github/workflows/integration-tests-core.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration-tests-core.yml b/.github/workflows/integration-tests-core.yml index 6d3ad2a7e50..29dccb6e1c9 100644 --- a/.github/workflows/integration-tests-core.yml +++ b/.github/workflows/integration-tests-core.yml @@ -45,17 +45,17 @@ jobs: run-core-integration-tests: runs-on: ${{ inputs.os_name }} steps: + - name: Check out code + uses: actions/checkout@v4 + with: + repository: 'firebolt-db/firebolt-python-sdk' + - name: Setup Firebolt Core id: setup-core uses: firebolt-db/action-setup-core@main with: tag_version: ${{ inputs.tag_version || vars.DEFAULT_CORE_IMAGE_TAG }} - - name: Check out code - uses: actions/checkout@v4 - with: - repository: 'firebolt-db/firebolt-python-sdk' - - name: Set up Python uses: actions/setup-python@v5 with: From 5c7716ced9c1487f46bc6d62265c0536ab993704 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Fri, 25 Jul 2025 10:42:14 +0100 Subject: [PATCH 3/5] remove redundant stop container --- .github/workflows/integration-tests-core.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/integration-tests-core.yml b/.github/workflows/integration-tests-core.yml index 29dccb6e1c9..345d9a43af8 100644 --- a/.github/workflows/integration-tests-core.yml +++ b/.github/workflows/integration-tests-core.yml @@ -92,11 +92,6 @@ jobs: run: | pytest -o log_cli=true -o log_cli_level=WARNING tests/integration -k "core" --alluredir=allure-results-https/ - - name: Stop container - if: always() - run: | - docker compose -f "$DOCKER_COMPOSE_FILE" down - # Need to pull the pages branch in order to fetch the previous runs - name: Get Allure history uses: actions/checkout@v4 From 4964babc716120f6bb32461ad0d386a27976bdec Mon Sep 17 00:00:00 2001 From: ptiurin Date: Fri, 25 Jul 2025 10:43:01 +0100 Subject: [PATCH 4/5] Correct test type --- .github/workflows/integration-tests-core.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests-core.yml b/.github/workflows/integration-tests-core.yml index 345d9a43af8..fa26a7faa35 100644 --- a/.github/workflows/integration-tests-core.yml +++ b/.github/workflows/integration-tests-core.yml @@ -107,7 +107,7 @@ jobs: continue-on-error: true with: github-key: ${{ secrets.GITHUB_TOKEN }} - test-type: integration + test-type: core allure-dir: allure-results pages-branch: gh-pages repository-name: python-sdk @@ -118,7 +118,7 @@ jobs: continue-on-error: true with: github-key: ${{ secrets.GITHUB_TOKEN }} - test-type: integration_https + test-type: core_https allure-dir: allure-results-https pages-branch: gh-pages repository-name: python-sdk \ No newline at end of file From 7912b1cdffa781f2b632df526edb409f66bbf0b8 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Fri, 25 Jul 2025 11:26:20 +0100 Subject: [PATCH 5/5] remove unused resources --- .github/resources/core/README.md | 26 --------------------- .github/resources/core/config.json | 7 ------ .github/resources/core/default.conf | 17 -------------- .github/resources/core/docker-compose.yml | 28 ----------------------- 4 files changed, 78 deletions(-) delete mode 100644 .github/resources/core/README.md delete mode 100644 .github/resources/core/config.json delete mode 100644 .github/resources/core/default.conf delete mode 100644 .github/resources/core/docker-compose.yml diff --git a/.github/resources/core/README.md b/.github/resources/core/README.md deleted file mode 100644 index c5e53379333..00000000000 --- a/.github/resources/core/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Running tests for firebolt core over https - -In order to run the integration tests using https protocol, we need to bring up an nginx reverse proxy. On this proxy we will install the server certificate. In our case it will be a self signed cert (localhost.pem) and its private key (localhost-key.pem). - -The cert and its key will be stored in GitHub since we cannot have secrets stored in code repository, and it would be easier to rotate them (no code changes required): -- FIREBOLT_CORE_DEV_CERT is the certificate -- FIREBOLT_CORE_DEV_CERT_PRIVATE_KEY is the private key - -This is the certificate information that we currently have. Note that it wille expire in 2 years (when it will expire we will generate a new certificate and key and set them in the Git repository secrets) - -Certificate Information: -- Common Name: -- Subject Alternative Names: localhost -- Organization: mkcert development certificate -- Organization Unit: george@Mac.localdomain (George's Blue Mac) -- Locality: -- State: -- Country: -- Valid From: May 22, 2025 -- Valid To: August 22, 2027 -- Issuer: mkcert george@Mac.localdomain (George's Blue Mac), mkcert development CA -- Key Size: 2048 bit -- Serial Number: 5b19e46ed1a5a912da2cef0129924c41 - - -The client will connect over https to the nginx server which will do the TLS handshake and the TLS termination. It will then call the firebolt-core over http on port 3473. diff --git a/.github/resources/core/config.json b/.github/resources/core/config.json deleted file mode 100644 index d7fe906fd12..00000000000 --- a/.github/resources/core/config.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "nodes": [ - { - "host": "firebolt-core" - } - ] -} \ No newline at end of file diff --git a/.github/resources/core/default.conf b/.github/resources/core/default.conf deleted file mode 100644 index 81591b38732..00000000000 --- a/.github/resources/core/default.conf +++ /dev/null @@ -1,17 +0,0 @@ -# this is the nginx configuration -# for http we are already exposing port 3473 on the docker so it will connect to the firebolt directly without going through nginx - -# HTTPS server -server { - listen 443 ssl; - server_name localhost; - - # On github these the localhost - ssl_certificate /etc/nginx/certs/server.pem; - ssl_certificate_key /etc/nginx/certs/server.key; - - location / { - proxy_pass http://firebolt-core:3473; - proxy_set_header Host $host; - } -} \ No newline at end of file diff --git a/.github/resources/core/docker-compose.yml b/.github/resources/core/docker-compose.yml deleted file mode 100644 index 85092e6add4..00000000000 --- a/.github/resources/core/docker-compose.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: firebolt-core - -services: - firebolt-core: - image: ghcr.io/firebolt-db/firebolt-core:${IMAGE_TAG} - container_name: firebolt-core - command: --node 0 - privileged: true - restart: no # equivalent to --rm (no persistence) - ulimits: - memlock: 8589934592 - ports: - - 3473:3473 - volumes: - # Mount the config file into the container. - - ${BASE_DIR}/.github/resources/core/config.json:/firebolt-core/config.json:ro - # Create an anonymous volume for Firebolt's internal database files. Not using a volume would have a performance impact. - - ${BASE_DIR}/firebolt-core:/firebolt-core/data - - nginx: - image: nginx:alpine - ports: - - "443:443" - volumes: - - ${BASE_DIR}/.github/resources/core/certs:/etc/nginx/certs:ro - - ${BASE_DIR}/.github/resources/core/default.conf:/etc/nginx/conf.d/default.conf:ro - depends_on: - - firebolt-core \ No newline at end of file