From 5964641f5cb7575d733c5669223e584ef6228f34 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 30 Oct 2025 15:37:22 -0400 Subject: [PATCH 01/21] chore: externalize c-chain-reexecution json based workflow into tasks --- Taskfile.yml | 104 ++++++++++++++++++++++-------- scripts/benchmark_cchain_range.sh | 24 +++++-- scripts/copy_dir.sh | 18 ++++-- 3 files changed, 112 insertions(+), 34 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index fa229d51d012..c69714938034 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -201,44 +201,42 @@ tasks: vars: CURRENT_STATE_DIR: '{{.CURRENT_STATE_DIR}}' BLOCK_DIR: '{{.BLOCK_DIR}}' - RUNNER_NAME: '{{.RUNNER_NAME | default "dev"}}' CONFIG: '{{.CONFIG | default ""}}' START_BLOCK: '{{.START_BLOCK}}' END_BLOCK: '{{.END_BLOCK}}' LABELS: '{{.LABELS | default ""}}' BENCHMARK_OUTPUT_FILE: '{{.BENCHMARK_OUTPUT_FILE | default ""}}' - METRICS_SERVER_ENABLED: '{{.METRICS_SERVER_ENABLED | default "false"}}' - METRICS_SERVER_PORT: '{{.METRICS_SERVER_PORT}}' - METRICS_COLLECTOR_ENABLED: '{{.METRICS_COLLECTOR_ENABLED | default "false"}}' + TIMESTAMP: '{{.TIMESTAMP | default (now | date "20060102-150405")}}' + EXECUTION_DATA_DIR: '{{.EXECUTION_DATA_DIR | default (printf "/tmp/%s-%s" .TASK_NAME .TIMESTAMP)}}' cmd: | CURRENT_STATE_DIR={{.CURRENT_STATE_DIR}} \ BLOCK_DIR={{.BLOCK_DIR}} \ - RUNNER_NAME='{{.RUNNER_NAME | default "dev"}}' \ CONFIG={{.CONFIG}} \ START_BLOCK={{.START_BLOCK}} \ END_BLOCK={{.END_BLOCK}} \ LABELS={{.LABELS}} \ BENCHMARK_OUTPUT_FILE={{.BENCHMARK_OUTPUT_FILE}} \ - METRICS_SERVER_ENABLED={{.METRICS_SERVER_ENABLED}} \ - METRICS_SERVER_PORT={{.METRICS_SERVER_PORT}} \ - METRICS_COLLECTOR_ENABLED={{.METRICS_COLLECTOR_ENABLED}} \ + EXECUTION_DATA_DIR={{.EXECUTION_DATA_DIR}} \ bash -x ./scripts/benchmark_cchain_range.sh + # Runtime context variables are read from environment by the script: + # - RUNNER_NAME (execution environment) + # - METRICS_SERVER_PORT (monitoring config) + # - METRICS_SERVER_ENABLED (runtime monitoring decision) + # - METRICS_COLLECTOR_ENABLED (runtime monitoring decision) + # - PROMETHEUS_URL, PROMETHEUS_USERNAME, PROMETHEUS_PASSWORD (monitoring config) + # - GH_REPO, GH_WORKFLOW, GH_RUN_ID, etc. (GitHub context) reexecute-cchain-range-with-copied-data: desc: Combines import-cchain-reexecute-range and reexecute-cchain-range vars: - EXECUTION_DATA_DIR: '{{.EXECUTION_DATA_DIR}}' - BLOCK_DIR_SRC: '{{.BLOCK_DIR_SRC | default "s3://avalanchego-bootstrap-testing/cchain-mainnet-blocks-1m-ldb/**"}}' - CURRENT_STATE_DIR_SRC: '{{.CURRENT_STATE_DIR_SRC | default "s3://avalanchego-bootstrap-testing/cchain-current-state-hashdb-full-100/**"}}' - RUNNER_NAME: '{{.RUNNER_NAME | default "dev"}}' - CONFIG: '{{.CONFIG | default ""}}' - START_BLOCK: '{{.START_BLOCK | default "101"}}' - END_BLOCK: '{{.END_BLOCK | default "250000"}}' - LABELS: '{{.LABELS | default ""}}' - BENCHMARK_OUTPUT_FILE: '{{.BENCHMARK_OUTPUT_FILE | default ""}}' - METRICS_SERVER_ENABLED: '{{.METRICS_SERVER_ENABLED | default "false"}}' - METRICS_SERVER_PORT: '{{.METRICS_SERVER_PORT}}' - METRICS_COLLECTOR_ENABLED: '{{.METRICS_COLLECTOR_ENABLED | default "false"}}' + TASK_NAME: '{{.TASK_NAME}}' + BLOCK_DIR_SRC: '{{.BLOCK_DIR_SRC}}' + CURRENT_STATE_DIR_SRC: '{{.CURRENT_STATE_DIR_SRC}}' + CONFIG: '{{.CONFIG}}' + START_BLOCK: '{{.START_BLOCK}}' + END_BLOCK: '{{.END_BLOCK}}' + TIMESTAMP: '{{.TIMESTAMP | default (now | date "20060102-150405")}}' + EXECUTION_DATA_DIR: '{{.EXECUTION_DATA_DIR | default (printf "/tmp/%s-%s" .TASK_NAME .TIMESTAMP)}}' cmds: - task: import-cchain-reexecute-range vars: @@ -247,17 +245,71 @@ tasks: EXECUTION_DATA_DIR: '{{.EXECUTION_DATA_DIR}}' - task: reexecute-cchain-range vars: + TASK_NAME: '{{.TASK_NAME}}' BLOCK_DIR: '{{.EXECUTION_DATA_DIR}}/blocks' CURRENT_STATE_DIR: '{{.EXECUTION_DATA_DIR}}/current-state' - RUNNER_NAME: '{{.RUNNER_NAME}}' CONFIG: '{{.CONFIG}}' START_BLOCK: '{{.START_BLOCK}}' END_BLOCK: '{{.END_BLOCK}}' - LABELS: '{{.LABELS}}' - BENCHMARK_OUTPUT_FILE: '{{.BENCHMARK_OUTPUT_FILE}}' - METRICS_SERVER_ENABLED: '{{.METRICS_SERVER_ENABLED}}' - METRICS_SERVER_PORT: '{{.METRICS_SERVER_PORT}}' - METRICS_COLLECTOR_ENABLED: '{{.METRICS_COLLECTOR_ENABLED}}' + EXECUTION_DATA_DIR: '{{.EXECUTION_DATA_DIR}}' + + c-chain-reexecution-hashdb-101-250k: + desc: C-Chain re-execution from block 101 to 250k with hashdb + cmds: + - task: reexecute-cchain-range-with-copied-data + vars: + TASK_NAME: '{{.TASK}}' + START_BLOCK: 101 + END_BLOCK: 250000 + BLOCK_DIR_SRC: 'cchain-mainnet-blocks-1m-ldb' + CURRENT_STATE_DIR_SRC: 'cchain-current-state-hashdb-full-100' + + c-chain-reexecution-hashdb-archive-101-250k: + desc: C-Chain re-execution from block 101 to 250k with hashdb archive + cmds: + - task: reexecute-cchain-range-with-copied-data + vars: + TASK_NAME: '{{.TASK}}' + START_BLOCK: 101 + END_BLOCK: 250000 + BLOCK_DIR_SRC: 'cchain-mainnet-blocks-1m-ldb' + CURRENT_STATE_DIR_SRC: 'cchain-current-state-hashdb-archive-100' + CONFIG: archive + + c-chain-reexecution-hashdb-33m-33m500k: + desc: C-Chain re-execution from block 33m to 33.5m with hashdb + cmds: + - task: reexecute-cchain-range-with-copied-data + vars: + TASK_NAME: '{{.TASK}}' + START_BLOCK: 33000001 + END_BLOCK: 33500000 + BLOCK_DIR_SRC: 'cchain-mainnet-blocks-30m-40m-ldb' + CURRENT_STATE_DIR_SRC: 'cchain-current-state-hashdb-full-33m' + + c-chain-reexecution-firewood-33m-33m500k: + desc: C-Chain re-execution from block 33m to 33.5m with firewood + cmds: + - task: reexecute-cchain-range-with-copied-data + vars: + TASK_NAME: '{{.TASK}}' + START_BLOCK: 33000001 + END_BLOCK: 33500000 + BLOCK_DIR_SRC: 'cchain-mainnet-blocks-30m-40m-ldb' + CURRENT_STATE_DIR_SRC: 'cchain-current-state-firewood-33m' + CONFIG: firewood + + c-chain-reexecution-firewood-101-250k: + desc: C-Chain re-execution from block 101 to 250k with firewood + cmds: + - task: reexecute-cchain-range-with-copied-data + vars: + TASK_NAME: '{{.TASK}}' + START_BLOCK: 101 + END_BLOCK: 250000 + BLOCK_DIR_SRC: 'cchain-mainnet-blocks-1m-ldb' + CURRENT_STATE_DIR_SRC: 'cchain-current-state-firewood-100' + CONFIG: firewood test-bootstrap-monitor-e2e: desc: Runs bootstrap monitor e2e tests diff --git a/scripts/benchmark_cchain_range.sh b/scripts/benchmark_cchain_range.sh index 0d9d951e7194..6d630f89157a 100755 --- a/scripts/benchmark_cchain_range.sh +++ b/scripts/benchmark_cchain_range.sh @@ -8,18 +8,34 @@ set -euo pipefail # CURRENT_STATE_DIR: Path or S3 URL to the current state directory or zip. # START_BLOCK: The starting block height (exclusive). # END_BLOCK: The ending block height (inclusive). +# RUNNER_NAME (optional): Name of the runner (defaults to "dev"). +# CONFIG (optional): Configuration name (defaults to "default"). # LABELS (optional): Comma-separated key=value pairs for metric labels. # BENCHMARK_OUTPUT_FILE (optional): If set, benchmark output is also written to this file. -# METRICS_SERVER_ENABLED (optional): If set, enables the metrics server. -# METRICS_SERVER_PORT (optional): If set, determines the port the metrics server will listen to. -# METRICS_COLLECTOR_ENABLED (optional): If set, enables the metrics collector. +# METRICS_SERVER_ENABLED (optional): If set to "true", enables the metrics server (defaults to "false"). +# METRICS_SERVER_PORT (optional): Port for the metrics server (defaults to "0"). +# METRICS_COLLECTOR_ENABLED (optional): If set to "true", enables the metrics collector (defaults to "false"). : "${BLOCK_DIR:?BLOCK_DIR must be set}" : "${CURRENT_STATE_DIR:?CURRENT_STATE_DIR must be set}" -: "${RUNNER_NAME:?RUNNER_NAME must be set}" : "${START_BLOCK:?START_BLOCK must be set}" : "${END_BLOCK:?END_BLOCK must be set}" +# Set defaults for optional variables +: "${RUNNER_NAME:=dev}" +: "${CONFIG:=default}" +: "${METRICS_SERVER_ENABLED:=false}" +: "${METRICS_SERVER_PORT:=0}" +: "${METRICS_COLLECTOR_ENABLED:=false}" + +echo "=== C-Chain Re-execution Benchmark Environment Variables ===" +echo "Runner: ${RUNNER_NAME}" +echo "Config: ${CONFIG}" +echo "Blocks: ${START_BLOCK} to ${END_BLOCK}" +echo "Metrics server: ${METRICS_SERVER_ENABLED}" +echo "Metrics server port: ${METRICS_SERVER_PORT}" +echo "Metrics collector: ${METRICS_COLLECTOR_ENABLED}" + cmd="go test -timeout=0 -v -benchtime=1x -bench=BenchmarkReexecuteRange -run=^$ github.com/ava-labs/avalanchego/tests/reexecute/c \ --block-dir=\"${BLOCK_DIR}\" \ --current-state-dir=\"${CURRENT_STATE_DIR}\" \ diff --git a/scripts/copy_dir.sh b/scripts/copy_dir.sh index 874b53c74769..5bdafc3dc2c4 100755 --- a/scripts/copy_dir.sh +++ b/scripts/copy_dir.sh @@ -3,13 +3,17 @@ set -euo pipefail # Usage: ./scripts/copy_dir.sh source_directory destination_directory -# Sources can be S3 URLs (s3://bucket/path) or a local file path +# Sources can be: +# - S3 URLs (s3://bucket/path) +# - S3 object keys (will be expanded to s3://avalanchego-bootstrap-testing//**) +# - Local file paths # Assumes s5cmd has been installed and is available in the PATH. # s5cmd is included in the nix dev shell. if [ $# -ne 2 ]; then echo "Usage: $0 " - echo "Import from S3 Example: $0 's3://bucket1/path1' /dest/dir" + echo "Import from S3 URL Example: $0 's3://bucket1/path1' /dest/dir" + echo "Import from S3 object key Example: $0 'cchain-mainnet-blocks-1m-ldb' /dest/dir" echo "Export to S3 Example: $0 '/local/path1' 's3://bucket2/path2'" echo "Local Example: $0 '/local/path1' /dest/dir" exit 1 @@ -18,11 +22,17 @@ fi SRC="$1" DST="$2" +# If SRC doesn't start with s3:// or /, assume it's an S3 object key +if [[ "$SRC" != s3://* ]] && [[ "$SRC" != /* ]]; then + SRC="s3://avalanchego-bootstrap-testing/${SRC}/**" + echo "Expanded object key to: $SRC" +fi + # Function to copy from a single source to destination function copy_source() { local source="$1" local dest="$2" - + # Check if source starts with s3:// if [[ "$source" == s3://* || "$dest" == s3://* ]]; then # Use s5cmd to copy from S3 @@ -30,7 +40,7 @@ function copy_source() { time s5cmd cp --show-progress "$source" "$dest" else # Use cp for local filesystem with recursive support - + # Ensure destination directory exists mkdir -p "$dest" From 70fbc9f143ef1c6ee05b193f6e5551ba7d16797a Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 30 Oct 2025 16:06:12 -0400 Subject: [PATCH 02/21] ci(c-chain-reexecution): remove relative imports to make this action reusable --- .../c-chain-reexecution-benchmark/action.yml | 154 +++++++++++------- .../output-metrics-url.sh | 23 +++ .../output-metrics-url.sh | 4 + 3 files changed, 126 insertions(+), 55 deletions(-) create mode 100755 .github/actions/c-chain-reexecution-benchmark/output-metrics-url.sh diff --git a/.github/actions/c-chain-reexecution-benchmark/action.yml b/.github/actions/c-chain-reexecution-benchmark/action.yml index 3385ff50a0c1..d19528969429 100644 --- a/.github/actions/c-chain-reexecution-benchmark/action.yml +++ b/.github/actions/c-chain-reexecution-benchmark/action.yml @@ -2,30 +2,18 @@ name: 'C-Chain Re-Execution Benchmark' description: 'Run C-Chain re-execution benchmark' inputs: + task: + description: 'Task name to execute from Taskfile.yml' + required: true runner_name: description: 'The name of the runner to use and include in the Golang Benchmark name.' required: true - config: - description: 'The config to pass to the VM for the benchmark. See BenchmarkReexecuteRange for details.' - default: '' - start-block: - description: 'The start block for the benchmark.' - default: '101' - end-block: - description: 'The end block for the benchmark.' - default: '250000' - block-dir-src: - description: 'The source block directory. Supports S3 directory/zip and local directories.' - default: 's3://avalanchego-bootstrap-testing/cchain-mainnet-blocks-1m-ldb/**' - current-state-dir-src: - description: 'The current state directory. Supports S3 directory/zip and local directories.' - default: 's3://avalanchego-bootstrap-testing/cchain-current-state-hashdb-full-100/**' aws-role: description: 'AWS role to assume for S3 access.' required: true aws-region: description: 'AWS region to use for S3 access.' - required: true + default: 'us-east-2' aws-role-duration-seconds: description: 'The duration of the AWS role to assume for S3 access.' required: true @@ -56,64 +44,120 @@ inputs: push-github-action-benchmark: description: 'Whether to push the benchmark result to GitHub.' required: true - default: false push-post-state: description: 'S3 destination to copy the current-state directory after completing re-execution. If empty, this will be skipped.' default: '' + # The following inputs need never be provided by the caller. They + # default to context values that the action's steps are unable to + # access directly. + repository-owner: + default: ${{ github.repository_owner }} + repository-name: + default: ${{ github.event.repository.name }} + workflow: + default: ${{ github.workflow }} + run-id: + default: ${{ github.run_id }} + run-number: + default: ${{ github.run_number }} + run-attempt: + default: ${{ github.run_attempt }} + job: + default: ${{ github.job }} + grafana-dashboard-id: + default: 'Gl1I20mnk/c-chain' runs: using: composite steps: - - name: Set task env + - uses: cachix/install-nix-action@02a151ada4993995686f9ed4f1be7cfbb229e56f #v31 + with: + github_access_token: ${{ inputs.github-token }} + - run: nix develop --command echo "dependencies installed" + shell: bash + # Cache Go modules (architecture-independent) + - uses: actions/cache@v4 + id: go-mod-cache + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-mod-${{ hashFiles('go.sum') }} + restore-keys: ${{ runner.os }}-go-mod- + # Cache Go build cache (architecture-specific) + - uses: actions/cache@v4 + with: + path: ~/.cache/go-build + key: ${{ runner.os }}-${{ runner.arch }}-go-build-${{ hashFiles('go.sum') }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-go-build- + # Download modules only on cache miss + - run: $GITHUB_ACTION_PATH/nix-develop.sh --command go mod download + if: steps.go-mod-cache.outputs.cache-hit != 'true' + shell: bash + - run: | + if [[ -f tools/go.mod ]]; then + nix develop --command go mod download -modfile=tools/go.mod + fi + if: steps.go-mod-cache.outputs.cache-hit != 'true' + shell: bash + - name: Notify of metrics availability + if: inputs.prometheus-username != '' shell: bash run: | - { - echo "EXECUTION_DATA_DIR=${{ inputs.workspace }}/reexecution-data" - echo "BENCHMARK_OUTPUT_FILE=output.txt" - echo "START_BLOCK=${{ inputs.start-block }}" - echo "END_BLOCK=${{ inputs.end-block }}" - echo "BLOCK_DIR_SRC=${{ inputs.block-dir-src }}" - echo "CURRENT_STATE_DIR_SRC=${{ inputs.current-state-dir-src }}" - } >> $GITHUB_ENV + metrics_url=$($GITHUB_ACTION_PATH/output-metrics-url.sh) + echo "Grafana: ${metrics_url}" + echo "🔗 [View Grafana Dashboard](${metrics_url})" >> "$GITHUB_STEP_SUMMARY" + env: + GRAFANA_URL: https://grafana-poc.avax-dev.network/d/${{ inputs.grafana_dashboard_id }}?orgId=1&refresh=10s&var-filter=is_ephemeral_node%7C%3D%7Cfalse&var-filter=gh_repo%7C%3D%7C${{ inputs.repository_owner }}%2F${{ inputs.repository_name }}&var-filter=gh_run_id%7C%3D%7C${{ inputs.run_id }}&var-filter=gh_run_attempt%7C%3D%7C${{ inputs.run_attempt }} + GH_JOB_ID: ${{ inputs.job }} + - name: Warn that collection of metrics and logs will not be performed + if: inputs.prometheus-username == '' + shell: bash + run: echo "::warning::Monitoring credentials not found. Skipping collector start. Is the PR from a fork branch?" - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ inputs.aws-role }} aws-region: ${{ inputs.aws-region }} role-duration-seconds: ${{ inputs.aws-role-duration-seconds }} - - name: Run C-Chain Re-Execution - uses: ./.github/actions/run-monitored-tmpnet-cmd - with: - run: | - ./scripts/run_task.sh reexecute-cchain-range-with-copied-data \ - CONFIG=${{ inputs.config }} \ - EXECUTION_DATA_DIR=${{ env.EXECUTION_DATA_DIR }} \ - BLOCK_DIR_SRC=${{ env.BLOCK_DIR_SRC }} \ - CURRENT_STATE_DIR_SRC=${{ env.CURRENT_STATE_DIR_SRC }} \ - START_BLOCK=${{ env.START_BLOCK }} \ - END_BLOCK=${{ env.END_BLOCK }} \ - LABELS=${{ env.LABELS }} \ - BENCHMARK_OUTPUT_FILE=${{ env.BENCHMARK_OUTPUT_FILE }} \ - RUNNER_NAME=${{ inputs.runner_name }} \ - METRICS_SERVER_ENABLED=true \ - METRICS_COLLECTOR_ENABLED=true - prometheus_url: ${{ inputs.prometheus-url }} - prometheus_push_url: ${{ inputs.prometheus-push-url }} - prometheus_username: ${{ inputs.prometheus-username }} - prometheus_password: ${{ inputs.prometheus-password }} - grafana_dashboard_id: 'Gl1I20mnk/c-chain' - runtime: "" # Set runtime input to empty string to disable log collection - + - name: Set task env + shell: bash + run: | + TIMESTAMP=$(date '+%Y%m%d-%H%M%S') + EXEC_DIR="/tmp/reexecution-data-${TIMESTAMP}" + echo "EXECUTION_DATA_DIR=${EXEC_DIR}" >> "$GITHUB_ENV" + - name: Run C-Chain Re-execution Benchmark + shell: nix develop --impure --command bash -x {0} + run: ./scripts/run_task.sh ${{ inputs.task }} \ + BENCHMARK_OUTPUT_FILE="benchmark-output.txt" \ + EXECUTION_DATA_DIR="${EXEC_DIR}" + env: + RUNNER_NAME: ${{ inputs.runner_name }} + TMPNET_START_METRICS_COLLECTOR: ${{ inputs.prometheus-username != '' }} + TMPNET_CHECK_METRICS_COLLECTED: ${{ inputs.prometheus-username != '' }} + METRICS_SERVER_ENABLED: ${{ inputs.prometheus-username != '' }} + METRICS_COLLECTOR_ENABLED: ${{ inputs.prometheus-username != '' }} + PROMETHEUS_URL: ${{ inputs.prometheus-url }} + PROMETHEUS_PUSH_URL: ${{ inputs.prometheus-push-url }} + PROMETHEUS_USERNAME: ${{ inputs.prometheus-username }} + PROMETHEUS_PASSWORD: ${{ inputs.prometheus-password }} + GH_REPO: ${{ inputs.repository_owner }}/${{ inputs.repository_name }} + GH_WORKFLOW: ${{ inputs.workflow }} + GH_RUN_ID: ${{ inputs.run_id }} + GH_RUN_NUMBER: ${{ inputs.run_number }} + GH_RUN_ATTEMPT: ${{ inputs.run_attempt }} + GH_JOB_ID: ${{ inputs.job }} - name: Compare Benchmark Results uses: benchmark-action/github-action-benchmark@v1 with: tool: 'go' - output-file-path: ${{ env.BENCHMARK_OUTPUT_FILE }} + output-file-path: benchmark-output.txt summary-always: true github-token: ${{ inputs.github-token }} auto-push: ${{ inputs.push-github-action-benchmark }} - - - name: Push Post-State to S3 (if not exists) - if: ${{ inputs.push-post-state != '' }} - shell: nix develop --command bash -x {0} - run: ./scripts/run_task.sh export-dir-to-s3 SRC=${{ env.EXECUTION_DATA_DIR }}/current-state/ DST=${{ inputs.push-post-state }} + - name: Push Post-State to S3 + if: inputs.push-post-state != '' + shell: bash + run: | + $GITHUB_ACTION_PATH/nix-develop.sh --command bash -x \ + ./scripts/run_task.sh export-dir-to-s3 \ + SRC=${{ env.EXECUTION_DATA_DIR }}/current-state/ \ + DST=${{ inputs.push-post-state }} diff --git a/.github/actions/c-chain-reexecution-benchmark/output-metrics-url.sh b/.github/actions/c-chain-reexecution-benchmark/output-metrics-url.sh new file mode 100755 index 000000000000..428971dec97a --- /dev/null +++ b/.github/actions/c-chain-reexecution-benchmark/output-metrics-url.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# WARNING: This file is duplication of: +# - .github/actions/run-monitored-tmpnet-cmd/output-metrics-url.sh (source of truth) +# Changes must be made to BOTH files. + +set -euo pipefail + +# Timestamps are in seconds +from_timestamp="$(date '+%s')" +monitoring_period=900 # 15 minutes +to_timestamp="$((from_timestamp + monitoring_period))" + +# Grafana expects microseconds, so pad timestamps with 3 zeros +metrics_url="${GRAFANA_URL}&var-filter=gh_job_id%7C%3D%7C${GH_JOB_ID}&from=${from_timestamp}000&to=${to_timestamp}000" + +# Optionally ensure that the link displays metrics only for the shared +# network rather than mixing it with the results for private networks. +if [[ -n "${FILTER_BY_OWNER:-}" ]]; then + metrics_url="${metrics_url}&var-filter=network_owner%7C%3D%7C${FILTER_BY_OWNER}" +fi + +echo "${metrics_url}" diff --git a/.github/actions/run-monitored-tmpnet-cmd/output-metrics-url.sh b/.github/actions/run-monitored-tmpnet-cmd/output-metrics-url.sh index ccecc34ac09c..5d2e8d59e8d1 100755 --- a/.github/actions/run-monitored-tmpnet-cmd/output-metrics-url.sh +++ b/.github/actions/run-monitored-tmpnet-cmd/output-metrics-url.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +# WARNING: This file is duplicated at: +# - .github/actions/c-chain-reexecution-benchmark/output-metrics-url.sh (copy) +# Changes must be made to BOTH files. + set -euo pipefail # Timestamps are in seconds From d5bc2272911839feeecd2f56c2a72513455dc1ba Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 30 Oct 2025 16:13:43 -0400 Subject: [PATCH 03/21] ci(c-chain-reexecution): use correct shells --- .../c-chain-reexecution-benchmark/action.yml | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/.github/actions/c-chain-reexecution-benchmark/action.yml b/.github/actions/c-chain-reexecution-benchmark/action.yml index d19528969429..e88351992d5d 100644 --- a/.github/actions/c-chain-reexecution-benchmark/action.yml +++ b/.github/actions/c-chain-reexecution-benchmark/action.yml @@ -73,8 +73,8 @@ runs: - uses: cachix/install-nix-action@02a151ada4993995686f9ed4f1be7cfbb229e56f #v31 with: github_access_token: ${{ inputs.github-token }} - - run: nix develop --command echo "dependencies installed" - shell: bash + - run: echo "dependencies installed" + shell: nix develop --command bash {0} # Cache Go modules (architecture-independent) - uses: actions/cache@v4 id: go-mod-cache @@ -89,15 +89,9 @@ runs: key: ${{ runner.os }}-${{ runner.arch }}-go-build-${{ hashFiles('go.sum') }} restore-keys: ${{ runner.os }}-${{ runner.arch }}-go-build- # Download modules only on cache miss - - run: $GITHUB_ACTION_PATH/nix-develop.sh --command go mod download - if: steps.go-mod-cache.outputs.cache-hit != 'true' - shell: bash - - run: | - if [[ -f tools/go.mod ]]; then - nix develop --command go mod download -modfile=tools/go.mod - fi + - run: go mod download if: steps.go-mod-cache.outputs.cache-hit != 'true' - shell: bash + shell: nix develop --command bash -x {0} - name: Notify of metrics availability if: inputs.prometheus-username != '' shell: bash @@ -127,8 +121,8 @@ runs: - name: Run C-Chain Re-execution Benchmark shell: nix develop --impure --command bash -x {0} run: ./scripts/run_task.sh ${{ inputs.task }} \ - BENCHMARK_OUTPUT_FILE="benchmark-output.txt" \ - EXECUTION_DATA_DIR="${EXEC_DIR}" + BENCHMARK_OUTPUT_FILE="benchmark-output.txt" \ + EXECUTION_DATA_DIR="${EXEC_DIR}" env: RUNNER_NAME: ${{ inputs.runner_name }} TMPNET_START_METRICS_COLLECTOR: ${{ inputs.prometheus-username != '' }} @@ -155,9 +149,8 @@ runs: auto-push: ${{ inputs.push-github-action-benchmark }} - name: Push Post-State to S3 if: inputs.push-post-state != '' - shell: bash + shell: nix develop --impure --command bash -x {0} run: | - $GITHUB_ACTION_PATH/nix-develop.sh --command bash -x \ - ./scripts/run_task.sh export-dir-to-s3 \ + ./scripts/run_task.sh export-dir-to-s3 \ SRC=${{ env.EXECUTION_DATA_DIR }}/current-state/ \ DST=${{ inputs.push-post-state }} From ff86c392f70ee4aaa14adb647f5293dcc327ebd0 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 30 Oct 2025 16:24:02 -0400 Subject: [PATCH 04/21] ci(c-chain-reexecution)!: --- ...chain-reexecution-benchmark-container.json | 24 ++-------- ...-chain-reexecution-benchmark-container.yml | 47 +++++-------------- ...chain-reexecution-benchmark-gh-native.json | 22 ++------- ...-chain-reexecution-benchmark-gh-native.yml | 47 +++++-------------- 4 files changed, 31 insertions(+), 109 deletions(-) diff --git a/.github/workflows/c-chain-reexecution-benchmark-container.json b/.github/workflows/c-chain-reexecution-benchmark-container.json index aa8edb0aac70..2ccfac027111 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-container.json +++ b/.github/workflows/c-chain-reexecution-benchmark-container.json @@ -3,20 +3,12 @@ "include": [ { "runner": "ubuntu-latest", - "config": "default", - "start-block": 101, - "end-block": 250000, - "block-dir-src": "s3://avalanchego-bootstrap-testing/cchain-mainnet-blocks-1m-ldb/**", - "current-state-dir-src": "s3://avalanchego-bootstrap-testing/cchain-current-state-hashdb-full-100/**", + "task": "c-chain-reexecution-hashdb-101-250k", "timeout-minutes": 30 }, { "runner": "avalanche-avalanchego-runner-2ti", - "config": "default", - "start-block": 101, - "end-block": 250000, - "block-dir-src": "s3://avalanchego-bootstrap-testing/cchain-mainnet-blocks-1m-ldb/**", - "current-state-dir-src": "s3://avalanchego-bootstrap-testing/cchain-current-state-hashdb-full-100/**", + "task": "c-chain-reexecution-hashdb-101-250k", "timeout-minutes": 30 } ] @@ -25,20 +17,12 @@ "include": [ { "runner": "avago-runner-m6i-4xlarge-ebs-fast", - "config": "default", - "start-block": 33000001, - "end-block": 33500000, - "block-dir-src": "s3://avalanchego-bootstrap-testing/cchain-mainnet-blocks-30m-40m-ldb/**", - "current-state-dir-src": "s3://avalanchego-bootstrap-testing/cchain-current-state-hashdb-full-33m/**", + "task": "c-chain-reexecution-hashdb-33m-33m500k", "timeout-minutes": 1440 }, { "runner": "avago-runner-i4i-4xlarge-local-ssd", - "config": "default", - "start-block": 33000001, - "end-block": 33500000, - "block-dir-src": "s3://avalanchego-bootstrap-testing/cchain-mainnet-blocks-30m-40m-ldb/**", - "current-state-dir-src": "s3://avalanchego-bootstrap-testing/cchain-current-state-hashdb-full-33m/**", + "task": "c-chain-reexecution-hashdb-33m-33m500k", "timeout-minutes": 1440 } ] diff --git a/.github/workflows/c-chain-reexecution-benchmark-container.yml b/.github/workflows/c-chain-reexecution-benchmark-container.yml index db12a98ad703..e90787ffd3ac 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-container.yml +++ b/.github/workflows/c-chain-reexecution-benchmark-container.yml @@ -4,30 +4,13 @@ on: pull_request: workflow_dispatch: inputs: - config: - description: 'The config to pass to the VM for the benchmark. See BenchmarkReexecuteRange for details.' - required: false - default: '' - start-block: - description: 'The start block for the benchmark.' - required: false - default: 101 - end-block: - description: 'The end block for the benchmark.' - required: false - default: 250000 - block-dir-src: - description: 'The source block directory. Supports S3 directory/zip and local directories.' - required: false - default: s3://avalanchego-bootstrap-testing/cchain-mainnet-blocks-1m-ldb/** - current-state-dir-src: - description: 'The current state directory. Supports S3 directory/zip and local directories.' - required: false - default: s3://avalanchego-bootstrap-testing/cchain-current-state-hashdb-full-100/** runner: description: 'Runner to execute the benchmark. Input to the runs-on field of the job.' required: false default: ubuntu-latest + task: + description: 'Taskfile task to execute (e.g., c-chain-reexecution-hashdb-101-250k)' + required: true push-post-state: description: 'S3 location to push post-execution state directory. Skips this step if left unpopulated.' default: '' @@ -55,12 +38,8 @@ jobs: if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then { echo "matrix< Date: Thu, 30 Oct 2025 16:27:52 -0400 Subject: [PATCH 05/21] copilot PR comments --- .github/actions/c-chain-reexecution-benchmark/action.yml | 2 +- .../actions/c-chain-reexecution-benchmark/output-metrics-url.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/c-chain-reexecution-benchmark/action.yml b/.github/actions/c-chain-reexecution-benchmark/action.yml index e88351992d5d..2c915e8b3d53 100644 --- a/.github/actions/c-chain-reexecution-benchmark/action.yml +++ b/.github/actions/c-chain-reexecution-benchmark/action.yml @@ -122,7 +122,7 @@ runs: shell: nix develop --impure --command bash -x {0} run: ./scripts/run_task.sh ${{ inputs.task }} \ BENCHMARK_OUTPUT_FILE="benchmark-output.txt" \ - EXECUTION_DATA_DIR="${EXEC_DIR}" + EXECUTION_DATA_DIR="${{ env.EXECUTION_DATA_DIR }}" env: RUNNER_NAME: ${{ inputs.runner_name }} TMPNET_START_METRICS_COLLECTOR: ${{ inputs.prometheus-username != '' }} diff --git a/.github/actions/c-chain-reexecution-benchmark/output-metrics-url.sh b/.github/actions/c-chain-reexecution-benchmark/output-metrics-url.sh index 428971dec97a..875f7d7fdb8d 100755 --- a/.github/actions/c-chain-reexecution-benchmark/output-metrics-url.sh +++ b/.github/actions/c-chain-reexecution-benchmark/output-metrics-url.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# WARNING: This file is duplication of: +# WARNING: This file is a duplication of: # - .github/actions/run-monitored-tmpnet-cmd/output-metrics-url.sh (source of truth) # Changes must be made to BOTH files. From c753cf9b15fe20a991577d6e8477644796d6a8df Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 30 Oct 2025 16:39:46 -0400 Subject: [PATCH 06/21] ci(c-chain-reexecution): set absolute path to benchmark output file resolving error on compare benchmark results job --- .github/actions/c-chain-reexecution-benchmark/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/c-chain-reexecution-benchmark/action.yml b/.github/actions/c-chain-reexecution-benchmark/action.yml index 2c915e8b3d53..61278707e330 100644 --- a/.github/actions/c-chain-reexecution-benchmark/action.yml +++ b/.github/actions/c-chain-reexecution-benchmark/action.yml @@ -117,11 +117,13 @@ runs: run: | TIMESTAMP=$(date '+%Y%m%d-%H%M%S') EXEC_DIR="/tmp/reexecution-data-${TIMESTAMP}" + BENCHMARK_FILE="${GITHUB_WORKSPACE}/benchmark-output.txt" echo "EXECUTION_DATA_DIR=${EXEC_DIR}" >> "$GITHUB_ENV" + echo "BENCHMARK_OUTPUT_FILE=${BENCHMARK_FILE}" >> "$GITHUB_ENV" - name: Run C-Chain Re-execution Benchmark shell: nix develop --impure --command bash -x {0} run: ./scripts/run_task.sh ${{ inputs.task }} \ - BENCHMARK_OUTPUT_FILE="benchmark-output.txt" \ + BENCHMARK_OUTPUT_FILE="${{ env.BENCHMARK_OUTPUT_FILE }}" \ EXECUTION_DATA_DIR="${{ env.EXECUTION_DATA_DIR }}" env: RUNNER_NAME: ${{ inputs.runner_name }} From 7b3a23187bb2d2ce3ae833e07e6f961b26d97452 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Mon, 3 Nov 2025 09:06:21 -0500 Subject: [PATCH 07/21] address pr review --- .../c-chain-reexecution-benchmark/action.yml | 8 ++------ .../c-chain-reexecution-benchmark-container.yml | 16 ++++++++-------- .../c-chain-reexecution-benchmark-gh-native.yml | 10 +++++----- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/.github/actions/c-chain-reexecution-benchmark/action.yml b/.github/actions/c-chain-reexecution-benchmark/action.yml index 61278707e330..dca62b086053 100644 --- a/.github/actions/c-chain-reexecution-benchmark/action.yml +++ b/.github/actions/c-chain-reexecution-benchmark/action.yml @@ -64,8 +64,6 @@ inputs: default: ${{ github.run_attempt }} job: default: ${{ github.job }} - grafana-dashboard-id: - default: 'Gl1I20mnk/c-chain' runs: using: composite @@ -100,7 +98,7 @@ runs: echo "Grafana: ${metrics_url}" echo "🔗 [View Grafana Dashboard](${metrics_url})" >> "$GITHUB_STEP_SUMMARY" env: - GRAFANA_URL: https://grafana-poc.avax-dev.network/d/${{ inputs.grafana_dashboard_id }}?orgId=1&refresh=10s&var-filter=is_ephemeral_node%7C%3D%7Cfalse&var-filter=gh_repo%7C%3D%7C${{ inputs.repository_owner }}%2F${{ inputs.repository_name }}&var-filter=gh_run_id%7C%3D%7C${{ inputs.run_id }}&var-filter=gh_run_attempt%7C%3D%7C${{ inputs.run_attempt }} + GRAFANA_URL: https://grafana-poc.avax-dev.network/d/Gl1I20mnk/c-chain?orgId=1&refresh=10s&var-filter=is_ephemeral_node%7C%3D%7Cfalse&var-filter=gh_repo%7C%3D%7C${{ inputs.repository_owner }}%2F${{ inputs.repository_name }}&var-filter=gh_run_id%7C%3D%7C${{ inputs.run_id }}&var-filter=gh_run_attempt%7C%3D%7C${{ inputs.run_attempt }} GH_JOB_ID: ${{ inputs.job }} - name: Warn that collection of metrics and logs will not be performed if: inputs.prometheus-username == '' @@ -128,8 +126,6 @@ runs: env: RUNNER_NAME: ${{ inputs.runner_name }} TMPNET_START_METRICS_COLLECTOR: ${{ inputs.prometheus-username != '' }} - TMPNET_CHECK_METRICS_COLLECTED: ${{ inputs.prometheus-username != '' }} - METRICS_SERVER_ENABLED: ${{ inputs.prometheus-username != '' }} METRICS_COLLECTOR_ENABLED: ${{ inputs.prometheus-username != '' }} PROMETHEUS_URL: ${{ inputs.prometheus-url }} PROMETHEUS_PUSH_URL: ${{ inputs.prometheus-push-url }} @@ -151,7 +147,7 @@ runs: auto-push: ${{ inputs.push-github-action-benchmark }} - name: Push Post-State to S3 if: inputs.push-post-state != '' - shell: nix develop --impure --command bash -x {0} + shell: nix develop --command bash -x {0} run: | ./scripts/run_task.sh export-dir-to-s3 \ SRC=${{ env.EXECUTION_DATA_DIR }}/current-state/ \ diff --git a/.github/workflows/c-chain-reexecution-benchmark-container.yml b/.github/workflows/c-chain-reexecution-benchmark-container.yml index e90787ffd3ac..80a215062c91 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-container.yml +++ b/.github/workflows/c-chain-reexecution-benchmark-container.yml @@ -4,13 +4,13 @@ on: pull_request: workflow_dispatch: inputs: + task: + description: 'Taskfile task to execute (e.g., c-chain-reexecution-hashdb-101-250k)' + required: true runner: description: 'Runner to execute the benchmark. Input to the runs-on field of the job.' required: false default: ubuntu-latest - task: - description: 'Taskfile task to execute (e.g., c-chain-reexecution-hashdb-101-250k)' - required: true push-post-state: description: 'S3 location to push post-execution state directory. Skips this step if left unpopulated.' default: '' @@ -80,13 +80,13 @@ jobs: uses: ./.github/actions/c-chain-reexecution-benchmark with: task: ${{ matrix.task }} - runner_name: ${{ matrix.runner }} - aws-role: ${{ github.event.inputs.push-post-state != '' && secrets.AWS_S3_RW_ROLE || secrets.AWS_S3_READ_ONLY_ROLE }} - aws-region: 'us-east-2' prometheus-url: ${{ secrets.PROMETHEUS_URL || '' }} prometheus-push-url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }} prometheus-username: ${{ secrets.PROMETHEUS_USERNAME || '' }} prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} - github-token: ${{ secrets.GITHUB_TOKEN }} push-github-action-benchmark: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.repository == 'ava-labs/avalanchego' && github.ref_name == 'master') }} - push-post-state: ${{ github.event.inputs.push-post-state || '' }} + aws-role: ${{ github.event.inputs.push-post-state != '' && secrets.AWS_S3_RW_ROLE || secrets.AWS_S3_READ_ONLY_ROLE }} + aws-region: 'us-east-2' + github-token: ${{ secrets.GITHUB_TOKEN }} + push-post-state: ${{ github.event.inputs.push-post-state }} + runner_name: ${{ matrix.runner }} diff --git a/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml b/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml index d1edfa20162d..fe0ea75cfd0d 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml +++ b/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml @@ -70,13 +70,13 @@ jobs: uses: ./.github/actions/c-chain-reexecution-benchmark with: task: ${{ matrix.task }} - runner_name: ${{ matrix.runner }} - aws-role: ${{ github.event.inputs.push-post-state != '' && secrets.AWS_S3_RW_ROLE || secrets.AWS_S3_READ_ONLY_ROLE }} - aws-region: 'us-east-2' prometheus-url: ${{ secrets.PROMETHEUS_URL || '' }} prometheus-push-url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }} prometheus-username: ${{ secrets.PROMETHEUS_USERNAME || '' }} prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} - github-token: ${{ secrets.GITHUB_TOKEN }} push-github-action-benchmark: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.repository == 'ava-labs/avalanchego' && github.ref_name == 'master') }} - push-post-state: ${{ github.event.inputs.push-post-state || '' }} + aws-role: ${{ github.event.inputs.push-post-state != '' && secrets.AWS_S3_RW_ROLE || secrets.AWS_S3_READ_ONLY_ROLE }} + aws-region: 'us-east-2' + github-token: ${{ secrets.GITHUB_TOKEN }} + push-post-state: ${{ github.event.inputs.push-post-state }} + runner_name: ${{ matrix.runner }} From c0281edf4c8898a6c76abeaa5aa5e7865649f9ef Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Mon, 3 Nov 2025 18:38:19 +0400 Subject: [PATCH 08/21] address PR --- scripts/benchmark_cchain_range.sh | 22 ++++------------------ tests/reexecute/c/vm_reexecute_test.go | 6 ++++++ 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/scripts/benchmark_cchain_range.sh b/scripts/benchmark_cchain_range.sh index 6d630f89157a..9534f12e2d39 100755 --- a/scripts/benchmark_cchain_range.sh +++ b/scripts/benchmark_cchain_range.sh @@ -8,33 +8,19 @@ set -euo pipefail # CURRENT_STATE_DIR: Path or S3 URL to the current state directory or zip. # START_BLOCK: The starting block height (exclusive). # END_BLOCK: The ending block height (inclusive). -# RUNNER_NAME (optional): Name of the runner (defaults to "dev"). -# CONFIG (optional): Configuration name (defaults to "default"). # LABELS (optional): Comma-separated key=value pairs for metric labels. # BENCHMARK_OUTPUT_FILE (optional): If set, benchmark output is also written to this file. -# METRICS_SERVER_ENABLED (optional): If set to "true", enables the metrics server (defaults to "false"). -# METRICS_SERVER_PORT (optional): Port for the metrics server (defaults to "0"). -# METRICS_COLLECTOR_ENABLED (optional): If set to "true", enables the metrics collector (defaults to "false"). +# METRICS_SERVER_ENABLED (optional): If set, enables the metrics server. +# METRICS_SERVER_PORT (optional): If set, determines the port the metrics server will listen to. +# METRICS_COLLECTOR_ENABLED (optional): If set, enables the metrics collector. : "${BLOCK_DIR:?BLOCK_DIR must be set}" : "${CURRENT_STATE_DIR:?CURRENT_STATE_DIR must be set}" : "${START_BLOCK:?START_BLOCK must be set}" : "${END_BLOCK:?END_BLOCK must be set}" -# Set defaults for optional variables +# Defaults to "dev" set at vm_reexecute_test.go `runnerNameArg` : "${RUNNER_NAME:=dev}" -: "${CONFIG:=default}" -: "${METRICS_SERVER_ENABLED:=false}" -: "${METRICS_SERVER_PORT:=0}" -: "${METRICS_COLLECTOR_ENABLED:=false}" - -echo "=== C-Chain Re-execution Benchmark Environment Variables ===" -echo "Runner: ${RUNNER_NAME}" -echo "Config: ${CONFIG}" -echo "Blocks: ${START_BLOCK} to ${END_BLOCK}" -echo "Metrics server: ${METRICS_SERVER_ENABLED}" -echo "Metrics server port: ${METRICS_SERVER_PORT}" -echo "Metrics collector: ${METRICS_COLLECTOR_ENABLED}" cmd="go test -timeout=0 -v -benchtime=1x -bench=BenchmarkReexecuteRange -run=^$ github.com/ava-labs/avalanchego/tests/reexecute/c \ --block-dir=\"${BLOCK_DIR}\" \ diff --git a/tests/reexecute/c/vm_reexecute_test.go b/tests/reexecute/c/vm_reexecute_test.go index 124ad589e571..7f0b538e359a 100644 --- a/tests/reexecute/c/vm_reexecute_test.go +++ b/tests/reexecute/c/vm_reexecute_test.go @@ -208,6 +208,12 @@ func benchmarkReexecuteRange( ) log.Info("re-executing block range with params", + zap.String("runner", runnerNameArg), + zap.String("config", configNameArg), + zap.String("labels", labelsArg), + zap.String("metrics-server-enabled", strconv.FormatBool(metricsServerEnabled)), + zap.Uint64("metrics-server-port", metricsPort), + zap.String("metrics-collector-enabled", strconv.FormatBool(metricsCollectorEnabled)), zap.String("block-dir", blockDir), zap.String("vm-db-dir", vmDBDir), zap.String("chain-data-dir", chainDataDir), From 10c5ef7d8c84b840479a3b2470b32ebe2bb3f290 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Mon, 3 Nov 2025 18:48:33 +0400 Subject: [PATCH 09/21] ci(c-chain-reexecution-benchmark): revert change on `Push Post-State to S3` --- .github/actions/c-chain-reexecution-benchmark/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/c-chain-reexecution-benchmark/action.yml b/.github/actions/c-chain-reexecution-benchmark/action.yml index dca62b086053..61073546086d 100644 --- a/.github/actions/c-chain-reexecution-benchmark/action.yml +++ b/.github/actions/c-chain-reexecution-benchmark/action.yml @@ -147,7 +147,7 @@ runs: auto-push: ${{ inputs.push-github-action-benchmark }} - name: Push Post-State to S3 if: inputs.push-post-state != '' - shell: nix develop --command bash -x {0} + shell: nix develop --impure --command bash -x {0} run: | ./scripts/run_task.sh export-dir-to-s3 \ SRC=${{ env.EXECUTION_DATA_DIR }}/current-state/ \ From c5b650a615e1e1c12c3a3c437ff8b70af6dbca93 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 6 Nov 2025 00:05:48 +0400 Subject: [PATCH 10/21] chore(benchamrk_cchain_range): avoid duplicating the default of `RUNNER_NAME` --- scripts/benchmark_cchain_range.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/benchmark_cchain_range.sh b/scripts/benchmark_cchain_range.sh index 9534f12e2d39..0eb0fe9ccb50 100755 --- a/scripts/benchmark_cchain_range.sh +++ b/scripts/benchmark_cchain_range.sh @@ -8,6 +8,7 @@ set -euo pipefail # CURRENT_STATE_DIR: Path or S3 URL to the current state directory or zip. # START_BLOCK: The starting block height (exclusive). # END_BLOCK: The ending block height (inclusive). +# RUNNER_NAME (optional): Runner name for the benchmark (defaults to "dev" in Go code). # LABELS (optional): Comma-separated key=value pairs for metric labels. # BENCHMARK_OUTPUT_FILE (optional): If set, benchmark output is also written to this file. # METRICS_SERVER_ENABLED (optional): If set, enables the metrics server. @@ -19,13 +20,10 @@ set -euo pipefail : "${START_BLOCK:?START_BLOCK must be set}" : "${END_BLOCK:?END_BLOCK must be set}" -# Defaults to "dev" set at vm_reexecute_test.go `runnerNameArg` -: "${RUNNER_NAME:=dev}" - cmd="go test -timeout=0 -v -benchtime=1x -bench=BenchmarkReexecuteRange -run=^$ github.com/ava-labs/avalanchego/tests/reexecute/c \ --block-dir=\"${BLOCK_DIR}\" \ --current-state-dir=\"${CURRENT_STATE_DIR}\" \ - --runner=\"${RUNNER_NAME}\" \ + ${RUNNER_NAME:+--runner=\"${RUNNER_NAME}\"} \ ${CONFIG:+--config=\"${CONFIG}\"} \ --start-block=\"${START_BLOCK}\" \ --end-block=\"${END_BLOCK}\" \ From 20c4ad057233afe5b8cd7ff4e9a06c78a0480a0c Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 6 Nov 2025 00:17:26 +0400 Subject: [PATCH 11/21] ci(c-chain-reexecution): - inline set task env - remove unrelated `TMPNET_START_METRICS_COLLECTOR` env var --- .github/actions/c-chain-reexecution-benchmark/action.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/actions/c-chain-reexecution-benchmark/action.yml b/.github/actions/c-chain-reexecution-benchmark/action.yml index 61073546086d..2b7c38067fb7 100644 --- a/.github/actions/c-chain-reexecution-benchmark/action.yml +++ b/.github/actions/c-chain-reexecution-benchmark/action.yml @@ -114,10 +114,8 @@ runs: shell: bash run: | TIMESTAMP=$(date '+%Y%m%d-%H%M%S') - EXEC_DIR="/tmp/reexecution-data-${TIMESTAMP}" - BENCHMARK_FILE="${GITHUB_WORKSPACE}/benchmark-output.txt" - echo "EXECUTION_DATA_DIR=${EXEC_DIR}" >> "$GITHUB_ENV" - echo "BENCHMARK_OUTPUT_FILE=${BENCHMARK_FILE}" >> "$GITHUB_ENV" + echo "EXECUTION_DATA_DIR=/tmp/reexecution-data-${TIMESTAMP}" >> "$GITHUB_ENV" + echo "BENCHMARK_OUTPUT_FILE=${GITHUB_WORKSPACE}/benchmark-output.txt" >> "$GITHUB_ENV" - name: Run C-Chain Re-execution Benchmark shell: nix develop --impure --command bash -x {0} run: ./scripts/run_task.sh ${{ inputs.task }} \ @@ -125,7 +123,6 @@ runs: EXECUTION_DATA_DIR="${{ env.EXECUTION_DATA_DIR }}" env: RUNNER_NAME: ${{ inputs.runner_name }} - TMPNET_START_METRICS_COLLECTOR: ${{ inputs.prometheus-username != '' }} METRICS_COLLECTOR_ENABLED: ${{ inputs.prometheus-username != '' }} PROMETHEUS_URL: ${{ inputs.prometheus-url }} PROMETHEUS_PUSH_URL: ${{ inputs.prometheus-push-url }} From 519e92a31da42f30d251edbc293b1f6ec90612d8 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Thu, 6 Nov 2025 00:18:23 +0400 Subject: [PATCH 12/21] chore: rm doc --- scripts/benchmark_cchain_range.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/benchmark_cchain_range.sh b/scripts/benchmark_cchain_range.sh index 0eb0fe9ccb50..896e68f8a559 100755 --- a/scripts/benchmark_cchain_range.sh +++ b/scripts/benchmark_cchain_range.sh @@ -8,7 +8,6 @@ set -euo pipefail # CURRENT_STATE_DIR: Path or S3 URL to the current state directory or zip. # START_BLOCK: The starting block height (exclusive). # END_BLOCK: The ending block height (inclusive). -# RUNNER_NAME (optional): Runner name for the benchmark (defaults to "dev" in Go code). # LABELS (optional): Comma-separated key=value pairs for metric labels. # BENCHMARK_OUTPUT_FILE (optional): If set, benchmark output is also written to this file. # METRICS_SERVER_ENABLED (optional): If set, enables the metrics server. From a800ff1545c0fedb598f686d4369ef21ab788925 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Fri, 7 Nov 2025 00:33:31 +0400 Subject: [PATCH 13/21] ci(c-chain-reexecution): support alternative approach to task based operations --- .../c-chain-reexecution-benchmark/action.yml | 57 +++++++++++++++++-- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/.github/actions/c-chain-reexecution-benchmark/action.yml b/.github/actions/c-chain-reexecution-benchmark/action.yml index 2b7c38067fb7..6ac3741ef37d 100644 --- a/.github/actions/c-chain-reexecution-benchmark/action.yml +++ b/.github/actions/c-chain-reexecution-benchmark/action.yml @@ -3,8 +3,24 @@ description: 'Run C-Chain re-execution benchmark' inputs: task: - description: 'Task name to execute from Taskfile.yml' - required: true + description: 'Task name to execute from Taskfile.yml. Leave empty to use granular inputs below.' + default: '' + # Custom inputs (alternative to task-based approach) + config: + description: 'The config to pass to the VM for the benchmark. See BenchmarkReexecuteRange for details.' + default: '' + start-block: + description: 'The start block for the benchmark.' + default: '' + end-block: + description: 'The end block for the benchmark.' + default: '' + block-dir-src: + description: 'The source block directory. Supports S3 directory/zip and local directories.' + default: '' + current-state-dir-src: + description: 'The current state directory. Supports S3 directory/zip and local directories.' + default: '' runner_name: description: 'The name of the runner to use and include in the Golang Benchmark name.' required: true @@ -110,6 +126,22 @@ runs: role-to-assume: ${{ inputs.aws-role }} aws-region: ${{ inputs.aws-region }} role-duration-seconds: ${{ inputs.aws-role-duration-seconds }} + - name: Validate inputs + shell: bash + run: | + if [[ -z "${{ inputs.task }}" ]]; then + # Granular mode - validate required inputs + missing=() + [[ -z "${{ inputs.block-dir-src }}" ]] && missing+=("block-dir-src") + [[ -z "${{ inputs.current-state-dir-src }}" ]] && missing+=("current-state-dir-src") + [[ -z "${{ inputs.start-block }}" ]] && missing+=("start-block") + [[ -z "${{ inputs.end-block }}" ]] && missing+=("end-block") + + if [[ ${#missing[@]} -gt 0 ]]; then + echo "::error::When 'task' is empty, the following inputs are required: ${missing[*]}" + exit 1 + fi + fi - name: Set task env shell: bash run: | @@ -118,9 +150,24 @@ runs: echo "BENCHMARK_OUTPUT_FILE=${GITHUB_WORKSPACE}/benchmark-output.txt" >> "$GITHUB_ENV" - name: Run C-Chain Re-execution Benchmark shell: nix develop --impure --command bash -x {0} - run: ./scripts/run_task.sh ${{ inputs.task }} \ - BENCHMARK_OUTPUT_FILE="${{ env.BENCHMARK_OUTPUT_FILE }}" \ - EXECUTION_DATA_DIR="${{ env.EXECUTION_DATA_DIR }}" + run: | + if [[ -n "${{ inputs.task }}" ]]; then + # Task-based approach + ./scripts/run_task.sh ${{ inputs.task }} \ + BENCHMARK_OUTPUT_FILE="${{ env.BENCHMARK_OUTPUT_FILE }}" \ + EXECUTION_DATA_DIR="${{ env.EXECUTION_DATA_DIR }}" + else + # Granular approach + ./scripts/run_task.sh reexecute-cchain-range-with-copied-data \ + CONFIG=${{ inputs.config }} \ + EXECUTION_DATA_DIR=${{ env.EXECUTION_DATA_DIR }} \ + BLOCK_DIR_SRC=${{ inputs.block-dir-src }} \ + CURRENT_STATE_DIR_SRC=${{ inputs.current-state-dir-src }} \ + START_BLOCK=${{ inputs.start-block }} \ + END_BLOCK=${{ inputs.end-block }} \ + BENCHMARK_OUTPUT_FILE="${{ env.BENCHMARK_OUTPUT_FILE }}" \ + TASK_NAME="custom-reexecution" + fi env: RUNNER_NAME: ${{ inputs.runner_name }} METRICS_COLLECTOR_ENABLED: ${{ inputs.prometheus-username != '' }} From f7f0ccfb0fcf27bedecad55cd7ffaf76a6998ca2 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Fri, 7 Nov 2025 00:35:21 +0400 Subject: [PATCH 14/21] ci(c-chain-reexecution): support both container and native altenrative (custom) approach to task based operations --- ...-chain-reexecution-benchmark-container.yml | 85 ++++++++++++++++++- ...-chain-reexecution-benchmark-gh-native.yml | 35 ++++++-- 2 files changed, 112 insertions(+), 8 deletions(-) diff --git a/.github/workflows/c-chain-reexecution-benchmark-container.yml b/.github/workflows/c-chain-reexecution-benchmark-container.yml index 80a215062c91..bda5dadc539c 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-container.yml +++ b/.github/workflows/c-chain-reexecution-benchmark-container.yml @@ -4,19 +4,93 @@ on: pull_request: workflow_dispatch: inputs: + config: + description: 'The config to pass to the VM for the benchmark. See BenchmarkReexecuteRange for details.' + default: '' + start-block: + description: 'The start block for the benchmark.' + default: '' + end-block: + description: 'The end block for the benchmark.' + default: '' + block-dir-src: + description: 'The source block directory. Supports S3 directory/zip and local directories.' + type: choice + default: '' + options: + - '' + - cchain-mainnet-blocks-200-ldb + - cchain-mainnet-blocks-10k-ldb + - cchain-mainnet-blocks-1m-ldb + - cchain-mainnet-blocks-1-10m-ldb + - cchain-mainnet-blocks-10m-20m-ldb + - cchain-mainnet-blocks-20m-30m-ldb + - cchain-mainnet-blocks-30m-40m-ldb + - cchain-mainnet-blocks-40m-50m-ldb + - cchain-mainnet-blocks-50m-ldb + - cchain-mainnet-blocks-50m-60m-ldb + - cchain-mainnet-blocks-60m-69m-ldb + - cchain-mainnet-blocks-69m-ldb + - cchain-mainnet-blocks-69m-70m-ldb + current-state-dir-src: + description: 'The current state directory. Supports S3 directory/zip and local directories.' + default: '' + type: choice + options: + - '' + - cchain-current-state-firewood-100 + - cchain-current-state-firewood-1m + - cchain-current-state-firewood-10m + - cchain-current-state-firewood-20m + - cchain-current-state-firewood-30m + - cchain-current-state-firewood-33m + - cchain-current-state-firewood-40m + - cchain-current-state-firewood-50m + - cchain-current-state-hashdb-full-100 + - cchain-current-state-hashdb-full-1k + - cchain-current-state-hashdb-full-1m + - cchain-current-state-hashdb-full-10m + - cchain-current-state-hashdb-full-20m + - cchain-current-state-hashdb-full-30m + - cchain-current-state-hashdb-full-33m + - cchain-current-state-hashdb-full-40m + - cchain-current-state-hashdb-full-50m + - cchain-current-state-hashdb-full-55m + - cchain-current-state-hashdb-full-60m + - cchain-current-state-hashdb-full-65m + - cchain-current-state-hashdb-full-68m + - cchain-current-state-hashdb-full-69m + - cchain-current-state-hashdb-archive-100 + - cchain-current-state-hashdb-archive-1k + - cchain-current-state-hashdb-archive-2k + - cchain-current-state-hashdb-archive-3k + - cchain-current-state-hashdb-archive-1m + - cchain-current-state-hashdb-archive-10m + - cchain-current-state-hashdb-archive-20m + - cchain-current-state-hashdb-archive-30m + - cchain-current-state-hashdb-archive-33m + - cchain-current-state-hashdb-archive-40m + - cchain-current-state-hashdb-archive-50m + - cchain-current-state-hashdb-archive-60m + - cchain-current-state-hashdb-archive-69m task: description: 'Taskfile task to execute (e.g., c-chain-reexecution-hashdb-101-250k)' - required: true + default: '' runner: description: 'Runner to execute the benchmark. Input to the runs-on field of the job.' - required: false default: ubuntu-latest + type: choice + options: + - ubuntu-latest + - blacksmith-4vcpu-ubuntu-2404 + - avalanche-avalanchego-runner-2ti + - avago-runner-m6i-4xlarge-ebs-fast + - avago-runner-i4i-4xlarge-local-ssd push-post-state: description: 'S3 location to push post-execution state directory. Skips this step if left unpopulated.' default: '' timeout-minutes: description: 'Timeout in minutes for the job.' - required: false default: 30 # Disabled because scheduled trigger is empty. To enable, uncomment and add at least one vector to the schedule @@ -80,6 +154,11 @@ jobs: uses: ./.github/actions/c-chain-reexecution-benchmark with: task: ${{ matrix.task }} + config: ${{ inputs.config }} + start-block: ${{ inputs.start-block }} + end-block: ${{ inputs.end-block }} + block-dir-src: ${{ inputs.block-dir-src }} + current-state-dir-src: ${{ inputs.current-state-dir-src }} prometheus-url: ${{ secrets.PROMETHEUS_URL || '' }} prometheus-push-url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }} prometheus-username: ${{ secrets.PROMETHEUS_USERNAME || '' }} diff --git a/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml b/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml index fe0ea75cfd0d..df298a7c141c 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml +++ b/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml @@ -4,19 +4,39 @@ on: pull_request: workflow_dispatch: inputs: + config: + description: 'The config to pass to the VM for the benchmark. See BenchmarkReexecuteRange for details.' + default: '' + start-block: + description: 'The start block for the benchmark.' + default: '' + end-block: + description: 'The end block for the benchmark.' + default: '' + block-dir-src: + description: 'The source block directory. Supports S3 directory/zip and local directories.' + default: '' + current-state-dir-src: + description: 'The current state directory. Supports S3 directory/zip and local directories.' + default: '' + task: + description: 'Taskfile task to execute (e.g., c-chain-reexecution-hashdb-101-250k)' + default: '' runner: description: 'Runner to execute the benchmark. Input to the runs-on field of the job.' - required: false default: ubuntu-latest - task: - description: 'Taskfile task to execute (e.g., c-chain-reexecution-hashdb-101-250k)' - required: true + type: choice + options: + - ubuntu-latest + - blacksmith-4vcpu-ubuntu-2404 + - avalanche-avalanchego-runner-2ti + - avago-runner-m6i-4xlarge-ebs-fast + - avago-runner-i4i-4xlarge-local-ssd push-post-state: description: 'S3 location to push post-execution state directory. Skips this step if left unpopulated.' default: '' timeout-minutes: description: 'Timeout in minutes for the job.' - required: false default: 30 # Disabled because scheduled trigger is empty. To enable, uncomment and add at least one vector to the schedule @@ -70,6 +90,11 @@ jobs: uses: ./.github/actions/c-chain-reexecution-benchmark with: task: ${{ matrix.task }} + config: ${{ inputs.config }} + start-block: ${{ inputs.start-block }} + end-block: ${{ inputs.end-block }} + block-dir-src: ${{ inputs.block-dir-src }} + current-state-dir-src: ${{ inputs.current-state-dir-src }} prometheus-url: ${{ secrets.PROMETHEUS_URL || '' }} prometheus-push-url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }} prometheus-username: ${{ secrets.PROMETHEUS_USERNAME || '' }} From 7921a03484ab48b639ba7cf26a50c99f1cbd24c2 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Fri, 7 Nov 2025 00:44:20 +0400 Subject: [PATCH 15/21] lint --- .../c-chain-reexecution-benchmark/action.yml | 2 +- ...-chain-reexecution-benchmark-container.yml | 54 ------------------- 2 files changed, 1 insertion(+), 55 deletions(-) diff --git a/.github/actions/c-chain-reexecution-benchmark/action.yml b/.github/actions/c-chain-reexecution-benchmark/action.yml index 6ac3741ef37d..bd628f322134 100644 --- a/.github/actions/c-chain-reexecution-benchmark/action.yml +++ b/.github/actions/c-chain-reexecution-benchmark/action.yml @@ -3,7 +3,7 @@ description: 'Run C-Chain re-execution benchmark' inputs: task: - description: 'Task name to execute from Taskfile.yml. Leave empty to use granular inputs below.' + description: 'Task name to execute from Taskfile.yml. Leave empty to use custom inputs below.' default: '' # Custom inputs (alternative to task-based approach) config: diff --git a/.github/workflows/c-chain-reexecution-benchmark-container.yml b/.github/workflows/c-chain-reexecution-benchmark-container.yml index bda5dadc539c..361eebe840b6 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-container.yml +++ b/.github/workflows/c-chain-reexecution-benchmark-container.yml @@ -15,64 +15,10 @@ on: default: '' block-dir-src: description: 'The source block directory. Supports S3 directory/zip and local directories.' - type: choice default: '' - options: - - '' - - cchain-mainnet-blocks-200-ldb - - cchain-mainnet-blocks-10k-ldb - - cchain-mainnet-blocks-1m-ldb - - cchain-mainnet-blocks-1-10m-ldb - - cchain-mainnet-blocks-10m-20m-ldb - - cchain-mainnet-blocks-20m-30m-ldb - - cchain-mainnet-blocks-30m-40m-ldb - - cchain-mainnet-blocks-40m-50m-ldb - - cchain-mainnet-blocks-50m-ldb - - cchain-mainnet-blocks-50m-60m-ldb - - cchain-mainnet-blocks-60m-69m-ldb - - cchain-mainnet-blocks-69m-ldb - - cchain-mainnet-blocks-69m-70m-ldb current-state-dir-src: description: 'The current state directory. Supports S3 directory/zip and local directories.' default: '' - type: choice - options: - - '' - - cchain-current-state-firewood-100 - - cchain-current-state-firewood-1m - - cchain-current-state-firewood-10m - - cchain-current-state-firewood-20m - - cchain-current-state-firewood-30m - - cchain-current-state-firewood-33m - - cchain-current-state-firewood-40m - - cchain-current-state-firewood-50m - - cchain-current-state-hashdb-full-100 - - cchain-current-state-hashdb-full-1k - - cchain-current-state-hashdb-full-1m - - cchain-current-state-hashdb-full-10m - - cchain-current-state-hashdb-full-20m - - cchain-current-state-hashdb-full-30m - - cchain-current-state-hashdb-full-33m - - cchain-current-state-hashdb-full-40m - - cchain-current-state-hashdb-full-50m - - cchain-current-state-hashdb-full-55m - - cchain-current-state-hashdb-full-60m - - cchain-current-state-hashdb-full-65m - - cchain-current-state-hashdb-full-68m - - cchain-current-state-hashdb-full-69m - - cchain-current-state-hashdb-archive-100 - - cchain-current-state-hashdb-archive-1k - - cchain-current-state-hashdb-archive-2k - - cchain-current-state-hashdb-archive-3k - - cchain-current-state-hashdb-archive-1m - - cchain-current-state-hashdb-archive-10m - - cchain-current-state-hashdb-archive-20m - - cchain-current-state-hashdb-archive-30m - - cchain-current-state-hashdb-archive-33m - - cchain-current-state-hashdb-archive-40m - - cchain-current-state-hashdb-archive-50m - - cchain-current-state-hashdb-archive-60m - - cchain-current-state-hashdb-archive-69m task: description: 'Taskfile task to execute (e.g., c-chain-reexecution-hashdb-101-250k)' default: '' From e703fe0a5bd411b5b1d01086cd84374abb0449d9 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Fri, 7 Nov 2025 00:58:51 +0400 Subject: [PATCH 16/21] refactor(scripts): extract S3 bucket config from copy_dir.sh to Taskfile env making copy_dir.sh reusable for any S3 bucket --- Taskfile.yml | 19 +++++++++++++++++-- scripts/copy_dir.sh | 9 +++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index c69714938034..8939d782b803 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -4,6 +4,9 @@ version: '3' +env: + S3_BOOTSTRAP_BUCKET: 's3://avalanchego-bootstrap-testing' + tasks: default: ./scripts/run_task.sh --list @@ -142,8 +145,8 @@ tasks: desc: Imports the C-Chain block and state data to re-execute. Defaults to import the first 200 and the current state created with the default config of the C-Chain (hashdb). vars: EXECUTION_DATA_DIR: '{{.EXECUTION_DATA_DIR}}' - BLOCK_DIR_SRC: '{{.BLOCK_DIR_SRC | default "s3://avalanchego-bootstrap-testing/cchain-mainnet-blocks-200-ldb/**"}}' - CURRENT_STATE_DIR_SRC: '{{.CURRENT_STATE_DIR_SRC | default "s3://avalanchego-bootstrap-testing/cchain-current-state-hashdb-full-100/**"}}' + BLOCK_DIR_SRC: '{{.BLOCK_DIR_SRC | default (printf "%s/cchain-mainnet-blocks-200-ldb/**" .S3_BOOTSTRAP_BUCKET)}}' + CURRENT_STATE_DIR_SRC: '{{.CURRENT_STATE_DIR_SRC | default (printf "%s/cchain-current-state-hashdb-full-100/**" .S3_BOOTSTRAP_BUCKET)}}' cmds: - task: import-s3-to-dir vars: @@ -299,6 +302,18 @@ tasks: CURRENT_STATE_DIR_SRC: 'cchain-current-state-firewood-33m' CONFIG: firewood + c-chain-reexecution-firewood-33m-40m: + desc: C-Chain re-execution from block 33m to 40m with firewood + cmds: + - task: reexecute-cchain-range-with-copied-data + vars: + TASK_NAME: '{{.TASK}}' + START_BLOCK: 33000001 + END_BLOCK: 40000000 + BLOCK_DIR_SRC: 'cchain-mainnet-blocks-30m-40m-ldb' + CURRENT_STATE_DIR_SRC: 'cchain-current-state-firewood-33m' + CONFIG: firewood + c-chain-reexecution-firewood-101-250k: desc: C-Chain re-execution from block 101 to 250k with firewood cmds: diff --git a/scripts/copy_dir.sh b/scripts/copy_dir.sh index 5bdafc3dc2c4..a6de180c1dc7 100755 --- a/scripts/copy_dir.sh +++ b/scripts/copy_dir.sh @@ -3,10 +3,6 @@ set -euo pipefail # Usage: ./scripts/copy_dir.sh source_directory destination_directory -# Sources can be: -# - S3 URLs (s3://bucket/path) -# - S3 object keys (will be expanded to s3://avalanchego-bootstrap-testing//**) -# - Local file paths # Assumes s5cmd has been installed and is available in the PATH. # s5cmd is included in the nix dev shell. @@ -24,8 +20,9 @@ DST="$2" # If SRC doesn't start with s3:// or /, assume it's an S3 object key if [[ "$SRC" != s3://* ]] && [[ "$SRC" != /* ]]; then - SRC="s3://avalanchego-bootstrap-testing/${SRC}/**" - echo "Expanded object key to: $SRC" + echo "Error: SRC must be either an S3 URL (s3://...), a local path (/...), or already expanded" + echo "If using an object key, expand it before calling this script" + exit 1 fi # Function to copy from a single source to destination From d359c7aeedbc02394db783eeb845d22c2049931b Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Fri, 7 Nov 2025 01:12:25 +0400 Subject: [PATCH 17/21] fix --- Taskfile.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 8939d782b803..746271999568 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -243,8 +243,8 @@ tasks: cmds: - task: import-cchain-reexecute-range vars: - BLOCK_DIR_SRC: '{{.BLOCK_DIR_SRC}}' - CURRENT_STATE_DIR_SRC: '{{.CURRENT_STATE_DIR_SRC}}' + BLOCK_DIR_SRC: '{{.S3_BOOTSTRAP_BUCKET}}/{{.BLOCK_DIR_SRC}}/**' + CURRENT_STATE_DIR_SRC: '{{.S3_BOOTSTRAP_BUCKET}}/{{.CURRENT_STATE_DIR_SRC}}/**' EXECUTION_DATA_DIR: '{{.EXECUTION_DATA_DIR}}' - task: reexecute-cchain-range vars: From 0cb04dc9def691259e08d66b93bff267985f74fe Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Wed, 12 Nov 2025 13:30:32 +0400 Subject: [PATCH 18/21] ci(c-chain-reexecution): use dynamic env variable for benchmark output file path --- .github/actions/c-chain-reexecution-benchmark/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/c-chain-reexecution-benchmark/action.yml b/.github/actions/c-chain-reexecution-benchmark/action.yml index bd628f322134..85a7c952af10 100644 --- a/.github/actions/c-chain-reexecution-benchmark/action.yml +++ b/.github/actions/c-chain-reexecution-benchmark/action.yml @@ -185,7 +185,7 @@ runs: uses: benchmark-action/github-action-benchmark@v1 with: tool: 'go' - output-file-path: benchmark-output.txt + output-file-path: ${{ env.BENCHMARK_OUTPUT_FILE }} summary-always: true github-token: ${{ inputs.github-token }} auto-push: ${{ inputs.push-github-action-benchmark }} From 3b2a82f1c8bc213b0aaa8371557aeb6001039b9f Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Wed, 12 Nov 2025 13:38:22 +0400 Subject: [PATCH 19/21] ci(c-chain-reexecution): remove unused `TASK_NAME` environment variable --- .github/actions/c-chain-reexecution-benchmark/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/c-chain-reexecution-benchmark/action.yml b/.github/actions/c-chain-reexecution-benchmark/action.yml index 85a7c952af10..6217ba3824d3 100644 --- a/.github/actions/c-chain-reexecution-benchmark/action.yml +++ b/.github/actions/c-chain-reexecution-benchmark/action.yml @@ -165,8 +165,7 @@ runs: CURRENT_STATE_DIR_SRC=${{ inputs.current-state-dir-src }} \ START_BLOCK=${{ inputs.start-block }} \ END_BLOCK=${{ inputs.end-block }} \ - BENCHMARK_OUTPUT_FILE="${{ env.BENCHMARK_OUTPUT_FILE }}" \ - TASK_NAME="custom-reexecution" + BENCHMARK_OUTPUT_FILE="${{ env.BENCHMARK_OUTPUT_FILE }}" fi env: RUNNER_NAME: ${{ inputs.runner_name }} From 3483737ff9af4ab0287822da1c0e51a5a403d6f8 Mon Sep 17 00:00:00 2001 From: "Elvis S." Date: Wed, 12 Nov 2025 14:17:27 +0400 Subject: [PATCH 20/21] ci(c-chain-reexecution): enforce required `runner` parameter by removing default options --- .../c-chain-reexecution-benchmark-container.yml | 9 +-------- .../c-chain-reexecution-benchmark-gh-native.yml | 9 +-------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/c-chain-reexecution-benchmark-container.yml b/.github/workflows/c-chain-reexecution-benchmark-container.yml index 361eebe840b6..83f0d16bd560 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-container.yml +++ b/.github/workflows/c-chain-reexecution-benchmark-container.yml @@ -24,14 +24,7 @@ on: default: '' runner: description: 'Runner to execute the benchmark. Input to the runs-on field of the job.' - default: ubuntu-latest - type: choice - options: - - ubuntu-latest - - blacksmith-4vcpu-ubuntu-2404 - - avalanche-avalanchego-runner-2ti - - avago-runner-m6i-4xlarge-ebs-fast - - avago-runner-i4i-4xlarge-local-ssd + required: true push-post-state: description: 'S3 location to push post-execution state directory. Skips this step if left unpopulated.' default: '' diff --git a/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml b/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml index df298a7c141c..d84033ee5232 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml +++ b/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml @@ -24,14 +24,7 @@ on: default: '' runner: description: 'Runner to execute the benchmark. Input to the runs-on field of the job.' - default: ubuntu-latest - type: choice - options: - - ubuntu-latest - - blacksmith-4vcpu-ubuntu-2404 - - avalanche-avalanchego-runner-2ti - - avago-runner-m6i-4xlarge-ebs-fast - - avago-runner-i4i-4xlarge-local-ssd + required: true push-post-state: description: 'S3 location to push post-execution state directory. Skips this step if left unpopulated.' default: '' From 9962085f33d0e51a9053064be0c6e7bed19b9128 Mon Sep 17 00:00:00 2001 From: maru Date: Fri, 14 Nov 2025 00:16:10 +0000 Subject: [PATCH 21/21] Simplify testing with arbitrary versions of firewood This change adds support for using an experimental tool called `polyrepo` to clone firewood and coreth and integrate them with the local avalanchego clone so that avalanchego tasks can be run against a specific combination of the 3 repos. To test-drive: - Clone this branch - Install nix - Use polyrepo to sync compatible versions of firewood and coreth - ./bin/polyrepo sync\ firewood@maru/ffi-reenable-ci-job coreth@maru/v0.16.0-firewood-v0.0.14 - Clones firewood and coreth and configures their use together with the local avalanchego clone - Ensure the shell is configured for AWS access (required for S3 downloads) - Okta -> AWS Access Portal -> Experimental -> Access Keys - Note: these credentials will expire after 1 hour - export AWS_ACCESS_KEY_ID= - export AWS_SECRET_ACCESS_KEY= - export AWS_SESSION_TOKEN= - export AWS_REGION=us-east-2 - Run a reexecution test - ./scripts/run_task.sh c-chain-reexecution-firewood-101-250k --- .gitignore | 5 +++++ bin/polyrepo | 1 + scripts/run_polyrepo.sh | 7 +++++++ 3 files changed, 13 insertions(+) create mode 120000 bin/polyrepo create mode 100755 scripts/run_polyrepo.sh diff --git a/.gitignore b/.gitignore index f2c2f521d92f..ff4bb7c76bce 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,8 @@ vendor # debug files __debug_* + +# polyrepo +polyrepo.log +firewood +coreth diff --git a/bin/polyrepo b/bin/polyrepo new file mode 120000 index 000000000000..d1d8f7bd1edf --- /dev/null +++ b/bin/polyrepo @@ -0,0 +1 @@ +../scripts/run_polyrepo.sh \ No newline at end of file diff --git a/scripts/run_polyrepo.sh b/scripts/run_polyrepo.sh new file mode 100755 index 000000000000..23ab4ec7bd1a --- /dev/null +++ b/scripts/run_polyrepo.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -euo pipefail + +POLYREPO_REVISION=0c4c6fcc92 +echo "Running polyrepo@${POLYREPO_REVISION} via go run..." +go run github.com/ava-labs/avalanchego/tests/fixture/polyrepo@"${POLYREPO_REVISION}" "${@}"