[WIP] Use bp-runner in benchmarks

.gitlab/benchmarks.yml

@@ -35,23 +35,23 @@ benchmarks-startup:
   script:
     - !reference [ .benchmarks, script ]
     - ./steps/capture-hardware-software-info.sh
-    - ./steps/run-benchmarks.sh startup
+    - BENCHMARK_TYPE=startup bp-runner "${CI_PROJECT_DIR:-.}/.gitlab/benchmarks/bp-runner.yml" --debug -t
     - ./steps/analyze-results.sh startup
 
 benchmarks-load:
   extends: .benchmarks
   script:
     - !reference [ .benchmarks, script ]
     - ./steps/capture-hardware-software-info.sh
-    - ./steps/run-benchmarks.sh load
+    - BENCHMARK_TYPE=load bp-runner "${CI_PROJECT_DIR:-.}/.gitlab/benchmarks/bp-runner.yml" --debug -t
     - ./steps/analyze-results.sh load
 
 benchmarks-dacapo:
   extends: .benchmarks
   script:
     - !reference [ .benchmarks, script ]
     - ./steps/capture-hardware-software-info.sh
-    - ./steps/run-benchmarks.sh dacapo
+    - BENCHMARK_TYPE=dacapo bp-runner "${CI_PROJECT_DIR:-.}/.gitlab/benchmarks/bp-runner.yml" --debug -t
     - ./steps/analyze-results.sh dacapo
 
 benchmarks-post-results:

.gitlab/benchmarks/bp-runner.yml

@@ -0,0 +1,10 @@
+# Usage: BENCHMARK_TYPE=<startup|load|dacapo> bp-runner .gitlab/benchmarks/bp-runner.yml --debug -t
+
+experiments:
+  - name: run-benchmarks
+    steps:
+      - name: benchmarks
+        cpus: 24-47
+        run: shell
+        script: /platform/steps/run-benchmarks.sh $BENCHMARK_TYPE
+

benchmark/Dockerfile

@@ -78,11 +78,7 @@ ENV JAVA_17_HOME=/usr/lib/jvm/17
 ENV JAVA_HOME=${JAVA_8_HOME}
 ENV PATH=${PATH}:${JAVA_HOME}/bin
 
-ARG SIRUN_VERSION=0.1.11
-RUN wget -O sirun.tar.gz https://github.com/DataDog/sirun/releases/download/v$SIRUN_VERSION/sirun-v$SIRUN_VERSION-x86_64-unknown-linux-musl.tar.gz \
-	&& tar -xzf sirun.tar.gz \
-	&& rm sirun.tar.gz \
-  && mv sirun /usr/bin/sirun
+# Install bp-runner somehow
 
 ARG K6_VERSION=0.45.1
 RUN wget -O k6.tar.gz https://github.com/grafana/k6/releases/download/v$K6_VERSION/k6-v$K6_VERSION-linux-amd64.tar.gz \

benchmark/README.MD

@@ -4,7 +4,7 @@ This directory contains different types of benchmarks.
 
 ## Running Benchmarks via Docker
 
-Docker allows the execution of benchmarks without needing to install and configure your development environment. For example, package installation and installation of sirun are performed automatically.
+Docker allows the execution of benchmarks without needing to install and configure your development environment.
 
 In order to run benchmarks using Docker, issue the following command from the `benchmark/` folder of this project:
 

benchmark/dacapo/bp-runner.yml

@@ -0,0 +1,115 @@
+# bp-runner configuration for dacapo benchmarks based on sirun's benchmark.json format
+#
+# Each variant (no_agent, tracing, profiling, appsec, iast, iast_GLOBAL) is defined
+# as a separate experiment.
+#
+# CPU pinning uses cores 24-47 for consistency with CI configuration.
+
+experiments:
+  # no-agent variant
+  - name: dacapo-no-agent
+    steps:
+      - name: run-dacapo-benchmark
+        cpus: 24-47
+        run: shell
+        script: |
+          export VARIANT="${NO_AGENT_VARIANT}"
+          export JAVA_OPTS=""
+          mkdir -p ${OUTPUT_DIR}/${VARIANT}
+
+          java ${JAVA_OPTS} \
+            -jar ${DACAPO} \
+            --converge \
+            --scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \
+            --latency-csv \
+            ${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log
+
+  # tracing variant
+  - name: dacapo-tracing
+    steps:
+      - name: run-dacapo-benchmark
+        cpus: 24-47
+        run: shell
+        script: |
+          export VARIANT="tracing"
+          export JAVA_OPTS="-javaagent:${TRACER}"
+          mkdir -p ${OUTPUT_DIR}/${VARIANT}
+
+          java ${JAVA_OPTS} \
+            -jar ${DACAPO} \
+            --converge \
+            --scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \
+            --latency-csv \
+            ${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log
+
+  # profiling variant
+  - name: dacapo-profiling
+    steps:
+      - name: run-dacapo-benchmark
+        cpus: 24-47
+        run: shell
+        script: |
+          export VARIANT="profiling"
+          export JAVA_OPTS="-javaagent:${TRACER} -Ddd.profiling.enabled=true"
+          mkdir -p ${OUTPUT_DIR}/${VARIANT}
+
+          java ${JAVA_OPTS} \
+            -jar ${DACAPO} \
+            --converge \
+            --scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \
+            --latency-csv \
+            ${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log
+
+  # appsec variant
+  - name: dacapo-appsec
+    steps:
+      - name: run-dacapo-benchmark
+        cpus: 24-47
+        run: shell
+        script: |
+          export VARIANT="appsec"
+          export JAVA_OPTS="-javaagent:${TRACER} -Ddd.appsec.enabled=true -Ddd.iast.enabled=false"
+          mkdir -p ${OUTPUT_DIR}/${VARIANT}
+
+          java ${JAVA_OPTS} \
+            -jar ${DACAPO} \
+            --converge \
+            --scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \
+            --latency-csv \
+            ${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log
+
+  # iast variant
+  - name: dacapo-iast
+    steps:
+      - name: run-dacapo-benchmark
+        cpus: 24-47
+        run: shell
+        script: |
+          export VARIANT="iast"
+          export JAVA_OPTS="-javaagent:${TRACER} -Ddd.iast.enabled=true"
+          mkdir -p ${OUTPUT_DIR}/${VARIANT}
+
+          java ${JAVA_OPTS} \
+            -jar ${DACAPO} \
+            --converge \
+            --scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \
+            --latency-csv \
+            ${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log
+
+  # iast_GLOBAL variant
+  - name: dacapo-iast-global
+    steps:
+      - name: run-dacapo-benchmark
+        cpus: 24-47
+        run: shell
+        script: |
+          export VARIANT="iast_GLOBAL"
+          export JAVA_OPTS="-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL"
+          mkdir -p ${OUTPUT_DIR}/${VARIANT}
+
+          java ${JAVA_OPTS} \
+            -jar ${DACAPO} \
+            --converge \
+            --scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \
+            --latency-csv \
+            ${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log

benchmark/dacapo/run.sh

@@ -19,14 +19,8 @@ run_benchmark() {
   export OUTPUT_DIR="${REPORTS_DIR}/dacapo/${type}"
   mkdir -p "${OUTPUT_DIR}"
 
-  # substitute environment variables in the json file
-  benchmark=$(mktemp)
-  # shellcheck disable=SC2046
-  # shellcheck disable=SC2016
-  envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1))" <benchmark.json >"${benchmark}"
-
-  # run the sirun test
-  sirun "${benchmark}" &>"${OUTPUT_DIR}/${type}.json"
+  # run the bp-runner test
+  bp-runner bp-runner.yml &>"${OUTPUT_DIR}/${type}.json"
 
   message "dacapo benchmark: ${type} finished"
 }

benchmark/startup/insecure-bank/bp-runner.yml

@@ -0,0 +1,53 @@
+# bp-runner configuration for insecure-bank startup benchmarks based on sirun's benchmark.json format
+#
+# Each variant (tracing, iast) is defined as a separate experiment.
+#
+# CPU pinning uses cores 24-47 for consistency with CI configuration.
+experiments:
+  # tracing variant
+  - name: startup-insecure-bank-tracing
+    setup:
+      - name: service-monitor
+        run: shell
+        script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080/login 'pkill java'
+
+    steps:
+      - name: run-startup-benchmark
+        cpus: 24-47
+        run: shell
+        script: |
+          export VARIANT="tracing"
+          export JAVA_OPTS=""
+          mkdir -p ${OUTPUT_DIR}/${VARIANT}
+
+          for i in $(seq 1 10); do
+            java -javaagent:${TRACER} \
+              -Ddd.benchmark.enabled=true \
+              -Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \
+              ${JAVA_OPTS} \
+              -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log || true
+          done
+
+  # iast variant
+  - name: startup-insecure-bank-iast
+    setup:
+      - name: service-monitor
+        run: shell
+        script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080/login 'pkill java'
+
+    steps:
+      - name: run-startup-benchmark
+        cpus: 24-47
+        run: shell
+        script: |
+          export VARIANT="iast"
+          export JAVA_OPTS="-Ddd.iast.enabled=true"
+          mkdir -p ${OUTPUT_DIR}/${VARIANT}
+
+          for i in $(seq 1 10); do
+            java -javaagent:${TRACER} \
+              -Ddd.benchmark.enabled=true \
+              -Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \
+              ${JAVA_OPTS} \
+              -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log || true
+          done

benchmark/startup/petclinic/bp-runner.yml

@@ -0,0 +1,102 @@
+# bp-runner configuration for petclinic startup benchmarks based on sirun's benchmark.json format
+#
+# Each variant (tracing, profiling, appsec, iast) is defined as a separate experiment.
+#
+# CPU pinning uses cores 24-47 for consistency with CI configuration.
+experiments:
+  # tracing variant
+  - name: startup-tracing
+    setup:
+      - name: service-monitor
+        run: shell
+        script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080 'pkill java'
+
+    steps:
+      - name: run-startup-benchmark
+        cpus: 24-47
+        run: shell
+        script: |
+          export VARIANT="tracing"
+          export JAVA_OPTS=""
+          mkdir -p ${OUTPUT_DIR}/${VARIANT}
+
+          # Run benchmark iterations (converted from sirun's iterations: 10)
+          for i in $(seq 1 10); do
+            java -javaagent:${TRACER} \
+              -Ddd.benchmark.enabled=true \
+              -Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \
+              ${JAVA_OPTS} \
+              -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log || true
+          done
+
+  # profiling variant
+  - name: startup-profiling
+    setup:
+      - name: service-monitor
+        run: shell
+        script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080 'pkill java'
+
+    steps:
+      - name: run-startup-benchmark
+        cpus: 24-47
+        run: shell
+        script: |
+          export VARIANT="profiling"
+          export JAVA_OPTS="-Ddd.profiling.enabled=true"
+          mkdir -p ${OUTPUT_DIR}/${VARIANT}
+
+          for i in $(seq 1 10); do
+            java -javaagent:${TRACER} \
+              -Ddd.benchmark.enabled=true \
+              -Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \
+              ${JAVA_OPTS} \
+              -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log || true
+          done
+
+  # appsec variant
+  - name: startup-appsec
+    setup:
+      - name: service-monitor
+        run: shell
+        script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080 'pkill java'
+
+    steps:
+      - name: run-startup-benchmark
+        cpus: 24-47
+        run: shell
+        script: |
+          export VARIANT="appsec"
+          export JAVA_OPTS="-Ddd.appsec.enabled=true"
+          mkdir -p ${OUTPUT_DIR}/${VARIANT}
+
+          for i in $(seq 1 10); do
+            java -javaagent:${TRACER} \
+              -Ddd.benchmark.enabled=true \
+              -Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \
+              ${JAVA_OPTS} \
+              -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log || true
+          done
+
+  # iast variant
+  - name: startup-iast
+    setup:
+      - name: service-monitor
+        run: shell
+        script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080 'pkill java'
+
+    steps:
+      - name: run-startup-benchmark
+        cpus: 24-47
+        run: shell
+        script: |
+          export VARIANT="iast"
+          export JAVA_OPTS="-Ddd.iast.enabled=true"
+          mkdir -p ${OUTPUT_DIR}/${VARIANT}
+
+          for i in $(seq 1 10); do
+            java -javaagent:${TRACER} \
+              -Ddd.benchmark.enabled=true \
+              -Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \
+              ${JAVA_OPTS} \
+              -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log || true
+          done

benchmark/startup/run.sh

@@ -2,4 +2,37 @@
 set -eu
 
 source "${UTILS_DIR}/update-java-version.sh" 17
-"${UTILS_DIR}/run-sirun-benchmarks.sh" "$@"
+
+# from run-sirun-benchmarks.sh, matches dacapo and load benchmark run.sh scripts
+function message() {
+  echo "$(date +"%T"): $1"
+}
+
+run_benchmark() {
+  local type=$1
+  local app=$2
+  if [[ -d "${app}" ]] && [[ -f "${app}/benchmark.json" ]]; then
+
+    message "${type} benchmark: ${app} started"
+    cd "${app}"
+
+    # create output folder for the test
+    export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}"
+    mkdir -p "${OUTPUT_DIR}"
+
+    # run the bp-runner test
+    bp-runner bp-runner.yml &>"${OUTPUT_DIR}/${app}.json"
+
+    message "${type} benchmark: ${app} finished"
+
+    cd ..
+  fi
+}
+
+if [ "$#" == '2' ]; then
+  run_benchmark "$@"
+else
+  for folder in *; do
+    run_benchmark "$1" "${folder}"
+  done
+fi