diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 28aef7ad58b..75e6f940f0b 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -35,7 +35,7 @@ benchmarks-startup: script: - !reference [ .benchmarks, script ] - ./steps/capture-hardware-software-info.sh - - ./steps/run-benchmarks.sh startup + - BENCHMARK_TYPE=startup bp-runner "${CI_PROJECT_DIR:-.}/.gitlab/benchmarks/bp-runner.yml" --debug -t - ./steps/analyze-results.sh startup benchmarks-load: @@ -43,7 +43,7 @@ benchmarks-load: script: - !reference [ .benchmarks, script ] - ./steps/capture-hardware-software-info.sh - - ./steps/run-benchmarks.sh load + - BENCHMARK_TYPE=load bp-runner "${CI_PROJECT_DIR:-.}/.gitlab/benchmarks/bp-runner.yml" --debug -t - ./steps/analyze-results.sh load benchmarks-dacapo: @@ -51,7 +51,7 @@ benchmarks-dacapo: script: - !reference [ .benchmarks, script ] - ./steps/capture-hardware-software-info.sh - - ./steps/run-benchmarks.sh dacapo + - BENCHMARK_TYPE=dacapo bp-runner "${CI_PROJECT_DIR:-.}/.gitlab/benchmarks/bp-runner.yml" --debug -t - ./steps/analyze-results.sh dacapo benchmarks-post-results: diff --git a/.gitlab/benchmarks/bp-runner.yml b/.gitlab/benchmarks/bp-runner.yml new file mode 100644 index 00000000000..578931fc0dd --- /dev/null +++ b/.gitlab/benchmarks/bp-runner.yml @@ -0,0 +1,10 @@ +# Usage: BENCHMARK_TYPE= bp-runner .gitlab/benchmarks/bp-runner.yml --debug -t + +experiments: + - name: run-benchmarks + steps: + - name: benchmarks + cpus: 24-47 + run: shell + script: /platform/steps/run-benchmarks.sh $BENCHMARK_TYPE + diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 0279186478a..863a540f511 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -78,11 +78,7 @@ ENV JAVA_17_HOME=/usr/lib/jvm/17 ENV JAVA_HOME=${JAVA_8_HOME} ENV PATH=${PATH}:${JAVA_HOME}/bin -ARG SIRUN_VERSION=0.1.11 -RUN wget -O sirun.tar.gz https://github.com/DataDog/sirun/releases/download/v$SIRUN_VERSION/sirun-v$SIRUN_VERSION-x86_64-unknown-linux-musl.tar.gz \ - && tar -xzf sirun.tar.gz \ - && rm sirun.tar.gz \ - && mv sirun /usr/bin/sirun +# Install bp-runner somehow ARG K6_VERSION=0.45.1 RUN wget -O k6.tar.gz https://github.com/grafana/k6/releases/download/v$K6_VERSION/k6-v$K6_VERSION-linux-amd64.tar.gz \ diff --git a/benchmark/README.MD b/benchmark/README.MD index 30f3bbcf864..cec1f7d3250 100644 --- a/benchmark/README.MD +++ b/benchmark/README.MD @@ -4,7 +4,7 @@ This directory contains different types of benchmarks. ## Running Benchmarks via Docker -Docker allows the execution of benchmarks without needing to install and configure your development environment. For example, package installation and installation of sirun are performed automatically. +Docker allows the execution of benchmarks without needing to install and configure your development environment. In order to run benchmarks using Docker, issue the following command from the `benchmark/` folder of this project: diff --git a/benchmark/dacapo/bp-runner.yml b/benchmark/dacapo/bp-runner.yml new file mode 100644 index 00000000000..1d501f617cc --- /dev/null +++ b/benchmark/dacapo/bp-runner.yml @@ -0,0 +1,115 @@ +# bp-runner configuration for dacapo benchmarks based on sirun's benchmark.json format +# +# Each variant (no_agent, tracing, profiling, appsec, iast, iast_GLOBAL) is defined +# as a separate experiment. +# +# CPU pinning uses cores 24-47 for consistency with CI configuration. + +experiments: + # no-agent variant + - name: dacapo-no-agent + steps: + - name: run-dacapo-benchmark + cpus: 24-47 + run: shell + script: | + export VARIANT="${NO_AGENT_VARIANT}" + export JAVA_OPTS="" + mkdir -p ${OUTPUT_DIR}/${VARIANT} + + java ${JAVA_OPTS} \ + -jar ${DACAPO} \ + --converge \ + --scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \ + --latency-csv \ + ${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log + + # tracing variant + - name: dacapo-tracing + steps: + - name: run-dacapo-benchmark + cpus: 24-47 + run: shell + script: | + export VARIANT="tracing" + export JAVA_OPTS="-javaagent:${TRACER}" + mkdir -p ${OUTPUT_DIR}/${VARIANT} + + java ${JAVA_OPTS} \ + -jar ${DACAPO} \ + --converge \ + --scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \ + --latency-csv \ + ${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log + + # profiling variant + - name: dacapo-profiling + steps: + - name: run-dacapo-benchmark + cpus: 24-47 + run: shell + script: | + export VARIANT="profiling" + export JAVA_OPTS="-javaagent:${TRACER} -Ddd.profiling.enabled=true" + mkdir -p ${OUTPUT_DIR}/${VARIANT} + + java ${JAVA_OPTS} \ + -jar ${DACAPO} \ + --converge \ + --scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \ + --latency-csv \ + ${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log + + # appsec variant + - name: dacapo-appsec + steps: + - name: run-dacapo-benchmark + cpus: 24-47 + run: shell + script: | + export VARIANT="appsec" + export JAVA_OPTS="-javaagent:${TRACER} -Ddd.appsec.enabled=true -Ddd.iast.enabled=false" + mkdir -p ${OUTPUT_DIR}/${VARIANT} + + java ${JAVA_OPTS} \ + -jar ${DACAPO} \ + --converge \ + --scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \ + --latency-csv \ + ${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log + + # iast variant + - name: dacapo-iast + steps: + - name: run-dacapo-benchmark + cpus: 24-47 + run: shell + script: | + export VARIANT="iast" + export JAVA_OPTS="-javaagent:${TRACER} -Ddd.iast.enabled=true" + mkdir -p ${OUTPUT_DIR}/${VARIANT} + + java ${JAVA_OPTS} \ + -jar ${DACAPO} \ + --converge \ + --scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \ + --latency-csv \ + ${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log + + # iast_GLOBAL variant + - name: dacapo-iast-global + steps: + - name: run-dacapo-benchmark + cpus: 24-47 + run: shell + script: | + export VARIANT="iast_GLOBAL" + export JAVA_OPTS="-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL" + mkdir -p ${OUTPUT_DIR}/${VARIANT} + + java ${JAVA_OPTS} \ + -jar ${DACAPO} \ + --converge \ + --scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch \ + --latency-csv \ + ${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log diff --git a/benchmark/dacapo/run.sh b/benchmark/dacapo/run.sh index ece44f9e5f0..81af6166f85 100755 --- a/benchmark/dacapo/run.sh +++ b/benchmark/dacapo/run.sh @@ -19,14 +19,8 @@ run_benchmark() { export OUTPUT_DIR="${REPORTS_DIR}/dacapo/${type}" mkdir -p "${OUTPUT_DIR}" - # substitute environment variables in the json file - benchmark=$(mktemp) - # shellcheck disable=SC2046 - # shellcheck disable=SC2016 - envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1))" "${benchmark}" - - # run the sirun test - sirun "${benchmark}" &>"${OUTPUT_DIR}/${type}.json" + # run the bp-runner test + bp-runner bp-runner.yml &>"${OUTPUT_DIR}/${type}.json" message "dacapo benchmark: ${type} finished" } diff --git a/benchmark/startup/insecure-bank/bp-runner.yml b/benchmark/startup/insecure-bank/bp-runner.yml new file mode 100644 index 00000000000..cd14e54f0d1 --- /dev/null +++ b/benchmark/startup/insecure-bank/bp-runner.yml @@ -0,0 +1,53 @@ +# bp-runner configuration for insecure-bank startup benchmarks based on sirun's benchmark.json format +# +# Each variant (tracing, iast) is defined as a separate experiment. +# +# CPU pinning uses cores 24-47 for consistency with CI configuration. +experiments: + # tracing variant + - name: startup-insecure-bank-tracing + setup: + - name: service-monitor + run: shell + script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080/login 'pkill java' + + steps: + - name: run-startup-benchmark + cpus: 24-47 + run: shell + script: | + export VARIANT="tracing" + export JAVA_OPTS="" + mkdir -p ${OUTPUT_DIR}/${VARIANT} + + for i in $(seq 1 10); do + java -javaagent:${TRACER} \ + -Ddd.benchmark.enabled=true \ + -Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \ + ${JAVA_OPTS} \ + -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log || true + done + + # iast variant + - name: startup-insecure-bank-iast + setup: + - name: service-monitor + run: shell + script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080/login 'pkill java' + + steps: + - name: run-startup-benchmark + cpus: 24-47 + run: shell + script: | + export VARIANT="iast" + export JAVA_OPTS="-Ddd.iast.enabled=true" + mkdir -p ${OUTPUT_DIR}/${VARIANT} + + for i in $(seq 1 10); do + java -javaagent:${TRACER} \ + -Ddd.benchmark.enabled=true \ + -Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \ + ${JAVA_OPTS} \ + -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log || true + done diff --git a/benchmark/startup/petclinic/bp-runner.yml b/benchmark/startup/petclinic/bp-runner.yml new file mode 100644 index 00000000000..8e48a957cb9 --- /dev/null +++ b/benchmark/startup/petclinic/bp-runner.yml @@ -0,0 +1,102 @@ +# bp-runner configuration for petclinic startup benchmarks based on sirun's benchmark.json format +# +# Each variant (tracing, profiling, appsec, iast) is defined as a separate experiment. +# +# CPU pinning uses cores 24-47 for consistency with CI configuration. +experiments: + # tracing variant + - name: startup-tracing + setup: + - name: service-monitor + run: shell + script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080 'pkill java' + + steps: + - name: run-startup-benchmark + cpus: 24-47 + run: shell + script: | + export VARIANT="tracing" + export JAVA_OPTS="" + mkdir -p ${OUTPUT_DIR}/${VARIANT} + + # Run benchmark iterations (converted from sirun's iterations: 10) + for i in $(seq 1 10); do + java -javaagent:${TRACER} \ + -Ddd.benchmark.enabled=true \ + -Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \ + ${JAVA_OPTS} \ + -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log || true + done + + # profiling variant + - name: startup-profiling + setup: + - name: service-monitor + run: shell + script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080 'pkill java' + + steps: + - name: run-startup-benchmark + cpus: 24-47 + run: shell + script: | + export VARIANT="profiling" + export JAVA_OPTS="-Ddd.profiling.enabled=true" + mkdir -p ${OUTPUT_DIR}/${VARIANT} + + for i in $(seq 1 10); do + java -javaagent:${TRACER} \ + -Ddd.benchmark.enabled=true \ + -Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \ + ${JAVA_OPTS} \ + -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log || true + done + + # appsec variant + - name: startup-appsec + setup: + - name: service-monitor + run: shell + script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080 'pkill java' + + steps: + - name: run-startup-benchmark + cpus: 24-47 + run: shell + script: | + export VARIANT="appsec" + export JAVA_OPTS="-Ddd.appsec.enabled=true" + mkdir -p ${OUTPUT_DIR}/${VARIANT} + + for i in $(seq 1 10); do + java -javaagent:${TRACER} \ + -Ddd.benchmark.enabled=true \ + -Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \ + ${JAVA_OPTS} \ + -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log || true + done + + # iast variant + - name: startup-iast + setup: + - name: service-monitor + run: shell + script: ${UTILS_DIR}/run-on-server-ready.sh http://localhost:8080 'pkill java' + + steps: + - name: run-startup-benchmark + cpus: 24-47 + run: shell + script: | + export VARIANT="iast" + export JAVA_OPTS="-Ddd.iast.enabled=true" + mkdir -p ${OUTPUT_DIR}/${VARIANT} + + for i in $(seq 1 10); do + java -javaagent:${TRACER} \ + -Ddd.benchmark.enabled=true \ + -Ddd.benchmark.output.dir=${OUTPUT_DIR}/${VARIANT} \ + ${JAVA_OPTS} \ + -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log || true + done diff --git a/benchmark/startup/run.sh b/benchmark/startup/run.sh index 432c65d3fd5..51e89c3a019 100755 --- a/benchmark/startup/run.sh +++ b/benchmark/startup/run.sh @@ -2,4 +2,37 @@ set -eu source "${UTILS_DIR}/update-java-version.sh" 17 -"${UTILS_DIR}/run-sirun-benchmarks.sh" "$@" + +# from run-sirun-benchmarks.sh, matches dacapo and load benchmark run.sh scripts +function message() { + echo "$(date +"%T"): $1" +} + +run_benchmark() { + local type=$1 + local app=$2 + if [[ -d "${app}" ]] && [[ -f "${app}/benchmark.json" ]]; then + + message "${type} benchmark: ${app} started" + cd "${app}" + + # create output folder for the test + export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}" + mkdir -p "${OUTPUT_DIR}" + + # run the bp-runner test + bp-runner bp-runner.yml &>"${OUTPUT_DIR}/${app}.json" + + message "${type} benchmark: ${app} finished" + + cd .. + fi +} + +if [ "$#" == '2' ]; then + run_benchmark "$@" +else + for folder in *; do + run_benchmark "$1" "${folder}" + done +fi