Skip to content

Commit 2d6aa39

Browse files
laserlaser
authored
Feat/821 and 814 benchmark improvements (#828)
* feat(combine benchmark output into single doc) - use bc for floating-point math - fix benchy such that it puts maximum RSS in correct place - modify CircleCI command to combine output into single JSON doc - don't allocate psuedo-tty in `ssh` command (which was causing stderr and stdout to be combined) * fixup(point at branch) * feat(add taper, taper-layers, and layer-challenges to zigzag input object) * bug(update "hash-constraints" string in both places) * feat(don't need to be so quiet) * refactor(move maximum rss around) * add composable with-dots.sh script * feat(add status-dots) * feat(persist JSON docs as CircleCI artifacts) * fixup(run full benchmarks) * refactor(move benchmark aggregator into standalone script)
1 parent e7ba5c7 commit 2d6aa39

File tree

11 files changed

+83
-17
lines changed

11 files changed

+83
-17
lines changed

‎.circleci/config.yml‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -195,16 +195,35 @@ jobs:
195195
command: apt-get install time jq -yqq
196196
- run:
197197
name: Run hash-constraints benchmarks on remote host
198-
command: ./fil-proofs-tooling/scripts/benchy-remote.sh "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}" hash-circuits
198+
command: |
199+
./fil-proofs-tooling/scripts/benchy-remote.sh "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}" hash-constraints > hash-constraints.json
200+
cat hash-constraints.json
199201
no_output_timeout: 60m
200202
- run:
201203
name: Run micro benchmarks
202-
command: ./fil-proofs-tooling/scripts/micro-remote.sh "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}"
204+
command: |
205+
./fil-proofs-tooling/scripts/micro-remote.sh "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}" > micro-benchmarks.json
206+
cat micro-benchmarks.json
203207
no_output_timeout: 60m
204208
- run:
205209
name: Run ZigZag benchmarks using 1GiB sectors
206-
command: ./fil-proofs-tooling/scripts/benchy-remote.sh "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}" zigzag --size=$((1024*1024))
210+
command: |
211+
./fil-proofs-tooling/scripts/benchy-remote.sh "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}" zigzag --size=$((1024*1024)) > zigzag-benchmarks.json
212+
cat zigzag-benchmarks.json
207213
no_output_timeout: 60m
214+
- run:
215+
name: Aggregate benchmarks into single JSON document
216+
command: |
217+
./fil-proofs-tooling/scripts/aggregate-benchmarks.sh zigzag-benchmarks.json micro-benchmarks.json hash-constraints.json > aggregated-benchmarks.json
218+
cat aggregated-benchmarks.json
219+
- store_artifacts:
220+
path: zigzag-benchmarks.json
221+
- store_artifacts:
222+
path: hash-constraints.json
223+
- store_artifacts:
224+
path: micro-benchmarks.json
225+
- store_artifacts:
226+
path: aggregated-benchmarks.json
208227

209228
rustfmt:
210229
docker:

‎fil-proofs-tooling/scripts/aggregate-benchmarks.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
zigzag_path=$1
6+
micro_path=$2
7+
hash_constraints_path=$3
8+
9+
jq --sort-keys -s '{ benchmarks: { "zigzag-benchmarks": { outputs: { "max-resident-set-size-kb": .[0] } } } } * .[1]' \
10+
<(jq '.["max-resident-set-size-kb"]' $zigzag_path) \
11+
<(jq -s '.[0] * { benchmarks: { "hash-constraints": .[1], "zigzag-benchmarks": .[2], "micro-benchmarks": .[3] } }' \
12+
<(jq 'del (.benchmarks)' $micro_path) \
13+
<(jq '.benchmarks' $hash_constraints_path) \
14+
<(jq '.benchmarks' $zigzag_path) \
15+
<(jq '.benchmarks' $micro_path))

‎fil-proofs-tooling/scripts/benchy-remote.sh‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,14 @@ set -e
44

55
CMDS=$(cat <<EOF
66
cd \$(mktemp -d)
7-
git clone -q https://github.com/filecoin-project/rust-fil-proofs.git
7+
git clone https://github.com/filecoin-project/rust-fil-proofs.git
88
cd rust-fil-proofs
99
git checkout -q master
1010
./fil-proofs-tooling/scripts/retry.sh 42 10 60000 \
1111
./fil-proofs-tooling/scripts/with-lock.sh 42 /tmp/benchmark \
12+
./fil-proofs-tooling/scripts/with-dots.sh \
1213
./fil-proofs-tooling/scripts/benchy.sh ${@:2}
1314
EOF
1415
)
1516

16-
ssh -t -q $1 "$CMDS"
17+
ssh -q $1 "$CMDS"

‎fil-proofs-tooling/scripts/benchy.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ GTIME_STDERR=$(mktemp)
77
JQ_STDERR=$(mktemp)
88

99
GTIME_BIN="env time"
10-
GTIME_ARG="-f '{ \"outputs\": { \"max-resident-set-size-kb\": %M } }' cargo run --quiet --bin benchy --release -- ${@}"
10+
GTIME_ARG="-f '{ \"max-resident-set-size-kb\": %M }' cargo run --quiet --bin benchy --release -- ${@}"
1111

1212
if [[ $(env time --version 2>&1) != *"GNU"* ]]; then
1313
if [[ $(/usr/bin/time --version 2>&1) != *"GNU"* ]]; then

‎fil-proofs-tooling/scripts/micro-remote.sh‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,14 @@ set -e
44

55
CMDS=$(cat <<EOF
66
cd \$(mktemp -d)
7-
git clone -q https://github.com/filecoin-project/rust-fil-proofs.git
7+
git clone https://github.com/filecoin-project/rust-fil-proofs.git
88
cd rust-fil-proofs
99
git checkout -q master
1010
./fil-proofs-tooling/scripts/retry.sh 42 10 60000 \
1111
./fil-proofs-tooling/scripts/with-lock.sh 42 /tmp/benchmark \
12+
./fil-proofs-tooling/scripts/with-dots.sh \
1213
./fil-proofs-tooling/scripts/micro.sh ${@:2}
1314
EOF
1415
)
1516

16-
ssh -t -q $1 "$CMDS"
17+
ssh -q $1 "$CMDS"

‎fil-proofs-tooling/scripts/micro.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ MICRO_SDERR=$(mktemp)
44
MICRO_SDOUT=$(mktemp)
55
JQ_STDERR=$(mktemp)
66

7-
CMD="cargo run --quiet --bin micro --release ${@}"
7+
CMD="cargo run --bin micro --release ${@}"
88

99
eval "RUST_BACKTRACE=1 RUSTFLAGS=\"-Awarnings -C target-cpu=native\" ${CMD}" 1> $MICRO_SDOUT 2> $MICRO_SDERR
1010

‎fil-proofs-tooling/scripts/retry.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,9 @@ for attempt in `seq 1 $attempts`; do
3535

3636
sleep_ms="$(($attempt * $attempt * $sleep_millis))"
3737

38-
echo >&2 "sleeping ${sleep_ms:0:-3}.${sleep_ms: -3}s and then retrying ($((attempt + 1))/${attempts})"
38+
sleep_seconds=$(echo "scale=2; ${sleep_ms}/1000" | bc)
3939

40-
sleep "${sleep_ms:0:-3}.${sleep_ms: -3}"
40+
echo >&2 "sleeping ${sleep_seconds}s and then retrying ($((attempt + 1))/${attempts})"
41+
42+
sleep "${sleep_seconds}"
4143
done

‎fil-proofs-tooling/scripts/with-dots.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env bash
2+
3+
trap cleanup EXIT
4+
5+
cleanup() {
6+
kill $DOT_PID
7+
}
8+
9+
(
10+
sleep 1
11+
while true; do
12+
(printf "." >&2)
13+
sleep 1
14+
done
15+
) &
16+
DOT_PID=$!
17+
18+
$@

‎fil-proofs-tooling/src/bin/benchy/main.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,8 @@ fn main() {
112112
.takes_value(true)
113113
);
114114

115-
let hash_cmd =
116-
SubCommand::with_name("hash-circuits").about("Benchmark hash function inside of a circuit");
115+
let hash_cmd = SubCommand::with_name("hash-constraints")
116+
.about("Benchmark hash function inside of a circuit");
117117

118118
let matches = App::new("benchy")
119119
.version("0.1")
@@ -137,7 +137,7 @@ fn main() {
137137
extract: m.is_present("extract"),
138138
groth: m.is_present("groth"),
139139
hasher: value_t!(m, "hasher", String)?,
140-
layers: value_t!(m, "layers", usize)?,
140+
layers,
141141
m: value_t!(m, "m", usize)?,
142142
no_bench: m.is_present("no-bench"),
143143
no_tmp: m.is_present("no-tmp"),
@@ -149,8 +149,8 @@ fn main() {
149149
})
150150
.expect("zigzag failed");
151151
}
152-
("hash-circuits", Some(_m)) => {
153-
hash_fns::run().expect("hash-circuits failed");
152+
("hash-constraints", Some(_m)) => {
153+
hash_fns::run().expect("hash-constraints failed");
154154
}
155155
_ => panic!("carnation"),
156156
}

‎fil-proofs-tooling/src/bin/benchy/zigzag.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,8 @@ struct Params {
7474
dump_proofs: bool,
7575
bench_only: bool,
7676
hasher: String,
77+
taper: f64,
78+
taper_layers: usize,
7779
}
7880

7981
impl From<Params> for Inputs {
@@ -88,6 +90,9 @@ impl From<Params> for Inputs {
8890
layers: p.layer_challenges.layers(),
8991
partition_challenges: p.layer_challenges.total_challenges(),
9092
total_challenges: p.layer_challenges.total_challenges() * p.partitions,
93+
layer_challenges: p.layer_challenges,
94+
taper: p.taper,
95+
taper_layers: p.taper_layers,
9196
}
9297
}
9398
}
@@ -446,6 +451,9 @@ struct Inputs {
446451
layers: usize,
447452
partition_challenges: usize,
448453
total_challenges: usize,
454+
taper: f64,
455+
taper_layers: usize,
456+
layer_challenges: LayerChallenges,
449457
}
450458

451459
#[derive(Serialize, Default)]
@@ -530,6 +538,8 @@ pub fn run(opts: RunOpts) -> Result<(), failure::Error> {
530538
extract: opts.extract,
531539
hasher: opts.hasher,
532540
samples: 5,
541+
taper: opts.taper,
542+
taper_layers: opts.taper_layers,
533543
};
534544

535545
let report = match params.hasher.as_ref() {

0 commit comments

Comments
 (0)