Skip to content

Commit 534d37e

Browse files
schomatisschomatis
authored
Merge branch 'master' into hotfix/decrease-post-sectors-count
2 parents 0f1fcb9 + b40d12a commit 534d37e

File tree

16 files changed

+130
-182
lines changed

16 files changed

+130
-182
lines changed

‎.circleci/config.yml‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -195,16 +195,35 @@ jobs:
195195
command: apt-get install time jq -yqq
196196
- run:
197197
name: Run hash-constraints benchmarks on remote host
198-
command: ./fil-proofs-tooling/scripts/benchy-remote.sh "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}" hash-circuits
198+
command: |
199+
./fil-proofs-tooling/scripts/benchy-remote.sh "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}" hash-constraints > hash-constraints.json
200+
cat hash-constraints.json
199201
no_output_timeout: 60m
200202
- run:
201203
name: Run micro benchmarks
202-
command: ./fil-proofs-tooling/scripts/micro-remote.sh "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}"
204+
command: |
205+
./fil-proofs-tooling/scripts/micro-remote.sh "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}" > micro-benchmarks.json
206+
cat micro-benchmarks.json
203207
no_output_timeout: 60m
204208
- run:
205209
name: Run ZigZag benchmarks using 1GiB sectors
206-
command: ./fil-proofs-tooling/scripts/benchy-remote.sh "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}" zigzag --size=$((1024*1024))
210+
command: |
211+
./fil-proofs-tooling/scripts/benchy-remote.sh "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}" zigzag --size=$((1024*1024)) > zigzag-benchmarks.json
212+
cat zigzag-benchmarks.json
207213
no_output_timeout: 60m
214+
- run:
215+
name: Aggregate benchmarks into single JSON document
216+
command: |
217+
./fil-proofs-tooling/scripts/aggregate-benchmarks.sh zigzag-benchmarks.json micro-benchmarks.json hash-constraints.json > aggregated-benchmarks.json
218+
cat aggregated-benchmarks.json
219+
- store_artifacts:
220+
path: zigzag-benchmarks.json
221+
- store_artifacts:
222+
path: hash-constraints.json
223+
- store_artifacts:
224+
path: micro-benchmarks.json
225+
- store_artifacts:
226+
path: aggregated-benchmarks.json
208227

209228
rustfmt:
210229
docker:

‎README.md‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ To check that it's working you can inspect the replication log to find `using pa
161161

162162
### Memory
163163

164-
We try to generate the MTs in parallel to speed up the process but that takes 2 sector sizes per each layer (e.g., a 1 GiB sector may require, in the worst case scenario, up to 20 GiB of memory to hold the MTs alone). To reduce that (at the cost of speed) we have the (experimental) `disk-trees` feature to offload the MTs to disk when we don't use them. For example, to run the `zigzag` example with this feature you'd need to indicate so to `cargo` *and* then indicate to the example (or any other code doing the replication) where they should be stored using the `FIL_PROOFS_REPLICATED_TREES_DIR` environmental variable (if set to a relative path, it will be relative to the current working directory of the process in which the replication happens, see [`create_dir`](https://doc.rust-lang.org/std/fs/fn.create_dir.html#platform-specific-behavior)),
164+
We try to generate the MTs in parallel to speed up the process but that takes 2 sector sizes per each layer (e.g., a 1 GiB sector may require, in the worst case scenario, up to 20 GiB of memory to hold the MTs alone). To reduce that (at the cost of speed) we have the (experimental) `disk-trees` feature to offload the MTs to disk when we don't use them. For example, to run the `zigzag` example with this feature you'd need to indicate so to `cargo`,
165165

166166
```
167167
# From inside the `storage-proofs` directory, where this feature
@@ -172,15 +172,10 @@ cargo build \
172172
--example zigzag \
173173
--features \
174174
disk-trees &&
175-
FIL_PROOFS_REPLICATED_TREES_DIR="<tree-dir>" \
176175
../target/release/examples/zigzag \
177176
--size 1048576
178177
```
179178

180-
To check if the feature is indeed working you can inspect the replication log to find the text `creating leaves tree mmap-file` indicating the path where each MT file is saved.
181-
182-
Note that at the moment we do *not* clean up `<tree-dir>` so you'll need to **remove old MT files** yourself (otherwise the disk will start to fill up pretty fast). To avoid collisions in the directory we append a random hexadecimal value to each file, to check which files belong to the latest replication run you can (besides checking modification times) inspect the replication log to find out the actual file names generated.
183-
184179
To optimize even more for memory there's another option (used in addition to the `disk-trees` feature) to generate all MTs in sequential order, to make sure we can offload them to disk before we start buildding the next one,
185180

186181
```

‎fil-proofs-tooling/scripts/aggregate-benchmarks.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
zigzag_path=$1
6+
micro_path=$2
7+
hash_constraints_path=$3
8+
9+
jq --sort-keys -s '{ benchmarks: { "zigzag-benchmarks": { outputs: { "max-resident-set-size-kb": .[0] } } } } * .[1]' \
10+
<(jq '.["max-resident-set-size-kb"]' $zigzag_path) \
11+
<(jq -s '.[0] * { benchmarks: { "hash-constraints": .[1], "zigzag-benchmarks": .[2], "micro-benchmarks": .[3] } }' \
12+
<(jq 'del (.benchmarks)' $micro_path) \
13+
<(jq '.benchmarks' $hash_constraints_path) \
14+
<(jq '.benchmarks' $zigzag_path) \
15+
<(jq '.benchmarks' $micro_path))

‎fil-proofs-tooling/scripts/benchy-remote.sh‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,14 @@ set -e
44

55
CMDS=$(cat <<EOF
66
cd \$(mktemp -d)
7-
git clone -q https://github.com/filecoin-project/rust-fil-proofs.git
7+
git clone https://github.com/filecoin-project/rust-fil-proofs.git
88
cd rust-fil-proofs
99
git checkout -q master
1010
./fil-proofs-tooling/scripts/retry.sh 42 10 60000 \
1111
./fil-proofs-tooling/scripts/with-lock.sh 42 /tmp/benchmark \
12+
./fil-proofs-tooling/scripts/with-dots.sh \
1213
./fil-proofs-tooling/scripts/benchy.sh ${@:2}
1314
EOF
1415
)
1516

16-
ssh -t -q $1 "$CMDS"
17+
ssh -q $1 "$CMDS"

‎fil-proofs-tooling/scripts/benchy.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ GTIME_STDERR=$(mktemp)
77
JQ_STDERR=$(mktemp)
88

99
GTIME_BIN="env time"
10-
GTIME_ARG="-f '{ \"outputs\": { \"max-resident-set-size-kb\": %M } }' cargo run --quiet --bin benchy --release -- ${@}"
10+
GTIME_ARG="-f '{ \"max-resident-set-size-kb\": %M }' cargo run --quiet --bin benchy --release -- ${@}"
1111

1212
if [[ $(env time --version 2>&1) != *"GNU"* ]]; then
1313
if [[ $(/usr/bin/time --version 2>&1) != *"GNU"* ]]; then

‎fil-proofs-tooling/scripts/micro-remote.sh‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,14 @@ set -e
44

55
CMDS=$(cat <<EOF
66
cd \$(mktemp -d)
7-
git clone -q https://github.com/filecoin-project/rust-fil-proofs.git
7+
git clone https://github.com/filecoin-project/rust-fil-proofs.git
88
cd rust-fil-proofs
99
git checkout -q master
1010
./fil-proofs-tooling/scripts/retry.sh 42 10 60000 \
1111
./fil-proofs-tooling/scripts/with-lock.sh 42 /tmp/benchmark \
12+
./fil-proofs-tooling/scripts/with-dots.sh \
1213
./fil-proofs-tooling/scripts/micro.sh ${@:2}
1314
EOF
1415
)
1516

16-
ssh -t -q $1 "$CMDS"
17+
ssh -q $1 "$CMDS"

‎fil-proofs-tooling/scripts/micro.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ MICRO_SDERR=$(mktemp)
44
MICRO_SDOUT=$(mktemp)
55
JQ_STDERR=$(mktemp)
66

7-
CMD="cargo run --quiet --bin micro --release ${@}"
7+
CMD="cargo run --bin micro --release ${@}"
88

99
eval "RUST_BACKTRACE=1 RUSTFLAGS=\"-Awarnings -C target-cpu=native\" ${CMD}" 1> $MICRO_SDOUT 2> $MICRO_SDERR
1010

‎fil-proofs-tooling/scripts/retry.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,9 @@ for attempt in `seq 1 $attempts`; do
3535

3636
sleep_ms="$(($attempt * $attempt * $sleep_millis))"
3737

38-
echo >&2 "sleeping ${sleep_ms:0:-3}.${sleep_ms: -3}s and then retrying ($((attempt + 1))/${attempts})"
38+
sleep_seconds=$(echo "scale=2; ${sleep_ms}/1000" | bc)
3939

40-
sleep "${sleep_ms:0:-3}.${sleep_ms: -3}"
40+
echo >&2 "sleeping ${sleep_seconds}s and then retrying ($((attempt + 1))/${attempts})"
41+
42+
sleep "${sleep_seconds}"
4143
done

‎fil-proofs-tooling/scripts/with-dots.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env bash
2+
3+
trap cleanup EXIT
4+
5+
cleanup() {
6+
kill $DOT_PID
7+
}
8+
9+
(
10+
sleep 1
11+
while true; do
12+
(printf "." >&2)
13+
sleep 1
14+
done
15+
) &
16+
DOT_PID=$!
17+
18+
$@

‎fil-proofs-tooling/src/bin/benchy/main.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,8 @@ fn main() {
112112
.takes_value(true)
113113
);
114114

115-
let hash_cmd =
116-
SubCommand::with_name("hash-circuits").about("Benchmark hash function inside of a circuit");
115+
let hash_cmd = SubCommand::with_name("hash-constraints")
116+
.about("Benchmark hash function inside of a circuit");
117117

118118
let matches = App::new("benchy")
119119
.version("0.1")
@@ -137,7 +137,7 @@ fn main() {
137137
extract: m.is_present("extract"),
138138
groth: m.is_present("groth"),
139139
hasher: value_t!(m, "hasher", String)?,
140-
layers: value_t!(m, "layers", usize)?,
140+
layers,
141141
m: value_t!(m, "m", usize)?,
142142
no_bench: m.is_present("no-bench"),
143143
no_tmp: m.is_present("no-tmp"),
@@ -149,8 +149,8 @@ fn main() {
149149
})
150150
.expect("zigzag failed");
151151
}
152-
("hash-circuits", Some(_m)) => {
153-
hash_fns::run().expect("hash-circuits failed");
152+
("hash-constraints", Some(_m)) => {
153+
hash_fns::run().expect("hash-constraints failed");
154154
}
155155
_ => panic!("carnation"),
156156
}

0 commit comments

Comments
 (0)