diff --git a/benchmark/commands.py b/benchmark/commands.py index 6433883..ff76a3c 100644 --- a/benchmark/commands.py +++ b/benchmark/commands.py @@ -28,6 +28,9 @@ def generate_key(filename, mechanism): return f"./node keys --filename {filename}" elif mechanism == "bullshark": return f"./node generate_keys --filename {filename}" + # mysticeti keypairs are generated via the benchmark-genesis command; skip here + else: + return "" @staticmethod def run_node(keys, committee, store, parameters, mechanism, debug=False): @@ -44,6 +47,8 @@ def run_node(keys, committee, store, parameters, mechanism, debug=False): f'~/cometbft/build/cometbft node --home ~/node i --p2p.persistent_peers="{persistent_peers}" ' f"--proxy_app=kvstore --consensus.create_empty_blocks=true" ) + elif mechanism == "mysticeti": + return f"./node {v} run --keys {keys} --committee {committee} " f"--store {store} --parameters {parameters}" @staticmethod def run_primary(keys, committee, store, parameters, debug=False): @@ -88,6 +93,8 @@ def run_client(address, size, rate, mechanism, timeout=None, nodes=[]): elif mechanism == "bullshark": nodes = f'--nodes {" ".join(nodes)}' if nodes else "" return f"./client {address} --size {size} --rate {rate} {nodes}" + elif mechanism == "mysticeti": + return f"./client {address} --size {size} --rate {rate} {nodes}" @staticmethod def kill(): @@ -102,4 +109,7 @@ def alias_binaries(origin, mechanism): node, client = "./cometbft/build/cometbft", "./cometbft/test/loadtime/build/load" elif mechanism == "bullshark": node, client = join(origin, "node"), join(origin, "benchmark_client") + elif mechanism == "mysticeti": + node, client = join(origin, "mysticeti"), join(origin, "client") + return f"rm node ; rm client ; ln -s {node} node ; ln -s {client} client" diff --git a/benchmark/mechanisms/mysticeti.py b/benchmark/mechanisms/mysticeti.py new file mode 100644 index 0000000..36cbd06 --- /dev/null +++ b/benchmark/mechanisms/mysticeti.py @@ -0,0 +1,33 @@ +from benchmark.commands import CommandMaker + + +class MysticetiMechanism: + def __init__(self, settings): + self.settings = settings + self.name = "mysticeti" + + self.install_cmd = [ + "sudo apt-get update", + "sudo apt-get -y upgrade", + "sudo apt-get -y autoremove", + # The following dependencies prevent the error: [error: linker `cc` not found]. + "sudo apt-get -y install build-essential", + "sudo apt-get -y install cmake", + # Install rust (non-interactive). + 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y', + "source $HOME/.cargo/env", + "rustup default stable", + # This is missing from the Rocksdb installer (needed for Rocksdb). + "sudo apt-get install -y clang", + # Clone the repo. + f"(git clone {self.settings.repo_url} || (cd {self.settings.repo_name} ; git pull))", + ] + + self.update_cmd = [ + f"(cd {self.settings.repo_name} && git fetch -f)", + f"(cd {self.settings.repo_name} && git checkout -f {self.settings.branch})", + f"(cd {self.settings.repo_name} && git pull -f)", + "source $HOME/.cargo/env", + f"(cd {self.settings.repo_name} && cargo build --quiet --release)", + CommandMaker.alias_binaries(f"./{self.settings.repo_name}/target/release/", self.name), + ] diff --git a/benchmark/remote.py b/benchmark/remote.py index f20c547..7ae18d7 100644 --- a/benchmark/remote.py +++ b/benchmark/remote.py @@ -32,6 +32,7 @@ ) from benchmark.mechanisms.cometbft import CometBftMechanism from benchmark.mechanisms.hotstuff import HotStuffMechanism +from benchmark.mechanisms.mysticeti import MysticetiMechanism from benchmark.utils import BenchError, PathMaker, Print, progress_bar, set_weight # import pandas as pd @@ -52,7 +53,7 @@ class ExecutionError(Exception): class Bench: def __init__(self, ctx, mechanism): - consensusMechanisms = ["cometbft", "hotstuff", "bullshark"] + consensusMechanisms = ["cometbft", "hotstuff", "bullshark", "mysticeti"] if mechanism not in consensusMechanisms: raise BenchError("Consensus mechanism support not available", e) @@ -65,6 +66,8 @@ def __init__(self, ctx, mechanism): self.mechanism = HotStuffMechanism(self.settings) elif mechanism == "bullshark": self.mechanism = BullsharkMechanism(self.settings) + elif mechanism == "mysticeti": + self.mechanism = MysticetiMechanism(self.settings) try: ctx.connect_kwargs.pkey = RSAKey.from_private_key_file(self.manager.settings.key_path) @@ -186,58 +189,75 @@ def _config(self, isGeoremote, hosts, node_parameters, bench_parameters=None): except Exception as e: Print.error(f"Failed to SCP config files to {host}: {e}") - else: - # Recompile the latest code. - cmd = CommandMaker.compile().split() - subprocess.run(cmd, check=True, cwd=PathMaker.node_crate_path(self.settings.repo_name)) - - # Create alias for the client and nodes binary. - cmd = CommandMaker.alias_binaries(PathMaker.binary_path(self.settings.repo_name), self.mechanism.name) - subprocess.run([cmd], shell=True) - - # Generate configuration files. + elif self.mechanism.name in ("hotstuff", "bullshark"): + # Recompile the latest code and alias binaries + subprocess.run( + CommandMaker.compile().split(), check=True, cwd=PathMaker.node_crate_path(self.settings.repo_name) + ) + subprocess.run( + [CommandMaker.alias_binaries(PathMaker.binary_path(self.settings.repo_name), self.mechanism.name)], + shell=True, + check=True, + ) + + # Generate keys and build committee keys = [] key_files = [PathMaker.key_file(i) for i in range(len(hosts))] for filename in key_files: - cmd = CommandMaker.generate_key(filename, self.mechanism.name).split() - subprocess.run(cmd, check=True) - keys += [Key.from_file(filename)] - - names = [x.name for x in keys] - + subprocess.run(CommandMaker.generate_key(filename, self.mechanism.name).split(), check=True) + keys.append(Key.from_file(filename)) + names = [k.name for k in keys] if self.mechanism.name == "hotstuff": - consensus_addr = [f'{x}:{self.settings.ports["consensus"]}' for x in hosts] - front_addr = [f'{x}:{self.settings.ports["front"]}' for x in hosts] - mempool_addr = [f'{x}:{self.settings.ports["mempool"]}' for x in hosts] + consensus_addr = [f'{h}:{self.settings.ports["consensus"]}' for h in hosts] + front_addr = [f'{h}:{self.settings.ports["front"]}' for h in hosts] + mempool_addr = [f'{h}:{self.settings.ports["mempool"]}' for h in hosts] committee = Committee(names, consensus_addr, front_addr, mempool_addr) - elif self.mechanism.name == "bullshark": + else: + # bullshark if bench_parameters.collocate: workers = bench_parameters.workers - addresses = OrderedDict((x, [y] * (workers + 1)) for x, y in zip(names, hosts)) + addresses = OrderedDict((n, [h] * (workers + 1)) for n, h in zip(names, hosts)) else: - addresses = OrderedDict((x, y) for x, y in zip(names, hosts)) + addresses = OrderedDict((n, h) for n, h in zip(names, hosts)) committee = BullsharkCommittee(addresses, self.settings.ports["base"]) + # Write committee and parameters committee.print(PathMaker.committee_file()) node_parameters.print(PathMaker.parameters_file()) - - # Always update stake weights from geo_input for voting power set_weight(self.mechanism.name, self.settings.geo_input) + # Upload files cmd = f"{CommandMaker.cleanup()} || true" - g = Group(*hosts, user=self.settings.key_name, connect_kwargs=self.connect) - g.run(cmd, hide=True) - - # NOTE Upload configuration files. - progress = progress_bar(hosts, prefix="Uploading config files:") - for i, host in enumerate(progress): - c = Connection(host, user=self.settings.key_name, connect_kwargs=self.connect) + Group(*hosts, user=self.settings.key_name, connect_kwargs=self.connect).run(cmd, hide=True) + for i, h in enumerate(progress_bar(hosts, prefix="Uploading config files:")): + c = Connection(h, user=self.settings.key_name, connect_kwargs=self.connect) c.put(PathMaker.committee_file(), ".") c.put(PathMaker.key_file(i), ".") c.put(PathMaker.parameters_file(), ".") - return committee + # Configuration template for mysticeti + elif self.mechanism.name == "mysticeti": + # Generate genesis & configs with mysticeti CLI + hosts_list = " ".join(hosts) + subprocess.run( + [ + f"./node benchmark-genesis --ips {hosts_list}" + f" --working-directory genesis" + f" --node-parameters-path {PathMaker.parameters_file()}" + ], + shell=True, + check=True, + ) + # Upload generated configs + for i, h in enumerate(progress_bar(hosts, prefix="Uploading mysticeti configs:")): + c = Connection(h, user=self.settings.key_name, connect_kwargs=self.connect) + c.put("genesis/committee.yaml", "committee.yaml") + c.put("genesis/public-config.yaml", "public-config.yaml") + c.put(f"genesis/private-config-{i}.yaml", f"private-config-{i}.yaml") + # No Committee object used + return None + def _run_single(self, hosts, rate, bench_parameters, node_parameters, debug=False, committee=[]): Print.info("Booting testbed...") @@ -366,6 +386,39 @@ def _run_single(self, hosts, rate, bench_parameters, node_parameters, debug=Fals log_file = PathMaker.worker_log_file(i, id) self._background_run(host, cmd, log_file) + elif self.mechanism.name == "mysticeti": + # Boot clients for mysticeti + Print.info("Booting clients... (mysticeti)") + addresses = [f'{x}:{self.settings.ports["base"]}' for x in hosts] + rate_share = ceil(rate / len(hosts)) + client_logs = [PathMaker.client_log_file(i) for i in range(len(hosts))] + for host, addr, log_file in zip(hosts, addresses, client_logs): + cmd = CommandMaker.run_client( + addr, + bench_parameters.tx_size, + rate_share, + self.mechanism.name, + bench_parameters.duration, + nodes=addresses, + ) + self._background_run(host, cmd, log_file) + + # Boot nodes for mysticeti + Print.info("Booting nodes... (mysticeti)") + key_files = [PathMaker.key_file(i) for i in range(len(hosts))] + dbs = [PathMaker.db_path(i) for i in range(len(hosts))] + node_logs = [PathMaker.node_log_file(i) for i in range(len(hosts))] + for host, key_file, db, log_file in zip(hosts, key_files, dbs, node_logs): + cmd = CommandMaker.run_node( + key_file, + PathMaker.committee_file(), + db, + PathMaker.parameters_file(), + debug=debug, + mechanism=self.mechanism.name, + ) + self._background_run(host, cmd, log_file) + # Wait for all transactions to be processed. duration = bench_parameters.duration for _ in progress_bar(range(20), prefix=f"Running benchmark ({duration} sec):"): diff --git a/fab-params.json b/fab-params.json index d72e192..b104cf6 100644 --- a/fab-params.json +++ b/fab-params.json @@ -66,6 +66,29 @@ "batch_size": 120000, "max_batch_delay": 500 } + }, + "mysticeti": { + "bench_params": { + "faults": 0, + "nodes": [4], + "rate": [1000], + "tx_size": 512, + "duration": 20, + "runs": 3 + }, + "node_params": { + "consensus": { + "timeout_delay": 2000, + "sync_retry_delay": 5000 + }, + "mempool": { + "gc_depth": 50, + "sync_retry_delay": 5000, + "sync_retry_nodes": 3, + "batch_size": 100000, + "max_batch_delay": 500 + } + } } } } diff --git a/settings.json b/settings.json index 9e018d5..01c2327 100644 --- a/settings.json +++ b/settings.json @@ -111,6 +111,42 @@ "provider": "google_compute_engine", "ip_file": "/home/ubuntu/geodec/rundata/ip_file.csv" } + }, + "mysticeti": { + "testbed": "mysticeti", + "key": { + "name": "ubuntu", + "path": "/home/ubuntu/.ssh/geodec" + }, + "ports": { + "base": 5000 + }, + "repo": { + "name": "mysticeti", + "url": "https://github.com/GeoDecConsensus/mysticeti", + "branch": "geodec" + }, + "geodec": { + "interface": "ens3", + "geo_input": "/home/ubuntu/geodec/rundata/aptos.csv", + "servers_file": "/home/ubuntu/geodec/frontend/public/servers.csv", + "pings_grouped_file": "/home/ubuntu/geodec/rundata/ping_grouped.csv", + "pings_file": "/home/ubuntu/geodec/rundata/pings.csv" + }, + "instances": { + "type": "m5d.8xlarge", + "regions": [ + "us-east-1", + "eu-north-1", + "ap-southeast-2", + "us-west-1", + "ap-northeast-1" + ] + }, + "configuration": { + "provider": "google_compute_engine", + "ip_file": "/home/ubuntu/geodec/rundata/ip_file.csv" + } } } } \ No newline at end of file