Feat/mysticeti

benchmark/commands.py

@@ -28,6 +28,9 @@ def generate_key(filename, mechanism):
             return f"./node keys --filename {filename}"
         elif mechanism == "bullshark":
             return f"./node generate_keys --filename {filename}"
+        # mysticeti keypairs are generated via the benchmark-genesis command; skip here
+        else:
+            return ""
 
     @staticmethod
     def run_node(keys, committee, store, parameters, mechanism, debug=False):
@@ -44,6 +47,8 @@ def run_node(keys, committee, store, parameters, mechanism, debug=False):
                 f'~/cometbft/build/cometbft node --home ~/node i --p2p.persistent_peers="{persistent_peers}" '
                 f"--proxy_app=kvstore --consensus.create_empty_blocks=true"
             )
+        elif mechanism == "mysticeti":
+            return f"./node {v} run --keys {keys} --committee {committee} " f"--store {store} --parameters {parameters}"
 
     @staticmethod
     def run_primary(keys, committee, store, parameters, debug=False):
@@ -88,6 +93,8 @@ def run_client(address, size, rate, mechanism, timeout=None, nodes=[]):
         elif mechanism == "bullshark":
             nodes = f'--nodes {" ".join(nodes)}' if nodes else ""
             return f"./client {address} --size {size} --rate {rate} {nodes}"
+        elif mechanism == "mysticeti":
+            return f"./client {address} --size {size} --rate {rate} {nodes}"
 
     @staticmethod
     def kill():
@@ -102,4 +109,7 @@ def alias_binaries(origin, mechanism):
             node, client = "./cometbft/build/cometbft", "./cometbft/test/loadtime/build/load"
         elif mechanism == "bullshark":
             node, client = join(origin, "node"), join(origin, "benchmark_client")
+        elif mechanism == "mysticeti":
+            node, client = join(origin, "mysticeti"), join(origin, "client")
+
         return f"rm node ; rm client ; ln -s {node} node ; ln -s {client} client"

benchmark/mechanisms/mysticeti.py

@@ -0,0 +1,33 @@
+from benchmark.commands import CommandMaker
+
+
+class MysticetiMechanism:
+    def __init__(self, settings):
+        self.settings = settings
+        self.name = "mysticeti"
+
+        self.install_cmd = [
+            "sudo apt-get update",
+            "sudo apt-get -y upgrade",
+            "sudo apt-get -y autoremove",
+            # The following dependencies prevent the error: [error: linker `cc` not found].
+            "sudo apt-get -y install build-essential",
+            "sudo apt-get -y install cmake",
+            # Install rust (non-interactive).
+            'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y',
+            "source $HOME/.cargo/env",
+            "rustup default stable",
+            # This is missing from the Rocksdb installer (needed for Rocksdb).
+            "sudo apt-get install -y clang",
+            # Clone the repo.
+            f"(git clone {self.settings.repo_url} || (cd {self.settings.repo_name} ; git pull))",
+        ]
+
+        self.update_cmd = [
+            f"(cd {self.settings.repo_name} && git fetch -f)",
+            f"(cd {self.settings.repo_name} && git checkout -f {self.settings.branch})",
+            f"(cd {self.settings.repo_name} && git pull -f)",
+            "source $HOME/.cargo/env",
+            f"(cd {self.settings.repo_name} && cargo build --quiet --release)",
+            CommandMaker.alias_binaries(f"./{self.settings.repo_name}/target/release/", self.name),
+        ]

benchmark/remote.py

@@ -32,6 +32,7 @@
 )
 from benchmark.mechanisms.cometbft import CometBftMechanism
 from benchmark.mechanisms.hotstuff import HotStuffMechanism
+from benchmark.mechanisms.mysticeti import MysticetiMechanism
 from benchmark.utils import BenchError, PathMaker, Print, progress_bar, set_weight
 
 # import pandas as pd
@@ -52,7 +53,7 @@ class ExecutionError(Exception):
 
 class Bench:
     def __init__(self, ctx, mechanism):
-        consensusMechanisms = ["cometbft", "hotstuff", "bullshark"]
+        consensusMechanisms = ["cometbft", "hotstuff", "bullshark", "mysticeti"]
         if mechanism not in consensusMechanisms:
             raise BenchError("Consensus mechanism support not available", e)
 
@@ -65,6 +66,8 @@ def __init__(self, ctx, mechanism):
             self.mechanism = HotStuffMechanism(self.settings)
         elif mechanism == "bullshark":
             self.mechanism = BullsharkMechanism(self.settings)
+        elif mechanism == "mysticeti":
+            self.mechanism = MysticetiMechanism(self.settings)
 
         try:
             ctx.connect_kwargs.pkey = RSAKey.from_private_key_file(self.manager.settings.key_path)
@@ -186,58 +189,75 @@ def _config(self, isGeoremote, hosts, node_parameters, bench_parameters=None):
                 except Exception as e:
                     Print.error(f"Failed to SCP config files to {host}: {e}")
 
-        else:
-            # Recompile the latest code.
-            cmd = CommandMaker.compile().split()
-            subprocess.run(cmd, check=True, cwd=PathMaker.node_crate_path(self.settings.repo_name))
-
-            # Create alias for the client and nodes binary.
-            cmd = CommandMaker.alias_binaries(PathMaker.binary_path(self.settings.repo_name), self.mechanism.name)
-            subprocess.run([cmd], shell=True)
-
-            # Generate configuration files.
+        elif self.mechanism.name in ("hotstuff", "bullshark"):
+            # Recompile the latest code and alias binaries
+            subprocess.run(
+                CommandMaker.compile().split(), check=True, cwd=PathMaker.node_crate_path(self.settings.repo_name)
+            )
+            subprocess.run(
+                [CommandMaker.alias_binaries(PathMaker.binary_path(self.settings.repo_name), self.mechanism.name)],
+                shell=True,
+                check=True,
+            )
+
+            # Generate keys and build committee
             keys = []
             key_files = [PathMaker.key_file(i) for i in range(len(hosts))]
             for filename in key_files:
-                cmd = CommandMaker.generate_key(filename, self.mechanism.name).split()
-                subprocess.run(cmd, check=True)
-                keys += [Key.from_file(filename)]
-
-            names = [x.name for x in keys]
-
+                subprocess.run(CommandMaker.generate_key(filename, self.mechanism.name).split(), check=True)
+                keys.append(Key.from_file(filename))
+            names = [k.name for k in keys]
             if self.mechanism.name == "hotstuff":
-                consensus_addr = [f'{x}:{self.settings.ports["consensus"]}' for x in hosts]
-                front_addr = [f'{x}:{self.settings.ports["front"]}' for x in hosts]
-                mempool_addr = [f'{x}:{self.settings.ports["mempool"]}' for x in hosts]
+                consensus_addr = [f'{h}:{self.settings.ports["consensus"]}' for h in hosts]
+                front_addr = [f'{h}:{self.settings.ports["front"]}' for h in hosts]
+                mempool_addr = [f'{h}:{self.settings.ports["mempool"]}' for h in hosts]
                 committee = Committee(names, consensus_addr, front_addr, mempool_addr)
-            elif self.mechanism.name == "bullshark":
+            else:
+                # bullshark
                 if bench_parameters.collocate:
                     workers = bench_parameters.workers
-                    addresses = OrderedDict((x, [y] * (workers + 1)) for x, y in zip(names, hosts))
+                    addresses = OrderedDict((n, [h] * (workers + 1)) for n, h in zip(names, hosts))
                 else:
-                    addresses = OrderedDict((x, y) for x, y in zip(names, hosts))
+                    addresses = OrderedDict((n, h) for n, h in zip(names, hosts))
                 committee = BullsharkCommittee(addresses, self.settings.ports["base"])
 
+            # Write committee and parameters
             committee.print(PathMaker.committee_file())
             node_parameters.print(PathMaker.parameters_file())
-
-            # Always update stake weights from geo_input for voting power
             set_weight(self.mechanism.name, self.settings.geo_input)
 
+            # Upload files
             cmd = f"{CommandMaker.cleanup()} || true"
-            g = Group(*hosts, user=self.settings.key_name, connect_kwargs=self.connect)
-            g.run(cmd, hide=True)
-
-            # NOTE Upload configuration files.
-            progress = progress_bar(hosts, prefix="Uploading config files:")
-            for i, host in enumerate(progress):
-                c = Connection(host, user=self.settings.key_name, connect_kwargs=self.connect)
+            Group(*hosts, user=self.settings.key_name, connect_kwargs=self.connect).run(cmd, hide=True)
+            for i, h in enumerate(progress_bar(hosts, prefix="Uploading config files:")):
+                c = Connection(h, user=self.settings.key_name, connect_kwargs=self.connect)
                 c.put(PathMaker.committee_file(), ".")
                 c.put(PathMaker.key_file(i), ".")
                 c.put(PathMaker.parameters_file(), ".")
-
             return committee
 
+        # Configuration template for mysticeti
+        elif self.mechanism.name == "mysticeti":
+            # Generate genesis & configs with mysticeti CLI
+            hosts_list = " ".join(hosts)
+            subprocess.run(
+                [
+                    f"./node benchmark-genesis --ips {hosts_list}"
+                    f" --working-directory genesis"
+                    f" --node-parameters-path {PathMaker.parameters_file()}"
+                ],
+                shell=True,
+                check=True,
+            )
+            # Upload generated configs
+            for i, h in enumerate(progress_bar(hosts, prefix="Uploading mysticeti configs:")):
+                c = Connection(h, user=self.settings.key_name, connect_kwargs=self.connect)
+                c.put("genesis/committee.yaml", "committee.yaml")
+                c.put("genesis/public-config.yaml", "public-config.yaml")
+                c.put(f"genesis/private-config-{i}.yaml", f"private-config-{i}.yaml")
+            # No Committee object used
+            return None
+
     def _run_single(self, hosts, rate, bench_parameters, node_parameters, debug=False, committee=[]):
         Print.info("Booting testbed...")
 
@@ -366,6 +386,39 @@ def _run_single(self, hosts, rate, bench_parameters, node_parameters, debug=Fals
                     log_file = PathMaker.worker_log_file(i, id)
                     self._background_run(host, cmd, log_file)
 
+        elif self.mechanism.name == "mysticeti":
+            # Boot clients for mysticeti
+            Print.info("Booting clients... (mysticeti)")
+            addresses = [f'{x}:{self.settings.ports["base"]}' for x in hosts]
+            rate_share = ceil(rate / len(hosts))
+            client_logs = [PathMaker.client_log_file(i) for i in range(len(hosts))]
+            for host, addr, log_file in zip(hosts, addresses, client_logs):
+                cmd = CommandMaker.run_client(
+                    addr,
+                    bench_parameters.tx_size,
+                    rate_share,
+                    self.mechanism.name,
+                    bench_parameters.duration,
+                    nodes=addresses,
+                )
+                self._background_run(host, cmd, log_file)
+
+            # Boot nodes for mysticeti
+            Print.info("Booting nodes... (mysticeti)")
+            key_files = [PathMaker.key_file(i) for i in range(len(hosts))]
+            dbs = [PathMaker.db_path(i) for i in range(len(hosts))]
+            node_logs = [PathMaker.node_log_file(i) for i in range(len(hosts))]
+            for host, key_file, db, log_file in zip(hosts, key_files, dbs, node_logs):
+                cmd = CommandMaker.run_node(
+                    key_file,
+                    PathMaker.committee_file(),
+                    db,
+                    PathMaker.parameters_file(),
+                    debug=debug,
+                    mechanism=self.mechanism.name,
+                )
+                self._background_run(host, cmd, log_file)
+
         # Wait for all transactions to be processed.
         duration = bench_parameters.duration
         for _ in progress_bar(range(20), prefix=f"Running benchmark ({duration} sec):"):

fab-params.json

@@ -66,6 +66,29 @@
                 "batch_size": 120000,
                 "max_batch_delay": 500
             }
+        },
+        "mysticeti": {
+            "bench_params": {
+                "faults": 0,
+                "nodes": [4],
+                "rate": [1000],
+                "tx_size": 512,
+                "duration": 20,
+                "runs": 3
+            },
+            "node_params": {
+                "consensus": {
+                    "timeout_delay": 2000,
+                    "sync_retry_delay": 5000
+                },
+                "mempool": {
+                    "gc_depth": 50,
+                    "sync_retry_delay": 5000,
+                    "sync_retry_nodes": 3,
+                    "batch_size": 100000,
+                    "max_batch_delay": 500
+                }
+            }
         }
     }
 }

settings.json

@@ -111,6 +111,42 @@
                 "provider": "google_compute_engine",
                 "ip_file": "/home/ubuntu/geodec/rundata/ip_file.csv"
             }
+        },
+        "mysticeti": {
+            "testbed": "mysticeti",
+            "key": {
+                "name": "ubuntu",
+                "path": "/home/ubuntu/.ssh/geodec"
+            },
+            "ports": {
+                "base": 5000
+            },
+            "repo": {
+                "name": "mysticeti",
+                "url": "https://github.com/GeoDecConsensus/mysticeti",
+                "branch": "geodec"
+            },
+            "geodec": {
+                "interface": "ens3",
+                "geo_input": "/home/ubuntu/geodec/rundata/aptos.csv",
+                "servers_file": "/home/ubuntu/geodec/frontend/public/servers.csv",
+                "pings_grouped_file": "/home/ubuntu/geodec/rundata/ping_grouped.csv",
+                "pings_file": "/home/ubuntu/geodec/rundata/pings.csv"
+            },
+            "instances": {
+                "type": "m5d.8xlarge",
+                "regions": [
+                    "us-east-1",
+                    "eu-north-1",
+                    "ap-southeast-2",
+                    "us-west-1",
+                    "ap-northeast-1"
+                ]
+            },
+            "configuration": {
+                "provider": "google_compute_engine",
+                "ip_file": "/home/ubuntu/geodec/rundata/ip_file.csv"
+            }
         }
     }
 }