More BoB-L1 refactors, prep for BoB-L2

Makefile

@@ -39,23 +39,30 @@ setup: ## Install dependencies (Linux only)
 
 # Build module
 build: check-perms setup ## Build the specified module
-	@$(WRAPPER) mkosi --force -I $(IMAGE).conf
+	$(WRAPPER) mkosi --force -I $(IMAGE).conf
 
 # Build module with devtools profile
 build-dev: check-perms setup ## Build module with development tools
-	@$(WRAPPER) mkosi --force --profile=devtools -I $(IMAGE).conf
+	$(WRAPPER) mkosi --force --profile=devtools -I $(IMAGE).conf
 
 ##@ Utilities
 
-# Run measured-boot on the EFI file
-measure: ## Export TDX measurements for the built image
+measure: ## Export TDX measurements for the built EFI file
 	@if [ ! -f build/tdx-debian.efi ]; then \
 		echo "Error: build/tdx-debian.efi not found. Run 'make build' first."; \
 		exit 1; \
 	fi
 	@$(WRAPPER) measured-boot build/tdx-debian.efi build/measurements.json --direct-uki
 	echo "Measurements exported to build/measurements.json"
 
+measure-gcp: ## Export TDX measurements for GCP
+	@if [ ! -f build/tdx-debian.efi ]; then \
+		echo "Error: build/tdx-debian.efi not found. Run 'make build' first."; \
+		exit 1; \
+	fi
+	@$(WRAPPER) dstack-mr -uki build/tdx-debian.efi -json > build/gcp_measurements.json
+	echo "GCP Measurements exported to build/gcp_measurements.json"
+
 # Clean build artifacts
 clean: ## Remove cache and build artifacts
 	rm -rf build/ mkosi.builddir/ mkosi.cache/ lima-nix/

base/debloat.sh

@@ -23,7 +23,6 @@ debloat_paths=(
     "/usr/share/bug"
     "/usr/share/menu"
     "/usr/share/systemd"
-    "/usr/share/bash-completion"
     "/usr/share/zsh"
     "/usr/share/mime"
     "/usr/lib/modules"
@@ -40,4 +39,13 @@ debloat_paths=(
     "/nix"
 )
 
-for p in "${debloat_paths[@]}"; do rm -rf $BUILDROOT$p; done
+if [[ ! "$PROFILES" == *"devtools"* ]]; then
+    debloat_paths+=(
+        "/usr/share/bash-completion"
+    )
+fi
+
+for p in "${debloat_paths[@]}"; do
+    echo "Debloating $p"
+    rm -rf $BUILDROOT$p
+done

base/mkosi.conf

@@ -42,7 +42,6 @@ BuildPackages=build-essential
               cmake
               pkg-config
               clang
-              cargo
               flex
               bison
               elfutils

bob-common/mkosi.conf

@@ -15,7 +15,6 @@ Packages=podman
          iproute2
          conntrack
          netfilter-persistent
-         openntpd
          curl
          jq
          logrotate

bob-common/mkosi.extra/etc/systemd/system/searcher-firewall.service

@@ -1,7 +1,7 @@
 [Unit]
 Description=Searcher Network and Firewall Rules
-After=azure-complete-provisioning.service
-Requires=azure-complete-provisioning.service
+After=network.target network-setup.service
+Requires=network-setup.service
 
 [Service]
 Type=oneshot

bob-common/mkosi.extra/usr/bin/init-container.sh

@@ -5,25 +5,30 @@ NAME=searcher-container
 
 # PORT FORWARDS
 SEARCHER_SSH_PORT=10022
-ENGINE_API_PORT=8551
 EL_P2P_PORT=30303
 SEARCHER_INPUT_CHANNEL=27017
 
+# Run extra commands which are customized per image,
+# see bob*/mkosi.extra/etc/bob/searcher-container-before-init
+#
+# `source` is not supported in dash
+. /etc/bob/searcher-container-before-init
+
+# BOB_SEARCHER_EXTRA_PODMAN_FLAGS is unescaped, it's sourced from trusted hardcoded file
+
 echo "Starting $NAME..."
 su -s /bin/sh searcher -c "cd ~ && podman run -d \
     --name $NAME --replace \
     --init \
     -p ${SEARCHER_SSH_PORT}:22 \
-    -p ${ENGINE_API_PORT}:${ENGINE_API_PORT} \
     -p ${EL_P2P_PORT}:${EL_P2P_PORT} \
     -p ${EL_P2P_PORT}:${EL_P2P_PORT}/udp \
     -p ${SEARCHER_INPUT_CHANNEL}:${SEARCHER_INPUT_CHANNEL}/udp \
     -v /persistent/searcher:/persistent:rw \
     -v /etc/searcher/ssh_hostkey:/etc/searcher/ssh_hostkey:rw \
     -v /persistent/searcher_logs:/var/log/searcher:rw \
-    -v /persistent/cl_logs:/var/log/cl:ro \
-    -v /tmp/jwt.hex:/secrets/jwt.hex:ro \
     -v /etc/searcher-logrotate.conf:/tmp/searcher.conf:ro \
+    $BOB_SEARCHER_EXTRA_PODMAN_FLAGS \
     docker.io/library/ubuntu:24.04 \
     /bin/sh -c ' \
         DEBIAN_FRONTEND=noninteractive apt-get update && \
@@ -41,7 +46,7 @@ su -s /bin/sh searcher -c "cd ~ && podman run -d \
         while true; do /usr/sbin/sshd -D -e; sleep 5; done'"
 
 # Attempt a quick check that the container is running
-for i in 1 2 3 4 5; do
+for i in $(seq 1 5); do
     status=$(su -s /bin/sh - searcher -c "podman inspect --format '{{.State.Status}}' $NAME 2>/dev/null || true")
     if [ "$status" = "running" ]; then
         break
@@ -63,24 +68,25 @@ if [ -z "$pid" ] || [ "$pid" = "0" ]; then
 fi
 
 echo "Applying iptables rules in $NAME (PID: $pid) network namespace..."
+ns_iptables() {
+    nsenter --target "$pid" --net iptables "$@"
+}
+
+ns_iptables -A OUTPUT -d 169.254.169.254 -j DROP
 
-# Enter network namespace and apply DROP rules on port 9000 TCP/UDP
-nsenter --target "$pid" --net iptables -A OUTPUT -p tcp --dport 9000 -j DROP
-nsenter --target "$pid" --net iptables -A OUTPUT -p udp --dport 9000 -j DROP
+ns_iptables -A OUTPUT -p tcp --dport 9000 -j DROP
+ns_iptables -A OUTPUT -p udp --dport 9000 -j DROP
 
-# Enter network namespace and apply DROP rule on port 123 UDP
-nsenter --target "$pid" --net iptables -A OUTPUT -p udp --dport 123 -j DROP
+ns_iptables -A OUTPUT -p udp --dport 123 -j DROP
 
-# Drop outbound traffic from SEARCHER_INPUT_CHANNEL
-nsenter --target "$pid" --net iptables -A OUTPUT -p udp --sport $SEARCHER_INPUT_CHANNEL -j DROP
-nsenter --target "$pid" --net iptables -A OUTPUT -p tcp --sport $SEARCHER_INPUT_CHANNEL -j DROP
+ns_iptables -A OUTPUT -p udp --sport $SEARCHER_INPUT_CHANNEL -j DROP
+ns_iptables -A OUTPUT -p tcp --sport $SEARCHER_INPUT_CHANNEL -j DROP
 
-echo "Injecting static hosts into $NAME..."
+# Helper, only used in sourced script below
+exec_in_container() {
+    su -s /bin/sh searcher -c "podman exec $NAME /bin/sh -c '$1'"
+}
 
-su -s /bin/sh searcher -c "podman exec $NAME /bin/sh -c '
-    echo \"3.149.14.12 tx.tee-searcher.flashbots.net\" >> /etc/hosts &&
-    echo \"3.136.107.142 tx.tee-searcher.flashbots.net\" >> /etc/hosts &&
-    echo \"18.221.59.61 backruns.tee-searcher.flashbots.net\" >> /etc/hosts &&
-    echo \"3.15.88.156 backruns.tee-searcher.flashbots.net\" >> /etc/hosts &&
-    echo \"52.207.17.217 fbtee.titanbuilder.xyz\" >> /etc/hosts
-'"
+# Run extra commands which are customized per image,
+# see bob*/mkosi.extra/etc/bob/searcher-container-after-init
+. /etc/bob/searcher-container-after-init

bob-common/mkosi.extra/usr/bin/init-firewall.sh

@@ -20,7 +20,7 @@ set -eu -o pipefail
 #    ├(loopback?)─> ACCEPT
 #    └─> default DROP
 #
-# - There are no ports opened in this file, refer to bob*/mkosi.extra/etc/firewall-config
+# - There are no ports opened in this file, refer to bob*/mkosi.extra/etc/bob/firewall-config
 #   for actual chain rules.
 # - Mode-specific ESTABLISHED/RELATED connections are killed by
 #   `conntrack -D ...` upon mode toggle.
@@ -98,7 +98,7 @@ iptables -A OUTPUT ! -o lo -d 127.0.0.0/8 -j DROP
 
 ###########################################################################
 #
-# Some helper functions to reduce boilerplate in /etc/firewall-config
+# Some helper functions to reduce boilerplate in /etc/bob/firewall-config
 #
 ###########################################################################
 accept_dst_port() {
@@ -124,14 +124,22 @@ accept_dst_ip_port() {
                          -m comment --comment "$comment"
 }
 
+drop_dst_ip() {
+    chain="$1"
+    ip="$2"
+    comment="$3"
+
+    iptables -A "$chain" -d "$ip" -j DROP \
+                         -m comment --comment "$comment"
+}
 
 ###########################################################################
 # (5) Load firewall rules in {MAINTENANCE,PRODUCTION}_{IN,OUT} chains.
-# Those are customized per image, see bob*/mkosi.extra/etc/firewall-config
+# Those are customized per image, see bob*/mkosi.extra/etc/bob/firewall-config
 #
 # `source` is not supported in dash
 ###########################################################################
-. /etc/firewall-config
+. /etc/bob/firewall-config
 
 ###########################################################################
 # (6) Start in Maintenance Mode

bob-common/mkosi.postinst

@@ -22,7 +22,6 @@ mkdir "$BUILDROOT/etc/dropbear"
 mkdir -p "$BUILDROOT/etc/systemd/system/minimal.target.wants"
 for service in \
     network-setup.service \
-    openntpd.service \
     logrotate.service \
     fluent-bit.service \
     wait-for-key.service \

bob-l1/mkosi.conf

@@ -1,15 +1,18 @@
 [Build]
-Environment=LIGHTHOUSE_BINARY KERNEL_CONFIG_SNIPPETS=bob-l1/kernel.config
+Environment=LIGHTHOUSE_BINARY KERNEL_CONFIG_SNIPPETS=bob-l1/kernel.config KERNEL_VERSION=6.13.12
 WithNetwork=true
 
 [Content]
 ExtraTrees=bob-l1/mkosi.extra
 PostInstallationScripts=bob-l1/mkosi.postinst
 BuildScripts=bob-l1/mkosi.build
 
+Packages=openntpd
+
 BuildPackages=build-essential
               git
               gcc
+              cargo
               zlib1g-dev
               libzstd-dev
               libleveldb-dev

bob-l1/mkosi.extra/etc/firewall-config → bob-l1/mkosi.extra/etc/bob/firewall-config

@@ -84,9 +84,7 @@ accept_dst_port $CHAIN_MAINTENANCE_IN udp $EL_P2P_PORT "EL P2P (UDP)"
 ###########################################################################
 
 # Block Flashbots protect tx endpoints during maintenance
-iptables -A $CHAIN_MAINTENANCE_OUT \
-    -d $FLASHBOTS_TX_STREAM_1,$FLASHBOTS_TX_STREAM_2 -j DROP \
-    -m comment --comment "Flashbots Protect (DROP before accept-all 443)"
+drop_dst_ip $CHAIN_MAINTENANCE_OUT $FLASHBOTS_TX_STREAM_1,$FLASHBOTS_TX_STREAM_2 "Flashbots Protect (DROP before accept-all rules)"
 
 accept_dst_port $CHAIN_MAINTENANCE_OUT udp $DNS_PORT "DNS (UDP)"
 accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $DNS_PORT "DNS (TCP)"

bob-l1/mkosi.extra/etc/bob/searcher-container-after-init

@@ -0,0 +1,12 @@
+# This script is sourced from init-container.sh and contains image-specific stuff
+# See also: bob-common/mkosi.extra/usr/bin/init-container.sh
+
+echo "Injecting static hosts into searcher container..."
+exec_in_container '
+    cat <<EOF >> /etc/hosts
+3.149.14.12   tx.tee-searcher.flashbots.net
+3.136.107.142 tx.tee-searcher.flashbots.net
+18.221.59.61  backruns.tee-searcher.flashbots.net
+3.15.88.156   backruns.tee-searcher.flashbots.net
+52.207.17.217 fbtee.titanbuilder.xyz
+EOF'

bob-l1/mkosi.extra/etc/bob/searcher-container-before-init

@@ -0,0 +1,10 @@
+# This script is sourced from init-container.sh and contains image-specific stuff
+# See also: bob-common/mkosi.extra/usr/bin/init-container.sh
+
+ENGINE_API_PORT=8551
+
+BOB_SEARCHER_EXTRA_PODMAN_FLAGS="\
+    -p ${ENGINE_API_PORT}:${ENGINE_API_PORT} \
+    -v /persistent/lighthouse_logs:/var/log/lighthouse:ro \
+    -v /tmp/jwt.hex:/secrets/jwt.hex:ro \
+"

bob-l1/mkosi.extra/etc/systemd/system/lighthouse.service

@@ -8,9 +8,9 @@ Type=exec
 User=lighthouse
 Group=eth
 ExecStartPre=+/usr/bin/lighthouse-init
-ExecStartPre=+/bin/mkdir -p /persistent/cl_logs
-ExecStartPre=+/bin/chown lighthouse:eth /persistent/cl_logs
-ExecStartPre=+/bin/chmod 755 /persistent/cl_logs
+ExecStartPre=+/bin/mkdir -p /persistent/lighthouse_logs
+ExecStartPre=+/bin/chown lighthouse:eth /persistent/lighthouse_logs
+ExecStartPre=+/bin/chmod 755 /persistent/lighthouse_logs
 ExecStart=/usr/bin/lighthouse bn \
 		--network mainnet \
 		--execution-endpoint http://localhost:8551 \
@@ -20,7 +20,7 @@ ExecStart=/usr/bin/lighthouse bn \
 		--datadir "/persistent/lighthouse" \
         --disable-optimistic-finalized-sync \
         --disable-quic \
-        --logfile-dir /persistent/cl_logs \
+        --logfile-dir /persistent/lighthouse_logs \
         --logfile-format JSON \
         --logfile-debug-level debug \
         --logfile-max-number 5 \

bob-l1/mkosi.extra/etc/systemd/system/searcher-firewall.service.d/require-azure-provision.conf

@@ -0,0 +1,3 @@
+[Unit]
+After=azure-complete-provisioning.service
+Requires=azure-complete-provisioning.service

bob-l1/mkosi.postinst

@@ -10,7 +10,12 @@ mkosi-chroot useradd -r -s /bin/false -G eth lighthouse
 # Install lighthouse
 install -m 755 services/bin/lighthouse-init "$BUILDROOT/usr/bin/"
 
-# Enable lighthouse service
+# Enable services
 mkdir -p "$BUILDROOT/etc/systemd/system/minimal.target.wants"
-mkosi-chroot systemctl enable lighthouse.service
-ln -sf /etc/systemd/system/lighthouse.service "$BUILDROOT/etc/systemd/system/minimal.target.wants/"
+for service in \
+    lighthouse.service \
+    openntpd.service
+do
+    mkosi-chroot systemctl enable "$service"
+    ln -sf "/etc/systemd/system/$service" "$BUILDROOT/etc/systemd/system/minimal.target.wants/"
+done

buildernet/mkosi.conf

@@ -16,7 +16,8 @@ Packages=prometheus
          libsnappy1v5
          netcat-openbsd
          bubblewrap
-BuildPackages=libleveldb-dev
+BuildPackages=cargo
+              libleveldb-dev
               libsnappy-dev
               zlib1g-dev
               libzstd-dev

flake.nix

@@ -38,6 +38,17 @@
       };
       vendorHash = "sha256-NrZjORe/MjfbRDcuYVOGjNMCo1JGWvJDNVEPojI3L/g=";
     };
+    measured-boot-gcp = pkgs.buildGoModule {
+      pname = "measured-boot-gcp";
+      version = "main";
+      src = pkgs.fetchFromGitHub {
+        owner = "flashbots";
+        repo = "dstack-mr-gcp";
+        rev = "3d718ab28599ea0c05e65d0f742fdee9fc17a5c7";
+        sha256 = "sha256-KFo9wcQuG98Hi4mlMr5VS6D6/STW7jzZ9y1DyqsI820=";
+      };
+      vendorHash = "sha256-MxOQSXLAbWC1SOCPzPrNcU20WElbe7eUVdCLTutSYM8=";
+    };
     mkosi = system: let
       pkgsForSystem = import nixpkgs {inherit system;};
       mkosi-unwrapped = pkgsForSystem.mkosi.override {
@@ -53,11 +64,13 @@
             mtools
             mustache-go
             cryptsetup
+            gptfdisk
             util-linux
             zstd
             which
             qemu-utils
             parted
+            unzip
           ]
           ++ [reprepro];
       };
@@ -76,7 +89,7 @@
     devShells = builtins.listToAttrs (map (system: {
       name = system;
       value.default = pkgs.mkShell {
-        nativeBuildInputs = [(mkosi system) measured-boot];
+        nativeBuildInputs = [(mkosi system) measured-boot measured-boot-gcp];
         shellHook = ''
           mkdir -p mkosi.packages mkosi.cache mkosi.builddir ~/.cache/mkosi
         '';