Merge pull request #41 from flashbots/ilya/more-bob-l1-refactors

More BoB-L1 refactors, prep for BoB-L2

Author: alexhulbert

Makefile

@@ -39,23 +39,30 @@ setup: ## Install dependencies (Linux only)
 
 # Build module
 build: check-perms setup ## Build the specified module
-	@$(WRAPPER) mkosi --force -I $(IMAGE).conf
+	$(WRAPPER) mkosi --force -I $(IMAGE).conf
 
 # Build module with devtools profile
 build-dev: check-perms setup ## Build module with development tools
-	@$(WRAPPER) mkosi --force --profile=devtools -I $(IMAGE).conf
+	$(WRAPPER) mkosi --force --profile=devtools -I $(IMAGE).conf
 
 ##@ Utilities
 
-# Run measured-boot on the EFI file
-measure: ## Export TDX measurements for the built image
+measure: ## Export TDX measurements for the built EFI file
 	@if [ ! -f build/tdx-debian.efi ]; then \
 		echo "Error: build/tdx-debian.efi not found. Run 'make build' first."; \
 		exit 1; \
 	fi
 	@$(WRAPPER) measured-boot build/tdx-debian.efi build/measurements.json --direct-uki
 	echo "Measurements exported to build/measurements.json"
 
+measure-gcp: ## Export TDX measurements for GCP
+	@if [ ! -f build/tdx-debian.efi ]; then \
+		echo "Error: build/tdx-debian.efi not found. Run 'make build' first."; \
+		exit 1; \
+	fi
+	@$(WRAPPER) dstack-mr -uki build/tdx-debian.efi -json > build/gcp_measurements.json
+	echo "GCP Measurements exported to build/gcp_measurements.json"
+
 # Clean build artifacts
 clean: ## Remove cache and build artifacts
 	rm -rf build/ mkosi.builddir/ mkosi.cache/ lima-nix/

base/debloat.sh

@@ -23,7 +23,6 @@ debloat_paths=(
     "/usr/share/bug"
     "/usr/share/menu"
     "/usr/share/systemd"
-    "/usr/share/bash-completion"
     "/usr/share/zsh"
     "/usr/share/mime"
     "/usr/lib/modules"
@@ -40,4 +39,13 @@ debloat_paths=(
     "/nix"
 )
 
-for p in "${debloat_paths[@]}"; do rm -rf $BUILDROOT$p; done
+if [[ ! "$PROFILES" == *"devtools"* ]]; then
+    debloat_paths+=(
+        "/usr/share/bash-completion"
+    )
+fi
+
+for p in "${debloat_paths[@]}"; do
+    echo "Debloating $p"
+    rm -rf $BUILDROOT$p
+done

base/mkosi.conf

@@ -42,7 +42,6 @@ BuildPackages=build-essential
               cmake
               pkg-config
               clang
-              cargo
               flex
               bison
               elfutils

bob-common/mkosi.conf

@@ -15,7 +15,6 @@ Packages=podman
          iproute2
          conntrack
          netfilter-persistent
-         openntpd
          curl
          jq
          logrotate

bob-common/mkosi.extra/etc/systemd/system/searcher-firewall.service

@@ -1,7 +1,7 @@
 [Unit]
 Description=Searcher Network and Firewall Rules
-After=azure-complete-provisioning.service
-Requires=azure-complete-provisioning.service
+After=network.target network-setup.service
+Requires=network-setup.service
 
 [Service]
 Type=oneshot

bob-common/mkosi.extra/usr/bin/init-container.sh

@@ -5,25 +5,30 @@ NAME=searcher-container
 
 # PORT FORWARDS
 SEARCHER_SSH_PORT=10022
-ENGINE_API_PORT=8551
 EL_P2P_PORT=30303
 SEARCHER_INPUT_CHANNEL=27017
 
+# Run extra commands which are customized per image,
+# see bob*/mkosi.extra/etc/bob/searcher-container-before-init
+#
+# `source` is not supported in dash
+. /etc/bob/searcher-container-before-init
+
+# BOB_SEARCHER_EXTRA_PODMAN_FLAGS is unescaped, it's sourced from trusted hardcoded file
+
 echo "Starting $NAME..."
 su -s /bin/sh searcher -c "cd ~ && podman run -d \
     --name $NAME --replace \
     --init \
     -p ${SEARCHER_SSH_PORT}:22 \
-    -p ${ENGINE_API_PORT}:${ENGINE_API_PORT} \
     -p ${EL_P2P_PORT}:${EL_P2P_PORT} \
     -p ${EL_P2P_PORT}:${EL_P2P_PORT}/udp \
     -p ${SEARCHER_INPUT_CHANNEL}:${SEARCHER_INPUT_CHANNEL}/udp \
     -v /persistent/searcher:/persistent:rw \
     -v /etc/searcher/ssh_hostkey:/etc/searcher/ssh_hostkey:rw \
     -v /persistent/searcher_logs:/var/log/searcher:rw \
-    -v /persistent/cl_logs:/var/log/cl:ro \
-    -v /tmp/jwt.hex:/secrets/jwt.hex:ro \
     -v /etc/searcher-logrotate.conf:/tmp/searcher.conf:ro \
+    $BOB_SEARCHER_EXTRA_PODMAN_FLAGS \
     docker.io/library/ubuntu:24.04 \
     /bin/sh -c ' \
         DEBIAN_FRONTEND=noninteractive apt-get update && \
@@ -41,7 +46,7 @@ su -s /bin/sh searcher -c "cd ~ && podman run -d \
         while true; do /usr/sbin/sshd -D -e; sleep 5; done'"
 
 # Attempt a quick check that the container is running
-for i in 1 2 3 4 5; do
+for i in $(seq 1 5); do
     status=$(su -s /bin/sh - searcher -c "podman inspect --format '{{.State.Status}}' $NAME 2>/dev/null || true")
     if [ "$status" = "running" ]; then
         break
@@ -63,24 +68,25 @@ if [ -z "$pid" ] || [ "$pid" = "0" ]; then
 fi
 
 echo "Applying iptables rules in $NAME (PID: $pid) network namespace..."
+ns_iptables() {
+    nsenter --target "$pid" --net iptables "$@"
+}
+
+ns_iptables -A OUTPUT -d 169.254.169.254 -j DROP
 
-# Enter network namespace and apply DROP rules on port 9000 TCP/UDP
-nsenter --target "$pid" --net iptables -A OUTPUT -p tcp --dport 9000 -j DROP
-nsenter --target "$pid" --net iptables -A OUTPUT -p udp --dport 9000 -j DROP
+ns_iptables -A OUTPUT -p tcp --dport 9000 -j DROP
+ns_iptables -A OUTPUT -p udp --dport 9000 -j DROP
 
-# Enter network namespace and apply DROP rule on port 123 UDP
-nsenter --target "$pid" --net iptables -A OUTPUT -p udp --dport 123 -j DROP
+ns_iptables -A OUTPUT -p udp --dport 123 -j DROP
 
-# Drop outbound traffic from SEARCHER_INPUT_CHANNEL
-nsenter --target "$pid" --net iptables -A OUTPUT -p udp --sport $SEARCHER_INPUT_CHANNEL -j DROP
-nsenter --target "$pid" --net iptables -A OUTPUT -p tcp --sport $SEARCHER_INPUT_CHANNEL -j DROP
+ns_iptables -A OUTPUT -p udp --sport $SEARCHER_INPUT_CHANNEL -j DROP
+ns_iptables -A OUTPUT -p tcp --sport $SEARCHER_INPUT_CHANNEL -j DROP
 
-echo "Injecting static hosts into $NAME..."
+# Helper, only used in sourced script below
+exec_in_container() {
+    su -s /bin/sh searcher -c "podman exec $NAME /bin/sh -c '$1'"
+}
 
-su -s /bin/sh searcher -c "podman exec $NAME /bin/sh -c '
-    echo \"3.149.14.12 tx.tee-searcher.flashbots.net\" >> /etc/hosts &&
-    echo \"3.136.107.142 tx.tee-searcher.flashbots.net\" >> /etc/hosts &&
-    echo \"18.221.59.61 backruns.tee-searcher.flashbots.net\" >> /etc/hosts &&
-    echo \"3.15.88.156 backruns.tee-searcher.flashbots.net\" >> /etc/hosts &&
-    echo \"52.207.17.217 fbtee.titanbuilder.xyz\" >> /etc/hosts
-'"
+# Run extra commands which are customized per image,
+# see bob*/mkosi.extra/etc/bob/searcher-container-after-init
+. /etc/bob/searcher-container-after-init

bob-common/mkosi.extra/usr/bin/init-firewall.sh

@@ -20,7 +20,7 @@ set -eu -o pipefail
 #    ├(loopback?)─> ACCEPT
 #    └─> default DROP
 #
-# - There are no ports opened in this file, refer to bob*/mkosi.extra/etc/firewall-config
+# - There are no ports opened in this file, refer to bob*/mkosi.extra/etc/bob/firewall-config
 #   for actual chain rules.
 # - Mode-specific ESTABLISHED/RELATED connections are killed by
 #   `conntrack -D ...` upon mode toggle.
@@ -98,7 +98,7 @@ iptables -A OUTPUT ! -o lo -d 127.0.0.0/8 -j DROP
 
 ###########################################################################
 #
-# Some helper functions to reduce boilerplate in /etc/firewall-config
+# Some helper functions to reduce boilerplate in /etc/bob/firewall-config
 #
 ###########################################################################
 accept_dst_port() {
@@ -124,14 +124,22 @@ accept_dst_ip_port() {
                          -m comment --comment "$comment"
 }
 
+drop_dst_ip() {
+    chain="$1"
+    ip="$2"
+    comment="$3"
+
+    iptables -A "$chain" -d "$ip" -j DROP \
+                         -m comment --comment "$comment"
+}
 
 ###########################################################################
 # (5) Load firewall rules in {MAINTENANCE,PRODUCTION}_{IN,OUT} chains.
-# Those are customized per image, see bob*/mkosi.extra/etc/firewall-config
+# Those are customized per image, see bob*/mkosi.extra/etc/bob/firewall-config
 #
 # `source` is not supported in dash
 ###########################################################################
-. /etc/firewall-config
+. /etc/bob/firewall-config
 
 ###########################################################################
 # (6) Start in Maintenance Mode

bob-common/mkosi.postinst

@@ -22,7 +22,6 @@ mkdir "$BUILDROOT/etc/dropbear"
 mkdir -p "$BUILDROOT/etc/systemd/system/minimal.target.wants"
 for service in \
     network-setup.service \
-    openntpd.service \
     logrotate.service \
     fluent-bit.service \
     wait-for-key.service \

bob-l1/mkosi.conf

@@ -1,15 +1,18 @@
 [Build]
-Environment=LIGHTHOUSE_BINARY KERNEL_CONFIG_SNIPPETS=bob-l1/kernel.config
+Environment=LIGHTHOUSE_BINARY KERNEL_CONFIG_SNIPPETS=bob-l1/kernel.config KERNEL_VERSION=6.13.12
 WithNetwork=true
 
 [Content]
 ExtraTrees=bob-l1/mkosi.extra
 PostInstallationScripts=bob-l1/mkosi.postinst
 BuildScripts=bob-l1/mkosi.build
 
+Packages=openntpd
+
 BuildPackages=build-essential
               git
               gcc
+              cargo
               zlib1g-dev
               libzstd-dev
               libleveldb-dev

bob-l1/mkosi.extra/etc/firewall-config renamed to bob-l1/mkosi.extra/etc/bob/firewall-config

@@ -84,9 +84,7 @@ accept_dst_port $CHAIN_MAINTENANCE_IN udp $EL_P2P_PORT "EL P2P (UDP)"
 ###########################################################################
 
 # Block Flashbots protect tx endpoints during maintenance
-iptables -A $CHAIN_MAINTENANCE_OUT \
-    -d $FLASHBOTS_TX_STREAM_1,$FLASHBOTS_TX_STREAM_2 -j DROP \
-    -m comment --comment "Flashbots Protect (DROP before accept-all 443)"
+drop_dst_ip $CHAIN_MAINTENANCE_OUT $FLASHBOTS_TX_STREAM_1,$FLASHBOTS_TX_STREAM_2 "Flashbots Protect (DROP before accept-all rules)"
 
 accept_dst_port $CHAIN_MAINTENANCE_OUT udp $DNS_PORT "DNS (UDP)"
 accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $DNS_PORT "DNS (TCP)"