Add cross-architecture remote Docker support with SSH tunneling and TLS certificate handling

Author: einyx

.gitignore

@@ -64,3 +64,4 @@ test/integration/testdata/minikube-linux-amd64-latest-stable
 hack/legacy_fill_db/gopogh_filldb_log.txt
 hack/legacy_fill_db/out/output_summary.json
 hack/legacy_fill_db/out/output.html
+.aider*

Makefile

@@ -24,7 +24,7 @@ KIC_VERSION ?= $(shell grep -E "Version =" pkg/drivers/kic/types.go | cut -d \"
 HUGO_VERSION ?= $(shell grep -E "HUGO_VERSION = \"" netlify.toml | cut -d \" -f2)
 
 # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
-ISO_VERSION ?= v1.36.0
+ISO_VERSION ?= v1.36.0-1749153077-20895
 
 # Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta
 DEB_VERSION ?= $(subst -,~,$(RAW_VERSION))
@@ -103,7 +103,7 @@ $(shell mkdir -p $(BUILD_DIR))
 CURRENT_GIT_BRANCH ?= $(shell git branch | grep \* | cut -d ' ' -f2)
 
 # Use system python if it exists, otherwise use Docker.
-PYTHON := $(shell command -v python || echo "docker run --rm -it -v $(shell pwd):/minikube:Z -w /minikube python python")
+PYTHON := $(shell command -v python || echo "docker run --rm -it -v $(shell pwd):/minikube -w /minikube python python")
 BUILD_OS := $(shell uname -s)
 
 SHA512SUM=$(shell command -v sha512sum || echo "shasum -a 512")
@@ -189,7 +189,7 @@ endef
 
 # $(call DOCKER, image, command)
 define DOCKER
-	docker run --rm -e GOCACHE=/app/.cache -e IN_DOCKER=1 --user $(shell id -u):$(shell id -g) -w /app -v $(PWD):/app:Z -v $(GOPATH):/go --init $(1) /bin/bash -c '$(2)'
+	docker run --rm -e GOCACHE=/app/.cache -e IN_DOCKER=1 --user $(shell id -u):$(shell id -g) -w /app -v $(PWD):/app -v $(GOPATH):/go --init $(1) /bin/bash -c '$(2)'
 endef
 
 ifeq ($(BUILD_IN_DOCKER),y)
@@ -341,13 +341,32 @@ out/minikube-%.iso: $(shell find "deploy/iso/minikube-iso" -type f)
 ifeq ($(IN_DOCKER),1)
 	$(MAKE) minikube-iso-$*
 else
-	docker run --rm --workdir /mnt --volume $(CURDIR):/mnt:Z $(ISO_DOCKER_EXTRA_ARGS) \
+	docker run --rm --workdir /mnt --volume $(CURDIR):/mnt $(ISO_DOCKER_EXTRA_ARGS) \
 		--user $(shell id -u):$(shell id -g) --env HOME=/tmp --env IN_DOCKER=1 \
 		$(ISO_BUILD_IMAGE) /bin/bash -lc '/usr/bin/make minikube-iso-$*'
 endif
 
+# Build ISO on remote frank server (169.254.1.1)
+.PHONY: iso-remote-%
+iso-remote-%: deploy/iso/minikube-iso/board/minikube/%/rootfs-overlay/usr/bin/auto-pause ## Build ISO on remote frank server
+	@echo "Building minikube ISO on frank (169.254.1.1) for architecture: $*"
+	@echo "Building auto-pause binary locally first..."
+	@make deploy/iso/minikube-iso/board/minikube/$*/rootfs-overlay/usr/bin/auto-pause
+	@echo "Syncing source code to frank..."
+	@rsync -av --delete --exclude 'out/' --exclude '.git/' --exclude 'test-results/' --exclude '.idea/' --exclude '*.swp' . 169.254.1.1:~/minikube-build/
+	@echo "Building ISO on frank (auto-pause already built)..."
+	@ssh 169.254.1.1 "cd ~/minikube-build && export GO111MODULE=on && touch deploy/iso/minikube-iso/board/minikube/$*/rootfs-overlay/usr/bin/auto-pause && make minikube-iso-$* HOSTCC=/usr/bin/gcc-13 HOSTCXX=/usr/bin/g++-13"
+	@echo "Copying ISO back from frank..."
+	@scp 169.254.1.1:~/minikube-build/out/minikube-$*.iso out/
+	@echo "ISO build complete: out/minikube-$*.iso"
+
+# Convenience targets for remote ISO builds
+.PHONY: iso-remote-amd64 iso-remote-arm64
+iso-remote-amd64: iso-remote-x86_64
+iso-remote-arm64: iso-remote-aarch64
+
 iso_in_docker:
-	docker run -it --rm --workdir /mnt --volume $(CURDIR):/mnt:Z $(ISO_DOCKER_EXTRA_ARGS) \
+	docker run -it --rm --workdir /mnt --volume $(CURDIR):/mnt $(ISO_DOCKER_EXTRA_ARGS) \
 		--user $(shell id -u):$(shell id -g) --env HOME=/tmp --env IN_DOCKER=1 \
 		$(ISO_BUILD_IMAGE) /bin/bash
 
@@ -523,7 +542,7 @@ out/linters/golangci-lint-$(GOLINT_VERSION):
 .PHONY: lint
 ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y)
 lint:
-	docker run --rm -v `pwd`:/app:Z -w /app golangci/golangci-lint:$(GOLINT_VERSION) \
+	docker run --rm -v `pwd`:/app -w /app golangci/golangci-lint:$(GOLINT_VERSION) \
 	golangci-lint run ${GOLINT_OPTIONS} ./..." 
 	# --skip-dirs "cmd/drivers/kvm|cmd/drivers/hyperkit|pkg/drivers/kvm|pkg/drivers/hyperkit" 
 	# The "--skip-dirs" parameter is no longer supported in the V2 version. If you need to skip the directory, 
@@ -657,7 +676,7 @@ out/docker-machine-driver-hyperkit:
 ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y)
 	docker run --rm -e GOCACHE=/app/.cache -e IN_DOCKER=1 \
 		--user $(shell id -u):$(shell id -g) -w /app \
-		-v $(PWD):/app:Z -v $(GOPATH):/go:Z --init --entrypoint "" \
+		-v $(PWD):/app -v $(GOPATH):/go --init --entrypoint "" \
 		$(HYPERKIT_BUILD_IMAGE) /bin/bash -c 'CC=o64-clang CXX=o64-clang++ /usr/bin/make $@'
 else
 	$(if $(quiet),@echo "  GO       $@")

README.md

@@ -21,6 +21,9 @@ minikube implements a local Kubernetes cluster on macOS, Linux, and Windows. min
 
 minikube runs the latest stable release of Kubernetes, with support for standard Kubernetes features like:
 
+### Remote Docker Support
+* **Cross-architecture remote Docker contexts** - Run ARM64 minikube on macOS to manage x86_64 Docker hosts via SSH or TCP
+
 * [LoadBalancer](https://minikube.sigs.k8s.io/docs/handbook/accessing/#loadbalancer-access) - using `minikube tunnel`
 * Multi-cluster - using `minikube start -p <name>`
 * [NodePorts](https://minikube.sigs.k8s.io/docs/handbook/accessing/#nodeport-access) - using `minikube service`
@@ -38,6 +41,29 @@ As well as developer-friendly features:
 * [AMD GPU support](https://minikube.sigs.k8s.io/docs/tutorials/amd/) - for machine learning
 * [Filesystem mounts](https://minikube.sigs.k8s.io/docs/handbook/mount/)
 
+## Remote Docker Support
+
+minikube supports running against remote Docker daemons, including cross-architecture scenarios. This allows you to:
+
+* Run minikube on ARM64 macOS while managing Kubernetes clusters on x86_64 Linux Docker hosts
+* Use SSH or TCP connections to remote Docker daemons
+* Automatically detect and download the correct architecture binaries for the remote host
+
+### Usage
+
+Configure a remote Docker context and minikube will automatically detect and use it:
+
+```bash
+# Create a remote Docker context via SSH
+docker context create remote-host --docker "host=ssh://user@remote-host"
+docker context use remote-host
+
+# Start minikube - it will automatically use the remote context
+minikube start --driver=docker
+```
+
+minikube will detect the remote architecture and download the appropriate Kubernetes binaries (kubelet, kubeadm, kubectl) for the target platform.
+
 **For more information, see the official [minikube website](https://minikube.sigs.k8s.io)**
 
 ## Installation

cmd/minikube/cmd/config/profile_list.go

@@ -40,8 +40,11 @@ import (
 	"k8s.io/klog/v2"
 )
 
-var profileOutput string
-var isLight bool
+var (
+	profileOutput string
+	isLight       bool
+	isDetailed    bool
+)
 
 var profileListCmd = &cobra.Command{
 	Use:   "list",
@@ -130,7 +133,13 @@ func profileStatus(p *config.Profile, api libmachine.API) cluster.State {
 
 func renderProfilesTable(ps [][]string) {
 	table := tablewriter.NewWriter(os.Stdout)
-	table.SetHeader([]string{"Profile", "VM Driver", "Runtime", "IP", "Port", "Version", "Status", "Nodes", "Active Profile", "Active Kubecontext"})
+	if isDetailed {
+		table.SetHeader([]string{"Profile", "Driver", "Runtime", "IP", "Port", "Version",
+			"Status", "Nodes", "Active Profile", "Active Kubecontext"})
+	} else {
+		table.SetHeader([]string{"Profile", "Driver", "Runtime", "IP", "Version", "Status",
+			"Nodes", "Active Profile", "Active Kubecontext"})
+	}
 	table.SetAutoFormatHeaders(false)
 	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
 	table.SetCenterSeparator("|")
@@ -164,7 +173,13 @@ func profilesToTableData(profiles []*config.Profile) [][]string {
 		if p.ActiveKubeContext {
 			k = "*"
 		}
-		data = append(data, []string{p.Name, p.Config.Driver, p.Config.KubernetesConfig.ContainerRuntime, cpIP, strconv.Itoa(cpPort), k8sVersion, p.Status, strconv.Itoa(len(p.Config.Nodes)), c, k})
+		if isDetailed {
+			data = append(data, []string{p.Name, p.Config.Driver, p.Config.KubernetesConfig.ContainerRuntime,
+				cpIP, strconv.Itoa(cpPort), k8sVersion, p.Status, strconv.Itoa(len(p.Config.Nodes)), c, k})
+		} else {
+			data = append(data, []string{p.Name, p.Config.Driver, p.Config.KubernetesConfig.ContainerRuntime,
+				cpIP, k8sVersion, p.Status, strconv.Itoa(len(p.Config.Nodes)), c, k})
+		}
 	}
 	return data
 }
@@ -213,5 +228,6 @@ func profilesOrDefault(profiles []*config.Profile) []*config.Profile {
 func init() {
 	profileListCmd.Flags().StringVarP(&profileOutput, "output", "o", "table", "The output format. One of 'json', 'table'")
 	profileListCmd.Flags().BoolVarP(&isLight, "light", "l", false, "If true, returns list of profiles faster by skipping validating the status of the cluster.")
+	profileListCmd.Flags().BoolVarP(&isDetailed, "detailed", "d", false, "If true, returns a detailed list of profiles.")
 	ProfileCmd.AddCommand(profileListCmd)
 }

cmd/minikube/cmd/service.go

@@ -163,18 +163,20 @@ You may select another namespace by using 'minikube service {{.service}} -n <nam
 		for _, svc := range noNodePortServices {
 			noNodePortSvcNames = append(noNodePortSvcNames, fmt.Sprintf("%s/%s", svc.Namespace, svc.Name))
 		}
-		if len(noNodePortServices) != 0 {
+		if len(noNodePortServices) > 0 {
 			out.WarningT("Services {{.svc_names}} have type \"ClusterIP\" not meant to be exposed, however for local development minikube allows you to access this !", out.V{"svc_names": noNodePortSvcNames})
 		}
 
-		if driver.NeedsPortForward(co.Config.Driver) && services != nil {
-			startKicServiceTunnel(services, cname, co.Config.Driver)
+		if driver.NeedsPortForward(co.Config.Driver) {
+			svcs := services
+			if len(svcs) == 0 && len(noNodePortServices) > 0 {
+				svcs = noNodePortServices
+			}
+			if len(svcs) > 0 {
+				startKicServiceTunnel(svcs, cname, co.Config.Driver)
+			}
 		} else if !serviceURLMode {
 			openURLs(data)
-			if len(noNodePortServices) != 0 {
-				startKicServiceTunnel(noNodePortServices, cname, co.Config.Driver)
-			}
-
 		}
 	},
 }

cmd/minikube/cmd/ssh.go

@@ -29,6 +29,8 @@ import (
 	"k8s.io/minikube/pkg/minikube/node"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/reason"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/klog/v2"
 )
 
 var nativeSSHClient bool
@@ -56,7 +58,16 @@ var sshCmd = &cobra.Command{
 			}
 		}
 
-		err = machine.CreateSSHShell(co.API, *co.Config, *n, args, nativeSSHClient)
+		// For remote Docker contexts, use docker exec instead of SSH
+		klog.Warningf("SSH Command: Driver=%s, IsRemoteDockerContext=%v", co.Config.Driver, oci.IsRemoteDockerContext())
+		if co.Config.Driver == driver.Docker && oci.IsRemoteDockerContext() {
+			klog.Warningf("Using CreateSSHTerminal for remote Docker context")
+			err = oci.CreateSSHTerminal(config.MachineName(*co.Config, *n), args)
+		} else {
+			klog.Warningf("Using standard SSH shell")
+			err = machine.CreateSSHShell(co.API, *co.Config, *n, args, nativeSSHClient)
+		}
+		
 		if err != nil {
 			// This is typically due to a non-zero exit code, so no need for flourish.
 			out.ErrLn("ssh: %v", err)

cmd/minikube/cmd/start.go

@@ -41,8 +41,8 @@ import (
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/pkg/errors"
-	"github.com/shirou/gopsutil/v3/cpu"
-	gopshost "github.com/shirou/gopsutil/v3/host"
+	"github.com/shirou/gopsutil/v4/cpu"
+	gopshost "github.com/shirou/gopsutil/v4/host"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"golang.org/x/text/cases"
@@ -113,10 +113,11 @@ func init() {
 
 // startCmd represents the start command
 var startCmd = &cobra.Command{
-	Use:   "start",
-	Short: "Starts a local Kubernetes cluster",
-	Long:  "Starts a local Kubernetes cluster",
-	Run:   runStart,
+	Use:     "start",
+	Aliases: []string{"create"},
+	Short:   "Starts a local Kubernetes cluster",
+	Long:    "Starts a local Kubernetes cluster",
+	Run:     runStart,
 }
 
 // platform generates a user-readable platform message
@@ -135,7 +136,12 @@ func platform() string {
 
 	vsys, vrole, err := gopshost.Virtualization()
 	if err != nil {
-		klog.Warningf("gopshost.Virtualization returned error: %v", err)
+		// Only log if it's a real error, not just "not implemented yet"
+		if !strings.Contains(err.Error(), "not implemented yet") {
+			klog.Warningf("gopshost.Virtualization returned error: %v", err)
+		} else {
+			klog.V(3).Infof("Virtualization detection not implemented for this platform (harmless)")
+		}
 	} else {
 		klog.Infof("virtualization: %s %s", vsys, vrole)
 	}
@@ -731,6 +737,14 @@ func selectDriver(existing *config.ClusterConfig) (registry.DriverState, []regis
 		return ds, nil, true
 	}
 
+	// Check for remote Docker context and auto-select docker driver
+	if oci.IsRemoteDockerContext() {
+		ds := driver.Status("docker")
+		out.Step(style.Sparkle, `Detected a remote Docker context, using the {{.driver}} driver`, out.V{"driver": ds.String()})
+		out.Infof("For remote Docker connections, you may need to run 'minikube tunnel-ssh' for API server access")
+		return ds, nil, true
+	}
+
 	choices := driver.Choices(viper.GetBool("vm"))
 	pick, alts, rejects := driver.Suggest(choices)
 	if pick.Name == "" {
@@ -1098,8 +1112,8 @@ func suggestMemoryAllocation(sysLimit, containerLimit, nodes int) int {
 		return mem
 	}
 
-	const fallback = 2200
-	maximum := 6000
+	const fallback = 3072
+	maximum := 6144
 
 	if sysLimit > 0 && fallback > sysLimit {
 		return sysLimit

cmd/minikube/cmd/start_flags.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/blang/semver/v4"
 	"github.com/pkg/errors"
-	"github.com/shirou/gopsutil/v3/cpu"
+	"github.com/shirou/gopsutil/v4/cpu"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"k8s.io/klog/v2"
@@ -61,8 +61,8 @@ const (
 	hostOnlyCIDR            = "host-only-cidr"
 	containerRuntime        = "container-runtime"
 	criSocket               = "cri-socket"
-	networkPlugin           = "network-plugin"
-	enableDefaultCNI        = "enable-default-cni"
+	networkPlugin           = "network-plugin"     // deprecated, use --cni instead
+	enableDefaultCNI        = "enable-default-cni" // deprecated, use --cni=bridge instead
 	cniFlag                 = "cni"
 	hypervVirtualSwitch     = "hyperv-virtual-switch"
 	hypervUseExternalSwitch = "hyperv-use-external-switch"
@@ -163,7 +163,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().Bool(dryRun, false, "dry-run mode. Validates configuration, but does not mutate system state")
 
 	startCmd.Flags().String(cpus, "2", fmt.Sprintf("Number of CPUs allocated to Kubernetes. Use %q to use the maximum number of CPUs. Use %q to not specify a limit (Docker/Podman only)", constants.MaxResources, constants.NoLimit))
-	startCmd.Flags().String(memory, "", fmt.Sprintf("Amount of RAM to allocate to Kubernetes (format: <number>[<unit>], where unit = b, k, m or g). Use %q to use the maximum amount of memory. Use %q to not specify a limit (Docker/Podman only)", constants.MaxResources, constants.NoLimit))
+	startCmd.Flags().StringP(memory, "m", "", fmt.Sprintf("Amount of RAM to allocate to Kubernetes (format: <number>[<unit>], where unit = b, k, m or g). Use %q to use the maximum amount of memory. Use %q to not specify a limit (Docker/Podman only)", constants.MaxResources, constants.NoLimit))
 	startCmd.Flags().String(humanReadableDiskSize, defaultDiskSize, "Disk size allocated to the minikube VM (format: <number>[<unit>], where unit = b, k, m or g).")
 	startCmd.Flags().Bool(downloadOnly, false, "If true, only download and cache files for later use - don't install or start anything.")
 	startCmd.Flags().Bool(cacheImages, true, "If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none.")
@@ -182,7 +182,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().Uint16(mountPortFlag, defaultMountPort, mountPortDescription)
 	startCmd.Flags().String(mountTypeFlag, defaultMountType, mountTypeDescription)
 	startCmd.Flags().String(mountUID, defaultMountUID, mountUIDDescription)
-	startCmd.Flags().StringSlice(config.AddonListFlag, nil, "Enable addons. see `minikube addons list` for a list of valid addon names.")
+	startCmd.Flags().StringSlice(config.AddonListFlag, nil, "Enable one or more addons, in a comma-separated format. See `minikube addons list` for a list of valid addon names.")
 	startCmd.Flags().String(criSocket, "", "The cri socket path to be used.")
 	startCmd.Flags().String(networkPlugin, "", "DEPRECATED: Replaced by --cni")
 	startCmd.Flags().Bool(enableDefaultCNI, false, "DEPRECATED: Replaced by --cni=bridge")
@@ -232,6 +232,20 @@ func initKubernetesFlags() {
 func initDriverFlags() {
 	startCmd.Flags().StringP("driver", "d", "", fmt.Sprintf("Driver is one of: %v (defaults to auto-detect)", driver.DisplaySupportedDrivers()))
 	startCmd.Flags().String("vm-driver", "", "DEPRECATED, use `driver` instead.")
+	// Hide the deprecated vm-driver flag from help text
+	if err := startCmd.Flags().MarkHidden("vm-driver"); err != nil {
+		klog.Warningf("Failed to hide vm-driver flag: %v\n", err)
+	}
+	// Hide the deprecated flag from help text so new users dont use it (still will be processed)
+	if err := startCmd.Flags().MarkHidden(enableDefaultCNI); err != nil {
+		klog.Warningf("Failed to hide %s flag: %v\n", enableDefaultCNI, err)
+	}
+
+	// Hide the deprecated flag from help text so new users dont use it (still will be processed)
+	if err := startCmd.Flags().MarkHidden(networkPlugin); err != nil {
+		klog.Warningf("Failed to hide %s flag: %v\n", networkPlugin, err)
+	}
+
 	startCmd.Flags().Bool(disableDriverMounts, false, "Disables the filesystem mounts provided by the hypervisors")
 	startCmd.Flags().Bool("vm", false, "Filter to use only VM Drivers")
 

cmd/minikube/cmd/start_test.go

@@ -277,26 +277,26 @@ func TestSuggestMemoryAllocation(t *testing.T) {
 		nodes          int
 		want           int
 	}{
-		{"128GB sys", 128000, 0, 1, 6000},
-		{"64GB sys", 64000, 0, 1, 6000},
-		{"32GB sys", 32768, 0, 1, 6000},
+		{"128GB sys", 128000, 0, 1, 6144},
+		{"64GB sys", 64000, 0, 1, 6144},
+		{"32GB sys", 32768, 0, 1, 6144},
 		{"16GB sys", 16384, 0, 1, 4000},
 		{"odd sys", 14567, 0, 1, 3600},
-		{"4GB sys", 4096, 0, 1, 2200},
+		{"4GB sys", 4096, 0, 1, 3072},
 		{"2GB sys", 2048, 0, 1, 2048},
-		{"Unable to poll sys", 0, 0, 1, 2200},
+		{"Unable to poll sys", 0, 0, 1, 3072},
 		{"128GB sys, 16GB container", 128000, 16384, 1, 16336},
 		{"64GB sys, 16GB container", 64000, 16384, 1, 16000},
 		{"16GB sys, 4GB container", 16384, 4096, 1, 4000},
 		{"4GB sys, 3.5GB container", 16384, 3500, 1, 3452},
 		{"16GB sys, 2GB container", 16384, 2048, 1, 2048},
 		{"16GB sys, unable to poll container", 16384, 0, 1, 4000},
-		{"128GB sys 2 nodes", 128000, 0, 2, 6000},
-		{"8GB sys 3 nodes", 8192, 0, 3, 2200},
-		{"16GB sys 2 nodes", 16384, 0, 2, 2200},
+		{"128GB sys 2 nodes", 128000, 0, 2, 6144},
+		{"8GB sys 3 nodes", 8192, 0, 3, 3072},
+		{"16GB sys 2 nodes", 16384, 0, 2, 3072},
 		{"32GB sys 2 nodes", 32768, 0, 2, 4050},
-		{"odd sys 2 nodes", 14567, 0, 2, 2200},
-		{"4GB sys 2 nodes", 4096, 0, 2, 2200},
+		{"odd sys 2 nodes", 14567, 0, 2, 3072},
+		{"4GB sys 2 nodes", 4096, 0, 2, 3072},
 		{"2GB sys 3 nodes", 2048, 0, 3, 2048},
 	}
 	for _, test := range tests {