CI: add qemu functional tests on github action

.github/workflows/pr.yml

@@ -115,6 +115,12 @@ jobs:
       - name: Docker Info
         shell: bash
         run: |
+          echo "--------------------------"
+          grep -i hypervisor /proc/cpuinfo || true
+          echo "--------------------------"
+          systemd-detect-virt || true
+          echo "--------------------------"
+          virt-what || true
           echo "--------------------------"
           docker version || true
           echo "--------------------------"
@@ -413,7 +419,122 @@ jobs:
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
           if [ "$numPass" -lt 36 ];then echo "*** Failed to pass at least 36 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
+  functional_qemu_macos_intel:
+    permissions:
+      contents: none
+    needs: [build_minikube_test_binaries]
+    env:
+      TIME_ELAPSED: time
+      JOB_NAME: "functional_qemu_macos_intel"
+      GOPOGH_RESULT: ""
+      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
+    runs-on: macos-13
+    steps:
+      - name: Install kubectl and docker-clie
+        shell: bash
+        run: |
+          brew install kubectl
+          brew install docker
+          kubectl version --client=true
+      - name: Install Qemu and socket_vmnet
+        shell: bash
+        run: |
+          brew install meson qemu cdrtools coreutils tree
+          fw=/usr/libexec/ApplicationFirewall/socketfilterfw
+          sudo $fw --remove /usr/libexec/bootpd
+          sudo $fw --add /usr/libexec/bootpd
+          sudo $fw --unblock /usr/libexec/bootpd
+          brew install socket_vmnet
+          HOMEBREW=$(which brew) && sudo ${HOMEBREW} services start socket_vmnet
+      - name: Info Block
+        shell: bash
+        run: |
+          set -x
+          sysctl -n hw.memsize
+          echo "$(sysctl -n hw.memsize) / 1024 / 1024 / 1024" | bc
+          sysctl -n hw.ncpu
+          sysctl -n machdep.cpu.brand_string
+          sysctl hw.model
+          sysctl -n kern.hv_vmm_present
+          sysctl -n kern.hv_support
+          system_profiler SPHardwareDataType
+          echo "--------------------------"
+          docker version || true
+          echo "--------------------------"
+          docker info || true      
+      - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5
+        with:
+          cache: false
+          go-version: ${{env.GO_VERSION}}
+      - name: Install gopogh
+        shell: bash
+        run: |
+          go install github.com/medyagh/gopogh/cmd/gopogh@v0.29.0
+      - name: Download Binaries
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
+        with:
+          name: minikube_binaries
+          path: minikube_binaries
+      - name: Display structure of downloaded files
+        run: ls -R 
+      - name: Run Integration Test
+        continue-on-error: false
+        # bash {0} to allow test to continue to next step. in case of
+        shell: bash {0}
+        run: |
+          set -x
+          pwd
+          cd minikube_binaries
+          chmod a+x e2e-*
+          chmod a+x minikube-*
+          mkdir -p report
+          # test home
+          mkdir -p /tmp/th
 
+          MINIKUBE_HOME=/tmp/th ./minikube-darwin-amd64 delete --all --purge
+          START_TIME=$(date -u +%s)
+          KUBECONFIG=$/tmp/th/kubeconfig MINIKUBE_HOME=/tmp/th ./e2e-darwin-amd64 -minikube-start-args="--vm-driver=qemu --memory=max --wait-timeout=15m --network socket_vmnet"  -test.run TestFunctional -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+          END_TIME=$(date -u +%s)
+          TIME_ELAPSED=$(($END_TIME-$START_TIME))
+          min=$((${TIME_ELAPSED}/60))
+          sec=$((${TIME_ELAPSED}%60))
+          TIME_ELAPSED="${min} min $sec seconds "
+          echo "TIME_ELAPSED=${TIME_ELAPSED}" >> $GITHUB_ENV
+      - name: Generate HTML Report
+        shell: bash
+        run: |
+          cd minikube_binaries
+          export PATH=${PATH}:`go env GOPATH`/bin
+          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
+          echo status: ${STAT}
+          FailNum=$(echo $STAT | jq '.NumberOfFail')
+          TestsNum=$(echo $STAT | jq '.NumberOfTests')
+          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
+          echo "GOPOGH_RESULT=${GOPOGH_RESULT}" >> $GITHUB_ENV
+          echo 'STAT<<EOF' >> $GITHUB_ENV
+          echo "${STAT}" >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: functional_qemu_macos_intel
+          path: minikube_binaries/report
+      - name: The End Result functional_qemu_macos_intel
+        shell: bash
+        run: |
+          echo ${GOPOGH_RESULT}
+          numFail=$(echo $STAT | jq '.NumberOfFail')
+          numPass=$(echo $STAT | jq '.NumberOfPass')
+          echo "*******************${numPass} Passes :) *******************"
+          echo $STAT | jq '.PassedTests' || true
+          echo "*******************************************************"
+          echo "---------------- ${numFail} Failures :( ----------------------------"
+          echo $STAT | jq '.FailedTests' || true
+          echo "-------------------------------------------------------"
+          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
+          if [ "$numPass" -lt 36 ];then echo "*** Failed to pass at least 36 ! ***";exit 2;fi
+          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi  
   functional_podman_ubuntu:
     permissions:
       contents: none
@@ -653,6 +774,7 @@ jobs:
         functional_docker_rootless_containerd_ubuntu,
         functional_podman_ubuntu,
         functional_baremetal_ubuntu22_04,
+        functional_qemu_macos_intel,
       ]
     runs-on: ubuntu-22.04
     steps:
@@ -669,6 +791,7 @@ jobs:
           cp -r ./functional_docker_rootless_containerd_ubuntu ./all_reports/
           cp -r ./functional_podman_ubuntu ./all_reports/
           cp -r ./functional_baremetal_ubuntu22_04 ./all_reports/
+          cp -r ./functional_qemu_macos_intel ./all_reports/
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
         with:
           name: all_reports

pkg/drivers/qemu/qemu.go

@@ -517,7 +517,11 @@ func (d *Driver) Start() error {
 			return nil
 		}
 		// Implement a retry loop because IP address isn't added to dhcp leases file immediately
-		for i := 0; i < 60; i++ {
+		multiplier := 1
+		if detect.NestedVM() {
+			multiplier = 3 // will help with running in Free github action Macos VMs (takes 112+ retries on average)
+		}
+		for i := 0; i < 60*multiplier; i++ {
 			log.Debugf("Attempt %d", i)
 			err = getIP()
 			if err == nil {

pkg/drivers/vfkit/vfkit.go

@@ -44,6 +44,7 @@ import (
 	"k8s.io/klog/v2"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
 	"k8s.io/minikube/pkg/drivers/vmnet"
+	"k8s.io/minikube/pkg/minikube/detect"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/firewall"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -309,7 +310,11 @@ func (d *Driver) setupIP(mac string) error {
 		return nil
 	}
 	// Implement a retry loop because IP address isn't added to dhcp leases file immediately
-	for i := 0; i < 60; i++ {
+	multiplier := 1
+	if detect.NestedVM() {
+		multiplier = 3 // will help with running in Free github action Macos VMs (takes 112+ retries on average)
+	}
+	for i := 0; i < 60*multiplier; i++ {
 		log.Debugf("Attempt %d", i)
 		err = getIP()
 		if err == nil {

pkg/minikube/detect/detect.go

@@ -111,6 +111,32 @@ func GithubActionRunner() bool {
 	return os.Getenv("GITHUB_ACTIONS") == "true"
 }
 
+// NestedVM returns true if the current machine is running a nested VM
+func NestedVM() bool {
+	if runtime.GOOS == "linux" {
+		c := exec.Command("systemd-detect-virt", "--quiet")
+		_, err := c.Output()
+		if err == nil {
+			klog.Infof("nested VM detected")
+			return true
+		}
+		return false // On a bare-metal system, it exits with code 1 and outputs "none".
+
+	}
+
+	if runtime.GOOS == "darwin" {
+		c := exec.Command("sysctl", "-n", "kern.hv_vmm_present")
+		o, err := c.Output()
+		if err != nil {
+			klog.Warningf("Failed to check for nested VM: %v", err)
+			return false
+		}
+		klog.Infof("nested VM detected, %s", o)
+		return true
+	}
+	return false
+}
+
 // ImageCacheDir returns the path in the minikube home directory to the container image cache for the current architecture
 func ImageCacheDir() string {
 	return filepath.Join(localpath.MakeMiniPath("cache", "images"), runtime.GOARCH)

test/integration/functional_test.go

@@ -986,7 +986,8 @@ func validateDryRun(ctx context.Context, t *testing.T, profile string) {
 
 	// docs: Run `minikube start --dry-run --memory 250MB`
 	// Too little memory!
-	startArgs := append([]string{"start", "-p", profile, "--dry-run", "--memory", "250MB", "--alsologtostderr"}, StartArgsWithContext(ctx)...)
+	// using StartArgsSkipMemoryWithContext to skip memory-related flags to avoid duplicate memory flag in this test
+	startArgs := append([]string{"start", "-p", profile, "--dry-run", "--memory", "250MB", "--alsologtostderr"}, StartArgsSkipMemoryWithContext(ctx)...)
 	c := exec.CommandContext(mctx, Target(), startArgs...)
 	rr, err := Run(t, c)
 
@@ -1028,7 +1029,7 @@ func validateInternationalLanguage(ctx context.Context, t *testing.T, profile st
 	defer cancel()
 
 	// Too little memory!
-	startArgs := append([]string{"start", "-p", profile, "--dry-run", "--memory", "250MB", "--alsologtostderr"}, StartArgsWithContext(ctx)...)
+	startArgs := append([]string{"start", "-p", profile, "--dry-run", "--memory", "250MB", "--alsologtostderr"}, StartArgsSkipMemoryWithContext(ctx)...)
 	c := exec.CommandContext(mctx, Target(), startArgs...)
 	// docs: Set environment variable `LC_ALL=fr` to enable minikube translation to French
 	c.Env = append(os.Environ(), "LC_ALL=fr")
@@ -1038,11 +1039,7 @@ func validateInternationalLanguage(ctx context.Context, t *testing.T, profile st
 
 	wantCode := reason.ExInsufficientMemory
 	if rr.ExitCode != wantCode {
-		if HyperVDriver() {
-			t.Skip("skipping this error on HyperV till this issue is solved https://github.com/kubernetes/minikube/issues/9785")
-		} else {
-			t.Errorf("dry-run(250MB) exit code = %d, wanted = %d: %v", rr.ExitCode, wantCode, err)
-		}
+		t.Errorf("dry-run(250MB) exit code = %d, wanted = %d: %v", rr.ExitCode, wantCode, err)
 	}
 	// docs: Make sure the dry-run output message is in French
 	if !strings.Contains(rr.Stdout.String(), "Utilisation du pilote") {

test/integration/main_test.go

@@ -130,6 +130,23 @@ func StartArgsWithContext(ctx context.Context) []string {
 	return res
 }
 
+// StartArgsSkipMemoryWithContext same as StartArgsWithContext but skips the --memory argument
+func StartArgsSkipMemoryWithContext(ctx context.Context) []string {
+	args := strings.Split(*startArgs, " ")
+	res := make([]string, 0, len(args))
+	for _, arg := range args {
+		if strings.HasPrefix(arg, "--memory") {
+			continue
+		}
+		res = append(res, arg)
+	}
+	value := ctx.Value(ContextKey("k8sVersion"))
+	if value != nil && value != "" {
+		res = append(res, fmt.Sprintf("--kubernetes-version=%s", value))
+	}
+	return res
+}
+
 // Target returns where the minikube binary can be found
 func Target() string {
 	return *binaryPath