[DO NOT SUBMIT] Repro TestJobControl flake and upload docker logs

PiperOrigin-RevId: 813873791

Author: shailend-g

.buildkite/hooks/post-command

@@ -8,6 +8,7 @@ rm -f repo.key
 # Upload all relevant test failures.
 make -s testlogs 2>/dev/null | grep // | sort | uniq | (
   declare log_count=0
+  local testjobcontrol_failed=false
   while read target log; do
     if test -z "${target}"; then
       continue
@@ -25,10 +26,31 @@ make -s testlogs 2>/dev/null | grep // | sort | uniq | (
         buildkite-agent annotate --style error --context failures --append
     else
       buildkite-agent artifact upload "${log}"
+      if grep -qP "TestJobControl" "${log}"; then
+        testjobcontrol_failed=true
+      fi
       echo " * [${target}](artifact://${log#/}) (${BUILDKITE_LABEL})" | \
         buildkite-agent annotate --style error --context failures --append
-    fi
+
+      # List the first 5 specific failing go tests as sub-bullets.
+      sed -n '/^--- FAIL:/ {s/^--- FAIL: \([^ ]*\) (.*)/\1/p}' "${log}" | \
+        grep -v 'main_test' | head -n 5 | xargs -I {} echo "   * {}" | \
+        buildkite-agent annotate --style error --context failures --append
+      # List the first 5 specific failing cpp tests as sub-bullets.
+      sed -n '/\[  FAILED  \]/ {s/.*\[  FAILED  \] \([^ ]*\) (.*)/\1/p}' "${log}" | \
+        grep -v 'main_test' | head -n 5 | xargs -I {} echo "   * {}" | \
+        buildkite-agent annotate --style error --context failures --append
+  fi
   done
+
+  if [[ "${testjobcontrol_failed}" == true ]]; then
+    sudo journalctl -u docker.service -p warning --since "2 hours ago" > /tmp/docker.log
+    echo "#############" >> /tmp/docker.log
+    docker --version >> /tmp/docker.log
+    uname -a >> /tmp/docker.log
+    echo "#############" >> /tmp/docker.log
+    buildkite-agent artifact upload /tmp/docker.log
+  fi
 )
 
 # Upload all profiles, and include in an annotation.

Makefile

@@ -369,6 +369,11 @@ docker-tests: load-basic $(RUNTIME_BIN)
 	@$(call test_runtime_cached,$(RUNTIME),$(INTEGRATION_TARGETS) --test_env=TEST_SAVE_RESTORE_NETSTACK=true //test/e2e:integration_runtime_test //test/e2e:runtime_in_docker_test)
 .PHONY: docker-tests
 
+flake-repro: load-basic $(RUNTIME_BIN)
+	@$(call install_runtime,$(RUNTIME),) # Clear flags.
+	@$(call test_runtime,$(RUNTIME),//test/e2e:integration_test)
+.PHONY: flake-repro
+
 plugin-network-tests: load-basic $(RUNTIME_BIN)
 	@$(call install_runtime,$(RUNTIME)-dpdk,--network=plugin)
 	@$(call test_runtime_cached,$(RUNTIME)-dpdk, --test_arg=-test.run=ConnectToSelf $(INTEGRATION_TARGETS))

test/e2e/integration_test.go

@@ -347,6 +347,13 @@ func TestNumCPU(t *testing.T) {
 
 // TestJobControl tests that job control characters are handled properly.
 func TestJobControl(t *testing.T) {
+	for i := 0; i < 100; i++ {
+		testJobControl(t)
+		time.Sleep(time.Second)
+	}
+}
+
+func testJobControl(t *testing.T) {
 	ctx := context.Background()
 	d := dockerutil.MakeContainer(ctx, t)
 	defer d.CleanUp(ctx)
@@ -360,6 +367,7 @@ func TestJobControl(t *testing.T) {
 	}
 	// Give shell a few seconds to start executing the sleep.
 	time.Sleep(2 * time.Second)
+	t.Logf("TestJobControl: container %v is sleeping", d.ID())
 
 	if _, err := p.Write(time.Second, []byte{0x03}); err != nil {
 		t.Fatalf("error exit: %v", err)