kubelet: Wait less for control-plane pods to restart

k8s/crds/kops.k8s.io_clusters.yaml

@@ -4115,6 +4115,12 @@ spec:
                     description: CpuManagerPolicy allows for changing the default
                       policy of None to static
                     type: string
+                  crashLoopBackOffMaxContainerRestartPeriod:
+                    description: CrashLoopBackOffMaxContainerRestartPeriod is the
+                      maximum duration the backoff delay can accrue to for container
+                      restarts, minimum 1 second, maximum 300 seconds. If not set,
+                      defaults to the internal crashloopbackoff maximum (300s).
+                    type: string
                   dockerDisableSharedPID:
                     description: DockerDisableSharedPID was removed.
                     type: boolean
@@ -4568,6 +4574,12 @@ spec:
                     description: CpuManagerPolicy allows for changing the default
                       policy of None to static
                     type: string
+                  crashLoopBackOffMaxContainerRestartPeriod:
+                    description: CrashLoopBackOffMaxContainerRestartPeriod is the
+                      maximum duration the backoff delay can accrue to for container
+                      restarts, minimum 1 second, maximum 300 seconds. If not set,
+                      defaults to the internal crashloopbackoff maximum (300s).
+                    type: string
                   dockerDisableSharedPID:
                     description: DockerDisableSharedPID was removed.
                     type: boolean

k8s/crds/kops.k8s.io_instancegroups.yaml

@@ -506,6 +506,12 @@ spec:
                     description: CpuManagerPolicy allows for changing the default
                       policy of None to static
                     type: string
+                  crashLoopBackOffMaxContainerRestartPeriod:
+                    description: CrashLoopBackOffMaxContainerRestartPeriod is the
+                      maximum duration the backoff delay can accrue to for container
+                      restarts, minimum 1 second, maximum 300 seconds. If not set,
+                      defaults to the internal crashloopbackoff maximum (300s).
+                    type: string
                   dockerDisableSharedPID:
                     description: DockerDisableSharedPID was removed.
                     type: boolean

nodeup/pkg/model/kubelet.go

@@ -26,10 +26,12 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/klog/v2"
@@ -242,6 +244,9 @@ func buildKubeletComponentConfig(kubeletConfig *kops.KubeletConfigSpec, provider
 	if providerID != "" {
 		componentConfig.ProviderID = providerID
 	}
+	componentConfig.CrashLoopBackOff = kubelet.CrashLoopBackOffConfig{
+		MaxContainerRestartPeriod: kubeletConfig.CrashLoopBackOffMaxContainerRestartPeriod,
+	}
 	if kubeletConfig.ShutdownGracePeriod != nil {
 		componentConfig.ShutdownGracePeriod = *kubeletConfig.ShutdownGracePeriod
 	}
@@ -648,6 +653,17 @@ func (b *KubeletBuilder) buildKubeletConfigSpec(ctx context.Context) (*kops.Kube
 
 	c.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt")
 
+	// Wait less for pods to restart, especially during the bootstrap sequence
+	if b.IsMaster && c.CrashLoopBackOffMaxContainerRestartPeriod == nil {
+		c.CrashLoopBackOffMaxContainerRestartPeriod = &metav1.Duration{Duration: time.Minute}
+	}
+	if c.CrashLoopBackOffMaxContainerRestartPeriod != nil {
+		if c.FeatureGates == nil {
+			c.FeatureGates = make(map[string]string)
+		}
+		c.FeatureGates["KubeletCrashLoopBackOffMax"] = "true"
+	}
+
 	// Respect any MaxPods value the user sets explicitly.
 	if (b.NodeupConfig.Networking.AmazonVPC != nil || (b.NodeupConfig.Networking.Cilium != nil && b.NodeupConfig.Networking.Cilium.IPAM == kops.CiliumIpamEni)) && c.MaxPods == nil {
 		config, err := awsconfig.LoadDefaultConfig(ctx)

pkg/apis/kops/componentconfig.go

@@ -243,6 +243,8 @@ type KubeletConfigSpec struct {
 	// MemorySwapBehavior defines how swap is used by container workloads.
 	// Supported values: LimitedSwap, "UnlimitedSwap.
 	MemorySwapBehavior string `json:"memorySwapBehavior,omitempty"`
+	// CrashLoopBackOffMaxContainerRestartPeriod is the maximum duration the backoff delay can accrue to for container restarts, minimum 1 second, maximum 300 seconds. If not set, defaults to the internal crashloopbackoff maximum (300s).
+	CrashLoopBackOffMaxContainerRestartPeriod *metav1.Duration `json:"crashLoopBackOffMaxContainerRestartPeriod,omitempty"`
 }
 
 // KubeProxyConfig defines the configuration for a proxy

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -243,6 +243,8 @@ type KubeletConfigSpec struct {
 	// MemorySwapBehavior defines how swap is used by container workloads.
 	// Supported values: LimitedSwap, "UnlimitedSwap.
 	MemorySwapBehavior string `json:"memorySwapBehavior,omitempty"`
+	// CrashLoopBackOffMaxContainerRestartPeriod is the maximum duration the backoff delay can accrue to for container restarts, minimum 1 second, maximum 300 seconds. If not set, defaults to the internal crashloopbackoff maximum (300s).
+	CrashLoopBackOffMaxContainerRestartPeriod *metav1.Duration `json:"crashLoopBackOffMaxContainerRestartPeriod,omitempty"`
 }
 
 // KubeProxyConfig defines the configuration for a proxy

pkg/apis/kops/v1alpha3/componentconfig.go

@@ -241,6 +241,8 @@ type KubeletConfigSpec struct {
 	// MemorySwapBehavior defines how swap is used by container workloads.
 	// Supported values: LimitedSwap, "UnlimitedSwap.
 	MemorySwapBehavior string `json:"memorySwapBehavior,omitempty"`
+	// CrashLoopBackOffMaxContainerRestartPeriod is the maximum duration the backoff delay can accrue to for container restarts, minimum 1 second, maximum 300 seconds. If not set, defaults to the internal crashloopbackoff maximum (300s).
+	CrashLoopBackOffMaxContainerRestartPeriod *metav1.Duration `json:"crashLoopBackOffMaxContainerRestartPeriod,omitempty"`
 }
 
 // KubeProxyConfig defines the configuration for a proxy

pkg/apis/kops/validation/validation.go

@@ -25,6 +25,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws/arn"
 	"github.com/blang/semver/v4"
@@ -966,6 +967,13 @@ func validateKubelet(k *kops.KubeletConfigSpec, c *kops.Cluster, kubeletPath *fi
 			}
 		}
 
+		containerRestartPeriod := k.CrashLoopBackOffMaxContainerRestartPeriod
+		if containerRestartPeriod != nil {
+			if containerRestartPeriod.Duration < time.Second || containerRestartPeriod.Duration > 300*time.Second {
+				allErrs = append(allErrs, field.Invalid(kubeletPath.Child("crashLoopBackOffMaxContainerRestartPeriod"), containerRestartPeriod.String(), "crashLoopBackOffMaxContainerRestartPeriod must be a value between 1s and 300s"))
+			}
+		}
+
 		if k.MemorySwapBehavior != "" {
 			allErrs = append(allErrs, IsValidValue(kubeletPath.Child("memorySwapBehavior"), &k.MemorySwapBehavior, []string{"LimitedSwap", "UnlimitedSwap"})...)
 		}