kubelet: Wait less for control-plane pods to restart

Author: hakman

k8s/crds/kops.k8s.io_clusters.yaml

@@ -4115,6 +4115,12 @@ spec:
                     description: CpuManagerPolicy allows for changing the default
                       policy of None to static
                     type: string
+                  crashLoopBackOffMaxContainerRestartPeriod:
+                    description: CrashLoopBackOffMaxContainerRestartPeriod is the
+                      maximum duration the backoff delay can accrue to for container
+                      restarts, minimum 1 second, maximum 300 seconds. If not set,
+                      defaults to the internal crashloopbackoff maximum (300s).
+                    type: string
                   dockerDisableSharedPID:
                     description: DockerDisableSharedPID was removed.
                     type: boolean
@@ -4568,6 +4574,12 @@ spec:
                     description: CpuManagerPolicy allows for changing the default
                       policy of None to static
                     type: string
+                  crashLoopBackOffMaxContainerRestartPeriod:
+                    description: CrashLoopBackOffMaxContainerRestartPeriod is the
+                      maximum duration the backoff delay can accrue to for container
+                      restarts, minimum 1 second, maximum 300 seconds. If not set,
+                      defaults to the internal crashloopbackoff maximum (300s).
+                    type: string
                   dockerDisableSharedPID:
                     description: DockerDisableSharedPID was removed.
                     type: boolean

k8s/crds/kops.k8s.io_instancegroups.yaml

@@ -506,6 +506,12 @@ spec:
                     description: CpuManagerPolicy allows for changing the default
                       policy of None to static
                     type: string
+                  crashLoopBackOffMaxContainerRestartPeriod:
+                    description: CrashLoopBackOffMaxContainerRestartPeriod is the
+                      maximum duration the backoff delay can accrue to for container
+                      restarts, minimum 1 second, maximum 300 seconds. If not set,
+                      defaults to the internal crashloopbackoff maximum (300s).
+                    type: string
                   dockerDisableSharedPID:
                     description: DockerDisableSharedPID was removed.
                     type: boolean

nodeup/pkg/model/kubelet.go

@@ -26,10 +26,12 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/klog/v2"
@@ -242,6 +244,9 @@ func buildKubeletComponentConfig(kubeletConfig *kops.KubeletConfigSpec, provider
 	if providerID != "" {
 		componentConfig.ProviderID = providerID
 	}
+	componentConfig.CrashLoopBackOff = kubelet.CrashLoopBackOffConfig{
+		MaxContainerRestartPeriod: kubeletConfig.CrashLoopBackOffMaxContainerRestartPeriod,
+	}
 	if kubeletConfig.ShutdownGracePeriod != nil {
 		componentConfig.ShutdownGracePeriod = *kubeletConfig.ShutdownGracePeriod
 	}
@@ -648,6 +653,11 @@ func (b *KubeletBuilder) buildKubeletConfigSpec(ctx context.Context) (*kops.Kube
 
 	c.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt")
 
+	// Wait less for pods to restart, especially during the bootstrap sequence
+	if b.IsMaster && c.CrashLoopBackOffMaxContainerRestartPeriod == nil {
+		c.CrashLoopBackOffMaxContainerRestartPeriod = &metav1.Duration{Duration: time.Minute}
+	}
+
 	// Respect any MaxPods value the user sets explicitly.
 	if (b.NodeupConfig.Networking.AmazonVPC != nil || (b.NodeupConfig.Networking.Cilium != nil && b.NodeupConfig.Networking.Cilium.IPAM == kops.CiliumIpamEni)) && c.MaxPods == nil {
 		config, err := awsconfig.LoadDefaultConfig(ctx)

pkg/apis/kops/componentconfig.go

@@ -243,6 +243,8 @@ type KubeletConfigSpec struct {
 	// MemorySwapBehavior defines how swap is used by container workloads.
 	// Supported values: LimitedSwap, "UnlimitedSwap.
 	MemorySwapBehavior string `json:"memorySwapBehavior,omitempty"`
+	// CrashLoopBackOffMaxContainerRestartPeriod is the maximum duration the backoff delay can accrue to for container restarts, minimum 1 second, maximum 300 seconds. If not set, defaults to the internal crashloopbackoff maximum (300s).
+	CrashLoopBackOffMaxContainerRestartPeriod *metav1.Duration `json:"crashLoopBackOffMaxContainerRestartPeriod,omitempty"`
 }
 
 // KubeProxyConfig defines the configuration for a proxy

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -243,6 +243,8 @@ type KubeletConfigSpec struct {
 	// MemorySwapBehavior defines how swap is used by container workloads.
 	// Supported values: LimitedSwap, "UnlimitedSwap.
 	MemorySwapBehavior string `json:"memorySwapBehavior,omitempty"`
+	// CrashLoopBackOffMaxContainerRestartPeriod is the maximum duration the backoff delay can accrue to for container restarts, minimum 1 second, maximum 300 seconds. If not set, defaults to the internal crashloopbackoff maximum (300s).
+	CrashLoopBackOffMaxContainerRestartPeriod *metav1.Duration `json:"crashLoopBackOffMaxContainerRestartPeriod,omitempty"`
 }
 
 // KubeProxyConfig defines the configuration for a proxy

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -241,6 +241,8 @@ type KubeletConfigSpec struct {
 	// MemorySwapBehavior defines how swap is used by container workloads.
 	// Supported values: LimitedSwap, "UnlimitedSwap.
 	MemorySwapBehavior string `json:"memorySwapBehavior,omitempty"`
+	// CrashLoopBackOffMaxContainerRestartPeriod is the maximum duration the backoff delay can accrue to for container restarts, minimum 1 second, maximum 300 seconds. If not set, defaults to the internal crashloopbackoff maximum (300s).
+	CrashLoopBackOffMaxContainerRestartPeriod *metav1.Duration `json:"crashLoopBackOffMaxContainerRestartPeriod,omitempty"`
 }
 
 // KubeProxyConfig defines the configuration for a proxy