Skip to content

Commit 4eade90

Browse files
phuhung273damdofaiq
authored
✨ Support EKS upgrade policy (#5471)
* Support EKS upgrade policy * Apply suggestions from code review Co-authored-by: Damiano Donati <damiano.donati@gmail.com> * regenerate * Apply suggestions from code review Co-authored-by: Damiano Donati <damiano.donati@gmail.com> * Update pkg/cloud/services/eks/cluster.go Co-authored-by: Damiano Donati <damiano.donati@gmail.com> * remove log * Update config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml Co-authored-by: Faiq <faiq.raza@nutanix.com> * docstring typo * set NotReady if cluster was automatically upgraded * Update pkg/cloud/services/eks/cluster.go Co-authored-by: Faiq <faiq.raza@nutanix.com> * fix version compare logic * e2e * Apply suggestions from code review Co-authored-by: Damiano Donati <damiano.donati@gmail.com> * syntax * WaitForEKSClusterUpgradePolicy fail early on NotFound --------- Co-authored-by: Damiano Donati <damiano.donati@gmail.com> Co-authored-by: Faiq <faiq.raza@nutanix.com>
1 parent 86e15f7 commit 4eade90

File tree

15 files changed

+400
-18
lines changed

15 files changed

+400
-18
lines changed

‎config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3243,6 +3243,17 @@ spec:
32433243
- iam-authenticator
32443244
- aws-cli
32453245
type: string
3246+
upgradePolicy:
3247+
description: |-
3248+
The cluster upgrade policy to use for the cluster.
3249+
(Official AWS docs for this policy: https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html)
3250+
`extended` upgrade policy indicates that the cluster will enter into extended support once the Kubernetes version reaches end of standard support. You will incur extended support charges with this setting. You can upgrade your cluster to a standard supported Kubernetes version to stop incurring extended support charges.
3251+
`standard` upgrade policy indicates that the cluster is eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but your EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support.
3252+
If omitted, new clusters will use the AWS default upgrade policy (which at the time of writing is "extended") and existing clusters will have their upgrade policy unchanged.
3253+
enum:
3254+
- extended
3255+
- standard
3256+
type: string
32463257
version:
32473258
description: |-
32483259
Version defines the desired Kubernetes version. If no version number

‎config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1044,6 +1044,17 @@ spec:
10441044
- iam-authenticator
10451045
- aws-cli
10461046
type: string
1047+
upgradePolicy:
1048+
description: |-
1049+
The cluster upgrade policy to use for the cluster.
1050+
(Official AWS docs for this policy: https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html)
1051+
`extended` upgrade policy indicates that the cluster will enter into extended support once the Kubernetes version reaches end of standard support. You will incur extended support charges with this setting. You can upgrade your cluster to a standard supported Kubernetes version to stop incurring extended support charges.
1052+
`standard` upgrade policy indicates that the cluster is eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but your EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support.
1053+
If omitted, new clusters will use the AWS default upgrade policy (which at the time of writing is "extended") and existing clusters will have their upgrade policy unchanged.
1054+
enum:
1055+
- extended
1056+
- standard
1057+
type: string
10471058
version:
10481059
description: |-
10491060
Version defines the desired Kubernetes version. If no version number

‎controlplane/eks/api/v1beta1/conversion.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ func (r *AWSManagedControlPlane) ConvertTo(dstRaw conversion.Hub) error {
122122
dst.Spec.RolePermissionsBoundary = restored.Spec.RolePermissionsBoundary
123123
dst.Status.Version = restored.Status.Version
124124
dst.Spec.BootstrapSelfManagedAddons = restored.Spec.BootstrapSelfManagedAddons
125+
dst.Spec.UpgradePolicy = restored.Spec.UpgradePolicy
125126
return nil
126127
}
127128

‎controlplane/eks/api/v1beta1/zz_generated.conversion.go‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,15 @@ type AWSManagedControlPlaneSpec struct { //nolint: maligned
212212

213213
// KubeProxy defines managed attributes of the kube-proxy daemonset
214214
KubeProxy KubeProxy `json:"kubeProxy,omitempty"`
215+
216+
// The cluster upgrade policy to use for the cluster.
217+
// (Official AWS docs for this policy: https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html)
218+
// `extended` upgrade policy indicates that the cluster will enter into extended support once the Kubernetes version reaches end of standard support. You will incur extended support charges with this setting. You can upgrade your cluster to a standard supported Kubernetes version to stop incurring extended support charges.
219+
// `standard` upgrade policy indicates that the cluster is eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but your EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support.
220+
// If omitted, new clusters will use the AWS default upgrade policy (which at the time of writing is "extended") and existing clusters will have their upgrade policy unchanged.
221+
// +kubebuilder:validation:Enum=extended;standard
222+
// +optional
223+
UpgradePolicy UpgradePolicy `json:"upgradePolicy,omitempty"`
215224
}
216225

217226
// KubeProxy specifies how the kube-proxy daemonset is managed.

‎controlplane/eks/api/v1beta2/types.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,24 @@ type AddonIssue struct {
241241
ResourceIDs []string `json:"resourceIds,omitempty"`
242242
}
243243

244+
// UpgradePolicy defines the support policy to use for the cluster.
245+
type UpgradePolicy string
246+
247+
var (
248+
// UpgradePolicyExtended indicates that the cluster will enter into extended support once the Kubernetes version reaches end of standard support.
249+
// You will incur extended support charges with this setting.
250+
// You can upgrade your cluster to a standard supported Kubernetes version to stop incurring extended support charges.
251+
UpgradePolicyExtended = UpgradePolicy("extended")
252+
253+
// UpgradePolicyStandard indicates that the cluster is eligible for automatic upgrade at the end of standard support.
254+
// You will not incur extended support charges with this setting but your EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support.
255+
UpgradePolicyStandard = UpgradePolicy("standard")
256+
)
257+
258+
func (e UpgradePolicy) String() string {
259+
return string(e)
260+
}
261+
244262
const (
245263
// SecurityGroupCluster is the security group for communication between EKS
246264
// control plane and managed node groups.

‎docs/book/src/topics/eks/creating-a-cluster.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ clusterctl generate cluster capi-eks-quickstart --flavor eks-managedmachinepool
1414

1515
NOTE: When creating an EKS cluster only the **MAJOR.MINOR** of the `-kubernetes-version` is taken into consideration.
1616

17+
By default CAPA relies on the default EKS cluster upgrade policy, which at the moment of writing is EXTENDED support.
18+
See more info about [cluster upgrade policy](https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html)
19+
1720
## Kubeconfig
1821

1922
When creating an EKS cluster 2 kubeconfigs are generated and stored as secrets in the management cluster. This is different to when you create a non-managed cluster using the AWS provider.

‎pkg/cloud/converters/eks.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -278,3 +278,11 @@ func AddonConflictResolutionFromSDK(conflict ekstypes.ResolveConflicts) *string
278278
}
279279
return aws.String(string(ekscontrolplanev1.AddonResolutionOverwrite))
280280
}
281+
282+
// SupportTypeToSDK converts CAPA upgrade support policy types to SDK types.
283+
func SupportTypeToSDK(input ekscontrolplanev1.UpgradePolicy) ekstypes.SupportType {
284+
if input == ekscontrolplanev1.UpgradePolicyStandard {
285+
return ekstypes.SupportTypeStandard
286+
}
287+
return ekstypes.SupportTypeExtended
288+
}

‎pkg/cloud/services/eks/cluster.go‎

Lines changed: 69 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"fmt"
2222
"net"
23+
"strings"
2324
"time"
2425

2526
"github.com/aws/aws-sdk-go-v2/aws"
@@ -35,6 +36,7 @@ import (
3536
infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
3637
ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2"
3738
"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/awserrors"
39+
"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/converters"
3840
"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/wait"
3941
"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/internal/cidr"
4042
"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/internal/cmp"
@@ -147,23 +149,7 @@ func (s *Service) reconcileCluster(ctx context.Context) error {
147149
// computeCurrentStatusVersion returns the computed current EKS cluster kubernetes version.
148150
// The computation has awareness of the fact that EKS clusters only return a major.minor kubernetes version,
149151
// and returns a compatible version for te status according to the one the user specified in the spec.
150-
func computeCurrentStatusVersion(specV *string, clusterV *string) *string {
151-
specVersion := ""
152-
if specV != nil {
153-
specVersion = *specV
154-
}
155-
156-
clusterVersion := ""
157-
if clusterV != nil {
158-
clusterVersion = *clusterV
159-
}
160-
161-
// Ignore parsing errors as these are already validated by the kubebuilder validation and the AWS API.
162-
// Also specVersion might not be specified in the spec.Version for AWSManagedControlPlane, this results in a "0.0.0" version.
163-
// Also clusterVersion might not yet be returned by the AWS EKS API, as the cluster might still be initializing, this results in a "0.0.0" version.
164-
specSemverVersion, _ := semver.ParseTolerant(specVersion)
165-
currentSemverVersion, _ := semver.ParseTolerant(clusterVersion)
166-
152+
func computeCurrentStatusVersion(clusterV *string, specSemverVersion semver.Version, currentSemverVersion semver.Version) *string {
167153
// If AWS EKS API is not returning a version, set the status.Version to empty string.
168154
if currentSemverVersion.String() == "0.0.0" {
169155
return ptr.To("")
@@ -187,9 +173,27 @@ func computeCurrentStatusVersion(specV *string, clusterV *string) *string {
187173
return clusterV
188174
}
189175

176+
// parseClusterVersionString parse a version string to semver version.
177+
// If the string cannot be parsed to semver, returning 0.0.0.
178+
func parseClusterVersionString(str *string) semver.Version {
179+
version := ""
180+
if str != nil {
181+
version = *str
182+
}
183+
184+
// Ignore parsing errors as these are already validated by the kubebuilder validation and the AWS API.
185+
semverVersion, _ := semver.ParseTolerant(version)
186+
return semverVersion
187+
}
188+
190189
func (s *Service) setStatus(cluster *ekstypes.Cluster) error {
190+
// specSemver might not be specified in the spec.Version for AWSManagedControlPlane, this results in a "0.0.0" version.
191+
specSemver := parseClusterVersionString(s.scope.ControlPlane.Spec.Version)
192+
// clusterSemver might not yet be returned by the AWS EKS API, as the cluster might still be initializing, this results in a "0.0.0" version.
193+
clusterSemver := parseClusterVersionString(cluster.Version)
194+
191195
// Set the current Kubernetes control plane version in the status.
192-
s.scope.ControlPlane.Status.Version = computeCurrentStatusVersion(s.scope.ControlPlane.Spec.Version, cluster.Version)
196+
s.scope.ControlPlane.Status.Version = computeCurrentStatusVersion(cluster.Version, specSemver, clusterSemver)
193197

194198
// Set the current cluster status in the control plane status.
195199
switch cluster.Status {
@@ -211,6 +215,19 @@ func (s *Service) setStatus(cluster *ekstypes.Cluster) error {
211215
conditions.MarkFalse(s.scope.ControlPlane, ekscontrolplanev1.EKSControlPlaneUpdatingCondition, "updated", clusterv1.ConditionSeverityInfo, "")
212216
record.Eventf(s.scope.ControlPlane, "SuccessfulUpdateEKSControlPlane", "Updated EKS control plane %s", s.scope.KubernetesClusterName())
213217
}
218+
if s.scope.ControlPlane.Spec.UpgradePolicy == ekscontrolplanev1.UpgradePolicyStandard &&
219+
(specSemver.Major < clusterSemver.Major ||
220+
(specSemver.Major == clusterSemver.Major && specSemver.Minor < clusterSemver.Minor)) {
221+
s.scope.ControlPlane.Status.Ready = false
222+
failureMsg := fmt.Sprintf(
223+
"EKS control plane %s was automatically upgraded to version %s because %s is out of standard support. "+
224+
"This can be fixed by changing to the version of the AWSManagedControlPlane to the one reported in the status",
225+
s.scope.KubernetesClusterName(),
226+
clusterSemver.String(),
227+
specSemver.String(),
228+
)
229+
s.scope.ControlPlane.Status.FailureMessage = &failureMsg
230+
}
214231
// TODO FailureReason
215232
case ekstypes.ClusterStatusCreating:
216233
s.scope.ControlPlane.Status.Ready = false
@@ -478,6 +495,14 @@ func (s *Service) createCluster(ctx context.Context, eksClusterName string) (*ek
478495
eksVersion = &v
479496
}
480497

498+
var upgradePolicy *ekstypes.UpgradePolicyRequest
499+
500+
if s.scope.ControlPlane.Spec.UpgradePolicy != "" {
501+
upgradePolicy = &ekstypes.UpgradePolicyRequest{
502+
SupportType: converters.SupportTypeToSDK(s.scope.ControlPlane.Spec.UpgradePolicy),
503+
}
504+
}
505+
481506
bootstrapAddon := s.scope.BootstrapSelfManagedAddons()
482507
input := &eks.CreateClusterInput{
483508
Name: aws.String(eksClusterName),
@@ -490,6 +515,7 @@ func (s *Service) createCluster(ctx context.Context, eksClusterName string) (*ek
490515
Tags: tags,
491516
KubernetesNetworkConfig: netConfig,
492517
BootstrapSelfManagedAddons: bootstrapAddon,
518+
UpgradePolicy: upgradePolicy,
493519
}
494520

495521
var out *eks.CreateClusterOutput
@@ -545,6 +571,11 @@ func (s *Service) reconcileClusterConfig(ctx context.Context, cluster *ekstypes.
545571
input.ResourcesVpcConfig = updateVpcConfig
546572
}
547573

574+
if updateUpgradePolicy := s.reconcileUpgradePolicy(cluster.UpgradePolicy); updateUpgradePolicy != nil {
575+
needsUpdate = true
576+
input.UpgradePolicy = updateUpgradePolicy
577+
}
578+
548579
if needsUpdate {
549580
if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) {
550581
if _, err := s.EKSClient.UpdateClusterConfig(ctx, input); err != nil {
@@ -782,6 +813,26 @@ func (s *Service) reconcileClusterVersion(ctx context.Context, cluster *ekstypes
782813
return nil
783814
}
784815

816+
func (s *Service) reconcileUpgradePolicy(upgradePolicy *ekstypes.UpgradePolicyResponse) *ekstypes.UpgradePolicyRequest {
817+
// Should not update when cluster upgrade policy is unknown
818+
if upgradePolicy == nil {
819+
return nil
820+
}
821+
822+
// Cluster stay unchanged when upgrade policy omitted
823+
if s.scope.ControlPlane.Spec.UpgradePolicy == "" {
824+
return nil
825+
}
826+
827+
if strings.ToLower(string(upgradePolicy.SupportType)) == s.scope.ControlPlane.Spec.UpgradePolicy.String() {
828+
return nil
829+
}
830+
831+
return &ekstypes.UpgradePolicyRequest{
832+
SupportType: converters.SupportTypeToSDK(s.scope.ControlPlane.Spec.UpgradePolicy),
833+
}
834+
}
835+
785836
func (s *Service) describeEKSCluster(ctx context.Context, eksClusterName string) (*ekstypes.Cluster, error) {
786837
input := &eks.DescribeClusterInput{
787838
Name: aws.String(eksClusterName),

‎pkg/cloud/services/eks/cluster_test.go‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -652,6 +652,7 @@ func TestCreateCluster(t *testing.T) {
652652
RoleName: tc.role,
653653
NetworkSpec: infrav1.NetworkSpec{Subnets: tc.subnets},
654654
BootstrapSelfManagedAddons: false,
655+
UpgradePolicy: ekscontrolplanev1.UpgradePolicyStandard,
655656
},
656657
},
657658
})
@@ -674,6 +675,9 @@ func TestCreateCluster(t *testing.T) {
674675
Tags: tc.tags,
675676
Version: version,
676677
BootstrapSelfManagedAddons: aws.Bool(false),
678+
UpgradePolicy: &ekstypes.UpgradePolicyRequest{
679+
SupportType: ekstypes.SupportTypeStandard,
680+
},
677681
}).Return(&eks.CreateClusterOutput{}, nil)
678682
}
679683
s := NewService(scope)
@@ -805,6 +809,91 @@ func TestReconcileEKSEncryptionConfig(t *testing.T) {
805809
}
806810
}
807811

812+
func TestReconcileUpgradePolicy(t *testing.T) {
813+
clusterName := "default.cluster"
814+
tests := []struct {
815+
name string
816+
oldUpgradePolicy *ekstypes.UpgradePolicyResponse
817+
newUpgradePolicy ekscontrolplanev1.UpgradePolicy
818+
expect *ekstypes.UpgradePolicyRequest
819+
expectError bool
820+
}{
821+
{
822+
name: "no update necessary - upgrade policy omitted",
823+
oldUpgradePolicy: &ekstypes.UpgradePolicyResponse{
824+
SupportType: ekstypes.SupportTypeStandard,
825+
},
826+
expect: nil,
827+
expectError: false,
828+
},
829+
{
830+
name: "no update necessary - cannot get cluster upgrade policy",
831+
newUpgradePolicy: ekscontrolplanev1.UpgradePolicyStandard,
832+
expect: nil,
833+
expectError: false,
834+
},
835+
{
836+
name: "no update necessary - upgrade policy unchanged",
837+
oldUpgradePolicy: &ekstypes.UpgradePolicyResponse{
838+
SupportType: ekstypes.SupportTypeStandard,
839+
},
840+
newUpgradePolicy: ekscontrolplanev1.UpgradePolicyStandard,
841+
expect: nil,
842+
expectError: false,
843+
},
844+
{
845+
name: "needs update",
846+
oldUpgradePolicy: &ekstypes.UpgradePolicyResponse{
847+
SupportType: ekstypes.SupportTypeStandard,
848+
},
849+
newUpgradePolicy: ekscontrolplanev1.UpgradePolicyExtended,
850+
expect: &ekstypes.UpgradePolicyRequest{
851+
SupportType: ekstypes.SupportTypeExtended,
852+
},
853+
expectError: false,
854+
},
855+
}
856+
857+
for _, tc := range tests {
858+
t.Run(tc.name, func(t *testing.T) {
859+
g := NewWithT(t)
860+
861+
mockControl := gomock.NewController(t)
862+
defer mockControl.Finish()
863+
864+
scheme := runtime.NewScheme()
865+
_ = infrav1.AddToScheme(scheme)
866+
_ = ekscontrolplanev1.AddToScheme(scheme)
867+
client := fake.NewClientBuilder().WithScheme(scheme).Build()
868+
scope, err := scope.NewManagedControlPlaneScope(scope.ManagedControlPlaneScopeParams{
869+
Client: client,
870+
Cluster: &clusterv1.Cluster{
871+
ObjectMeta: metav1.ObjectMeta{
872+
Namespace: "ns",
873+
Name: clusterName,
874+
},
875+
},
876+
ControlPlane: &ekscontrolplanev1.AWSManagedControlPlane{
877+
Spec: ekscontrolplanev1.AWSManagedControlPlaneSpec{
878+
Version: aws.String("1.16"),
879+
UpgradePolicy: tc.newUpgradePolicy,
880+
},
881+
},
882+
})
883+
g.Expect(err).To(BeNil())
884+
885+
s := NewService(scope)
886+
887+
upgradePolicyRequest := s.reconcileUpgradePolicy(tc.oldUpgradePolicy)
888+
if tc.expectError {
889+
g.Expect(err).To(HaveOccurred())
890+
return
891+
}
892+
g.Expect(upgradePolicyRequest).To(Equal(tc.expect))
893+
})
894+
}
895+
}
896+
808897
func TestCreateIPv6Cluster(t *testing.T) {
809898
g := NewWithT(t)
810899

0 commit comments

Comments
 (0)