Skip to content

Commit 6fa7de7

Browse files
bryan-coxclaude
andcommitted
Add zone verification to private cluster E2E test
- Remove separate zone-redundant LB test - Add API server LB zone verification inside AzurePrivateClusterSpec - Verify zones in both AzureCluster spec and Azure infrastructure Updates #5709 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
1 parent 186ea25 commit 6fa7de7

File tree

2 files changed

+56
-149
lines changed

2 files changed

+56
-149
lines changed

test/e2e/azure_privatecluster.go

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ import (
4040
"sigs.k8s.io/cluster-api/util"
4141
"sigs.k8s.io/controller-runtime/pkg/client"
4242

43+
infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
4344
azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure"
4445
)
4546

@@ -202,6 +203,61 @@ func AzurePrivateClusterSpec(ctx context.Context, inputGetter func() AzurePrivat
202203

203204
Expect(err).NotTo(HaveOccurred())
204205
}
206+
207+
// Check that API server load balancer zones are configured correctly.
208+
{
209+
By("verifying the API server load balancer zones are configured correctly")
210+
cred, err := azidentity.NewDefaultAzureCredential(nil)
211+
Expect(err).NotTo(HaveOccurred())
212+
213+
mgmtClient := publicClusterProxy.GetClient()
214+
Expect(mgmtClient).NotTo(BeNil())
215+
216+
azureCluster := &infrav1.AzureCluster{}
217+
err = mgmtClient.Get(ctx, client.ObjectKey{
218+
Namespace: input.Namespace.Name,
219+
Name: clusterName,
220+
}, azureCluster)
221+
Expect(err).NotTo(HaveOccurred())
222+
223+
expectedZones := []string{"1", "2", "3"}
224+
225+
// Verify API Server Load Balancer zones in spec
226+
if azureCluster.Spec.NetworkSpec.APIServerLB != nil {
227+
Expect(azureCluster.Spec.NetworkSpec.APIServerLB.AvailabilityZones).To(Equal(expectedZones),
228+
"APIServerLB should have zones configured in AzureCluster spec")
229+
230+
// Verify zones in Azure
231+
resourceGroupName := azureCluster.Spec.ResourceGroup
232+
Expect(resourceGroupName).NotTo(BeEmpty())
233+
234+
lbClient, err := armnetwork.NewLoadBalancersClient(getSubscriptionID(Default), cred, nil)
235+
Expect(err).NotTo(HaveOccurred())
236+
237+
lbName := azureCluster.Spec.NetworkSpec.APIServerLB.Name
238+
Eventually(func(g Gomega) {
239+
lb, err := lbClient.Get(ctx, resourceGroupName, lbName, nil)
240+
g.Expect(err).NotTo(HaveOccurred())
241+
g.Expect(lb.Properties).NotTo(BeNil())
242+
g.Expect(lb.Properties.FrontendIPConfigurations).NotTo(BeEmpty())
243+
244+
for _, frontendIP := range lb.Properties.FrontendIPConfigurations {
245+
g.Expect(frontendIP.Zones).NotTo(BeNil(), "Frontend IP should have zones configured")
246+
g.Expect(frontendIP.Zones).To(HaveLen(3), "Frontend IP should have 3 zones")
247+
248+
zonesMap := make(map[string]bool)
249+
for _, zone := range frontendIP.Zones {
250+
if zone != nil {
251+
zonesMap[*zone] = true
252+
}
253+
}
254+
for _, expectedZone := range expectedZones {
255+
g.Expect(zonesMap[expectedZone]).To(BeTrue(), "Zone %s should be configured", expectedZone)
256+
}
257+
}
258+
}, retryableOperationTimeout, retryableOperationSleepBetweenRetries).Should(Succeed())
259+
}
260+
}
205261
}
206262

207263
// SetupExistingVNet creates a resource group and a VNet to be used by a workload cluster.

test/e2e/azure_test.go

Lines changed: 0 additions & 149 deletions
Original file line numberDiff line numberDiff line change
@@ -1431,154 +1431,5 @@ var _ = Describe("Workload cluster creation", func() {
14311431
})
14321432
})
14331433

1434-
Context("Creating a cluster with zone-redundant load balancers [OPTIONAL]", func() {
1435-
It("with zone-redundant API server, node outbound, and control plane outbound load balancers", func() {
1436-
clusterName = getClusterName(clusterNamePrefix, "lb-zones")
1437-
1438-
// Set up zone-redundant load balancer configuration
1439-
Expect(os.Setenv("EXP_APISERVER_ILB", "true")).To(Succeed())
1440-
Expect(os.Setenv("AZURE_INTERNAL_LB_PRIVATE_IP", "40.0.0.100")).To(Succeed())
1441-
Expect(os.Setenv("AZURE_VNET_CIDR", "40.0.0.0/8")).To(Succeed())
1442-
Expect(os.Setenv("AZURE_CP_SUBNET_CIDR", "40.0.0.0/16")).To(Succeed())
1443-
Expect(os.Setenv("AZURE_NODE_SUBNET_CIDR", "40.1.0.0/16")).To(Succeed())
1444-
Expect(os.Setenv("AZURE_LB_ZONES", "1,2,3")).To(Succeed())
1445-
1446-
clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput(
1447-
specName,
1448-
withFlavor("apiserver-ilb"),
1449-
withNamespace(namespace.Name),
1450-
withClusterName(clusterName),
1451-
withControlPlaneMachineCount(3),
1452-
withWorkerMachineCount(2),
1453-
withControlPlaneInterval(specName, "wait-control-plane-ha"),
1454-
withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
1455-
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
1456-
}),
1457-
withPostMachinesProvisioned(func() {
1458-
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
1459-
return DaemonsetsSpecInput{
1460-
BootstrapClusterProxy: bootstrapClusterProxy,
1461-
Namespace: namespace,
1462-
ClusterName: clusterName,
1463-
}
1464-
})
1465-
}),
1466-
), result)
1467-
1468-
By("Verifying load balancer zones are configured correctly in Azure", func() {
1469-
expectedZones := []string{"1", "2", "3"}
1470-
1471-
subscriptionID := getSubscriptionID(Default)
1472-
cred, err := azidentity.NewDefaultAzureCredential(nil)
1473-
Expect(err).NotTo(HaveOccurred())
1474-
1475-
mgmtClient := bootstrapClusterProxy.GetClient()
1476-
Expect(mgmtClient).NotTo(BeNil())
1477-
1478-
azureCluster := &infrav1.AzureCluster{}
1479-
err = mgmtClient.Get(ctx, client.ObjectKey{
1480-
Namespace: namespace.Name,
1481-
Name: clusterName,
1482-
}, azureCluster)
1483-
Expect(err).NotTo(HaveOccurred())
1484-
1485-
resourceGroupName := azureCluster.Spec.ResourceGroup
1486-
Expect(resourceGroupName).NotTo(BeEmpty())
1487-
1488-
lbClient, err := armnetwork.NewLoadBalancersClient(subscriptionID, cred, nil)
1489-
Expect(err).NotTo(HaveOccurred())
1490-
1491-
// Verify API Server Load Balancer zones
1492-
if azureCluster.Spec.NetworkSpec.APIServerLB != nil {
1493-
Expect(azureCluster.Spec.NetworkSpec.APIServerLB.AvailabilityZones).To(Equal(expectedZones),
1494-
"APIServerLB should have zones configured in AzureCluster spec")
1495-
1496-
lbName := azureCluster.Spec.NetworkSpec.APIServerLB.Name
1497-
Eventually(func(g Gomega) {
1498-
lb, err := lbClient.Get(ctx, resourceGroupName, lbName, nil)
1499-
g.Expect(err).NotTo(HaveOccurred())
1500-
g.Expect(lb.Properties).NotTo(BeNil())
1501-
g.Expect(lb.Properties.FrontendIPConfigurations).NotTo(BeEmpty())
1502-
1503-
for _, frontendIP := range lb.Properties.FrontendIPConfigurations {
1504-
g.Expect(frontendIP.Zones).NotTo(BeNil(), "Frontend IP should have zones configured")
1505-
g.Expect(frontendIP.Zones).To(HaveLen(3), "Frontend IP should have 3 zones")
1506-
1507-
zonesMap := make(map[string]bool)
1508-
for _, zone := range frontendIP.Zones {
1509-
if zone != nil {
1510-
zonesMap[*zone] = true
1511-
}
1512-
}
1513-
for _, expectedZone := range expectedZones {
1514-
g.Expect(zonesMap[expectedZone]).To(BeTrue(), "Zone %s should be configured", expectedZone)
1515-
}
1516-
}
1517-
}, retryableOperationTimeout, retryableOperationSleepBetweenRetries).Should(Succeed())
1518-
}
1519-
1520-
// Verify Node Outbound Load Balancer zones
1521-
if azureCluster.Spec.NetworkSpec.NodeOutboundLB != nil {
1522-
Expect(azureCluster.Spec.NetworkSpec.NodeOutboundLB.AvailabilityZones).To(Equal(expectedZones),
1523-
"NodeOutboundLB should have zones configured in AzureCluster spec")
1524-
1525-
lbName := azureCluster.Spec.NetworkSpec.NodeOutboundLB.Name
1526-
Eventually(func(g Gomega) {
1527-
lb, err := lbClient.Get(ctx, resourceGroupName, lbName, nil)
1528-
g.Expect(err).NotTo(HaveOccurred())
1529-
g.Expect(lb.Properties).NotTo(BeNil())
1530-
g.Expect(lb.Properties.FrontendIPConfigurations).NotTo(BeEmpty())
1531-
1532-
for _, frontendIP := range lb.Properties.FrontendIPConfigurations {
1533-
g.Expect(frontendIP.Zones).NotTo(BeNil(), "Frontend IP should have zones configured")
1534-
g.Expect(frontendIP.Zones).To(HaveLen(3), "Frontend IP should have 3 zones")
1535-
1536-
zonesMap := make(map[string]bool)
1537-
for _, zone := range frontendIP.Zones {
1538-
if zone != nil {
1539-
zonesMap[*zone] = true
1540-
}
1541-
}
1542-
for _, expectedZone := range expectedZones {
1543-
g.Expect(zonesMap[expectedZone]).To(BeTrue(), "Zone %s should be configured", expectedZone)
1544-
}
1545-
}
1546-
}, retryableOperationTimeout, retryableOperationSleepBetweenRetries).Should(Succeed())
1547-
}
1548-
1549-
// Verify Control Plane Outbound Load Balancer zones
1550-
if azureCluster.Spec.NetworkSpec.ControlPlaneOutboundLB != nil {
1551-
Expect(azureCluster.Spec.NetworkSpec.ControlPlaneOutboundLB.AvailabilityZones).To(Equal(expectedZones),
1552-
"ControlPlaneOutboundLB should have zones configured in AzureCluster spec")
1553-
1554-
lbName := azureCluster.Spec.NetworkSpec.ControlPlaneOutboundLB.Name
1555-
Eventually(func(g Gomega) {
1556-
lb, err := lbClient.Get(ctx, resourceGroupName, lbName, nil)
1557-
g.Expect(err).NotTo(HaveOccurred())
1558-
g.Expect(lb.Properties).NotTo(BeNil())
1559-
g.Expect(lb.Properties.FrontendIPConfigurations).NotTo(BeEmpty())
1560-
1561-
for _, frontendIP := range lb.Properties.FrontendIPConfigurations {
1562-
g.Expect(frontendIP.Zones).NotTo(BeNil(), "Frontend IP should have zones configured")
1563-
g.Expect(frontendIP.Zones).To(HaveLen(3), "Frontend IP should have 3 zones")
1564-
1565-
zonesMap := make(map[string]bool)
1566-
for _, zone := range frontendIP.Zones {
1567-
if zone != nil {
1568-
zonesMap[*zone] = true
1569-
}
1570-
}
1571-
for _, expectedZone := range expectedZones {
1572-
g.Expect(zonesMap[expectedZone]).To(BeTrue(), "Zone %s should be configured", expectedZone)
1573-
}
1574-
}
1575-
}, retryableOperationTimeout, retryableOperationSleepBetweenRetries).Should(Succeed())
1576-
}
1577-
})
1578-
1579-
By("PASSED!")
1580-
})
1581-
})
1582-
15831434
// TODO: add a same test as above for a windows cluster
15841435
})

0 commit comments

Comments
 (0)