feat: Implement ZoneAware loadbalancing - ZoneAware Lb Config (splitup #6482)

internal/gatewayapi/route.go

@@ -1426,7 +1426,7 @@ func (t *Translator) processDestination(name string, backendRefContext BackendRe
 		ds = t.processServiceDestinationSetting(name, backendRef.BackendObjectReference, backendNamespace, protocol, resources, envoyProxy)
 		svc := resources.GetService(backendNamespace, string(backendRef.Name))
 		ds.IPFamily = getServiceIPFamily(svc)
-		ds.ZoneAwareRouting = processZoneAwareRouting(svc)
+		ds.PreferLocal = processPreferLocalZone(svc)
 
 	case egv1a1.KindBackend:
 		ds = t.processBackendDestinationSetting(name, backendRef.BackendObjectReference, backendNamespace, protocol, resources)
@@ -1599,12 +1599,12 @@ func (t *Translator) processServiceDestinationSetting(
 	}
 
 	return &ir.DestinationSetting{
-		Name:             name,
-		Protocol:         protocol,
-		Endpoints:        endpoints,
-		AddressType:      addrType,
-		ZoneAwareRouting: processZoneAwareRouting(service),
-		Metadata:         buildResourceMetadata(service, ptr.To(gwapiv1.SectionName(strconv.Itoa(int(*backendRef.Port))))),
+		Name:        name,
+		Protocol:    protocol,
+		Endpoints:   endpoints,
+		AddressType: addrType,
+		PreferLocal: processPreferLocalZone(service),
+		Metadata:    buildResourceMetadata(service, ptr.To(gwapiv1.SectionName(strconv.Itoa(int(*backendRef.Port))))),
 	}
 }
 
@@ -1624,14 +1624,17 @@ func getBackendFilters(routeType gwapiv1.Kind, backendRefContext BackendRefConte
 	return nil
 }
 
-func processZoneAwareRouting(svc *corev1.Service) *ir.ZoneAwareRouting {
+func processPreferLocalZone(svc *corev1.Service) *ir.PreferLocalZone {
 	if svc == nil {
 		return nil
 	}
 
 	if trafficDist := svc.Spec.TrafficDistribution; trafficDist != nil {
-		return &ir.ZoneAwareRouting{
-			MinSize: 1,
+		return &ir.PreferLocalZone{
+			MinEndpointsThreshold: ptr.To[uint64](1),
+			Force: &ir.ForceLocalZone{
+				MinEndpointsInZoneThreshold: ptr.To[uint32](1),
+			},
 		}
 	}
 
@@ -1640,8 +1643,11 @@ func processZoneAwareRouting(svc *corev1.Service) *ir.ZoneAwareRouting {
 	// https://kubernetes.io/docs/concepts/services-networking/topology-aware-routing/#enabling-topology-aware-routing
 	// https://github.com/kubernetes/kubernetes/blob/9d9e1afdf78bce0a517cc22557457f942040ca19/staging/src/k8s.io/endpointslice/utils.go#L355-L368
 	if val, ok := svc.Annotations[corev1.AnnotationTopologyMode]; ok && val == "Auto" || val == "auto" {
-		return &ir.ZoneAwareRouting{
-			MinSize: 3,
+		return &ir.PreferLocalZone{
+			MinEndpointsThreshold: ptr.To[uint64](3),
+			Force: &ir.ForceLocalZone{
+				MinEndpointsInZoneThreshold: ptr.To[uint32](3),
+			},
 		}
 	}
 

internal/gatewayapi/testdata/httproute-with-enable-zone-discovery.out.yaml

@@ -157,10 +157,12 @@ xdsIR:
               namespace: default
               sectionName: "8080"
             name: httproute/default/httproute-1/rule/1/backend/0
+            preferLocal:
+              force:
+                minEndpointsInZoneThreshold: 1
+              minEndpointsThreshold: 1
             protocol: HTTP
             weight: 1
-            zoneAwareRouting:
-              minSize: 1
         headerMatches:
         - distinct: false
           exact: bar
@@ -195,10 +197,12 @@ xdsIR:
               namespace: default
               sectionName: "8080"
             name: httproute/default/httproute-1/rule/0/backend/0
+            preferLocal:
+              force:
+                minEndpointsInZoneThreshold: 1
+              minEndpointsThreshold: 1
             protocol: HTTP
             weight: 1
-            zoneAwareRouting:
-              minSize: 1
         hostname: '*'
         isHTTP2: false
         metadata:

internal/ir/xds.go

@@ -832,6 +832,15 @@ func (h *HTTPRoute) GetRetry() *Retry {
 	return nil
 }
 
+func (h *HTTPRoute) NeedsClusterPerSetting() bool {
+	if h.Traffic != nil &&
+		h.Traffic.LoadBalancer != nil &&
+		h.Traffic.LoadBalancer.PreferLocal != nil {
+		return true
+	}
+	return h.Destination.NeedsClusterPerSetting()
+}
+
 // DNS contains configuration options for DNS resolution.
 // +k8s:deepcopy-gen=true
 type DNS struct {
@@ -1582,7 +1591,7 @@ func (r *RouteDestination) Validate() error {
 func (r *RouteDestination) NeedsClusterPerSetting() bool {
 	return r.HasMixedEndpoints() ||
 		r.HasFiltersInSettings() ||
-		(len(r.Settings) > 1 && r.HasZoneAwareRouting())
+		(len(r.Settings) > 1 && r.HasPreferLocalZone())
 }
 
 // HasMixedEndpoints returns true if the RouteDestination has endpoints of multiple types
@@ -1607,10 +1616,10 @@ func (r *RouteDestination) HasFiltersInSettings() bool {
 	return false
 }
 
-// HasZoneAwareRouting returns true if any setting in the destination has ZoneAwareRoutingEnabled set
-func (r *RouteDestination) HasZoneAwareRouting() bool {
+// HasPreferLocalZone returns true if any setting in the destination has PreferLocalZone set
+func (r *RouteDestination) HasPreferLocalZone() bool {
 	for _, setting := range r.Settings {
-		if setting.ZoneAwareRouting != nil {
+		if setting.PreferLocal != nil {
 			return true
 		}
 	}
@@ -1673,11 +1682,9 @@ type DestinationSetting struct {
 	IPFamily *egv1a1.IPFamily    `json:"ipFamily,omitempty" yaml:"ipFamily,omitempty"`
 	TLS      *TLSUpstreamConfig  `json:"tls,omitempty" yaml:"tls,omitempty"`
 	Filters  *DestinationFilters `json:"filters,omitempty" yaml:"filters,omitempty"`
-	// ZoneAwareRouting specifies whether to enable Zone Aware Routing for this destination's endpoints.
+	// PreferLocal specifies whether to enable Zone Aware Routing for this destination's endpoints.
 	// This is derived from the backend service and depends on having Kubernetes Topology Aware Routing or Traffic Distribution enabled.
-	//
-	// +optional
-	ZoneAwareRouting *ZoneAwareRouting `json:"zoneAwareRouting,omitempty" yaml:"zoneAwareRouting,omitempty"`
+	PreferLocal *PreferLocalZone `json:"preferLocal,omitempty" yaml:"preferLocal,omitempty"`
 	// Metadata is used to enrich envoy route metadata with user and provider-specific information
 	// The primary metadata for DestinationSettings comes from the Backend resource reference in BackendRef
 	Metadata *ResourceMetadata `json:"metadata,omitempty" yaml:"metadata,omitempty"`
@@ -2523,6 +2530,8 @@ type LoadBalancer struct {
 	Random *Random `json:"random,omitempty" yaml:"random,omitempty"`
 	// ConsistentHash load balancer policy
 	ConsistentHash *ConsistentHash `json:"consistentHash,omitempty" yaml:"consistentHash,omitempty"`
+	// PreferLocal defines the configuration related to the distribution of requests between locality zones.
+	PreferLocal *PreferLocalZone `json:"preferLocal,omitempty" yaml:"preferLocal,omitempty"`
 }
 
 // Validate the fields within the LoadBalancer structure
@@ -3203,8 +3212,21 @@ type RequestBuffer struct {
 	Limit resource.Quantity `json:"limit" yaml:"limit"`
 }
 
-// ZoneAwareRouting holds the zone aware routing configuration
+// PreferLocalZone configures zone-aware routing to prefer sending traffic to the local locality zone.
+// +k8s:deepcopy-gen=true
+type PreferLocalZone struct {
+	// ForceLocalZone defines override configuration for forcing all traffic to stay within the local zone instead of the default behavior
+	// which maintains equal distribution among upstream endpoints while sending as much traffic as possible locally.
+	Force *ForceLocalZone `json:"force,omitempty" yaml:"force,omitempty"`
+	// MinEndpointsThreshold is the minimum number of total upstream endpoints across all zones required to enable zone-aware routing.
+	MinEndpointsThreshold *uint64 `json:"minEndpointsThreshold,omitempty" yaml:"minEndpointsThreshold,omitempty"`
+}
+
+// ForceLocalZone defines override configuration for forcing all traffic to stay within the local zone instead of the default behavior
+// which maintains equal distribution among upstream endpoints while sending as much traffic as possible locally.
 // +k8s:deepcopy-gen=true
-type ZoneAwareRouting struct {
-	MinSize int `json:"minSize" yaml:"minSize"`
+type ForceLocalZone struct {
+	// MinEndpointsInZoneThreshold is the minimum number of upstream endpoints in the local zone required to honor the forceLocalZone
+	// override. This is useful for protecting zones with fewer endpoints.
+	MinEndpointsInZoneThreshold *uint32 `json:"minEndpointsInZoneThreshold,omitempty" yaml:"minEndpointsInZoneThreshold,omitempty"`
 }

internal/ir/xds_test.go

@@ -1125,8 +1125,10 @@ func TestRouteDestination_NeedsClusterPerSetting(t *testing.T) {
 								Port: 8080,
 							},
 						},
-						AddressType:      ptr.To(FQDN),
-						ZoneAwareRouting: &ZoneAwareRouting{MinSize: 1},
+						AddressType: ptr.To(FQDN),
+						PreferLocal: &PreferLocalZone{
+							Force: &ForceLocalZone{MinEndpointsInZoneThreshold: ptr.To[uint32](1)},
+						},
 					},
 					{
 						Endpoints: []*DestinationEndpoint{
@@ -1153,8 +1155,10 @@ func TestRouteDestination_NeedsClusterPerSetting(t *testing.T) {
 								Port: 8080,
 							},
 						},
-						AddressType:      ptr.To(FQDN),
-						ZoneAwareRouting: &ZoneAwareRouting{MinSize: 1},
+						AddressType: ptr.To(FQDN),
+						PreferLocal: &PreferLocalZone{
+							Force: &ForceLocalZone{MinEndpointsInZoneThreshold: ptr.To[uint32](1)},
+						},
 					},
 				},
 			},

internal/xds/translator/cluster.go

@@ -228,25 +228,7 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) {
 		cluster.TypedExtensionProtocolOptions = epo
 	}
 
-	// Set default localityLbConfig
-	localityLbConfig := &commonv3.LocalityLbConfig{
-		LocalityConfigSpecifier: &commonv3.LocalityLbConfig_LocalityWeightedLbConfig_{
-			LocalityWeightedLbConfig: &commonv3.LocalityLbConfig_LocalityWeightedLbConfig{},
-		},
-	}
-
-	// Override LocalityWeightedLbConfig if zone aware routing is enabled.
-	// Zone aware enabled backendRefs always have a single DestinationSetting per-cluster.
-	if len(args.settings) == 1 && args.settings[0].ZoneAwareRouting != nil {
-		localityLbConfig.LocalityConfigSpecifier = &commonv3.LocalityLbConfig_ZoneAwareLbConfig_{
-			ZoneAwareLbConfig: &commonv3.LocalityLbConfig_ZoneAwareLbConfig{
-				MinClusterSize: wrapperspb.UInt64(1),
-				ForceLocalZone: &commonv3.LocalityLbConfig_ZoneAwareLbConfig_ForceLocalZone{
-					MinSize: wrapperspb.UInt32(uint32(args.settings[0].ZoneAwareRouting.MinSize)),
-				},
-			},
-		}
-	}
+	localityLbConfig := buildLocalityLbConfig(args)
 
 	// Set Load Balancer policy
 	//nolint:gocritic
@@ -439,6 +421,48 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) {
 	}, nil
 }
 
+func buildLocalityLbConfig(args *xdsClusterArgs) *commonv3.LocalityLbConfig {
+	// Default to LocalityWeightedLbConfig
+	localityLbConfig := &commonv3.LocalityLbConfig{
+		LocalityConfigSpecifier: &commonv3.LocalityLbConfig_LocalityWeightedLbConfig_{
+			LocalityWeightedLbConfig: &commonv3.LocalityLbConfig_LocalityWeightedLbConfig{},
+		},
+	}
+
+	// Check for LoadBalancer.PreferLocal configuration or if backendRef enables
+	// PreferLocal (such as Topology Aware Routing or Traffic Distribution)
+	if preferLocal := ptr.Deref(args.loadBalancer, ir.LoadBalancer{}).PreferLocal; preferLocal != nil {
+		if cfg := buildZoneAwareLbConfig(preferLocal); cfg != nil {
+			localityLbConfig.LocalityConfigSpecifier = cfg
+		}
+		// Zone aware enabled backendRefs use weighted clusters and
+		// always have a single DestinationSetting per-cluster.
+	} else if len(args.settings) == 1 && args.settings[0].PreferLocal != nil {
+		if cfg := buildZoneAwareLbConfig(args.settings[0].PreferLocal); cfg != nil {
+			localityLbConfig.LocalityConfigSpecifier = cfg
+		}
+	}
+
+	return localityLbConfig
+}
+
+func buildZoneAwareLbConfig(preferLocal *ir.PreferLocalZone) *commonv3.LocalityLbConfig_ZoneAwareLbConfig_ {
+	if preferLocal == nil {
+		return nil
+	}
+	lbConfig := &commonv3.LocalityLbConfig_ZoneAwareLbConfig_{
+		ZoneAwareLbConfig: &commonv3.LocalityLbConfig_ZoneAwareLbConfig{
+			MinClusterSize: wrapperspb.UInt64(ptr.Deref(preferLocal.MinEndpointsThreshold, 6)),
+		},
+	}
+	if preferLocal.Force != nil {
+		lbConfig.ZoneAwareLbConfig.ForceLocalZone = &commonv3.LocalityLbConfig_ZoneAwareLbConfig_ForceLocalZone{
+			MinSize: wrapperspb.UInt32(ptr.Deref(preferLocal.Force.MinEndpointsInZoneThreshold, 1)),
+		}
+	}
+	return lbConfig
+}
+
 func buildXdsHealthCheck(healthcheck *ir.ActiveHealthCheck) []*corev3.HealthCheck {
 	hc := &corev3.HealthCheck{
 		Timeout:  durationpb.New(healthcheck.Timeout.Duration),
@@ -650,7 +674,7 @@ func buildXdsClusterLoadAssignment(clusterName string, destSettings []*ir.Destin
 		// if multiple backendRefs exist. This pushes part of the routing logic higher up the stack which can
 		// limit host selection controls during retries and session affinity.
 		// For more details see https://github.com/envoyproxy/gateway/issues/5307#issuecomment-2688767482
-		if ds.ZoneAwareRouting != nil {
+		if ds.PreferLocal != nil {
 			localities = append(localities, buildZonalLocalities(metadata, ds)...)
 		} else {
 			localities = append(localities, buildWeightedLocalities(metadata, ds))