feat: watch-based route reconciliation (#970)

Switch to manually created dependencies for `cloud-provider` and
`controller-manager`. These patches contain a feature gate, which
enables watch-based reconciliation in the routes' controller of
cloud-controller-manager, reducing API calls. Previously the routes
where reconciled every ten seconds.

This feature is for an upcoming preview release, which is used to test
our current upstream contribution at Kubernetes. Two forks of
`cloud-provider` and `controller-manager` are used. The tag `v0.33.2` is
used as a base, with the necessary patches applied manually.

This feature will be revered upon completing our tests.

Upstream WIP PR: kubernetes/kubernetes#131220
Kubernetes Enhancement Proposal:
kubernetes/enhancements#5289
Fork of `cloud-provider`: https://github.com/lukasmetzner/cloud-provider
Fork of `controller-manager`:
https://github.com/lukasmetzner/controller-manager

---------

Co-authored-by: Julian Tölle <julian.toelle@hetzner-cloud.de>

Author: lukasmetzner

chart/.snapshots/default.yaml

@@ -64,6 +64,7 @@ spec:
           args:
             - "--allow-untagged-cloud"
             - "--cloud-provider=hcloud"
+            - "--feature-gates=CloudControllerManagerWatchBasedRoutesReconciliation=false"
             - "--route-reconciliation-period=30s"
             - "--webhook-secure-port=0"
             - "--leader-elect=false"

chart/.snapshots/full.daemonset.yaml

@@ -74,6 +74,7 @@ spec:
             - "/bin/hcloud-cloud-controller-manager"
             - "--allow-untagged-cloud"
             - "--cloud-provider=hcloud"
+            - "--feature-gates=CloudControllerManagerWatchBasedRoutesReconciliation=false"
             - "--route-reconciliation-period=30s"
             - "--webhook-secure-port=0"
           env:

chart/values.yaml

@@ -16,6 +16,13 @@ args:
   # https://github.com/hetznercloud/hcloud-cloud-controller-manager/issues/492
   webhook-secure-port: "0"
 
+  # Experimental feature, which enables route reconciliation based on the following Kubernetes events:
+  # - Node added
+  # - Node deleted
+  # - Node PodCIDRs, or Addresses updated
+  # This replaces the fixed route reconcile interval of 30s
+  feature-gates: "CloudControllerManagerWatchBasedRoutesReconciliation=false"
+
 # Change deployment kind from "Deployment" to "DaemonSet"
 kind: Deployment
 

deploy/ccm-networks.yaml

@@ -63,6 +63,7 @@ spec:
           args:
             - "--allow-untagged-cloud"
             - "--cloud-provider=hcloud"
+            - "--feature-gates=CloudControllerManagerWatchBasedRoutesReconciliation=false"
             - "--route-reconciliation-period=30s"
             - "--webhook-secure-port=0"
             - "--allocate-node-cidrs=true"

deploy/ccm.yaml

@@ -62,6 +62,7 @@ spec:
           args:
             - "--allow-untagged-cloud"
             - "--cloud-provider=hcloud"
+            - "--feature-gates=CloudControllerManagerWatchBasedRoutesReconciliation=false"
             - "--route-reconciliation-period=30s"
             - "--webhook-secure-port=0"
             - "--leader-elect=false"

go.mod

@@ -24,7 +24,6 @@ require (
 	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
-	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
@@ -42,21 +41,17 @@ require (
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/cel-go v0.23.2 // indirect
 	github.com/google/gnostic-models v0.6.9 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
-	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect
-	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/moby/term v0.5.0 // indirect
@@ -102,7 +97,6 @@ require (
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiserver v0.33.2 // indirect
 	k8s.io/component-helpers v0.33.2 // indirect
@@ -116,3 +110,7 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
+
+replace k8s.io/cloud-provider v0.33.2 => github.com/lukasmetzner/cloud-provider v1.33.2-watch-based-route-reconcile
+
+replace k8s.io/controller-manager v0.33.2 => github.com/lukasmetzner/controller-manager v1.33.2-watch-based-route-reconcile