Keystone Auth: Remove authorization-* args from apiserver bootstrap (#600)

In kubernetes 1.29 kubeadm is not able to authorize the admin
user because it is trying to find a webhook that doesn't yet
exist.

Solution: don't add webhook at init time but instead wait till
after kubeadm has completed and then use kustomize to patch
the kube-apiserver with the authorization-mode argument to add
"Webhook". It is possible to add this option to the kube-apiserver
arguments separately so it can just be appended to the list of args
along with the authorization/authentication config paths.

Related issue: kubernetes/cloud-provider-openstack#2575

Co-authored-by: Travis Holton <travisholton@catalystcloud.nz>

Author: dalees

charts/openstack-cluster/templates/_helpers.tpl

@@ -392,9 +392,16 @@ ignition:
 Create folders necessary for webhook integration.
 */}}
 {{- define "openstack-cluster.webhookPatches" }}
+{{- $authWebhook := .Values.authWebhook }}
   preKubeadmCommands:
     - mkdir -p /etc/kubernetes/webhooks
     - mkdir -p /etc/kubernetes/patches
+{{- if eq $authWebhook "k8s-keystone-auth" }}
+    - mkdir -p /etc/kubernetes/keystone-auth
+  postKubeadmCommands:
+    - cp /etc/kubernetes/manifests/kube-apiserver.yaml /etc/kubernetes/keystone-auth/kube-apiserver.yaml
+    - kubectl kustomize /etc/kubernetes/keystone-auth -o /etc/kubernetes/manifests/kube-apiserver.yaml
+{{- end }}
 {{- end }}
 
 {{/*
@@ -409,11 +416,7 @@ webhooks and policies for audit logging can be added here.
       extraArgs:
         v: {{ $ctx.Values.apiServer.logLevel | quote }}
 {{- if ne $authWebhook "none" }}
-{{- if eq $authWebhook "k8s-keystone-auth" }}
-        authorization-mode: Node,Webhook,RBAC
-        authentication-token-webhook-config-file: /etc/kubernetes/webhooks/keystone_webhook_config.yaml
-        authorization-webhook-config-file: /etc/kubernetes/webhooks/keystone_webhook_config.yaml
-{{- else if eq $authWebhook "azimuth-authorization-webhook" }}
+{{- if eq $authWebhook "azimuth-authorization-webhook" }}
         authorization-config: /etc/kubernetes/webhooks/authorization_config.yaml
 {{/*
 Add else if blocks with other webhooks and apiServer arguments (i.e. audit logging) 
@@ -466,6 +469,25 @@ Produces integration for k8s-keystone-auth webhook on apiserver
 {{- define "openstack-cluster.k8sKeystoneAuthWebhook" }}
   files:
 {{- include "openstack-cluster.webhookMountDirectoryFile" . }}
+    - path: /etc/kubernetes/keystone-auth/kustomization.yml
+      permissions: "0644"
+      owner: root:root
+      content: |
+        resources:
+        - kube-apiserver.yaml
+        patches:
+        - patch: |-
+            - op: add
+              path: /spec/containers/0/command/-
+              value: --authentication-token-webhook-config-file=/etc/kubernetes/webhooks/keystone_webhook_config.yaml
+            - op: add
+              path: /spec/containers/0/command/-
+              value: --authorization-webhook-config-file=/etc/kubernetes/webhooks/keystone_webhook_config.yaml
+            - op: add
+              path: /spec/containers/0/command/-
+              value: --authorization-mode=Webhook
+          target:
+            kind: Pod
     - path: /etc/kubernetes/webhooks/keystone_webhook_config.yaml
       content: |
         ---