From cf657e5d6504e598a5fb63e34214a5ae64707742 Mon Sep 17 00:00:00 2001 From: Sri Roopa Ramesh Babu Date: Tue, 23 Sep 2025 14:57:44 -0400 Subject: [PATCH] Postgres reconciliation on certificate rotation. #Updating hash trigger. --- internal/controller/constants.go | 3 + .../controller/ols_app_postgres_assets.go | 22 +++-- .../ols_app_postgres_reconciliator.go | 84 ++++++++++++++++++- .../controller/ols_app_server_deployment.go | 15 ++-- .../ols_app_server_reconciliator.go | 17 ++-- internal/controller/olsconfig_controller.go | 7 +- internal/controller/resource_watchers.go | 21 +++++ 7 files changed, 144 insertions(+), 25 deletions(-) diff --git a/internal/controller/constants.go b/internal/controller/constants.go index ff059997d..6fb2dca29 100644 --- a/internal/controller/constants.go +++ b/internal/controller/constants.go @@ -169,6 +169,8 @@ const ( // PostgresSecretHashKey is the key of the hash value of OLS Postgres secret // #nosec G101 PostgresSecretHashKey = "hash/postgres-secret" + // PostgresCAHashKey is the key of the hash value of the OLS Postgres CA certificate + PostgresCAHashKey = "hash/postgres-ca" // PostgresServiceName is the name of OLS application Postgres server service PostgresServiceName = "lightspeed-postgres-server" // PostgresSecretName is the name of OLS application Postgres secret @@ -255,6 +257,7 @@ ssl_ca_file = '/etc/certs/cm-olspostgresca/service-ca.crt' OperatorDeploymentName = "lightspeed-operator-controller-manager" OLSDefaultCacheType = "postgres" PostgresConfigHashStateCacheKey = "olspostgresconfig-hash" + PostgresCAHashStateCacheKey = "olspostgresca-hash" // #nosec G101 PostgresSecretHashStateCacheKey = "olspostgressecret-hash" // OperatorNetworkPolicyName is the name of the network policy for the operator diff --git a/internal/controller/ols_app_postgres_assets.go b/internal/controller/ols_app_postgres_assets.go index 4e90abbe3..b095f2bab 100644 --- a/internal/controller/ols_app_postgres_assets.go +++ b/internal/controller/ols_app_postgres_assets.go @@ -256,16 +256,21 @@ func (r *OLSConfigReconciler) updatePostgresDeployment(ctx context.Context, exis // Validate deployment annotations. if existingDeployment.Annotations == nil || existingDeployment.Annotations[PostgresConfigHashKey] != r.stateCache[PostgresConfigHashStateCacheKey] || - existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] { - updateDeploymentAnnotations(existingDeployment, map[string]string{ + existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] || + existingDeployment.Annotations[PostgresCAHashKey] != r.stateCache[PostgresCAHashStateCacheKey] { + annotations := map[string]string{ PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey], PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) + PostgresCAHashKey: r.stateCache[PostgresCAHashStateCacheKey], + } + updateDeploymentAnnotations(existingDeployment, annotations) // update the deployment template annotation triggers the rolling update - updateDeploymentTemplateAnnotations(existingDeployment, map[string]string{ + templateAnnotations := map[string]string{ PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey], PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) + PostgresCAHashKey: r.stateCache[PostgresCAHashStateCacheKey], + } + updateDeploymentTemplateAnnotations(existingDeployment, templateAnnotations) if _, err := setDeploymentContainerEnvs(existingDeployment, desiredDeployment.Spec.Template.Spec.Containers[0].Env, PostgresDeploymentName); err != nil { return err @@ -454,7 +459,12 @@ func (r *OLSConfigReconciler) storageDefaults(s *olsv1alpha1.Storage) error { func (r *OLSConfigReconciler) generatePostgresPVC(cr *olsv1alpha1.OLSConfig) (*corev1.PersistentVolumeClaim, error) { - storage := cr.Spec.OLSConfig.Storage + // Create a copy of the storage configuration to avoid modifying the original CR + storage := &olsv1alpha1.Storage{} + if cr.Spec.OLSConfig.Storage != nil { + storage.Size = cr.Spec.OLSConfig.Storage.Size + storage.Class = cr.Spec.OLSConfig.Storage.Class + } if err := r.storageDefaults(storage); err != nil { return nil, err } diff --git a/internal/controller/ols_app_postgres_reconciliator.go b/internal/controller/ols_app_postgres_reconciliator.go index 21552dd38..a1d9f84ba 100644 --- a/internal/controller/ols_app_postgres_reconciliator.go +++ b/internal/controller/ols_app_postgres_reconciliator.go @@ -30,6 +30,10 @@ func (r *OLSConfigReconciler) reconcilePostgresServer(ctx context.Context, olsco Name: "reconcile Postgres Secret", Task: r.reconcilePostgresSecret, }, + { + Name: "reconcile Postgres CA Secret", + Task: r.reconcilePostgresCA, + }, { Name: "reconcile Postgres Service", Task: r.reconcilePostgresService, @@ -70,14 +74,18 @@ func (r *OLSConfigReconciler) reconcilePostgresDeployment(ctx context.Context, c existingDeployment := &appsv1.Deployment{} err = r.Get(ctx, client.ObjectKey{Name: PostgresDeploymentName, Namespace: r.Options.Namespace}, existingDeployment) if err != nil && errors.IsNotFound(err) { - updateDeploymentAnnotations(desiredDeployment, map[string]string{ + annotations := map[string]string{ PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey], PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) - updateDeploymentTemplateAnnotations(desiredDeployment, map[string]string{ + PostgresCAHashKey: r.stateCache[PostgresCAHashStateCacheKey], + } + updateDeploymentAnnotations(desiredDeployment, annotations) + templateAnnotations := map[string]string{ PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey], PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) + PostgresCAHashKey: r.stateCache[PostgresCAHashStateCacheKey], + } + updateDeploymentTemplateAnnotations(desiredDeployment, templateAnnotations) r.logger.Info("creating a new OLS postgres deployment", "deployment", desiredDeployment.Name) err = r.Create(ctx, desiredDeployment) if err != nil { @@ -273,3 +281,71 @@ func (r *OLSConfigReconciler) reconcilePostgresNetworkPolicy(ctx context.Context r.logger.Info("OLS postgres network policy reconciled", "network policy", networkPolicy.Name) return nil } + +func (r *OLSConfigReconciler) reconcilePostgresCA(ctx context.Context, cr *olsv1alpha1.OLSConfig) error { + var caConfigMap *corev1.ConfigMap + var servingCertSecret *corev1.Secret + certBytes := []byte{} + hasAnyInput := false + + tmpCM := &corev1.ConfigMap{} + err := r.Client.Get(ctx, client.ObjectKey{Name: OLSCAConfigMap, Namespace: r.Options.Namespace}, tmpCM) + if err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("failed to get openshift-service-ca.crt ConfigMap: %w", err) + } + r.logger.Info("openshift-service-ca.crt ConfigMap not found, skipping CA bundle") + } else { + caConfigMap = tmpCM + if caCert, exists := caConfigMap.Data["service-ca.crt"]; exists { + certBytes = append(certBytes, []byte("service-ca.crt")...) + certBytes = append(certBytes, []byte(caCert)...) + hasAnyInput = true + } + } + + // Serving cert Secret + tmpSec := &corev1.Secret{} + err = r.Client.Get(ctx, client.ObjectKey{Name: PostgresCertsSecretName, Namespace: r.Options.Namespace}, tmpSec) + if err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("failed to get %s Secret: %w", PostgresCertsSecretName, err) + } + r.logger.Info("serving cert Secret not found, skipping server certificate", "secret", PostgresCertsSecretName) + } else { + servingCertSecret = tmpSec + if tlsCert, exists := servingCertSecret.Data["tls.crt"]; exists { + certBytes = append(certBytes, []byte("tls.crt")...) + certBytes = append(certBytes, tlsCert...) + hasAnyInput = true + } + } + + // Calculate hash based on available inputs + var combinedHash string + if !hasAnyInput { + // No cert inputs available - use empty hash + combinedHash = "" + } else { + var err error + combinedHash, err = hashBytes(certBytes) + if err != nil { + return fmt.Errorf("failed to generate Postgres CA hash: %w", err) + } + } + + // Store existing hash before updating + existingHash := r.stateCache[PostgresCAHashStateCacheKey] + + // Always update state cache to ensure it's set, even if value hasn't changed + r.stateCache[PostgresCAHashStateCacheKey] = combinedHash + + // Check if hash changed (including changes to/from empty string) + if combinedHash == existingHash { + return nil + } + + r.logger.Info("Postgres CA hash updated,deployment will be updated via updatePostgresDeployment") + + return nil +} diff --git a/internal/controller/ols_app_server_deployment.go b/internal/controller/ols_app_server_deployment.go index 46933f46f..ccf78a58c 100644 --- a/internal/controller/ols_app_server_deployment.go +++ b/internal/controller/ols_app_server_deployment.go @@ -438,22 +438,27 @@ func (r *OLSConfigReconciler) updateOLSDeployment(ctx context.Context, existingD existingDeployment.Annotations[OLSConfigHashKey] != r.stateCache[OLSConfigHashStateCacheKey] || existingDeployment.Annotations[OLSAppTLSHashKey] != r.stateCache[OLSAppTLSHashStateCacheKey] || existingDeployment.Annotations[LLMProviderHashKey] != r.stateCache[LLMProviderHashStateCacheKey] || - existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] { - updateDeploymentAnnotations(existingDeployment, map[string]string{ + existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] || + existingDeployment.Annotations[PostgresCAHashKey] != r.stateCache[PostgresCAHashStateCacheKey] { + annotations := map[string]string{ OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], AdditionalCAHashKey: r.stateCache[AdditionalCAHashStateCacheKey], PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) + PostgresCAHashKey: r.stateCache[PostgresCAHashStateCacheKey], + } + updateDeploymentAnnotations(existingDeployment, annotations) // update the deployment template annotation triggers the rolling update - updateDeploymentTemplateAnnotations(existingDeployment, map[string]string{ + templateAnnotations := map[string]string{ OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], AdditionalCAHashKey: r.stateCache[AdditionalCAHashStateCacheKey], PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) + PostgresCAHashKey: r.stateCache[PostgresCAHashStateCacheKey], + } + updateDeploymentTemplateAnnotations(existingDeployment, templateAnnotations) changed = true } diff --git a/internal/controller/ols_app_server_reconciliator.go b/internal/controller/ols_app_server_reconciliator.go index 77e0795ff..797624a2c 100644 --- a/internal/controller/ols_app_server_reconciliator.go +++ b/internal/controller/ols_app_server_reconciliator.go @@ -140,8 +140,9 @@ func (r *OLSConfigReconciler) reconcileOLSConfigMap(ctx context.Context, cr *ols func (r *OLSConfigReconciler) reconcileOLSAdditionalCAConfigMap(ctx context.Context, cr *olsv1alpha1.OLSConfig) error { if cr.Spec.OLSConfig.AdditionalCAConfigMapRef == nil { - // no additional CA certs, skip - r.logger.Info("Additional CA not configured, reconciliation skipped") + // no additional CA certs, set empty hash + r.logger.Info("Additional CA not configured, setting empty hash") + r.stateCache[AdditionalCAHashStateCacheKey] = "" return nil } @@ -278,18 +279,22 @@ func (r *OLSConfigReconciler) reconcileDeployment(ctx context.Context, cr *olsv1 existingDeployment := &appsv1.Deployment{} err = r.Get(ctx, client.ObjectKey{Name: OLSAppServerDeploymentName, Namespace: r.Options.Namespace}, existingDeployment) if err != nil && errors.IsNotFound(err) { - updateDeploymentAnnotations(desiredDeployment, map[string]string{ + annotations := map[string]string{ OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) - updateDeploymentTemplateAnnotations(desiredDeployment, map[string]string{ + PostgresCAHashKey: r.stateCache[PostgresCAHashStateCacheKey], + } + updateDeploymentAnnotations(desiredDeployment, annotations) + templateAnnotations := map[string]string{ OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) + PostgresCAHashKey: r.stateCache[PostgresCAHashStateCacheKey], + } + updateDeploymentTemplateAnnotations(desiredDeployment, templateAnnotations) r.logger.Info("creating a new deployment", "deployment", desiredDeployment.Name) err = r.Create(ctx, desiredDeployment) if err != nil { diff --git a/internal/controller/olsconfig_controller.go b/internal/controller/olsconfig_controller.go index 764ec4566..3a674a838 100644 --- a/internal/controller/olsconfig_controller.go +++ b/internal/controller/olsconfig_controller.go @@ -33,10 +33,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" olsv1alpha1 "github.com/openshift/lightspeed-operator/api/v1alpha1" @@ -338,10 +336,9 @@ func (r *OLSConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { r.stateCache = make(map[string]string) r.NextReconcileTime = time.Now() - generationChanged := builder.WithPredicates(predicate.GenerationChangedPredicate{}) return ctrl.NewControllerManagedBy(mgr). For(&olsv1alpha1.OLSConfig{}). - Owns(&appsv1.Deployment{}, generationChanged). + Owns(&appsv1.Deployment{}). Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.ClusterRole{}). Owns(&rbacv1.ClusterRoleBinding{}). @@ -351,9 +348,11 @@ func (r *OLSConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.PersistentVolumeClaim{}). Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(secretWatcherFilter)). Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(telemetryPullSecretWatcherFilter)). + Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(postgresCAWatcherFilter)). Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { return r.configMapWatcherFilter(ctx, obj) })). + Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(postgresCAWatcherFilter)). Owns(&consolev1.ConsolePlugin{}). Owns(&monv1.ServiceMonitor{}). Owns(&monv1.PrometheusRule{}). diff --git a/internal/controller/resource_watchers.go b/internal/controller/resource_watchers.go index 493ae8ab5..47c323430 100644 --- a/internal/controller/resource_watchers.go +++ b/internal/controller/resource_watchers.go @@ -124,3 +124,24 @@ func (r *OLSConfigReconciler) restartAppServer(ctx context.Context, inCluster bo } return nil } + +// postgresCAWatcherFilter watches for changes to PostgreSQL CA certificate resources +func postgresCAWatcherFilter(ctx context.Context, obj client.Object) []reconcile.Request { + // Watch the openshift-service-ca.crt ConfigMap + if obj.GetName() == OLSCAConfigMap { + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Name: OLSConfigName, + }}, + } + } + // Watch the PostgreSQL serving certificate Secret + if obj.GetName() == PostgresCertsSecretName { + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Name: OLSConfigName, + }}, + } + } + return nil +}