OLS-1775: Postgres reconciliation on certificate rotation.

internal/controller/constants.go

@@ -169,6 +169,8 @@ const (
 	// PostgresSecretHashKey is the key of the hash value of OLS Postgres secret
 	// #nosec G101
 	PostgresSecretHashKey = "hash/postgres-secret"
+	// PostgresCAHashKey is the key of the hash value of the OLS Postgres CA certificate
+	PostgresCAHashKey = "hash/postgres-ca"
 	// PostgresServiceName is the name of OLS application Postgres server service
 	PostgresServiceName = "lightspeed-postgres-server"
 	// PostgresSecretName is the name of OLS application Postgres secret
@@ -255,6 +257,7 @@ ssl_ca_file = '/etc/certs/cm-olspostgresca/service-ca.crt'
 	OperatorDeploymentName          = "lightspeed-operator-controller-manager"
 	OLSDefaultCacheType             = "postgres"
 	PostgresConfigHashStateCacheKey = "olspostgresconfig-hash"
+	PostgresCAHashStateCacheKey     = "olspostgresca-hash"
 	// #nosec G101
 	PostgresSecretHashStateCacheKey = "olspostgressecret-hash"
 	// OperatorNetworkPolicyName is the name of the network policy for the operator

internal/controller/ols_app_postgres_assets.go

@@ -256,16 +256,21 @@ func (r *OLSConfigReconciler) updatePostgresDeployment(ctx context.Context, exis
 	// Validate deployment annotations.
 	if existingDeployment.Annotations == nil ||
 		existingDeployment.Annotations[PostgresConfigHashKey] != r.stateCache[PostgresConfigHashStateCacheKey] ||
-		existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] {
-		updateDeploymentAnnotations(existingDeployment, map[string]string{
+		existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] ||
+		existingDeployment.Annotations[PostgresCAHashKey] != r.stateCache[PostgresCAHashStateCacheKey] {
+		annotations := map[string]string{
 			PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey],
 			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
+			PostgresCAHashKey:     r.stateCache[PostgresCAHashStateCacheKey],
+		}
+		updateDeploymentAnnotations(existingDeployment, annotations)
 		// update the deployment template annotation triggers the rolling update
-		updateDeploymentTemplateAnnotations(existingDeployment, map[string]string{
+		templateAnnotations := map[string]string{
 			PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey],
 			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
+			PostgresCAHashKey:     r.stateCache[PostgresCAHashStateCacheKey],
+		}
+		updateDeploymentTemplateAnnotations(existingDeployment, templateAnnotations)
 
 		if _, err := setDeploymentContainerEnvs(existingDeployment, desiredDeployment.Spec.Template.Spec.Containers[0].Env, PostgresDeploymentName); err != nil {
 			return err
@@ -454,7 +459,12 @@ func (r *OLSConfigReconciler) storageDefaults(s *olsv1alpha1.Storage) error {
 
 func (r *OLSConfigReconciler) generatePostgresPVC(cr *olsv1alpha1.OLSConfig) (*corev1.PersistentVolumeClaim, error) {
 
-	storage := cr.Spec.OLSConfig.Storage
+	// Create a copy of the storage configuration to avoid modifying the original CR
+	storage := &olsv1alpha1.Storage{}
+	if cr.Spec.OLSConfig.Storage != nil {
+		storage.Size = cr.Spec.OLSConfig.Storage.Size
+		storage.Class = cr.Spec.OLSConfig.Storage.Class
+	}
 	if err := r.storageDefaults(storage); err != nil {
 		return nil, err
 	}

internal/controller/ols_app_postgres_reconciliator.go

@@ -30,6 +30,10 @@ func (r *OLSConfigReconciler) reconcilePostgresServer(ctx context.Context, olsco
 			Name: "reconcile Postgres Secret",
 			Task: r.reconcilePostgresSecret,
 		},
+		{
+			Name: "reconcile Postgres CA Secret",
+			Task: r.reconcilePostgresCA,
+		},
 		{
 			Name: "reconcile Postgres Service",
 			Task: r.reconcilePostgresService,
@@ -70,14 +74,18 @@ func (r *OLSConfigReconciler) reconcilePostgresDeployment(ctx context.Context, c
 	existingDeployment := &appsv1.Deployment{}
 	err = r.Get(ctx, client.ObjectKey{Name: PostgresDeploymentName, Namespace: r.Options.Namespace}, existingDeployment)
 	if err != nil && errors.IsNotFound(err) {
-		updateDeploymentAnnotations(desiredDeployment, map[string]string{
+		annotations := map[string]string{
 			PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey],
 			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
-		updateDeploymentTemplateAnnotations(desiredDeployment, map[string]string{
+			PostgresCAHashKey:     r.stateCache[PostgresCAHashStateCacheKey],
+		}
+		updateDeploymentAnnotations(desiredDeployment, annotations)
+		templateAnnotations := map[string]string{
 			PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey],
 			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
+			PostgresCAHashKey:     r.stateCache[PostgresCAHashStateCacheKey],
+		}
+		updateDeploymentTemplateAnnotations(desiredDeployment, templateAnnotations)
 		r.logger.Info("creating a new OLS postgres deployment", "deployment", desiredDeployment.Name)
 		err = r.Create(ctx, desiredDeployment)
 		if err != nil {
@@ -273,3 +281,71 @@ func (r *OLSConfigReconciler) reconcilePostgresNetworkPolicy(ctx context.Context
 	r.logger.Info("OLS postgres network policy reconciled", "network policy", networkPolicy.Name)
 	return nil
 }
+
+func (r *OLSConfigReconciler) reconcilePostgresCA(ctx context.Context, cr *olsv1alpha1.OLSConfig) error {
+	var caConfigMap *corev1.ConfigMap
+	var servingCertSecret *corev1.Secret
+	certBytes := []byte{}
+	hasAnyInput := false
+
+	tmpCM := &corev1.ConfigMap{}
+	err := r.Client.Get(ctx, client.ObjectKey{Name: OLSCAConfigMap, Namespace: r.Options.Namespace}, tmpCM)
+	if err != nil {
+		if !errors.IsNotFound(err) {
+			return fmt.Errorf("failed to get openshift-service-ca.crt ConfigMap: %w", err)
+		}
+		r.logger.Info("openshift-service-ca.crt ConfigMap not found, skipping CA bundle")
+	} else {
+		caConfigMap = tmpCM
+		if caCert, exists := caConfigMap.Data["service-ca.crt"]; exists {
+			certBytes = append(certBytes, []byte("service-ca.crt")...)
+			certBytes = append(certBytes, []byte(caCert)...)
+			hasAnyInput = true
+		}
+	}
+
+	// Serving cert Secret
+	tmpSec := &corev1.Secret{}
+	err = r.Client.Get(ctx, client.ObjectKey{Name: PostgresCertsSecretName, Namespace: r.Options.Namespace}, tmpSec)
+	if err != nil {
+		if !errors.IsNotFound(err) {
+			return fmt.Errorf("failed to get %s Secret: %w", PostgresCertsSecretName, err)
+		}
+		r.logger.Info("serving cert Secret not found, skipping server certificate", "secret", PostgresCertsSecretName)
+	} else {
+		servingCertSecret = tmpSec
+		if tlsCert, exists := servingCertSecret.Data["tls.crt"]; exists {
+			certBytes = append(certBytes, []byte("tls.crt")...)
+			certBytes = append(certBytes, tlsCert...)
+			hasAnyInput = true
+		}
+	}
+
+	// Calculate hash based on available inputs
+	var combinedHash string
+	if !hasAnyInput {
+		// No cert inputs available - use empty hash
+		combinedHash = ""
+	} else {
+		var err error
+		combinedHash, err = hashBytes(certBytes)
+		if err != nil {
+			return fmt.Errorf("failed to generate Postgres CA hash: %w", err)
+		}
+	}
+
+	// Store existing hash before updating
+	existingHash := r.stateCache[PostgresCAHashStateCacheKey]
+
+	// Always update state cache to ensure it's set, even if value hasn't changed
+	r.stateCache[PostgresCAHashStateCacheKey] = combinedHash
+
+	// Check if hash changed (including changes to/from empty string)
+	if combinedHash == existingHash {
+		return nil
+	}
+
+	r.logger.Info("Postgres CA hash updated,deployment will be updated via updatePostgresDeployment")
+
+	return nil
+}

internal/controller/ols_app_server_deployment.go

@@ -438,22 +438,27 @@ func (r *OLSConfigReconciler) updateOLSDeployment(ctx context.Context, existingD
 		existingDeployment.Annotations[OLSConfigHashKey] != r.stateCache[OLSConfigHashStateCacheKey] ||
 		existingDeployment.Annotations[OLSAppTLSHashKey] != r.stateCache[OLSAppTLSHashStateCacheKey] ||
 		existingDeployment.Annotations[LLMProviderHashKey] != r.stateCache[LLMProviderHashStateCacheKey] ||
-		existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] {
-		updateDeploymentAnnotations(existingDeployment, map[string]string{
+		existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] ||
+		existingDeployment.Annotations[PostgresCAHashKey] != r.stateCache[PostgresCAHashStateCacheKey] {
+		annotations := map[string]string{
 			OLSConfigHashKey:      r.stateCache[OLSConfigHashStateCacheKey],
 			OLSAppTLSHashKey:      r.stateCache[OLSAppTLSHashStateCacheKey],
 			LLMProviderHashKey:    r.stateCache[LLMProviderHashStateCacheKey],
 			AdditionalCAHashKey:   r.stateCache[AdditionalCAHashStateCacheKey],
 			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
+			PostgresCAHashKey:     r.stateCache[PostgresCAHashStateCacheKey],
+		}
+		updateDeploymentAnnotations(existingDeployment, annotations)
 		// update the deployment template annotation triggers the rolling update
-		updateDeploymentTemplateAnnotations(existingDeployment, map[string]string{
+		templateAnnotations := map[string]string{
 			OLSConfigHashKey:      r.stateCache[OLSConfigHashStateCacheKey],
 			OLSAppTLSHashKey:      r.stateCache[OLSAppTLSHashStateCacheKey],
 			LLMProviderHashKey:    r.stateCache[LLMProviderHashStateCacheKey],
 			AdditionalCAHashKey:   r.stateCache[AdditionalCAHashStateCacheKey],
 			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
+			PostgresCAHashKey:     r.stateCache[PostgresCAHashStateCacheKey],
+		}
+		updateDeploymentTemplateAnnotations(existingDeployment, templateAnnotations)
 		changed = true
 	}
 

internal/controller/ols_app_server_reconciliator.go

@@ -140,8 +140,9 @@ func (r *OLSConfigReconciler) reconcileOLSConfigMap(ctx context.Context, cr *ols
 
 func (r *OLSConfigReconciler) reconcileOLSAdditionalCAConfigMap(ctx context.Context, cr *olsv1alpha1.OLSConfig) error {
 	if cr.Spec.OLSConfig.AdditionalCAConfigMapRef == nil {
-		// no additional CA certs, skip
-		r.logger.Info("Additional CA not configured, reconciliation skipped")
+		// no additional CA certs, set empty hash
+		r.logger.Info("Additional CA not configured, setting empty hash")
+		r.stateCache[AdditionalCAHashStateCacheKey] = ""
 		return nil
 	}
 
@@ -278,18 +279,22 @@ func (r *OLSConfigReconciler) reconcileDeployment(ctx context.Context, cr *olsv1
 	existingDeployment := &appsv1.Deployment{}
 	err = r.Get(ctx, client.ObjectKey{Name: OLSAppServerDeploymentName, Namespace: r.Options.Namespace}, existingDeployment)
 	if err != nil && errors.IsNotFound(err) {
-		updateDeploymentAnnotations(desiredDeployment, map[string]string{
+		annotations := map[string]string{
 			OLSConfigHashKey:      r.stateCache[OLSConfigHashStateCacheKey],
 			OLSAppTLSHashKey:      r.stateCache[OLSAppTLSHashStateCacheKey],
 			LLMProviderHashKey:    r.stateCache[LLMProviderHashStateCacheKey],
 			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
-		updateDeploymentTemplateAnnotations(desiredDeployment, map[string]string{
+			PostgresCAHashKey:     r.stateCache[PostgresCAHashStateCacheKey],
+		}
+		updateDeploymentAnnotations(desiredDeployment, annotations)
+		templateAnnotations := map[string]string{
 			OLSConfigHashKey:      r.stateCache[OLSConfigHashStateCacheKey],
 			OLSAppTLSHashKey:      r.stateCache[OLSAppTLSHashStateCacheKey],
 			LLMProviderHashKey:    r.stateCache[LLMProviderHashStateCacheKey],
 			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
+			PostgresCAHashKey:     r.stateCache[PostgresCAHashStateCacheKey],
+		}
+		updateDeploymentTemplateAnnotations(desiredDeployment, templateAnnotations)
 		r.logger.Info("creating a new deployment", "deployment", desiredDeployment.Name)
 		err = r.Create(ctx, desiredDeployment)
 		if err != nil {

internal/controller/olsconfig_controller.go

@@ -33,10 +33,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/util/retry"
 	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
-	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	olsv1alpha1 "github.com/openshift/lightspeed-operator/api/v1alpha1"
@@ -338,10 +336,9 @@ func (r *OLSConfigReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	r.stateCache = make(map[string]string)
 	r.NextReconcileTime = time.Now()
 
-	generationChanged := builder.WithPredicates(predicate.GenerationChangedPredicate{})
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&olsv1alpha1.OLSConfig{}).
-		Owns(&appsv1.Deployment{}, generationChanged).
+		Owns(&appsv1.Deployment{}).
 		Owns(&corev1.ServiceAccount{}).
 		Owns(&rbacv1.ClusterRole{}).
 		Owns(&rbacv1.ClusterRoleBinding{}).
@@ -351,9 +348,11 @@ func (r *OLSConfigReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&corev1.PersistentVolumeClaim{}).
 		Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(secretWatcherFilter)).
 		Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(telemetryPullSecretWatcherFilter)).
+		Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(postgresCAWatcherFilter)).
 		Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request {
 			return r.configMapWatcherFilter(ctx, obj)
 		})).
+		Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(postgresCAWatcherFilter)).
 		Owns(&consolev1.ConsolePlugin{}).
 		Owns(&monv1.ServiceMonitor{}).
 		Owns(&monv1.PrometheusRule{}).

internal/controller/resource_watchers.go

@@ -124,3 +124,24 @@ func (r *OLSConfigReconciler) restartAppServer(ctx context.Context, inCluster bo
 	}
 	return nil
 }
+
+// postgresCAWatcherFilter watches for changes to PostgreSQL CA certificate resources
+func postgresCAWatcherFilter(ctx context.Context, obj client.Object) []reconcile.Request {
+	// Watch the openshift-service-ca.crt ConfigMap
+	if obj.GetName() == OLSCAConfigMap {
+		return []reconcile.Request{
+			{NamespacedName: types.NamespacedName{
+				Name: OLSConfigName,
+			}},
+		}
+	}
+	// Watch the PostgreSQL serving certificate Secret
+	if obj.GetName() == PostgresCertsSecretName {
+		return []reconcile.Request{
+			{NamespacedName: types.NamespacedName{
+				Name: OLSConfigName,
+			}},
+		}
+	}
+	return nil
+}