Skip to content

Commit 1ae29da

Browse files
dsessler7dsessler7
committed
Allow user to set an annotation that will specify an existing PVC to be mounted to cloud backup jobs so that the backup logs can be persisted.
1 parent 899c4a0 commit 1ae29da

File tree

8 files changed

+397
-32
lines changed

8 files changed

+397
-32
lines changed

‎internal/controller/postgrescluster/pgbackrest.go

Lines changed: 50 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ import (
3838
"github.com/crunchydata/postgres-operator/internal/pgbackrest"
3939
"github.com/crunchydata/postgres-operator/internal/pki"
4040
"github.com/crunchydata/postgres-operator/internal/postgres"
41+
"github.com/crunchydata/postgres-operator/internal/util"
4142
"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
4243
)
4344

@@ -771,7 +772,7 @@ func (r *Reconciler) generateRepoVolumeIntent(postgresCluster *v1beta1.PostgresC
771772
}
772773

773774
// generateBackupJobSpecIntent generates a JobSpec for a pgBackRest backup job
774-
func generateBackupJobSpecIntent(ctx context.Context, postgresCluster *v1beta1.PostgresCluster,
775+
func (r *Reconciler) generateBackupJobSpecIntent(ctx context.Context, postgresCluster *v1beta1.PostgresCluster,
775776
repo v1beta1.PGBackRestRepo, serviceAccountName string,
776777
labels, annotations map[string]string, opts ...string) *batchv1.JobSpec {
777778

@@ -873,6 +874,27 @@ func generateBackupJobSpecIntent(ctx context.Context, postgresCluster *v1beta1.P
873874
// to read certificate files
874875
jobSpec.Template.Spec.SecurityContext = postgres.PodSecurityContext(postgresCluster)
875876
pgbackrest.AddConfigToCloudBackupJob(postgresCluster, &jobSpec.Template)
877+
878+
// If the user has specified a PVC to use as a log volume via the PGBackRestCloudLogVolume
879+
// annotation, check for the PVC. If we find it, mount it to the backup job.
880+
// Otherwise, create a warning event.
881+
if logVolumeName := postgresCluster.Annotations[naming.PGBackRestCloudLogVolume]; logVolumeName != "" {
882+
logVolume := &corev1.PersistentVolumeClaim{
883+
ObjectMeta: metav1.ObjectMeta{
884+
Name: logVolumeName,
885+
Namespace: postgresCluster.GetNamespace(),
886+
},
887+
}
888+
err := errors.WithStack(r.Client.Get(ctx,
889+
client.ObjectKeyFromObject(logVolume), logVolume))
890+
if err != nil {
891+
// PVC not retrieved, create warning event
892+
r.Recorder.Event(postgresCluster, corev1.EventTypeWarning, "PGBackRestCloudLogVolumeNotFound", err.Error())
893+
} else {
894+
// We successfully found the specified PVC, so we will add it to the backup job
895+
util.AddVolumeAndMountsToPod(&jobSpec.Template.Spec, logVolume)
896+
}
897+
}
876898
}
877899

878900
return jobSpec
@@ -2040,8 +2062,31 @@ func (r *Reconciler) reconcilePGBackRestConfig(ctx context.Context,
20402062
repoHostName, configHash, serviceName, serviceNamespace string,
20412063
instanceNames []string) error {
20422064

2065+
// If the user has specified a PVC to use as a log volume for cloud backups via the
2066+
// PGBackRestCloudLogVolume annotation, check for the PVC. If we find it, set the cloud
2067+
// log path. If the user has specified a PVC, but we can't find it, create a warning event.
2068+
cloudLogPath := ""
2069+
if logVolumeName := postgresCluster.Annotations[naming.PGBackRestCloudLogVolume]; logVolumeName != "" {
2070+
logVolume := &corev1.PersistentVolumeClaim{
2071+
ObjectMeta: metav1.ObjectMeta{
2072+
Name: logVolumeName,
2073+
Namespace: postgresCluster.GetNamespace(),
2074+
},
2075+
}
2076+
err := errors.WithStack(r.Client.Get(ctx,
2077+
client.ObjectKeyFromObject(logVolume), logVolume))
2078+
if err != nil {
2079+
// PVC not retrieved, create warning event
2080+
r.Recorder.Event(postgresCluster, corev1.EventTypeWarning,
2081+
"PGBackRestCloudLogVolumeNotFound", err.Error())
2082+
} else {
2083+
// We successfully found the specified PVC, so we will set the log path
2084+
cloudLogPath = "/volumes/" + logVolumeName
2085+
}
2086+
}
2087+
20432088
backrestConfig, err := pgbackrest.CreatePGBackRestConfigMapIntent(ctx, postgresCluster, repoHostName,
2044-
configHash, serviceName, serviceNamespace, instanceNames)
2089+
configHash, serviceName, serviceNamespace, cloudLogPath, instanceNames)
20452090
if err != nil {
20462091
return err
20472092
}
@@ -2454,7 +2499,7 @@ func (r *Reconciler) reconcileManualBackup(ctx context.Context,
24542499
backupJob.Labels = labels
24552500
backupJob.Annotations = annotations
24562501

2457-
spec := generateBackupJobSpecIntent(ctx, postgresCluster, repo,
2502+
spec := r.generateBackupJobSpecIntent(ctx, postgresCluster, repo,
24582503
serviceAccount.GetName(), labels, annotations, backupOpts...)
24592504

24602505
backupJob.Spec = *spec
@@ -2631,7 +2676,7 @@ func (r *Reconciler) reconcileReplicaCreateBackup(ctx context.Context,
26312676
backupJob.Labels = labels
26322677
backupJob.Annotations = annotations
26332678

2634-
spec := generateBackupJobSpecIntent(ctx, postgresCluster, replicaCreateRepo,
2679+
spec := r.generateBackupJobSpecIntent(ctx, postgresCluster, replicaCreateRepo,
26352680
serviceAccount.GetName(), labels, annotations)
26362681

26372682
backupJob.Spec = *spec
@@ -3058,7 +3103,7 @@ func (r *Reconciler) reconcilePGBackRestCronJob(
30583103
// set backup type (i.e. "full", "diff", "incr")
30593104
backupOpts := []string{"--type=" + backupType}
30603105

3061-
jobSpec := generateBackupJobSpecIntent(ctx, cluster, repo,
3106+
jobSpec := r.generateBackupJobSpecIntent(ctx, cluster, repo,
30623107
serviceAccount.GetName(), labels, annotations, backupOpts...)
30633108

30643109
// Suspend cronjobs when shutdown or read-only. Any jobs that have already

‎internal/controller/postgrescluster/pgbackrest_test.go

Lines changed: 179 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ import (
4040
"github.com/crunchydata/postgres-operator/internal/pgbackrest"
4141
"github.com/crunchydata/postgres-operator/internal/pki"
4242
"github.com/crunchydata/postgres-operator/internal/testing/cmp"
43+
"github.com/crunchydata/postgres-operator/internal/testing/events"
4344
"github.com/crunchydata/postgres-operator/internal/testing/require"
4445
"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
4546
)
@@ -2601,6 +2602,17 @@ func TestCopyConfigurationResources(t *testing.T) {
26012602
}
26022603

26032604
func TestGenerateBackupJobIntent(t *testing.T) {
2605+
_, cc := setupKubernetes(t)
2606+
require.ParallelCapacity(t, 0)
2607+
ns := setupNamespace(t, cc)
2608+
2609+
recorder := events.NewRecorder(t, runtime.Scheme)
2610+
r := &Reconciler{
2611+
Client: cc,
2612+
Recorder: recorder,
2613+
Owner: ControllerName,
2614+
}
2615+
26042616
ctx := context.Background()
26052617
cluster := v1beta1.PostgresCluster{}
26062618
cluster.Name = "hippo-test"
@@ -2609,7 +2621,7 @@ func TestGenerateBackupJobIntent(t *testing.T) {
26092621
// If repo.Volume is nil, the code interprets this as a cloud repo backup,
26102622
// therefore, an "empty" input results in a job spec for a cloud repo backup
26112623
t.Run("empty", func(t *testing.T) {
2612-
spec := generateBackupJobSpecIntent(ctx,
2624+
spec := r.generateBackupJobSpecIntent(ctx,
26132625
&cluster, v1beta1.PGBackRestRepo{},
26142626
"",
26152627
nil, nil,
@@ -2670,7 +2682,7 @@ volumes:
26702682
})
26712683

26722684
t.Run("volumeRepo", func(t *testing.T) {
2673-
spec := generateBackupJobSpecIntent(ctx,
2685+
spec := r.generateBackupJobSpecIntent(ctx,
26742686
&cluster, v1beta1.PGBackRestRepo{
26752687
Volume: &v1beta1.RepoPVC{
26762688
VolumeClaimSpec: v1beta1.VolumeClaimSpec{},
@@ -2747,7 +2759,7 @@ volumes:
27472759
ImagePullPolicy: corev1.PullAlways,
27482760
},
27492761
}
2750-
job := generateBackupJobSpecIntent(ctx,
2762+
job := r.generateBackupJobSpecIntent(ctx,
27512763
cluster, v1beta1.PGBackRestRepo{},
27522764
"",
27532765
nil, nil,
@@ -2762,7 +2774,7 @@ volumes:
27622774
cluster.Spec.Backups = v1beta1.Backups{
27632775
PGBackRest: v1beta1.PGBackRestArchive{},
27642776
}
2765-
job := generateBackupJobSpecIntent(ctx,
2777+
job := r.generateBackupJobSpecIntent(ctx,
27662778
cluster, v1beta1.PGBackRestRepo{},
27672779
"",
27682780
nil, nil,
@@ -2779,7 +2791,7 @@ volumes:
27792791
},
27802792
},
27812793
}
2782-
job := generateBackupJobSpecIntent(ctx,
2794+
job := r.generateBackupJobSpecIntent(ctx,
27832795
cluster, v1beta1.PGBackRestRepo{},
27842796
"",
27852797
nil, nil,
@@ -2818,7 +2830,7 @@ volumes:
28182830
},
28192831
},
28202832
}
2821-
job := generateBackupJobSpecIntent(ctx,
2833+
job := r.generateBackupJobSpecIntent(ctx,
28222834
cluster, v1beta1.PGBackRestRepo{},
28232835
"",
28242836
nil, nil,
@@ -2831,7 +2843,7 @@ volumes:
28312843
cluster.Spec.Backups.PGBackRest.Jobs = &v1beta1.BackupJobs{
28322844
PriorityClassName: initialize.String("some-priority-class"),
28332845
}
2834-
job := generateBackupJobSpecIntent(ctx,
2846+
job := r.generateBackupJobSpecIntent(ctx,
28352847
cluster, v1beta1.PGBackRestRepo{},
28362848
"",
28372849
nil, nil,
@@ -2849,7 +2861,7 @@ volumes:
28492861
cluster.Spec.Backups.PGBackRest.Jobs = &v1beta1.BackupJobs{
28502862
Tolerations: tolerations,
28512863
}
2852-
job := generateBackupJobSpecIntent(ctx,
2864+
job := r.generateBackupJobSpecIntent(ctx,
28532865
cluster, v1beta1.PGBackRestRepo{},
28542866
"",
28552867
nil, nil,
@@ -2863,14 +2875,14 @@ volumes:
28632875
t.Run("Undefined", func(t *testing.T) {
28642876
cluster.Spec.Backups.PGBackRest.Jobs = nil
28652877

2866-
spec := generateBackupJobSpecIntent(ctx,
2878+
spec := r.generateBackupJobSpecIntent(ctx,
28672879
cluster, v1beta1.PGBackRestRepo{}, "", nil, nil,
28682880
)
28692881
assert.Assert(t, spec.TTLSecondsAfterFinished == nil)
28702882

28712883
cluster.Spec.Backups.PGBackRest.Jobs = &v1beta1.BackupJobs{}
28722884

2873-
spec = generateBackupJobSpecIntent(ctx,
2885+
spec = r.generateBackupJobSpecIntent(ctx,
28742886
cluster, v1beta1.PGBackRestRepo{}, "", nil, nil,
28752887
)
28762888
assert.Assert(t, spec.TTLSecondsAfterFinished == nil)
@@ -2881,7 +2893,7 @@ volumes:
28812893
TTLSecondsAfterFinished: initialize.Int32(0),
28822894
}
28832895

2884-
spec := generateBackupJobSpecIntent(ctx,
2896+
spec := r.generateBackupJobSpecIntent(ctx,
28852897
cluster, v1beta1.PGBackRestRepo{}, "", nil, nil,
28862898
)
28872899
if assert.Check(t, spec.TTLSecondsAfterFinished != nil) {
@@ -2894,14 +2906,169 @@ volumes:
28942906
TTLSecondsAfterFinished: initialize.Int32(100),
28952907
}
28962908

2897-
spec := generateBackupJobSpecIntent(ctx,
2909+
spec := r.generateBackupJobSpecIntent(ctx,
28982910
cluster, v1beta1.PGBackRestRepo{}, "", nil, nil,
28992911
)
29002912
if assert.Check(t, spec.TTLSecondsAfterFinished != nil) {
29012913
assert.Equal(t, *spec.TTLSecondsAfterFinished, int32(100))
29022914
}
29032915
})
29042916
})
2917+
2918+
t.Run("CloudLogVolumeAnnotationNoPvc", func(t *testing.T) {
2919+
cluster.Namespace = ns.Name
2920+
cluster.Annotations = map[string]string{}
2921+
cluster.Annotations[naming.PGBackRestCloudLogVolume] = "some-pvc"
2922+
spec := r.generateBackupJobSpecIntent(ctx,
2923+
&cluster, v1beta1.PGBackRestRepo{},
2924+
"",
2925+
nil, nil,
2926+
)
2927+
assert.Assert(t, cmp.MarshalMatches(spec.Template.Spec, `
2928+
containers:
2929+
- command:
2930+
- /bin/pgbackrest
2931+
- backup
2932+
- --stanza=db
2933+
- --repo=
2934+
name: pgbackrest
2935+
resources: {}
2936+
securityContext:
2937+
allowPrivilegeEscalation: false
2938+
capabilities:
2939+
drop:
2940+
- ALL
2941+
privileged: false
2942+
readOnlyRootFilesystem: true
2943+
runAsNonRoot: true
2944+
seccompProfile:
2945+
type: RuntimeDefault
2946+
volumeMounts:
2947+
- mountPath: /etc/pgbackrest/conf.d
2948+
name: pgbackrest-config
2949+
readOnly: true
2950+
- mountPath: /tmp
2951+
name: tmp
2952+
enableServiceLinks: false
2953+
restartPolicy: Never
2954+
securityContext:
2955+
fsGroup: 26
2956+
fsGroupChangePolicy: OnRootMismatch
2957+
volumes:
2958+
- name: pgbackrest-config
2959+
projected:
2960+
sources:
2961+
- configMap:
2962+
items:
2963+
- key: pgbackrest_cloud.conf
2964+
path: pgbackrest_cloud.conf
2965+
name: hippo-test-pgbackrest-config
2966+
- secret:
2967+
items:
2968+
- key: pgbackrest.ca-roots
2969+
path: ~postgres-operator/tls-ca.crt
2970+
- key: pgbackrest-client.crt
2971+
path: ~postgres-operator/client-tls.crt
2972+
- key: pgbackrest-client.key
2973+
mode: 384
2974+
path: ~postgres-operator/client-tls.key
2975+
name: hippo-test-pgbackrest
2976+
- emptyDir:
2977+
sizeLimit: 16Mi
2978+
name: tmp
2979+
`))
2980+
2981+
assert.Equal(t, len(recorder.Events), 1)
2982+
assert.Equal(t, recorder.Events[0].Regarding.Name, cluster.Name)
2983+
assert.Equal(t, recorder.Events[0].Reason, "PGBackRestCloudLogVolumeNotFound")
2984+
assert.Equal(t, recorder.Events[0].Note, "persistentvolumeclaims \"some-pvc\" not found")
2985+
})
2986+
2987+
t.Run("CloudLogVolumeAnnotationPvcInPlace", func(t *testing.T) {
2988+
cluster.Namespace = ns.Name
2989+
cluster.Annotations = map[string]string{}
2990+
cluster.Annotations[naming.PGBackRestCloudLogVolume] = "another-pvc"
2991+
2992+
pvc := &corev1.PersistentVolumeClaim{
2993+
ObjectMeta: metav1.ObjectMeta{
2994+
Name: "another-pvc",
2995+
Namespace: ns.Name,
2996+
},
2997+
Spec: corev1.PersistentVolumeClaimSpec(testVolumeClaimSpec()),
2998+
}
2999+
err := r.Client.Create(ctx, pvc)
3000+
assert.NilError(t, err)
3001+
3002+
spec := r.generateBackupJobSpecIntent(ctx,
3003+
&cluster, v1beta1.PGBackRestRepo{},
3004+
"",
3005+
nil, nil,
3006+
)
3007+
assert.Assert(t, cmp.MarshalMatches(spec.Template.Spec, `
3008+
containers:
3009+
- command:
3010+
- /bin/pgbackrest
3011+
- backup
3012+
- --stanza=db
3013+
- --repo=
3014+
name: pgbackrest
3015+
resources: {}
3016+
securityContext:
3017+
allowPrivilegeEscalation: false
3018+
capabilities:
3019+
drop:
3020+
- ALL
3021+
privileged: false
3022+
readOnlyRootFilesystem: true
3023+
runAsNonRoot: true
3024+
seccompProfile:
3025+
type: RuntimeDefault
3026+
volumeMounts:
3027+
- mountPath: /etc/pgbackrest/conf.d
3028+
name: pgbackrest-config
3029+
readOnly: true
3030+
- mountPath: /tmp
3031+
name: tmp
3032+
- mountPath: /volumes/another-pvc
3033+
name: another-pvc
3034+
enableServiceLinks: false
3035+
restartPolicy: Never
3036+
securityContext:
3037+
fsGroup: 26
3038+
fsGroupChangePolicy: OnRootMismatch
3039+
volumes:
3040+
- name: pgbackrest-config
3041+
projected:
3042+
sources:
3043+
- configMap:
3044+
items:
3045+
- key: pgbackrest_cloud.conf
3046+
path: pgbackrest_cloud.conf
3047+
name: hippo-test-pgbackrest-config
3048+
- secret:
3049+
items:
3050+
- key: pgbackrest.ca-roots
3051+
path: ~postgres-operator/tls-ca.crt
3052+
- key: pgbackrest-client.crt
3053+
path: ~postgres-operator/client-tls.crt
3054+
- key: pgbackrest-client.key
3055+
mode: 384
3056+
path: ~postgres-operator/client-tls.key
3057+
name: hippo-test-pgbackrest
3058+
- emptyDir:
3059+
sizeLimit: 16Mi
3060+
name: tmp
3061+
- name: another-pvc
3062+
persistentVolumeClaim:
3063+
claimName: another-pvc
3064+
`))
3065+
3066+
// No new events
3067+
assert.Equal(t, len(recorder.Events), 1)
3068+
assert.Equal(t, recorder.Events[0].Regarding.Name, cluster.Name)
3069+
assert.Equal(t, recorder.Events[0].Reason, "PGBackRestCloudLogVolumeNotFound")
3070+
assert.Equal(t, recorder.Events[0].Note, "persistentvolumeclaims \"some-pvc\" not found")
3071+
})
29053072
}
29063073

29073074
func TestGenerateRepoHostIntent(t *testing.T) {

0 commit comments

Comments
 (0)