[HWORKS-2433] Configure cleanup for old backups

Author: maismail

files/scripts/backups/cleanup_old_backups.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Copyright (c) 2024-2025 Hopsworks AB. All rights reserved.
+
+set -e
+
+{{- include "rondb.backups.defineBackupIdEnv" . }}
+
+REMOTE_BACKUP_BASE_DIR={{ include "rondb.rcloneBackupRemoteName" . }}:{{ include "rondb.backups.bucketName" (dict "backupConfig" .Values.backups "global" .Values.global) }}/{{ include "rondb.takeBackupPathPrefix" . }}
+
+if [ -z "$TTL" ]; then
+  echo "No TTL configuration found."
+  exit 0
+fi
+
+echo "Check expired backups in $REMOTE_BACKUP_BASE_DIR with TTL $TTL "
+
+TTL_EXPIRED=$(
+  rclone lsjson --recursive --files-only "$REMOTE_BACKUP_BASE_DIR" --min-age "$TTL" \
+  | jq -r '.[].Path | split("/") | .[0]' \
+  | sort -u
+)
+
+if [ -z "$TTL_EXPIRED" ]; then
+  echo "No TTL expired backups found."
+  exit 0
+fi
+
+echo "TTL expired backups detected:"
+echo "$TTL_EXPIRED"
+
+echo "Deleting TTL-expired backups from object storage"
+
+{{- if include "rondb.backups.metadataStore.configMapName" . }}
+CONFIGMAPS=$(kubectl get cm -n {{ .Release.Namespace }} \
+  -l "app=backups-metadata,service=rondb,managed-by=cronjob" \
+  -o jsonpath='{.items[*].metadata.name}')
+{{- end }}
+
+for id in $TTL_EXPIRED; do
+  if [ "$id" = "$BACKUP_ID" ]; then
+      echo "Skipping $id since this is the last active backup"
+      continue
+  fi
+  BACKUP_PATH="$REMOTE_BACKUP_BASE_DIR/$id"
+  echo "Deleting $BACKUP_PATH"
+  rclone delete -v "$BACKUP_PATH" --rmdirs
+  
+{{- if include "rondb.backups.metadataStore.configMapName" . }}
+  PATCH_JSON="{\"data\": {\"$id\": null}}"
+  for cm in $CONFIGMAPS; do
+    echo "Cleaning metadata from ConfigMap: $cm with $PATCH_JSON"
+    kubectl patch cm "$cm" -n {{ .Release.Namespace }} --type merge -p "$PATCH_JSON" || true
+  done
+{{- end }}
+done

templates/backups/create.yaml

@@ -71,7 +71,9 @@ spec:
             - name: backup-id
               mountPath: /home/hopsworks/backup-id
               readOnly: true
+{{- if not (include "rondb.backups.ttl" .) }}
           containers:
+{{- end }}
           - name: upload-native-backups
             image: {{ include "image_address" (dict "image" $.Values.images.toolbox) }}
             imagePullPolicy: {{ $.Values.imagePullPolicy }}
@@ -86,6 +88,32 @@ spec:
             - name: backup-id
               mountPath: /home/hopsworks/backup-id
               readOnly: true
+{{- if include "rondb.backups.ttl" . }}
+          containers:
+          - name: cleanup-old-backups
+            image: {{ include "image_address" (dict "image" $.Values.images.toolbox) }}
+            imagePullPolicy: {{ $.Values.imagePullPolicy }}
+{{ include "rondb.ContainerSecurityContext" $ | indent 12 }}
+            workingDir: /home/hopsworks
+            command:
+            - /bin/bash
+            - -c
+            - |
+{{ tpl (.Files.Get "files/scripts/backups/cleanup_old_backups.sh") . | indent 14 }}
+            env:
+            - name: TTL
+              value: {{ include "rondb.backups.ttl" . }}
+            - name: RCLONE_CONFIG
+              value: /home/hopsworks/rclone.conf
+{{- include "rondb.backup.credentials" (dict "backupConfig" $.Values.backups "namespace" $.Release.Namespace "global" $.Values.global) | indent 12 }}
+            volumeMounts:
+            - name: rclone-configs
+              mountPath: /home/hopsworks/rclone.conf
+              subPath: rclone.conf
+            - name: backup-id
+              mountPath: /home/hopsworks/backup-id
+              readOnly: true
+{{- end }}
           volumes:
           - name: rclone-configs
             configMap:

templates/shared_templates/_helpers.tpl

@@ -528,3 +528,11 @@ endpoint = {{ .global._hopsworks.managedObjectStorage.s3.endpoint }}
 {{- "s3:/" -}}
 {{- end -}}
 {{- end -}}
+
+{{- define "rondb.backups.ttl" -}}
+{{- if .Values.backups.ttl -}}
+{{- .Values.backups.ttl  -}}
+{{- else if and (include "rondb.global.backupsEnabled" .) .Values.global._hopsworks.backups.ttl -}}
+{{- .Values.global._hopsworks.backups.ttl -}}
+{{- end -}}
+{{- end -}}

values.schema.json

@@ -1634,6 +1634,15 @@
                     "description": "The name of the configmap to be used to store the backups metadata information.",
                     "type": ["string", "null"],
                     "default": null
+                },
+                "ttl": {
+                  "default": null,
+                  "description": "time to live to control when to clean up backups. It is a number followed by either d (days) or h (hours) suffix.",
+                  "pattern": "^\\d+[dh]$",
+                  "type": [
+                    "string",
+                    "null"
+                  ]
                 }
             }
         },

values.yaml

@@ -22,6 +22,7 @@ backups:
       name: null
     serverSideEncryption: null
   schedule: null
+  ttl: null
 benchmarking:
   dbt2:
     numWarehouses: 4