Support "run-finally" in backup-section & profile (#70)

Author: jkellerer

README.md

@@ -665,24 +665,29 @@ documents:
   run-before: "echo == run-before profile $PROFILE_NAME command $PROFILE_COMMAND"
   run-after: "echo == run-after profile $PROFILE_NAME command $PROFILE_COMMAND"
   run-after-fail: "echo == Error in profile $PROFILE_NAME command $PROFILE_COMMAND: $ERROR"
+  run-finally: "echo == run-finally $PROFILE_NAME command $PROFILE_COMMAND"
   backup:
     run-before: "echo === run-before backup profile $PROFILE_NAME command $PROFILE_COMMAND"
     run-after: "echo === run-after backup profile $PROFILE_NAME command $PROFILE_COMMAND"
+    run-finally: "echo == run-finally $PROFILE_NAME command $PROFILE_COMMAND"
     source: ~/Documents
 ```
 
-`run-before`, `run-after` and `run-after-fail` can be a string, or an array of strings if you need to run more than one command
+`run-before`, `run-after`, `run-after-fail` and `run-finally` can be a string, or an array of strings if you need to run more than one command
 
 A few environment variables will be set before running these commands:
 - `PROFILE_NAME`
 - `PROFILE_COMMAND`: backup, check, forget, etc.
 
-Additionally for the `run-after-fail` commands, these environment variables will also be available:
+Additionally, for the `run-after-fail` commands, these environment variables will also be available:
 - `ERROR` containing the latest error message
 - `ERROR_COMMANDLINE` containing the command line that failed
 - `ERROR_EXIT_CODE` containing the exit code of the command line that failed
 - `ERROR_STDERR` containing any message that the failed command sent to the standard error (stderr)
 
+The commands of `run-finally` get the environment of `run-after-fail` when `run-before`, `run-after` or `restic` failed. 
+Failures in `run-finally` are logged but do not influence environment or return code.
+
 ## run before and after order during a backup
 
 The commands will be running in this order **during a backup**:
@@ -691,6 +696,9 @@ The commands will be running in this order **during a backup**:
 - run the restic backup (with check and retention if configured) - if error, go to `run-after-fail`
 - `run-after` from the backup section - if error, go to `run-after-fail`
 - `run-after` from the profile - if error, go to `run-after-fail`
+- If error: `run-after-fail` from the profile - if error, go to `run-finally`
+- `run-finally` from the backup section - if error, log and continue with next
+- `run-finally` from the profile - if error, log and continue with next
 
 # Warnings from restic
 

config/profile.go

@@ -29,6 +29,7 @@ type Profile struct {
 	RunBefore            []string                     `mapstructure:"run-before"`
 	RunAfter             []string                     `mapstructure:"run-after"`
 	RunAfterFail         []string                     `mapstructure:"run-after-fail"`
+	RunFinally           []string                     `mapstructure:"run-finally"`
 	StatusFile           string                       `mapstructure:"status-file"`
 	PrometheusSaveToFile string                       `mapstructure:"prometheus-save-to-file"`
 	PrometheusPush       string                       `mapstructure:"prometheus-push"`
@@ -51,6 +52,7 @@ type BackupSection struct {
 	CheckAfter       bool                   `mapstructure:"check-after"`
 	RunBefore        []string               `mapstructure:"run-before"`
 	RunAfter         []string               `mapstructure:"run-after"`
+	RunFinally       []string               `mapstructure:"run-finally"`
 	UseStdin         bool                   `mapstructure:"stdin" argument:"stdin"`
 	Source           []string               `mapstructure:"source"`
 	Exclude          []string               `mapstructure:"exclude" argument:"exclude" argument-type:"no-glob"`

wrapper.go

@@ -189,6 +189,9 @@ func (r *resticWrapper) runProfile() error {
 			func(err error) {
 				_ = r.runProfilePostFailCommand(err)
 			},
+			func(err error) {
+				r.runFinalCommand(r.command, err)
+			},
 		)
 	})
 	if err != nil {
@@ -420,22 +423,7 @@ func (r *resticWrapper) runProfilePostFailCommand(fail error) error {
 	}
 	env := append(os.Environ(), r.getEnvironment()...)
 	env = append(env, r.getProfileEnvironment()...)
-	env = append(env, fmt.Sprintf("ERROR=%s", fail.Error()))
-
-	if fail, ok := fail.(*commandError); ok {
-		exitCode := -1
-		if code, err := fail.ExitCode(); err == nil {
-			exitCode = code
-		}
-
-		env = append(env,
-			fmt.Sprintf("ERROR_COMMANDLINE=%s", fail.Commandline()),
-			fmt.Sprintf("ERROR_EXIT_CODE=%d", exitCode),
-			fmt.Sprintf("ERROR_STDERR=%s", fail.Stderr()),
-			// Deprecated: STDERR can originate from (pre/post)-command which doesn't need to be restic
-			fmt.Sprintf("RESTIC_STDERR=%s", fail.Stderr()),
-		)
-	}
+	env = append(env, r.getFailEnvironment(fail)...)
 
 	for i, postCommand := range r.profile.RunAfterFail {
 		clog.Debugf("starting 'run-after-fail' profile command %d/%d", i+1, len(r.profile.RunAfterFail))
@@ -451,6 +439,37 @@ func (r *resticWrapper) runProfilePostFailCommand(fail error) error {
 	return nil
 }
 
+func (r *resticWrapper) runFinalCommand(command string, fail error) {
+	var commands []string
+
+	if command == constants.CommandBackup && r.profile.Backup != nil && r.profile.Backup.RunFinally != nil {
+		commands = append(commands, r.profile.Backup.RunFinally...)
+	}
+	if r.profile.RunFinally != nil {
+		commands = append(commands, r.profile.RunFinally...)
+	}
+
+	env := append(os.Environ(), r.getEnvironment()...)
+	env = append(env, r.getProfileEnvironment()...)
+	env = append(env, r.getFailEnvironment(fail)...)
+
+	for i := len(commands) - 1; i >= 0; i-- {
+		// Using defer stack for "finally" to ensure every command is run even on panic
+		defer func(index int, cmd string) {
+			clog.Debugf("starting final command %d/%d", index+1, len(commands))
+			rCommand := newShellCommand(cmd, nil, env, r.dryRun, r.sigChan, r.setPID)
+			// stdout are stderr are coming from the default terminal (in case they're redirected)
+			rCommand.stdout = term.GetOutput()
+			rCommand.stderr = term.GetErrorOutput()
+			_, _, err := runShellCommand(rCommand)
+			if err != nil {
+				clog.Errorf("run-finally command %d/%d failed ('%s' on profile '%s'): %w",
+					index+1, len(commands), command, r.profile.Name, err)
+			}
+		}(i, commands[i])
+	}
+}
+
 // getEnvironment returns the environment variables defined in the profile configuration
 func (r *resticWrapper) getEnvironment() []string {
 	if r.profile.Environment == nil || len(r.profile.Environment) == 0 {
@@ -477,6 +496,31 @@ func (r *resticWrapper) getProfileEnvironment() []string {
 	}
 }
 
+// getFailEnvironment returns additional environment variables describing the fail reason
+func (r *resticWrapper) getFailEnvironment(err error) (env []string) {
+	if err == nil {
+		return
+	}
+
+	env = []string{fmt.Sprintf("ERROR=%s", err.Error())}
+
+	if fail, ok := err.(*commandError); ok {
+		exitCode := -1
+		if code, err := fail.ExitCode(); err == nil {
+			exitCode = code
+		}
+
+		env = append(env,
+			fmt.Sprintf("ERROR_COMMANDLINE=%s", fail.Commandline()),
+			fmt.Sprintf("ERROR_EXIT_CODE=%d", exitCode),
+			fmt.Sprintf("ERROR_STDERR=%s", fail.Stderr()),
+			// Deprecated: STDERR can originate from (pre/post)-command which doesn't need to be restic
+			fmt.Sprintf("RESTIC_STDERR=%s", fail.Stderr()),
+		)
+	}
+	return
+}
+
 // canSucceedAfterError returns true if an error reported by running restic in runCommand can be counted as success
 func (r *resticWrapper) canSucceedAfterError(command string, summary progress.Summary, err error) bool {
 	if err == nil {
@@ -689,12 +733,20 @@ func logLockWait(lockName string, started, lastLogged time.Time, maxLockWait tim
 }
 
 // runOnFailure will run the onFailure function if an error occurred in the run function
-func runOnFailure(run func() error, onFailure func(error)) error {
-	err := run()
+func runOnFailure(run func() error, onFailure func(error), finally func(error)) (err error) {
+	// Using "defer" for finally to ensure it runs even on panic
+	if finally != nil {
+		defer func() {
+			finally(err)
+		}()
+	}
+
+	err = run()
 	if err != nil {
 		onFailure(err)
 	}
-	return err
+
+	return
 }
 
 func asExitError(err error) (*exec.ExitError, bool) {

wrapper_test.go

@@ -111,6 +111,51 @@ func TestPostFailProfile(t *testing.T) {
 	_ = os.Remove(testFile)
 }
 
+func TestFinallyProfile(t *testing.T) {
+	testFile := "TestFinallyProfile.txt"
+	defer os.Remove(testFile)
+
+	var profile *config.Profile
+	newProfile := func() {
+		_ = os.Remove(testFile)
+		profile = config.NewProfile(nil, "name")
+		profile.RunFinally = []string{"echo finally > " + testFile}
+		profile.Backup = &config.BackupSection{}
+		profile.Backup.RunFinally = []string{"echo finally-backup > " + testFile}
+	}
+
+	assertFileEquals := func(t *testing.T, expected string) {
+		content, err := os.ReadFile(testFile)
+		require.NoError(t, err)
+		assert.Equal(t, strings.TrimSpace(string(content)), expected)
+	}
+
+	t.Run("backup-before-profile", func(t *testing.T) {
+		newProfile()
+		wrapper := newResticWrapper("echo", false, profile, "backup", nil, nil)
+		err := wrapper.runProfile()
+		assert.NoError(t, err)
+		assertFileEquals(t, "finally")
+	})
+
+	t.Run("on-backup-only", func(t *testing.T) {
+		newProfile()
+		profile.RunFinally = nil
+		wrapper := newResticWrapper("echo", false, profile, "backup", nil, nil)
+		err := wrapper.runProfile()
+		assert.NoError(t, err)
+		assertFileEquals(t, "finally-backup")
+	})
+
+	t.Run("on-error", func(t *testing.T) {
+		newProfile()
+		wrapper := newResticWrapper("exit", false, profile, "1", nil, nil)
+		err := wrapper.runProfile()
+		assert.EqualError(t, err, "1 on profile 'name': exit status 1")
+		assertFileEquals(t, "finally")
+	})
+}
+
 func Example_runProfile() {
 	term.SetOutput(os.Stdout)
 	profile := config.NewProfile(nil, "name")