Add support for copy command (#73)

* add copy section (with schedules)

* docs: add section on copy command

Author: creativeprojects

.vscode/settings.json

@@ -1,5 +1,7 @@
 {
     "cSpell.words": [
+        "crond",
+        "ionice",
         "launchd",
         "restic",
         "resticprofile",

README.md

@@ -31,9 +31,10 @@ With resticprofile:
 * You can generate a simple status file to send to some monitoring software and make sure your backups are running fine 
 * You can use a template syntax in your configuration file
 * You can generate scheduled tasks using *crond*
-* **[new for v0.12.0]** Get backup statistics in your status file
+* Get backup statistics in your status file
 * **[new for v0.14.0]** Automatically clear up [stale locks](#locks)
 * **[new for v0.15.0]** Export a **prometheus** file after a backup, or send the report to a push gateway automatically
+* **[new for v0.16.0]** Full support for the copy command (with scheduling)
 
 The configuration file accepts various formats:
 * [TOML](https://github.com/toml-lang/toml) : configuration file with extension _.toml_ and _.conf_ to keep compatibility with versions before 0.6.0
@@ -65,7 +66,8 @@ For the rest of the documentation, I'll be showing examples using different form
   * [Simple YAML configuration](#simple-yaml-configuration)
   * [More complex configuration in TOML](#more-complex-configuration-in-toml)
   * [TOML configuration example for Windows](#toml-configuration-example-for-windows)
-    * [Use stdin in configuration](#use-stdin-in-configuration)
+  * [Use stdin in configuration](#use-stdin-in-configuration)
+  * [Special case for the copy command section](#special-case-for-the-copy-command-section)
 * [Configuration paths](#configuration-paths)
   * [macOS X](#macos-x)
   * [Other unixes (Linux and BSD)](#other-unixes-linux-and-bsd)
@@ -212,7 +214,7 @@ Installation using Ansible is not supported out of the box yet, but since I'm us
 
 ## Installation from source
 
-You can download the source code and recompile it, it's actually very easy! all you need to have on your machine is:
+You can download the source code and compile it, it's actually very easy! all you need to have on your machine is:
 - `git`
 - [go compiler](https://golang.org/dl/)
 - `GNU Make` which is installed by default on many unix boxes. On debian based distributions (Ubuntu included) the package is called `build-essential`.
@@ -555,7 +557,7 @@ run-after = "echo All Done!"
 no-error-on-warning = true
 ```
 
-### Use stdin in configuration
+## Use stdin in configuration
 
 Simple example sending a file via stdin
 
@@ -572,6 +574,37 @@ tag = [ 'stdin' ]
 
 ```
 
+## Special case for the `copy` command section
+
+The copy command needs two repository (and quite likely 2 different set of keys). You can configure a `copy` section like this:
+
+```toml
+[default]
+initialize = false
+repository = "/backup/original"
+password-file = "key"
+
+    [default.copy]
+    initialize = true
+    repository = "/backup/copy"
+    password-file = "other_key"
+```
+
+You will note that the secondary repository doesn't need to have a `2` behind its flags (`repository2`, `password-file2`, etc.). It's because the flags are well separated in the configuration.
+
+Here's the same configuration in YAML format:
+
+```yaml
+default:
+    initialize: false
+    repository: "/backup/original"
+    password-file: key
+    copy:
+        initialize: true
+        repository: "/backup/copy"
+        password-file: other_key
+```
+
 # Configuration paths
 
 The default name for the configuration file is `profiles`, without an extension.
@@ -957,13 +990,14 @@ global:
 
 Each profile can be scheduled independently (groups are not available for scheduling yet).
 
-These 4 profile sections are accepting a schedule configuration:
+These 5 profile sections are accepting a schedule configuration:
 - backup
 - check
 - forget (version 0.11.0)
 - prune (version 0.11.0)
+- copy (version 0.16.0)
 
-which mean you can schedule `backup`, `forget`, `prune` and `check` independently (I recommend to use a local `lock` in this case).
+which mean you can schedule `backup`, `forget`, `prune`, `check` and `copy` independently (I recommend to use a local `lock` in this case).
 
 ## retention schedule is deprecated
 **Important**:
@@ -2075,6 +2109,7 @@ Flags passed to the restic command line
 * **password-file**: string
 * **quiet**: true / false
 * **repository**: string **(will be passed as 'repo' to the command line)**
+* **repository-file**: string
 * **tls-client-cert**: string
 * **verbose**: true / false OR integer
 
@@ -2223,6 +2258,24 @@ Flags passed to the restic command line
 * **snapshot-template**: string
 * **tag**: string OR list of strings
 
+`[profile.copy]`
+
+Flags used by resticprofile only
+
+* **initialize**: true / false
+
+
+Flags passed to the restic command line
+
+* **key-hint**: string
+* **password-command**: command
+* **password-file**: string
+* **path**: string OR list of strings
+* **repository**: repository
+* **repository-file**: string
+* **tag**: string OR list of strings
+
+
 # Appendix
 
 As an example, here's a similar configuration file in YAML:

config/confidential.go

@@ -40,7 +40,7 @@ func (c *ConfidentialValue) hideSubmatches(pattern *regexp.Regexp) {
 		return
 	}
 
-	if matches := pattern.FindStringSubmatchIndex(c.confidential); matches != nil && len(matches) > 2 {
+	if matches := pattern.FindStringSubmatchIndex(c.confidential); len(matches) > 2 {
 		c.public = c.confidential
 
 		for i := len(matches) - 2; i > 1; i -= 2 {
@@ -121,11 +121,9 @@ func getAllConfidentialValues(profile *Profile) []*ConfidentialValue {
 }
 
 func convertToNonConfidential(confidentials []*ConfidentialValue, value string) string {
-	if confidentials != nil {
-		for _, c := range confidentials {
-			if c != nil && c.IsConfidential() && value == c.Value() {
-				return c.String()
-			}
+	for _, c := range confidentials {
+		if c != nil && c.IsConfidential() && value == c.Value() {
+			return c.String()
 		}
 	}
 	return value

config/profile.go

@@ -18,6 +18,7 @@ type Profile struct {
 	Quiet                bool                         `mapstructure:"quiet" argument:"quiet"`
 	Verbose              bool                         `mapstructure:"verbose" argument:"verbose"`
 	Repository           ConfidentialValue            `mapstructure:"repository" argument:"repo"`
+	RepositoryFile       string                       `mapstructure:"repository-file" argument:"repository-file"`
 	PasswordFile         string                       `mapstructure:"password-file" argument:"password-file"`
 	CacheDir             string                       `mapstructure:"cache-dir" argument:"cache-dir"`
 	CACert               string                       `mapstructure:"cacert" argument:"cacert"`
@@ -43,6 +44,7 @@ type Profile struct {
 	Snapshots            map[string]interface{}       `mapstructure:"snapshots"`
 	Forget               *OtherSectionWithSchedule    `mapstructure:"forget"`
 	Mount                map[string]interface{}       `mapstructure:"mount"`
+	Copy                 *CopySection                 `mapstructure:"copy"`
 }
 
 // BackupSection contains the specific configuration to the 'backup' command
@@ -90,6 +92,18 @@ type ScheduleSection struct {
 	ScheduleLockWait   time.Duration `mapstructure:"schedule-lock-wait"`
 }
 
+// CopySection contains the destination parameters for a copy command
+type CopySection struct {
+	Initialize      bool              `mapstructure:"initialize"`
+	Repository      ConfidentialValue `mapstructure:"repository" argument:"repo2"`
+	RepositoryFile  string            `mapstructure:"repository-file" argument:"repository-file2"`
+	PasswordFile    string            `mapstructure:"password-file" argument:"password-file2"`
+	PasswordCommand string            `mapstructure:"password-command" argument:"password-command2"`
+	KeyHint         string            `mapstructure:"key-hint" argument:"key-hint2"`
+	ScheduleSection `mapstructure:",squash"`
+	OtherFlags      map[string]interface{} `mapstructure:",remain"`
+}
+
 // NewProfile instantiates a new blank profile
 func NewProfile(c *Config, name string) *Profile {
 	return &Profile{
@@ -108,6 +122,7 @@ func (p *Profile) SetRootPath(rootPath string) {
 
 	p.Lock = fixPath(p.Lock, expandEnv, absolutePrefix(rootPath))
 	p.PasswordFile = fixPath(p.PasswordFile, expandEnv, absolutePrefix(rootPath))
+	p.RepositoryFile = fixPath(p.RepositoryFile, expandEnv, absolutePrefix(rootPath))
 	p.CacheDir = fixPath(p.CacheDir, expandEnv, absolutePrefix(rootPath))
 	p.CACert = fixPath(p.CACert, expandEnv, absolutePrefix(rootPath))
 	p.TLSClientCert = fixPath(p.TLSClientCert, expandEnv, absolutePrefix(rootPath))
@@ -137,6 +152,11 @@ func (p *Profile) SetRootPath(rootPath string) {
 			p.Backup.Iexclude = fixPaths(p.Backup.Iexclude, expandEnv)
 		}
 	}
+
+	if p.Copy != nil {
+		p.Copy.PasswordFile = fixPath(p.Copy.PasswordFile, expandEnv, absolutePrefix(rootPath))
+		p.Copy.RepositoryFile = fixPath(p.Copy.RepositoryFile, expandEnv, absolutePrefix(rootPath))
+	}
 }
 
 // SetHost will replace any host value from a boolean to the hostname
@@ -205,6 +225,14 @@ func (p *Profile) GetCommandFlags(command string) *shell.Args {
 		if p.Mount != nil {
 			flags = addOtherArgs(flags, p.Mount)
 		}
+
+	case constants.CommandCopy:
+		if p.Copy != nil {
+			flags = convertStructToArgs(*p.Copy, flags)
+			if p.Copy.OtherFlags != nil {
+				flags = addOtherArgs(flags, p.Copy.OtherFlags)
+			}
+		}
 	}
 
 	return flags
@@ -329,6 +357,7 @@ func (p *Profile) allSchedulableSections() map[string]interface{} {
 		constants.CommandCheck:                  p.Check,
 		constants.CommandForget:                 p.Forget,
 		constants.CommandPrune:                  p.Prune,
+		constants.CommandCopy:                   p.Copy,
 	}
 }
 
@@ -339,6 +368,8 @@ func getScheduleSection(section interface{}) *ScheduleSection {
 			return &v.ScheduleSection
 		case *RetentionSection:
 			return &v.ScheduleSection
+		case *CopySection:
+			return &v.ScheduleSection
 		case *OtherSectionWithSchedule:
 			return &v.ScheduleSection
 		}

config/profile_test.go

@@ -223,6 +223,8 @@ exclude-file = "exclude"
 files-from = "include"
 exclude = "exclude"
 iexclude = "iexclude"
+[profile.copy]
+password-file = "key"
 `
 	profile, err := getProfile("toml", testConfig, "profile")
 	if err != nil {
@@ -240,6 +242,7 @@ iexclude = "iexclude"
 	assert.ElementsMatch(t, []string{"/wd/include"}, profile.Backup.FilesFrom)
 	assert.ElementsMatch(t, []string{"exclude"}, profile.Backup.Exclude)
 	assert.ElementsMatch(t, []string{"iexclude"}, profile.Backup.Iexclude)
+	assert.Equal(t, "/wd/key", profile.Copy.PasswordFile)
 }
 
 func TestHostInProfile(t *testing.T) {
@@ -556,7 +559,7 @@ initialize = true
 	}
 
 	sections := NewProfile(nil, "").SchedulableCommands()
-	assert.Len(sections, 5)
+	assert.Len(sections, 6)
 
 	for _, command := range sections {
 		// Check that schedule is supported
@@ -711,6 +714,8 @@ other-flag-forget = true
 other-flag-prune = true
 [profile.mount]
 other-flag-mount = true
+[profile.copy]
+other-flag-copy = true
 `},
 		{"json", `
 {
@@ -722,7 +727,8 @@ other-flag-mount = true
     "check": {"other-flag-check": true},
     "forget": {"other-flag-forget": true},
     "prune": {"other-flag-prune": true},
-    "mount": {"other-flag-mount": true}
+    "mount": {"other-flag-mount": true},
+    "copy": {"other-flag-copy": true}
   }
 }`},
 		{"yaml", `---
@@ -742,6 +748,8 @@ profile:
     other-flag-prune: true
   mount:
     other-flag-mount: true
+  copy:
+    other-flag-copy: true
 `},
 		{"hcl", `
 "profile" = {
@@ -767,6 +775,9 @@ profile:
 	mount = {
 		other-flag-mount = true
 	}
+	copy = {
+		other-flag-copy = true
+	}
 }
 `},
 	}
@@ -786,6 +797,7 @@ profile:
 			require.NotNil(t, profile.Mount)
 			require.NotNil(t, profile.Prune)
 			require.NotNil(t, profile.Snapshots)
+			require.NotNil(t, profile.Copy)
 
 			flags := profile.GetCommonFlags()
 			assert.Equal(t, 1, len(flags.ToMap()))
@@ -831,6 +843,12 @@ profile:
 			assert.ElementsMatch(t, []string{"1"}, flags.ToMap()["other-flag"])
 			_, found = flags.ToMap()["other-flag-mount"]
 			assert.True(t, found)
+
+			flags = profile.GetCommandFlags("copy")
+			assert.Equal(t, 2, len(flags.ToMap()))
+			assert.ElementsMatch(t, []string{"1"}, flags.ToMap()["other-flag"])
+			_, found = flags.ToMap()["other-flag-copy"]
+			assert.True(t, found)
 		})
 	}
 }

constants/command.go

@@ -10,4 +10,5 @@ const (
 	CommandSnapshots = "snapshots"
 	CommandUnlock    = "unlock"
 	CommandMount     = "mount"
+	CommandCopy      = "copy"
 )

examples/dev.yaml

@@ -25,6 +25,9 @@ default:
     password-file: key
     repository: "/Volumes/RAMDisk/{{ .Profile.Name }}"
     lock: "/Volumes/RAMDisk/resticprofile-{{ .Profile.Name }}.lock"
+    copy:
+        password-file: key
+        repository: "/Volumes/RAMDisk/{{ .Profile.Name }}-copy"
 
 space:
     description: Repository contains space
@@ -105,12 +108,16 @@ self:
       - "echo restic stderr = ${RESTIC_STDERR}"
     check:
         schedule:
-          - "*:15,45"
+          - "*:15"
     retention:
         after-backup: true
     forget:
         schedule: "weekly"
         schedule-priority: standard
+    copy:
+        initialize: true
+        schedule:
+          - "*:45"
 
 prom:
     force-inactive-lock: true

main.go

@@ -26,7 +26,7 @@ import (
 
 // These fields are populated by the goreleaser build
 var (
-	version = "0.15.0-dev"
+	version = "0.16.0-dev"
 	commit  = ""
 	date    = ""
 	builtBy = ""