squash: added option to retain n latest versions (#747)

Signed-off-by: Frederic BIDON <frederic@oneconcern.com>

Signed-off-by: Frederic BIDON <frederic@oneconcern.com>

Author: fredbi

cmd/datamon/cmd/context_squash.go

@@ -69,6 +69,7 @@ func applyRepoSquash(remoteStores context2.Stores, datamonFlags *flagsT, logger
 		return core.RepoSquash(remoteStores, repo.Name,
 			core.WithRetainTags(datamonFlags.squash.RetainTags),
 			core.WithRetainSemverTags(datamonFlags.squash.RetainSemverTags),
+			core.WithRetainNLatest(datamonFlags.squash.RetainNLatest),
 			core.ConcurrentList(datamonFlags.core.ConcurrencyFactor),
 			core.BatchSize(datamonFlags.core.BatchSize),
 		)
@@ -81,6 +82,7 @@ func init() {
 	)
 	addRetainTagsFlag(contextSquash)
 	addRetainSemverTagsFlag(contextSquash)
+	addRetainNLatestFlag(contextSquash)
 	addCoreConcurrencyFactorFlag(contextSquash, 500)
 	addBatchSizeFlag(contextSquash)
 

cmd/datamon/cmd/flags.go

@@ -105,6 +105,7 @@ type flagsT struct {
 	squash struct {
 		RetainTags       bool
 		RetainSemverTags bool
+		RetainNLatest    int
 	}
 }
 
@@ -641,6 +642,14 @@ func addRepoSizeFlag(cmd *cobra.Command) string {
 	return c
 }
 
+func addRetainNLatestFlag(cmd *cobra.Command) string {
+	const c = "retain-n-latest"
+	if cmd != nil {
+		cmd.Flags().IntVar(&datamonFlags.squash.RetainNLatest, c, 1, "Squash past bundles and retain n latest versions. May be combined with retain-tags and retain-semver-flags")
+	}
+	return c
+}
+
 /** parameters struct from other formats */
 
 // apply config file + env vars to structure used to parse cli flags

cmd/datamon/cmd/repo_squash.go

@@ -35,6 +35,7 @@ Optionally, the squashing may also retain past tagged bundles, or only past tagg
 		err = core.RepoSquash(remoteStores, datamonFlags.repo.RepoName,
 			core.WithRetainTags(datamonFlags.squash.RetainTags),
 			core.WithRetainSemverTags(datamonFlags.squash.RetainSemverTags),
+			core.WithRetainNLatest(datamonFlags.squash.RetainNLatest),
 			core.ConcurrentList(datamonFlags.core.ConcurrencyFactor),
 			core.BatchSize(datamonFlags.core.BatchSize),
 		)
@@ -67,6 +68,7 @@ func init() {
 	)
 	addRetainTagsFlag(repoSquash)
 	addRetainSemverTagsFlag(repoSquash)
+	addRetainNLatestFlag(repoSquash)
 	addCoreConcurrencyFactorFlag(repoSquash, 500)
 	addBatchSizeFlag(repoSquash)
 

cmd/datamon/cmd/repo_squash_test.go

@@ -222,15 +222,25 @@ func TestSquashRepo(t *testing.T) {
 		require.Contains(t, lines[2], "sixth and last commit for the repo")
 	})
 
-	t.Run("should squash repo13 with no tags retained and get only the last commit", func(t *testing.T) {
+	runCmd(t, []string{"bundle",
+		"upload",
+		"--path", input3,
+		"--message", "keep this commit",
+		"--repo", repo13,
+		"--context", dcontext,
+	}, fmt.Sprintf("added bundle at %q", input3), false)
+
+	t.Run("should squash repo13 with no tags retained and get only the 2 last commits", func(t *testing.T) {
 		r, w := startCapture(t)
 		runCmd(t, []string{"repo",
 			"squash",
+			"--retain-n-latest", "2",
 			"--repo", repo13,
 		}, fmt.Sprintf("squashed repo %q", repo13), false)
 		lines := endCapture(t, r, w, []string{})
-		require.Len(t, lines, 1)
+		require.Len(t, lines, 2)
 		require.Contains(t, lines[0], "sixth and last commit for the repo")
+		require.Contains(t, lines[1], "keep this commit")
 	})
 
 	t.Run("with squash context", func(t *testing.T) {

pkg/core/options.go

@@ -24,6 +24,7 @@ type Settings struct {
 	ignoreCorruptedMetadata bool
 	retainTags              bool
 	retainSemverTags        bool
+	retainNLatest           int
 	// m *M // TODO(fred): enable metrics for list operations
 }
 
@@ -113,10 +114,19 @@ func WithRetainSemverTags(enabled bool) Option {
 	}
 }
 
+func WithRetainNLatest(n int) Option {
+	return func(s *Settings) {
+		if n > 0 {
+			s.retainNLatest = n
+		}
+	}
+}
+
 func defaultSettings() Settings {
 	return Settings{
 		concurrentList: defaultListConcurrency,
 		batchSize:      defaultBatchSize,
 		memProfDir:     ".",
+		retainNLatest:  1,
 	}
 }

pkg/core/repo_squash.go

@@ -13,15 +13,18 @@ func RepoSquash(stores context2.Stores, repoName string, opts ...Option) error {
 		return err
 	}
 
-	if len(bundles) < 2 {
-		// nothing to be squashed
-		return nil
-	}
-
 	settings := defaultSettings()
 	for _, bApply := range opts {
 		bApply(&settings)
 	}
+	if settings.retainNLatest == 0 {
+		settings.retainNLatest = 1
+	}
+
+	if len(bundles) < settings.retainNLatest+1 {
+		// nothing to be squashed
+		return nil
+	}
 
 	labelsIndex := make(map[string]struct{}, 10)
 	if settings.retainTags || settings.retainSemverTags {
@@ -52,7 +55,7 @@ func RepoSquash(stores context2.Stores, repoName string, opts ...Option) error {
 	sort.SliceStable(bundles, func(i, j int) bool {
 		return bundles[i].Timestamp.Before(bundles[j].Timestamp)
 	})
-	for _, bundle := range bundles[:len(bundles)-1] {
+	for _, bundle := range bundles[:len(bundles)-settings.retainNLatest] {
 		if settings.retainTags || settings.retainSemverTags {
 			if _, retain := labelsIndex[bundle.ID]; retain {
 				continue