PBM-1649 Basic tests for cleanup and delete-backup components

Author: jcechace

pbm/backup/delete_test.go

@@ -0,0 +1,375 @@
+package backup
+
+import (
+	"context"
+	"errors"
+	"log"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/percona/percona-backup-mongodb/pbm/config"
+	"github.com/percona/percona-backup-mongodb/pbm/connect"
+	"github.com/percona/percona-backup-mongodb/pbm/ctrl"
+	"github.com/percona/percona-backup-mongodb/pbm/defs"
+	pbmlog "github.com/percona/percona-backup-mongodb/pbm/log"
+	"github.com/percona/percona-backup-mongodb/pbm/storage"
+	"github.com/percona/percona-backup-mongodb/pbm/storage/fs"
+	"github.com/percona/percona-backup-mongodb/pbm/topo"
+	"github.com/percona/percona-backup-mongodb/pbm/version"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/testcontainers/testcontainers-go/modules/mongodb"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type TestEnvironment struct {
+	Mongo       *TestMongo
+	StoragePath string
+	Brief       topo.NodeBrief
+}
+
+type TestMongo struct {
+	Container *mongodb.MongoDBContainer
+	ReplSet   string
+	CS        string
+	Client    connect.Client
+	image     string
+	csParams  string
+}
+
+var TestEnv = newTestEnv()
+
+func newTestEnv() *TestEnvironment {
+	mongo := &TestMongo{
+		ReplSet:  "rs0",
+		image:    "perconalab/percona-server-mongodb:8.0.4-multi",
+		csParams: "directConnection=true",
+	}
+	tenv := &TestEnvironment{
+		Mongo: mongo,
+	}
+
+	err := tenv.InitStorage()
+	if err != nil {
+		log.Fatalf("failed to create temp dir: %s", err)
+	}
+
+	return tenv
+}
+
+func (tenv *TestEnvironment) InitStorage() error {
+	stg, err := os.MkdirTemp("", "pbm_tests_*")
+	if err != nil {
+		return err
+	}
+	tenv.StoragePath = stg
+	return nil
+}
+
+func (tenv *TestEnvironment) GetStorageTmpDir(name string) (string, error) {
+	if name == "" {
+		name = "default"
+	}
+	return os.MkdirTemp(tenv.StoragePath, name+"_*")
+}
+
+func (tenv *TestEnvironment) StartMongo(ctx context.Context) error {
+	cnt, err := mongodb.Run(ctx, tenv.Mongo.image, mongodb.WithReplicaSet(tenv.Mongo.ReplSet))
+	if err != nil {
+		return err
+	}
+
+	cs, err := cnt.ConnectionString(ctx)
+	if err != nil {
+		return err
+	}
+	cs += "&" + tenv.Mongo.csParams
+
+	client, err := connect.Connect(ctx, cs, "test")
+	if err != nil {
+		return err
+	}
+
+	tenv.Mongo.CS = cs
+	tenv.Mongo.Client = client
+	tenv.Mongo.Container = cnt
+
+	info, err := topo.GetNodeInfo(ctx, client.MongoClient())
+	if err != nil {
+		return err
+	}
+	mongoVersion, err := version.GetMongoVersion(ctx, client.MongoClient())
+	if err != nil {
+		return err
+	}
+	TestEnv.Brief = topo.NodeBrief{
+		URI:       cs,
+		SetName:   info.SetName,
+		Me:        info.Me,
+		Sharded:   info.IsSharded(),
+		ConfigSvr: info.IsConfigSrv(),
+		Version:   mongoVersion,
+	}
+
+	return nil
+}
+
+func (tm *TestMongo) StopMongo(ctx context.Context) error {
+	return tm.Container.Terminate(ctx)
+}
+
+func (tenv *TestEnvironment) Cleanup(ctx context.Context) error {
+	mongoErr := tenv.Mongo.StopMongo(ctx)
+	stgErr := tenv.cleanupStorage()
+
+	return errors.Join(mongoErr, stgErr)
+}
+
+func (tenv *TestEnvironment) Reset(t *testing.T) {
+	if err := tenv.resetStorage(); err != nil {
+		assert.FailNow(t, "failed to reset storage: %v", err)
+	}
+	if err := tenv.resetMongo(t.Context()); err != nil {
+		assert.FailNow(t, "failed to reset mongo: %v", err)
+	}
+	if err := tenv.SetConfigWithStorage(nil); err != nil {
+		assert.FailNow(t, "failed to reset config: %v", err)
+	}
+}
+
+func (tenv *TestEnvironment) SetConfigWithStorage(cfg *config.Config) error {
+	if cfg == nil {
+		cfg = &config.Config{}
+	}
+
+	path, err := tenv.GetStorageTmpDir(cfg.Name)
+	if err != nil {
+		return err
+	}
+
+	cfg.Storage = config.StorageConf{
+		Type: storage.Filesystem,
+		Filesystem: &fs.Config{
+			Path: path,
+		},
+	}
+
+	if cfg.IsProfile {
+		err = config.AddProfile(context.Background(), tenv.Mongo.Client, cfg)
+	} else {
+		err = config.SetConfig(context.Background(), tenv.Mongo.Client, cfg)
+	}
+
+	return err
+}
+
+func (tenv *TestEnvironment) AddStorageProfile(t *testing.T, name string) {
+	err := tenv.SetConfigWithStorage(&config.Config{IsProfile: true, Name: name})
+	if err != nil {
+		assert.FailNow(t, "failed to add storage profile: %v", err)
+	}
+}
+
+func (tenv *TestEnvironment) cleanupStorage() error {
+	err := os.RemoveAll(tenv.StoragePath)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (tenv *TestEnvironment) resetStorage() error {
+	err := tenv.cleanupStorage()
+	if err != nil {
+		return err
+	}
+	return tenv.InitStorage()
+}
+
+func (tenv *TestEnvironment) resetMongo(ctx context.Context) error {
+	// clear admin.pbm* collections
+	mongo := tenv.Mongo.Client.MongoClient()
+
+	// list and drop databases except local, config and admin
+	sysDBs := bson.A{"local", "config", "admin", defs.DB}
+	sysFilter := bson.D{{Key: "name", Value: bson.D{{Key: "$nin", Value: sysDBs}}}}
+	dbs, err := mongo.ListDatabaseNames(ctx, sysFilter)
+	if err != nil {
+		return err
+	}
+
+	for _, db := range dbs {
+		err = mongo.Database(db).Drop(ctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	// drop PBM collections
+	db := mongo.Database(defs.DB)
+	pbmColFilter := bson.D{{Key: "name", Value: primitive.Regex{Pattern: "^pbm"}}}
+	collections, err := db.ListCollectionNames(ctx, pbmColFilter)
+	if err != nil {
+		return err
+	}
+	for _, coll := range collections {
+		err = db.Collection(coll).Drop(ctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func TestMain(m *testing.M) {
+	ctx := context.Background()
+
+	err := TestEnv.StartMongo(ctx)
+	if err != nil {
+		log.Fatalf("failed to start test MongoDB: %s", err)
+	}
+
+	defer func() {
+		err = TestEnv.Cleanup(ctx)
+		if err != nil {
+			log.Fatalf("failed to cleanup test environment: %s", err)
+		}
+	}()
+
+	code := m.Run()
+	os.Exit(code)
+}
+
+func TestDeleteBackup(t *testing.T) {
+	TestEnv.Reset(t)
+	TestEnv.AddStorageProfile(t, "secondary")
+
+	// run backups
+	runBackup(t, t.Context(), TestEnv.Mongo.Client, TestEnv.Brief, "")
+	runBackup(t, t.Context(), TestEnv.Mongo.Client, TestEnv.Brief, "secondary")
+	assertBackupCount(t, TestEnv.Mongo.Client, 1, "")
+	assertBackupCount(t, TestEnv.Mongo.Client, 1, "secondary")
+
+	t.Run("MakeCleanupInfo", func(t *testing.T) {
+		infos := runCleanupInfo(t, t.Context(), TestEnv.Mongo.Client, "")
+		assertBackupList(t, infos.Backups, "", 1)
+
+		infos = runCleanupInfo(t, t.Context(), TestEnv.Mongo.Client, "secondary")
+		assertBackupList(t, infos.Backups, "secondary", 1)
+		assert.Len(t, infos.Chunks, 0, "expected no chunks in profile cleanup info")
+	})
+
+	t.Run("listDeleteBackupBefore", func(t *testing.T) {
+		bcps := runListBefore(t, t.Context(), TestEnv.Mongo.Client, "")
+		assertBackupList(t, bcps, "", 1)
+
+		bcps = runListBefore(t, t.Context(), TestEnv.Mongo.Client, "secondary")
+		assertBackupList(t, bcps, "secondary", 1)
+	})
+
+	t.Run("DeleteBackupBefore-default", func(t *testing.T) {
+		runDeleteBefore(t, t.Context(), TestEnv.Mongo.Client, "")
+		assertBackupCount(t, TestEnv.Mongo.Client, 0, "")
+		assertBackupCount(t, TestEnv.Mongo.Client, 1, "secondary")
+	})
+
+	runBackup(t, t.Context(), TestEnv.Mongo.Client, TestEnv.Brief, "")
+	assertBackupCount(t, TestEnv.Mongo.Client, 1, "")
+
+	t.Run("DeleteBackupBefore-profile", func(t *testing.T) {
+		runDeleteBefore(t, t.Context(), TestEnv.Mongo.Client, "secondary")
+		assertBackupCount(t, TestEnv.Mongo.Client, 1, "")
+		assertBackupCount(t, TestEnv.Mongo.Client, 0, "secondary")
+	})
+}
+
+func assertBackupList(t *testing.T, bcps []BackupMeta, profile string, count int) {
+	assert.Len(t, bcps, count, "expected %d backups, got %d", count, len(bcps))
+	for _, bcp := range bcps {
+		isProfile := profile != ""
+		assert.Equal(t, bcp.Store.Name, profile,
+			"expected backup %q store profile to be %q, got %q", bcp.Name, profile, bcp.Store.Name,
+		)
+		assert.Equal(t, bcp.Store.IsProfile, isProfile,
+			"expected backup %q profile to be %v, got %v", bcp.Name, isProfile, bcp.Store.IsProfile,
+		)
+	}
+
+}
+
+func runDeleteBefore(t *testing.T, ctx context.Context, client connect.Client, profile string) {
+	ts := time.Now().UTC()
+	err := DeleteBackupBefore(ctx, client, ts, "", profile, TestEnv.Brief.Me)
+	require.NoError(t, err)
+}
+
+func runCleanupInfo(t *testing.T, ctx context.Context, client connect.Client, profile string) CleanupInfo {
+	ts := time.Now().UTC().Unix()
+	bsonTs := primitive.Timestamp{T: uint32(ts)}
+	infos, err := MakeCleanupInfo(ctx, client, bsonTs, profile)
+	require.NoError(t, err)
+
+	return infos
+}
+
+func runListBefore(t *testing.T, ctx context.Context, client connect.Client, profile string) []BackupMeta {
+	ts := time.Now().UTC().Unix()
+	bsonTs := primitive.Timestamp{T: uint32(ts)}
+	bcps, err := ListDeleteBackupBefore(ctx, client, bsonTs, "", profile)
+	require.NoError(t, err)
+
+	return bcps
+}
+
+func runBackup(t *testing.T, ctx context.Context, client connect.Client, brief topo.NodeBrief, profile string) {
+	ts := time.Now().UTC().Format(time.RFC3339)
+	opid := ctrl.OPID(primitive.NilObjectID)
+	cmd := ctrl.BackupCmd{
+		Type: defs.LogicalBackup,
+		Name: ts,
+	}
+
+	cfgFull, err := config.GetProfiledConfig(ctx, client, profile)
+	require.NoError(t, err)
+
+	bcp := New(client, client.MongoClient(), brief, 1)
+	bcp.SetConfig(cfgFull)
+
+	err = bcp.Init(ctx, &cmd, opid, topo.BalancerModeOff)
+	require.NoError(t, err)
+
+	err = bcp.Run(ctx, &cmd, opid, pbmlog.DiscardEvent)
+	require.NoError(t, err)
+}
+
+func assertBackupCount(t *testing.T, client connect.Client, count int, profile string) {
+	cfg, err := config.GetProfiledConfig(t.Context(), client, profile)
+	require.NoError(t, err)
+
+	f := bson.D{{Key: "store.profile", Value: nil}}
+	if profile != "" {
+		f = bson.D{
+			{Key: "store.profile", Value: true},
+			{Key: "store.name", Value: profile},
+		}
+	}
+	cur, err := client.BcpCollection().Find(t.Context(), f)
+	assert.Nil(t, err)
+
+	var bcps []BackupMeta
+	err = cur.All(t.Context(), &bcps)
+	require.NoError(t, err)
+	assert.Len(t, bcps, count, "expected %d backups, got %d", count, len(bcps))
+
+	stg, err := fs.New(cfg.Storage.Filesystem)
+	require.NoError(t, err)
+
+	for _, bcp := range bcps {
+		name := bcp.Name
+		err := CheckBackupFiles(t.Context(), stg, name)
+		assert.Nil(t, err, "backup %q files check failed", name)
+	}
+}