diff --git a/cmd/backup/config.go b/cmd/backup/config.go index b5a98d95..08232a8b 100644 --- a/cmd/backup/config.go +++ b/cmd/backup/config.go @@ -87,6 +87,11 @@ type Config struct { DropboxAppSecret string `split_words:"true"` DropboxRemotePath string `split_words:"true"` DropboxConcurrencyLevel NaturalNumber `split_words:"true" default:"6"` + GoogleDriveCredentialsJSON string `split_words:"true"` + GoogleDriveFolderID string `split_words:"true"` + GoogleDriveImpersonateSubject string `split_words:"true"` + GoogleDriveEndpoint string `split_words:"true"` + GoogleDriveTokenURL string `split_words:"true"` source string additionalEnvVars map[string]string } diff --git a/cmd/backup/script.go b/cmd/backup/script.go index 56b8af87..bcc83f04 100644 --- a/cmd/backup/script.go +++ b/cmd/backup/script.go @@ -16,6 +16,7 @@ import ( "github.com/offen/docker-volume-backup/internal/storage" "github.com/offen/docker-volume-backup/internal/storage/azure" "github.com/offen/docker-volume-backup/internal/storage/dropbox" + "github.com/offen/docker-volume-backup/internal/storage/googledrive" "github.com/offen/docker-volume-backup/internal/storage/local" "github.com/offen/docker-volume-backup/internal/storage/s3" "github.com/offen/docker-volume-backup/internal/storage/ssh" @@ -59,12 +60,13 @@ func newScript(c *Config) *script { StartTime: time.Now(), LogOutput: logBuffer, Storages: map[string]StorageStats{ - "S3": {}, - "WebDAV": {}, - "SSH": {}, - "Local": {}, - "Azure": {}, - "Dropbox": {}, + "S3": {}, + "WebDAV": {}, + "SSH": {}, + "Local": {}, + "Azure": {}, + "Dropbox": {}, + "GoogleDrive": {}, }, }, } @@ -225,6 +227,21 @@ func (s *script) init() error { s.storages = append(s.storages, dropboxBackend) } + if s.c.GoogleDriveCredentialsJSON != "" { + googleDriveConfig := googledrive.Config{ + CredentialsJSON: s.c.GoogleDriveCredentialsJSON, + FolderID: s.c.GoogleDriveFolderID, + ImpersonateSubject: s.c.GoogleDriveImpersonateSubject, + Endpoint: s.c.GoogleDriveEndpoint, + TokenURL: s.c.GoogleDriveTokenURL, + } + googleDriveBackend, err := googledrive.NewStorageBackend(googleDriveConfig, logFunc) + if err != nil { + return errwrap.Wrap(err, "error creating googledrive storage backend") + } + s.storages = append(s.storages, googleDriveBackend) + } + if s.c.EmailNotificationRecipient != "" { emailURL := fmt.Sprintf( "smtp://%s:%s@%s:%d/?from=%s&to=%s", diff --git a/docs/reference/index.md b/docs/reference/index.md index 4009315d..08e27e02 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -386,6 +386,57 @@ The values for each key currently match its default. # DROPBOX_REFRESH_TOKEN="" +########### GOOGLE DRIVE STORAGE + +# The JSON credentials for a Google service account with access to Google Drive. +# You can provide either: +# 1. The actual JSON content directly +# 2. Use the _FILE suffix to load from a file (e.g., GOOGLE_DRIVE_CREDENTIALS_JSON_FILE) +# +# Examples: +# Option 1 - JSON content: +# docker run [...] \ +# -e GOOGLE_DRIVE_CREDENTIALS_JSON='{"type":"service_account",...}' +# +# Option 2 - Using _FILE suffix (recommended for Docker Secrets): +# docker run [...] \ +# -v ./credentials.json:/creds/google-credentials.json \ +# -e GOOGLE_DRIVE_CREDENTIALS_JSON_FILE=/creds/google-credentials.json +# +# GOOGLE_DRIVE_CREDENTIALS_JSON="" + +# --- + +# The ID of the Google Drive folder where backups will be uploaded. +# You can find the folder ID in the URL when viewing the folder in Google Drive. +# +# Example: "1A2B3C4D5E6F7G8H9I0J" +# +# GOOGLE_DRIVE_FOLDER_ID="" + +# --- + +# The email address of the user to impersonate when accessing Google Drive (domain-wide delegation). +# This is required becasue your service account needs to act on behalf of a user in your organization in order to upload files. +# How to: https://support.google.com/a/answer/162106 +# Example: "user@example.com" +# +# GOOGLE_DRIVE_IMPERSONATE_SUBJECT="" + +# --- + +# (Optional) Custom Google Drive API endpoint. This is primarily for testing with a mock server. +# Example: "http://localhost:8080/drive/v3" +# +# GOOGLE_DRIVE_ENDPOINT="" + +# --- + +# (Optional) Custom token URL for Google Drive authentication. This is primarily for testing with a mock server. +# Example: "http://localhost:8080/token" +# +# GOOGLE_DRIVE_TOKEN_URL="" + ########### LOCAL FILE STORAGE # In addition to storing backups remotely, you can also keep local copies. diff --git a/go.mod b/go.mod index 3eec8cf0..3143efcf 100644 --- a/go.mod +++ b/go.mod @@ -23,10 +23,14 @@ require ( golang.org/x/crypto v0.39.0 golang.org/x/oauth2 v0.30.0 golang.org/x/sync v0.16.0 + google.golang.org/api v0.242.0 mvdan.cc/sh/v3 v3.12.0 ) require ( + cloud.google.com/go/auth v0.16.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.7.0 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect github.com/cloudflare/circl v1.6.1 // indirect @@ -40,7 +44,9 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect + github.com/googleapis/gax-go/v2 v2.14.2 // indirect github.com/minio/crc64nvme v1.0.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect @@ -48,15 +54,15 @@ require ( github.com/philhofer/fwd v1.2.0 // indirect github.com/tinylib/msgp v1.3.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect go.opentelemetry.io/otel v1.36.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 // indirect go.opentelemetry.io/otel/metric v1.36.0 // indirect - go.opentelemetry.io/otel/sdk v1.26.0 // indirect go.opentelemetry.io/otel/trace v1.36.0 // indirect - golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + golang.org/x/time v0.12.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect + google.golang.org/grpc v1.73.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect ) require ( diff --git a/go.sum b/go.sum index cb3ba014..acd2472a 100644 --- a/go.sum +++ b/go.sum @@ -15,12 +15,18 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/auth v0.16.2 h1:QvBAGFPLrDeoiNjyfVunhQ10HKNYuOwZ5noee0M5df4= +cloud.google.com/go/auth v0.16.2/go.mod h1:sRBas2Y1fB1vZTdurouM0AzuYQBMZinrUYL8EufhtEA= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= +cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -184,10 +190,16 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= +github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3GqO0k0= +github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -311,8 +323,8 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0/go.mod h1:vy+2G/6NvVMpwGX/NyLqcC41fxepnuKHk16E6IZUcJc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 h1:1u/AyyOqAWzy+SkPxDpahCNZParHV8Vid1RnI2clyDE= @@ -321,8 +333,10 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 h1:1wp/g go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0/go.mod h1:gbTHmghkGgqxMomVQQMur1Nba4M0MQ8AYThXDUjsJ38= go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= -go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8= -go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs= +go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= +go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= +go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= +go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= @@ -508,8 +522,8 @@ golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -579,6 +593,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.242.0 h1:7Lnb1nfnpvbkCiZek6IXKdJ0MFuAZNAJKQfA1ws62xg= +google.golang.org/api v0.242.0/go.mod h1:cOVEm2TpdAGHL2z+UwyS+kmlGr3bVWQQ6sYEqkKje50= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -614,10 +630,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= -google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 h1:1tXaIXCracvtsRxSBsYDiSBN0cuJvM7QYW+MrpIRY78= +google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:49MsLSx0oWMOZqcpB3uL8ZOkAh1+TndpJ8ONoCBWiZk= +google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 h1:vPV0tzlsK6EzEDHNNH5sa7Hs9bd7iXR7B1tSiPepkV0= +google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:pKLAc5OolXC3ViWGI62vvC0n10CpwAtRcTNCFwTKBEw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -630,8 +648,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.67.0 h1:IdH9y6PF5MPSdAntIcpjQ+tXO41pcQsfZV2RxtQgVcw= -google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= +google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -642,8 +660,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/internal/storage/googledrive/googledrive.go b/internal/storage/googledrive/googledrive.go new file mode 100644 index 00000000..89635be9 --- /dev/null +++ b/internal/storage/googledrive/googledrive.go @@ -0,0 +1,174 @@ +// Copyright 2025 - The Gemini CLI authors +// SPDX-License-Identifier: MPL-2.0 + +package googledrive + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/offen/docker-volume-backup/internal/errwrap" + "github.com/offen/docker-volume-backup/internal/storage" + "golang.org/x/oauth2/google" + "google.golang.org/api/drive/v3" + "google.golang.org/api/option" + "golang.org/x/oauth2" + "net/http" + "crypto/tls" +) + +type googleDriveStorage struct { + storage.StorageBackend + client *drive.Service +} + +// Config allows to configure a Google Drive storage backend. +type Config struct { + CredentialsJSON string + FolderID string + ImpersonateSubject string + Endpoint string + TokenURL string +} + +// NewStorageBackend creates and initializes a new Google Drive storage backend. +func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) { + ctx := context.Background() + + credentialsBytes := []byte(opts.CredentialsJSON) + + config, err := google.JWTConfigFromJSON(credentialsBytes, drive.DriveScope) + if err != nil { + return nil, errwrap.Wrap(err, "unable to parse credentials") + } + if opts.ImpersonateSubject != "" { + config.Subject = opts.ImpersonateSubject + } + if opts.TokenURL != "" { + config.TokenURL = opts.TokenURL + } + + var clientOptions []option.ClientOption + if opts.Endpoint != "" { + clientOptions = append(clientOptions, option.WithEndpoint(opts.Endpoint)) + // Insecure transport for http mock server + insecureTransport := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + insecureClient := &http.Client{Transport: insecureTransport} + ctx = context.WithValue(ctx, oauth2.HTTPClient, insecureClient) + } + clientOptions = append(clientOptions, option.WithTokenSource(config.TokenSource(ctx))) + + srv, err := drive.NewService(ctx, clientOptions...) + if err != nil { + return nil, errwrap.Wrap(err, "unable to create Drive client") + } + + return &googleDriveStorage{ + StorageBackend: storage.StorageBackend{ + DestinationPath: opts.FolderID, + Log: logFunc, + }, + client: srv, + }, nil +} + +// Name returns the name of the storage backend +func (b *googleDriveStorage) Name() string { + return "GoogleDrive" +} + +// Copy copies the given file to the Google Drive storage backend. +func (b *googleDriveStorage) Copy(file string) error { + _, name := filepath.Split(file) + b.Log(storage.LogLevelInfo, b.Name(), "Starting upload for backup '%s'.", name) + + f, err := os.Open(file) + if err != nil { + return errwrap.Wrap(err, fmt.Sprintf("failed to open file %s", file)) + } + defer f.Close() + + driveFile := &drive.File{Name: name} + if b.DestinationPath != "" { + driveFile.Parents = []string{b.DestinationPath} + } else { + driveFile.Parents = []string{"root"} + } + + createCall := b.client.Files.Create(driveFile).SupportsAllDrives(true).Fields("id") + created, err := createCall.Media(f).Do() + if err != nil { + return errwrap.Wrap(err, fmt.Sprintf("failed to upload %s", name)) + } + + b.Log(storage.LogLevelInfo, b.Name(), "Finished upload for %s. File ID: %s", name, created.Id) + return nil +} + +// Prune rotates away backups according to the configuration and provided deadline for the Google Drive storage backend. +func (b *googleDriveStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) { + parentID := b.DestinationPath + if parentID == "" { + parentID = "root" + } + + query := fmt.Sprintf("name contains '%s' and trashed = false", pruningPrefix) + if parentID != "root" { + query = fmt.Sprintf("'%s' in parents and (%s)", parentID, query) + } + + var allFiles []*drive.File + pageToken := "" + for { + req := b.client.Files.List().Q(query).SupportsAllDrives(true).Fields("files(id, name, createdTime, parents)").PageToken(pageToken) + res, err := req.Do() + if err != nil { + return nil, errwrap.Wrap(err, "listing files") + } + allFiles = append(allFiles, res.Files...) + pageToken = res.NextPageToken + if pageToken == "" { + break + } + } + + var matches []*drive.File + var lenCandidates int + for _, f := range allFiles { + if !strings.HasPrefix(f.Name, pruningPrefix) { + continue + } + lenCandidates++ + created, err := time.Parse(time.RFC3339, f.CreatedTime) + if err != nil { + b.Log(storage.LogLevelWarning, b.Name(), "Could not parse time for backup %s: %v", f.Name, err) + continue + } + if created.Before(deadline) { + matches = append(matches, f) + } + } + + stats := &storage.PruneStats{ + Total: uint(lenCandidates), + Pruned: uint(len(matches)), + } + + pruneErr := b.DoPrune(b.Name(), len(matches), lenCandidates, deadline, func() error { + for _, file := range matches { + b.Log(storage.LogLevelInfo, b.Name(), "Deleting old backup file: %s", file.Name) + if err := b.client.Files.Delete(file.Id).SupportsAllDrives(true).Do(); err != nil { + b.Log(storage.LogLevelWarning, b.Name(), "Error deleting %s: %v", file.Name, err) + } + } + return nil + }) + + return stats, pruneErr +} diff --git a/test/googledrive/credentials.json b/test/googledrive/credentials.json new file mode 100644 index 00000000..c6de7f7d --- /dev/null +++ b/test/googledrive/credentials.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "dummy-project", + "private_key_id": "dummykeyid", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCus0CDXvrHhl6a\nLBj7onfU3vRExQQAPstSovS4x3/3BLJNbdMUjrxWnmV5I+Y/U1iw18+8I87CMJDA\n+rIG37tSQ6WYhj2d9ym31O2EgVDQJMkVack/rdXCoWYWn6o7dZcv4K5MEtwW8uWQ\n5PEw0wbK7NIHSSotB9RajzHnLFkSu2XcEThlOp+wkfpTCYGg6+uCBJcMwUBR45eJ\nBLcvifBJVpWaAdj7DcYqWSxRQxensqB5wzCTatwwxDZo3KxnXsf2XRU+C3B71e5q\nb26XTkuIe9W04pj9Fp3fM7RgPSJpElMRFnPUliRhkyppspfYJBYQlpdzDdqKGkGK\nLMDu2c8DAgMBAAECggEAARG8QQ+HJqWNF4VSKCXPO0+C8RtD/IULCNX3NhJzTO4c\nI3ezrp9mlGsUWvPAPAarHmYbgBJtU2I+EZsmse4TaWhcIyVnMm+Dpy1ECucpZoeU\nqIgWe90iW9daBiC3NtRXIlSQNVGjM0mpX8olZM924am6o5/wNh2CP+hsRayBAkqf\nZojppQxYnI+WNNqOlke0T8FoWWm1ZX1gHAJQAeiLpDG675lckP5WxK0RmmKOW/UM\nFU/D4+csMG3eJPhT/Qm3LyAB+pNGpfzHuQXD5jubUhUq2uSsH4ko23wSl0nGHXRW\nX3YhlMDbK4bZtG7YNHQTmh05l6HvEQVbxgHTQLN9gQKBgQDTDDlBQEkLLCWyjmja\nTNt6308CZWZIrWMVtlrpY7S0a6NKm0YGhnXsDGRY4UCNqfMv7xmIw0efN4x90JoX\nglOVeODWgCJHqt6Zzsl8zbEOgbBEvcUO0dMa5PdpMzqd2Y2WghDH1PcrXueMVNXO\nUdf7Rs157LXx5+NouzfGZVmBwQKBgQDT6RwjWV04cxXsCg3QJ06q6YsVeoAawtQE\nWLQ13e0Soa2sBH5TbuOkEQIXVRAVeGSlPfL7N5FsSiZz+ozIhRdTTgNAHqF/TJCf\nEuLEb32Sfw/krLon0LoHBf6GgP+lWqvG4K2YCoAJwBlyHKoQuvbxGer7quuQ29V1\nDqmRL8g5wwKBgQDC0UjU/BOxVYpi/mS6BzKfhR35F0NJGY0a0N+xDBIWbjopN5Z3\nlY2rXXEQPraJTvWnLO8EOUeXKP7ucS6dPvgLRa8/Mr7yK0Aa+TEznOixfHQLsKYE\nXRqje/MLUHfumJHD+sKkxOl5Rr015GYNc62NTjmFMEZwTN+2oQQGhy4NwQKBgBrA\n6W6FD8Hatb/RHSFUdRga2BZkGtxGEKJj2IycchvSEa0P/CroaxEBnLP5Z0hupLY/\n9fdFcrSrP+OQlEmUk/dOeBaWR2lc7z1GEx8dvErMg+Mo82+naHUOiq3Mh3oG0n0P\nTJtPaA7TE+NWPxpRoG+cCBCx6X+mYXKf4USVNcAlAoGBAMH2a8qlnU/lrXSNGcrd\na2TNVi2qDfy0fU6IVFGEydmLMB3wuUUCUcBS6n1d62FqdJY9Rf1wKVIeZgtqJbCv\nOculz64WaXP8TSVrXnqfW8rUsYSTIdV+/P8gxJ9gYGS8E8KZSW5a8yRDc0jcKGI6\nzUJ8tz0Q5jEWC4MdDm7G1XrG\n-----END PRIVATE KEY-----\n", + "client_email": "dummy@dummy-project.iam.gserviceaccount.com", + "client_id": "dummyclientid", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/dummy%40dummy-project.iam.gserviceaccount.com" +} \ No newline at end of file diff --git a/test/googledrive/docker-compose.yml b/test/googledrive/docker-compose.yml new file mode 100644 index 00000000..41df5d02 --- /dev/null +++ b/test/googledrive/docker-compose.yml @@ -0,0 +1,52 @@ +services: + openapi_mock: + image: muonsoft/openapi-mock:0.3.9 + environment: + OPENAPI_MOCK_USE_EXAMPLES: if_present + OPENAPI_MOCK_SPECIFICATION_URL: '/etc/openapi/googledrive_v3.yaml' + ports: + - 8080:8080 + volumes: + - ${SPEC_FILE:-./googledrive_v3.yaml}:/etc/openapi/googledrive_v3.yaml + + oauth2_mock: + image: ghcr.io/navikt/mock-oauth2-server:1.0.0 + ports: + - 8090:8090 + environment: + PORT: 8090 + JSON_CONFIG_PATH: '/etc/oauth2/config.json' + volumes: + - ./oauth2_config.json:/etc/oauth2/config.json + + backup: + image: offen/docker-volume-backup:${TEST_VERSION:-canary} + hostname: hostnametoken + depends_on: + - openapi_mock + - oauth2_mock + restart: always + environment: + BACKUP_FILENAME_EXPAND: 'true' + BACKUP_FILENAME: test-$$HOSTNAME.tar.gz + BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? + BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} + BACKUP_PRUNING_LEEWAY: 5s + BACKUP_PRUNING_PREFIX: test + GOOGLE_DRIVE_ENDPOINT: http://openapi_mock:8080 + GOOGLE_DRIVE_TOKEN_URL: http://oauth2_mock:8090/issuer1/token + GOOGLE_DRIVE_CREDENTIALS_JSON_FILE: /etc/gdrive/credentials.json + GOOGLE_DRIVE_FOLDER_ID: "root" + volumes: + - app_data:/backup/app_data:ro + - ./credentials.json:/etc/gdrive/credentials.json + + offen: + image: offen/offen:latest + labels: + - docker-volume-backup.stop-during-backup=true + volumes: + - app_data:/var/opt/offen + +volumes: + app_data: diff --git a/test/googledrive/googledrive_v3.yaml b/test/googledrive/googledrive_v3.yaml new file mode 100644 index 00000000..2ce47fe6 --- /dev/null +++ b/test/googledrive/googledrive_v3.yaml @@ -0,0 +1,139 @@ +openapi: 3.0.1 +info: + title: Minimal Google Drive API Mock + version: 1.0.0 + description: Minimal mock implementation of Google Drive API v3 for testing +servers: + - url: / +paths: + /upload/drive/v3/files: + post: + summary: Upload file to Google Drive + parameters: + - name: uploadType + in: query + schema: + type: string + - name: fields + in: query + schema: + type: string + - name: supportsAllDrives + in: query + schema: + type: boolean + - name: alt + in: query + schema: + type: string + - name: prettyPrint + in: query + schema: + type: boolean + requestBody: + content: + multipart/related: + schema: + type: string + format: binary + responses: + '200': + description: File uploaded successfully + content: + application/json: + schema: + type: object + properties: + id: + type: string + description: "The ID of the file" + name: + type: string + description: "The name of the file (extracted from request.metadata.name)" + mimeType: + type: string + description: "The MIME type of the file" + size: + type: string + description: "The size of the file in bytes" + examples: + UploadSuccess: + summary: "Response when file is uploaded successfully" + description: "The response includes the filename from the request metadata" + value: + id: "1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms" + name: "test-backup.tar.gz" + mimeType: "application/gzip" + /files: + get: + summary: List files in Google Drive + parameters: + - name: q + in: query + schema: + type: string + description: "A query for filtering the file results" + - name: fields + in: query + schema: + type: string + - name: supportsAllDrives + in: query + schema: + type: boolean + - name: includeItemsFromAllDrives + in: query + schema: + type: boolean + responses: + '200': + description: Files listed successfully + content: + application/json: + schema: + type: object + properties: + files: + type: array + items: + type: object + properties: + id: + type: string + description: "The ID of the file" + name: + type: string + description: "The name of the file" + mimeType: + type: string + description: "The MIME type of the file" + createdTime: + type: string + description: "The time the file was created" + examples: + FilesList: + value: + files: + - id: "1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms" + name: "test-hostnametoken.tar.gz" + createdTime: "CREATED_TIME_1" + - id: "jgmUUqptlbs74OgvE2upms1BxiMVs0XRA5nFMdKvBdBZ" + name: "test-hostnametoken-old.tar.gz" + createdTime: "CREATED_TIME_2" + + /files/{fileId}: + delete: + summary: Delete a file from Google Drive + parameters: + - name: fileId + in: path + required: true + schema: + type: string + - name: supportsAllDrives + in: query + schema: + type: boolean + responses: + '204': + description: File deleted successfully diff --git a/test/googledrive/oauth2_config.json b/test/googledrive/oauth2_config.json new file mode 100644 index 00000000..c56d54ff --- /dev/null +++ b/test/googledrive/oauth2_config.json @@ -0,0 +1,37 @@ +{ + "interactiveLogin": true, + "httpServer": "NettyWrapper", + "tokenCallbacks": [ + { + "issuerId": "issuer1", + "tokenExpiry": 120, + "requestMappings": [ + { + "requestParam": "scope", + "match": "scope1", + "claims": { + "sub": "subByScope", + "aud": [ + "audByScope" + ] + } + } + ] + }, + { + "issuerId": "issuer2", + "requestMappings": [ + { + "requestParam": "someparam", + "match": "somevalue", + "claims": { + "sub": "subBySomeParam", + "aud": [ + "audBySomeParam" + ] + } + } + ] + } + ] +} diff --git a/test/googledrive/run.sh b/test/googledrive/run.sh new file mode 100755 index 00000000..c471d6bc --- /dev/null +++ b/test/googledrive/run.sh @@ -0,0 +1,59 @@ +#!/bin/sh + +set -e + +cd "$(dirname "$0")" +. ../util.sh +current_test=$(basename $(pwd)) + +export SPEC_FILE=$(mktemp -d)/googledrive_v3.yaml +cp googledrive_v3.yaml $SPEC_FILE +sed -i 's/CREATED_TIME_1/'"$(date "+%Y-%m-%dT%H:%M:%SZ")/g" $SPEC_FILE +sed -i 's/CREATED_TIME_2/'"$(date "+%Y-%m-%dT%H:%M:%SZ" -d "14 days ago")/g" $SPEC_FILE + +docker compose up -d --quiet-pull +sleep 5 + +logs=$(docker compose exec backup backup | tee /dev/stderr) + +sleep 5 + +expect_running_containers "4" + +if echo "$logs" | grep -q "ERROR"; then + fail "Backup failed, check logs for error" +else + pass "Backup succeeded, no errors reported." +fi + +# The second part of this test checks if backups get deleted when the retention +# is set to 0 days (which it should not as it would mean all backups get deleted) +BACKUP_RETENTION_DAYS="0" docker compose up -d +sleep 5 + +logs=$(docker compose exec -T backup backup | tee /dev/stderr) + +if echo "$logs" | grep -q "Refusing to do so, please check your configuration"; then + pass "Remote backups have not been deleted." +else + fail "Remote backups would have been deleted: $logs" +fi + +# The third part of this test checks if old backups get deleted when the retention +# is set to 7 days (which it should) +BACKUP_RETENTION_DAYS="7" docker compose up -d +sleep 5 + +info "Create second backup and prune" + +logs=$(docker compose exec -T backup backup | tee /dev/stderr) + +if echo "$logs" | grep -q "Pruned 1 out of 2 backups as they were older"; then + pass "Old remote backup has been pruned, new one is still present." +elif echo "$logs" | grep -q "ERROR"; then + fail "Pruning failed, errors reported: $logs" +elif echo "$logs" | grep -q "None of 1 existing backups were pruned"; then + fail "Pruning failed, old backup has not been pruned: $logs" +else + fail "Pruning failed, unknown result: $logs" +fi