diff --git a/README.md b/README.md index c422860..d2bac98 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ mysql-backup is a simple way to do MySQL database backups and restores, as well It has the following features: * dump and restore -* dump to local filesystem or to SMB server +* dump to supported targets * select database user and password * connect to any container running on the same system * select how often to run a dump @@ -96,7 +96,7 @@ __You should consider the [use of `--env-file=`](https://docs.docker.com/engine/ * `DB_PASS`: password for the database * `DB_DUMP_INCLUDE`: names of databases to restore separated by spaces. Required if `SINGLE_DATABASE=true`. * `SINGLE_DATABASE`: If is set to `true`, `DB_DUMP_INCLUDE` is required and must contain exactly one database name. Mysql command will then run with `--database=$DB_DUMP_INCLUDE` flag. This avoids the need of `USE ;` statement, which is useful when restoring from a file saved with `SINGLE_DATABASE` set to `true`. -* `DB_RESTORE_TARGET`: path to the actual restore file, which should be a compressed dump file. The target can be an absolute path, which should be volume mounted, an smb or S3 URL, similar to the target. +* `DB_RESTORE_TARGET`: path to the actual restore file, which should be a compressed dump file. The target can be any valid backup target. * `DB_DEBUG`: if `true`, dump copious outputs to the container logs while restoring. * To use the S3 driver `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_DEFAULT_REGION` will need to be defined. diff --git a/docs/backup.md b/docs/backup.md index 91070ec..dbc64b9 100644 --- a/docs/backup.md +++ b/docs/backup.md @@ -6,6 +6,7 @@ to a target. That target can be one of: * local file * SMB remote file * S3 bucket +* SCP target ## Instructions and Examples for Backup Configuration Options @@ -124,6 +125,7 @@ The value of the environment variable or CLI target can be one of three formats, * Local: If it starts with a `/` character or `file:///` url, it will dump to a local path. If in a container, you should have it volume-mounted. * SMB: If it is a URL of the format `smb://hostname/share/path/` then it will connect via SMB. * S3: If it is a URL of the format `s3://bucketname.fqdn.com/path` then it will connect via using the S3 protocol. +* SCP: If it is a URL of the format `scp://user@hostname:/path` then it will connect via SCP. In addition, you can send to multiple targets by separating them with a whitespace for the environment variable, or native multiple options for other configuration options. For example, to send to a local directory and an SMB share: @@ -190,7 +192,21 @@ Note that if you have multiple S3-compatible backup targets, each with its own s or endpoint, then you _must_ use the config file. There is no way to distinguish between multiple sets of credentials via the environment variables or CLI flags, while the config file provides credentials for each target. - + +##### SCP + +If it is a URL of the format `scp://user@hostname/path` then it will connect via SCP. If you leave off the `user` +i.e. `scp://hostname/path`, it will use the default ssh protocol for determining the user. +The default port is `22`; you can override it with `scp://hostname:port/path`. + +The `scp` implementation respects the following configuration: + +* user's ssh config file, by default `$HOME/.ssh/config` +* user's identity keys, by default `$HOME/.ssh/id_rsa`, `$HOME/.ssh/id_dsa`, etc. +* override the directory for finding `config` and identity key files via `SSH_HOME` + +As of this writing, the SCP implementation is basic and may not support all features of the SCP protocol. + #### Configuration File The configuration file is the most flexible way to configure the dump target. It allows you to specify diff --git a/go.mod b/go.mod index c8c431c..082e997 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/docker/go-connections v0.4.0 github.com/go-sql-driver/mysql v1.7.1 github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877 - github.com/moby/moby v28.3.3+incompatible github.com/robfig/cron/v3 v3.0.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 @@ -33,8 +32,12 @@ require ( require ( filippo.io/age v1.2.1 github.com/InfiniteLoopSpace/go_S-MIME v0.0.0-20181221134359-3f58f9a4b2b6 + github.com/bramvdbogaerde/go-scp v1.5.0 github.com/databacker/api/go/api v0.0.0-20250818102239-219c793f2151 github.com/google/go-cmp v0.7.0 + github.com/kevinburke/ssh_config v1.2.0 + github.com/moby/go-archive v0.1.0 + github.com/pkg/sftp v1.13.9 go.opentelemetry.io/otel v1.31.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 @@ -43,17 +46,19 @@ require ( ) require ( + github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/gliderlabs/ssh v0.3.8 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect + github.com/kr/fs v0.1.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/go-archive v0.1.0 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect diff --git a/go.sum b/go.sum index 053328f..6e80943 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,8 @@ github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4= github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= @@ -58,6 +60,8 @@ github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bramvdbogaerde/go-scp v1.5.0 h1:a9BinAjTfQh273eh7vd3qUgmBC+bx+3TRDtkZWmIpzM= +github.com/bramvdbogaerde/go-scp v1.5.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -76,8 +80,6 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/databacker/api/go/api v0.0.0-20250423183243-7775066c265e h1:5K7IbijS9p+dezx9m45CjFCR2Sf6BfT/tb540aEw66k= -github.com/databacker/api/go/api v0.0.0-20250423183243-7775066c265e/go.mod h1:bQhbl71Lk1ATni0H+u249hjoQ8ShAdVNcNjnw6z+SbE= github.com/databacker/api/go/api v0.0.0-20250818102239-219c793f2151 h1:WuQNmzJiLSR0d2IpeifwK0E6eOLZQDxzbuHWIEN2/9U= github.com/databacker/api/go/api v0.0.0-20250818102239-219c793f2151/go.mod h1:bQhbl71Lk1ATni0H+u249hjoQ8ShAdVNcNjnw6z+SbE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -104,6 +106,8 @@ github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbS github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -130,6 +134,7 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -157,6 +162,8 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -165,6 +172,8 @@ github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zt github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -182,8 +191,6 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= -github.com/moby/moby v28.3.3+incompatible h1:nzkZIIn9bQP9S553kNmJ+U8PBhdS2ciFWphV2vX/Zp4= -github.com/moby/moby v28.3.3+incompatible/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= @@ -213,6 +220,8 @@ github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.9 h1:4NGkvGudBL7GteO3m6qnaQ4pC0Kvf0onSVc9gR3EWBw= +github.com/pkg/sftp v1.13.9/go.mod h1:OBN7bVXdstkFFN/gdnHPUb5TE8eb8G1Rp9wCItqjkkA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -260,12 +269,15 @@ github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs= github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= @@ -305,6 +317,10 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -314,6 +330,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -328,6 +347,10 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -339,6 +362,10 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -356,19 +383,36 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= +golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -386,6 +430,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/storage/parse.go b/pkg/storage/parse.go index 97c55ef..6adf858 100644 --- a/pkg/storage/parse.go +++ b/pkg/storage/parse.go @@ -7,6 +7,7 @@ import ( "github.com/databacker/mysql-backup/pkg/storage/credentials" "github.com/databacker/mysql-backup/pkg/storage/file" "github.com/databacker/mysql-backup/pkg/storage/s3" + "github.com/databacker/mysql-backup/pkg/storage/scp" "github.com/databacker/mysql-backup/pkg/storage/smb" "github.com/databacker/mysql-backup/pkg/util" "gopkg.in/yaml.v3" @@ -54,6 +55,8 @@ func ParseURL(url string, creds credentials.Creds) (Storage, error) { opts = append(opts, s3.WithPathStyle()) } store = s3.New(*u, opts...) + case "scp": + store = scp.New(*u) default: return nil, fmt.Errorf("unknown url protocol: %s", u.Scheme) } diff --git a/pkg/storage/scp/scp.go b/pkg/storage/scp/scp.go new file mode 100644 index 0000000..73d41a8 --- /dev/null +++ b/pkg/storage/scp/scp.go @@ -0,0 +1,318 @@ +package scp + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/fs" + "net" + "net/url" + "os" + "path/filepath" + "time" + + scp "github.com/bramvdbogaerde/go-scp" + "github.com/kevinburke/ssh_config" + "github.com/pkg/sftp" + log "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/agent" +) + +var baseIdentityFileNames = []string{ + "id_ed25519", + "id_ecdsa", + "id_ecdsa_sk", // FIDO2 + "id_ed25519_sk", // FIDO2 + "id_rsa", // still common, though SHA-1 is discouraged +} + +func getIdentityFiles() []string { + idFileDir := os.Getenv("SSH_HOME") + if idFileDir == "" { + idFileDir = filepath.Join(os.Getenv("HOME"), ".ssh") + } + var files []string + for _, name := range baseIdentityFileNames { + filename := filepath.Join(idFileDir, name) + stat, err := os.Stat(filename) // check if file exists + if err == nil && !stat.IsDir() { + files = append(files, filename) + } + } + return files +} + +type SCP struct { + url url.URL +} + +func New(u url.URL) *SCP { + return &SCP{u} +} + +func (s *SCP) Pull(ctx context.Context, source, target string, logger *log.Entry) (int64, error) { + client, err := s.getSCPClient() + if err != nil { + return 0, fmt.Errorf("failed to create SCP client: %w", err) + } + f, err := os.OpenFile(target, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return 0, fmt.Errorf("failed to open target file %s: %w", target, err) + } + + defer func() { + // Close client connection after the file has been copied + client.Close() + // Close the file after it has been copied + _ = f.Close() + }() + + if err := client.CopyFromRemote(ctx, f, source); err != nil { + return 0, fmt.Errorf("failed to copy file from SCP server: %w", err) + } + stat, err := f.Stat() + if err != nil { + return 0, fmt.Errorf("failed to get file stat: %w", err) + } + return stat.Size(), nil +} + +func (s *SCP) Push(ctx context.Context, target, source string, logger *log.Entry) (int64, error) { + client, err := s.getSCPClient() + if err != nil { + return 0, fmt.Errorf("failed to create SCP client: %w", err) + } + + f, err := os.Open(source) + if err != nil { + return 0, fmt.Errorf("failed to open source file %s: %w", source, err) + } + + defer func() { + // Close client connection after the file has been copied + client.Close() + // Close the file after it has been copied + _ = f.Close() + }() + + if err := client.CopyFromFile(ctx, *f, target, "0644"); err != nil { + return 0, fmt.Errorf("failed to copy file to SCP server: %w", err) + } + stat, err := f.Stat() + if err != nil { + return 0, fmt.Errorf("failed to get file stat: %w", err) + } + return stat.Size(), nil +} + +func (s *SCP) Clean(filename string) string { + return filename +} + +func (s *SCP) Protocol() string { + return "scp" +} + +func (s *SCP) URL() string { + return s.url.String() +} + +func (s *SCP) ReadDir(ctx context.Context, dirname string, logger *log.Entry) ([]fs.FileInfo, error) { + client, err := s.getSSHClient() + if err != nil { + return nil, err + } + sftpClient, err := sftp.NewClient(client) + if err != nil { + return nil, err + } + defer func() { _ = sftpClient.Close() }() + infos, err := sftpClient.ReadDir(dirname) + if err != nil { + return nil, fmt.Errorf("failed to read remote directory %s over sftp: %w", dirname, err) + } + out := make([]fs.FileInfo, 0, len(infos)) + out = append(out, infos...) + return out, nil +} + +func (s *SCP) Remove(ctx context.Context, target string, logger *log.Entry) error { + client, err := s.getSSHClient() + if err != nil { + return err + } + sftpClient, err := sftp.NewClient(client) + if err != nil { + return fmt.Errorf("new sftp: %w", err) + } + defer func() { _ = sftpClient.Close() }() + + if err := sftpClient.Remove(target); err != nil { + return fmt.Errorf("remove %q: %w", target, err) + } + return nil +} + +// command run a command over ssh +// +//nolint:unparam,unused +func (s *SCP) command(ctx context.Context, cmd string) (stdout string, stderr string, err error) { + client, err := s.getSSHClient() + if err != nil { + return "", "", fmt.Errorf("failed to create SSH client: %w", err) + } + // Step 1: Create session + session, err := client.NewSession() + if err != nil { + return "", "", err + } + defer func() { _ = session.Close() }() + + // Step 2: Capture stdout and stderr + var stdoutBuf, stderrBuf bytes.Buffer + session.Stdout = &stdoutBuf + session.Stderr = &stderrBuf + + // Step 3: Run command + err = session.Run(cmd) // blocks until command finishes + + // Step 4: Return captured outputs + return stdoutBuf.String(), stderrBuf.String(), err + +} + +func (s *SCP) getSSHClient() (*ssh.Client, error) { + // read ssh config file + sshConfig, err := loadSSHConfig() + if err != nil { + return nil, fmt.Errorf("failed to load SSH config: %w", err) + } + // check items as provided against ssh config, URL override, defaults + hostname := s.url.Hostname() + port := s.url.Port() + username := s.url.User.Username() + var identityFiles []string + configPort, err := sshConfig.Get(hostname, "Port") + if err != nil { + return nil, fmt.Errorf("error getting hostname from SSH config: %w", err) + } + configHostname, err := sshConfig.Get(hostname, "HostName") + if err != nil { + return nil, fmt.Errorf("error getting port from SSH config: %w", err) + } + configIdentityFile, err := sshConfig.Get(hostname, "IdentityFile") + if err != nil { + return nil, fmt.Errorf("error getting identity file from SSH config: %w", err) + } + configUsername, err := sshConfig.Get(hostname, "User") + if err != nil { + return nil, fmt.Errorf("error getting username from SSH config: %w", err) + } + if configPort != "" { + port = configPort + } + if configHostname != "" { + hostname = configHostname + } + if configUsername != "" { + username = configUsername + } + if configIdentityFile != "" { + identityFiles = append(identityFiles, configIdentityFile) + } + // look for fixed identity files, if none explicitly specified + if len(identityFiles) == 0 { + identityFiles = getIdentityFiles() + } + authMethods, err := authMethodsFromAgentAndFiles(identityFiles) + if err != nil { + return nil, fmt.Errorf("failed to get SSH auth methods: %w", err) + } + clientConfig := &ssh.ClientConfig{ + User: username, + Auth: authMethods, + Timeout: 15 * time.Second, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), // TODO: use a proper host key callback in production + } + + client, err := ssh.Dial("tcp", net.JoinHostPort(hostname, port), clientConfig) + if err != nil { + return nil, fmt.Errorf("failed to connect to SSH server: %w", err) + } + return client, nil +} + +func (s *SCP) getSCPClient() (*scp.Client, error) { + sshClient, err := s.getSSHClient() + if err != nil { + return nil, fmt.Errorf("failed to create SSH client: %w", err) + } + // Create a new SCP client + client, err := scp.NewClientBySSH(sshClient) + if err != nil { + return nil, fmt.Errorf("failed to connect to SCP server: %w", err) + } + return &client, nil +} + +func loadSSHConfig() (*ssh_config.Config, error) { + idFileDir := os.Getenv("SSH_HOME") + if idFileDir == "" { + idFileDir = filepath.Join(os.Getenv("HOME"), ".ssh") + } + path := filepath.Join(idFileDir, "config") + f, err := os.Open(path) + if err != nil { + // No config is fine; act like empty config. + return &ssh_config.Config{}, nil + } + defer func() { _ = f.Close() }() + return ssh_config.Decode(f) +} + +func authMethodsFromAgentAndFiles(identityFiles []string) ([]ssh.AuthMethod, error) { + var ( + methods []ssh.AuthMethod + signers []ssh.Signer + ) + // ssh-agent + if sock := os.Getenv("SSH_AUTH_SOCK"); sock != "" { + if conn, err := net.Dial("unix", sock); err == nil { + agentSigners, err := agent.NewClient(conn).Signers() + if err != nil { + return nil, fmt.Errorf("failed to get signers from SSH agent: %v", err) + } + signers = append(signers, agentSigners...) + } + } + + // Identity files + for _, p := range identityFiles { + key, err := os.ReadFile(p) + // skip missing files gracefully + if err != nil { + continue + } + raw, err := ssh.ParseRawPrivateKey(key) + if err != nil && errors.Is(err, &ssh.PassphraseMissingError{}) { + // ignore encrypted keys + continue + } + if err != nil { + return nil, fmt.Errorf("failed to parse private key %s: %w", p, err) + } + signer, err := ssh.NewSignerFromKey(raw) + if err != nil { + return nil, fmt.Errorf("failed to create signer from key %s: %w", p, err) + } + signers = append(signers, signer) + } + methods = append(methods, ssh.PublicKeysCallback(func() ([]ssh.Signer, error) { + // assemble signers here (file first, then agent) + return signers, nil + })) + + return methods, nil +} diff --git a/pkg/storage/scp/scp_test.go b/pkg/storage/scp/scp_test.go new file mode 100644 index 0000000..7d5eedc --- /dev/null +++ b/pkg/storage/scp/scp_test.go @@ -0,0 +1,681 @@ +package scp + +import ( + "bufio" + "context" + "crypto" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "io" + "net" + "net/url" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "testing" + "time" + + gliderssh "github.com/gliderlabs/ssh" + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" +) + +const ( + testUser = "alice" + testPass = "secret" +) + +func TestPull(t *testing.T) { + const ( + aFile = "a.txt" + bFile = "b.txt" + ) + files := map[string][]byte{ + aFile: []byte("hello"), + bFile: []byte("world"), + } + // start the server + server := testStartServerWithKeys(t) + + // Prepare some files in the server's root dir + for f, content := range files { + if err := os.WriteFile(filepath.Join(server.RootDir, f), content, 0600); err != nil { + t.Fatalf("failed to write file %s: %v", f, err) + } + } + + // check the files + handler := New(url.URL{Scheme: "scp", Host: server.Addr}) + tmpDir := t.TempDir() + + for f, content := range files { + n, err := handler.Pull(context.Background(), f, filepath.Join(tmpDir, f), nil) + if err != nil { + t.Fatalf("failed to pull file %s: %v", f, err) + } + if n != int64(len(content)) { + t.Errorf("pulled %d bytes instead of expected %d", n, len(content)) + } + localContent, err := os.ReadFile(filepath.Join(tmpDir, f)) + if err != nil { + t.Fatalf("failed to read local file %s: %v", f, err) + } + if string(localContent) != string(content) { + t.Errorf("local file %s content mismatch: got %q, want %q", f, localContent, content) + } + } +} + +func TestPush(t *testing.T) { + const ( + aFile = "a.txt" + bFile = "b.txt" + ) + files := map[string][]byte{ + aFile: []byte("hello"), + bFile: []byte("world"), + } + // start the server + server := testStartServerWithKeys(t) + + // Prepare some files in the sethe localrver's root dir + tmpDir := t.TempDir() + for f, content := range files { + _ = os.WriteFile(filepath.Join(tmpDir, f), content, 0600) + } + + // check the files + handler := New(url.URL{Scheme: "scp", Host: server.Addr}) + + for f, content := range files { + localFile := filepath.Join(tmpDir, f) + n, err := handler.Push(context.Background(), f, localFile, nil) + if err != nil { + t.Fatalf("failed to push file %s: %v", localFile, err) + } + if n != int64(len(content)) { + t.Errorf("pushed %d bytes instead of expected %d", n, len(content)) + } + serverFile := filepath.Join(server.RootDir, f) + foundContent, err := os.ReadFile(serverFile) + if err != nil { + t.Fatalf("failed to read server file %s: %v", serverFile, err) + } + if string(foundContent) != string(content) { + t.Errorf("server file %s content mismatch: got %q, want %q", f, foundContent, content) + } + } +} + +func TestReadDir(t *testing.T) { + const ( + aFile = "a.txt" + bFile = "b.txt" + ) + files := map[string][]byte{ + aFile: []byte("hello"), + bFile: []byte("world"), + } + // start the server + server := testStartServerWithKeys(t) + + // Prepare some files in the server's root dir + for f, content := range files { + _ = os.WriteFile(filepath.Join(server.RootDir, f), content, 0600) + } + + // check the files + handler := New(url.URL{Scheme: "scp", Host: server.Addr}) + // sftp would require a lot of work on our end to make it chrooted, so we are not bothering + fileInfo, err := handler.ReadDir(context.Background(), server.RootDir, nil) + if err != nil { + t.Fatalf("failed to read remote directory: %v", err) + } + if len(fileInfo) != len(files) { + t.Errorf("unexpected number of files: got %d, want %d", len(fileInfo), len(files)) + } + // sort and compare + var filenames []string + for f := range files { + filenames = append(filenames, f) + } + sort.Strings(filenames) + sort.Slice(fileInfo, func(i, j int) bool { + return fileInfo[i].Name() < fileInfo[j].Name() + }) + for i, fi := range fileInfo { + if fi.Name() != filenames[i] { + t.Errorf("file %d: got %s, want %s", i, fi.Name(), filenames[i]) + } + } +} + +func TestRemove(t *testing.T) { + const ( + aFile = "a.txt" + bFile = "b.txt" + ) + files := map[string][]byte{ + aFile: []byte("hello"), + bFile: []byte("world"), + } + // start the server + server := testStartServerWithKeys(t) + + // Prepare some files in the server's root dir + for f, content := range files { + _ = os.WriteFile(filepath.Join(server.RootDir, f), content, 0600) + } + + // check the files + handler := New(url.URL{Scheme: "scp", Host: server.Addr}) + err := handler.Remove(context.Background(), aFile, nil) + if err != nil { + t.Fatalf("failed to remove file %s: %v", aFile, err) + } + // see that it no longer exists + _, err = os.Stat(filepath.Join(server.RootDir, aFile)) + if err == nil { + t.Fatalf("file %s still exists after removal", aFile) + } +} + +func TestConnection(t *testing.T) { + t.Run("no keyfile found", func(t *testing.T) { + server := testStartServer(t) + tmpDir := t.TempDir() + if err := os.Setenv("SSH_HOME", tmpDir); err != nil { + t.Fatalf("failed to set SSH_HOME: %v", err) + } + t.Cleanup(func() { _ = os.Unsetenv("SSH_HOME") }) + + handler := New(url.URL{Scheme: "scp", Host: server.Addr}) + _, err := handler.getSSHClient() + if err == nil { + t.Fatal("expected error getting SSH client, got none") + } + t.Logf("%+v", err) + }) + t.Run("invalid keyfile found", func(t *testing.T) { + server := testStartServer(t) + tmpDir := t.TempDir() + if err := os.Setenv("SSH_HOME", tmpDir); err != nil { + t.Fatalf("failed to set SSH_HOME: %v", err) + } + t.Cleanup(func() { _ = os.Unsetenv("SSH_HOME") }) + + handler := New(url.URL{Scheme: "scp", Host: server.Addr}) + _, err := handler.getSSHClient() + if err == nil { + t.Fatal("expected error getting SSH client, got none") + } + t.Logf("%+v", err) + }) + t.Run("valid keyfile found in directory", func(t *testing.T) { + server := testStartServer(t) + tmpDir := t.TempDir() + if err := os.Setenv("SSH_HOME", tmpDir); err != nil { + t.Fatalf("failed to set SSH_HOME: %v", err) + } + t.Cleanup(func() { _ = os.Unsetenv("SSH_HOME") }) + + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate keypair: %v", err) + } + if err := testWriteKeypairToDir(pub, priv, tmpDir); err != nil { + t.Fatalf("write keypair: %v", err) + } + sshPub, err := ssh.NewPublicKey(pub) + if err != nil { + t.Fatalf("failed to create SSH public key: %v", err) + } + server.UserKeys = append(server.UserKeys, sshPub) + + handler := New(url.URL{Scheme: "scp", Host: server.Addr}) + _, err = handler.getSSHClient() + if err != nil { + t.Fatalf("unexpected error getting SSH client: %v", err) + } + }) + t.Run("explicitly specified in ssh_config", func(t *testing.T) { + server := testStartServer(t) + tmpDir := t.TempDir() + if err := os.Setenv("SSH_HOME", tmpDir); err != nil { + t.Fatalf("failed to set SSH_HOME: %v", err) + } + t.Cleanup(func() { _ = os.Unsetenv("SSH_HOME") }) + + // create ssh config + configFilename := filepath.Join(tmpDir, "config") + keyFilename := filepath.Join(tmpDir, "unique") + config := fmt.Sprintf(` +Host testhost + HostName %s + Port %d + IdentityFile %s +`, server.Hostname(), server.Port(), keyFilename) + _ = os.WriteFile(configFilename, []byte(config), 0600) + + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate keypair: %v", err) + } + if err := testWriteKeypairToDir(pub, priv, tmpDir); err != nil { + t.Fatalf("write keypair: %v", err) + } + sshPub, err := ssh.NewPublicKey(pub) + if err != nil { + t.Fatalf("failed to create SSH public key: %v", err) + } + server.UserKeys = append(server.UserKeys, sshPub) + + handler := New(url.URL{Scheme: "scp", Host: server.Addr}) + _, err = handler.getSSHClient() + if err != nil { + t.Fatalf("unexpected error getting SSH client: %v", err) + } + }) +} + +func testWriteKeypairToDir(pubkey crypto.PublicKey, privkey crypto.PrivateKey, dir string) error { + privBytes, err := x509.MarshalPKCS8PrivateKey(privkey) + if err != nil { + return fmt.Errorf("marshal priv: %w", err) + } + privPem := pem.EncodeToMemory(&pem.Block{ + Type: "PRIVATE KEY", + Bytes: privBytes, + }) + + pubKey, err := ssh.NewPublicKey(pubkey) + if err != nil { + return fmt.Errorf("marshal pub: %w", err) + } + pubBytes := ssh.MarshalAuthorizedKey(pubKey) + + // 4. Write files + privPath := filepath.Join(dir, "id_ed25519") + pubPath := filepath.Join(dir, "id_ed25519.pub") + + if err := os.WriteFile(privPath, privPem, 0600); err != nil { + return fmt.Errorf("write priv: %w", err) + } + if err := os.WriteFile(pubPath, pubBytes, 0644); err != nil { + return fmt.Errorf("write pub: %w", err) + } + + return nil +} + +// testStartServer starts a server +func testStartServer(t *testing.T) *Server { + t.Helper() + root := t.TempDir() + // Generate an ECDSA host key (ephemeral, per test) + priv, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + signer, err := ssh.NewSignerFromKey(priv) + if err != nil { + t.Fatalf("host key: %v", err) + } + + s := &Server{ + User: testUser, + Password: testPass, + Key: signer, + RootDir: root, + } + pwAuth := func(ctx gliderssh.Context, password string) bool { + return ctx.User() == testUser && password == testPass + } + pubAuth := func(ctx gliderssh.Context, key gliderssh.PublicKey) bool { + for _, ak := range s.UserKeys { + if ssh.FingerprintSHA256(ak) == ssh.FingerprintSHA256(key) { + return true + } + } + return false + } + + // SFTP subsystem handler + subsystemHandlers := map[string]gliderssh.SubsystemHandler{ + "sftp": func(sess gliderssh.Session) { + // Start an sftp server bound to the session + s, err := sftp.NewServer(sess, sftp.WithDebug(nil), sftp.WithServerWorkingDirectory(root)) + if err != nil { + _, _ = io.WriteString(sess, "sftp start error: "+err.Error()) + return + } + + defer func() { _ = s.Close() }() + _ = s.Serve() // blocks until client closes + }, + } + + handler := scpExecHandler(s.RootDir) + server := &gliderssh.Server{ + HostSigners: []gliderssh.Signer{s.Key}, + Addr: "127.0.0.1:0", + PasswordHandler: pwAuth, + PublicKeyHandler: pubAuth, + SubsystemHandlers: subsystemHandlers, + Handler: handler, + } + // Bind + lis, err := net.Listen("tcp", server.Addr) + if err != nil { + t.Fatalf("listen: %v", err) + } + s.lis = lis + s.Addr = lis.Addr().String() + s.srv = server + + go func() { + _ = server.Serve(lis) // stops when Close() is called + }() + // Wait for it to be ready + deadline := time.Now().Add(1 * time.Second) + for time.Now().Before(deadline) { + conn, err := net.Dial("tcp", s.Addr) + if err == nil { + _ = conn.Close() + break + } + time.Sleep(20 * time.Millisecond) + } + return s +} + +// testStartServerWithKeys starts a server with user keys +func testStartServerWithKeys(t *testing.T) *Server { + t.Helper() + server := testStartServer(t) + tmpDir := t.TempDir() + if err := os.Setenv("SSH_HOME", tmpDir); err != nil { + t.Fatalf("failed to set SSH_HOME: %v", err) + } + t.Cleanup(func() { + _ = os.Unsetenv("SSH_HOME") + }) + + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate keypair: %v", err) + } + if err := testWriteKeypairToDir(pub, priv, tmpDir); err != nil { + t.Fatalf("write keypair: %v", err) + } + sshPub, err := ssh.NewPublicKey(pub) + if err != nil { + t.Fatalf("failed to create SSH public key: %v", err) + } + server.UserKeys = append(server.UserKeys, sshPub) + return server +} + +type Server struct { + Addr string // host:port to connect to + User string // default "alice" + Password string // default "secret" + Key ssh.Signer // server host key + UserKeys []ssh.PublicKey // authorized user keys, can be added later as well + RootDir string // temp dir used as chroot-like base for SFTP & exec + srv *gliderssh.Server + lis net.Listener +} + +func (s *Server) Hostname() string { + host, _, _ := net.SplitHostPort(s.Addr) + return host +} + +func (s *Server) Port() int { + _, port, _ := net.SplitHostPort(s.Addr) + p, _ := strconv.Atoi(port) + return p +} + +func (s *Server) Close() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = s.srv.Shutdown(ctx) + _ = s.lis.Close() +} + +// ClientConfig returns a client config matching the test server creds. +func (s *Server) ClientConfig() *ssh.ClientConfig { + return &ssh.ClientConfig{ + User: s.User, + Auth: []ssh.AuthMethod{ssh.Password(s.Password)}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), // for tests + Timeout: 3 * time.Second, + } +} + +// minimal flag parsing + both directions (-f and -t) +type scpOpts struct { + from bool // -f (client pulls from server) + to bool // -t (client pushes to server) + recursive bool // -r (ignored in this minimal impl) + preserve bool // -p (ignored here; timestamps line `T` is parsed/ignored) + target string +} + +// parse "scp [flags] " +func parseScpArgs(args []string) (scpOpts, error) { + var o scpOpts + // skip "scp" + for i := 1; i < len(args); i++ { + a := args[i] + if a == "--" { + if i+1 < len(args) { + o.target = args[i+1] + } + break + } + if strings.HasPrefix(a, "-") { + if strings.Contains(a, "f") { + o.from = true + } + if strings.Contains(a, "t") { + o.to = true + } + if strings.Contains(a, "r") { + o.recursive = true + } + if strings.Contains(a, "p") { + o.preserve = true + } + continue + } + o.target = a + } + if !o.from && !o.to { + return o, fmt.Errorf("unsupported scp mode: need -f or -t") + } + if o.target == "" { + return o, fmt.Errorf("missing target path") + } + return o, nil +} + +// ---------- receive path: implement `scp -t ` (upload) ---------- + +func scpReceive(sess gliderssh.Session, root, target string) error { + // RFC-ish handshake helpers + readByte := func() (byte, error) { + var b [1]byte + _, err := io.ReadFull(sess, b[:]) + return b[0], err + } + sendAck := func() error { _, err := sess.Write([]byte{0}); return err } + sendErr := func(msg string) error { + _, _ = fmt.Fprintf(sess.Stderr(), "scp: %s\n", msg) + _, err := sess.Write([]byte{2}) // fatal + return err + } + + // Tell the client we're ready + if err := sendAck(); err != nil { + return err + } + + // We implement a tiny subset: + // optional: T 0 0\n (ignore but ack) + // required: C \n + // then bytes of data + // then a single 0 byte from client (end-of-file) + // we ack (0) + // end (client closes or sends next file; we handle one file) + r := bufio.NewReader(sess) + + // If client sends T line, consume & ack (ignored) + // Peek first byte; if 'T' consume the line, ack, continue + if b, _ := r.Peek(1); len(b) == 1 && b[0] == 'T' { + if _, err := r.ReadString('\n'); err != nil { + return err + } + if err := sendAck(); err != nil { + return err + } + } + + // Expect C line + hdr, err := r.ReadString('\n') + if err != nil { + return err + } + if !strings.HasPrefix(hdr, "C") { + _ = sendErr("only single-file uploads (C...) supported") + return fmt.Errorf("unexpected header: %q", hdr) + } + + // Parse: C%04o \n + var modeStr string + var size int64 + var name string + _, err = fmt.Sscanf(hdr, "C%s %d %s\n", &modeStr, &size, &name) + if err != nil { + _ = sendErr("bad C header") + return fmt.Errorf("parse header: %w", err) + } + // Ack header + if err := sendAck(); err != nil { + return err + } + + // Open destination file under root/target + dstPath := filepath.Clean(filepath.Join(root, target)) + // If target is a directory, place into it using the sent name + if fi, statErr := os.Stat(dstPath); statErr == nil && fi.IsDir() { + dstPath = filepath.Join(dstPath, name) + } + // Parse mode + m, _ := strconv.ParseUint(modeStr, 8, 32) + dst, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(m)&0777) + if err != nil { + _ = sendErr("cannot open dest") + return err + } + defer func() { _ = dst.Close() }() + + // Copy exactly bytes + if _, err := io.CopyN(dst, r, size); err != nil { + _ = sendErr("short write") + return err + } + + // Consume file-end byte from client + b, err := readByte() + if err != nil && !errors.Is(err, io.EOF) { + _ = sendErr("missing file-end ack") + return fmt.Errorf("expected end-of-file 0, got %v err=%v", b, err) + } + // Final ack + if err := sendAck(); err != nil { + return err + } + + return nil +} + +// ---------- your exec handler using the parser + both paths ---------- + +func scpExecHandler(root string) gliderssh.Handler { + return func(sess gliderssh.Session) { + args := sess.Command() + if len(args) == 0 || args[0] != "scp" { + _, _ = io.WriteString(sess.Stderr(), "unsupported command\n") + _ = sess.Exit(127) + return + } + + opts, err := parseScpArgs(args) + if err != nil { + _, _ = io.WriteString(sess.Stderr(), "scp: "+err.Error()+"\n") + _ = sess.Exit(2) + return + } + + if opts.from { + // existing send code (download): scp -f + fp := filepath.Clean(filepath.Join(root, opts.target)) + f, err := os.Open(fp) + if err != nil { + _ = sess.Exit(1) + return + } + defer func() { _ = f.Close() }() + fi, _ := f.Stat() + // 1) wait for initial OK + var b [1]byte + if _, err := io.ReadFull(sess, b[:]); err != nil || b[0] != 0 { + _ = sess.Exit(1) + return + } + // 2) header + _, _ = fmt.Fprintf(sess, "C%04o %d %s\n", fi.Mode()&0777, fi.Size(), filepath.Base(fp)) + // 3) wait OK + if _, err := io.ReadFull(sess, b[:]); err != nil || b[0] != 0 { + _ = sess.Exit(1) + return + } + // 4) data + if _, err := io.Copy(sess, f); err != nil { + _ = sess.Exit(1) + return + } + // 5) end + wait final OK + _, _ = sess.Write([]byte{0}) + _, _ = io.ReadFull(sess, b[:]) + _ = sess.Exit(0) + return + } + + if opts.to { + // NEW: receive upload: scp -t + if opts.recursive { + _, _ = io.WriteString(sess.Stderr(), "scp: -r not supported\n") + _ = sess.Exit(2) + return + } + if err := scpReceive(sess, root, opts.target); err != nil { + _ = sess.Exit(1) + return + } + _ = sess.Exit(0) + return + } + + _, _ = io.WriteString(sess.Stderr(), "unsupported scp mode\n") + _ = sess.Exit(2) + } +}