From 332b539d26a5c086e434122868422f68a956c03e Mon Sep 17 00:00:00 2001 From: FH3095 Date: Tue, 15 Apr 2025 20:51:11 +0200 Subject: [PATCH 1/7] Add function to upload log-files to a remote server --- .vscode/launch.json | 8 +++++++ config/global.go | 1 + logger.go | 53 ++++++++++++++++++++++++++++++++++++++++----- main.go | 10 +++++++-- 4 files changed, 64 insertions(+), 8 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index f1eeafa01..aac3a5507 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,6 +11,14 @@ "mode": "auto", "program": "${fileDirname}", "args": ["-c", "examples/include.conf"] + }, + { + "name": "Private profiles.toml", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${fileDirname}", + "args": ["-c", "examples/private/profiles.toml"] } ] } \ No newline at end of file diff --git a/config/global.go b/config/global.go index cd19e7ca6..be26134cf 100644 --- a/config/global.go +++ b/config/global.go @@ -27,6 +27,7 @@ type Global struct { Scheduler string `mapstructure:"scheduler" default:"auto" examples:"auto;launchd;systemd;taskscheduler;crond;crond:/usr/bin/crontab;crontab:*:/etc/cron.d/resticprofile" description:"Selects the scheduler. Blank or \"auto\" uses the default scheduler of your operating system: \"launchd\", \"systemd\", \"taskscheduler\" or \"crond\" (as fallback). Alternatively you can set \"crond\" for cron compatible schedulers supporting the crontab executable API or \"crontab:[user:]file\" to write into a crontab file directly. The need for a user is detected if missing and can be set to a name, \"-\" (no user) or \"*\" (current user)."` ScheduleDefaults *ScheduleBaseConfig `mapstructure:"schedule-defaults" default:"" description:"Sets defaults for all schedules"` Log string `mapstructure:"log" default:"" examples:"/resticprofile.log;syslog-tcp://syslog-server:514;syslog:server;syslog:" description:"Sets the default log destination to be used if not specified in \"--log\" or \"schedule-log\" - see https://creativeprojects.github.io/resticprofile/configuration/logs/"` + LogUploadUrl string `mapstructure:"log-upload-url" description:"A URL to where to log-file will be uploaded after the job ran"` CommandOutput string `mapstructure:"command-output" default:"auto" enum:"auto;log;console;all" description:"Sets the destination for command output (stderr/stdout). \"log\" sends output to the log file (if specified), \"console\" sends it to the console instead. \"auto\" sends it to \"both\" if console is a terminal otherwise to \"log\" only - see https://creativeprojects.github.io/resticprofile/configuration/logs/"` LegacyArguments bool `mapstructure:"legacy-arguments" default:"false" deprecated:"0.20.0" description:"Legacy, broken arguments mode of resticprofile before version 0.15"` SystemdUnitTemplate string `mapstructure:"systemd-unit-template" default:"" description:"File containing the go template to generate a systemd unit - see https://creativeprojects.github.io/resticprofile/schedules/systemd/"` diff --git a/logger.go b/logger.go index 9bb26c9ab..9996e13a9 100644 --- a/logger.go +++ b/logger.go @@ -1,9 +1,11 @@ package main import ( + "bytes" "fmt" "io" "log" + "net/http" "os" "path/filepath" "slices" @@ -50,18 +52,20 @@ func setupRemoteLogger(flags commandLineFlags, client *remote.Client) { clog.SetDefaultLogger(logger) } -func setupTargetLogger(flags commandLineFlags, logTarget, commandOutput string) (io.Closer, error) { +func setupTargetLogger(flags commandLineFlags, logTarget, logUploadTarget, commandOutput string) (io.Closer, error) { var ( - handler LogCloser - file io.Writer - err error + handler LogCloser + file io.Writer + filepath string + err error ) if scheme, hostPort, isURL := dial.GetAddr(logTarget); isURL { handler, file, err = getSyslogHandler(scheme, hostPort) } else if dial.IsURL(logTarget) { err = fmt.Errorf("unsupported URL: %s", logTarget) } else { - handler, file, err = getFileHandler(logTarget) + filepath = getLogfilePath(logTarget) + handler, file, err = getFileHandler(filepath) } if err != nil { return nil, err @@ -79,11 +83,45 @@ func setupTargetLogger(flags commandLineFlags, logTarget, commandOutput string) } else if toLog { term.SetAllOutput(file) } + if logUploadTarget != "" && filepath != "" { + handler = createSendLogWrappedLogHandler(handler, filepath, logUploadTarget) + } } // and return the handler (so we can close it at the end) return handler, nil } +type wrappedLogCloser struct { + LogCloser + logfilePath string + logUploadTarget string +} + +func (w wrappedLogCloser) Close() error { + err := w.LogCloser.Close() + if err != nil { + return err + } + logData, err := os.ReadFile(w.logfilePath) + if err != nil { + return err + } + resp, err := http.Post(w.logUploadTarget, "application/octet-stream", bytes.NewReader(logData)) + defer resp.Body.Close() + if err != nil { + return err + } + if resp.StatusCode < 200 || resp.StatusCode > 299 { + respBody, _ := io.ReadAll(resp.Body) + return fmt.Errorf("log-upload: Got invalid http status %v: %v", resp.StatusCode, string(respBody)) + } + return nil +} + +func createSendLogWrappedLogHandler(handler LogCloser, logfilePath string, logUploadTarget string) LogCloser { + return wrappedLogCloser{LogCloser: handler, logfilePath: logfilePath, logUploadTarget: logUploadTarget} +} + func parseCommandOutput(commandOutput string) (all, log bool) { if strings.TrimSpace(commandOutput) == "auto" { if term.OsStdoutIsTerminal() { @@ -98,7 +136,7 @@ func parseCommandOutput(commandOutput string) (all, log bool) { return } -func getFileHandler(logfile string) (*clog.StandardLogHandler, io.Writer, error) { +func getLogfilePath(logfile string) string { if strings.HasPrefix(logfile, constants.TemporaryDirMarker) { if tempDir, err := util.TempDir(); err == nil { logfile = logfile[len(constants.TemporaryDirMarker):] @@ -109,7 +147,10 @@ func getFileHandler(logfile string) (*clog.StandardLogHandler, io.Writer, error) _ = os.MkdirAll(filepath.Dir(logfile), 0755) } } + return logfile +} +func getFileHandler(logfile string) (*clog.StandardLogHandler, io.Writer, error) { // create a platform aware log file appender keepOpen, appender := true, appendFunc(nil) if platform.IsWindows() { diff --git a/main.go b/main.go index 6a3d02c38..dcee596d5 100644 --- a/main.go +++ b/main.go @@ -118,8 +118,14 @@ func main() { term.PrintToError = flags.stderr } if logTarget != "" && logTarget != "-" { - if closer, err := setupTargetLogger(flags, logTarget, commandOutput); err == nil { - logCloser = func() { _ = closer.Close() } + if closer, err := setupTargetLogger(flags, logTarget, ctx.global.LogUploadUrl, commandOutput); err == nil { + logCloser = func() { + err := closer.Close() + if err != nil { + // Log is already closed. Write to stderr as last resort + fmt.Fprintf(os.Stderr, "Error closing logfile: %v", err) + } + } } else { // fallback to a console logger setupConsoleLogger(flags) From ee68d232bd2bc27623b1e0ed0026dd5dcca0edcd Mon Sep 17 00:00:00 2001 From: FH3095 Date: Tue, 15 Apr 2025 21:25:29 +0200 Subject: [PATCH 2/7] Add Log-Upload tests --- logger_test.go | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/logger_test.go b/logger_test.go index 32fb6fe8b..0ba2c072d 100644 --- a/logger_test.go +++ b/logger_test.go @@ -2,9 +2,13 @@ package main import ( "bufio" + "bytes" "fmt" + "net/http" + "net/http/httptest" "os" "path/filepath" + "strings" "testing" "time" @@ -38,7 +42,8 @@ func TestFileHandlerWithTemporaryDirMarker(t *testing.T) { logFile := filepath.Join(util.MustGetTempDir(), "sub", "file.log") assert.NoFileExists(t, logFile) - handler, _, err := getFileHandler(filepath.Join(constants.TemporaryDirMarker, "sub", "file.log")) + filepath := getLogfilePath(filepath.Join(constants.TemporaryDirMarker, "sub", "file.log")) + handler, _, err := getFileHandler(filepath) require.NoError(t, err) assert.FileExists(t, logFile) @@ -142,3 +147,33 @@ func TestCloseFileHandler(t *testing.T) { handler.Close() assert.Error(t, handler.LogEntry(clog.LogEntry{Level: clog.LevelInfo, Format: "log-line-2"})) } + +func TestLogUploadFailed(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.Body.Close() + w.WriteHeader(500) + }) + server := httptest.NewServer(handler) + defer server.Close() + closer, err := setupTargetLogger(commandLineFlags{}, filepath.Join(constants.TemporaryDirMarker, "file.log"), server.URL, "log") + assert.NoError(t, err) + assert.ErrorContains(t, closer.Close(), "log-upload: Got invalid http status 500") +} + +func TestLogUpload(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + buffer := bytes.Buffer{} + _, err := buffer.ReadFrom(r.Body) + assert.NoError(t, err) + r.Body.Close() + + w.WriteHeader(200) + assert.Equal(t, strings.Trim(buffer.String(), "\r\n"), "TestLogLine") + }) + server := httptest.NewServer(handler) + defer server.Close() + closer, err := setupTargetLogger(commandLineFlags{}, filepath.Join(constants.TemporaryDirMarker, "file.log"), server.URL, "log") + _, err = term.Println("TestLogLine") + assert.NoError(t, err) + assert.NoError(t, closer.Close()) +} From 91b4f5389e7e63c72804e8025b937879f4c9a730 Mon Sep 17 00:00:00 2001 From: FH3095 Date: Tue, 15 Apr 2025 21:34:21 +0200 Subject: [PATCH 3/7] Correct launch.json --- .vscode/launch.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index aac3a5507..2d3fcc50c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -16,8 +16,8 @@ "name": "Private profiles.toml", "type": "go", "request": "launch", - "mode": "auto", - "program": "${fileDirname}", + "mode": "debug", + "program": "${workspaceFolder}", "args": ["-c", "examples/private/profiles.toml"] } ] From 9239d14532d9d2258ac0dc414e549b59f99f605a Mon Sep 17 00:00:00 2001 From: FH3095 Date: Tue, 15 Apr 2025 21:44:44 +0200 Subject: [PATCH 4/7] Comments, also use Reader directly to send logfile --- logger.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/logger.go b/logger.go index 9996e13a9..6479e945b 100644 --- a/logger.go +++ b/logger.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "fmt" "io" "log" @@ -84,33 +83,38 @@ func setupTargetLogger(flags commandLineFlags, logTarget, logUploadTarget, comma term.SetAllOutput(file) } if logUploadTarget != "" && filepath != "" { - handler = createSendLogWrappedLogHandler(handler, filepath, logUploadTarget) + handler = createLogUploadingLogHandler(handler, filepath, logUploadTarget) } } // and return the handler (so we can close it at the end) return handler, nil } -type wrappedLogCloser struct { +type logUploadingLogCloser struct { LogCloser logfilePath string logUploadTarget string } -func (w wrappedLogCloser) Close() error { +// Try to close the original handler +// Also upload the log to the configured log-upload-url +func (w logUploadingLogCloser) Close() error { err := w.LogCloser.Close() if err != nil { return err } - logData, err := os.ReadFile(w.logfilePath) + // Open logfile for reading + logData, err := os.Open(w.logfilePath) if err != nil { return err } - resp, err := http.Post(w.logUploadTarget, "application/octet-stream", bytes.NewReader(logData)) + // Upload logfile to server + resp, err := http.Post(w.logUploadTarget, "application/octet-stream", logData) defer resp.Body.Close() if err != nil { return err } + // HTTP-Status-Codes 200-299 signal success, return an error for everything else if resp.StatusCode < 200 || resp.StatusCode > 299 { respBody, _ := io.ReadAll(resp.Body) return fmt.Errorf("log-upload: Got invalid http status %v: %v", resp.StatusCode, string(respBody)) @@ -118,8 +122,8 @@ func (w wrappedLogCloser) Close() error { return nil } -func createSendLogWrappedLogHandler(handler LogCloser, logfilePath string, logUploadTarget string) LogCloser { - return wrappedLogCloser{LogCloser: handler, logfilePath: logfilePath, logUploadTarget: logUploadTarget} +func createLogUploadingLogHandler(handler LogCloser, logfilePath string, logUploadTarget string) LogCloser { + return logUploadingLogCloser{LogCloser: handler, logfilePath: logfilePath, logUploadTarget: logUploadTarget} } func parseCommandOutput(commandOutput string) (all, log bool) { From 9805d2a5f8f3f54e82b89d5ae7d4e7c8e9cbdbfc Mon Sep 17 00:00:00 2001 From: FH3095 Date: Wed, 16 Apr 2025 19:40:33 +0200 Subject: [PATCH 5/7] Linter --- logger.go | 8 ++++++-- logger_test.go | 8 +++++--- main.go | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/logger.go b/logger.go index 6479e945b..1bc8d9060 100644 --- a/logger.go +++ b/logger.go @@ -83,6 +83,9 @@ func setupTargetLogger(flags commandLineFlags, logTarget, logUploadTarget, comma term.SetAllOutput(file) } if logUploadTarget != "" && filepath != "" { + if !dial.IsURL(logUploadTarget) { + return nil, fmt.Errorf("log-upload: No valid URL %v", logUploadTarget) + } handler = createLogUploadingLogHandler(handler, filepath, logUploadTarget) } } @@ -109,11 +112,12 @@ func (w logUploadingLogCloser) Close() error { return err } // Upload logfile to server - resp, err := http.Post(w.logUploadTarget, "application/octet-stream", logData) - defer resp.Body.Close() + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Post(w.logUploadTarget, "application/octet-stream", logData) if err != nil { return err } + defer resp.Body.Close() // HTTP-Status-Codes 200-299 signal success, return an error for everything else if resp.StatusCode < 200 || resp.StatusCode > 299 { respBody, _ := io.ReadAll(resp.Body) diff --git a/logger_test.go b/logger_test.go index 0ba2c072d..f20498c52 100644 --- a/logger_test.go +++ b/logger_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -151,13 +152,13 @@ func TestCloseFileHandler(t *testing.T) { func TestLogUploadFailed(t *testing.T) { handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { r.Body.Close() - w.WriteHeader(500) + w.WriteHeader(http.StatusInternalServerError) }) server := httptest.NewServer(handler) defer server.Close() closer, err := setupTargetLogger(commandLineFlags{}, filepath.Join(constants.TemporaryDirMarker, "file.log"), server.URL, "log") assert.NoError(t, err) - assert.ErrorContains(t, closer.Close(), "log-upload: Got invalid http status 500") + assert.ErrorContains(t, closer.Close(), "log-upload: Got invalid http status "+strconv.Itoa(http.StatusInternalServerError)) } func TestLogUpload(t *testing.T) { @@ -167,12 +168,13 @@ func TestLogUpload(t *testing.T) { assert.NoError(t, err) r.Body.Close() - w.WriteHeader(200) + w.WriteHeader(http.StatusOK) assert.Equal(t, strings.Trim(buffer.String(), "\r\n"), "TestLogLine") }) server := httptest.NewServer(handler) defer server.Close() closer, err := setupTargetLogger(commandLineFlags{}, filepath.Join(constants.TemporaryDirMarker, "file.log"), server.URL, "log") + assert.NoError(t, err) _, err = term.Println("TestLogLine") assert.NoError(t, err) assert.NoError(t, closer.Close()) diff --git a/main.go b/main.go index dcee596d5..e77e996a5 100644 --- a/main.go +++ b/main.go @@ -123,7 +123,7 @@ func main() { err := closer.Close() if err != nil { // Log is already closed. Write to stderr as last resort - fmt.Fprintf(os.Stderr, "Error closing logfile: %v", err) + fmt.Fprintf(os.Stderr, "Error closing logfile: %v\n", err) } } } else { From 48a89310ab336d24d933836f18b585a4a97ef1f8 Mon Sep 17 00:00:00 2001 From: FH3095 Date: Wed, 16 Apr 2025 19:44:01 +0200 Subject: [PATCH 6/7] Change POST to form with context --- logger.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/logger.go b/logger.go index 1bc8d9060..3a21bed5e 100644 --- a/logger.go +++ b/logger.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "io" "log" @@ -112,8 +113,12 @@ func (w logUploadingLogCloser) Close() error { return err } // Upload logfile to server + req, err := http.NewRequestWithContext(context.TODO(), "POST", w.logUploadTarget, logData) + if err != nil { + return err + } client := &http.Client{Timeout: 30 * time.Second} - resp, err := client.Post(w.logUploadTarget, "application/octet-stream", logData) + resp, err := client.Do(req) if err != nil { return err } From b0746a1648e6e99095ab3c03bf5bb6631615b054 Mon Sep 17 00:00:00 2001 From: FH3095 Date: Wed, 16 Apr 2025 19:54:34 +0200 Subject: [PATCH 7/7] Proper HTTP-Context --- logger.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/logger.go b/logger.go index 3a21bed5e..13b76b3ad 100644 --- a/logger.go +++ b/logger.go @@ -113,10 +113,14 @@ func (w logUploadingLogCloser) Close() error { return err } // Upload logfile to server - req, err := http.NewRequestWithContext(context.TODO(), "POST", w.logUploadTarget, logData) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, w.logUploadTarget, logData) if err != nil { return err } + req.Header.Set("Content-Type", "application/octet-stream") + client := &http.Client{Timeout: 30 * time.Second} resp, err := client.Do(req) if err != nil {