From b14d86f952c122c65838aa10eaeba96ea8260e05 Mon Sep 17 00:00:00 2001 From: MarcWort <113890636+MarcWort@users.noreply.github.com> Date: Mon, 31 Aug 2026 22:14:04 +0200 Subject: [PATCH 1/2] fix: report failed backup monitoring when stdin-command fails before restic starts --- docs/content/monitoring/prometheus.md | 8 +++++++- wrapper.go | 6 +++++- wrapper_test.go | 17 +++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/docs/content/monitoring/prometheus.md b/docs/content/monitoring/prometheus.md index bde4cf03f..19d002625 100644 --- a/docs/content/monitoring/prometheus.md +++ b/docs/content/monitoring/prometheus.md @@ -131,6 +131,12 @@ resticprofile_build_info{goversion="go1.24.3",profile="prom",version="0.31.0"} 1 ``` +## Failure behavior + +The metrics reflects the most recent `backup` run, whether it succeeded or failed. A failed run is reported with `resticprofile_backup_status` set to `0` + +The `*_bytes`, `*_files`, `*_dir` and `duration_seconds` gauges are derived from the summary that restic prints on completion. When a backup is aborted before restic prints that summary (for example when a `stdin-command` exits with an error), those gauges are written as `0`. + ## Prometheus Pushgateway Prometheus Pushgateway uses the job label as a grouping key. Metrics with the same grouping key are replaced when pushed. To prevent overwriting metrics from different profiles, the default job label is set to `.` (e.g., `root.backup`). @@ -229,4 +235,4 @@ root: {{< /tabs >}} -This adds the `host` label to all your metrics. \ No newline at end of file +This adds the `host` label to all your metrics. diff --git a/wrapper.go b/wrapper.go index 7364bbfc5..2392f9da5 100644 --- a/wrapper.go +++ b/wrapper.go @@ -558,7 +558,11 @@ func (r *resticWrapper) runCommand(command string) error { rCommand.stdin = streamSource } } else { - return newCommandError(rCommand, "", fmt.Errorf("%s on profile '%s': %w", r.command, r.profile.Name, err)) + // Report the failure so that a backup which never started does not leave the + // previous (successful) result in the status file and the prometheus metrics. + err = newCommandError(rCommand, "", fmt.Errorf("%s on profile '%s': %w", r.command, r.profile.Name, err)) + r.summary(r.command, monitor.Summary{}, "", err) + return err } } diff --git a/wrapper_test.go b/wrapper_test.go index deadc8aa2..504acbedf 100644 --- a/wrapper_test.go +++ b/wrapper_test.go @@ -22,6 +22,7 @@ import ( "github.com/creativeprojects/resticprofile/constants" "github.com/creativeprojects/resticprofile/monitor" "github.com/creativeprojects/resticprofile/monitor/mocks" + "github.com/creativeprojects/resticprofile/monitor/prom" "github.com/creativeprojects/resticprofile/monitor/status" "github.com/creativeprojects/resticprofile/platform" "github.com/creativeprojects/resticprofile/restic" @@ -768,6 +769,22 @@ func TestBackupWithStreamSource(t *testing.T) { assert.EqualError(t, err, "stdin-test on profile 'name': 'stdin-command' on profile 'name': exit status 2") }) + t.Run("StreamSourceFailureReportsMonitoring", func(t *testing.T) { + profile, wrapper := profileAndWrapper(t) + wrapper.command = constants.CommandBackup + profile.PrometheusSaveToFile = filepath.Join(t.TempDir(), "metrics.prom") + profile.Backup.StdinCommand = []string{"exit 2"} + profile.ResolveConfiguration() + wrapper.addProgress(prom.NewProgress(profile, prom.NewMetrics(profile.Name, "", version, "", nil))) + + err := wrapper.runCommand(constants.CommandBackup) + + require.Error(t, err) + content, readErr := os.ReadFile(profile.PrometheusSaveToFile) + require.NoError(t, readErr, "prometheus metrics file should be written even if restic never started") + assert.Contains(t, string(content), "resticprofile_backup_status{profile=\"name\"} 0") + }) + t.Run("StreamSourceWorksWithDryRun", func(t *testing.T) { profile, wrapper := profileAndWrapper(t) wrapper.dryRun = true From 14a1e8e38460444c2aade16041dc6c2b98da24fc Mon Sep 17 00:00:00 2001 From: MarcWort <113890636+MarcWort@users.noreply.github.com> Date: Mon, 31 Aug 2026 22:57:06 +0200 Subject: [PATCH 2/2] fix: report failed backup monitoring when a run-before command fails --- docs/content/monitoring/prometheus.md | 5 +-- wrapper.go | 9 +++-- wrapper_test.go | 50 +++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/docs/content/monitoring/prometheus.md b/docs/content/monitoring/prometheus.md index 19d002625..7fa17d309 100644 --- a/docs/content/monitoring/prometheus.md +++ b/docs/content/monitoring/prometheus.md @@ -133,9 +133,10 @@ resticprofile_build_info{goversion="go1.24.3",profile="prom",version="0.31.0"} 1 ## Failure behavior -The metrics reflects the most recent `backup` run, whether it succeeded or failed. A failed run is reported with `resticprofile_backup_status` set to `0` +The metrics reflects the most recent `backup` run, whether it succeeded or failed. A failed run is reported with `resticprofile_backup_status` set to `0`. +This also applies to a backup that never started because a `run-before` command or a `stdin-command` exited with an error. -The `*_bytes`, `*_files`, `*_dir` and `duration_seconds` gauges are derived from the summary that restic prints on completion. When a backup is aborted before restic prints that summary (for example when a `stdin-command` exits with an error), those gauges are written as `0`. +The `*_bytes`, `*_files`, `*_dir` and `duration_seconds` gauges are derived from the summary that restic prints on completion. When a backup is aborted before restic prints that summary (for example when a `run-before` or `stdin-command` exits with an error), those gauges are written as `0`. ## Prometheus Pushgateway diff --git a/wrapper.go b/wrapper.go index 2392f9da5..994b619b3 100644 --- a/wrapper.go +++ b/wrapper.go @@ -121,11 +121,14 @@ func (r *resticWrapper) summary(command string, summary monitor.Summary, stderr func (r *resticWrapper) runnerWithBeforeAndAfter(commands config.RunShellCommandsSection, command string, action func() error) func() error { return func() (err error) { err = r.runBeforeCommands(commands, command) - - if err == nil { - err = action() + if err != nil { + // Report the failure so that a run aborted by a failing "run-before" command does not + // leave the previous (successful) result in the status file and the prometheus metrics. + r.summary(r.command, monitor.Summary{}, "", err) + return } + err = action() if err == nil { err = r.runAfterCommands(commands, command) } diff --git a/wrapper_test.go b/wrapper_test.go index 504acbedf..acf318ccb 100644 --- a/wrapper_test.go +++ b/wrapper_test.go @@ -241,6 +241,56 @@ func TestPostProfileScriptFail(t *testing.T) { assert.EqualError(t, err, "run-after on profile 'name': exit status 1") } +func TestRunBeforeFailureReportsMonitoring(t *testing.T) { + t.Parallel() + + newWrapper := func(t *testing.T, runBefore *config.RunShellCommandsSection) (string, *resticWrapper) { + t.Helper() + profile := config.NewProfile(nil, "name") + profile.Backup = &config.BackupSection{} + if runBefore != nil { + profile.Backup.RunBefore = runBefore.RunBefore + } + promFile := filepath.Join(t.TempDir(), "metrics.prom") + profile.PrometheusSaveToFile = promFile + ctx := &Context{ + binary: mockBinary, + profile: profile, + command: constants.CommandBackup, + terminal: term.NewTerminal(), + } + wrapper := newResticWrapper(ctx) + wrapper.addProgress(prom.NewProgress(profile, prom.NewMetrics(profile.Name, "", version, "", nil))) + return promFile, wrapper + } + + readStatus := func(t *testing.T, file string) string { + t.Helper() + content, err := os.ReadFile(file) + require.NoError(t, err, "prometheus metrics file should be written") + return string(content) + } + + t.Run("ProfileRunBefore", func(t *testing.T) { + promFile, wrapper := newWrapper(t, nil) + wrapper.profile.RunBefore = []string{"exit 2"} + + err := wrapper.runProfile() + + require.Error(t, err) + assert.Contains(t, readStatus(t, promFile), `resticprofile_backup_status{profile="name"} 0`) + }) + + t.Run("SectionRunBefore", func(t *testing.T) { + promFile, wrapper := newWrapper(t, &config.RunShellCommandsSection{RunBefore: []string{"exit 2"}}) + + err := wrapper.runProfile() + + require.Error(t, err) + assert.Contains(t, readStatus(t, promFile), `resticprofile_backup_status{profile="name"} 0`) + }) +} + func TestRunEchoProfile(t *testing.T) { t.Parallel()