diff --git a/docs/content/monitoring/prometheus.md b/docs/content/monitoring/prometheus.md index bde4cf03f..7fa17d309 100644 --- a/docs/content/monitoring/prometheus.md +++ b/docs/content/monitoring/prometheus.md @@ -131,6 +131,13 @@ resticprofile_build_info{goversion="go1.24.3",profile="prom",version="0.31.0"} 1 ``` +## Failure behavior + +The metrics reflects the most recent `backup` run, whether it succeeded or failed. A failed run is reported with `resticprofile_backup_status` set to `0`. +This also applies to a backup that never started because a `run-before` command or a `stdin-command` exited with an error. + +The `*_bytes`, `*_files`, `*_dir` and `duration_seconds` gauges are derived from the summary that restic prints on completion. When a backup is aborted before restic prints that summary (for example when a `run-before` or `stdin-command` exits with an error), those gauges are written as `0`. + ## Prometheus Pushgateway Prometheus Pushgateway uses the job label as a grouping key. Metrics with the same grouping key are replaced when pushed. To prevent overwriting metrics from different profiles, the default job label is set to `.` (e.g., `root.backup`). @@ -229,4 +236,4 @@ root: {{< /tabs >}} -This adds the `host` label to all your metrics. \ No newline at end of file +This adds the `host` label to all your metrics. diff --git a/wrapper.go b/wrapper.go index 7364bbfc5..994b619b3 100644 --- a/wrapper.go +++ b/wrapper.go @@ -121,11 +121,14 @@ func (r *resticWrapper) summary(command string, summary monitor.Summary, stderr func (r *resticWrapper) runnerWithBeforeAndAfter(commands config.RunShellCommandsSection, command string, action func() error) func() error { return func() (err error) { err = r.runBeforeCommands(commands, command) - - if err == nil { - err = action() + if err != nil { + // Report the failure so that a run aborted by a failing "run-before" command does not + // leave the previous (successful) result in the status file and the prometheus metrics. + r.summary(r.command, monitor.Summary{}, "", err) + return } + err = action() if err == nil { err = r.runAfterCommands(commands, command) } @@ -558,7 +561,11 @@ func (r *resticWrapper) runCommand(command string) error { rCommand.stdin = streamSource } } else { - return newCommandError(rCommand, "", fmt.Errorf("%s on profile '%s': %w", r.command, r.profile.Name, err)) + // Report the failure so that a backup which never started does not leave the + // previous (successful) result in the status file and the prometheus metrics. + err = newCommandError(rCommand, "", fmt.Errorf("%s on profile '%s': %w", r.command, r.profile.Name, err)) + r.summary(r.command, monitor.Summary{}, "", err) + return err } } diff --git a/wrapper_test.go b/wrapper_test.go index deadc8aa2..acf318ccb 100644 --- a/wrapper_test.go +++ b/wrapper_test.go @@ -22,6 +22,7 @@ import ( "github.com/creativeprojects/resticprofile/constants" "github.com/creativeprojects/resticprofile/monitor" "github.com/creativeprojects/resticprofile/monitor/mocks" + "github.com/creativeprojects/resticprofile/monitor/prom" "github.com/creativeprojects/resticprofile/monitor/status" "github.com/creativeprojects/resticprofile/platform" "github.com/creativeprojects/resticprofile/restic" @@ -240,6 +241,56 @@ func TestPostProfileScriptFail(t *testing.T) { assert.EqualError(t, err, "run-after on profile 'name': exit status 1") } +func TestRunBeforeFailureReportsMonitoring(t *testing.T) { + t.Parallel() + + newWrapper := func(t *testing.T, runBefore *config.RunShellCommandsSection) (string, *resticWrapper) { + t.Helper() + profile := config.NewProfile(nil, "name") + profile.Backup = &config.BackupSection{} + if runBefore != nil { + profile.Backup.RunBefore = runBefore.RunBefore + } + promFile := filepath.Join(t.TempDir(), "metrics.prom") + profile.PrometheusSaveToFile = promFile + ctx := &Context{ + binary: mockBinary, + profile: profile, + command: constants.CommandBackup, + terminal: term.NewTerminal(), + } + wrapper := newResticWrapper(ctx) + wrapper.addProgress(prom.NewProgress(profile, prom.NewMetrics(profile.Name, "", version, "", nil))) + return promFile, wrapper + } + + readStatus := func(t *testing.T, file string) string { + t.Helper() + content, err := os.ReadFile(file) + require.NoError(t, err, "prometheus metrics file should be written") + return string(content) + } + + t.Run("ProfileRunBefore", func(t *testing.T) { + promFile, wrapper := newWrapper(t, nil) + wrapper.profile.RunBefore = []string{"exit 2"} + + err := wrapper.runProfile() + + require.Error(t, err) + assert.Contains(t, readStatus(t, promFile), `resticprofile_backup_status{profile="name"} 0`) + }) + + t.Run("SectionRunBefore", func(t *testing.T) { + promFile, wrapper := newWrapper(t, &config.RunShellCommandsSection{RunBefore: []string{"exit 2"}}) + + err := wrapper.runProfile() + + require.Error(t, err) + assert.Contains(t, readStatus(t, promFile), `resticprofile_backup_status{profile="name"} 0`) + }) +} + func TestRunEchoProfile(t *testing.T) { t.Parallel() @@ -768,6 +819,22 @@ func TestBackupWithStreamSource(t *testing.T) { assert.EqualError(t, err, "stdin-test on profile 'name': 'stdin-command' on profile 'name': exit status 2") }) + t.Run("StreamSourceFailureReportsMonitoring", func(t *testing.T) { + profile, wrapper := profileAndWrapper(t) + wrapper.command = constants.CommandBackup + profile.PrometheusSaveToFile = filepath.Join(t.TempDir(), "metrics.prom") + profile.Backup.StdinCommand = []string{"exit 2"} + profile.ResolveConfiguration() + wrapper.addProgress(prom.NewProgress(profile, prom.NewMetrics(profile.Name, "", version, "", nil))) + + err := wrapper.runCommand(constants.CommandBackup) + + require.Error(t, err) + content, readErr := os.ReadFile(profile.PrometheusSaveToFile) + require.NoError(t, readErr, "prometheus metrics file should be written even if restic never started") + assert.Contains(t, string(content), "resticprofile_backup_status{profile=\"name\"} 0") + }) + t.Run("StreamSourceWorksWithDryRun", func(t *testing.T) { profile, wrapper := profileAndWrapper(t) wrapper.dryRun = true