feat(xtest): compare benchmark candidates with each other

.github/workflows/check.yml

@@ -52,7 +52,8 @@ jobs:
           uv run --frozen --no-build pytest --no-header -q
           test_bench_stats.py test_bench_measure.py test_bench_runner.py
           test_bench_arms.py test_bench_config.py test_bench_report.py
-          test_bench_installation.py test_sdk_commands.py test_tdfs_units.py test_encryption_units.py
+          test_bench_installation.py test_bench_conclusions.py
+          test_sdk_commands.py test_tdfs_units.py test_encryption_units.py
           test_sizes_units.py test_zip64_units.py test_assertions_units.py
         working-directory: xtest
       - name: Lint and test otdf-local

xtest/conftest.py

@@ -630,6 +630,8 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int):
     if reporter is not None:
         reporter.write_sep("=", "benchmark results")
         reporter.write_line(report.headline(recorder, gate))
+        for conclusion in recorder.conclusions(bench_config, gate):
+            reporter.write_line(report.conclusion_line(conclusion))
         reporter.write_line(f"raw samples and statistics: {json_path}")
 
     if config.getoption("--bench-no-gate", default=False):

xtest/perf/README.md

@@ -51,9 +51,11 @@ The artifact is `"schema": 2`:
 
 - Each cell names its `arms` (`id` and build `label`) and its `reference`,
   which is an arm id. `samples` is keyed by arm id.
-- The statistics live under `contrasts`, one entry per candidate. Each has `a`
-  (the reference's arm id), `b` (the candidate's), and `metrics`, whose
-  per-metric results carry `a_median` and `b_median`.
+- The statistics live under `contrasts`: one entry per candidate against the
+  reference, then one per pair of candidates (see
+  [Comparing candidates](#comparing-candidates-with-each-other)). Each has `a`
+  and `b`, read as "`b` against `a`", and `metrics`, whose per-metric results
+  carry `a_median` and `b_median`.
 - Two-arm cells also carry the `baseline` and `candidate` labels.
 - `regressions`, `improvements` and `noise_floor_by_control` are keyed
   `<cell>/<b>_vs_<a>/<metric>`, e.g. `go-encrypt-1MiB/main_vs_v0.29.0/wall`.
@@ -158,15 +160,47 @@ lists the reason for each cell.
 > Check with `otdf-sdk-mgr versions resolve <sdk> main latest` — one entry back
 > instead of two means there is nothing to measure until `main` moves.
 
+### Comparing candidates with each other
+
+With more than one candidate, each pair of candidates is also compared
+directly — the same within-round ratio, so the comparison is as fair as the
+one against the reference. These comparisons **never fail the build**: there
+is no incumbent, so "regression" means nothing between them. They are listed
+under **Candidate comparisons** in the summary and in the JSON `conclusions`.
+
+**FASTER** / **SLOWER** — the interval lies entirely outside the
+`[1/threshold, threshold]` band and the matching one-sided adjusted p-value is
+below `alpha / 2`. Half alpha because the question is two-sided: neither
+candidate is privileged, and testing both tails at alpha would double the
+chance of ranking two equal builds.
+
+**TIED** — the interval lies entirely *inside* the band: any real difference
+is smaller than the threshold. A positive finding, and deliberately not PASS,
+which only claims "not slower".
+
+**inconclusive** — anything else, including a missing or underpowered A/A
+control.
+
+Reversing a pair turns FASTER into SLOWER and leaves TIED alone, so the order
+the candidates were listed in does not change any conclusion. They form their
+own correction family: adding candidates costs the regression gate no power.
+
+A **winner** is named for a cell and metric only when one candidate is FASTER
+than *every* other candidate in their own contrasts, and the A/A control did
+not trip. A tie, an inconclusive pair involving a contender, or a tripped
+control — however narrow the interval — means no winner, and the summary says
+which of those it was. Ranking by point estimates is not evidence that two
+candidates differ.
+
 ### The A/A control
 
 Each SDK gets a control cell that compares the reference build **against itself**
 through the identical pipeline. Its true ratio is exactly 1.0 by construction, so
 whatever it reports is the harness's own error on this runner. With K arms the
 control runs K copies of the reference: a K-arm round is longer, and the two arms
 of a contrast sit further apart in it on average, so a two-arm control would
-understate the noise. Of its K-1 A/A contrasts, the worst one stands as the
-floor. It does two jobs:
+understate the noise. Of its C(K,2) copy-to-copy contrasts, the worst one stands
+as the floor. It does two jobs:
 
 - If the control *trips* — its own A/A comparison looks like a real effect — then
   something is systematically biased and **the whole run stops being able to fail
@@ -264,6 +298,7 @@ things; the noise floor will tell you whether you succeeded.
 | `runner.py` | The paired round loop, the stopping rule, the budget, `analyze()` |
 | `stats.py` | Pure functions: log-ratios, bootstrap CI, Wilcoxon, BH, the decision rule |
 | `report.py` | Session recorder, JSON artifact (`SCHEMA_VERSION`), step-summary markdown |
+| `conclusions.py` | Candidate-to-candidate conclusions and winner eligibility, read by every renderer |
 | `config.py` | Option parsing and validation shared by pytest and CI: `BenchConfig`, `--bench-refs`, the arm ceiling, the budget default. No pytest |
 | `installation.py` | Reads the installation manifest `otdf-sdk-mgr install benchmark` writes. No pytest |
 | `docs/` | The figures in this README, and the Monte Carlo behind them. Documentation only; imported by nothing in the harness |
@@ -588,6 +623,8 @@ two builds doing different amounts of work) is invisible in the output.
 10. Never hand-edit the figures. Change `docs/make_figures.py` and regenerate;
     if a number moves, `--verify` is what tells you whether the figure or the
     world changed.
+11. Never let a comparison between two candidates fail the build, share the
+    gate's correction family, or name a winner the conclusions model did not.
 
 Every one of these fails *silently* and *plausibly* when broken: the numbers
 still look like numbers. That is why they are written down.

xtest/perf/conclusions.py

@@ -0,0 +1,167 @@
+"""What a multi-candidate run concluded about its candidates.
+
+The gate answers "did any candidate get slower than the reference". This
+module answers the other question a run with several candidates is asked:
+"how do the candidates compare with each other, and is one of them the best".
+It never changes the gate. A build must not go red because one candidate lost
+to another.
+
+Pairwise conclusions come first. Point-estimate order is not evidence: two
+candidates whose ratios against the reference are 1.02 and 1.05 have not been
+shown to differ unless their own contrast says so. A unique winner is named
+only when it is supported against *every* other candidate, and never when the
+run's control tripped.
+
+Winner eligibility is decided here, once, from the analysed result. The JSON
+artifact, the terminal line, and the markdown summary all read these objects,
+so they cannot disagree about the conclusion.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from perf import stats
+from perf.runner import BenchConfig, CellResult, contrast_key
+
+
+@dataclass(frozen=True, slots=True)
+class Pairwise:
+    """One candidate-to-candidate contrast, read as "``b`` vs ``a``"."""
+
+    a: str
+    b: str
+    verdict: stats.Verdict
+    ratio: float
+    ci_low: float
+    ci_high: float
+    note: str
+
+    def faster(self) -> str | None:
+        """The arm shown to be faster, or None if neither was."""
+        if self.verdict is stats.Verdict.FASTER:
+            return self.b
+        if self.verdict is stats.Verdict.SLOWER:
+            return self.a
+        return None
+
+
+@dataclass(frozen=True, slots=True)
+class Conclusion:
+    """The candidates of one cell compared on one metric."""
+
+    cell_id: str
+    metric: str
+    #: Arm id -> build label, for rendering.
+    labels: dict[str, str]
+    pairwise: tuple[Pairwise, ...]
+    #: The candidate supported against every other, or None.
+    winner: str | None
+    #: Why there is or is not a winner, in one sentence.
+    reason: str
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "cell": self.cell_id,
+            "metric": self.metric,
+            "pairwise": [
+                {
+                    "a": p.a,
+                    "b": p.b,
+                    "verdict": str(p.verdict),
+                    "note": p.note,
+                }
+                for p in self.pairwise
+            ],
+            "winner": self.winner,
+            "reason": self.reason,
+        }
+
+
+def conclude(
+    results: Sequence[CellResult], config: BenchConfig, gate: stats.GateResult
+) -> list[Conclusion]:
+    """Compare the candidates of every cell that measured more than one.
+
+    Empty for a two-arm run: one candidate has nothing to be compared with,
+    and the gate has already said everything there is to say about it.
+    """
+    out: list[Conclusion] = []
+    for r in results:
+        if r.control or len(r.candidates) < 2:
+            continue
+        for metric in config.gated_metrics:
+            pairwise = tuple(
+                _pairwise(a, b, gate.comparisons[key])
+                for a, b in r.contrast_pairs()
+                if not r.is_reference_pair(a, b)
+                and (key := contrast_key(r.cell_id, a, b, metric)) in gate.comparisons
+            )
+            winner, reason = _winner(r.candidates, pairwise, gate)
+            out.append(
+                Conclusion(
+                    cell_id=r.cell_id,
+                    metric=metric,
+                    labels=dict(r.arm_labels),
+                    pairwise=pairwise,
+                    winner=winner,
+                    reason=reason,
+                )
+            )
+    return out
+
+
+def _pairwise(a: str, b: str, c: stats.PairedComparison) -> Pairwise:
+    return Pairwise(
+        a=a,
+        b=b,
+        verdict=c.verdict,
+        ratio=c.ratio,
+        ci_low=c.ci_low,
+        ci_high=c.ci_high,
+        note=c.note,
+    )
+
+
+def _winner(
+    candidates: Sequence[str], pairwise: Sequence[Pairwise], gate: stats.GateResult
+) -> tuple[str | None, str]:
+    """Name the candidate faster than every other, or say why there is none.
+
+    Every requirement is checked, in an order that gives the most useful
+    reason first:
+
+    1. The control must not have tripped. A biased harness can produce a
+       narrow, decisive-looking interval, so a tripped control suppresses the
+       claim however clean the contrast looks.
+    2. Some candidate must be shown FASTER than each of the others in its own
+       contrast. Beating the runner-up is not enough when a third candidate's
+       contrast is unresolved: that competitor may be the faster one.
+    """
+    if not gate.trustworthy:
+        return None, (
+            "no winner: the A/A control failed its own comparison, so no "
+            "ranking from this run can be trusted"
+        )
+    beats: dict[str, set[str]] = {c: set() for c in candidates}
+    for p in pairwise:
+        faster = p.faster()
+        if faster is not None:
+            beats[faster].add(p.b if faster == p.a else p.a)
+    for c in candidates:
+        if beats[c] == set(candidates) - {c}:
+            return c, f"faster than every other candidate ({len(candidates) - 1})"
+
+    ties = [p for p in pairwise if p.verdict is stats.Verdict.TIED]
+    unresolved = [p for p in pairwise if p.verdict is stats.Verdict.INCONCLUSIVE]
+    if unresolved:
+        p = unresolved[0]
+        return None, (
+            f"no winner: `{p.b}` vs `{p.a}` is unresolved"
+            + (f" ({p.note})" if p.note else "")
+        )
+    if ties:
+        p = ties[0]
+        return None, f"no winner: `{p.a}` and `{p.b}` are tied"
+    return None, "no winner: no candidate is faster than every other"

xtest/perf/report.py

@@ -29,6 +29,7 @@
 
 from perf import stats
 from perf.cells import BenchCell
+from perf.conclusions import Conclusion, conclude
 from perf.measure import METRIC_LABELS, METRICS, format_metric
 from perf.runner import BenchConfig, CellResult, analyze, contrast_key
 
@@ -73,6 +74,11 @@ def skip(self, cell_id: str, reason: str) -> None:
     def gate(self, config: BenchConfig) -> stats.GateResult:
         return analyze(self.results, config)
 
+    def conclusions(
+        self, config: BenchConfig, gate: stats.GateResult
+    ) -> list[Conclusion]:
+        return conclude(self.results, config, gate)
+
 
 def to_dict(
     recorder: BenchmarkRecorder, config: BenchConfig, gate: stats.GateResult
@@ -104,6 +110,7 @@ def to_dict(
         "regressions": gate.regressions,
         "improvements": gate.improvements,
         "summary": gate.summary,
+        "conclusions": [c.to_dict() for c in recorder.conclusions(config, gate)],
         "skipped": recorder.skipped,
         "cells": [_cell_dict(r, gate) for r in recorder.results],
     }
@@ -257,6 +264,8 @@ def markdown(
     ]
     for result in recorder.results:
         for a, b in result.contrast_pairs():
+            if not result.is_reference_pair(a, b):
+                continue
             candidate = f"A/A {b}" if result.control else f"`{result.arm_labels[b]}`"
             for metric in METRICS:
                 c = gate.comparisons[contrast_key(result.cell_id, a, b, metric)]
@@ -270,6 +279,8 @@ def markdown(
                     f"| {_verdict_cell(c)} |"
                 )
 
+    lines += _candidate_comparisons(recorder.conclusions(config, gate))
+
     references = sorted(
         {r.arm_labels[r.reference] for r in recorder.results if not r.control}
     )
@@ -290,6 +301,46 @@ def markdown(
     return "\n".join(lines) + "\n"
 
 
+def _candidate_comparisons(conclusions: list[Conclusion]) -> list[str]:
+    """The candidate-to-candidate section: pairwise verdicts, then any winner."""
+    if not conclusions:
+        return []
+    lines = [
+        "",
+        "### Candidate comparisons",
+        "",
+        "Each pair of candidates, compared directly. These never fail the build. "
+        "A winner is named only when it is faster than every other candidate "
+        "and the A/A control held.",
+        "",
+        "| cell | metric | comparison | ratio (95% CI) | verdict |",
+        "| --- | --- | --- | --- | --- |",
+    ]
+    for c in conclusions:
+        for p in c.pairwise:
+            interval = (
+                f"{p.ratio:.3f}x [{p.ci_low:.3f}, {p.ci_high:.3f}]"
+                if math.isfinite(p.ci_low) and math.isfinite(p.ci_high)
+                else "-"
+            )
+            verdict = f"{p.verdict} ({p.note})" if p.note else str(p.verdict)
+            lines.append(
+                f"| {c.cell_id} | {METRIC_LABELS[c.metric][0]} "
+                f"| `{c.labels[p.b]}` vs `{c.labels[p.a]}` | {interval} | {verdict} |"
+            )
+    lines.append("")
+    lines += [conclusion_line(c) for c in conclusions]
+    return lines
+
+
+def conclusion_line(c: Conclusion) -> str:
+    """One line per cell and metric: the winner, or why there is none."""
+    head = f"- **{c.cell_id}** ({METRIC_LABELS[c.metric][0]}): "
+    if c.winner is not None:
+        return head + f"`{c.labels[c.winner]}` wins, {c.reason}."
+    return head + f"{c.reason}."
+
+
 def _ratio_cell(c: stats.PairedComparison) -> str:
     if not math.isfinite(c.ratio):
         return "-"
@@ -317,6 +368,9 @@ def _p_cell(c: stats.PairedComparison) -> str:
     stats.Verdict.REGRESSION: "**REGRESSION**",
     stats.Verdict.IMPROVED: "IMPROVED",
     stats.Verdict.INCONCLUSIVE: "inconclusive",
+    stats.Verdict.FASTER: "FASTER",
+    stats.Verdict.SLOWER: "SLOWER",
+    stats.Verdict.TIED: "TIED",
 }