Wire nine observability fixtures and fix the dead metrics invariants

AGENTS.md

@@ -75,6 +75,36 @@ by convention.
   `SubgraphNode.run`, projection variants, frozen-state mutation, etc.
 - `tests/test_smoke.py` — version sync.
 
+### Activating a conformance fixture is not done when it passes
+
+A green conformance run is the **null result**, not evidence. A fixture
+wired into a driver that ignores half its `expected` block passes exactly
+like one that is fully asserted, so "it passes" distinguishes nothing.
+
+The acceptance criterion is therefore inverted: a fixture is wired when
+you have **broken the behaviour it covers and watched it go red**.
+
+- Mutate the src behaviour, not the test. Confirm the mutation actually
+  landed before trusting the result: if the anchor text occurs twice and
+  you changed one, a partial mutation reads exactly like a surviving one.
+- A fixture that passes the moment you add a dispatch entry, with no
+  harness work, is the highest-risk case, not the easiest win.
+- Diagnose by running, not by reading. A deferral reason reached by
+  reasoning about which driver "looks" suitable is routinely wrong; a
+  ten-second `await _run_x(_load(path))` settles it.
+- Prefer a structural guard over remembering any of this, but check its
+  reach before trusting it. The `_DRIVER_EXPECTED_KEYS` check in
+  `tests/conformance/test_observability.py` catches "wired into a driver
+  that drops a directive" — it caught a live error in the commit that
+  added it — but only for the drivers registered in that map, which is a
+  minority of them. It is **fail-open**: a fixture routed to an
+  unregistered driver returns silently, so a green run there still means
+  nothing. Registering the driver is what turns the guard on.
+
+This is written down because the rule existed as guidance and was still
+missed repeatedly across one session: four fixtures were reported wired
+while asserting nothing, and three deferral reasons were wrong.
+
 ## Tooling
 
 - `uv` for everything. Don't use `pip` directly.

tests/conformance/harness/capabilities.py

@@ -149,3 +149,30 @@ def assert_some_case_ran(
     "capability_skip_reason",
     "report_recognized_skip",
 ]
+
+
+def assert_case_asserts_something(fixture_stem: str, case_name: str, expected: Mapping[str, object]) -> None:
+    """Fail a case whose ``expected`` block carries nothing but ``invariants``."""
+    # Invariant blocks in this corpus are DOCUMENTARY: they restate in prose what
+    # a concrete directive (span_tree / langfuse_trace / metrics / event_count)
+    # already pins, and runners deliberately do not implement most of them. That
+    # convention is only safe while something concrete is present to do the
+    # pinning. A case whose expected block is invariants-ONLY has no such backing,
+    # so it asserts exactly nothing while reading like coverage.
+    #
+    # Matches no ACTIVATED fixture. That is weaker than "no fixture": an
+    # invariants-only case does exist in the corpus and escapes only because it
+    # is skipped upstream, which makes this guard's placement relative to the
+    # skip chain load-bearing rather than incidental. It exists because the metrics driver reached that state
+    # by a different route -- reading `metrics` but never `invariants`, on two
+    # fixtures whose invariants were their only negative assertion -- and both
+    # passed while checking nothing.
+    concrete = set(expected) - {"invariants"}
+    if concrete or not expected.get("invariants"):
+        return
+    raise AssertionError(
+        f"{fixture_stem}::{case_name}: `expected` declares only `invariants` and no concrete "
+        f"directive, so nothing is asserted unless the runner implements every one of them. "
+        f"Add a concrete assertion, or implement the invariants and register them in the "
+        f"driver's known-invariant set."
+    )

tests/conformance/test_capability_gate_wiring.py

@@ -320,3 +320,106 @@ def test_every_gated_fixture_belongs_to_a_runner_that_implements_the_gate() -> N
         f"runner implements the gate. Wire it into their runner "
         f"(tests/conformance/harness/capabilities.py) before those fixtures are activated."
     )
+
+
+# -- invariants-only guard --------------------------------------------------
+
+
+def test_invariants_only_expected_block_is_rejected() -> None:
+    from .harness.capabilities import assert_case_asserts_something
+
+    with pytest.raises(AssertionError, match="only `invariants`"):
+        assert_case_asserts_something(
+            "999-synthetic",
+            "case_with_no_concrete_assertion",
+            {"invariants": {"some_absence_claim": True}},
+        )
+
+
+@pytest.mark.parametrize(
+    "expected",
+    [
+        {"span_tree": [], "invariants": {"x": True}},
+        {"metrics": [], "invariants": {"x": True}},
+        {"langfuse_trace": {}, "invariants": {"x": True}},
+        {"invariants": {}},
+        {},
+    ],
+)
+def test_a_case_with_any_concrete_directive_is_accepted(expected: dict[str, object]) -> None:
+    from .harness.capabilities import assert_case_asserts_something
+
+    assert_case_asserts_something("999-synthetic", "case", expected)
+
+
+@pytest.mark.parametrize(
+    ("module_name", "runner_attr", "fixture_stem"),
+    [
+        # Each dispatch path x each fixture SHAPE. A single-case fixture puts
+        # `expected` at the top level with no `cases` list -- the shape the first
+        # version of this wiring skipped entirely -- while a multi-case fixture
+        # exercises the `spec["cases"][0]` branch of the graft below. Naming only
+        # single-case fixtures left that branch unexecuted, so the two shapes are
+        # now covered explicitly rather than assumed from a fixture id.
+        ("tests.conformance.test_observability", "test_observability_fixture", "001-otel-basic-trace"),
+        (
+            "tests.conformance.test_observability",
+            "test_observability_fixture",
+            "005-otel-llm-provider-span-nested",
+        ),
+        (
+            "tests.conformance.test_observability_langfuse",
+            "test_langfuse_fixture",
+            "022-langfuse-basic-trace",
+        ),
+        (
+            "tests.conformance.test_observability_langfuse",
+            "test_langfuse_fixture",
+            "023-langfuse-generation-rendering",
+        ),
+    ],
+)
+@pytest.mark.asyncio
+async def test_the_guard_is_wired_into_each_runner(
+    monkeypatch: pytest.MonkeyPatch, module_name: str, runner_attr: str, fixture_stem: str
+) -> None:
+    # Non-vacuity for the WIRING, not the predicate. The rejecting test above
+    # already proves the predicate; this one failed to prove anything, because it
+    # only counted fixtures on disk and passed identically with both call sites
+    # deleted. Graft an invariants-only expected block onto a real fixture and
+    # push it through each runner's actual entry point.
+    import importlib
+
+    module = importlib.import_module(module_name)
+    path = next(p for p in module._fixture_paths() if p.stem == fixture_stem)  # noqa: SLF001
+    original_load = module._load  # noqa: SLF001
+    spec_shape = original_load(path)
+    multi_case = bool(spec_shape.get("cases"))
+
+    def _grafted(p: Any) -> Any:
+        spec = original_load(p)
+        target = spec["cases"][0] if spec.get("cases") else spec
+        target["expected"] = {"invariants": {"synthetic_unbacked_claim": True}}
+        return spec
+
+    monkeypatch.setattr(module, "_load", _grafted)
+    # Both runners `pytest.skip` for a fixture they do not currently drive, and
+    # `Skipped` derives from BaseException, so it sails through `pytest.raises`
+    # and turns this whole test into a silent skip. That is the exact failure
+    # this file exists to prevent, one level up: the day a named fixture leaves
+    # its runner's supported set, a skip must read as a wiring regression.
+    try:
+        with pytest.raises(AssertionError, match="only `invariants`"):
+            await getattr(module, runner_attr)(path)
+    except pytest.skip.Exception as exc:  # pragma: no cover - regression guard
+        pytest.fail(
+            f"{runner_attr} skipped {fixture_stem} ({exc}), so this test asserted nothing about "
+            f"the guard. Repoint it at a fixture that runner still drives."
+        )
+    # Non-vacuity on the parametrization itself: the two shapes must actually
+    # differ, or both rows exercise the same branch of the graft above.
+    multi_case_rows = {"005-otel-llm-provider-span-nested", "023-langfuse-generation-rendering"}
+    assert multi_case == (fixture_stem in multi_case_rows), (
+        f"{fixture_stem} no longer has the fixture shape this row was chosen for "
+        f"(multi_case={multi_case}); the single-case and multi-case branches are no longer both covered."
+    )

tests/conformance/test_harness_fidelity.py

@@ -0,0 +1,203 @@
+"""Structural guards on the conformance harness itself."""
+
+# Nothing here asserts engine behaviour. These tests assert that the harness's
+# own allowlists still describe what its code does, because both have already
+# drifted in ways no fixture run could reveal: a green conformance run is
+# identical whether an assertion is live or dead. See the AGENTS.md note
+# "Activating a conformance fixture is not done when it passes".
+
+from __future__ import annotations
+
+import ast
+import inspect
+from typing import Any
+
+import pytest
+
+from . import test_observability as otel_runner
+from . import test_observability_langfuse as langfuse_runner
+
+# Drivers whose `expected` keys are deliberately NOT declared in
+# `_DRIVER_EXPECTED_KEYS`, with the reason they are safe to leave unguarded.
+# The guard is fail-open, so an unregistered driver is invisible to it; listing
+# them here is what turns "invisible" into "decided". A new driver appears in
+# neither map and fails the test below, forcing that decision at the point it is
+# added rather than leaving it dark indefinitely.
+#
+# A single-fixture driver cannot suffer the defect the guard exists to catch --
+# a fixture routed to a SHARED driver that ignores one of its directives -- since
+# it was written against exactly one fixture's `expected` block.
+_UNGUARDED_SINGLE_FIXTURE_DRIVERS = frozenset(
+    {
+        "_run_fixture_001",
+        "_run_fixture_002",
+        "_run_fixture_003",
+        "_run_fixture_004",
+        "_run_fixture_005",
+        "_run_fixture_006",
+        "_run_fixture_007",
+        "_run_fixture_008",
+        "_run_fixture_009",
+        "_run_fixture_010",
+        "_run_fixture_011",
+        "_run_fixture_028",
+        "_run_fixture_038",
+        "_run_fixture_050",
+        "_run_fixture_051",
+        "_run_fixture_052",
+        "_run_fixture_053",
+        "_run_fixture_054",
+        "_run_fixture_055",
+        "_run_fixture_056",
+        "_run_fixture_058",
+        "_run_fixture_084",
+        "_run_fixture_110",
+        "_run_fixture_132",
+        "_run_fixture_133",
+        "_run_structured_output_error_span_fixture",
+    }
+)
+
+# Shared drivers that ARE exposed to the mis-routing defect and are not yet
+# registered. This is a real gap, recorded rather than hidden: registering one
+# means deriving its read set from its body (and its callees), which is the step
+# that produced a wrong `observers` claim when done by inspection alone.
+_UNGUARDED_SHARED_DRIVERS = frozenset(
+    {
+        "_run_get_invocation_metadata_fixture",
+        "_run_llm_cache_fixture",
+        "_run_tool_fixture",
+        "_run_typed_event_chain_cases",
+    }
+)
+
+
+def test_every_dispatched_driver_is_registered_or_explicitly_unguarded() -> None:
+    dispatched = set(otel_runner._driver_for_fixture().values())  # noqa: SLF001
+    accounted = (
+        set(otel_runner._DRIVER_EXPECTED_KEYS)  # noqa: SLF001
+        | _UNGUARDED_SINGLE_FIXTURE_DRIVERS
+        | _UNGUARDED_SHARED_DRIVERS
+    )
+    unaccounted = sorted(dispatched - accounted)
+    assert not unaccounted, (
+        f"drivers {unaccounted} are dispatched to but appear in neither `_DRIVER_EXPECTED_KEYS` nor "
+        f"the unguarded lists in this file. Register the driver's `expected` keys, or add it to an "
+        f"unguarded list with the reason -- an unregistered driver silently exempts its fixtures "
+        f"from the mis-routing guard."
+    )
+    # Non-vacuity on the INPUT: an empty dispatch map satisfies the subset check
+    # above trivially, which is how a guard over nothing reads as a clean pass.
+    assert dispatched, "the dispatch map is empty, so the check above compared nothing"
+    # And the lists must not accumulate names for drivers that no longer exist.
+    stale = sorted((_UNGUARDED_SINGLE_FIXTURE_DRIVERS | _UNGUARDED_SHARED_DRIVERS) - dispatched)
+    assert not stale, f"unguarded lists name drivers nothing dispatches to: {stale}"
+
+
+def _invariant_names_read_by(func: object, param: str) -> set[str]:
+    """String literals `func` looks up on the mapping named `param`."""
+    tree = ast.parse(inspect.getsource(func))  # pyright: ignore[reportArgumentType]
+    names: set[str] = set()
+    for node in ast.walk(tree):
+        if (
+            isinstance(node, ast.Call)
+            and isinstance(node.func, ast.Attribute)
+            and node.func.attr == "get"
+            and isinstance(node.func.value, ast.Name)
+            and node.func.value.id == param
+            and node.args
+            and isinstance(node.args[0], ast.Constant)
+            and isinstance(node.args[0].value, str)
+        ):
+            names.add(node.args[0].value)
+        elif (
+            isinstance(node, ast.Subscript)
+            and isinstance(node.value, ast.Name)
+            and node.value.id == param
+            and isinstance(node.slice, ast.Constant)
+            and isinstance(node.slice.value, str)
+        ):
+            names.add(node.slice.value)
+    return names
+
+
+def _span_dependent_invariants_in(func: object) -> set[str]:
+    """Invariant names whose `if invariants.get(...)` body reads the span set."""
+    tree = ast.parse(inspect.getsource(func))  # pyright: ignore[reportArgumentType]
+    found: set[str] = set()
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.If):
+            continue
+        names = {
+            call.args[0].value
+            for call in ast.walk(node.test)
+            if isinstance(call, ast.Call)
+            and isinstance(call.func, ast.Attribute)
+            and call.func.attr == "get"
+            and isinstance(call.func.value, ast.Name)
+            and call.func.value.id == "invariants"
+            and call.args
+            and isinstance(call.args[0], ast.Constant)
+            and isinstance(call.args[0].value, str)
+        }
+        if not names:
+            continue
+        body = ast.Module(body=node.body, type_ignores=[])
+        if any(isinstance(n, ast.Name) and n.id in {"spans", "_spans_carrying"} for n in ast.walk(body)):
+            found |= names
+    return found
+
+
+def test_span_dependent_invariants_lists_every_span_reading_claim() -> None:
+    # `_assert_span_anchor` only fires for names in `_SPAN_DEPENDENT_INVARIANTS`.
+    # A span-reading claim missing from that set gets no positive anchor, so an
+    # empty span set satisfies it trivially -- which is what the `startswith`
+    # heuristic this set replaced actually did to the two
+    # `exceeded_span_signal_*` names.
+    derived = _span_dependent_invariants_in(
+        otel_runner._assert_metrics_invariants  # noqa: SLF001
+    ) | _span_dependent_invariants_in(otel_runner._assert_token_budget_invariants)  # noqa: SLF001
+    assert derived, "parsed no span-reading invariant blocks, so this compared nothing"
+    missing = sorted(derived - otel_runner._SPAN_DEPENDENT_INVARIANTS)  # noqa: SLF001
+    assert not missing, (
+        f"{missing} read the span set but are absent from `_SPAN_DEPENDENT_INVARIANTS`, so no "
+        f"positive anchor runs for them and a zero-span run satisfies them vacuously."
+    )
+    stale = sorted(otel_runner._SPAN_DEPENDENT_INVARIANTS - derived)  # noqa: SLF001
+    assert not stale, f"`_SPAN_DEPENDENT_INVARIANTS` names claims that no longer read spans: {stale}"
+
+
+def _drive_metrics_guard(case: dict[str, Any]) -> None:
+    otel_runner._assert_metrics_invariants(case, [], [])  # noqa: SLF001
+
+
+def _drive_token_budget_guard(case: dict[str, Any]) -> None:
+    otel_runner._assert_token_budget_invariants(case, [], {}, [])  # noqa: SLF001
+
+
+@pytest.mark.parametrize("drive", [_drive_metrics_guard, _drive_token_budget_guard])
+@pytest.mark.parametrize("invariant", sorted(otel_runner._SPAN_DEPENDENT_INVARIANTS))  # noqa: SLF001
+def test_a_zero_span_run_cannot_satisfy_a_span_absence_claim(invariant: str, drive: Any) -> None:
+    # The test above keeps `_SPAN_DEPENDENT_INVARIANTS` honest about which names
+    # read spans; this one keeps the ANCHOR ITSELF live. Deleting
+    # `_assert_span_anchor`, or its call from either guard, leaves every shipping
+    # fixture green -- both fixtures that declare a span-absence claim also
+    # declare `span_tree`, whose own root-span check fails first, so nothing in
+    # the corpus would notice. Driving each guard directly is what makes the
+    # anchor's removal detectable at all.
+    with pytest.raises(AssertionError, match="pass vacuously"):
+        drive({"expected": {"invariants": {invariant: True}}})
+
+
+def test_per_trace_invariants_covers_everything_assert_trace_reads() -> None:
+    # `_assert_multi_traces` filters the fixture's invariants down to
+    # `_PER_TRACE_INVARIANTS` before delegating. A name `_assert_trace` checks
+    # but that is absent from the set is therefore dropped on that path, and the
+    # fixture declaring it still passes.
+    read = _invariant_names_read_by(langfuse_runner._assert_trace, "expected_invariants")  # noqa: SLF001
+    assert read, "parsed no invariant lookups out of _assert_trace, so this compared nothing"
+    missing = sorted(read - set(langfuse_runner._PER_TRACE_INVARIANTS))  # noqa: SLF001
+    assert not missing, (
+        f"`_assert_trace` checks {missing}, but `_PER_TRACE_INVARIANTS` omits them, so the "
+        f"multi-trace runner discards those claims silently."
+    )