diff --git a/AGENTS.md b/AGENTS.md index c3ace78..446660a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,6 +75,36 @@ by convention. `SubgraphNode.run`, projection variants, frozen-state mutation, etc. - `tests/test_smoke.py` — version sync. +### Activating a conformance fixture is not done when it passes + +A green conformance run is the **null result**, not evidence. A fixture +wired into a driver that ignores half its `expected` block passes exactly +like one that is fully asserted, so "it passes" distinguishes nothing. + +The acceptance criterion is therefore inverted: a fixture is wired when +you have **broken the behaviour it covers and watched it go red**. + +- Mutate the src behaviour, not the test. Confirm the mutation actually + landed before trusting the result: if the anchor text occurs twice and + you changed one, a partial mutation reads exactly like a surviving one. +- A fixture that passes the moment you add a dispatch entry, with no + harness work, is the highest-risk case, not the easiest win. +- Diagnose by running, not by reading. A deferral reason reached by + reasoning about which driver "looks" suitable is routinely wrong; a + ten-second `await _run_x(_load(path))` settles it. +- Prefer a structural guard over remembering any of this, but check its + reach before trusting it. The `_DRIVER_EXPECTED_KEYS` check in + `tests/conformance/test_observability.py` catches "wired into a driver + that drops a directive" — it caught a live error in the commit that + added it — but only for the drivers registered in that map, which is a + minority of them. It is **fail-open**: a fixture routed to an + unregistered driver returns silently, so a green run there still means + nothing. Registering the driver is what turns the guard on. + +This is written down because the rule existed as guidance and was still +missed repeatedly across one session: four fixtures were reported wired +while asserting nothing, and three deferral reasons were wrong. + ## Tooling - `uv` for everything. Don't use `pip` directly. diff --git a/tests/conformance/harness/capabilities.py b/tests/conformance/harness/capabilities.py index f18c4d4..f98f608 100644 --- a/tests/conformance/harness/capabilities.py +++ b/tests/conformance/harness/capabilities.py @@ -149,3 +149,30 @@ def assert_some_case_ran( "capability_skip_reason", "report_recognized_skip", ] + + +def assert_case_asserts_something(fixture_stem: str, case_name: str, expected: Mapping[str, object]) -> None: + """Fail a case whose ``expected`` block carries nothing but ``invariants``.""" + # Invariant blocks in this corpus are DOCUMENTARY: they restate in prose what + # a concrete directive (span_tree / langfuse_trace / metrics / event_count) + # already pins, and runners deliberately do not implement most of them. That + # convention is only safe while something concrete is present to do the + # pinning. A case whose expected block is invariants-ONLY has no such backing, + # so it asserts exactly nothing while reading like coverage. + # + # Matches no ACTIVATED fixture. That is weaker than "no fixture": an + # invariants-only case does exist in the corpus and escapes only because it + # is skipped upstream, which makes this guard's placement relative to the + # skip chain load-bearing rather than incidental. It exists because the metrics driver reached that state + # by a different route -- reading `metrics` but never `invariants`, on two + # fixtures whose invariants were their only negative assertion -- and both + # passed while checking nothing. + concrete = set(expected) - {"invariants"} + if concrete or not expected.get("invariants"): + return + raise AssertionError( + f"{fixture_stem}::{case_name}: `expected` declares only `invariants` and no concrete " + f"directive, so nothing is asserted unless the runner implements every one of them. " + f"Add a concrete assertion, or implement the invariants and register them in the " + f"driver's known-invariant set." + ) diff --git a/tests/conformance/test_capability_gate_wiring.py b/tests/conformance/test_capability_gate_wiring.py index 588bc88..c42528a 100644 --- a/tests/conformance/test_capability_gate_wiring.py +++ b/tests/conformance/test_capability_gate_wiring.py @@ -320,3 +320,106 @@ def test_every_gated_fixture_belongs_to_a_runner_that_implements_the_gate() -> N f"runner implements the gate. Wire it into their runner " f"(tests/conformance/harness/capabilities.py) before those fixtures are activated." ) + + +# -- invariants-only guard -------------------------------------------------- + + +def test_invariants_only_expected_block_is_rejected() -> None: + from .harness.capabilities import assert_case_asserts_something + + with pytest.raises(AssertionError, match="only `invariants`"): + assert_case_asserts_something( + "999-synthetic", + "case_with_no_concrete_assertion", + {"invariants": {"some_absence_claim": True}}, + ) + + +@pytest.mark.parametrize( + "expected", + [ + {"span_tree": [], "invariants": {"x": True}}, + {"metrics": [], "invariants": {"x": True}}, + {"langfuse_trace": {}, "invariants": {"x": True}}, + {"invariants": {}}, + {}, + ], +) +def test_a_case_with_any_concrete_directive_is_accepted(expected: dict[str, object]) -> None: + from .harness.capabilities import assert_case_asserts_something + + assert_case_asserts_something("999-synthetic", "case", expected) + + +@pytest.mark.parametrize( + ("module_name", "runner_attr", "fixture_stem"), + [ + # Each dispatch path x each fixture SHAPE. A single-case fixture puts + # `expected` at the top level with no `cases` list -- the shape the first + # version of this wiring skipped entirely -- while a multi-case fixture + # exercises the `spec["cases"][0]` branch of the graft below. Naming only + # single-case fixtures left that branch unexecuted, so the two shapes are + # now covered explicitly rather than assumed from a fixture id. + ("tests.conformance.test_observability", "test_observability_fixture", "001-otel-basic-trace"), + ( + "tests.conformance.test_observability", + "test_observability_fixture", + "005-otel-llm-provider-span-nested", + ), + ( + "tests.conformance.test_observability_langfuse", + "test_langfuse_fixture", + "022-langfuse-basic-trace", + ), + ( + "tests.conformance.test_observability_langfuse", + "test_langfuse_fixture", + "023-langfuse-generation-rendering", + ), + ], +) +@pytest.mark.asyncio +async def test_the_guard_is_wired_into_each_runner( + monkeypatch: pytest.MonkeyPatch, module_name: str, runner_attr: str, fixture_stem: str +) -> None: + # Non-vacuity for the WIRING, not the predicate. The rejecting test above + # already proves the predicate; this one failed to prove anything, because it + # only counted fixtures on disk and passed identically with both call sites + # deleted. Graft an invariants-only expected block onto a real fixture and + # push it through each runner's actual entry point. + import importlib + + module = importlib.import_module(module_name) + path = next(p for p in module._fixture_paths() if p.stem == fixture_stem) # noqa: SLF001 + original_load = module._load # noqa: SLF001 + spec_shape = original_load(path) + multi_case = bool(spec_shape.get("cases")) + + def _grafted(p: Any) -> Any: + spec = original_load(p) + target = spec["cases"][0] if spec.get("cases") else spec + target["expected"] = {"invariants": {"synthetic_unbacked_claim": True}} + return spec + + monkeypatch.setattr(module, "_load", _grafted) + # Both runners `pytest.skip` for a fixture they do not currently drive, and + # `Skipped` derives from BaseException, so it sails through `pytest.raises` + # and turns this whole test into a silent skip. That is the exact failure + # this file exists to prevent, one level up: the day a named fixture leaves + # its runner's supported set, a skip must read as a wiring regression. + try: + with pytest.raises(AssertionError, match="only `invariants`"): + await getattr(module, runner_attr)(path) + except pytest.skip.Exception as exc: # pragma: no cover - regression guard + pytest.fail( + f"{runner_attr} skipped {fixture_stem} ({exc}), so this test asserted nothing about " + f"the guard. Repoint it at a fixture that runner still drives." + ) + # Non-vacuity on the parametrization itself: the two shapes must actually + # differ, or both rows exercise the same branch of the graft above. + multi_case_rows = {"005-otel-llm-provider-span-nested", "023-langfuse-generation-rendering"} + assert multi_case == (fixture_stem in multi_case_rows), ( + f"{fixture_stem} no longer has the fixture shape this row was chosen for " + f"(multi_case={multi_case}); the single-case and multi-case branches are no longer both covered." + ) diff --git a/tests/conformance/test_harness_fidelity.py b/tests/conformance/test_harness_fidelity.py new file mode 100644 index 0000000..3393920 --- /dev/null +++ b/tests/conformance/test_harness_fidelity.py @@ -0,0 +1,203 @@ +"""Structural guards on the conformance harness itself.""" + +# Nothing here asserts engine behaviour. These tests assert that the harness's +# own allowlists still describe what its code does, because both have already +# drifted in ways no fixture run could reveal: a green conformance run is +# identical whether an assertion is live or dead. See the AGENTS.md note +# "Activating a conformance fixture is not done when it passes". + +from __future__ import annotations + +import ast +import inspect +from typing import Any + +import pytest + +from . import test_observability as otel_runner +from . import test_observability_langfuse as langfuse_runner + +# Drivers whose `expected` keys are deliberately NOT declared in +# `_DRIVER_EXPECTED_KEYS`, with the reason they are safe to leave unguarded. +# The guard is fail-open, so an unregistered driver is invisible to it; listing +# them here is what turns "invisible" into "decided". A new driver appears in +# neither map and fails the test below, forcing that decision at the point it is +# added rather than leaving it dark indefinitely. +# +# A single-fixture driver cannot suffer the defect the guard exists to catch -- +# a fixture routed to a SHARED driver that ignores one of its directives -- since +# it was written against exactly one fixture's `expected` block. +_UNGUARDED_SINGLE_FIXTURE_DRIVERS = frozenset( + { + "_run_fixture_001", + "_run_fixture_002", + "_run_fixture_003", + "_run_fixture_004", + "_run_fixture_005", + "_run_fixture_006", + "_run_fixture_007", + "_run_fixture_008", + "_run_fixture_009", + "_run_fixture_010", + "_run_fixture_011", + "_run_fixture_028", + "_run_fixture_038", + "_run_fixture_050", + "_run_fixture_051", + "_run_fixture_052", + "_run_fixture_053", + "_run_fixture_054", + "_run_fixture_055", + "_run_fixture_056", + "_run_fixture_058", + "_run_fixture_084", + "_run_fixture_110", + "_run_fixture_132", + "_run_fixture_133", + "_run_structured_output_error_span_fixture", + } +) + +# Shared drivers that ARE exposed to the mis-routing defect and are not yet +# registered. This is a real gap, recorded rather than hidden: registering one +# means deriving its read set from its body (and its callees), which is the step +# that produced a wrong `observers` claim when done by inspection alone. +_UNGUARDED_SHARED_DRIVERS = frozenset( + { + "_run_get_invocation_metadata_fixture", + "_run_llm_cache_fixture", + "_run_tool_fixture", + "_run_typed_event_chain_cases", + } +) + + +def test_every_dispatched_driver_is_registered_or_explicitly_unguarded() -> None: + dispatched = set(otel_runner._driver_for_fixture().values()) # noqa: SLF001 + accounted = ( + set(otel_runner._DRIVER_EXPECTED_KEYS) # noqa: SLF001 + | _UNGUARDED_SINGLE_FIXTURE_DRIVERS + | _UNGUARDED_SHARED_DRIVERS + ) + unaccounted = sorted(dispatched - accounted) + assert not unaccounted, ( + f"drivers {unaccounted} are dispatched to but appear in neither `_DRIVER_EXPECTED_KEYS` nor " + f"the unguarded lists in this file. Register the driver's `expected` keys, or add it to an " + f"unguarded list with the reason -- an unregistered driver silently exempts its fixtures " + f"from the mis-routing guard." + ) + # Non-vacuity on the INPUT: an empty dispatch map satisfies the subset check + # above trivially, which is how a guard over nothing reads as a clean pass. + assert dispatched, "the dispatch map is empty, so the check above compared nothing" + # And the lists must not accumulate names for drivers that no longer exist. + stale = sorted((_UNGUARDED_SINGLE_FIXTURE_DRIVERS | _UNGUARDED_SHARED_DRIVERS) - dispatched) + assert not stale, f"unguarded lists name drivers nothing dispatches to: {stale}" + + +def _invariant_names_read_by(func: object, param: str) -> set[str]: + """String literals `func` looks up on the mapping named `param`.""" + tree = ast.parse(inspect.getsource(func)) # pyright: ignore[reportArgumentType] + names: set[str] = set() + for node in ast.walk(tree): + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "get" + and isinstance(node.func.value, ast.Name) + and node.func.value.id == param + and node.args + and isinstance(node.args[0], ast.Constant) + and isinstance(node.args[0].value, str) + ): + names.add(node.args[0].value) + elif ( + isinstance(node, ast.Subscript) + and isinstance(node.value, ast.Name) + and node.value.id == param + and isinstance(node.slice, ast.Constant) + and isinstance(node.slice.value, str) + ): + names.add(node.slice.value) + return names + + +def _span_dependent_invariants_in(func: object) -> set[str]: + """Invariant names whose `if invariants.get(...)` body reads the span set.""" + tree = ast.parse(inspect.getsource(func)) # pyright: ignore[reportArgumentType] + found: set[str] = set() + for node in ast.walk(tree): + if not isinstance(node, ast.If): + continue + names = { + call.args[0].value + for call in ast.walk(node.test) + if isinstance(call, ast.Call) + and isinstance(call.func, ast.Attribute) + and call.func.attr == "get" + and isinstance(call.func.value, ast.Name) + and call.func.value.id == "invariants" + and call.args + and isinstance(call.args[0], ast.Constant) + and isinstance(call.args[0].value, str) + } + if not names: + continue + body = ast.Module(body=node.body, type_ignores=[]) + if any(isinstance(n, ast.Name) and n.id in {"spans", "_spans_carrying"} for n in ast.walk(body)): + found |= names + return found + + +def test_span_dependent_invariants_lists_every_span_reading_claim() -> None: + # `_assert_span_anchor` only fires for names in `_SPAN_DEPENDENT_INVARIANTS`. + # A span-reading claim missing from that set gets no positive anchor, so an + # empty span set satisfies it trivially -- which is what the `startswith` + # heuristic this set replaced actually did to the two + # `exceeded_span_signal_*` names. + derived = _span_dependent_invariants_in( + otel_runner._assert_metrics_invariants # noqa: SLF001 + ) | _span_dependent_invariants_in(otel_runner._assert_token_budget_invariants) # noqa: SLF001 + assert derived, "parsed no span-reading invariant blocks, so this compared nothing" + missing = sorted(derived - otel_runner._SPAN_DEPENDENT_INVARIANTS) # noqa: SLF001 + assert not missing, ( + f"{missing} read the span set but are absent from `_SPAN_DEPENDENT_INVARIANTS`, so no " + f"positive anchor runs for them and a zero-span run satisfies them vacuously." + ) + stale = sorted(otel_runner._SPAN_DEPENDENT_INVARIANTS - derived) # noqa: SLF001 + assert not stale, f"`_SPAN_DEPENDENT_INVARIANTS` names claims that no longer read spans: {stale}" + + +def _drive_metrics_guard(case: dict[str, Any]) -> None: + otel_runner._assert_metrics_invariants(case, [], []) # noqa: SLF001 + + +def _drive_token_budget_guard(case: dict[str, Any]) -> None: + otel_runner._assert_token_budget_invariants(case, [], {}, []) # noqa: SLF001 + + +@pytest.mark.parametrize("drive", [_drive_metrics_guard, _drive_token_budget_guard]) +@pytest.mark.parametrize("invariant", sorted(otel_runner._SPAN_DEPENDENT_INVARIANTS)) # noqa: SLF001 +def test_a_zero_span_run_cannot_satisfy_a_span_absence_claim(invariant: str, drive: Any) -> None: + # The test above keeps `_SPAN_DEPENDENT_INVARIANTS` honest about which names + # read spans; this one keeps the ANCHOR ITSELF live. Deleting + # `_assert_span_anchor`, or its call from either guard, leaves every shipping + # fixture green -- both fixtures that declare a span-absence claim also + # declare `span_tree`, whose own root-span check fails first, so nothing in + # the corpus would notice. Driving each guard directly is what makes the + # anchor's removal detectable at all. + with pytest.raises(AssertionError, match="pass vacuously"): + drive({"expected": {"invariants": {invariant: True}}}) + + +def test_per_trace_invariants_covers_everything_assert_trace_reads() -> None: + # `_assert_multi_traces` filters the fixture's invariants down to + # `_PER_TRACE_INVARIANTS` before delegating. A name `_assert_trace` checks + # but that is absent from the set is therefore dropped on that path, and the + # fixture declaring it still passes. + read = _invariant_names_read_by(langfuse_runner._assert_trace, "expected_invariants") # noqa: SLF001 + assert read, "parsed no invariant lookups out of _assert_trace, so this compared nothing" + missing = sorted(read - set(langfuse_runner._PER_TRACE_INVARIANTS)) # noqa: SLF001 + assert not missing, ( + f"`_assert_trace` checks {missing}, but `_PER_TRACE_INVARIANTS` omits them, so the " + f"multi-trace runner discards those claims silently." + ) diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py index 2cdd5a3..7a7add4 100644 --- a/tests/conformance/test_observability.py +++ b/tests/conformance/test_observability.py @@ -32,8 +32,10 @@ from __future__ import annotations import copy +import functools import re -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence +from collections.abc import Set as AbstractSet from pathlib import Path from typing import TYPE_CHECKING, Any, cast @@ -194,6 +196,7 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None: "065-llm-completion-event-active-prompt-null", "067-llm-completion-event-call-id-always-present-and-distinct", "068-llm-completion-event-response-model-distinct-from-request", + "144-otel-llm-span-omits-input-usage-on-null-counter", "071-llm-failure-event-call-id-distinct-from-completion-event", "072-llm-failure-event-mutual-exclusion-with-completion-event", # proposal 0082: the LlmFailedEvent response-side surface on a @@ -227,6 +230,7 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None: # Captured via a private MeterProvider + InMemoryMetricReader (the # §6.9 metric-capture primitive). 089 (embeddings) is deferred. "088-llm-metrics-token-and-duration", + "145-llm-metrics-no-input-token-observation-on-null-counter", "090-metrics-error-type-on-duration", "091-metrics-disabled-no-measurements", # proposal 0082: the OTel error span (124) renders the response-side @@ -245,6 +249,10 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None: # NOT on a no-usage provider_unavailable failure. The Langfuse WARNING # (130) runs in the dedicated test_observability_langfuse harness. "126-token-budget-input-exceeded", + "146-token-budget-input-bound-not-evaluated-on-null-counter", + # Declares observers + span_tree as well as metrics; the metrics driver + # reads neither, so its span claims were dropped entirely. + "147-otel-null-counter-reaches-no-span-histogram-or-budget", "127-token-budget-total-exceeded", "128-token-budget-under-budget-no-warning", "129-token-budget-absent-unchanged", @@ -288,12 +296,14 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None: # harvested error message ABSENT with error_type retained, via the # metadata_absent directive that landed in #267. "137-langfuse-embedding-failure-observation", + "150-langfuse-embedding-failure-literal-error-fields", "139-otel-embedding-no-usage-input-tokens-omitted", "140-langfuse-embedding-no-usage-usagedetails-omitted", # proposal 0067 §11 embedding metrics: token.usage (input only) + # operation.duration, operation="embeddings", recorded from the terminal # embedding event. 089 (usage) + 143 (no-usage: duration only). "089-embedding-metrics-token-and-duration", + "154-embedding-metrics-on-failure", "143-embedding-metrics-no-usage-no-token-observation", # v0.16.0 — proposal 0060 rerank observability (0060b). A calls_rerank # node awaits CohereRerankProvider.rerank() inside the node body; the @@ -315,6 +325,7 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None: "107-otel-rerank-span-attributes", "108-langfuse-rerank-observation", "138-langfuse-rerank-failure-observation", + "151-langfuse-rerank-failure-literal-error-fields", "141-otel-rerank-no-usage-attributes-omitted", "142-langfuse-rerank-no-usage-usagedetails-omitted", "109-rerank-metrics-token-and-duration", @@ -380,10 +391,6 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None: # attempt_index-under-node-retry coverage round-out -- mixes a graph-style # shape the cross-capability parser doesn't model (cf. 110) and has no # dedicated runner here yet; defer until the harness wires it. - "119-otel-callable-branch-attempt-index-under-node-retry": ( - "0075 shipped v0.15.0; runner for this callable-branch-attempt-index-under-retry " - "case not yet wired -- harness gap, not unimplemented" - ), # Proposal 0082 (structured-output failure diagnostics, spec v0.77.0). # The event surface (120-122), the OTel error-span rendering (124) and the # §11 token-usage metric (125) run here; the Langfuse failed-Generation @@ -417,55 +424,46 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None: # conformance fixture wiring rides the v0.17.0 fixture-wiring PR. # Proposal 0101 (spec v0.96.0) malformed usage counter -> not reported, # threaded through the span / metric / budget observability surfaces. - "144-otel-llm-span-omits-input-usage-on-null-counter": ( - "Proposal 0101 null-counter observability; harness wiring rides the v0.17.0 fixture-wiring PR" - ), - "145-llm-metrics-no-input-token-observation-on-null-counter": ( - "Proposal 0101 null-counter observability; harness wiring rides the v0.17.0 fixture-wiring PR" - ), - "146-token-budget-input-bound-not-evaluated-on-null-counter": ( - "Proposal 0101 null-counter observability; harness wiring rides the v0.17.0 fixture-wiring PR" - ), - "147-otel-null-counter-reaches-no-span-histogram-or-budget": ( - "Proposal 0101 null-counter observability; harness wiring rides the v0.17.0 fixture-wiring PR" - ), - "148-langfuse-generation-usage-omits-input-on-null-counter": ( - "Proposal 0101 null-counter observability; harness wiring rides the v0.17.0 fixture-wiring PR" - ), - "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span": ( - "Proposal 0101 null-counter observability; harness wiring rides the v0.17.0 fixture-wiring PR" - ), # Proposal 0107 (spec v0.102.0) mock_embedding / mock_rerank raises # sub-directive -> literal error-field assertion. # The 0118 half of these two resolved at this pin (both cases moved to # disable_provider_payload=false, since asserting the message literally is # their purpose). What remains is the 0107 mock-raises harness wiring. - "150-langfuse-embedding-failure-literal-error-fields": ( - "Proposal 0107 mock_embedding raises sub-directive not yet wired; rides the " - "remaining v0.17.0 fixture-wiring PR (observability 144-156)" - ), - "151-langfuse-rerank-failure-literal-error-fields": ( - "Proposal 0107 mock_rerank raises sub-directive not yet wired; rides the " - "remaining v0.17.0 fixture-wiring PR (observability 144-156)" - ), # Spec v0.103.1 conformance coverage (0084 orphan-fallback arms + the # embedding failure-metrics counterpart). + # Diagnosed against their sibling drivers rather than guessed: each reaches a + # driver and fails on a concrete gap, not on "no driver". + "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span": ( + "declares BOTH expected.span_tree and expected.observers.contains_event, and no driver " + "serves both: _run_typed_event_cases reads observers only, _run_llm_payload_case reads " + "span_tree only, and _run_token_budget_case reads both but builds the graph from a mock " + "shape that does not reproduce 149's response model / response id, so its span-tree match " + "fails on attributes rather than on behaviour. Needs a driver threading 149's mock_llm " + "through a typed collector AND a span exporter; the event half is the fixture's stated " + "discriminator, so wiring it under a span-only driver drops the assertion it exists for" + ), + "119-otel-callable-branch-attempt-index-under-node-retry": ( + "the conformance adapter never translates a fixture node's YAML `middleware:` block, so " + "the retry is never installed and the transient failure is terminal: only attempt 0 runs " + "and the branch exception escapes. The fix is YAML middleware parsing in adapter.py plus " + "threading per_node_mw into add_parallel_branches_node, NOT a failure path in the 110 " + "driver -- that driver is reached and would still see an unretried raise" + ), "152-otel-parallel-branch-orphan-llm-fallback": ( - "Spec v0.103.1 orphan-fallback coverage; harness wiring rides the v0.17.0 fixture-wiring PR" + "reuses fixture 133's orphan-fallback driver, which does not build the `subgraphs` block " + "152 adds (KeyError: 'subgraphs')" ), "153-otel-mixed-nesting-orphan-llm-fallback": ( - "Spec v0.103.1 orphan-fallback coverage; harness wiring rides the v0.17.0 fixture-wiring PR" + "same driver gap as 152, one nesting level deeper (KeyError: 'leaf_sg')" ), - "154-embedding-metrics-on-failure": ( - "Spec v0.103.1 embedding failure-metrics coverage; harness wiring rides the v0.17.0 fixture-wiring PR" + "148-langfuse-generation-usage-omits-input-on-null-counter": ( + "the sibling Langfuse runner DOES drive mock_llm generations asserting usage (023, 155, " + "156), so plumbing is not the gap. The gap is its `usage` comparator: it iterates the " + "EXPECTED keys, a subset match, so an OMITTED `input` key -- the whole claim -- cannot " + "be expressed. Wiring it needs an exact-map or usage_absent comparator first, or the " + "fixture passes against an impl emitting usage.input = null" ), # Proposal 0109 (spec v0.104.0) token-budget failure-path parity. - "155-langfuse-token-budget-exceeded-flag-on-failure": ( - "Proposal 0109 token_budget_exceeded flag; harness wiring rides the v0.17.0 fixture-wiring PR" - ), - "156-langfuse-token-budget-under-budget-flag-false": ( - "Proposal 0109 token_budget_exceeded flag; harness wiring rides the v0.17.0 fixture-wiring PR" - ), } @@ -491,6 +489,8 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None: "036-caller-invocation-id-non-uuid", "037-langfuse-trace-input-output", "059-implementation-attribution-langfuse", + "155-langfuse-token-budget-exceeded-flag-on-failure", + "156-langfuse-token-budget-under-budget-flag-false", # 134 -- proposal 0084 Langfuse parent resolution (nested exact-match + # orphan fallback). Driven by a dedicated hand-built runner in the # sibling harness; the generic topology path cannot model the @@ -668,6 +668,83 @@ def _reject_unsupported_capability_gate(fixture_id: str, spec: Mapping[str, Any] ) +# Which `expected` keys each OTel driver actually READS. Derived from the driver +# bodies, not from fixture usage: this exists because 147 and 149 were wired into +# drivers that never look at `span_tree`, so their span claims were dropped while +# both counted as covered. A fixture routed to a driver that ignores one of its +# directives now fails at wiring time instead of passing hollow. +_DRIVER_EXPECTED_KEYS: dict[str, frozenset[str]] = { + "_run_typed_event_cases": frozenset({"observers", "invariants", "node_completed_event_carries_error"}), + "_run_metrics_fixture": frozenset({"metrics", "invariants"}), + "_run_embedding_fixture": frozenset( + {"span_tree", "metrics", "invariants", "observers", "langfuse_trace"} + ), + "_run_rerank_fixture": frozenset({"span_tree", "metrics", "invariants", "observers", "langfuse_trace"}), + "_run_token_budget_fixture": frozenset({"span_tree", "metrics", "invariants", "observers"}), + # `invariants` is documentary here; `observers` is deliberately ABSENT because + # _run_llm_payload_case reads span_tree ONLY. Claiming observers is what let + # 149 be re-routed here with its contains_event half silently dropped. + "_run_llm_payload_fixture": frozenset({"span_tree", "invariants"}), +} + + +@functools.cache +def _driver_for_fixture() -> dict[str, str]: + """Map each fixture id to the driver the dispatch chain sends it to.""" + # Parsed from THIS file's own dispatch chain rather than hand-maintained, so + # the map cannot drift from what actually runs. A hand-written copy is the + # allowlist-derived-from-usage mistake one level up. + import ast # noqa: PLC0415 + import inspect # noqa: PLC0415 + + tree = ast.parse(inspect.getsource(test_observability_fixture)) + out: dict[str, str] = {} + for node in ast.walk(tree): + if not isinstance(node, ast.If): + continue + called = [ + c.func.id + for c in ast.walk(ast.Module(body=node.body, type_ignores=[])) + if isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id.startswith("_run_") + ] + if not called: + continue + ids: list[str] = [] + for cmp_node in ast.walk(node.test): + if isinstance(cmp_node, ast.Constant) and isinstance(cmp_node.value, str): + ids.append(cmp_node.value) + for fid in ids: + out.setdefault(fid, called[0]) + return out + + +def _assert_driver_reads_every_expected_key(fixture_id: str, spec: Mapping[str, Any]) -> None: + """Fail when a fixture declares an `expected` key its driver does not read.""" + # Resolved lazily: the dispatch function it parses is defined below this point. + driver = _driver_for_fixture().get(fixture_id) + if driver is None: + return + handled = _DRIVER_EXPECTED_KEYS.get(driver) + if handled is None: + # FAIL-OPEN, and deliberately narrow. Most drivers are unregistered, so + # this exempts the majority; registering one is what turns the guard on + # for its fixtures. Recorded here rather than left implicit because a + # guard that is dark for two thirds of the corpus must not read as + # blanket protection -- see the note in AGENTS.md. + return + # `or [spec]`, not `or []`: a single-case fixture carries `expected` at the top + # level with no `cases` list, and `or []` made this guard a no-op for all seven. + for case in cast("list[dict[str, Any]]", spec.get("cases") or [spec]): + declared = set(cast("dict[str, Any]", case.get("expected") or {})) + unread = sorted(declared - handled) + assert not unread, ( + f"{fixture_id}::{case.get('name')}: declares expected key(s) {unread} that " + f"{driver} does not read, so those assertions would be silently dropped while the " + f"fixture still counts as covered. Route it to a driver that reads them, or " + f"implement them and register the key in _DRIVER_EXPECTED_KEYS." + ) + + @pytest.mark.parametrize("fixture_path", _fixture_paths(), ids=_fixture_id) async def test_observability_fixture(fixture_path: Path) -> None: fixture_id = fixture_path.stem @@ -688,6 +765,18 @@ async def test_observability_fixture(fixture_path: Path) -> None: spec = _load(fixture_path) _reject_unsupported_capability_gate(fixture_id, spec) + from .harness.capabilities import assert_case_asserts_something # noqa: PLC0415 + + _assert_driver_reads_every_expected_key(fixture_id, spec) + + # `or [spec]` because a single-case fixture puts `expected` at the top + # level with no `cases` list; `or []` skipped seven of them entirely. + for _case in cast("list[dict[str, Any]]", spec.get("cases") or [spec]): + assert_case_asserts_something( + fixture_id, + cast("str", _case.get("name") or ""), + cast("Mapping[str, Any]", _case.get("expected") or {}), + ) if fixture_id == "001-otel-basic-trace": await _run_fixture_001(spec) elif fixture_id == "002-otel-subgraph-hierarchy": @@ -714,11 +803,15 @@ async def test_observability_fixture(fixture_path: Path) -> None: await _run_fixture_028(spec) elif fixture_id == "038-otel-parallel-branches-dispatch-span": await _run_fixture_038(spec) - elif fixture_id == "110-otel-callable-branch-span": + elif fixture_id in { + "110-otel-callable-branch-span", + }: await _run_fixture_110(spec) elif fixture_id == "132-otel-nested-fan-out-span-keying-and-llm-exact-match": await _run_fixture_132(spec) - elif fixture_id == "133-otel-nested-fan-out-orphan-llm-fallback": + elif fixture_id in { + "133-otel-nested-fan-out-orphan-llm-fallback", + }: await _run_fixture_133(spec) elif fixture_id in { "040-llm-cache-attribute-emission", @@ -782,10 +875,15 @@ async def test_observability_fixture(fixture_path: Path) -> None: "085-llm-tool-call-request-attributes", "086-llm-tool-call-request-absent", "087-llm-tool-call-request-survives-payload-gating", + # 0101 null-counter span claims. This driver walks expected.span_tree + # including attributes_absent -- the half 149 silently dropped under the + # typed-event driver, and the driver 144 was wrongly deferred for lacking. + "144-otel-llm-span-omits-input-usage-on-null-counter", }: await _run_llm_payload_fixture(spec) elif fixture_id in { "088-llm-metrics-token-and-duration", + "145-llm-metrics-no-input-token-observation-on-null-counter", "090-metrics-error-type-on-duration", "091-metrics-disabled-no-measurements", "125-metrics-token-usage-on-structured-output-failure", @@ -795,6 +893,10 @@ async def test_observability_fixture(fixture_path: Path) -> None: await _run_structured_output_error_span_fixture(spec) elif fixture_id in { "126-token-budget-input-exceeded", + "146-token-budget-input-bound-not-evaluated-on-null-counter", + # Declares observers + span_tree as well as metrics; the metrics driver + # reads neither, so its span claims were dropped entirely. + "147-otel-null-counter-reaches-no-span-histogram-or-budget", "127-token-budget-total-exceeded", "128-token-budget-under-budget-no-warning", "129-token-budget-absent-unchanged", @@ -826,7 +928,9 @@ async def test_observability_fixture(fixture_path: Path) -> None: "139-otel-embedding-no-usage-input-tokens-omitted", "140-langfuse-embedding-no-usage-usagedetails-omitted", "089-embedding-metrics-token-and-duration", + "154-embedding-metrics-on-failure", "143-embedding-metrics-no-usage-no-token-observation", + "150-langfuse-embedding-failure-literal-error-fields", }: await _run_embedding_fixture(spec) elif fixture_id in { @@ -844,6 +948,7 @@ async def test_observability_fixture(fixture_path: Path) -> None: "141-otel-rerank-no-usage-attributes-omitted", "142-langfuse-rerank-no-usage-usagedetails-omitted", "109-rerank-metrics-token-and-duration", + "151-langfuse-rerank-failure-literal-error-fields", }: await _run_rerank_fixture(spec) elif fixture_id in { @@ -4518,6 +4623,203 @@ async def _run_metrics_case(case: Mapping[str, Any]) -> None: points = _collect_metric_points(reader) expected_metrics = cast("list[dict[str, Any]]", case["expected"].get("metrics") or []) _assert_metric_points(points, expected_metrics) + # `_assert_metrics_invariants` is the ONLY guard on this path, so the + # token-budget family is deliberately not recognized here: a fixture routed + # to this driver that declares one must fail, not be silently dropped. + _assert_invariants_recognized(case, _METRICS_INVARIANTS, "metrics") + _assert_metrics_invariants(case, points, exporter.get_finished_spans()) + + +# Names `_assert_token_budget_invariants` implements. Overlaps the metrics set by +# one name that BOTH guards check, so this is not a disjoint "only" set. +_TOKEN_BUDGET_INVARIANTS = { + "no_token_budget_exceeded_observation_when_under_budget", + "utilization_records_under_budget", + "no_token_budget_instrument_observations_when_no_budget", + "token_budget_null_on_completion_event", + "no_token_budget_instrument_observations_without_usage", + "token_budget_populated_but_not_evaluated_without_usage", + "exception_propagates_alongside_typed_event", + # Proposal 0101 null-counter arms (fixture 146). + "input_bound_not_evaluated_when_prompt_tokens_null", + "exceeded_span_signal_absent_not_false_when_only_bound_unevaluable", + "no_input_token_usage_observation_when_prompt_tokens_null", + "total_bound_evaluated_when_total_tokens_sound", + "no_input_kind_budget_observation_when_prompt_tokens_null", +} + +_METRICS_INVARIANTS = { + # Proposal 0101 null-counter arms (fixtures 145 / 147). + "no_input_token_usage_observation_when_prompt_tokens_null", + "output_token_usage_observation_recorded_when_completion_sound", + "null_prompt_tokens_reaches_no_span_histogram_or_budget_surface", + "span_omits_both_input_usage_attributes", + "no_input_token_usage_observation", + "no_token_budget_instrument_observation_for_unevaluable_input_bound", + "exceeded_span_signal_absent_not_false", + # Pre-existing metrics fixtures (088 / 090 / 091 / 125). These were being + # silently dropped too: the driver read no invariants at all, so adding the + # guard above surfaced them. + "one_duration_observation_per_call", + "two_token_usage_observations_input_and_output", + "errored_call_records_duration_with_error_type", + "no_token_usage_observation_on_failure", + "no_measurements_recorded_when_metrics_off", + "duration_records_error_type_on_failure", + "token_usage_recorded_on_structured_output_failure", + # Documentary: states that the duration VALUE is deliberately not asserted + # (it is wall-clock). Recognized so it cannot read as unhandled, but there is + # nothing to check. + "duration_value_not_asserted", +} + +# Invariant names whose claim is "some attribute is absent from a SPAN". Each is +# satisfied trivially by a run that produced no spans at all, so declaring one +# obliges the positive anchor below. Enumerated rather than matched by prefix: a +# `startswith` heuristic silently skipped the anchor for the two +# `exceeded_span_signal_*` names, which are exactly this shape. +_SPAN_DEPENDENT_INVARIANTS = { + "span_omits_both_input_usage_attributes", + "exceeded_span_signal_absent_not_false", + "exceeded_span_signal_absent_not_false_when_only_bound_unevaluable", + "null_prompt_tokens_reaches_no_span_histogram_or_budget_surface", +} + + +def _assert_invariants_recognized( + case: Mapping[str, Any], + recognized: AbstractSet[str], + label: str, +) -> None: + """Fail loudly on a declared invariant name no guard on this driver's path + implements, so a fixture's new invariant cannot pass vacuously.""" + # `recognized` is supplied by the RUNNER, not by an individual guard: it is + # the union over the guards that runner actually calls. Validating inside a + # guard against a wider union is what let a token-budget name declared on a + # metrics fixture read as recognized while nothing checked it. + invariants = cast("dict[str, Any]", case["expected"].get("invariants") or {}) + unknown = set(invariants) - recognized + assert not unknown, f"unhandled {label} invariant(s): {sorted(unknown)}" + + +def _assert_span_anchor(invariants: Mapping[str, Any], spans: Sequence[Any]) -> None: + """Assert the LLM span exists before any span-absence predicate reads it.""" + # Every span-dependent claim is of the form "X is absent", which an empty + # span set satisfies trivially -- deleting the LLM span outright made 147 + # green. Deliberately NOT gated on `spans` being non-empty: no spans at all + # is the very case this exists to catch. + if not (set(invariants) & _SPAN_DEPENDENT_INVARIANTS): + return + llm_spans = [s for s in spans if s.name == "openarmature.llm.complete"] + assert llm_spans, ( + f"no openarmature.llm.complete span was recorded, so the span-absence invariants " + f"would pass vacuously; got {[s.name for s in spans]}" + ) + + +def _assert_metrics_invariants( + case: Mapping[str, Any], + points: Sequence[tuple[str, float, int, dict[str, Any]]], + spans: Sequence[Any], +) -> None: + """Evaluate the named invariants on a metrics fixture.""" + # These are ABSENCE predicates, which the `metrics:` list shape cannot + # express -- it asserts presence only. 145 and 147 say so in their own + # comments, so for those two the invariants ARE the assertion: without this + # the fixtures pass while checking nothing but the positive rows. Mirrors the + # token-budget driver's guard, whose absence here is what let that happen. + invariants = cast("dict[str, Any]", case["expected"].get("invariants") or {}) + if not invariants: + return + _assert_span_anchor(invariants, spans) + + usage_points = [p for p in points if p[0] == "openarmature.gen_ai.client.token.usage"] + input_usage = [p for p in usage_points if p[3].get("openarmature.gen_ai.token.type") == "input"] + output_usage = [p for p in usage_points if p[3].get("openarmature.gen_ai.token.type") == "output"] + budget_points = [ + p + for p in points + if p[0] + in ( + "openarmature.gen_ai.client.token_budget.exceeded", + "openarmature.gen_ai.client.token_budget.utilization", + ) + ] + input_attrs = ("openarmature.gen_ai.usage.input_tokens", "gen_ai.usage.input_tokens") + + def _spans_carrying(*names: str) -> list[tuple[str, str]]: + return [(s.name, n) for s in spans for n in names if n in dict(s.attributes or {})] + + if invariants.get("no_input_token_usage_observation_when_prompt_tokens_null") or invariants.get( + "no_input_token_usage_observation" + ): + assert not input_usage, ( + f"a null prompt_tokens MUST record no input token.usage observation; got {input_usage}" + ) + if invariants.get("output_token_usage_observation_recorded_when_completion_sound"): + # The non-vacuity partner: without it the absence above is satisfied by + # an implementation that recorded no usage observations at all. + assert output_usage, ( + f"a sound completion_tokens MUST still record an output token.usage " + f"observation; got only {usage_points}" + ) + if invariants.get("span_omits_both_input_usage_attributes"): + carrying = _spans_carrying(*input_attrs) + assert not carrying, ( + f"a null prompt_tokens MUST leave both input-usage span attributes absent; present as {carrying}" + ) + if invariants.get("no_token_budget_instrument_observation_for_unevaluable_input_bound"): + input_budget = [ + p for p in budget_points if p[3].get("openarmature.gen_ai.token_budget.kind") == "input" + ] + assert not input_budget, ( + f"an unevaluable input bound MUST record no token-budget observation; got {input_budget}" + ) + if invariants.get("exceeded_span_signal_absent_not_false"): + attr = "openarmature.llm.token_budget.exceeded" + carrying = _spans_carrying(attr) + assert not carrying, ( + f"with no evaluable bound the {attr} span attribute MUST be absent rather than " + f"false; present on {carrying}" + ) + duration_points = [p for p in points if p[0] == "openarmature.gen_ai.client.operation.duration"] + + if invariants.get("one_duration_observation_per_call"): + # p[2] is the histogram's observation COUNT. Asserting only that one data + # point exists cannot fail the double-record regression this is named for: + # two observations aggregate into a single point with count 2. + assert len(duration_points) == 1 and duration_points[0][2] == 1, ( + f"expected exactly one operation.duration observation per call; got {duration_points}" + ) + if invariants.get("two_token_usage_observations_input_and_output"): + assert ( + len(input_usage) == 1 + and len(output_usage) == 1 + and input_usage[0][2] == 1 + and output_usage[0][2] == 1 + ), f"expected one input and one output token.usage observation; got {usage_points}" + if invariants.get("errored_call_records_duration_with_error_type") or invariants.get( + "duration_records_error_type_on_failure" + ): + assert duration_points, "a failed call MUST still record operation.duration" + assert all(p[3].get("error.type") for p in duration_points), ( + f"a failed call's duration observation MUST carry error.type; got {duration_points}" + ) + if invariants.get("no_token_usage_observation_on_failure"): + assert not usage_points, ( + f"a failure with no usage MUST record no token.usage observation; got {usage_points}" + ) + if invariants.get("token_usage_recorded_on_structured_output_failure"): + # A structured-output failure DOES carry usage, so the counterpart above + # must not be over-applied to every failure. + assert usage_points, "a structured-output failure carries usage, so token.usage MUST still record" + if invariants.get("no_measurements_recorded_when_metrics_off"): + assert not points, f"metrics disabled MUST record no measurements at all; got {points}" + if invariants.get("null_prompt_tokens_reaches_no_span_histogram_or_budget_surface"): + # The umbrella claim: none of the three surfaces sees the null counter. + assert not input_usage, f"histogram surface: {input_usage}" + assert not _spans_carrying(*input_attrs), "span surface carries an input-usage attribute" + assert not budget_points, f"budget surface: {budget_points}" # --------------------------------------------------------------------------- @@ -4622,32 +4924,25 @@ async def _run_token_budget_case(case: Mapping[str, Any]) -> None: expected_metrics = cast("list[dict[str, Any]] | None", expected.get("metrics")) if expected_metrics is not None: _assert_metric_points(points, expected_metrics) - _assert_token_budget_invariants(case, points, collectors) + # Both guards run on this path, so the recognized set is their union: 147 + # routes here and declares metrics-family names exclusively. + _assert_invariants_recognized(case, _METRICS_INVARIANTS | _TOKEN_BUDGET_INVARIANTS, "token-budget") + _assert_token_budget_invariants(case, points, collectors, spans) + _assert_metrics_invariants(case, points, spans) def _assert_token_budget_invariants( case: Mapping[str, Any], points: Sequence[tuple[str, float, int, dict[str, Any]]], collectors: Mapping[str, Any], + spans: Sequence[Any], ) -> None: """Evaluate the token-budget named invariants (128/129/131) as absence predicates over the captured measurement set + the typed events. The ``metrics:`` list shape asserts presence only, so these cover the "no observation recorded" claims the list cannot express.""" invariants = cast("dict[str, Any]", case["expected"].get("invariants") or {}) - # Fail loudly on a declared invariant this runner does not map, so a future - # fixture's new invariant name cannot pass vacuously (harness-fidelity). - _known_invariants = { - "no_token_budget_exceeded_observation_when_under_budget", - "utilization_records_under_budget", - "no_token_budget_instrument_observations_when_no_budget", - "token_budget_null_on_completion_event", - "no_token_budget_instrument_observations_without_usage", - "token_budget_populated_but_not_evaluated_without_usage", - "exception_propagates_alongside_typed_event", - } - unknown = set(invariants) - _known_invariants - assert not unknown, f"unhandled token-budget invariant(s): {sorted(unknown)}" + _assert_span_anchor(invariants, spans) exceeded_points = [p for p in points if p[0] == "openarmature.gen_ai.client.token_budget.exceeded"] utilization_points = [p for p in points if p[0] == "openarmature.gen_ai.client.token_budget.utilization"] @@ -4694,6 +4989,50 @@ def _assert_token_budget_invariants( assert not exceeded_points and not utilization_points, ( "a no-usage failure MUST NOT evaluate the budget (no token-budget metric observation)" ) + # ---- proposal 0101: a null prompt_tokens makes the INPUT bound unevaluable + # while leaving a sound total bound fully evaluated. Absence predicates, + # because the `metrics:` list shape asserts presence only. + _BUDGET_KIND = "openarmature.gen_ai.token_budget.kind" + _TOKEN_TYPE = "openarmature.gen_ai.token.type" + input_budget_points = [ + p for p in exceeded_points + utilization_points if p[3].get(_BUDGET_KIND) == "input" + ] + + if invariants.get("input_bound_not_evaluated_when_prompt_tokens_null") or invariants.get( + "no_input_kind_budget_observation_when_prompt_tokens_null" + ): + assert not input_budget_points, ( + "a null prompt_tokens leaves the INPUT bound unevaluable, so it MUST record no " + f"token-budget observation of kind=input; got {input_budget_points}" + ) + if invariants.get("total_bound_evaluated_when_total_tokens_sound"): + # The independence half. Without it the case above is satisfied by an + # implementation that skipped budget evaluation entirely. + total_points = [p for p in utilization_points if p[3].get(_BUDGET_KIND) == "total"] + assert total_points, ( + "a sound total_tokens MUST still evaluate the total bound; got no kind=total " + f"utilization observation among {utilization_points}" + ) + if invariants.get("no_input_token_usage_observation_when_prompt_tokens_null"): + input_usage = [ + p + for p in points + if p[0] == "openarmature.gen_ai.client.token.usage" and p[3].get(_TOKEN_TYPE) == "input" + ] + assert not input_usage, ( + f"a null prompt_tokens MUST record no input token.usage observation; got {input_usage}" + ) + if invariants.get("exceeded_span_signal_absent_not_false_when_only_bound_unevaluable"): + # ABSENT, not present-and-false. A `False` would assert "evaluated, not + # breached", which is a different and wrong claim when nothing was + # evaluable at all. + attr = "openarmature.llm.token_budget.exceeded" + carrying = [s for s in spans if attr in dict(s.attributes or {})] + assert not carrying, ( + f"with no evaluable bound the {attr} span attribute MUST be absent rather than false; " + f"present on {[(s.name, dict(s.attributes or {}).get(attr)) for s in carrying]}" + ) + if invariants.get("exception_propagates_alongside_typed_event"): # The runner's pytest.raises already asserts the exception propagated; # assert the typed failure event fired alongside it (both, not either). @@ -5195,15 +5534,68 @@ async def _run_tool_case(case: Mapping[str, Any]) -> None: # appears, mirroring the tool path (_run_tool_case). +def _raise_mock_provider_error( + spec_resp: Mapping[str, Any], + classify: Callable[[Any], Any], +) -> None: + """Honour a ``raises`` sub-directive on a mock_embedding / mock_rerank entry.""" + # Proposal 0107 (conformance-adapter §5.15), the retrieval analogue of the + # tool path's `mock_tool: {raises: {...}}`. Fixtures 137 / 138 can only assert + # error_type / error_message by FORMAT, because both are impl-derived: the + # class name comes from whichever exception the provider's own classifier + # picked, and the message from the vendor body. `raises` supplies literal + # values so 150 / 151 can assert them exactly. + # + # The category must still come from `status`. So rather than restate the + # status-to-category table here, which would silently drift from src, run the + # REAL classifier on a probe response of that status and subclass whatever it + # returns. `error_category` on the event reads `exc.category`, which the + # subclass inherits, while `error_type` reads `type(exc).__name__` and + # `error_message` reads `str(exc)`, which the rename and the message supply. + # + # Raised from inside the mock transport, which works because the exception is + # an LlmProviderError rather than an httpx.HTTPError: the provider's + # `except httpx.HTTPError` around the post does not catch it, so it reaches + # the `except LlmProviderError` that builds the failure event. + import json + + import httpx + + raises = spec_resp.get("raises") + if not isinstance(raises, Mapping): + return + raises_map = cast("Mapping[str, Any]", raises) + status = int(spec_resp.get("status", 500)) + probe = httpx.Response( + status, + content=json.dumps(spec_resp.get("body") or {}).encode("utf-8"), + headers={"Content-Type": "application/json"}, + ) + classified = cast("Exception", classify(probe)) + error_type = cast("str", raises_map.get("error_type") or type(classified).__name__) + renamed: type[Exception] = type(error_type, (type(classified),), {}) + raise renamed(cast("str", raises_map.get("message", ""))) + + def _embedding_model_from_first_response(case: Mapping[str, Any]) -> str | None: """Return the ``model`` on the first ``mock_embedding`` response body, so the provider binds to the model the fixture's expected event reports.""" responses = cast("list[dict[str, Any]] | None", case.get("mock_embedding")) or [] - if not responses: - return None - body = cast("dict[str, Any] | None", responses[0].get("body")) or {} - model = body.get("model") - return model if isinstance(model, str) else None + body = cast("dict[str, Any] | None", responses[0].get("body")) if responses else None + model = (body or {}).get("model") + if isinstance(model, str): + return model + # A failure-path fixture (154) has no response body to carry a model, so bind + # to the one its expected metric dimensions report instead. Without this the + # provider binds to the harness default and every dimension match fails on a + # model name the fixture never mentions. + expected = cast("dict[str, Any]", case.get("expected") or {}) + for point in cast("list[dict[str, Any]]", expected.get("metrics") or []): + dims = cast("dict[str, Any]", point.get("dimensions") or {}) + declared = dims.get("gen_ai.request.model") + if isinstance(declared, str): + return declared + return None def _build_embedding_runtime_config(config_spec: Mapping[str, Any] | None) -> Any: @@ -5260,6 +5652,7 @@ def _build_embedding_graph( from openarmature.graph import END, GraphBuilder from openarmature.retrieval import OpenAIEmbeddingProvider + from openarmature.retrieval.providers.openai import _classify_embedding_http_error from .adapter import build_state_cls @@ -5274,6 +5667,7 @@ def _handler(_request: httpx.Request) -> httpx.Response: if not mock_responses: raise AssertionError("mock_embedding queue exhausted") spec_resp = mock_responses.pop(0) + _raise_mock_provider_error(spec_resp, _classify_embedding_http_error) body = cast("dict[str, Any]", spec_resp.get("body") or {}) return httpx.Response( int(spec_resp.get("status", 200)), @@ -5438,6 +5832,9 @@ def _assert_embedding_metrics_invariants( "duration_recorded_when_usage_absent", "embedding_records_input_token_only", "no_output_token_observation_for_embedding", + # 154: the failure-path counterpart of 089 / 143. + "errored_embedding_records_duration_with_error_type", + "no_token_usage_observation_on_failure", } unknown = set(invariants) - _known assert not unknown, f"unhandled embedding-metrics invariant(s): {sorted(unknown)}" @@ -5460,6 +5857,19 @@ def _assert_embedding_metrics_invariants( assert not output_token_points, ( f"embedding MUST record no output token observation; got {output_token_points}" ) + if invariants.get("errored_embedding_records_duration_with_error_type"): + # The §11 baseline survives the failure, carrying the §7 category. Both + # halves matter: a duration that records without error.type would lose + # the only dimension distinguishing a failed call from a slow one. + assert duration_points, "a failed embedding MUST still record operation.duration" + assert all(p[3].get("error.type") for p in duration_points), ( + f"a failed embedding's duration MUST carry error.type; got {duration_points}" + ) + if invariants.get("no_token_usage_observation_on_failure"): + assert not token_points, ( + f"a failed embedding received no response, so it MUST record no token.usage " + f"observation; got {token_points}" + ) # A calls_rerank node constructs a CohereRerankProvider on an httpx.MockTransport @@ -5540,6 +5950,7 @@ def _build_rerank_graph( from openarmature.graph import END, GraphBuilder from openarmature.retrieval import CohereRerankProvider + from openarmature.retrieval.providers.cohere import _classify_cohere_http_error from .adapter import build_state_cls @@ -5554,6 +5965,7 @@ def _handler(_request: httpx.Request) -> httpx.Response: if not mock_responses: raise AssertionError("mock_rerank queue exhausted") spec_resp = mock_responses.pop(0) + _raise_mock_provider_error(spec_resp, _classify_cohere_http_error) body = cast("dict[str, Any]", spec_resp.get("body") or {}) return httpx.Response( int(spec_resp.get("status", 200)), diff --git a/tests/conformance/test_observability_langfuse.py b/tests/conformance/test_observability_langfuse.py index 8c3b248..270c928 100644 --- a/tests/conformance/test_observability_langfuse.py +++ b/tests/conformance/test_observability_langfuse.py @@ -52,6 +52,7 @@ from .adapter import build_graph, build_state_cls from .harness.capabilities import ( + assert_case_asserts_something, assert_some_case_ran, capability_skip_reason, report_recognized_skip, @@ -158,6 +159,8 @@ # to metadata.token_budget.*. Drives the input-exceeded path via # renders_prompt + the prompt's token_budget block. "130-langfuse-token-budget-warning-level", + "155-langfuse-token-budget-exceeded-flag-on-failure", + "156-langfuse-token-budget-under-budget-flag-false", # 134 (proposal 0084): the Langfuse Generation parent resolves by the same # chain-aware §5.5 rule as the OTel span parent -- both the nested # exact-match (case 1, mirrors OTel 132) and the orphan fallback (case 2, @@ -558,6 +561,14 @@ async def fetch( async def test_langfuse_fixture(fixture_path: Path) -> None: spec = _load(fixture_path) fixture_stem = fixture_path.stem + # `or [spec]` because a single-case fixture puts `expected` at the top + # level with no `cases` list; `or []` skipped seven of them entirely. + for _case in cast("list[dict[str, Any]]", spec.get("cases") or [spec]): + assert_case_asserts_something( + fixture_stem, + cast("str", _case.get("name") or ""), + cast("Mapping[str, Any]", _case.get("expected") or {}), + ) if fixture_stem in _ISOLATION_FIXTURES: cases_157 = cast("list[dict[str, Any]]", spec["cases"]) excluded_157: dict[str, str] = {} @@ -2270,7 +2281,15 @@ def _runtime_config_from_spec(config_spec: dict[str, Any] | None) -> RuntimeConf # assertions; the rest (``distinct_trace_ids``, # ``correlation_id_consistent_across_traces``, etc.) stay in # ``_assert_multi_traces`` as cross-Trace checks. -_PER_TRACE_INVARIANTS = frozenset({"trace_id_equals_invocation_id", "correlation_id_consistency"}) +# +# Must list EVERY name ``_assert_trace`` reads: a name it checks but that is +# missing here is silently discarded on the multi-trace path, so a fixture +# declaring it there passes without the claim being evaluated. +# ``test_harness_fidelity.py`` derives the read set from the function body and +# fails on any omission -- ``no_warning_level_under_budget`` was one. +_PER_TRACE_INVARIANTS = frozenset( + {"trace_id_equals_invocation_id", "correlation_id_consistency", "no_warning_level_under_budget"} +) def _assert_multi_traces( @@ -2391,6 +2410,17 @@ def _assert_trace( *, expected_invariants: dict[str, Any], ) -> None: + if expected_invariants.get("no_warning_level_under_budget"): + # Implemented, unlike its neighbours, because it is UNBACKED: 155 pins its + # level concretely (`level: ERROR`) so its invariants are documentary, but + # 156's expected block declares no level at all, leaving this its only + # expression. A blanket guard here would be wrong -- it would demand + # implementation of the documentary majority. + offending = [o for o in trace.observations if str(getattr(o, "level", "") or "").upper() == "WARNING"] + assert not offending, ( + f"an under-budget call MUST NOT render a WARNING-level observation; got " + f"{[(o.name, o.level) for o in offending]}" + ) expected_id = expected.get("id") if expected_id is not None and not _is_placeholder(expected_id): # Fixtures 035/036: a LITERAL trace.id is the DERIVED Langfuse id; the