From 40b204dbb529bffaf65e0894492113db33cde5a2 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Wed, 12 Aug 2026 22:44:34 -0700 Subject: [PATCH 1/3] Add the conformance requires_capability audience gate A fixture case may declare which adapter class it applies to; a case selecting the other arm is a recognized skip rather than a failure. Declares langfuse_bound_provider_detection in the conformance manifest, wires the gate into the Langfuse runner's three case paths, and rejects an unimplemented gate in the OTel runner. Both ways of excluding a case are deliberately quiet, so a fixture that ends up running nothing would otherwise pass green. The guard counts executions rather than inferring them from the exclusion map, which also covers cases that share a name, and each recognized skip is reported where a passing run still shows it. --- conformance.toml | 21 ++ tests/conformance/harness/capabilities.py | 137 ++++++++ .../test_capability_gate_wiring.py | 303 ++++++++++++++++++ tests/conformance/test_observability.py | 20 ++ .../test_observability_langfuse.py | 41 ++- .../unit/test_conformance_capability_gate.py | 127 ++++++++ 6 files changed, 647 insertions(+), 2 deletions(-) create mode 100644 tests/conformance/harness/capabilities.py create mode 100644 tests/conformance/test_capability_gate_wiring.py create mode 100644 tests/unit/test_conformance_capability_gate.py diff --git a/conformance.toml b/conformance.toml index fc03a38..54d61ef 100644 --- a/conformance.toml +++ b/conformance.toml @@ -34,6 +34,27 @@ implementation = "openarmature-python" spec_pin = "v0.107.0" +# Adapter-level conformance capabilities (conformance-adapter §5.5, proposal +# 0116). A fixture case may carry `requires_capability`, which selects the arm of +# a contract that applies to this adapter; a case whose requirement this table +# contradicts is a recognized skip rather than a failure. +# +# The section cited here arrives with the pin bump: at spec_pin v0.107.0 no +# fixture carries `requires_capability` yet, so this table is declared ahead of +# the fixtures that read it rather than orphaned. +# +# §5.5 lets an adapter leave a capability undeclared and be treated as +# non-capable. This harness is stricter and errors on an undeclared name instead +# — silence reads the same whether the capability is genuinely absent or the +# table has not caught up with a newly-added one, and in the second case every +# case gating on it switches itself off while still reporting green. +[adapter_capabilities] +# Whether the adapter can establish which TracerProvider a Langfuse client is +# bound to. True here: the v4 SDK exposes it via `client._resources`, so this +# implementation takes 0116's raise arm on a detected shared provider rather +# than the portable suppress floor. +langfuse_bound_provider_detection = true + # Status values: # implemented — shipped behavior matches the proposal's contract # partial — partial impl; consult `note` for what's missing diff --git a/tests/conformance/harness/capabilities.py b/tests/conformance/harness/capabilities.py new file mode 100644 index 0000000..3c4aa4f --- /dev/null +++ b/tests/conformance/harness/capabilities.py @@ -0,0 +1,137 @@ +"""Adapter-capability declaration and the per-case `requires_capability` gate.""" + +# Spec basis: conformance-adapter §5.5 (proposal 0116). Some contracts have arms +# only one kind of adapter can take -- the raise arm needs the adapter to be able +# to establish a Langfuse client's bound provider, which is not portably +# guaranteed -- so a fixture case carries `requires_capability` and each adapter +# declares what it can do. A case whose gate this adapter does not match is a +# RECOGNIZED skip (§8.1): the contract's other arm covers it, so not running it +# is correct rather than a coverage hole. +# +# The declaration lives in conformance.toml rather than here so the public +# conformance record and the test gate cannot disagree: it is the same statement, +# read once. + +from __future__ import annotations + +import tomllib +import warnings +from collections.abc import Mapping +from functools import cache +from pathlib import Path +from typing import Any + +_CONFORMANCE_TOML = Path(__file__).resolve().parents[3] / "conformance.toml" + + +class RecognizedSkip(UserWarning): + """A fixture case that did not run because its capability arm is not ours.""" + + +def report_recognized_skip(fixture_stem: str, case_name: str, reason: str) -> None: + """Record a gated-out case where a PASSING run still shows it.""" + # §5.5 calls a gated-out case "a recognized skip, not a silently-omitted + # case". Writing the reason into a local dict does not satisfy that: on a + # passing run the dict is dropped and the output is identical whether the + # fixture asserted every case or a third of them. A warning lands in pytest's + # summary either way, so a gate that quietly swallows the MUST-bearing cases + # of a fixture is visible without having to already suspect it. + warnings.warn( + f"{fixture_stem}::{case_name} did not run -- {reason}", + RecognizedSkip, + stacklevel=2, + ) + + +@cache +def adapter_capabilities() -> Mapping[str, bool]: + """The capabilities this adapter declares, from conformance.toml.""" + # Values are required to be real booleans rather than coerced. bool("false") + # is True, so a quoted TOML value would read as capable in the gate while the + # published record renders false -- the two disagreeing is precisely what + # single-sourcing the declaration is supposed to make impossible. + with _CONFORMANCE_TOML.open("rb") as handle: + manifest = tomllib.load(handle) + declared: dict[str, Any] = manifest.get("adapter_capabilities") or {} + for name, value in declared.items(): + if not isinstance(value, bool): + raise AssertionError( + f"conformance.toml [adapter_capabilities] {name} = {value!r} is " + f"{type(value).__name__}, not a boolean. Write it unquoted (true / false) so " + f"the published record and the gate cannot disagree." + ) + return dict(declared) + + +def capability_skip_reason(requires: Mapping[str, Any] | None) -> str | None: + """Return why this case is gated out, or ``None`` when it should run.""" + # Pure by design: the caller decides how to record the skip. The conformance + # runners iterate a fixture's cases inside a single parametrized test, so a + # `pytest.skip` here would abandon the SIBLING cases too -- the reason the + # existing per-case deferral hook `continue`s rather than skips. + # + # A case naming a capability the manifest does not declare raises. §5.5 lets + # an adapter leave `langfuse_bound_provider_detection` undeclared and be + # treated as non-capable; we require the value to be written down anyway, + # because silence reads identically whether the adapter genuinely lacks the + # capability or the manifest has not caught up with a newly-added capability + # name -- and in the second case every gating case switches itself off while + # still reporting green. + if not requires: + return None + declared = adapter_capabilities() + for name, required in requires.items(): + if not isinstance(required, bool): + raise AssertionError( + f"case requires_capability {name} = {required!r} is {type(required).__name__}, " + f"not a boolean; a coerced value would select the wrong arm silently." + ) + if name not in declared: + raise AssertionError( + f"case requires capability {name!r}, which conformance.toml " + f"[adapter_capabilities] does not declare. Declare it (true or false) so the " + f"gate is a decision rather than an accident; declared: {sorted(declared)}" + ) + if declared[name] is not required: + return ( + f"requires {name}={required}, this adapter declares " + f"{name}={declared[name]} (recognized skip: the contract's other arm)" + ) + return None + + +def assert_some_case_ran( + fixture_stem: str, + ran: int, + excluded: Mapping[str, str], +) -> None: + """Fail when a fixture executed no cases, whatever excluded them.""" + # A fixture that ran nothing is a vacuous pass wearing a green tick, and both + # ways of excluding a case -- a capability gate and a per-case deferral -- are + # designed to be quiet, which is what makes the empty run invisible. So the + # check spans both channels rather than each policing itself: an all-deferred + # fixture belongs in the fixture-level deferral table, where it skips visibly. + # + # `ran` counts EXECUTIONS, not the difference between a case total and the + # size of `excluded`. Keying off `excluded` would miss a fixture whose cases + # share a name (or are unnamed): two excluded cases collapse to one dict + # entry, the count comes up short, and the empty run passes. `excluded` is + # for the message only. + if ran > 0: + return + detail = "; ".join(f"{name}: {reason}" for name, reason in sorted(excluded.items())) or "no cases at all" + raise AssertionError( + f"{fixture_stem}: executed no cases, so this fixture asserted nothing. Either an " + f"exclusion is wrong (a stale deferral, a bad capability declaration) or the fixture " + f"no longer applies to this adapter and should be de-activated at fixture level, " + f"where it skips visibly. Excluded -- {detail}" + ) + + +__all__ = [ + "RecognizedSkip", + "adapter_capabilities", + "assert_some_case_ran", + "capability_skip_reason", + "report_recognized_skip", +] diff --git a/tests/conformance/test_capability_gate_wiring.py b/tests/conformance/test_capability_gate_wiring.py new file mode 100644 index 0000000..021d213 --- /dev/null +++ b/tests/conformance/test_capability_gate_wiring.py @@ -0,0 +1,303 @@ +"""The `requires_capability` gate as wired into the fixture runners.""" + +# Spec basis: conformance-adapter §5.5 (proposal 0116). The gate's own logic is +# covered by tests/unit/test_conformance_capability_gate.py; what is covered here +# is that the RUNNERS consult it, which their fixtures cannot yet demonstrate -- +# no fixture at the current spec pin carries `requires_capability` (157 / 158 +# arrive with the pin bump). Rather than let the wiring sit unexercised until +# then, these tests inject a gate into a real activated fixture. + +from __future__ import annotations + +import warnings +from pathlib import Path +from typing import Any, cast + +import pytest +import yaml + +from . import test_observability as otel_runner +from . import test_observability_langfuse as langfuse_runner +from .harness.capabilities import RecognizedSkip + +# A two-case activated Langfuse fixture; both cases pass ungated, so any failure +# below is the gate's doing rather than the fixture's. +_FIXTURE = "023-langfuse-generation-rendering" +_CASES = ("generation_rendering", "generation_rendering_truncated") + + +def _fixture_path() -> Path: + return langfuse_runner.CONFORMANCE_DIR / f"{_FIXTURE}.yaml" + + +def _patch_load( + monkeypatch: pytest.MonkeyPatch, + module: Any, + inject: dict[str, dict[str, Any]], +) -> None: + """Load the real fixture, then graft `requires_capability` onto named cases.""" + original = module._load + + def _loader(path: Path) -> dict[str, Any]: + spec = original(path) + for case in spec.get("cases", []): + gate = inject.get(case.get("name", "")) + if gate is not None: + case["requires_capability"] = gate + return spec + + monkeypatch.setattr(module, "_load", _loader) + + +def _spy_run_case(monkeypatch: pytest.MonkeyPatch) -> list[str]: + """Record the names of the cases the runner actually executes.""" + ran: list[str] = [] + original = langfuse_runner._run_case + + async def _recorder(case: Any, **kwargs: Any) -> None: + ran.append(case.get("name", "")) + await original(case, **kwargs) + + monkeypatch.setattr(langfuse_runner, "_run_case", _recorder) + return ran + + +async def test_mismatched_gate_skips_only_that_case(monkeypatch: pytest.MonkeyPatch) -> None: + # One case gated out, its sibling still runs -- the in-loop `continue`, not a + # pytest.skip that would abandon the sibling silently. Asserted on the cases + # that actually executed, so "it did not raise" cannot stand in for it. + _patch_load( + monkeypatch, + langfuse_runner, + {_CASES[0]: {"langfuse_bound_provider_detection": False}}, + ) + ran = _spy_run_case(monkeypatch) + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + assert ran == [_CASES[1]] + + +async def test_gated_case_is_reported_on_a_passing_run(monkeypatch: pytest.MonkeyPatch) -> None: + # §5.5 calls a gated-out case a recognized skip, not a silently-omitted one. + # The run below PASSES -- the surviving sibling carries it -- so without the + # warning there would be no output anywhere distinguishing this from a run + # that asserted both cases. + _patch_load( + monkeypatch, + langfuse_runner, + {_CASES[0]: {"langfuse_bound_provider_detection": False}}, + ) + with pytest.warns(RecognizedSkip, match=_CASES[0]): + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + + +async def test_no_skip_is_reported_when_every_case_runs(monkeypatch: pytest.MonkeyPatch) -> None: + # Non-vacuity for the test above: the warning tracks actual exclusions rather + # than firing on every fixture, so its presence carries information. + _patch_load( + monkeypatch, + langfuse_runner, + dict.fromkeys(_CASES, {"langfuse_bound_provider_detection": True}), + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + assert [w for w in caught if issubclass(w.category, RecognizedSkip)] == [] + + +async def test_matching_gate_still_runs_the_case(monkeypatch: pytest.MonkeyPatch) -> None: + # Non-vacuity for the test above: with a MATCHING gate both cases run, so the + # skip there came from the mismatch and not from the injection itself. + _patch_load( + monkeypatch, + langfuse_runner, + dict.fromkeys(_CASES, {"langfuse_bound_provider_detection": True}), + ) + ran = _spy_run_case(monkeypatch) + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + assert ran == list(_CASES) + + +async def test_gating_every_case_fails_rather_than_passing_vacuously( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_load( + monkeypatch, + langfuse_runner, + dict.fromkeys(_CASES, {"langfuse_bound_provider_detection": False}), + ) + with pytest.raises(AssertionError, match="asserted nothing"): + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + + +async def test_undeclared_capability_reaches_the_runner_as_an_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_load(monkeypatch, langfuse_runner, {_CASES[0]: {"some_future_capability": True}}) + with pytest.raises(AssertionError, match="does not declare"): + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + + +async def test_deferred_sibling_does_not_mask_the_vacuity_check( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # One case deferred, the other gated out: nothing ran, and the check must say + # so. Counting only gated cases would let a single deferral put any fixture + # permanently out of its reach. + monkeypatch.setattr(langfuse_runner, "_DEFERRED_CASES", frozenset({(_FIXTURE, _CASES[0])})) + _patch_load( + monkeypatch, + langfuse_runner, + {_CASES[1]: {"langfuse_bound_provider_detection": False}}, + ) + with pytest.raises(AssertionError, match="asserted nothing"): + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + + +async def test_duplicate_case_names_cannot_hide_an_empty_run( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # End-to-end guard on how the RUNNER counts. Both cases carry the same name + # and both are gated out, so they collapse into ONE entry in `excluded`. A + # count derived from (total - len(excluded)) reads 1 and passes; only + # counting actual executions reaches 0 and fails. Nothing in the fixture set + # at this pin has colliding names, so without this the runner's arithmetic is + # untested against the case it was rewritten for. + original = langfuse_runner._load + + def _loader(path: Path) -> dict[str, Any]: + spec = original(path) + for case in spec["cases"]: + case["name"] = "same-name" + case["requires_capability"] = {"langfuse_bound_provider_detection": False} + return spec + + monkeypatch.setattr(langfuse_runner, "_load", _loader) + with pytest.raises(AssertionError, match="asserted nothing"): + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + + +async def test_wholly_deferred_fixture_does_not_pass_green( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # No capability gate involved at all -- the deferral hook on its own could + # empty a fixture and report success. A fixture with nothing left to run + # belongs in the fixture-level deferral table, which skips visibly. + monkeypatch.setattr(langfuse_runner, "_DEFERRED_CASES", frozenset({(_FIXTURE, name) for name in _CASES})) + with pytest.raises(AssertionError, match="asserted nothing"): + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + + +async def test_one_deferral_still_lets_the_sibling_run(monkeypatch: pytest.MonkeyPatch) -> None: + # Non-vacuity for the two above: a deferral is still a deferral, so a fixture + # with one live case runs it and passes. + monkeypatch.setattr(langfuse_runner, "_DEFERRED_CASES", frozenset({(_FIXTURE, _CASES[0])})) + ran = _spy_run_case(monkeypatch) + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + assert ran == [_CASES[1]] + + +async def test_hand_built_134_runner_consults_the_gate(monkeypatch: pytest.MonkeyPatch) -> None: + # 134 dispatches to a hand-built runner that returns before the generic case + # loop, so it needs the gate wired separately -- otherwise a gated case there + # would run the arm belonging to a different adapter class. + fixture = langfuse_runner._FIXTURE_134 + ran: list[str] = [] + original = langfuse_runner._run_langfuse_134 + + async def _recorder(case: Any) -> None: + ran.append(case.get("name", "")) + await original(case) + + monkeypatch.setattr(langfuse_runner, "_run_langfuse_134", _recorder) + + path = langfuse_runner.CONFORMANCE_DIR / f"{fixture}.yaml" + all_cases = [c["name"] for c in langfuse_runner._load(path)["cases"]] + _patch_load( + monkeypatch, + langfuse_runner, + {all_cases[0]: {"langfuse_bound_provider_detection": False}}, + ) + await langfuse_runner.test_langfuse_fixture(path) + assert ran == all_cases[1:] + + +def test_otel_runner_rejects_a_gate_it_does_not_implement() -> None: + # The OTel runner implements no gate, so an appearance must fail rather than + # be ignored -- ignoring it would assert an arm meant for another adapter. + # + # Called directly rather than through test_observability_fixture: that entry + # point runs four pytest.skip branches first, and pytest.skip raises Skipped, + # an OutcomeException deriving from BaseException that pytest.raises does not + # catch. Driving it end-to-end would therefore report SKIPPED (green) rather + # than FAILED the day the chosen fixture leaves _SUPPORTED_FIXTURES. + gated = {"cases": [{"name": "c", "requires_capability": {"langfuse_bound_provider_detection": True}}]} + with pytest.raises(AssertionError, match="does not implement"): + otel_runner._reject_unsupported_capability_gate("007-x", gated) + + # The single-case shape (no `cases` key) has to be rejected too. + single = {"name": "c", "requires_capability": {"langfuse_bound_provider_detection": True}} + with pytest.raises(AssertionError, match="does not implement"): + otel_runner._reject_unsupported_capability_gate("007-x", single) + + # Non-vacuity: an ungated spec passes through both shapes. + otel_runner._reject_unsupported_capability_gate("007-x", {"cases": [{"name": "c"}]}) + otel_runner._reject_unsupported_capability_gate("007-x", {"name": "c"}) + + +def _gated_fixtures(spec_root: Path) -> list[str]: + """Fixture paths, relative to spec_root, carrying `requires_capability`.""" + found: list[str] = [] + for fixture in sorted(spec_root.glob("*/conformance/*.yaml")): + try: + loaded: Any = yaml.safe_load(fixture.read_text()) + except yaml.YAMLError: # pragma: no cover - malformed fixtures are the parse suite's job + continue + if not isinstance(loaded, dict): + continue + spec = cast("dict[str, Any]", loaded) + raw = spec.get("cases") + cases: list[Any] = cast("list[Any]", raw) if isinstance(raw, list) else [spec] + if any(isinstance(c, dict) and cast("dict[str, Any]", c).get("requires_capability") for c in cases): + found.append(str(fixture.relative_to(spec_root))) + return found + + +def test_gated_fixture_detector_finds_both_case_shapes(tmp_path: Path) -> None: + # Non-vacuity for the sweep below. At the current pin NO fixture carries + # `requires_capability` (157/158 arrive with the bump), so the sweep is + # searching an empty haystack and would pass with a detector that always + # returned nothing. This pins the detector against a synthetic tree instead, + # so the sweep starts working the moment a real gated fixture lands. + (tmp_path / "llm-provider" / "conformance").mkdir(parents=True) + (tmp_path / "observability" / "conformance").mkdir(parents=True) + (tmp_path / "checkpointing" / "conformance").mkdir(parents=True) + + (tmp_path / "llm-provider" / "conformance" / "900-multi.yaml").write_text( + "cases:\n - name: a\n - name: b\n requires_capability: {some_capability: true}\n" + ) + (tmp_path / "checkpointing" / "conformance" / "901-single.yaml").write_text( + "name: solo\nrequires_capability: {some_capability: false}\n" + ) + (tmp_path / "observability" / "conformance" / "902-ungated.yaml").write_text("cases:\n - name: a\n") + + assert _gated_fixtures(tmp_path) == [ + "checkpointing/conformance/901-single.yaml", + "llm-provider/conformance/900-multi.yaml", + ] + + +def test_every_gated_fixture_belongs_to_a_runner_that_implements_the_gate() -> None: + # Repo-wide companion to the per-runner rejection. `requires_capability` is a + # general §5.5 directive and CaseSpec allows extra keys, so a gate landing in + # an llm-provider / retrieval / checkpoint / prompt-management fixture would + # parse cleanly and then be dropped on the floor by a runner that never looks + # for it -- executing an arm meant for a different adapter class, silently. + # Only the Langfuse runner implements the gate today, so any gated fixture + # outside observability/ is a wiring gap this must surface. + spec_root = langfuse_runner.CONFORMANCE_DIR.parents[2] + unhandled = [f for f in _gated_fixtures(spec_root) if not f.startswith("observability/")] + assert not unhandled, ( + f"fixtures {unhandled} carry `requires_capability`, but only the observability Langfuse " + f"runner implements the gate. Wire it into their runner " + f"(tests/conformance/harness/capabilities.py) before those fixtures are activated." + ) diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py index 0d0b873..c8c2ded 100644 --- a/tests/conformance/test_observability.py +++ b/tests/conformance/test_observability.py @@ -641,6 +641,25 @@ def test_observability_fixture_coverage_is_complete() -> None: # --------------------------------------------------------------------------- +def _reject_unsupported_capability_gate(fixture_id: str, spec: Mapping[str, Any]) -> None: + """Fail if a fixture routed here carries an audience gate this runner ignores.""" + # `requires_capability` (conformance-adapter §5.5) selects which adapter class + # a case applies to. This runner implements no gate, so ignoring one would + # silently assert the arm meant for a different adapter class rather than + # skip -- an appearance is an error until this runner grows the same per-case + # handling the Langfuse runner has. The repo-wide companion check is + # test_capability_gate_wiring.py's sweep over every fixture directory, which + # catches a gate landing in a runner that never reaches this function. + cases = cast("list[Mapping[str, Any]]", spec.get("cases") or [spec]) + gated = [cast("str", c.get("name") or "") for c in cases if c.get("requires_capability")] + if gated: + raise AssertionError( + f"{fixture_id}: cases {gated} carry `requires_capability`, which this runner does " + f"not implement. Wire the gate (tests/conformance/harness/capabilities.py) before " + f"activating them -- ignoring it asserts an arm meant for a different adapter class." + ) + + @pytest.mark.parametrize("fixture_path", _fixture_paths(), ids=_fixture_id) async def test_observability_fixture(fixture_path: Path) -> None: fixture_id = fixture_path.stem @@ -660,6 +679,7 @@ async def test_observability_fixture(fixture_path: Path) -> None: pytest.skip(f"{fixture_id}: unaccounted -- see the coverage guard") spec = _load(fixture_path) + _reject_unsupported_capability_gate(fixture_id, spec) if fixture_id == "001-otel-basic-trace": await _run_fixture_001(spec) elif fixture_id == "002-otel-subgraph-hierarchy": diff --git a/tests/conformance/test_observability_langfuse.py b/tests/conformance/test_observability_langfuse.py index e1a6dd1..f5b5593 100644 --- a/tests/conformance/test_observability_langfuse.py +++ b/tests/conformance/test_observability_langfuse.py @@ -46,6 +46,11 @@ from openarmature.prompts.context import with_active_prompt from .adapter import build_graph, build_state_cls +from .harness.capabilities import ( + assert_some_case_ran, + capability_skip_reason, + report_recognized_skip, +) CONFORMANCE_DIR = ( Path(__file__).resolve().parents[2] / "openarmature-spec" / "spec" / "observability" / "conformance" @@ -486,12 +491,22 @@ async def test_langfuse_fixture(fixture_path: Path) -> None: # 134's nested double fan-out + `calls_llm_from_wrapper` orphan primitive # are not modeled by the generic `_run_case` topology path; each case is # driven by a dedicated hand-built runner (cf. the 039 hand-built path). - for case in cast("list[dict[str, Any]]", spec["cases"]): + cases_134 = cast("list[dict[str, Any]]", spec["cases"]) + excluded_134: dict[str, str] = {} + ran_134 = 0 + for case in cases_134: case_name = cast("str", case.get("name") or "") + gate = capability_skip_reason(cast("Mapping[str, Any] | None", case.get("requires_capability"))) + if gate is not None: + excluded_134[case_name] = gate + report_recognized_skip(fixture_stem, case_name, gate) + continue + ran_134 += 1 try: await _run_langfuse_134(case) except AssertionError as e: raise AssertionError(f"case {case_name!r}: {e}") from e + assert_some_case_ran(fixture_stem, ran_134, excluded_134) return if "cases" in spec: # Fold fixture-level ``subgraphs`` / ``inner_subgraphs`` into @@ -501,14 +516,30 @@ async def test_langfuse_fixture(fixture_path: Path) -> None: # resolve ``branches.fraud_check.subgraph: fraud_check``. fixture_subgraphs = cast("dict[str, Any] | None", spec.get("subgraphs")) fixture_inner_subgraphs = cast("dict[str, Any] | None", spec.get("inner_subgraphs")) - for case in cast("list[dict[str, Any]]", spec["cases"]): + cases = cast("list[dict[str, Any]]", spec["cases"]) + # Why a case was left out, for the failure message below. The load-bearing + # count is `ran`, not the size of this dict: cases sharing a name would + # collapse into one entry here and let an empty run look partial. + excluded: dict[str, str] = {} + ran = 0 + for case in cases: case_name = cast("str", case.get("name") or "") if (fixture_stem, case_name) in _DEFERRED_CASES: # Per-case deferral. Skipping inside the loop rather # than emitting a separate pytest.skip lets us keep the # surrounding cases running under the same parametrized # test id. + excluded[case_name] = "per-case deferral (_DEFERRED_CASES)" + continue + gate = capability_skip_reason(cast("Mapping[str, Any] | None", case.get("requires_capability"))) + if gate is not None: + # Recognized skip: this case's arm belongs to an adapter class we + # are not. Same in-loop `continue` as a deferral, for the same + # reason -- and warned about, so a passing run still shows it. + excluded[case_name] = gate + report_recognized_skip(fixture_stem, case_name, gate) continue + ran += 1 if fixture_subgraphs is not None and "subgraphs" not in case: case["subgraphs"] = fixture_subgraphs if fixture_inner_subgraphs is not None and "inner_subgraphs" not in case: @@ -517,7 +548,13 @@ async def test_langfuse_fixture(fixture_path: Path) -> None: await _run_case(case, fixture_stem=fixture_stem) except AssertionError as e: raise AssertionError(f"case {case_name!r}: {e}") from e + assert_some_case_ran(fixture_stem, ran, excluded) else: + # Single-case fixture: no siblings to abandon, so a gate here is a real + # pytest skip -- visible in the run summary rather than a silent no-op. + gate = capability_skip_reason(cast("Mapping[str, Any] | None", spec.get("requires_capability"))) + if gate is not None: + pytest.skip(f"{fixture_stem}: {gate}") await _run_case(spec, fixture_stem=fixture_stem) diff --git a/tests/unit/test_conformance_capability_gate.py b/tests/unit/test_conformance_capability_gate.py new file mode 100644 index 0000000..e9ec912 --- /dev/null +++ b/tests/unit/test_conformance_capability_gate.py @@ -0,0 +1,127 @@ +"""The conformance harness's `requires_capability` audience gate.""" + +# Spec basis: conformance-adapter §5.5 (proposal 0116). The gate decides whether a +# fixture case applies to this adapter. Its failure mode is silence -- a gate that +# skips too much reports green while asserting nothing -- so the not-silent +# properties are tested directly rather than inferred from the fixture runs. + +from __future__ import annotations + +import pytest + +from tests.conformance.harness.capabilities import ( + adapter_capabilities, + assert_some_case_ran, + capability_skip_reason, +) + + +def test_declared_detection_capability_matches_the_installed_sdk() -> None: + # The declaration is a public claim in conformance.toml, and asserting it + # against its own literal would only prove the file parses. What makes this + # adapter detection-capable is the probe in the adapter: it reads + # client._resources.tracer_provider, an SDK internal the adapter itself + # treats as possibly-absent. So the claim is checked against the SDK actually + # installed -- an upgrade that drops or renames that attribute silently + # demotes every client to `undetectable`, and this is what goes red. + import inspect + + langfuse = pytest.importorskip( + "langfuse", + reason="the declared capability is a claim about the installed Langfuse SDK", + ) + resource_manager = pytest.importorskip("langfuse._client.resource_manager") + + assert adapter_capabilities()["langfuse_bound_provider_detection"] is True + + # The two hops of _classify_isolation's probe: client._resources, then + # .tracer_provider on it. A rename or removal of either changes this source, + # which is the point -- the declaration must break loudly rather than let + # every client quietly classify `undetectable`. + client_src = inspect.getsource(langfuse.Langfuse) + manager_src = inspect.getsource(resource_manager.LangfuseResourceManager) + hint = ( + "conformance.toml declares langfuse_bound_provider_detection = true, but the installed " + "langfuse SDK no longer exposes the internal adapter._classify_isolation reads. The raise " + "arm that declaration advertises is unreachable and every client would classify " + "`undetectable`; fix the probe or drop the claim." + ) + assert "self._resources" in client_src, f"Langfuse no longer sets _resources. {hint}" + assert "self.tracer_provider" in manager_src, ( + f"LangfuseResourceManager no longer sets tracer_provider. {hint}" + ) + + +def test_ungated_case_runs() -> None: + assert capability_skip_reason(None) is None + assert capability_skip_reason({}) is None + + +def test_matching_gate_runs() -> None: + assert capability_skip_reason({"langfuse_bound_provider_detection": True}) is None + + +def test_mismatched_gate_is_a_recognized_skip() -> None: + reason = capability_skip_reason({"langfuse_bound_provider_detection": False}) + assert reason is not None + assert "recognized skip" in reason + + +def test_undeclared_capability_is_an_error_not_a_skip() -> None: + # The load-bearing property. §5.5 permits treating an undeclared capability as + # absent, which would make a newly-added capability name switch off every case + # that gates on it -- green, and asserting nothing. Here it raises instead. + with pytest.raises(AssertionError, match="does not declare"): + capability_skip_reason({"some_future_capability": True}) + + +def test_non_boolean_requirement_is_rejected_not_coerced() -> None: + # bool("false") is True, so coercing would select the capable arm for a + # quoted fixture value and assert the wrong half of the contract in silence. + with pytest.raises(AssertionError, match="not a boolean"): + capability_skip_reason({"langfuse_bound_provider_detection": "false"}) + + +def test_fixture_that_ran_nothing_fails() -> None: + with pytest.raises(AssertionError, match="asserted nothing"): + assert_some_case_ran("158-x", 0, {"a": "gated", "b": "gated"}) + + +def test_wholly_deferred_fixture_fails_too() -> None: + # The exclusion channels are checked together: a fixture every one of whose + # cases is deferred is as empty as one entirely gated out, and reads as green + # either way. It belongs in the fixture-level deferral table instead. + with pytest.raises(AssertionError, match="asserted nothing"): + assert_some_case_ran("158-x", 0, {"a": "per-case deferral", "b": "per-case deferral"}) + + +def test_duplicate_case_names_cannot_hide_an_empty_run() -> None: + # Two excluded cases sharing a name collapse to ONE entry in `excluded`. A + # guard comparing len(excluded) against the case total would come up short + # and pass; counting executions is immune to how the names collide. + with pytest.raises(AssertionError, match="asserted nothing"): + assert_some_case_ran("158-x", 0, {"": "gated"}) + + +def test_zero_case_fixture_fails() -> None: + # A fixture with `cases: []` loops zero times and excludes nothing, so the + # message has nothing to quote -- but it still asserted nothing. + with pytest.raises(AssertionError, match="no cases at all"): + assert_some_case_ran("158-x", 0, {}) + + +def test_failure_message_names_each_exclusion_reason() -> None: + # The message has to distinguish the channels, or a reader cannot tell a stale + # deferral from a wrong capability declaration. + with pytest.raises(AssertionError) as excinfo: + assert_some_case_ran("158-x", 0, {"a": "per-case deferral", "b": "recognized skip: ..."}) + assert "a: per-case deferral" in str(excinfo.value) + assert "b: recognized skip" in str(excinfo.value) + + +def test_one_execution_is_enough_to_pass() -> None: + assert_some_case_ran("158-x", 1, {"a": "gated"}) + + +def test_unexcluded_fixture_passes() -> None: + assert_some_case_ran("157-x", 2, {}) From 55fdeeb15007872c586313d1d256530ae95ca4c8 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Wed, 12 Aug 2026 23:31:47 -0700 Subject: [PATCH 2/3] Reject a malformed capability gate and stop skipping on a renamed SDK internal Two review findings, both the failure this PR exists to prevent. A requires_capability that is not a mapping now raises. The empty list was the dangerous spelling: it is falsy, so it read as ungated and ran the case against an arm meant for a different adapter class with nothing going red. The type is checked before the falsy test, and the parameter is typed as object because the value comes from YAML. The manifest test no longer skips when the Langfuse SDK is installed but the internal the adapter probes has been renamed or removed. That is precisely the case the test exists to catch, so it raises; only an absent SDK is still a legitimate skip. --- tests/conformance/harness/capabilities.py | 20 +++++++++++++--- .../test_observability_langfuse.py | 6 ++--- .../unit/test_conformance_capability_gate.py | 24 ++++++++++++++++++- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/tests/conformance/harness/capabilities.py b/tests/conformance/harness/capabilities.py index 3c4aa4f..f18c4d4 100644 --- a/tests/conformance/harness/capabilities.py +++ b/tests/conformance/harness/capabilities.py @@ -19,7 +19,7 @@ from collections.abc import Mapping from functools import cache from pathlib import Path -from typing import Any +from typing import Any, cast _CONFORMANCE_TOML = Path(__file__).resolve().parents[3] / "conformance.toml" @@ -63,7 +63,7 @@ def adapter_capabilities() -> Mapping[str, bool]: return dict(declared) -def capability_skip_reason(requires: Mapping[str, Any] | None) -> str | None: +def capability_skip_reason(requires: object) -> str | None: """Return why this case is gated out, or ``None`` when it should run.""" # Pure by design: the caller decides how to record the skip. The conformance # runners iterate a fixture's cases inside a single parametrized test, so a @@ -77,10 +77,24 @@ def capability_skip_reason(requires: Mapping[str, Any] | None) -> str | None: # capability or the manifest has not caught up with a newly-added capability # name -- and in the second case every gating case switches itself off while # still reporting green. + if requires is None: + return None + if not isinstance(requires, Mapping): + # Checked BEFORE the falsy test, which a malformed value would otherwise + # slip through: `requires_capability: []` is falsy, so it would read as + # ungated and run the case against an arm meant for a different adapter + # class, silently. A non-empty malformed value would instead surface as + # an AttributeError from `.items()`, which says nothing useful about the + # fixture-authoring mistake behind it. + raise AssertionError( + f"case requires_capability must be a mapping of capability name to boolean, got " + f"{type(requires).__name__}: {requires!r}. A non-mapping would otherwise read as " + f"ungated and assert the wrong adapter arm." + ) if not requires: return None declared = adapter_capabilities() - for name, required in requires.items(): + for name, required in cast("Mapping[str, Any]", requires).items(): if not isinstance(required, bool): raise AssertionError( f"case requires_capability {name} = {required!r} is {type(required).__name__}, " diff --git a/tests/conformance/test_observability_langfuse.py b/tests/conformance/test_observability_langfuse.py index f5b5593..34947d5 100644 --- a/tests/conformance/test_observability_langfuse.py +++ b/tests/conformance/test_observability_langfuse.py @@ -496,7 +496,7 @@ async def test_langfuse_fixture(fixture_path: Path) -> None: ran_134 = 0 for case in cases_134: case_name = cast("str", case.get("name") or "") - gate = capability_skip_reason(cast("Mapping[str, Any] | None", case.get("requires_capability"))) + gate = capability_skip_reason(case.get("requires_capability")) if gate is not None: excluded_134[case_name] = gate report_recognized_skip(fixture_stem, case_name, gate) @@ -531,7 +531,7 @@ async def test_langfuse_fixture(fixture_path: Path) -> None: # test id. excluded[case_name] = "per-case deferral (_DEFERRED_CASES)" continue - gate = capability_skip_reason(cast("Mapping[str, Any] | None", case.get("requires_capability"))) + gate = capability_skip_reason(case.get("requires_capability")) if gate is not None: # Recognized skip: this case's arm belongs to an adapter class we # are not. Same in-loop `continue` as a deferral, for the same @@ -552,7 +552,7 @@ async def test_langfuse_fixture(fixture_path: Path) -> None: else: # Single-case fixture: no siblings to abandon, so a gate here is a real # pytest skip -- visible in the run summary rather than a silent no-op. - gate = capability_skip_reason(cast("Mapping[str, Any] | None", spec.get("requires_capability"))) + gate = capability_skip_reason(spec.get("requires_capability")) if gate is not None: pytest.skip(f"{fixture_stem}: {gate}") await _run_case(spec, fixture_stem=fixture_stem) diff --git a/tests/unit/test_conformance_capability_gate.py b/tests/unit/test_conformance_capability_gate.py index e9ec912..ad48c1e 100644 --- a/tests/unit/test_conformance_capability_gate.py +++ b/tests/unit/test_conformance_capability_gate.py @@ -26,11 +26,23 @@ def test_declared_detection_capability_matches_the_installed_sdk() -> None: # demotes every client to `undetectable`, and this is what goes red. import inspect + # Only the SDK's ABSENCE is a legitimate skip: without langfuse installed the + # claim is untestable. A langfuse that no longer exposes the internal is the + # very failure this test exists for, so it raises rather than skipping -- an + # importorskip on the internal path would turn the load-bearing case green. langfuse = pytest.importorskip( "langfuse", reason="the declared capability is a claim about the installed Langfuse SDK", ) - resource_manager = pytest.importorskip("langfuse._client.resource_manager") + try: + from langfuse._client import resource_manager + except ImportError as exc: + raise AssertionError( + "langfuse is installed but langfuse._client.resource_manager is gone. " + "conformance.toml declares langfuse_bound_provider_detection = true, and the probe in " + "adapter._classify_isolation reads that module's tracer_provider. Re-point the probe " + "or drop the claim; a skip here would let the stale claim ship." + ) from exc assert adapter_capabilities()["langfuse_bound_provider_detection"] is True @@ -75,6 +87,16 @@ def test_undeclared_capability_is_an_error_not_a_skip() -> None: capability_skip_reason({"some_future_capability": True}) +@pytest.mark.parametrize("malformed", [[], ["langfuse_bound_provider_detection"], "true", 1]) +def test_non_mapping_requirement_is_rejected(malformed: object) -> None: + # A fixture comes from YAML, so the gate can be handed any shape. The empty + # list is the dangerous one: it is falsy, so a bare truthiness check would + # read it as ungated and run the case against the wrong adapter arm with + # nothing going red. + with pytest.raises(AssertionError, match="must be a mapping"): + capability_skip_reason(malformed) + + def test_non_boolean_requirement_is_rejected_not_coerced() -> None: # bool("false") is True, so coercing would select the capable arm for a # quoted fixture value and assert the wrong half of the contract in silence. From e0c29fd7dbabfb27ea10a15c2d81126ecac312dd Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Wed, 12 Aug 2026 23:46:38 -0700 Subject: [PATCH 3/3] Point the gated-fixture sweep at the capability folders The sweep globbed from the submodule root, which holds no capability folders, so it matched nothing and would have reported no gated fixtures outside observability however many landed. A companion test already pinned the detector against a synthetic tree, and its passing is what made the wrong root look verified: the detector worked while the haystack was empty. So the sweep now also asserts its scan reached the corpus, by requiring the observability conformance directory to be among the scanned parents. --- .../test_capability_gate_wiring.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/conformance/test_capability_gate_wiring.py b/tests/conformance/test_capability_gate_wiring.py index 021d213..588bc88 100644 --- a/tests/conformance/test_capability_gate_wiring.py +++ b/tests/conformance/test_capability_gate_wiring.py @@ -244,10 +244,15 @@ def test_otel_runner_rejects_a_gate_it_does_not_implement() -> None: otel_runner._reject_unsupported_capability_gate("007-x", {"name": "c"}) +def _fixture_files(spec_root: Path) -> list[Path]: + """Every conformance fixture under a `/conformance/` tree.""" + return sorted(spec_root.glob("*/conformance/*.yaml")) + + def _gated_fixtures(spec_root: Path) -> list[str]: """Fixture paths, relative to spec_root, carrying `requires_capability`.""" found: list[str] = [] - for fixture in sorted(spec_root.glob("*/conformance/*.yaml")): + for fixture in _fixture_files(spec_root): try: loaded: Any = yaml.safe_load(fixture.read_text()) except yaml.YAMLError: # pragma: no cover - malformed fixtures are the parse suite's job @@ -294,7 +299,21 @@ def test_every_gated_fixture_belongs_to_a_runner_that_implements_the_gate() -> N # for it -- executing an arm meant for a different adapter class, silently. # Only the Langfuse runner implements the gate today, so any gated fixture # outside observability/ is a wiring gap this must surface. - spec_root = langfuse_runner.CONFORMANCE_DIR.parents[2] + # + # CONFORMANCE_DIR is `/spec/observability/conformance`, so the + # directory holding the capability folders is parents[1] (`/spec`), + # NOT parents[2] (the submodule root, which holds no capability folders at + # all and globs nothing). + spec_root = langfuse_runner.CONFORMANCE_DIR.parents[1] + scanned = _fixture_files(spec_root) + # Non-vacuity on the sweep's INPUT, not merely on its detector. The detector + # is pinned separately against a synthetic tree, and that test passing is + # exactly what made a wrong root look verified: the detector worked while the + # haystack was empty, so the sweep reported a clean pass over nothing. + assert langfuse_runner.CONFORMANCE_DIR in {f.parent for f in scanned}, ( + f"the sweep root {spec_root} does not contain the observability conformance directory, " + f"so it scanned {len(scanned)} fixtures and would report no gated fixtures whatever lands." + ) unhandled = [f for f in _gated_fixtures(spec_root) if not f.startswith("observability/")] assert not unhandled, ( f"fixtures {unhandled} carry `requires_capability`, but only the observability Langfuse "