diff --git a/conformance.toml b/conformance.toml index fc03a38..54d61ef 100644 --- a/conformance.toml +++ b/conformance.toml @@ -34,6 +34,27 @@ implementation = "openarmature-python" spec_pin = "v0.107.0" +# Adapter-level conformance capabilities (conformance-adapter §5.5, proposal +# 0116). A fixture case may carry `requires_capability`, which selects the arm of +# a contract that applies to this adapter; a case whose requirement this table +# contradicts is a recognized skip rather than a failure. +# +# The section cited here arrives with the pin bump: at spec_pin v0.107.0 no +# fixture carries `requires_capability` yet, so this table is declared ahead of +# the fixtures that read it rather than orphaned. +# +# §5.5 lets an adapter leave a capability undeclared and be treated as +# non-capable. This harness is stricter and errors on an undeclared name instead +# — silence reads the same whether the capability is genuinely absent or the +# table has not caught up with a newly-added one, and in the second case every +# case gating on it switches itself off while still reporting green. +[adapter_capabilities] +# Whether the adapter can establish which TracerProvider a Langfuse client is +# bound to. True here: the v4 SDK exposes it via `client._resources`, so this +# implementation takes 0116's raise arm on a detected shared provider rather +# than the portable suppress floor. +langfuse_bound_provider_detection = true + # Status values: # implemented — shipped behavior matches the proposal's contract # partial — partial impl; consult `note` for what's missing diff --git a/tests/conformance/harness/capabilities.py b/tests/conformance/harness/capabilities.py new file mode 100644 index 0000000..f18c4d4 --- /dev/null +++ b/tests/conformance/harness/capabilities.py @@ -0,0 +1,151 @@ +"""Adapter-capability declaration and the per-case `requires_capability` gate.""" + +# Spec basis: conformance-adapter §5.5 (proposal 0116). Some contracts have arms +# only one kind of adapter can take -- the raise arm needs the adapter to be able +# to establish a Langfuse client's bound provider, which is not portably +# guaranteed -- so a fixture case carries `requires_capability` and each adapter +# declares what it can do. A case whose gate this adapter does not match is a +# RECOGNIZED skip (§8.1): the contract's other arm covers it, so not running it +# is correct rather than a coverage hole. +# +# The declaration lives in conformance.toml rather than here so the public +# conformance record and the test gate cannot disagree: it is the same statement, +# read once. + +from __future__ import annotations + +import tomllib +import warnings +from collections.abc import Mapping +from functools import cache +from pathlib import Path +from typing import Any, cast + +_CONFORMANCE_TOML = Path(__file__).resolve().parents[3] / "conformance.toml" + + +class RecognizedSkip(UserWarning): + """A fixture case that did not run because its capability arm is not ours.""" + + +def report_recognized_skip(fixture_stem: str, case_name: str, reason: str) -> None: + """Record a gated-out case where a PASSING run still shows it.""" + # §5.5 calls a gated-out case "a recognized skip, not a silently-omitted + # case". Writing the reason into a local dict does not satisfy that: on a + # passing run the dict is dropped and the output is identical whether the + # fixture asserted every case or a third of them. A warning lands in pytest's + # summary either way, so a gate that quietly swallows the MUST-bearing cases + # of a fixture is visible without having to already suspect it. + warnings.warn( + f"{fixture_stem}::{case_name} did not run -- {reason}", + RecognizedSkip, + stacklevel=2, + ) + + +@cache +def adapter_capabilities() -> Mapping[str, bool]: + """The capabilities this adapter declares, from conformance.toml.""" + # Values are required to be real booleans rather than coerced. bool("false") + # is True, so a quoted TOML value would read as capable in the gate while the + # published record renders false -- the two disagreeing is precisely what + # single-sourcing the declaration is supposed to make impossible. + with _CONFORMANCE_TOML.open("rb") as handle: + manifest = tomllib.load(handle) + declared: dict[str, Any] = manifest.get("adapter_capabilities") or {} + for name, value in declared.items(): + if not isinstance(value, bool): + raise AssertionError( + f"conformance.toml [adapter_capabilities] {name} = {value!r} is " + f"{type(value).__name__}, not a boolean. Write it unquoted (true / false) so " + f"the published record and the gate cannot disagree." + ) + return dict(declared) + + +def capability_skip_reason(requires: object) -> str | None: + """Return why this case is gated out, or ``None`` when it should run.""" + # Pure by design: the caller decides how to record the skip. The conformance + # runners iterate a fixture's cases inside a single parametrized test, so a + # `pytest.skip` here would abandon the SIBLING cases too -- the reason the + # existing per-case deferral hook `continue`s rather than skips. + # + # A case naming a capability the manifest does not declare raises. §5.5 lets + # an adapter leave `langfuse_bound_provider_detection` undeclared and be + # treated as non-capable; we require the value to be written down anyway, + # because silence reads identically whether the adapter genuinely lacks the + # capability or the manifest has not caught up with a newly-added capability + # name -- and in the second case every gating case switches itself off while + # still reporting green. + if requires is None: + return None + if not isinstance(requires, Mapping): + # Checked BEFORE the falsy test, which a malformed value would otherwise + # slip through: `requires_capability: []` is falsy, so it would read as + # ungated and run the case against an arm meant for a different adapter + # class, silently. A non-empty malformed value would instead surface as + # an AttributeError from `.items()`, which says nothing useful about the + # fixture-authoring mistake behind it. + raise AssertionError( + f"case requires_capability must be a mapping of capability name to boolean, got " + f"{type(requires).__name__}: {requires!r}. A non-mapping would otherwise read as " + f"ungated and assert the wrong adapter arm." + ) + if not requires: + return None + declared = adapter_capabilities() + for name, required in cast("Mapping[str, Any]", requires).items(): + if not isinstance(required, bool): + raise AssertionError( + f"case requires_capability {name} = {required!r} is {type(required).__name__}, " + f"not a boolean; a coerced value would select the wrong arm silently." + ) + if name not in declared: + raise AssertionError( + f"case requires capability {name!r}, which conformance.toml " + f"[adapter_capabilities] does not declare. Declare it (true or false) so the " + f"gate is a decision rather than an accident; declared: {sorted(declared)}" + ) + if declared[name] is not required: + return ( + f"requires {name}={required}, this adapter declares " + f"{name}={declared[name]} (recognized skip: the contract's other arm)" + ) + return None + + +def assert_some_case_ran( + fixture_stem: str, + ran: int, + excluded: Mapping[str, str], +) -> None: + """Fail when a fixture executed no cases, whatever excluded them.""" + # A fixture that ran nothing is a vacuous pass wearing a green tick, and both + # ways of excluding a case -- a capability gate and a per-case deferral -- are + # designed to be quiet, which is what makes the empty run invisible. So the + # check spans both channels rather than each policing itself: an all-deferred + # fixture belongs in the fixture-level deferral table, where it skips visibly. + # + # `ran` counts EXECUTIONS, not the difference between a case total and the + # size of `excluded`. Keying off `excluded` would miss a fixture whose cases + # share a name (or are unnamed): two excluded cases collapse to one dict + # entry, the count comes up short, and the empty run passes. `excluded` is + # for the message only. + if ran > 0: + return + detail = "; ".join(f"{name}: {reason}" for name, reason in sorted(excluded.items())) or "no cases at all" + raise AssertionError( + f"{fixture_stem}: executed no cases, so this fixture asserted nothing. Either an " + f"exclusion is wrong (a stale deferral, a bad capability declaration) or the fixture " + f"no longer applies to this adapter and should be de-activated at fixture level, " + f"where it skips visibly. Excluded -- {detail}" + ) + + +__all__ = [ + "RecognizedSkip", + "adapter_capabilities", + "assert_some_case_ran", + "capability_skip_reason", + "report_recognized_skip", +] diff --git a/tests/conformance/test_capability_gate_wiring.py b/tests/conformance/test_capability_gate_wiring.py new file mode 100644 index 0000000..588bc88 --- /dev/null +++ b/tests/conformance/test_capability_gate_wiring.py @@ -0,0 +1,322 @@ +"""The `requires_capability` gate as wired into the fixture runners.""" + +# Spec basis: conformance-adapter §5.5 (proposal 0116). The gate's own logic is +# covered by tests/unit/test_conformance_capability_gate.py; what is covered here +# is that the RUNNERS consult it, which their fixtures cannot yet demonstrate -- +# no fixture at the current spec pin carries `requires_capability` (157 / 158 +# arrive with the pin bump). Rather than let the wiring sit unexercised until +# then, these tests inject a gate into a real activated fixture. + +from __future__ import annotations + +import warnings +from pathlib import Path +from typing import Any, cast + +import pytest +import yaml + +from . import test_observability as otel_runner +from . import test_observability_langfuse as langfuse_runner +from .harness.capabilities import RecognizedSkip + +# A two-case activated Langfuse fixture; both cases pass ungated, so any failure +# below is the gate's doing rather than the fixture's. +_FIXTURE = "023-langfuse-generation-rendering" +_CASES = ("generation_rendering", "generation_rendering_truncated") + + +def _fixture_path() -> Path: + return langfuse_runner.CONFORMANCE_DIR / f"{_FIXTURE}.yaml" + + +def _patch_load( + monkeypatch: pytest.MonkeyPatch, + module: Any, + inject: dict[str, dict[str, Any]], +) -> None: + """Load the real fixture, then graft `requires_capability` onto named cases.""" + original = module._load + + def _loader(path: Path) -> dict[str, Any]: + spec = original(path) + for case in spec.get("cases", []): + gate = inject.get(case.get("name", "")) + if gate is not None: + case["requires_capability"] = gate + return spec + + monkeypatch.setattr(module, "_load", _loader) + + +def _spy_run_case(monkeypatch: pytest.MonkeyPatch) -> list[str]: + """Record the names of the cases the runner actually executes.""" + ran: list[str] = [] + original = langfuse_runner._run_case + + async def _recorder(case: Any, **kwargs: Any) -> None: + ran.append(case.get("name", "")) + await original(case, **kwargs) + + monkeypatch.setattr(langfuse_runner, "_run_case", _recorder) + return ran + + +async def test_mismatched_gate_skips_only_that_case(monkeypatch: pytest.MonkeyPatch) -> None: + # One case gated out, its sibling still runs -- the in-loop `continue`, not a + # pytest.skip that would abandon the sibling silently. Asserted on the cases + # that actually executed, so "it did not raise" cannot stand in for it. + _patch_load( + monkeypatch, + langfuse_runner, + {_CASES[0]: {"langfuse_bound_provider_detection": False}}, + ) + ran = _spy_run_case(monkeypatch) + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + assert ran == [_CASES[1]] + + +async def test_gated_case_is_reported_on_a_passing_run(monkeypatch: pytest.MonkeyPatch) -> None: + # §5.5 calls a gated-out case a recognized skip, not a silently-omitted one. + # The run below PASSES -- the surviving sibling carries it -- so without the + # warning there would be no output anywhere distinguishing this from a run + # that asserted both cases. + _patch_load( + monkeypatch, + langfuse_runner, + {_CASES[0]: {"langfuse_bound_provider_detection": False}}, + ) + with pytest.warns(RecognizedSkip, match=_CASES[0]): + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + + +async def test_no_skip_is_reported_when_every_case_runs(monkeypatch: pytest.MonkeyPatch) -> None: + # Non-vacuity for the test above: the warning tracks actual exclusions rather + # than firing on every fixture, so its presence carries information. + _patch_load( + monkeypatch, + langfuse_runner, + dict.fromkeys(_CASES, {"langfuse_bound_provider_detection": True}), + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + assert [w for w in caught if issubclass(w.category, RecognizedSkip)] == [] + + +async def test_matching_gate_still_runs_the_case(monkeypatch: pytest.MonkeyPatch) -> None: + # Non-vacuity for the test above: with a MATCHING gate both cases run, so the + # skip there came from the mismatch and not from the injection itself. + _patch_load( + monkeypatch, + langfuse_runner, + dict.fromkeys(_CASES, {"langfuse_bound_provider_detection": True}), + ) + ran = _spy_run_case(monkeypatch) + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + assert ran == list(_CASES) + + +async def test_gating_every_case_fails_rather_than_passing_vacuously( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_load( + monkeypatch, + langfuse_runner, + dict.fromkeys(_CASES, {"langfuse_bound_provider_detection": False}), + ) + with pytest.raises(AssertionError, match="asserted nothing"): + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + + +async def test_undeclared_capability_reaches_the_runner_as_an_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_load(monkeypatch, langfuse_runner, {_CASES[0]: {"some_future_capability": True}}) + with pytest.raises(AssertionError, match="does not declare"): + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + + +async def test_deferred_sibling_does_not_mask_the_vacuity_check( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # One case deferred, the other gated out: nothing ran, and the check must say + # so. Counting only gated cases would let a single deferral put any fixture + # permanently out of its reach. + monkeypatch.setattr(langfuse_runner, "_DEFERRED_CASES", frozenset({(_FIXTURE, _CASES[0])})) + _patch_load( + monkeypatch, + langfuse_runner, + {_CASES[1]: {"langfuse_bound_provider_detection": False}}, + ) + with pytest.raises(AssertionError, match="asserted nothing"): + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + + +async def test_duplicate_case_names_cannot_hide_an_empty_run( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # End-to-end guard on how the RUNNER counts. Both cases carry the same name + # and both are gated out, so they collapse into ONE entry in `excluded`. A + # count derived from (total - len(excluded)) reads 1 and passes; only + # counting actual executions reaches 0 and fails. Nothing in the fixture set + # at this pin has colliding names, so without this the runner's arithmetic is + # untested against the case it was rewritten for. + original = langfuse_runner._load + + def _loader(path: Path) -> dict[str, Any]: + spec = original(path) + for case in spec["cases"]: + case["name"] = "same-name" + case["requires_capability"] = {"langfuse_bound_provider_detection": False} + return spec + + monkeypatch.setattr(langfuse_runner, "_load", _loader) + with pytest.raises(AssertionError, match="asserted nothing"): + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + + +async def test_wholly_deferred_fixture_does_not_pass_green( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # No capability gate involved at all -- the deferral hook on its own could + # empty a fixture and report success. A fixture with nothing left to run + # belongs in the fixture-level deferral table, which skips visibly. + monkeypatch.setattr(langfuse_runner, "_DEFERRED_CASES", frozenset({(_FIXTURE, name) for name in _CASES})) + with pytest.raises(AssertionError, match="asserted nothing"): + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + + +async def test_one_deferral_still_lets_the_sibling_run(monkeypatch: pytest.MonkeyPatch) -> None: + # Non-vacuity for the two above: a deferral is still a deferral, so a fixture + # with one live case runs it and passes. + monkeypatch.setattr(langfuse_runner, "_DEFERRED_CASES", frozenset({(_FIXTURE, _CASES[0])})) + ran = _spy_run_case(monkeypatch) + await langfuse_runner.test_langfuse_fixture(_fixture_path()) + assert ran == [_CASES[1]] + + +async def test_hand_built_134_runner_consults_the_gate(monkeypatch: pytest.MonkeyPatch) -> None: + # 134 dispatches to a hand-built runner that returns before the generic case + # loop, so it needs the gate wired separately -- otherwise a gated case there + # would run the arm belonging to a different adapter class. + fixture = langfuse_runner._FIXTURE_134 + ran: list[str] = [] + original = langfuse_runner._run_langfuse_134 + + async def _recorder(case: Any) -> None: + ran.append(case.get("name", "")) + await original(case) + + monkeypatch.setattr(langfuse_runner, "_run_langfuse_134", _recorder) + + path = langfuse_runner.CONFORMANCE_DIR / f"{fixture}.yaml" + all_cases = [c["name"] for c in langfuse_runner._load(path)["cases"]] + _patch_load( + monkeypatch, + langfuse_runner, + {all_cases[0]: {"langfuse_bound_provider_detection": False}}, + ) + await langfuse_runner.test_langfuse_fixture(path) + assert ran == all_cases[1:] + + +def test_otel_runner_rejects_a_gate_it_does_not_implement() -> None: + # The OTel runner implements no gate, so an appearance must fail rather than + # be ignored -- ignoring it would assert an arm meant for another adapter. + # + # Called directly rather than through test_observability_fixture: that entry + # point runs four pytest.skip branches first, and pytest.skip raises Skipped, + # an OutcomeException deriving from BaseException that pytest.raises does not + # catch. Driving it end-to-end would therefore report SKIPPED (green) rather + # than FAILED the day the chosen fixture leaves _SUPPORTED_FIXTURES. + gated = {"cases": [{"name": "c", "requires_capability": {"langfuse_bound_provider_detection": True}}]} + with pytest.raises(AssertionError, match="does not implement"): + otel_runner._reject_unsupported_capability_gate("007-x", gated) + + # The single-case shape (no `cases` key) has to be rejected too. + single = {"name": "c", "requires_capability": {"langfuse_bound_provider_detection": True}} + with pytest.raises(AssertionError, match="does not implement"): + otel_runner._reject_unsupported_capability_gate("007-x", single) + + # Non-vacuity: an ungated spec passes through both shapes. + otel_runner._reject_unsupported_capability_gate("007-x", {"cases": [{"name": "c"}]}) + otel_runner._reject_unsupported_capability_gate("007-x", {"name": "c"}) + + +def _fixture_files(spec_root: Path) -> list[Path]: + """Every conformance fixture under a `/conformance/` tree.""" + return sorted(spec_root.glob("*/conformance/*.yaml")) + + +def _gated_fixtures(spec_root: Path) -> list[str]: + """Fixture paths, relative to spec_root, carrying `requires_capability`.""" + found: list[str] = [] + for fixture in _fixture_files(spec_root): + try: + loaded: Any = yaml.safe_load(fixture.read_text()) + except yaml.YAMLError: # pragma: no cover - malformed fixtures are the parse suite's job + continue + if not isinstance(loaded, dict): + continue + spec = cast("dict[str, Any]", loaded) + raw = spec.get("cases") + cases: list[Any] = cast("list[Any]", raw) if isinstance(raw, list) else [spec] + if any(isinstance(c, dict) and cast("dict[str, Any]", c).get("requires_capability") for c in cases): + found.append(str(fixture.relative_to(spec_root))) + return found + + +def test_gated_fixture_detector_finds_both_case_shapes(tmp_path: Path) -> None: + # Non-vacuity for the sweep below. At the current pin NO fixture carries + # `requires_capability` (157/158 arrive with the bump), so the sweep is + # searching an empty haystack and would pass with a detector that always + # returned nothing. This pins the detector against a synthetic tree instead, + # so the sweep starts working the moment a real gated fixture lands. + (tmp_path / "llm-provider" / "conformance").mkdir(parents=True) + (tmp_path / "observability" / "conformance").mkdir(parents=True) + (tmp_path / "checkpointing" / "conformance").mkdir(parents=True) + + (tmp_path / "llm-provider" / "conformance" / "900-multi.yaml").write_text( + "cases:\n - name: a\n - name: b\n requires_capability: {some_capability: true}\n" + ) + (tmp_path / "checkpointing" / "conformance" / "901-single.yaml").write_text( + "name: solo\nrequires_capability: {some_capability: false}\n" + ) + (tmp_path / "observability" / "conformance" / "902-ungated.yaml").write_text("cases:\n - name: a\n") + + assert _gated_fixtures(tmp_path) == [ + "checkpointing/conformance/901-single.yaml", + "llm-provider/conformance/900-multi.yaml", + ] + + +def test_every_gated_fixture_belongs_to_a_runner_that_implements_the_gate() -> None: + # Repo-wide companion to the per-runner rejection. `requires_capability` is a + # general §5.5 directive and CaseSpec allows extra keys, so a gate landing in + # an llm-provider / retrieval / checkpoint / prompt-management fixture would + # parse cleanly and then be dropped on the floor by a runner that never looks + # for it -- executing an arm meant for a different adapter class, silently. + # Only the Langfuse runner implements the gate today, so any gated fixture + # outside observability/ is a wiring gap this must surface. + # + # CONFORMANCE_DIR is `/spec/observability/conformance`, so the + # directory holding the capability folders is parents[1] (`/spec`), + # NOT parents[2] (the submodule root, which holds no capability folders at + # all and globs nothing). + spec_root = langfuse_runner.CONFORMANCE_DIR.parents[1] + scanned = _fixture_files(spec_root) + # Non-vacuity on the sweep's INPUT, not merely on its detector. The detector + # is pinned separately against a synthetic tree, and that test passing is + # exactly what made a wrong root look verified: the detector worked while the + # haystack was empty, so the sweep reported a clean pass over nothing. + assert langfuse_runner.CONFORMANCE_DIR in {f.parent for f in scanned}, ( + f"the sweep root {spec_root} does not contain the observability conformance directory, " + f"so it scanned {len(scanned)} fixtures and would report no gated fixtures whatever lands." + ) + unhandled = [f for f in _gated_fixtures(spec_root) if not f.startswith("observability/")] + assert not unhandled, ( + f"fixtures {unhandled} carry `requires_capability`, but only the observability Langfuse " + f"runner implements the gate. Wire it into their runner " + f"(tests/conformance/harness/capabilities.py) before those fixtures are activated." + ) diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py index 0d0b873..c8c2ded 100644 --- a/tests/conformance/test_observability.py +++ b/tests/conformance/test_observability.py @@ -641,6 +641,25 @@ def test_observability_fixture_coverage_is_complete() -> None: # --------------------------------------------------------------------------- +def _reject_unsupported_capability_gate(fixture_id: str, spec: Mapping[str, Any]) -> None: + """Fail if a fixture routed here carries an audience gate this runner ignores.""" + # `requires_capability` (conformance-adapter §5.5) selects which adapter class + # a case applies to. This runner implements no gate, so ignoring one would + # silently assert the arm meant for a different adapter class rather than + # skip -- an appearance is an error until this runner grows the same per-case + # handling the Langfuse runner has. The repo-wide companion check is + # test_capability_gate_wiring.py's sweep over every fixture directory, which + # catches a gate landing in a runner that never reaches this function. + cases = cast("list[Mapping[str, Any]]", spec.get("cases") or [spec]) + gated = [cast("str", c.get("name") or "") for c in cases if c.get("requires_capability")] + if gated: + raise AssertionError( + f"{fixture_id}: cases {gated} carry `requires_capability`, which this runner does " + f"not implement. Wire the gate (tests/conformance/harness/capabilities.py) before " + f"activating them -- ignoring it asserts an arm meant for a different adapter class." + ) + + @pytest.mark.parametrize("fixture_path", _fixture_paths(), ids=_fixture_id) async def test_observability_fixture(fixture_path: Path) -> None: fixture_id = fixture_path.stem @@ -660,6 +679,7 @@ async def test_observability_fixture(fixture_path: Path) -> None: pytest.skip(f"{fixture_id}: unaccounted -- see the coverage guard") spec = _load(fixture_path) + _reject_unsupported_capability_gate(fixture_id, spec) if fixture_id == "001-otel-basic-trace": await _run_fixture_001(spec) elif fixture_id == "002-otel-subgraph-hierarchy": diff --git a/tests/conformance/test_observability_langfuse.py b/tests/conformance/test_observability_langfuse.py index e1a6dd1..34947d5 100644 --- a/tests/conformance/test_observability_langfuse.py +++ b/tests/conformance/test_observability_langfuse.py @@ -46,6 +46,11 @@ from openarmature.prompts.context import with_active_prompt from .adapter import build_graph, build_state_cls +from .harness.capabilities import ( + assert_some_case_ran, + capability_skip_reason, + report_recognized_skip, +) CONFORMANCE_DIR = ( Path(__file__).resolve().parents[2] / "openarmature-spec" / "spec" / "observability" / "conformance" @@ -486,12 +491,22 @@ async def test_langfuse_fixture(fixture_path: Path) -> None: # 134's nested double fan-out + `calls_llm_from_wrapper` orphan primitive # are not modeled by the generic `_run_case` topology path; each case is # driven by a dedicated hand-built runner (cf. the 039 hand-built path). - for case in cast("list[dict[str, Any]]", spec["cases"]): + cases_134 = cast("list[dict[str, Any]]", spec["cases"]) + excluded_134: dict[str, str] = {} + ran_134 = 0 + for case in cases_134: case_name = cast("str", case.get("name") or "") + gate = capability_skip_reason(case.get("requires_capability")) + if gate is not None: + excluded_134[case_name] = gate + report_recognized_skip(fixture_stem, case_name, gate) + continue + ran_134 += 1 try: await _run_langfuse_134(case) except AssertionError as e: raise AssertionError(f"case {case_name!r}: {e}") from e + assert_some_case_ran(fixture_stem, ran_134, excluded_134) return if "cases" in spec: # Fold fixture-level ``subgraphs`` / ``inner_subgraphs`` into @@ -501,14 +516,30 @@ async def test_langfuse_fixture(fixture_path: Path) -> None: # resolve ``branches.fraud_check.subgraph: fraud_check``. fixture_subgraphs = cast("dict[str, Any] | None", spec.get("subgraphs")) fixture_inner_subgraphs = cast("dict[str, Any] | None", spec.get("inner_subgraphs")) - for case in cast("list[dict[str, Any]]", spec["cases"]): + cases = cast("list[dict[str, Any]]", spec["cases"]) + # Why a case was left out, for the failure message below. The load-bearing + # count is `ran`, not the size of this dict: cases sharing a name would + # collapse into one entry here and let an empty run look partial. + excluded: dict[str, str] = {} + ran = 0 + for case in cases: case_name = cast("str", case.get("name") or "") if (fixture_stem, case_name) in _DEFERRED_CASES: # Per-case deferral. Skipping inside the loop rather # than emitting a separate pytest.skip lets us keep the # surrounding cases running under the same parametrized # test id. + excluded[case_name] = "per-case deferral (_DEFERRED_CASES)" + continue + gate = capability_skip_reason(case.get("requires_capability")) + if gate is not None: + # Recognized skip: this case's arm belongs to an adapter class we + # are not. Same in-loop `continue` as a deferral, for the same + # reason -- and warned about, so a passing run still shows it. + excluded[case_name] = gate + report_recognized_skip(fixture_stem, case_name, gate) continue + ran += 1 if fixture_subgraphs is not None and "subgraphs" not in case: case["subgraphs"] = fixture_subgraphs if fixture_inner_subgraphs is not None and "inner_subgraphs" not in case: @@ -517,7 +548,13 @@ async def test_langfuse_fixture(fixture_path: Path) -> None: await _run_case(case, fixture_stem=fixture_stem) except AssertionError as e: raise AssertionError(f"case {case_name!r}: {e}") from e + assert_some_case_ran(fixture_stem, ran, excluded) else: + # Single-case fixture: no siblings to abandon, so a gate here is a real + # pytest skip -- visible in the run summary rather than a silent no-op. + gate = capability_skip_reason(spec.get("requires_capability")) + if gate is not None: + pytest.skip(f"{fixture_stem}: {gate}") await _run_case(spec, fixture_stem=fixture_stem) diff --git a/tests/unit/test_conformance_capability_gate.py b/tests/unit/test_conformance_capability_gate.py new file mode 100644 index 0000000..ad48c1e --- /dev/null +++ b/tests/unit/test_conformance_capability_gate.py @@ -0,0 +1,149 @@ +"""The conformance harness's `requires_capability` audience gate.""" + +# Spec basis: conformance-adapter §5.5 (proposal 0116). The gate decides whether a +# fixture case applies to this adapter. Its failure mode is silence -- a gate that +# skips too much reports green while asserting nothing -- so the not-silent +# properties are tested directly rather than inferred from the fixture runs. + +from __future__ import annotations + +import pytest + +from tests.conformance.harness.capabilities import ( + adapter_capabilities, + assert_some_case_ran, + capability_skip_reason, +) + + +def test_declared_detection_capability_matches_the_installed_sdk() -> None: + # The declaration is a public claim in conformance.toml, and asserting it + # against its own literal would only prove the file parses. What makes this + # adapter detection-capable is the probe in the adapter: it reads + # client._resources.tracer_provider, an SDK internal the adapter itself + # treats as possibly-absent. So the claim is checked against the SDK actually + # installed -- an upgrade that drops or renames that attribute silently + # demotes every client to `undetectable`, and this is what goes red. + import inspect + + # Only the SDK's ABSENCE is a legitimate skip: without langfuse installed the + # claim is untestable. A langfuse that no longer exposes the internal is the + # very failure this test exists for, so it raises rather than skipping -- an + # importorskip on the internal path would turn the load-bearing case green. + langfuse = pytest.importorskip( + "langfuse", + reason="the declared capability is a claim about the installed Langfuse SDK", + ) + try: + from langfuse._client import resource_manager + except ImportError as exc: + raise AssertionError( + "langfuse is installed but langfuse._client.resource_manager is gone. " + "conformance.toml declares langfuse_bound_provider_detection = true, and the probe in " + "adapter._classify_isolation reads that module's tracer_provider. Re-point the probe " + "or drop the claim; a skip here would let the stale claim ship." + ) from exc + + assert adapter_capabilities()["langfuse_bound_provider_detection"] is True + + # The two hops of _classify_isolation's probe: client._resources, then + # .tracer_provider on it. A rename or removal of either changes this source, + # which is the point -- the declaration must break loudly rather than let + # every client quietly classify `undetectable`. + client_src = inspect.getsource(langfuse.Langfuse) + manager_src = inspect.getsource(resource_manager.LangfuseResourceManager) + hint = ( + "conformance.toml declares langfuse_bound_provider_detection = true, but the installed " + "langfuse SDK no longer exposes the internal adapter._classify_isolation reads. The raise " + "arm that declaration advertises is unreachable and every client would classify " + "`undetectable`; fix the probe or drop the claim." + ) + assert "self._resources" in client_src, f"Langfuse no longer sets _resources. {hint}" + assert "self.tracer_provider" in manager_src, ( + f"LangfuseResourceManager no longer sets tracer_provider. {hint}" + ) + + +def test_ungated_case_runs() -> None: + assert capability_skip_reason(None) is None + assert capability_skip_reason({}) is None + + +def test_matching_gate_runs() -> None: + assert capability_skip_reason({"langfuse_bound_provider_detection": True}) is None + + +def test_mismatched_gate_is_a_recognized_skip() -> None: + reason = capability_skip_reason({"langfuse_bound_provider_detection": False}) + assert reason is not None + assert "recognized skip" in reason + + +def test_undeclared_capability_is_an_error_not_a_skip() -> None: + # The load-bearing property. §5.5 permits treating an undeclared capability as + # absent, which would make a newly-added capability name switch off every case + # that gates on it -- green, and asserting nothing. Here it raises instead. + with pytest.raises(AssertionError, match="does not declare"): + capability_skip_reason({"some_future_capability": True}) + + +@pytest.mark.parametrize("malformed", [[], ["langfuse_bound_provider_detection"], "true", 1]) +def test_non_mapping_requirement_is_rejected(malformed: object) -> None: + # A fixture comes from YAML, so the gate can be handed any shape. The empty + # list is the dangerous one: it is falsy, so a bare truthiness check would + # read it as ungated and run the case against the wrong adapter arm with + # nothing going red. + with pytest.raises(AssertionError, match="must be a mapping"): + capability_skip_reason(malformed) + + +def test_non_boolean_requirement_is_rejected_not_coerced() -> None: + # bool("false") is True, so coercing would select the capable arm for a + # quoted fixture value and assert the wrong half of the contract in silence. + with pytest.raises(AssertionError, match="not a boolean"): + capability_skip_reason({"langfuse_bound_provider_detection": "false"}) + + +def test_fixture_that_ran_nothing_fails() -> None: + with pytest.raises(AssertionError, match="asserted nothing"): + assert_some_case_ran("158-x", 0, {"a": "gated", "b": "gated"}) + + +def test_wholly_deferred_fixture_fails_too() -> None: + # The exclusion channels are checked together: a fixture every one of whose + # cases is deferred is as empty as one entirely gated out, and reads as green + # either way. It belongs in the fixture-level deferral table instead. + with pytest.raises(AssertionError, match="asserted nothing"): + assert_some_case_ran("158-x", 0, {"a": "per-case deferral", "b": "per-case deferral"}) + + +def test_duplicate_case_names_cannot_hide_an_empty_run() -> None: + # Two excluded cases sharing a name collapse to ONE entry in `excluded`. A + # guard comparing len(excluded) against the case total would come up short + # and pass; counting executions is immune to how the names collide. + with pytest.raises(AssertionError, match="asserted nothing"): + assert_some_case_ran("158-x", 0, {"": "gated"}) + + +def test_zero_case_fixture_fails() -> None: + # A fixture with `cases: []` loops zero times and excludes nothing, so the + # message has nothing to quote -- but it still asserted nothing. + with pytest.raises(AssertionError, match="no cases at all"): + assert_some_case_ran("158-x", 0, {}) + + +def test_failure_message_names_each_exclusion_reason() -> None: + # The message has to distinguish the channels, or a reader cannot tell a stale + # deferral from a wrong capability declaration. + with pytest.raises(AssertionError) as excinfo: + assert_some_case_ran("158-x", 0, {"a": "per-case deferral", "b": "recognized skip: ..."}) + assert "a: per-case deferral" in str(excinfo.value) + assert "b: recognized skip" in str(excinfo.value) + + +def test_one_execution_is_enough_to_pass() -> None: + assert_some_case_ran("158-x", 1, {"a": "gated"}) + + +def test_unexcluded_fixture_passes() -> None: + assert_some_case_ran("157-x", 2, {})