From c6b1f44f804fe9570c25ac387a2947dcfcad8bd3 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Tue, 21 Jul 2026 20:48:46 -0700 Subject: [PATCH 1/2] Implement 0080 PromptGroup arity enforcement PromptGroup already rejected fewer than two members, but with a bare ValueError that pydantic folds into a ValidationError carrying no error category, so the conformance harness had nothing to assert against. Add a categorized PromptGroupInvalid (new prompt_group_invalid category) and raise it from the arity validator. Because PromptGroupInvalid is a PromptError and not a ValueError, pydantic propagates it unwrapped from the model validator, so the caller sees .category. It is non-transient and exported from openarmature.prompts. Wire the rejection fixture 035: make PromptManagementFixture.backends optional (035 is cases-only with per-case backends, one case needing none) and rework the cases-runner to build per-case backends when a case declares its own backends key (else share the top-level, so 016 is unchanged) and to run a manager-less case's direct-target calls instead of skipping them (the empty-group construct needs no manager). Update the two arity unit tests, the concepts/prompts doc, and the 0.17.0 changelog entry. --- CHANGELOG.md | 1 + conformance.toml | 8 ++-- docs/concepts/prompts.md | 9 +++-- src/openarmature/prompts/__init__.py | 4 ++ src/openarmature/prompts/errors.py | 15 ++++++++ src/openarmature/prompts/group.py | 7 +++- .../conformance/harness/prompt_management.py | 5 ++- tests/conformance/test_prompt_management.py | 37 ++++++++++++------- tests/unit/test_prompts.py | 12 +++++- 9 files changed, 74 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c35912dd..3a2ada0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). The ### Added - **PromptManager service-wide default cache TTL** (proposal 0086, prompt-management §6, spec v0.79.0). `PromptManager` construction accepts an optional `default_cache_ttl_seconds`, applied to any `fetch` or `get` that omits a per-call `cache_ttl_seconds`. Resolution follows a precedence chain: an explicit per-call value (including `0` force-fresh) wins; otherwise the manager default applies; otherwise nothing is forwarded and the backend's own caching governs. An omitted or explicit-`None` per-call value both select the default, so resolution does not depend on argument presence. A negative default is rejected at construction, and the per-call negative rejection is unchanged. `get` delegates to `fetch` and inherits the chain, and the bundled caching backends need no change (they already honor a resolved `cache_ttl_seconds`). Conformance fixture 036 is un-deferred. +- **PromptGroup arity enforcement** (proposal 0080, prompt-management §10 / §11, spec v0.75.0). Constructing a `PromptGroup` with fewer than two members (an empty or single-member group) now raises a categorized `PromptGroupInvalid` at construction, before any render or LLM call. This replaces the previous bare `ValueError` (which pydantic folded into a `ValidationError` carrying no error category) and adds a `prompt_group_invalid` category to the prompt-management error set. `PromptGroupInvalid` is non-transient and is exported from `openarmature.prompts`. Conformance fixture 035 is un-deferred. ## [0.16.0] — 2026-07-18 diff --git a/conformance.toml b/conformance.toml index f311c0b5..57c8594b 100644 --- a/conformance.toml +++ b/conformance.toml @@ -795,10 +795,12 @@ note = "Retrieval-provider OpenAI-compatible embeddings wire mapping (§8 / §8. # Spec v0.75.0 (proposal 0080). PromptGroup arity enforcement # (prompt-management §10 / §11 -- construct-time raise plus the new -# prompt_group_invalid error category). Not-yet; prompt-management -# fixture 035 defers with it. +# prompt_group_invalid error category). Implemented since 0.17.0; +# prompt-management fixture 035 pins it. [proposals."0080"] -status = "not-yet" +status = "implemented" +since = "0.17.0" +note = "PromptGroup already rejected fewer than two members, but as a bare ValueError (pydantic folds it into a ValidationError, which has no category). Now raises a categorized PromptGroupInvalid (new category prompt_group_invalid, prompt-management §11) at construction: PromptGroupInvalid is a PromptError (not a ValueError), so pydantic propagates it unwrapped from the model validator and the caller sees .category. Non-transient, not in PROMPT_TRANSIENT_CATEGORIES; exported from openarmature.prompts. Fixture 035 (single-member + empty group, each asserting raises.category = prompt_group_invalid) is un-deferred. Harness: PromptManagementFixture.backends is now optional (035 is cases-only with per-case backends, one case needing none) and the cases-runner builds per-case backends when a case declares its own backends key (else shares the top-level, preserving 016) and runs a manager-less case's direct-target calls instead of skipping them (035's empty-group construct_prompt_group needs no manager)." # Spec v0.76.0 (proposal 0081). Conformance-adapter value-matcher # vocabulary (§5.10) -- ratifies the fixture matcher tokens already in diff --git a/docs/concepts/prompts.md b/docs/concepts/prompts.md index 4ef93750..e5b6d1a6 100644 --- a/docs/concepts/prompts.md +++ b/docs/concepts/prompts.md @@ -329,10 +329,11 @@ Canonical patterns the primitive covers: - **Map-reduce over chunks**: `[chunk_classify_1..N, synthesize]`. The N=2 case ("classifier + follow-up") is the simplest; -larger groups work under the same primitive. The group rejects -empty and single-member shapes; single-prompt tagging is -already served by the per-prompt observability attributes -below. +larger groups work under the same primitive. Constructing a +group with fewer than two members raises `PromptGroupInvalid` +at construction time, before any render or call; single-prompt +tagging is already served by the per-prompt observability +attributes below. ## Observability propagation diff --git a/src/openarmature/prompts/__init__.py b/src/openarmature/prompts/__init__.py index 3feed15d..dc5f07e1 100644 --- a/src/openarmature/prompts/__init__.py +++ b/src/openarmature/prompts/__init__.py @@ -9,11 +9,13 @@ with_active_prompt_group, ) from .errors import ( + PROMPT_GROUP_INVALID, PROMPT_NOT_FOUND, PROMPT_RENDER_ERROR, PROMPT_STORE_UNAVAILABLE, PROMPT_TRANSIENT_CATEGORIES, PromptError, + PromptGroupInvalid, PromptNotFound, PromptRenderError, PromptStoreUnavailable, @@ -38,6 +40,7 @@ ) __all__ = [ + "PROMPT_GROUP_INVALID", "PROMPT_NOT_FOUND", "PROMPT_RENDER_ERROR", "PROMPT_STORE_UNAVAILABLE", @@ -57,6 +60,7 @@ "PromptBackend", "PromptError", "PromptGroup", + "PromptGroupInvalid", "PromptManager", "PromptNotFound", "PromptRenderError", diff --git a/src/openarmature/prompts/errors.py b/src/openarmature/prompts/errors.py index 68843556..9611bb76 100644 --- a/src/openarmature/prompts/errors.py +++ b/src/openarmature/prompts/errors.py @@ -7,6 +7,7 @@ PROMPT_NOT_FOUND = "prompt_not_found" PROMPT_RENDER_ERROR = "prompt_render_error" PROMPT_STORE_UNAVAILABLE = "prompt_store_unavailable" +PROMPT_GROUP_INVALID = "prompt_group_invalid" # Mirrors openarmature.llm.errors.TRANSIENT_CATEGORIES. Retry-middleware # classifiers MAY import this to identify transient prompt-management @@ -123,3 +124,17 @@ def __init__( self.label = label self.backends_tried = backends_tried self.causes = causes + + +class PromptGroupInvalid(PromptError): + """Raised when a ``PromptGroup`` construction violates a + group-validity rule. Currently raised when ``members`` contains + fewer than two elements (an empty or single-member group). Raised + at construction time, before any render or LLM call, so an invalid + group never reaches observability emission. + + Non-transient: a caller-contract violation. Constructing again with + the same members will not succeed without changing them. + """ + + category = PROMPT_GROUP_INVALID diff --git a/src/openarmature/prompts/group.py b/src/openarmature/prompts/group.py index c4977f1f..37d6473b 100644 --- a/src/openarmature/prompts/group.py +++ b/src/openarmature/prompts/group.py @@ -4,6 +4,7 @@ from pydantic import BaseModel, ConfigDict, model_validator +from .errors import PromptGroupInvalid from .prompt import PromptResult @@ -31,6 +32,10 @@ class PromptGroup(BaseModel): @model_validator(mode="after") def _check_min_two_members(self) -> PromptGroup: + # PromptGroupInvalid is not a ValueError, so pydantic propagates it + # unwrapped from the validator (only ValueError / AssertionError get + # folded into a ValidationError) -- the categorized error reaches the + # caller with its ``category``. if len(self.members) < 2: - raise ValueError("prompt group: members MUST contain at least two PromptResult instances") + raise PromptGroupInvalid("prompt group: members MUST contain at least two PromptResult instances") return self diff --git a/tests/conformance/harness/prompt_management.py b/tests/conformance/harness/prompt_management.py index dae8ec59..6e722a8b 100644 --- a/tests/conformance/harness/prompt_management.py +++ b/tests/conformance/harness/prompt_management.py @@ -235,7 +235,10 @@ class FixtureExpectedTopLevel(_PermissiveModel): class PromptManagementFixture(_StrictModel): - backends: list[FixtureBackendSpec] + # Optional: fixture 035 is cases-only with per-case backends (and an + # empty-group case that needs no backend at all), so the top level + # declares none (proposal 0080). + backends: list[FixtureBackendSpec] = [] # Fixture 016 uses a top-level ``cases:`` list to split into # independent sub-cases that share the backends declaration but # each have their own manager + calls. The runner walks the list diff --git a/tests/conformance/test_prompt_management.py b/tests/conformance/test_prompt_management.py index e7e50905..1cd1cd80 100644 --- a/tests/conformance/test_prompt_management.py +++ b/tests/conformance/test_prompt_management.py @@ -620,12 +620,6 @@ def _message_to_dict_for_compare(message: Message) -> dict[str, Any]: "032-cross-variable-substring-stability": ( "Proposal 0047 wire-byte stability (expected_shared_prefix directive); queued for v0.13.0" ), - # ----- v0.16.0 spec-pin bump (v0.70.1 -> v0.84.0) ------------------- - # Proposal 0080 (PromptGroup arity enforcement, spec v0.75.0) -- fixture - # 035 uses a cases-only shape (no backends) the PM fixture model doesn't - # accept, and asserts the construct-time prompt_group_invalid raise that - # python does not yet implement. Defers until a later v0.16.0 PR. - "035-prompt-group-arity-rejection": ("Proposal 0080 PromptGroup arity enforcement; not implemented"), } @@ -671,8 +665,11 @@ async def test_prompt_management_fixture(fixture_path: Path) -> None: if call.capture_as is not None and raised is None: captures[call.capture_as] = result - # Cases-form fixtures (016) split into independent sub-cases that - # share the backends but use their own per-case manager + calls. + # Cases-form fixtures split into independent sub-cases sharing the + # captures dict. Fixture 016 shares the top-level backends across cases; + # fixture 035 (proposal 0080) declares backends per case (one case needs + # none), so a case that brings its own ``backends`` key gets fresh + # backends and any other shares the top-level set. cases = raw.get("cases") if cases: for case in cases: @@ -684,18 +681,32 @@ async def test_prompt_management_fixture(fixture_path: Path) -> None: **{k: v for k, v in case.items() if k not in {"name", "description"}}, } case_fixture = PromptManagementFixture.model_validate(case_payload) + # A case that declares its own ``backends`` REPLACES the top-level + # set (not a union) -- the payload merge already gives case keys + # override semantics. No current fixture needs a case to add to the + # shared backends; if one ever does, merge here instead. + case_backends = ( + {spec.name: MockPromptBackend(spec) for spec in case_fixture.backends} + if "backends" in case + else backends + ) case_manager_pairs = [ (case_fixture.manager, case_fixture.calls), (case_fixture.secondary_manager, case_fixture.secondary_calls), (case_fixture.tertiary_manager, case_fixture.tertiary_calls), ] for manager_spec, manager_calls in case_manager_pairs: - if manager_spec is None: - continue - manager = _build_manager(manager_spec, backends, resolvers_map) + # Build a manager only when one is declared; a manager-less + # case still runs its direct-target calls (035's empty-group + # construct_prompt_group needs no manager). + manager = ( + _build_manager(manager_spec, case_backends, resolvers_map) + if manager_spec is not None + else None + ) for call in manager_calls: - result, raised = await _run_call(call, backends, manager, captures) - _assert_per_call(call, result, raised, backends) + result, raised = await _run_call(call, case_backends, manager, captures) + _assert_per_call(call, result, raised, case_backends) if call.capture_as is not None and raised is None: captures[call.capture_as] = result if case_fixture.expected is not None: diff --git a/tests/unit/test_prompts.py b/tests/unit/test_prompts.py index 8df97a5a..8d157a37 100644 --- a/tests/unit/test_prompts.py +++ b/tests/unit/test_prompts.py @@ -28,6 +28,7 @@ Prompt, PromptError, PromptGroup, + PromptGroupInvalid, PromptManager, PromptNotFound, PromptRenderError, @@ -51,6 +52,7 @@ def test_error_categories_match_spec() -> None: assert PromptNotFound.category == "prompt_not_found" assert PromptRenderError.category == "prompt_render_error" assert PromptStoreUnavailable.category == "prompt_store_unavailable" + assert PromptGroupInvalid.category == "prompt_group_invalid" assert PROMPT_NOT_FOUND == "prompt_not_found" assert PROMPT_RENDER_ERROR == "prompt_render_error" assert PROMPT_STORE_UNAVAILABLE == "prompt_store_unavailable" @@ -167,8 +169,13 @@ def test_prompt_result_rejects_empty_messages() -> None: def test_prompt_group_rejects_zero_members() -> None: - with pytest.raises(ValueError, match="at least two"): + # Categorized prompt_group_invalid (proposal 0080), not a bare ValueError: + # PromptGroupInvalid is not a ValueError, so pydantic propagates it + # unwrapped from the model validator rather than folding it into a + # ValidationError. + with pytest.raises(PromptGroupInvalid, match="at least two") as exc_info: PromptGroup(group_name="g", members=[]) + assert exc_info.value.category == "prompt_group_invalid" def test_prompt_group_rejects_one_member() -> None: @@ -184,8 +191,9 @@ def test_prompt_group_rejects_one_member() -> None: fetched_at=prompt.fetched_at, rendered_at=datetime.now(UTC), ) - with pytest.raises(ValueError, match="at least two"): + with pytest.raises(PromptGroupInvalid, match="at least two") as exc_info: PromptGroup(group_name="g", members=[pr]) + assert exc_info.value.category == "prompt_group_invalid" def test_prompt_group_accepts_two_or_more_members() -> None: From 0986a31e87a6a84c77e5fd6b81d39f27ab152d74 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Wed, 22 Jul 2026 10:52:26 -0700 Subject: [PATCH 2/2] Assert PROMPT_GROUP_INVALID constant in category test Address CoPilot review on #228: test_error_categories_match_spec pinned PromptGroupInvalid.category but not the exported PROMPT_GROUP_INVALID constant, so a regression in the constant value or export could slip through. Add the constant assertion (and its import) alongside the other three categories. Test-only; no behavior change. --- tests/unit/test_prompts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/test_prompts.py b/tests/unit/test_prompts.py index 8d157a37..591af3dc 100644 --- a/tests/unit/test_prompts.py +++ b/tests/unit/test_prompts.py @@ -20,6 +20,7 @@ from openarmature.llm.messages import Message, UserMessage from openarmature.prompts import ( + PROMPT_GROUP_INVALID, PROMPT_NOT_FOUND, PROMPT_RENDER_ERROR, PROMPT_STORE_UNAVAILABLE, @@ -56,6 +57,7 @@ def test_error_categories_match_spec() -> None: assert PROMPT_NOT_FOUND == "prompt_not_found" assert PROMPT_RENDER_ERROR == "prompt_render_error" assert PROMPT_STORE_UNAVAILABLE == "prompt_store_unavailable" + assert PROMPT_GROUP_INVALID == "prompt_group_invalid" def test_transient_categories_contains_only_store_unavailable() -> None: