Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion CHANGELOG.md

Large diffs are not rendered by default.

41 changes: 40 additions & 1 deletion conformance.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@

[manifest]
implementation = "openarmature-python"
spec_pin = "v0.107.0"
spec_pin = "v0.112.0"

# Adapter-level conformance capabilities (conformance-adapter §5.5, proposal
# 0116). A fixture case may carry `requires_capability`, which selects the arm of
Expand Down Expand Up @@ -1093,3 +1093,42 @@ note = "0111 scoped the collect-channel warning to inputs-based seeding; it now
status = "implemented"
since = "0.17.0"
note = "0113 pins the malformed case of the §6 Managed-field collision MERGE arm: a caller's extras value that is not the field's expected list shape, or a list containing any element not of the expected element type/shape, is treated as ABSENT (the mapping sends only the value(s) it would send with no such extra present -- all-or-nothing, no partial salvage, no raise / diagnostic); malformation is judged structurally (no semantic vocabulary check). The shared apply_managed_extras merge arm gates on well-formedness (a string, or a list whose every element is a string): a malformed extra is skipped and the managed value stands alone. Fixture 081 (llm stop) exercises the general rule; the retrieval-provider §8.4 embedding_types arm (a malformed extras embedding_types -> wire [\"float\"], fixture 053) already shipped via 0099's bespoke Cohere merge."

# ---- v0.17.0 spec-pin bump v0.107.0 -> v0.112.0 (proposals 0114-0118).
# The Langfuse client-ownership and provider-isolation arc, opened by a
# registered consumer's leak report and settled across five spec releases.

# Spec v0.108.0 (proposal 0114). Langfuse client ownership and
# tracer-provider isolation (observability §6 / §8.9).
[proposals."0114"]
status = "implemented"
since = "0.17.0"
note = "Pins the two-mode Langfuse client ownership model. Mode (b): LangfuseObserver.from_credentials(public_key=..., secret_key=..., host=...) (over LangfuseSDKAdapter.from_credentials) builds an OA-owned client on a dedicated TracerProvider, so its observations no longer bind the globally-registered provider and export onto the application's OTel backend. Mode (a), a caller-supplied client, is never mutated: the caller stays responsible for isolating its tracer_provider and OA documents the remedy. secret_key is a pydantic SecretStr, masked in OA's reprs and logs with the plaintext read only at the SDK call; a blank credential is rejected at the boundary rather than falling through to the SDK's ambient LANGFUSE_* environment fallback. Fixture 157 is deferred: it needs the §6.4 provider-faithful fake and the langfuse_client construction directive, and is unit-tested meanwhile in tests/unit/test_langfuse_provider_isolation.py."

# Spec v0.109.0 (proposal 0115). Conformance primitives for Langfuse
# provider isolation (conformance-adapter §5.5 / §6.4).
[proposals."0115"]
status = "partial"
since = "0.17.0"
note = "The adapter-capability half is implemented: conformance.toml declares langfuse_bound_provider_detection and the harness gates a case on requires_capability, treating a mismatched arm as a recognized skip and an undeclared capability name as an error. The §6.4 provider-faithful Langfuse fake (records observation content AND emits it as OTel spans through its bound TracerProvider, so a leak to a shared provider is catchable) and the langfuse_client construction directive are NOT yet built, so fixtures 157 / 158 stay deferred. The four leak assertions are declared in the expectations model so those fixtures parse at this pin, and the Langfuse runner fails loudly if an activated fixture reaches for one, so a declared-but-unimplemented assertion cannot read as coverage."

# Spec v0.110.0 (proposal 0116). Fail closed when Langfuse payloads
# would reach a shared provider (observability §6).
[proposals."0116"]
status = "implemented"
since = "0.17.0"
note = "The Langfuse v4 SDK caches one client per public_key, so mode (b)'s dedicated provider takes effect only when OA is the first constructor for that credential; otherwise the SDK returns the cached client on its original provider and discards OA's. OA reuses one isolated provider per credential and reads the actual binding back after construction. Where a construction-determinable payload channel is live and OA establishes the client is bound to a provider it did not isolate, construction raises a categorized LangfuseProviderIsolationUnavailable before any observation is emitted; where OA cannot establish the binding at all it suppresses every channel and logs one WARNING; accept_shared_provider=True is the single opt-out, binding the provider the application already registered rather than letting the SDK construct and globally register one. With no channel live (the default posture) an un-isolatable client neither raises nor warns. Fixture 158 is deferred on the same §6.4 machinery as 157."

# Spec v0.111.0 (proposal 0117). Payload-leak invariant broadened to all
# harvested-payload channels (observability §6 / §8.4).
[proposals."0117"]
status = "implemented"
since = "0.17.0"
note = "0116 guarded only the provider-payload channel. A sweep of OA's Langfuse observation surface found two more channels carrying payload OA harvests from the runtime: the Trace-level state input / output (disable_state_payload plus the trace_input_from_state / trace_output_from_state hooks, which emit regardless of the knob) and a failed observation's error_message. All three are covered. The scoping clause exempting caller-ATTACHED dimensions (correlation_id, session_id, the promoted userId, trace name, arbitrary caller metadata) is honored: those are opaque cross-backend join keys the caller owns and they ride verbatim. Fixture 158's six added cases ride with it."

# Spec v0.112.0 (proposal 0118). The harvested error message comes under
# the payload flag (observability §5.5.4 / §8.4 / §8.7).
[proposals."0118"]
status = "implemented"
since = "0.17.0"
note = "disable_provider_payload now gates a failed provider call's harvested exception TEXT on all four provider observations, closing a hole in the flag's promise: a structured_output_invalid message commonly quotes the model output that failed validation, so a caller could set the flag, watch generation.output disappear as promised, and still receive that text in metadata.error_message. error_type is NOT gated: it is a classification token, so suppressing it buys no privacy while removing the only failure discriminator on a Tool observation, which has no error category at all; for the same reason it no longer counts toward the payload-bearing classification. The §8.4 exhaustive-mapping rule is honored too: the failure-isolation marker span carried the caught exception's message although no §8.4.x table maps it, which is non-conforming over-emission, and it now carries only error_category. Fixtures 098 / 137 / 138 are un-deferred and reconciled at this pin, and 159 is wired."
2 changes: 1 addition & 1 deletion openarmature-spec
Submodule openarmature-spec updated 39 files
+69 −0 CHANGELOG.md
+2 −2 README.md
+13 −2 docs/compatibility.md
+56 −0 docs/open-questions.md
+2 −2 docs/openarmature.md
+123 −0 docs/privacy-controls.md
+25 −20 docs/proposals.md
+1 −0 docs/proposals/0114-langfuse-client-provider-isolation.md
+1 −0 docs/proposals/0115-langfuse-provider-isolation-conformance.md
+1 −0 docs/proposals/0116-langfuse-isolation-fail-loud.md
+1 −0 docs/proposals/0117-payload-leak-invariant-channels.md
+1 −0 docs/proposals/0118-llm-error-message-channel.md
+3 −0 mkdocs.yml
+213 −0 proposals/0114-langfuse-client-provider-isolation.md
+170 −0 proposals/0115-langfuse-provider-isolation-conformance.md
+392 −0 proposals/0116-langfuse-isolation-fail-loud.md
+327 −0 proposals/0117-payload-leak-invariant-channels.md
+310 −0 proposals/0118-llm-error-message-channel.md
+76 −0 spec/conformance-adapter/spec.md
+35 −1 spec/observability/conformance/001-otel-basic-trace.yaml
+22 −2 spec/observability/conformance/098-langfuse-tool-observation.md
+70 −2 spec/observability/conformance/098-langfuse-tool-observation.yaml
+8 −0 spec/observability/conformance/123-langfuse-failed-generation-renders-output-usage-finish-reason.md
+11 −0 spec/observability/conformance/123-langfuse-failed-generation-renders-output-usage-finish-reason.yaml
+8 −4 spec/observability/conformance/137-langfuse-embedding-failure-observation.md
+13 −6 spec/observability/conformance/137-langfuse-embedding-failure-observation.yaml
+10 −5 spec/observability/conformance/138-langfuse-rerank-failure-observation.md
+13 −7 spec/observability/conformance/138-langfuse-rerank-failure-observation.yaml
+7 −0 spec/observability/conformance/150-langfuse-embedding-failure-literal-error-fields.md
+9 −2 spec/observability/conformance/150-langfuse-embedding-failure-literal-error-fields.yaml
+7 −0 spec/observability/conformance/151-langfuse-rerank-failure-literal-error-fields.md
+11 −2 spec/observability/conformance/151-langfuse-rerank-failure-literal-error-fields.yaml
+32 −0 spec/observability/conformance/157-langfuse-provider-isolation.md
+135 −0 spec/observability/conformance/157-langfuse-provider-isolation.yaml
+126 −0 spec/observability/conformance/158-langfuse-payload-leak-fail-closed.md
+650 −0 spec/observability/conformance/158-langfuse-payload-leak-fail-closed.yaml
+51 −0 spec/observability/conformance/159-langfuse-llm-failure-error-message.md
+157 −0 spec/observability/conformance/159-langfuse-llm-failure-error-message.yaml
+304 −14 spec/observability/spec.md
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ Specification = "https://github.com/LunarCommand/openarmature-spec"
openarmature = "openarmature.cli:main"

[tool.openarmature]
spec_version = "0.107.0"
spec_version = "0.112.0"

[dependency-groups]
dev = [
Expand Down
4 changes: 2 additions & 2 deletions src/openarmature/AGENTS.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# OpenArmature — Agent documentation

*This is the agent guide bundled with the openarmature Python package, version 0.16.0 (spec v0.107.0). For the full docs site see [openarmature.ai](https://openarmature.ai). For the canonical spec text see [openarmature.org/capabilities](https://openarmature.org/capabilities/). For project-specific conventions for the code you're editing, see the host project's `AGENTS.md` or `CLAUDE.md`.*
*This is the agent guide bundled with the openarmature Python package, version 0.16.0 (spec v0.112.0). For the full docs site see [openarmature.ai](https://openarmature.ai). For the canonical spec text see [openarmature.org/capabilities](https://openarmature.org/capabilities/). For project-specific conventions for the code you're editing, see the host project's `AGENTS.md` or `CLAUDE.md`.*

## TL;DR

Expand All @@ -10,7 +10,7 @@ OpenArmature is a workflow framework for LLM pipelines and tool-calling agents:

## Capability contracts

_Sourced from openarmature-spec v0.107.0. Each entry below reproduces §1 (Purpose) and §2 (Concepts) of the capability's `spec.md` verbatim — including additions from accepted proposals that this Python implementation may not yet ship. For per-proposal implementation status (implemented / partial / textual-only / not-yet), see the `conformance.toml` manifest at the repo root. For the full spec text (execution model, error semantics, determinism, observer hooks, etc.) see the linked docs site._
_Sourced from openarmature-spec v0.112.0. Each entry below reproduces §1 (Purpose) and §2 (Concepts) of the capability's `spec.md` verbatim — including additions from accepted proposals that this Python implementation may not yet ship. For per-proposal implementation status (implemented / partial / textual-only / not-yet), see the `conformance.toml` manifest at the repo root. For the full spec text (execution model, error semantics, determinism, observer hooks, etc.) see the linked docs site._

### Capability: `graph-engine`

Expand Down
2 changes: 1 addition & 1 deletion src/openarmature/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
"""

__version__ = "0.16.0"
__spec_version__ = "0.107.0"
__spec_version__ = "0.112.0"
# Proposal 0052 (spec observability §5.1 / §8.4.1): canonical
# package-registry name for this implementation. Surfaces on every
# OTel invocation span as ``openarmature.implementation.name`` and on
Expand Down
17 changes: 17 additions & 0 deletions tests/conformance/harness/expectations.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,17 @@ class ObservabilityExpected(_ForbidExtras):
no_openarmature_spans_on_global: bool | None = None
no_edge_spans: bool | None = None
no_llm_provider_span: bool | None = None
# Langfuse provider-isolation leak assertions (proposals 0115 / 0116,
# fixtures 157 / 158). Declared so those fixtures PARSE at this pin; the
# assertions themselves are not implemented yet, and both fixtures are
# deferred accordingly. `_UNIMPLEMENTED_OBSERVABILITY_ASSERTIONS` in the
# Langfuse runner fails loudly if an ACTIVATED fixture reaches for one, so a
# declared-but-unimplemented key cannot pass as coverage.
no_langfuse_observations_on_global: bool | None = None
no_langfuse_observations_on_private: bool | None = None
langfuse_observations_on_global: bool | None = None
no_payload_bearing_langfuse_observations_on_global: bool | None = None
payload_bearing_langfuse_observations_on_global: bool | None = None
# Invariants block (fixture 011 determinism).
invariants: dict[str, Any] | None = None
determinism_check: dict[str, Any] | None = None
Expand Down Expand Up @@ -259,6 +270,12 @@ class ObservabilityExpected(_ForbidExtras):
"no_openarmature_spans_on_global",
"no_edge_spans",
"no_llm_provider_span",
# proposals 0115 / 0116 (fixtures 157 / 158) Langfuse leak assertions.
"no_langfuse_observations_on_global",
"no_langfuse_observations_on_private",
"langfuse_observations_on_global",
"no_payload_bearing_langfuse_observations_on_global",
"payload_bearing_langfuse_observations_on_global",
"determinism_check",
"invocation_count",
# proposal 0064 (fixture 084) Langfuse Trace-level expectations
Expand Down
61 changes: 41 additions & 20 deletions tests/conformance/test_observability.py
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,12 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
"095-tool-call-id-links-to-llm-request",
"096-tool-call-payload-gating",
"097-otel-tool-span-attributes",
# 098 (proposal 0118, spec v0.112.0): reconciled at this pin. Its
# failure case moves to disable_provider_payload=false (asserting the
# message literally is its purpose) and it gains a default-posture
# failure case carrying the Tool anti-smuggling clause, which 158
# cannot cover for a detection-capable adapter (that arm raises first).
"098-langfuse-tool-observation",
# v0.16.0 — proposal 0059 embedding observability (0059b). A
# calls_embed node awaits OpenAIEmbeddingProvider.embed() inside the
# node body; the typed EmbeddingEvent / EmbeddingFailedEvent drive the
Expand All @@ -278,6 +284,10 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
"081-embedding-event-active-prompt-populated",
"082-otel-embedding-span-attributes",
"083-langfuse-embedding-observation",
# 137 / 138 (proposal 0118): the payload-suppressed cases now assert the
# harvested error message ABSENT with error_type retained, via the
# metadata_absent directive that landed in #267.
"137-langfuse-embedding-failure-observation",
"139-otel-embedding-no-usage-input-tokens-omitted",
"140-langfuse-embedding-no-usage-usagedetails-omitted",
# proposal 0067 §11 embedding metrics: token.usage (input only) +
Expand All @@ -304,6 +314,7 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
"106-rerank-event-active-prompt-populated",
"107-otel-rerank-span-attributes",
"108-langfuse-rerank-observation",
"138-langfuse-rerank-failure-observation",
"141-otel-rerank-no-usage-attributes-omitted",
"142-langfuse-rerank-no-usage-usagedetails-omitted",
"109-rerank-metrics-token-and-duration",
Expand Down Expand Up @@ -338,18 +349,6 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
# pin bump is now the only remaining prerequisite; the source behavior here is
# covered meanwhile by the default-posture unit tests in
# tests/unit/test_langfuse_provider_isolation.py.
"098-langfuse-tool-observation": (
"Proposal 0118 error-message gating: fixture asserts the pre-0118 shape; "
"reconciled at the v0.112.0 pin bump"
),
"137-langfuse-embedding-failure-observation": (
"Proposal 0118 error-message gating: fixture asserts the pre-0118 shape; "
"reconciled at the v0.112.0 pin bump"
),
"138-langfuse-rerank-failure-observation": (
"Proposal 0118 error-message gating: fixture asserts the pre-0118 shape; "
"reconciled at the v0.112.0 pin bump"
),
# Proposal 0045 IS implemented (v0.11.0), but the nested-case Langfuse
# fixture stays deferred: it needs runtime-state item-list lookup for
# nested fan-outs plus an augment_metadata_from_outer_item directive
Expand Down Expand Up @@ -438,17 +437,35 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
),
# Proposal 0107 (spec v0.102.0) mock_embedding / mock_rerank raises
# sub-directive -> literal error-field assertion.
# The 0118 half of these two resolved at this pin (both cases moved to
# disable_provider_payload=false, since asserting the message literally is
# their purpose). What remains is the 0107 mock-raises harness wiring.
"150-langfuse-embedding-failure-literal-error-fields": (
"Proposal 0107 mock-raises literal error fields; harness wiring rides the v0.17.0 "
"fixture-wiring PR. Also asserts the pre-0118 shape at this pin: spec moves both "
"cases to disable_provider_payload=false at v0.112.0, since asserting the message "
"literally is their purpose"
"Proposal 0107 mock_embedding raises sub-directive not yet wired; rides the "
"remaining v0.17.0 fixture-wiring PR (observability 144-156)"
),
"151-langfuse-rerank-failure-literal-error-fields": (
"Proposal 0107 mock-raises literal error fields; harness wiring rides the v0.17.0 "
"fixture-wiring PR. Also asserts the pre-0118 shape at this pin: spec moves both "
"cases to disable_provider_payload=false at v0.112.0, since asserting the message "
"literally is their purpose"
"Proposal 0107 mock_rerank raises sub-directive not yet wired; rides the "
"remaining v0.17.0 fixture-wiring PR (observability 144-156)"
),
# 157 / 158 (proposals 0115 / 0116 / 0117 / 0118, spec v0.109.0-v0.112.0).
# The src side shipped ahead of this pin and is unit-tested; what these need
# is harness machinery: the provider-faithful Langfuse fake of
# conformance-adapter 6.4 (records observation content AND emits it through
# its bound TracerProvider, so a leak to a shared provider is catchable), the
# langfuse_client construction directive, expected_construction_error, and
# the four payload-scoped leak assertions. 158 additionally gates arms on the
# requires_capability audience gate.
"157-langfuse-provider-isolation": (
"needs the conformance-adapter 6.4 provider-faithful Langfuse fake and the "
"langfuse_client construction directive; src side is unit-tested meanwhile in "
"tests/unit/test_langfuse_provider_isolation.py"
),
"158-langfuse-payload-leak-fail-closed": (
"needs the provider-faithful fake, the langfuse_client singleton sub-directives, "
"expected_construction_error, and the payload-scoped leak assertions; src side is "
"unit-tested meanwhile in tests/unit/test_langfuse_provider_isolation.py and "
"tests/unit/test_langfuse_payload_leak_canary.py"
),
# Spec v0.103.1 conformance coverage (0084 orphan-fallback arms + the
# embedding failure-metrics counterpart).
Expand Down Expand Up @@ -498,6 +515,10 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
# sibling harness; the generic topology path cannot model the
# `calls_llm_from_wrapper` orphan-call primitive.
"134-langfuse-nested-fan-out-parent-resolution",
# 159 (proposal 0118): the failed Generation's harvested error message
# under `disable_provider_payload`. Its langfuse_trace shape lives in the
# sibling harness, like 123 and 130.
"159-langfuse-llm-failure-error-message",
}
)

Expand Down
Loading