Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions conformance.toml
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,42 @@ spec_pin = "v0.112.0"
# than the portable suppress floor.
langfuse_bound_provider_detection = true

# External dependencies this implementation requires, and the private upstream
# surface it reaches into (coord thread discuss-external-dependency-tracking).
#
# The split from openarmature.org/compatibility's matrix is deliberate: that
# matrix records what the SPEC's normative text is written against, which is one
# implementation-independent row. This records what THIS implementation requires,
# has verified, and depends on, which differs per implementation. Spec renders a
# per-implementation block from these keys.
#
# `verified_on` is the date the pin was last deliberately moved or re-verified,
# NOT the date CI last passed. A date that sits well back while `requires` still
# admits newer versions is telling the reader something true, and is not nudged
# forward by runs that merely pass.
#
# `internals` is the SOURCE the guard in tests/unit/test_langfuse_sdk_internals.py
# parametrizes over, so the published list and the enforced list cannot drift:
# adding a path here without it existing upstream fails, and losing one upstream
# fails too.
[external_dependencies.langfuse]
requires = ">=4.6,<5"
verified = "4.7.1"
verified_on = "2026-08-13"
note = "The declared range currently resolves as far as 4.14.x, well past `verified`. Every listed internal still exists there and the suite passes against it, but 4.7.1 is the version this implementation deliberately tests against. Losing one of these internals does not raise to the caller: the graph observer isolates observer errors, so an observation simply stops being emitted and a leak assertion reads clean, which is why they are guarded rather than trusted."
Comment thread
chris-colinsky marked this conversation as resolved.
internals = [
"langfuse._client.client.Langfuse._resources",
"langfuse._client.client.Langfuse._tracing_enabled",
"langfuse._client.client.Langfuse._otel_tracer",
"langfuse._client.client.Langfuse._create_remote_parent_span",
"langfuse._client.resource_manager.LangfuseResourceManager.tracer_provider",
"langfuse._client.resource_manager.LangfuseResourceManager._instances",
"langfuse._client.span.LangfuseGeneration",
"langfuse._client.span.LangfuseTool",
"langfuse._client.span.LangfuseEmbedding",
"langfuse._client.span.LangfuseRetriever",
]

# Status values:
# implemented — shipped behavior matches the proposal's contract
# partial — partial impl; consult `note` for what's missing
Expand Down
13 changes: 9 additions & 4 deletions src/openarmature/observability/langfuse/adapter.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
# Bridges the langfuse Python SDK (v4.6+) onto the LangfuseClient
# Protocol. Validated against langfuse==4.7.0; the [langfuse] extras
# pin to `>=4.6,<5`. SDK churn before v4 (v2/v3 API removed in v4) is
# not supported — projects on v2/v3 should write their own adapter or
# upgrade.
# Protocol. The version this adapter is verified against, and the private
# SDK surface it depends on, are published in `conformance.toml` under
# `[external_dependencies.langfuse]` and enforced by
# `tests/unit/test_langfuse_sdk_internals.py`. Named there rather than
# here so there is one authority: a version in a comment goes stale
# silently, which is exactly what that manifest entry exists to prevent.
# The [langfuse] extras pin to `>=4.6,<5`. SDK churn before v4 (v2/v3
# API removed in v4) is not supported — projects on v2/v3 should write
# their own adapter or upgrade.
#
# Shape mismatch the adapter handles:
# - v4 has no explicit `client.trace(...)` — traces are auto-created
Expand Down
120 changes: 99 additions & 21 deletions tests/unit/test_langfuse_sdk_internals.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,17 @@

from __future__ import annotations

import importlib
import inspect
import re
import tomllib
from pathlib import Path
from typing import Any, cast

import pytest

_CONFORMANCE_TOML = Path(__file__).resolve().parents[2] / "conformance.toml"

langfuse = pytest.importorskip("langfuse", reason="the langfuse extra is optional")


Expand Down Expand Up @@ -64,41 +71,112 @@
assert isinstance(LangfuseResourceManager._instances, dict)


@pytest.mark.parametrize(
("symbol", "spelling"),
[("_otel_tracer", "self._otel_tracer"), ("_create_remote_parent_span", "def _create_remote_parent_span")],
)
def test_the_back_dated_observation_internals_exist(symbol: str, spelling: str) -> None:
# Every provider observation carrying a start_time goes through
# _start_back_dated_observation, which needs both. Losing either does not
# raise to the caller: the graph observer isolates observer errors, so the
# observation simply stops being emitted and a leak assertion reads clean.
def _declared() -> dict[str, Any]:
# Read at collection time to parametrize, so a missing section surfaces as a
# named failure rather than a bare KeyError from inside pytest's collector.
with _CONFORMANCE_TOML.open("rb") as handle:
manifest = tomllib.load(handle)
entry = cast("dict[str, Any]", manifest.get("external_dependencies", {})).get("langfuse")
assert entry is not None, (
"conformance.toml has no [external_dependencies.langfuse] section. It is the published "
"record of the private SDK surface this implementation depends on, and the source these "
"guards parametrize over; without it nothing checks that surface."
)
missing = sorted({"requires", "verified", "verified_on", "internals"} - set(entry))
assert not missing, f"[external_dependencies.langfuse] is missing required keys: {missing}"
return cast("dict[str, Any]", entry)


def _resolve(path: str) -> tuple[Any, str]:
"""Split a dotted path into the deepest importable owner and the final name."""
parts = path.split(".")
for cut in range(len(parts) - 1, 0, -1):
try:
owner: Any = importlib.import_module(".".join(parts[:cut]))
except ImportError:
continue
for attr in parts[cut:-1]:
# Asserted rather than left to getattr: a renamed intermediate (a
# class, say) would otherwise surface as a bare AttributeError, and
# the message IS this guard's product. Losing one of these paths shows
# up nowhere else, because the graph observer swallows observer errors.
assert hasattr(owner, attr), (
f"{path}: {attr!r} is missing from {owner!r}, so the rest of the path cannot be "
f"resolved. openarmature's shipped adapter depends on this path."
)
owner = getattr(owner, attr)
return owner, parts[-1]
raise AssertionError(
f"no importable module in {path!r}; the declared internal names a module that no "
f"longer exists in the installed SDK"
)


@pytest.mark.parametrize("path", _declared()["internals"])
def test_each_declared_internal_still_exists(path: str) -> None:
# Parametrized over conformance.toml's `internals` list rather than a copy of
# it, so the PUBLISHED surface and the ENFORCED surface are the same list.
# Restating it here would let the public record drift from what is checked,
# which is the failure this whole guard exists to prevent.
#
# Checked in the source rather than with hasattr, because `_otel_tracer` is an
# INSTANCE attribute and a class-level hasattr would report it missing on a
# perfectly good SDK.
assert spelling in inspect.getsource(langfuse.Langfuse), (
f"langfuse.Langfuse.{symbol} is gone; adapter._start_back_dated_observation depends "
f"on it, and its absence surfaces only as a swallowed observer warning"
# Losing any of these does not raise to the caller: the graph observer
# isolates observer errors, so an observation simply stops being emitted and
# a leak assertion reads clean.
owner, name = _resolve(path)
if hasattr(owner, name):
return
# Instance attributes (`self._resources`, `self._otel_tracer`) are not on the
# class, so a bare hasattr would report a perfectly good SDK as broken.
source = inspect.getsource(owner)
assert f"self.{name}" in source, (
f"{path} is gone from the installed langfuse SDK. openarmature's shipped adapter "
f"depends on it, and its absence surfaces only as a swallowed observer warning."
)


@pytest.mark.parametrize(
"name", ["LangfuseGeneration", "LangfuseTool", "LangfuseEmbedding", "LangfuseRetriever"]
)
def test_the_span_classes_the_adapter_constructs_exist(name: str) -> None:
span_module = pytest.importorskip("langfuse._client.span")
assert hasattr(span_module, name)
def test_the_declared_internals_cover_what_the_adapter_imports() -> None:
# The completeness half: the per-path checks above verify what IS declared and
# can say nothing about what the adapter depends on but nobody declared.
#
# Parsed from the actual import statements rather than matched against names
# written out here. A hardcoded list is not a completeness check: an earlier
# version enumerated four span classes, so a fifth added later would have gone
# unnoticed by the very test meant to catch that. A substring search over the
# module source would also match a name in prose, and a suffix match would
# accept the same name exported by a different module.
adapter_source = inspect.getsource(importlib.import_module("openarmature.observability.langfuse.adapter"))
imported = set(re.findall(r"from\s+langfuse\._client\.span\s+import\s+(\w+)", adapter_source))
assert imported, (
"found no `from langfuse._client.span import ...` in the adapter. Either it stopped "
"importing private span classes, in which case the declared internals should shrink, "
"or this pattern no longer matches and the check is reading nothing."
)
declared = set(_declared()["internals"])
missing = sorted(
f"langfuse._client.span.{name}"
for name in imported
if f"langfuse._client.span.{name}" not in declared
)
assert not missing, (
f"the adapter imports {missing} but conformance.toml does not declare them, so a rename "
f"upstream would go both unguarded and unpublished"
)


def test_the_installed_version_is_within_the_declared_range() -> None:
# Non-vacuity for everything above: the checks are only meaningful against a
# version we claim to support. This also surfaces drift between what is
# installed and what openarmature.org/compatibility records as verified.
import re

Check notice

Code scanning / CodeQL

Module is imported more than once Note test

This import of module re is redundant, as it was previously imported
on line 26
.
from importlib.metadata import version

installed = version("langfuse")
declared = _declared()
assert installed == declared["verified"], (
f"conformance.toml publishes langfuse {declared['verified']} as the verified version "
f"but {installed} is installed. Either move the pin deliberately and update `verified` "
f"and `verified_on`, or restore the lock; the published number must be the tested one."
)
# Regex rather than int() on the split parts: a PEP 440 two-component
# pre-release such as 4.7rc1 attaches its suffix to the MINOR, so splitting
# raises ValueError on a version that is inside our declared range. A bare
Expand Down