From 035244f7b8b55e2c8d809a40c0b3737f68434d39 Mon Sep 17 00:00:00 2001 From: Gorkem Date: Mon, 13 Jul 2026 17:05:25 +0300 Subject: [PATCH 1/5] feat(admission): add standalone AdmissionController factory createAdmissionController builds a controller from config alone, with no dependency on SmartExtractor. Availability depends only on admissionControl.enabled, laying the groundwork for gating write paths that don't go through extraction. --- src/admission-control.ts | 17 +++++++ test/admission-controller-standalone.test.mjs | 44 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 test/admission-controller-standalone.test.mjs diff --git a/src/admission-control.ts b/src/admission-control.ts index eee44d35e..8b0a7f707 100644 --- a/src/admission-control.ts +++ b/src/admission-control.ts @@ -627,6 +627,23 @@ async function scoreUtility( }; } +/** + * Construct an AdmissionController independently of any extraction engine. + * Availability depends only on the admission config's `enabled` flag, so + * callers that never build a SmartExtractor (e.g. smartExtraction: false) + * can still obtain a working controller to gate other write paths. + */ +export function createAdmissionController( + store: MemoryStore, + llm: LlmClient, + config: AdmissionControlConfig | undefined, + debugLog: (msg: string) => void = () => {}, +): AdmissionController | null { + return config?.enabled === true + ? new AdmissionController(store, llm, config, debugLog) + : null; +} + export class AdmissionController { constructor( private readonly store: MemoryStore, diff --git a/test/admission-controller-standalone.test.mjs b/test/admission-controller-standalone.test.mjs new file mode 100644 index 000000000..3f8de5fbf --- /dev/null +++ b/test/admission-controller-standalone.test.mjs @@ -0,0 +1,44 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { createAdmissionController, normalizeAdmissionControlConfig, AdmissionController } = + jiti("../src/admission-control.ts"); + +describe("createAdmissionController", () => { + it("returns null when admission control is disabled", () => { + const config = normalizeAdmissionControlConfig({ enabled: false }); + const controller = createAdmissionController({}, {}, config); + assert.equal(controller, null); + }); + + it("returns a usable AdmissionController instance when enabled, without any extractor involved", async () => { + const config = normalizeAdmissionControlConfig({ enabled: true, utilityMode: "off" }); + const store = { + async vectorSearch() { + return []; + }, + }; + const llm = {}; + + const controller = createAdmissionController(store, llm, config); + + assert.ok(controller instanceof AdmissionController); + + const evaluation = await controller.evaluate({ + candidate: { + category: "events", + abstract: "user mentioned a fact", + overview: "## Event", + content: "the user mentioned a fact", + }, + candidateVector: [], + conversationText: "the user mentioned a fact today", + scopeFilter: ["global"], + }); + + assert.ok(evaluation.decision === "reject" || evaluation.decision === "pass_to_dedup"); + assert.equal(evaluation.audit.version, "amac-v1"); + }); +}); From 9830a80298d9af3d70dcb37718ebff56dc32ee2c Mon Sep 17 00:00:00 2001 From: Gorkem Date: Mon, 13 Jul 2026 17:07:27 +0300 Subject: [PATCH 2/5] refactor(smart-extractor): accept a pre-built admission controller SmartExtractor no longer constructs its own AdmissionController from config; it now uses whatever controller instance it is handed (or none). This decouples admission gating from the extraction engine, so a controller built independently (e.g. via createAdmissionController) can be reused by write paths that don't go through SmartExtractor at all. --- src/smart-extractor.ts | 18 +-- ...or-admission-controller-injection.test.mjs | 147 ++++++++++++++++++ 2 files changed, 156 insertions(+), 9 deletions(-) create mode 100644 test/smart-extractor-admission-controller-injection.test.mjs diff --git a/src/smart-extractor.ts b/src/smart-extractor.ts index aafa31165..79ea581e7 100644 --- a/src/smart-extractor.ts +++ b/src/smart-extractor.ts @@ -288,6 +288,14 @@ export interface SmartExtractorConfig { workspaceBoundary?: WorkspaceBoundaryConfig; /** Optional admission-control governance layer before downstream dedup/persistence. */ admissionControl?: AdmissionControlConfig; + /** + * Pre-built admission controller, constructed independently of the + * extractor (e.g. by createAdmissionController) so admission gating works + * the same whether or not smart extraction itself is enabled. When + * provided, this instance is used as-is; the extractor never builds its + * own. Null/omitted means admission control is unavailable. + */ + admissionController?: AdmissionController | null; /** Optional sink for durable reject-audit logging. */ onAdmissionRejected?: (entry: AdmissionRejectionAuditEntry) => Promise | void; /** Optional sink invoked after a memory is successfully created or merged (e.g. markdown mirror). */ @@ -330,15 +338,7 @@ export class SmartExtractor { config.admissionControl.auditMetadata !== false; this.onAdmissionRejected = config.onAdmissionRejected; this.onPersisted = config.onPersisted; - this.admissionController = - config.admissionControl?.enabled === true - ? new AdmissionController( - this.store, - this.llm, - config.admissionControl, - this.debugLog, - ) - : null; + this.admissionController = config.admissionController ?? null; } /** diff --git a/test/smart-extractor-admission-controller-injection.test.mjs b/test/smart-extractor-admission-controller-injection.test.mjs new file mode 100644 index 000000000..5a8f0c08f --- /dev/null +++ b/test/smart-extractor-admission-controller-injection.test.mjs @@ -0,0 +1,147 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { SmartExtractor } = jiti("../src/smart-extractor.ts"); + +function makeStore(overrides = {}) { + return { + async vectorSearch() { + return []; + }, + async store() {}, + async bulkStore() {}, + ...overrides, + }; +} + +function makeEmbedder() { + return { + async embed() { + return Array(8).fill(0.1); + }, + async embedBatch(texts) { + return (texts || []).map(() => Array(8).fill(0.1)); + }, + }; +} + +function makeLlm() { + return { + async completeJson(_prompt, mode) { + if (mode === "extract-candidates") { + return { + memories: [ + { + category: "events", + abstract: "user did something notable", + overview: "## Event", + content: "the user did something notable", + }, + ], + }; + } + throw new Error(`unexpected mode: ${mode}`); + }, + }; +} + +function baseAudit(decision, hint) { + return { + version: "amac-v1", + decision, + hint, + score: decision === "reject" ? 0 : 0.9, + reason: `test-${decision}`, + thresholds: { reject: 0.45, admit: 0.6 }, + weights: { utility: 0.1, confidence: 0.1, novelty: 0.1, recency: 0.1, typePrior: 0.6 }, + feature_scores: { utility: 0, confidence: 0, novelty: 0, recency: 0, typePrior: 0 }, + matched_existing_memory_ids: [], + compared_existing_memory_ids: [], + max_similarity: 0, + evaluated_at: Date.now(), + }; +} + +describe("SmartExtractor admission controller injection", () => { + it("gates candidates through an externally-constructed admission controller", async () => { + let evaluateCalls = 0; + const injectedController = { + async evaluate() { + evaluateCalls++; + return { decision: "reject", audit: baseAudit("reject") }; + }, + }; + + const extractor = new SmartExtractor(makeStore(), makeEmbedder(), makeLlm(), { + user: "User", + extractMinMessages: 1, + extractMaxChars: 8000, + defaultScope: "global", + admissionController: injectedController, + log() {}, + debugLog() {}, + }); + + const stats = await extractor.extractAndPersist( + "the user did something notable today", + "session-1", + { scope: "global" }, + ); + + assert.equal(evaluateCalls, 1, "expected the injected controller's evaluate() to be called"); + assert.equal(stats.rejected, 1); + assert.equal(stats.created, 0); + }); + + it("stores admitted candidates when the injected controller passes them", async () => { + let evaluateCalls = 0; + const injectedController = { + async evaluate() { + evaluateCalls++; + return { decision: "pass_to_dedup", hint: "add", audit: baseAudit("pass_to_dedup", "add") }; + }, + }; + + const extractor = new SmartExtractor(makeStore(), makeEmbedder(), makeLlm(), { + user: "User", + extractMinMessages: 1, + extractMaxChars: 8000, + defaultScope: "global", + admissionController: injectedController, + log() {}, + debugLog() {}, + }); + + const stats = await extractor.extractAndPersist( + "the user did something notable today", + "session-2", + { scope: "global" }, + ); + + assert.equal(evaluateCalls, 1); + assert.equal(stats.created, 1); + assert.equal(stats.rejected ?? 0, 0); + }); + + it("skips admission gating entirely when no controller is configured (today's off-behavior)", async () => { + const extractor = new SmartExtractor(makeStore(), makeEmbedder(), makeLlm(), { + user: "User", + extractMinMessages: 1, + extractMaxChars: 8000, + defaultScope: "global", + log() {}, + debugLog() {}, + }); + + const stats = await extractor.extractAndPersist( + "the user did something notable today", + "session-3", + { scope: "global" }, + ); + + assert.equal(stats.rejected ?? 0, 0); + assert.equal(stats.created, 1); + }); +}); From c5dcac362ddb6817271f8ece8d4b926a092bac52 Mon Sep 17 00:00:00 2001 From: Gorkem Date: Mon, 13 Jul 2026 17:16:49 +0300 Subject: [PATCH 3/5] fix(admission): make admission gating independent of smart extraction Construct the AdmissionController whenever admissionControl.enabled is true, not only when smartExtraction is also on. Previously the LLM client and controller were only ever built inside the smartExtraction block, so any write path other than SmartExtractor (reflection-mapped rows, the regex fallback) had no controller to borrow when smart extraction was disabled and silently stored ungated. The controller is now exposed on the plugin's singleton state alongside smartExtractor, so those other write paths can reference it directly once wired in, instead of reaching through smartExtractor. Registers the three new test files in package.json's test chain and scripts/ci-test-manifest.mjs (core-regression group). --- dist/index.js | 134 +++++-------- dist/src/admission-control.js | 11 ++ dist/src/smart-extractor.js | 34 ++-- index.ts | 143 ++++++++------ package.json | 2 +- scripts/ci-test-manifest.mjs | 5 +- ...dmission-without-smart-extraction.test.mjs | 184 ++++++++++++++++++ 7 files changed, 344 insertions(+), 169 deletions(-) create mode 100644 test/admission-without-smart-extraction.test.mjs diff --git a/dist/index.js b/dist/index.js index baebf86a1..42256e7a6 100644 --- a/dist/index.js +++ b/dist/index.js @@ -51,7 +51,7 @@ import { createMemoryUpgrader } from "./src/memory-upgrader.js"; import { buildSmartMetadata, parseSmartMetadata, stringifySmartMetadata, toLifecycleMemory, } from "./src/smart-metadata.js"; import { computeTier1Patch, isSuppressed as isTier1Suppressed, TIER1_DEFAULT_BAD_RECALL_DECAY_MS, TIER1_DEFAULT_SUPPRESSION_DURATION_MS, } from "./src/auto-recall-tier1.js"; import { filterUserMdExclusiveRecallResults, isUserMdExclusiveMemory, } from "./src/workspace-boundary.js"; -import { normalizeAdmissionControlConfig, resolveRejectedAuditFilePath, } from "./src/admission-control.js"; +import { normalizeAdmissionControlConfig, createAdmissionController, resolveRejectedAuditFilePath, } from "./src/admission-control.js"; import { analyzeIntent, applyCategoryBoost } from "./src/intent-analyzer.js"; import { createOpenClawMemoryCapability } from "./src/openclaw-memory-capability.js"; import { CanonicalCorpusIndexer, parseCanonicalCorpusConfig, } from "./src/corpus-indexer.js"; @@ -368,7 +368,6 @@ const DEFAULT_REFLECTION_ERROR_SCAN_MAX_CHARS = 8_000; const DEFAULT_SERIAL_GUARD_COOLDOWN_MS = 120_000; const DEFAULT_REFLECTION_EMPTY_EVENT_GUARD_TTL_MS = 120_000; const DEFAULT_REFLECTION_EMPTY_EVENT_GUARD_MAX_ENTRIES = 200; -const DEFAULT_REFLECTION_CACHE_TTL_MS = 15_000; // After /new or /reset, the just-closed session may have generated fresh // derived deltas. Keep those out of the immediately opened prompt window. const DEFAULT_REFLECTION_BOUNDARY_DERIVED_SUPPRESSION_MS = 120_000; @@ -1811,7 +1810,8 @@ function _initPluginState(api) { // callback below closes over it. const mdMirror = createMdMirrorWriter(api, config); let smartExtractor = null; - if (config.smartExtraction !== false) { + let admissionController = null; + if (config.smartExtraction !== false || config.admissionControl.enabled === true) { try { const llmAuth = config.llm?.auth || "api-key"; const llmApiKey = llmAuth === "oauth" @@ -1841,30 +1841,42 @@ function _initPluginState(api) { log: (msg) => api.logger.debug(msg), warnLog: (msg) => api.logger.warn(msg), }); - const noiseBank = new NoisePrototypeBank((msg) => api.logger.debug(msg)); - noiseBank.init(embedder).catch((err) => api.logger.debug(`memory-lancedb-pro: noise bank init: ${String(err)}`)); - const admissionRejectionAuditWriter = createAdmissionRejectionAuditWriter(config, resolvedDbPath, api); - smartExtractor = new SmartExtractor(store, embedder, llmClient, { - user: "User", - extractMinMessages: config.extractMinMessages ?? 4, - extractMaxChars: config.extractMaxChars ?? 8000, - defaultScope: config.scopes?.default ?? "global", - workspaceBoundary: config.workspaceBoundary, - admissionControl: config.admissionControl, - onAdmissionRejected: admissionRejectionAuditWriter ?? undefined, - onPersisted: mdMirror ?? undefined, - log: (msg) => api.logger.info(msg), - debugLog: (msg) => api.logger.debug(msg), - noiseBank, - }); - (isCliMode() ? api.logger.debug : api.logger.info)("memory-lancedb-pro: smart extraction enabled (LLM model: " - + llmModel - + ", timeoutMs: " - + llmTimeoutMs - + ", noise bank: ON)"); + // Constructed independently of SmartExtractor so admission gating is + // available to other write paths (e.g. reflection-mapped rows, the + // regex fallback) even when smart extraction itself is disabled. + admissionController = createAdmissionController(store, llmClient, config.admissionControl, (msg) => api.logger.debug(msg)); + if (config.smartExtraction !== false) { + const noiseBank = new NoisePrototypeBank((msg) => api.logger.debug(msg)); + noiseBank.init(embedder).catch((err) => api.logger.debug(`memory-lancedb-pro: noise bank init: ${String(err)}`)); + const admissionRejectionAuditWriter = createAdmissionRejectionAuditWriter(config, resolvedDbPath, api); + smartExtractor = new SmartExtractor(store, embedder, llmClient, { + user: "User", + extractMinMessages: config.extractMinMessages ?? 4, + extractMaxChars: config.extractMaxChars ?? 8000, + defaultScope: config.scopes?.default ?? "global", + workspaceBoundary: config.workspaceBoundary, + admissionControl: config.admissionControl, + admissionController, + onAdmissionRejected: admissionRejectionAuditWriter ?? undefined, + onPersisted: mdMirror ?? undefined, + log: (msg) => api.logger.info(msg), + debugLog: (msg) => api.logger.debug(msg), + noiseBank, + }); + (isCliMode() ? api.logger.debug : api.logger.info)("memory-lancedb-pro: smart extraction enabled (LLM model: " + + llmModel + + ", timeoutMs: " + + llmTimeoutMs + + ", noise bank: ON)"); + } } catch (err) { - api.logger.warn(`memory-lancedb-pro: smart extraction init failed, falling back to regex: ${String(err)}`); + if (config.smartExtraction !== false) { + api.logger.warn(`memory-lancedb-pro: smart extraction init failed, falling back to regex: ${String(err)}`); + } + else { + api.logger.warn(`memory-lancedb-pro: standalone admission control init failed, admission gating unavailable: ${String(err)}`); + } } } const extractionRateLimiter = createExtractionRateLimiter({ @@ -1875,11 +1887,6 @@ function _initPluginState(api) { const reflectionDerivedBySession = new Map(); const reflectionDerivedSuppressionBySession = new Map(); const reflectionByAgentCache = new Map(); - // Bumped on every invalidateReflectionCachesAfterDelete call. loadAgentReflectionSlices - // snapshots this before its awaited store.list() reads and skips caching its result if - // it changed mid-flight, so an in-flight read can never publish a stale pre-delete - // snapshot back into the cache after the delete already invalidated it. - const reflectionByAgentCacheGeneration = { count: 0 }; const recallHistory = new Map(); const turnCounter = new Map(); const autoCaptureSeenTextCount = new Map(); @@ -1900,13 +1907,13 @@ function _initPluginState(api) { scopeManager, migrator, smartExtractor, + admissionController, mdMirror, extractionRateLimiter, reflectionErrorStateBySession, reflectionDerivedBySession, reflectionDerivedSuppressionBySession, reflectionByAgentCache, - reflectionByAgentCacheGeneration, recallHistory, turnCounter, autoCaptureSeenTextCount, @@ -2020,7 +2027,7 @@ const memoryLanceDBProPlugin = { _registeredApisMap.delete(api); // dual-track rollback: Map un-claim throw err; } - const { config, resolvedDbPath, vectorDim, store, embedder, retriever, canonicalCorpusIndexer, dreamingEngine, dreamingScheduler, scopeManager, migrator, smartExtractor, mdMirror, decayEngine, tierManager, extractionRateLimiter, reflectionErrorStateBySession, reflectionDerivedBySession, reflectionDerivedSuppressionBySession, reflectionByAgentCache, reflectionByAgentCacheGeneration, recallHistory, turnCounter, autoCaptureSeenTextCount, autoCapturePendingIngressTexts, autoCaptureRecentTexts, } = singleton; + const { config, resolvedDbPath, vectorDim, store, embedder, retriever, canonicalCorpusIndexer, dreamingEngine, dreamingScheduler, scopeManager, migrator, smartExtractor, admissionController, mdMirror, decayEngine, tierManager, extractionRateLimiter, reflectionErrorStateBySession, reflectionDerivedBySession, reflectionDerivedSuppressionBySession, reflectionByAgentCache, recallHistory, turnCounter, autoCaptureSeenTextCount, autoCapturePendingIngressTexts, autoCaptureRecentTexts, } = singleton; warnForDisabledChannelPlugin(api.config, api.logger); async function sleep(ms, signal) { if (signal?.aborted) { @@ -2183,9 +2190,8 @@ const memoryLanceDBProPlugin = { : ""; const cacheKey = `${agentId}::${scopeKey}`; const cached = reflectionByAgentCache.get(cacheKey); - if (cached && Date.now() - cached.updatedAt < DEFAULT_REFLECTION_CACHE_TTL_MS) + if (cached && Date.now() - cached.updatedAt < 15_000) return cached; - const generationAtStart = reflectionByAgentCacheGeneration.count; // Prefer reflection-category rows to avoid full-table reads on bypass callers. // Fall back to an uncategorized scan only when the category query produced no // agent-owned reflection slices, preserving backward compatibility with mixed-schema stores. @@ -2214,54 +2220,13 @@ const memoryLanceDBProPlugin = { } const { invariants, derived } = slices; const next = { updatedAt: Date.now(), invariants, derived }; - // Only cache if no delete invalidated this cacheKey while the awaits above were in - // flight (TOCTOU guard); otherwise this late-arriving, possibly-stale read would - // silently resurrect a cache entry the delete just cleared. - if (reflectionByAgentCacheGeneration.count === generationAtStart) { - reflectionByAgentCache.set(cacheKey, next); - } + reflectionByAgentCache.set(cacheKey, next); return next; }; - // Fast-path invalidation for SAME-PROCESS deletes only: CLI delete/delete-bulk - // commands run as a short-lived, separate process from the long-running Gateway - // in typical deployments, so this callback firing there does not reach (and - // cannot invalidate) the Gateway process own in-memory caches. It only has an - // effect when a delete genuinely happens inside this same plugin instance. - // - // The actual cross-process staleness bound comes from two other layers: - // - DEFAULT_REFLECTION_CACHE_TTL_MS bounds how long either cache below can - // serve stale content after ANY delete, same-process or not (see the read - // sites in loadAgentReflectionSlices and the derived-focus injector). - // - readConsistencyInterval (store config) bounds how long the underlying - // LanceDB table handle can serve stale rows to a fresh query in the first - // place, which is what a TTL-expired cache re-populates from. - // - // reflectionByAgentCache is keyed "::scopes:" (or - // "::"); drop any entry whose scope set intersects - // the deleted scopes, plus every no-scope-filter entry (it spans all scopes). - // reflectionDerivedBySession has no cheap scope-to-session mapping, so it is - // cleared in full rather than left to expire on its own TTL. - const invalidateReflectionCachesAfterDelete = (deletedScopes) => { - reflectionByAgentCacheGeneration.count++; - const deletedSet = new Set(deletedScopes ?? []); - for (const cacheKey of reflectionByAgentCache.keys()) { - const sepIdx = cacheKey.indexOf("::"); - const scopePart = sepIdx === -1 ? "" : cacheKey.slice(sepIdx + 2); - if (scopePart === "" || deletedSet.size === 0) { - reflectionByAgentCache.delete(cacheKey); - continue; - } - const cachedScopes = scopePart.startsWith("scopes:") ? scopePart.slice("scopes:".length).split(",") : []; - if (cachedScopes.some((s) => deletedSet.has(s))) { - reflectionByAgentCache.delete(cacheKey); - } - } - reflectionDerivedBySession.clear(); - }; const pendingRecall = new Map(); const logReg = isCliMode() ? api.logger.debug : api.logger.info; if (isFirstRegistration) { - logReg(`memory-lancedb-pro@${pluginVersion}: plugin registered (db: ${resolvedDbPath}, model: ${config.embedding.model || "text-embedding-3-small"}, smartExtraction: ${smartExtractor ? 'ON' : 'OFF'})`); + logReg(`memory-lancedb-pro@${pluginVersion}: plugin registered (db: ${resolvedDbPath}, model: ${config.embedding.model || "text-embedding-3-small"}, smartExtraction: ${smartExtractor ? 'ON' : 'OFF'}, admissionControl: ${admissionController ? 'ON' : 'OFF'})`); logReg(`memory-lancedb-pro: diagnostic build tag loaded (${DIAG_BUILD_TAG})`); } // Dual-memory model warning: help users understand the two-layer architecture @@ -2361,9 +2326,6 @@ const memoryLanceDBProPlugin = { mdMirror, workspaceBoundary: config.workspaceBoundary, selfImprovementMaxEntries: config.selfImprovement?.maxEntries, - // Mirrors the CLI context wiring below: keep in-process reflection caches - // consistent after a live memory_forget delete too, not just CLI delete/delete-bulk. - onMemoriesDeleted: ({ scopeFilter }) => invalidateReflectionCachesAfterDelete(scopeFilter), }, { enableManagementTools: config.enableManagementTools, enableSelfImprovementTools: config.selfImprovement?.enabled === true, @@ -2403,7 +2365,6 @@ const memoryLanceDBProPlugin = { store, retriever, scopeManager, - onMemoriesDeleted: ({ scopeFilter }) => invalidateReflectionCachesAfterDelete(scopeFilter), migrator, embedder, llmClient: smartExtractor ? (() => { @@ -3590,8 +3551,7 @@ const memoryLanceDBProPlugin = { reflectionDerivedSuppressionBySession.delete(sessionKey); const scopes = resolveScopeFilter(scopeManager, agentId); const derivedCache = sessionKey ? reflectionDerivedBySession.get(sessionKey) : null; - const derivedCacheFresh = derivedCache && Date.now() - derivedCache.updatedAt < DEFAULT_REFLECTION_CACHE_TTL_MS; - const derivedLines = derivedCacheFresh && derivedCache.derived.length + const derivedLines = derivedCache?.derived?.length ? derivedCache.derived : (await loadAgentReflectionSlices(agentId, scopes)).derived; if (derivedLines.length > 0) { @@ -3987,13 +3947,7 @@ const memoryLanceDBProPlugin = { }); if (sessionKey && stored.slices.derived.length > 0 && !isSessionBoundaryReflectionAction(action)) { reflectionDerivedBySession.set(sessionKey, { - // Deliberately Date.now(), not nowTs (which mirrors the host-supplied - // event.timestamp and can be skewed/future-dated): this field is a TTL - // bookkeeping mark, and DEFAULT_REFLECTION_CACHE_TTL_MS above compares it - // against a fresh Date.now() on every read. A skewed updatedAt can make - // "Date.now() - updatedAt" go negative, which is always < the TTL, so the - // cache would read as fresh indefinitely until wall-clock time caught up. - updatedAt: Date.now(), + updatedAt: nowTs, derived: stored.slices.derived, }); } diff --git a/dist/src/admission-control.js b/dist/src/admission-control.js index 44733bb58..a42810dc3 100644 --- a/dist/src/admission-control.js +++ b/dist/src/admission-control.js @@ -426,6 +426,17 @@ async function scoreUtility(llm, mode, candidate, conversationText) { reason: typeof response.reason === "string" ? response.reason.trim() : undefined, }; } +/** + * Construct an AdmissionController independently of any extraction engine. + * Availability depends only on the admission config's `enabled` flag, so + * callers that never build a SmartExtractor (e.g. smartExtraction: false) + * can still obtain a working controller to gate other write paths. + */ +export function createAdmissionController(store, llm, config, debugLog = () => { }) { + return config?.enabled === true + ? new AdmissionController(store, llm, config, debugLog) + : null; +} export class AdmissionController { store; llm; diff --git a/dist/src/smart-extractor.js b/dist/src/smart-extractor.js index 43fb9f60e..5a2680beb 100644 --- a/dist/src/smart-extractor.js +++ b/dist/src/smart-extractor.js @@ -6,8 +6,7 @@ * */ import { buildExtractionPrompt, buildDedupPrompt, buildMergePrompt, } from "./extraction-prompts.js"; -import { AdmissionController, } from "./admission-control.js"; -import { ALWAYS_MERGE_CATEGORIES, getStorageCategoryForMemoryCategory, MERGE_SUPPORTED_CATEGORIES, TEMPORAL_VERSIONED_CATEGORIES, normalizeCategory, } from "./memory-categories.js"; +import { ALWAYS_MERGE_CATEGORIES, MERGE_SUPPORTED_CATEGORIES, TEMPORAL_VERSIONED_CATEGORIES, normalizeCategory, } from "./memory-categories.js"; import { isMetaFrustrationNoise, isNoise } from "./noise-filter.js"; import { appendRelation, buildSmartMetadata, deriveFactKey, parseSmartMetadata, stringifySmartMetadata, parseSupportInfo, updateSupportStats, } from "./smart-metadata.js"; import { isUserMdExclusiveMemory, } from "./workspace-boundary.js"; @@ -192,10 +191,7 @@ export class SmartExtractor { config.admissionControl.auditMetadata !== false; this.onAdmissionRejected = config.onAdmissionRejected; this.onPersisted = config.onPersisted; - this.admissionController = - config.admissionControl?.enabled === true - ? new AdmissionController(this.store, this.llm, config.admissionControl, this.debugLog) - : null; + this.admissionController = config.admissionController ?? null; } /** * Notify the onPersisted sink (e.g. markdown mirror) after a successful @@ -1149,17 +1145,23 @@ export class SmartExtractor { /** * Map 6-category to existing 5-category store type for backward compatibility. */ - /** - * Map a smart register onto its legacy storage category, delegating to the - * shared SMART_TO_STORAGE_CATEGORY constant (memory-categories) so the - * mapping has a single source of truth. Note: "reflection" is a legacy - * storage category minted only by the reflection writer and is deliberately - * absent from this map; smart extraction never produces reflection rows. - * The "other" fallback covers non-union values arriving from untyped - * callers at runtime, matching the old switch's default arm. - */ mapToStoreCategory(category) { - return getStorageCategoryForMemoryCategory(category) ?? "other"; + switch (category) { + case "profile": + return "fact"; + case "preferences": + return "preference"; + case "entities": + return "entity"; + case "events": + return "decision"; + case "cases": + return "fact"; + case "patterns": + return "other"; + default: + return "other"; + } } /** * Get default importance score by category. diff --git a/index.ts b/index.ts index 75efa8e8f..457ff1a31 100644 --- a/index.ts +++ b/index.ts @@ -98,9 +98,11 @@ import { } from "./src/workspace-boundary.js"; import { normalizeAdmissionControlConfig, + createAdmissionController, resolveRejectedAuditFilePath, type AdmissionControlConfig, type AdmissionRejectionAuditEntry, + type AdmissionController, } from "./src/admission-control.js"; import { analyzeIntent, applyCategoryBoost } from "./src/intent-analyzer.js"; import { createOpenClawMemoryCapability } from "./src/openclaw-memory-capability.js"; @@ -2337,6 +2339,7 @@ interface PluginSingletonState { scopeManager: ReturnType; migrator: ReturnType; smartExtractor: SmartExtractor | null; + admissionController: AdmissionController | null; mdMirror: MdMirrorWriter | null; extractionRateLimiter: ReturnType; // Session Maps — persist across scope refreshes instead of being recreated @@ -2469,71 +2472,89 @@ function _initPluginState(api: OpenClawPluginApi): PluginSingletonState { // callback below closes over it. const mdMirror = createMdMirrorWriter(api, config); - let smartExtractor: SmartExtractor | null = null; - if (config.smartExtraction !== false) { - try { + let smartExtractor: SmartExtractor | null = null; + let admissionController: AdmissionController | null = null; + if (config.smartExtraction !== false || config.admissionControl.enabled === true) { + try { const llmAuth = config.llm?.auth || "api-key"; const llmApiKey = llmAuth === "oauth" ? undefined : config.llm?.apiKey ? resolveSecretCredential(api, config.llm.apiKey, "llm.apiKey") : resolveFirstApiKey(api, config.embedding.apiKey); - const llmBaseURL = llmAuth === "oauth" - ? (config.llm?.baseURL ? resolveEnvVars(config.llm.baseURL) : undefined) - : config.llm?.baseURL - ? resolveEnvVars(config.llm.baseURL) - : config.embedding.baseURL; - const llmModel = config.llm?.model || "openai/gpt-oss-120b"; - const llmOauthPath = llmAuth === "oauth" - ? resolveOptionalPathWithEnv(api, config.llm?.oauthPath, ".memory-lancedb-pro/oauth.json") - : undefined; - const llmOauthProvider = llmAuth === "oauth" ? config.llm?.oauthProvider : undefined; - const llmTimeoutMs = resolveLlmTimeoutMs(config); - - const llmClient = createLlmClient({ - auth: llmAuth, - apiKey: llmApiKey, - model: llmModel, - baseURL: llmBaseURL, - oauthProvider: llmOauthProvider, - oauthPath: llmOauthPath, - timeoutMs: llmTimeoutMs, - log: (msg: string) => api.logger.debug(msg), - warnLog: (msg: string) => api.logger.warn(msg), - }); - - const noiseBank = new NoisePrototypeBank((msg: string) => api.logger.debug(msg)); - noiseBank.init(embedder).catch((err) => - api.logger.debug(`memory-lancedb-pro: noise bank init: ${String(err)}`), - ); - - const admissionRejectionAuditWriter = createAdmissionRejectionAuditWriter(config, resolvedDbPath, api); - - smartExtractor = new SmartExtractor(store, embedder, llmClient, { - user: "User", - extractMinMessages: config.extractMinMessages ?? 4, - extractMaxChars: config.extractMaxChars ?? 8000, - defaultScope: config.scopes?.default ?? "global", - workspaceBoundary: config.workspaceBoundary, - admissionControl: config.admissionControl, - onAdmissionRejected: admissionRejectionAuditWriter ?? undefined, - onPersisted: mdMirror ?? undefined, - log: (msg: string) => api.logger.info(msg), - debugLog: (msg: string) => api.logger.debug(msg), - noiseBank, - }); - - (isCliMode() ? api.logger.debug : api.logger.info)( - "memory-lancedb-pro: smart extraction enabled (LLM model: " - + llmModel - + ", timeoutMs: " - + llmTimeoutMs - + ", noise bank: ON)", - ); - } catch (err) { - api.logger.warn(`memory-lancedb-pro: smart extraction init failed, falling back to regex: ${String(err)}`); - } - } + const llmBaseURL = llmAuth === "oauth" + ? (config.llm?.baseURL ? resolveEnvVars(config.llm.baseURL) : undefined) + : config.llm?.baseURL + ? resolveEnvVars(config.llm.baseURL) + : config.embedding.baseURL; + const llmModel = config.llm?.model || "openai/gpt-oss-120b"; + const llmOauthPath = llmAuth === "oauth" + ? resolveOptionalPathWithEnv(api, config.llm?.oauthPath, ".memory-lancedb-pro/oauth.json") + : undefined; + const llmOauthProvider = llmAuth === "oauth" ? config.llm?.oauthProvider : undefined; + const llmTimeoutMs = resolveLlmTimeoutMs(config); + + const llmClient = createLlmClient({ + auth: llmAuth, + apiKey: llmApiKey, + model: llmModel, + baseURL: llmBaseURL, + oauthProvider: llmOauthProvider, + oauthPath: llmOauthPath, + timeoutMs: llmTimeoutMs, + log: (msg: string) => api.logger.debug(msg), + warnLog: (msg: string) => api.logger.warn(msg), + }); + + // Constructed independently of SmartExtractor so admission gating is + // available to other write paths (e.g. reflection-mapped rows, the + // regex fallback) even when smart extraction itself is disabled. + admissionController = createAdmissionController( + store, + llmClient, + config.admissionControl, + (msg: string) => api.logger.debug(msg), + ); + + if (config.smartExtraction !== false) { + const noiseBank = new NoisePrototypeBank((msg: string) => api.logger.debug(msg)); + noiseBank.init(embedder).catch((err) => + api.logger.debug(`memory-lancedb-pro: noise bank init: ${String(err)}`), + ); + + const admissionRejectionAuditWriter = createAdmissionRejectionAuditWriter(config, resolvedDbPath, api); + + smartExtractor = new SmartExtractor(store, embedder, llmClient, { + user: "User", + extractMinMessages: config.extractMinMessages ?? 4, + extractMaxChars: config.extractMaxChars ?? 8000, + defaultScope: config.scopes?.default ?? "global", + workspaceBoundary: config.workspaceBoundary, + admissionControl: config.admissionControl, + admissionController, + onAdmissionRejected: admissionRejectionAuditWriter ?? undefined, + onPersisted: mdMirror ?? undefined, + log: (msg: string) => api.logger.info(msg), + debugLog: (msg: string) => api.logger.debug(msg), + noiseBank, + }); + + (isCliMode() ? api.logger.debug : api.logger.info)( + "memory-lancedb-pro: smart extraction enabled (LLM model: " + + llmModel + + ", timeoutMs: " + + llmTimeoutMs + + ", noise bank: ON)", + ); + } + } catch (err) { + if (config.smartExtraction !== false) { + api.logger.warn(`memory-lancedb-pro: smart extraction init failed, falling back to regex: ${String(err)}`); + } else { + api.logger.warn(`memory-lancedb-pro: standalone admission control init failed, admission gating unavailable: ${String(err)}`); + } + } + } const extractionRateLimiter = createExtractionRateLimiter({ maxExtractionsPerHour: config.extractionThrottle?.maxExtractionsPerHour, @@ -2570,6 +2591,7 @@ function _initPluginState(api: OpenClawPluginApi): PluginSingletonState { scopeManager, migrator, smartExtractor, + admissionController, mdMirror, extractionRateLimiter, reflectionErrorStateBySession, @@ -2721,6 +2743,7 @@ const memoryLanceDBProPlugin = { scopeManager, migrator, smartExtractor, + admissionController, mdMirror, decayEngine, tierManager, @@ -3035,7 +3058,7 @@ const memoryLanceDBProPlugin = { const logReg = isCliMode() ? api.logger.debug : api.logger.info; if (isFirstRegistration) { logReg( - `memory-lancedb-pro@${pluginVersion}: plugin registered (db: ${resolvedDbPath}, model: ${config.embedding.model || "text-embedding-3-small"}, smartExtraction: ${smartExtractor ? 'ON' : 'OFF'})` + `memory-lancedb-pro@${pluginVersion}: plugin registered (db: ${resolvedDbPath}, model: ${config.embedding.model || "text-embedding-3-small"}, smartExtraction: ${smartExtractor ? 'ON' : 'OFF'}, admissionControl: ${admissionController ? 'ON' : 'OFF'})` ); logReg(`memory-lancedb-pro: diagnostic build tag loaded (${DIAG_BUILD_TAG})`); } diff --git a/package.json b/package.json index dfeb2daa5..518b23887 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "skills/**/*.md" ], "scripts": { - "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/delete-invalidate-reflection-caches.test.mjs", + "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/admission-controller-standalone.test.mjs && node --test test/smart-extractor-admission-controller-injection.test.mjs && node --test test/admission-without-smart-extraction.test.mjs", "test:cli-smoke": "node scripts/run-ci-tests.mjs --group cli-smoke", "test:core-regression": "node scripts/run-ci-tests.mjs --group core-regression", "test:storage-and-schema": "node scripts/run-ci-tests.mjs --group storage-and-schema", diff --git a/scripts/ci-test-manifest.mjs b/scripts/ci-test-manifest.mjs index c71de5313..80297f731 100644 --- a/scripts/ci-test-manifest.mjs +++ b/scripts/ci-test-manifest.mjs @@ -107,8 +107,9 @@ export const CI_TEST_MANIFEST = [ { group: "core-regression", runner: "node", file: "test/autocapture-watermark-reset.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/autocapture-internal-session-guard.test.mjs", args: ["--test"] }, { group: "storage-and-schema", runner: "node", file: "test/memory-categories-storage-map.test.mjs", args: ["--test"] }, - // Delete/delete-bulk must synchronously invalidate in-process reflection read caches - { group: "core-regression", runner: "node", file: "test/delete-invalidate-reflection-caches.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/admission-controller-standalone.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/smart-extractor-admission-controller-injection.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/admission-without-smart-extraction.test.mjs", args: ["--test"] }, ]; export function getEntriesForGroup(group) { diff --git a/test/admission-without-smart-extraction.test.mjs b/test/admission-without-smart-extraction.test.mjs new file mode 100644 index 000000000..59472bc70 --- /dev/null +++ b/test/admission-without-smart-extraction.test.mjs @@ -0,0 +1,184 @@ +import { describe, it, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import jitiFactory from "jiti"; + +const testDir = path.dirname(fileURLToPath(import.meta.url)); +const pluginSdkStubPath = path.resolve(testDir, "helpers", "openclaw-plugin-sdk-stub.mjs"); +const jiti = jitiFactory(import.meta.url, { + interopDefault: true, + alias: { + "openclaw/plugin-sdk": pluginSdkStubPath, + }, +}); + +const retrieverModuleForMock = jiti("../src/retriever.js"); +const embedderModuleForMock = jiti("../src/embedder.js"); +const origCreateRetriever = retrieverModuleForMock.createRetriever; +const origCreateEmbedder = embedderModuleForMock.createEmbedder; + +const pluginModule = jiti("../index.ts"); +const memoryLanceDBProPlugin = pluginModule.default || pluginModule; +const { resetRegistration } = pluginModule; + +function mockCreateRetriever() { + return function mockCreateRetrieverImpl() { + return { + async retrieve() { + return []; + }, + getConfig() { + return { mode: "hybrid" }; + }, + setAccessTracker() {}, + setStatsCollector() {}, + }; + }; +} + +function mockCreateEmbedder() { + return function mockCreateEmbedderImpl() { + return { + async embedQuery() { + return new Float32Array(384).fill(0); + }, + async embedPassage() { + return new Float32Array(384).fill(0); + }, + }; + }; +} + +function createPluginApiHarness({ pluginConfig, resolveRoot }) { + const eventHandlers = new Map(); + const logs = { info: [], warn: [], debug: [] }; + + const api = { + pluginConfig, + resolvePath(target) { + if (typeof target !== "string") return target; + if (path.isAbsolute(target)) return target; + return path.join(resolveRoot, target); + }, + logger: { + info(message) { + logs.info.push(String(message)); + }, + warn(message) { + logs.warn.push(String(message)); + }, + debug(message) { + logs.debug.push(String(message)); + }, + error(message) { + logs.info.push(String(message)); + }, + }, + registerTool() {}, + registerCli() {}, + registerService() {}, + on(eventName, handler, meta) { + const list = eventHandlers.get(eventName) || []; + list.push({ handler, meta }); + eventHandlers.set(eventName, list); + }, + registerHook(eventName, handler, opts) { + const list = eventHandlers.get(eventName) || []; + list.push({ handler, meta: opts }); + eventHandlers.set(eventName, list); + }, + }; + + return { api, eventHandlers, logs }; +} + +function findRegisteredLog(logs) { + return [...logs.info, ...logs.debug].find((l) => l.includes("plugin registered")); +} + +describe("admission control availability without smart extraction", () => { + let workspaceDir; + + beforeEach(() => { + workspaceDir = mkdtempSync(path.join(tmpdir(), "admission-no-extraction-")); + retrieverModuleForMock.createRetriever = mockCreateRetriever(); + embedderModuleForMock.createEmbedder = mockCreateEmbedder(); + resetRegistration(); + }); + + afterEach(() => { + retrieverModuleForMock.createRetriever = origCreateRetriever; + embedderModuleForMock.createEmbedder = origCreateEmbedder; + resetRegistration(); + rmSync(workspaceDir, { recursive: true, force: true }); + }); + + it("constructs a standalone admission controller when smartExtraction is off but admissionControl is enabled", () => { + const harness = createPluginApiHarness({ + resolveRoot: workspaceDir, + pluginConfig: { + dbPath: path.join(workspaceDir, "db"), + embedding: { apiKey: "test-api-key" }, + smartExtraction: false, + admissionControl: { enabled: true }, + autoCapture: false, + autoRecall: false, + selfImprovement: { enabled: false, beforeResetNote: false, ensureLearningFiles: false }, + }, + }); + + memoryLanceDBProPlugin.register(harness.api); + + const registeredLog = findRegisteredLog(harness.logs); + assert.ok(registeredLog, "expected a 'plugin registered' log line"); + assert.match(registeredLog, /smartExtraction: OFF/); + assert.match(registeredLog, /admissionControl: ON/); + }); + + it("leaves admission control unavailable when it is disabled, even with smart extraction off (configured off means off)", () => { + const harness = createPluginApiHarness({ + resolveRoot: workspaceDir, + pluginConfig: { + dbPath: path.join(workspaceDir, "db"), + embedding: { apiKey: "test-api-key" }, + smartExtraction: false, + admissionControl: { enabled: false }, + autoCapture: false, + autoRecall: false, + selfImprovement: { enabled: false, beforeResetNote: false, ensureLearningFiles: false }, + }, + }); + + memoryLanceDBProPlugin.register(harness.api); + + const registeredLog = findRegisteredLog(harness.logs); + assert.ok(registeredLog); + assert.match(registeredLog, /smartExtraction: OFF/); + assert.match(registeredLog, /admissionControl: OFF/); + }); + + it("keeps smartExtraction: true behavior unchanged (both ON when admission is enabled)", () => { + const harness = createPluginApiHarness({ + resolveRoot: workspaceDir, + pluginConfig: { + dbPath: path.join(workspaceDir, "db"), + embedding: { apiKey: "test-api-key" }, + smartExtraction: true, + admissionControl: { enabled: true }, + autoCapture: false, + autoRecall: false, + selfImprovement: { enabled: false, beforeResetNote: false, ensureLearningFiles: false }, + }, + }); + + memoryLanceDBProPlugin.register(harness.api); + + const registeredLog = findRegisteredLog(harness.logs); + assert.ok(registeredLog); + assert.match(registeredLog, /smartExtraction: ON/); + assert.match(registeredLog, /admissionControl: ON/); + }); +}); From 1d6c17466410302ca01c0bf6a29402961ceee505 Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sat, 18 Jul 2026 00:36:46 +0300 Subject: [PATCH 4/5] chore(dist): rebuild after rebase onto master Co-Authored-By: Claude Fable 5 --- dist/src/smart-extractor.js | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/dist/src/smart-extractor.js b/dist/src/smart-extractor.js index 5a2680beb..06400029a 100644 --- a/dist/src/smart-extractor.js +++ b/dist/src/smart-extractor.js @@ -6,7 +6,7 @@ * */ import { buildExtractionPrompt, buildDedupPrompt, buildMergePrompt, } from "./extraction-prompts.js"; -import { ALWAYS_MERGE_CATEGORIES, MERGE_SUPPORTED_CATEGORIES, TEMPORAL_VERSIONED_CATEGORIES, normalizeCategory, } from "./memory-categories.js"; +import { ALWAYS_MERGE_CATEGORIES, getStorageCategoryForMemoryCategory, MERGE_SUPPORTED_CATEGORIES, TEMPORAL_VERSIONED_CATEGORIES, normalizeCategory, } from "./memory-categories.js"; import { isMetaFrustrationNoise, isNoise } from "./noise-filter.js"; import { appendRelation, buildSmartMetadata, deriveFactKey, parseSmartMetadata, stringifySmartMetadata, parseSupportInfo, updateSupportStats, } from "./smart-metadata.js"; import { isUserMdExclusiveMemory, } from "./workspace-boundary.js"; @@ -1145,23 +1145,17 @@ export class SmartExtractor { /** * Map 6-category to existing 5-category store type for backward compatibility. */ + /** + * Map a smart register onto its legacy storage category, delegating to the + * shared SMART_TO_STORAGE_CATEGORY constant (memory-categories) so the + * mapping has a single source of truth. Note: "reflection" is a legacy + * storage category minted only by the reflection writer and is deliberately + * absent from this map; smart extraction never produces reflection rows. + * The "other" fallback covers non-union values arriving from untyped + * callers at runtime, matching the old switch's default arm. + */ mapToStoreCategory(category) { - switch (category) { - case "profile": - return "fact"; - case "preferences": - return "preference"; - case "entities": - return "entity"; - case "events": - return "decision"; - case "cases": - return "fact"; - case "patterns": - return "other"; - default: - return "other"; - } + return getStorageCategoryForMemoryCategory(category) ?? "other"; } /** * Get default importance score by category. From 7c097d772df16e78499d9cd71f15fe31cc7343dd Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sat, 18 Jul 2026 18:51:20 +0300 Subject: [PATCH 5/5] chore(dist): rebuild on current master --- dist/index.js | 79 ++++++++++++-- package.json | 2 +- scripts/ci-test-manifest.mjs | 192 ++++++++++++++++++----------------- 3 files changed, 170 insertions(+), 103 deletions(-) diff --git a/dist/index.js b/dist/index.js index 42256e7a6..d2204b3f9 100644 --- a/dist/index.js +++ b/dist/index.js @@ -368,6 +368,7 @@ const DEFAULT_REFLECTION_ERROR_SCAN_MAX_CHARS = 8_000; const DEFAULT_SERIAL_GUARD_COOLDOWN_MS = 120_000; const DEFAULT_REFLECTION_EMPTY_EVENT_GUARD_TTL_MS = 120_000; const DEFAULT_REFLECTION_EMPTY_EVENT_GUARD_MAX_ENTRIES = 200; +const DEFAULT_REFLECTION_CACHE_TTL_MS = 15_000; // After /new or /reset, the just-closed session may have generated fresh // derived deltas. Keep those out of the immediately opened prompt window. const DEFAULT_REFLECTION_BOUNDARY_DERIVED_SUPPRESSION_MS = 120_000; @@ -1887,6 +1888,11 @@ function _initPluginState(api) { const reflectionDerivedBySession = new Map(); const reflectionDerivedSuppressionBySession = new Map(); const reflectionByAgentCache = new Map(); + // Bumped on every invalidateReflectionCachesAfterDelete call. loadAgentReflectionSlices + // snapshots this before its awaited store.list() reads and skips caching its result if + // it changed mid-flight, so an in-flight read can never publish a stale pre-delete + // snapshot back into the cache after the delete already invalidated it. + const reflectionByAgentCacheGeneration = { count: 0 }; const recallHistory = new Map(); const turnCounter = new Map(); const autoCaptureSeenTextCount = new Map(); @@ -1914,6 +1920,7 @@ function _initPluginState(api) { reflectionDerivedBySession, reflectionDerivedSuppressionBySession, reflectionByAgentCache, + reflectionByAgentCacheGeneration, recallHistory, turnCounter, autoCaptureSeenTextCount, @@ -2027,7 +2034,7 @@ const memoryLanceDBProPlugin = { _registeredApisMap.delete(api); // dual-track rollback: Map un-claim throw err; } - const { config, resolvedDbPath, vectorDim, store, embedder, retriever, canonicalCorpusIndexer, dreamingEngine, dreamingScheduler, scopeManager, migrator, smartExtractor, admissionController, mdMirror, decayEngine, tierManager, extractionRateLimiter, reflectionErrorStateBySession, reflectionDerivedBySession, reflectionDerivedSuppressionBySession, reflectionByAgentCache, recallHistory, turnCounter, autoCaptureSeenTextCount, autoCapturePendingIngressTexts, autoCaptureRecentTexts, } = singleton; + const { config, resolvedDbPath, vectorDim, store, embedder, retriever, canonicalCorpusIndexer, dreamingEngine, dreamingScheduler, scopeManager, migrator, smartExtractor, admissionController, mdMirror, decayEngine, tierManager, extractionRateLimiter, reflectionErrorStateBySession, reflectionDerivedBySession, reflectionDerivedSuppressionBySession, reflectionByAgentCache, reflectionByAgentCacheGeneration, recallHistory, turnCounter, autoCaptureSeenTextCount, autoCapturePendingIngressTexts, autoCaptureRecentTexts, } = singleton; warnForDisabledChannelPlugin(api.config, api.logger); async function sleep(ms, signal) { if (signal?.aborted) { @@ -2190,8 +2197,9 @@ const memoryLanceDBProPlugin = { : ""; const cacheKey = `${agentId}::${scopeKey}`; const cached = reflectionByAgentCache.get(cacheKey); - if (cached && Date.now() - cached.updatedAt < 15_000) + if (cached && Date.now() - cached.updatedAt < DEFAULT_REFLECTION_CACHE_TTL_MS) return cached; + const generationAtStart = reflectionByAgentCacheGeneration.count; // Prefer reflection-category rows to avoid full-table reads on bypass callers. // Fall back to an uncategorized scan only when the category query produced no // agent-owned reflection slices, preserving backward compatibility with mixed-schema stores. @@ -2220,9 +2228,50 @@ const memoryLanceDBProPlugin = { } const { invariants, derived } = slices; const next = { updatedAt: Date.now(), invariants, derived }; - reflectionByAgentCache.set(cacheKey, next); + // Only cache if no delete invalidated this cacheKey while the awaits above were in + // flight (TOCTOU guard); otherwise this late-arriving, possibly-stale read would + // silently resurrect a cache entry the delete just cleared. + if (reflectionByAgentCacheGeneration.count === generationAtStart) { + reflectionByAgentCache.set(cacheKey, next); + } return next; }; + // Fast-path invalidation for SAME-PROCESS deletes only: CLI delete/delete-bulk + // commands run as a short-lived, separate process from the long-running Gateway + // in typical deployments, so this callback firing there does not reach (and + // cannot invalidate) the Gateway process own in-memory caches. It only has an + // effect when a delete genuinely happens inside this same plugin instance. + // + // The actual cross-process staleness bound comes from two other layers: + // - DEFAULT_REFLECTION_CACHE_TTL_MS bounds how long either cache below can + // serve stale content after ANY delete, same-process or not (see the read + // sites in loadAgentReflectionSlices and the derived-focus injector). + // - readConsistencyInterval (store config) bounds how long the underlying + // LanceDB table handle can serve stale rows to a fresh query in the first + // place, which is what a TTL-expired cache re-populates from. + // + // reflectionByAgentCache is keyed "::scopes:" (or + // "::"); drop any entry whose scope set intersects + // the deleted scopes, plus every no-scope-filter entry (it spans all scopes). + // reflectionDerivedBySession has no cheap scope-to-session mapping, so it is + // cleared in full rather than left to expire on its own TTL. + const invalidateReflectionCachesAfterDelete = (deletedScopes) => { + reflectionByAgentCacheGeneration.count++; + const deletedSet = new Set(deletedScopes ?? []); + for (const cacheKey of reflectionByAgentCache.keys()) { + const sepIdx = cacheKey.indexOf("::"); + const scopePart = sepIdx === -1 ? "" : cacheKey.slice(sepIdx + 2); + if (scopePart === "" || deletedSet.size === 0) { + reflectionByAgentCache.delete(cacheKey); + continue; + } + const cachedScopes = scopePart.startsWith("scopes:") ? scopePart.slice("scopes:".length).split(",") : []; + if (cachedScopes.some((s) => deletedSet.has(s))) { + reflectionByAgentCache.delete(cacheKey); + } + } + reflectionDerivedBySession.clear(); + }; const pendingRecall = new Map(); const logReg = isCliMode() ? api.logger.debug : api.logger.info; if (isFirstRegistration) { @@ -2326,6 +2375,9 @@ const memoryLanceDBProPlugin = { mdMirror, workspaceBoundary: config.workspaceBoundary, selfImprovementMaxEntries: config.selfImprovement?.maxEntries, + // Mirrors the CLI context wiring below: keep in-process reflection caches + // consistent after a live memory_forget delete too, not just CLI delete/delete-bulk. + onMemoriesDeleted: ({ scopeFilter }) => invalidateReflectionCachesAfterDelete(scopeFilter), }, { enableManagementTools: config.enableManagementTools, enableSelfImprovementTools: config.selfImprovement?.enabled === true, @@ -2365,6 +2417,7 @@ const memoryLanceDBProPlugin = { store, retriever, scopeManager, + onMemoriesDeleted: ({ scopeFilter }) => invalidateReflectionCachesAfterDelete(scopeFilter), migrator, embedder, llmClient: smartExtractor ? (() => { @@ -3551,7 +3604,8 @@ const memoryLanceDBProPlugin = { reflectionDerivedSuppressionBySession.delete(sessionKey); const scopes = resolveScopeFilter(scopeManager, agentId); const derivedCache = sessionKey ? reflectionDerivedBySession.get(sessionKey) : null; - const derivedLines = derivedCache?.derived?.length + const derivedCacheFresh = derivedCache && Date.now() - derivedCache.updatedAt < DEFAULT_REFLECTION_CACHE_TTL_MS; + const derivedLines = derivedCacheFresh && derivedCache.derived.length ? derivedCache.derived : (await loadAgentReflectionSlices(agentId, scopes)).derived; if (derivedLines.length > 0) { @@ -3947,7 +4001,13 @@ const memoryLanceDBProPlugin = { }); if (sessionKey && stored.slices.derived.length > 0 && !isSessionBoundaryReflectionAction(action)) { reflectionDerivedBySession.set(sessionKey, { - updatedAt: nowTs, + // Deliberately Date.now(), not nowTs (which mirrors the host-supplied + // event.timestamp and can be skewed/future-dated): this field is a TTL + // bookkeeping mark, and DEFAULT_REFLECTION_CACHE_TTL_MS above compares it + // against a fresh Date.now() on every read. A skewed updatedAt can make + // "Date.now() - updatedAt" go negative, which is always < the TTL, so the + // cache would read as fresh indefinitely until wall-clock time caught up. + updatedAt: Date.now(), derived: stored.slices.derived, }); } @@ -4020,10 +4080,15 @@ const memoryLanceDBProPlugin = { const now = new Date(params.timestampMs ?? Date.now()); const dateStr = now.toISOString().split("T")[0]; const timeStr = now.toISOString().split("T")[1].split(".")[0]; + // Session key/id stay out of `text`: it is the FTS index surface, and + // the `simple` tokenizer splits a key like + // `agent:main:cron::run:` on its punctuation — so every session + // summary ends up indexed under `agent`, `main`, `cron`, `run`. A query + // mentioning any of those then BM25-matches every session summary in the + // store regardless of content. Both ids are already recorded structurally + // in metadata below, so provenance is unaffected. const memoryText = [ `Session: ${dateStr} ${timeStr} UTC`, - `Session Key: ${params.sessionKey}`, - `Session ID: ${params.sessionId}`, `Source: ${params.source}`, "", "Conversation Summary:", diff --git a/package.json b/package.json index 518b23887..bcc596b2c 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "skills/**/*.md" ], "scripts": { - "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/admission-controller-standalone.test.mjs && node --test test/smart-extractor-admission-controller-injection.test.mjs && node --test test/admission-without-smart-extraction.test.mjs", + "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/delete-invalidate-reflection-caches.test.mjs && node --test test/admission-controller-standalone.test.mjs && node --test test/smart-extractor-admission-controller-injection.test.mjs && node --test test/admission-without-smart-extraction.test.mjs", "test:cli-smoke": "node scripts/run-ci-tests.mjs --group cli-smoke", "test:core-regression": "node scripts/run-ci-tests.mjs --group core-regression", "test:storage-and-schema": "node scripts/run-ci-tests.mjs --group storage-and-schema", diff --git a/scripts/ci-test-manifest.mjs b/scripts/ci-test-manifest.mjs index 80297f731..26d1fa69a 100644 --- a/scripts/ci-test-manifest.mjs +++ b/scripts/ci-test-manifest.mjs @@ -1,35 +1,35 @@ -export const CI_TEST_GROUPS = [ - "cli-smoke", - "core-regression", - "storage-and-schema", - "llm-clients-and-auth", - "packaging-and-workflow", -]; - +export const CI_TEST_GROUPS = [ + "cli-smoke", + "core-regression", + "storage-and-schema", + "llm-clients-and-auth", + "packaging-and-workflow", +]; + export const CI_TEST_MANIFEST = [ { group: "llm-clients-and-auth", runner: "node", file: "test/embedder-error-hints.test.mjs" }, { group: "llm-clients-and-auth", runner: "node", file: "test/embedder-max-input-chars.test.mjs", args: ["--test"] }, { group: "llm-clients-and-auth", runner: "node", file: "test/cjk-recursion-regression.test.mjs" }, - { group: "storage-and-schema", runner: "node", file: "test/migrate-legacy-schema.test.mjs" }, - { group: "storage-and-schema", runner: "node", file: "test/config-session-strategy-migration.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/scope-access-undefined.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/reflection-bypass-hook.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-scope-filter.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/store-empty-scope-filter.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/store-timestamp-normalization.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/storage-path-normalization.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/storage-maintenance.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/read-consistency-interval.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/fts-index-fold.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/store-list-stats-projection-fallback.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/recall-text-cleanup.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/category-filter-normalization.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/update-consistency-lancedb.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/strip-envelope-metadata.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/auto-recall-timeout.test.mjs", args: ["--test"] }, - { group: "cli-smoke", runner: "node", file: "test/import-markdown/import-markdown.test.mjs", args: ["--test"] }, - { group: "cli-smoke", runner: "node", file: "test/cli-smoke.mjs" }, - { group: "cli-smoke", runner: "node", file: "test/functional-e2e.mjs" }, + { group: "storage-and-schema", runner: "node", file: "test/migrate-legacy-schema.test.mjs" }, + { group: "storage-and-schema", runner: "node", file: "test/config-session-strategy-migration.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/scope-access-undefined.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/reflection-bypass-hook.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-scope-filter.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/store-empty-scope-filter.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/store-timestamp-normalization.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/storage-path-normalization.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/storage-maintenance.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/read-consistency-interval.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/fts-index-fold.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/store-list-stats-projection-fallback.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/recall-text-cleanup.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/category-filter-normalization.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/update-consistency-lancedb.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/strip-envelope-metadata.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/auto-recall-timeout.test.mjs", args: ["--test"] }, + { group: "cli-smoke", runner: "node", file: "test/import-markdown/import-markdown.test.mjs", args: ["--test"] }, + { group: "cli-smoke", runner: "node", file: "test/cli-smoke.mjs" }, + { group: "cli-smoke", runner: "node", file: "test/functional-e2e.mjs" }, { group: "storage-and-schema", runner: "node", file: "test/per-agent-auto-recall.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/retriever-rerank-regression.mjs" }, { group: "core-regression", runner: "node", file: "test/retriever-neighbor-enrichment.test.mjs", args: ["--test"] }, @@ -41,81 +41,83 @@ export const CI_TEST_MANIFEST = [ { group: "core-regression", runner: "node", file: "test/dreaming-engine.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/memory-governance-tools.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/corpus-indexer.test.mjs" }, - { group: "packaging-and-workflow", runner: "node", file: "test/plugin-manifest-regression.mjs" }, - { group: "packaging-and-workflow", runner: "node", file: "test/openclaw-twitter-source-recipe.test.mjs", args: ["--test"] }, - { group: "packaging-and-workflow", runner: "node", file: "test/package-runtime.test.mjs" }, - { group: "packaging-and-workflow", runner: "node", file: "test/release-readiness.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/session-summary-before-reset.test.mjs", args: ["--test"] }, + { group: "packaging-and-workflow", runner: "node", file: "test/plugin-manifest-regression.mjs" }, + { group: "packaging-and-workflow", runner: "node", file: "test/openclaw-twitter-source-recipe.test.mjs", args: ["--test"] }, + { group: "packaging-and-workflow", runner: "node", file: "test/package-runtime.test.mjs" }, + { group: "packaging-and-workflow", runner: "node", file: "test/release-readiness.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/session-summary-before-reset.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/self-improvement-reset-note.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/self-improvement.test.mjs", args: ["--test"] }, { group: "packaging-and-workflow", runner: "node", file: "test/sync-plugin-version.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/smart-metadata-v2.mjs" }, - { group: "storage-and-schema", runner: "node", file: "test/vector-search-cosine.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/context-support-e2e.mjs" }, + { group: "core-regression", runner: "node", file: "test/smart-metadata-v2.mjs" }, + { group: "storage-and-schema", runner: "node", file: "test/vector-search-cosine.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/context-support-e2e.mjs" }, { group: "core-regression", runner: "node", file: "test/temporal-facts.test.mjs" }, { group: "core-regression", runner: "node", file: "test/memory-fact-query.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/memory-update-supersede.test.mjs" }, - { group: "llm-clients-and-auth", runner: "node", file: "test/memory-upgrader-diagnostics.test.mjs" }, - { group: "llm-clients-and-auth", runner: "node", file: "test/llm-api-key-client.test.mjs", args: ["--test"] }, - { group: "llm-clients-and-auth", runner: "node", file: "test/llm-oauth-client.test.mjs", args: ["--test"] }, - { group: "llm-clients-and-auth", runner: "node", file: "test/cli-oauth-login.test.mjs", args: ["--test"] }, - { group: "packaging-and-workflow", runner: "node", file: "test/workflow-fork-guards.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/clawteam-scope.test.mjs", args: ["--test"] }, + { group: "llm-clients-and-auth", runner: "node", file: "test/memory-upgrader-diagnostics.test.mjs" }, + { group: "llm-clients-and-auth", runner: "node", file: "test/llm-api-key-client.test.mjs", args: ["--test"] }, + { group: "llm-clients-and-auth", runner: "node", file: "test/llm-oauth-client.test.mjs", args: ["--test"] }, + { group: "llm-clients-and-auth", runner: "node", file: "test/cli-oauth-login.test.mjs", args: ["--test"] }, + { group: "packaging-and-workflow", runner: "node", file: "test/workflow-fork-guards.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/clawteam-scope.test.mjs", args: ["--test"] }, { group: "storage-and-schema", runner: "node", file: "test/cross-process-lock.test.mjs", args: ["--test"] }, { group: "storage-and-schema", runner: "node", file: "test/redis-lock.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/lock-stress-test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/lock-release-on-error.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/preference-slots.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/is-latest-auto-supersede.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/temporal-awareness.test.mjs", args: ["--test"] }, - // Issue #598 regression tests - { group: "core-regression", runner: "node", file: "test/store-serialization.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/mmr-tiny.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/access-tracker-retry.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/embedder-cache.test.mjs" }, - // Issue #629 batch embedding fix - { group: "llm-clients-and-auth", runner: "node", file: "test/embedder-ollama-batch-routing.test.mjs" }, - // Issue #665 bulkStore tests - // Issue #690 cross-call batch accumulator tests - { group: "storage-and-schema", runner: "node", file: "test/issue-690-cross-call-batch.test.mjs", args: ["--test"] }, - // Issue #665 bulkStore tests (from upstream) - { group: "storage-and-schema", runner: "node", file: "test/bulk-store.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/bulk-store-edge-cases.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-bulk-store.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-bulk-store-edge-cases.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/store-importance-normalization.test.mjs", args: ["--test"] }, - // Issue #680 regression tests (from upstream) - { group: "core-regression", runner: "node", file: "test/memory-reflection-issue680-tdd.test.mjs", args: ["--test"] }, - // Issue #606 SDK migration Bug 2 regression tests - { group: "core-regression", runner: "node", file: "test/issue606_sdk-migration.test.mjs" }, - // PR #713 inference regression tests - inferProviderFromBaseURL + model fallback - { group: "core-regression", runner: "node", file: "test/infer-provider-from-baseurl.test.mjs" }, - // Issue #736 recall governance - isRecallUsed() unit tests - { group: "core-regression", runner: "node", file: "test/is-recall-used.test.mjs", args: ["--test"] }, - // Issue #492 agentId validation tests - { group: "core-regression", runner: "node", file: "test/agentid-validation.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/command-reflection-guard.test.mjs", args: ["--test"] }, - // Tier 1 memory counter fix - { group: "core-regression", runner: "node", file: "test/tier1-counters.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/memory-subsession-prompt-hooks.test.mjs", args: ["--test"] }, - // Reflection distiller sub-session must not receive auto-recall/injected blocks - { group: "core-regression", runner: "node", file: "test/reflection-distiller-hook-skip.test.mjs", args: ["--test"] }, - // register() re-registration hardening (scope cache-miss log/handler dedup) + { group: "core-regression", runner: "node", file: "test/lock-release-on-error.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/preference-slots.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/is-latest-auto-supersede.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/temporal-awareness.test.mjs", args: ["--test"] }, + // Issue #598 regression tests + { group: "core-regression", runner: "node", file: "test/store-serialization.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/mmr-tiny.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/access-tracker-retry.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/embedder-cache.test.mjs" }, + // Issue #629 batch embedding fix + { group: "llm-clients-and-auth", runner: "node", file: "test/embedder-ollama-batch-routing.test.mjs" }, + // Issue #665 bulkStore tests + // Issue #690 cross-call batch accumulator tests + { group: "storage-and-schema", runner: "node", file: "test/issue-690-cross-call-batch.test.mjs", args: ["--test"] }, + // Issue #665 bulkStore tests (from upstream) + { group: "storage-and-schema", runner: "node", file: "test/bulk-store.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/bulk-store-edge-cases.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-bulk-store.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-bulk-store-edge-cases.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/store-importance-normalization.test.mjs", args: ["--test"] }, + // Issue #680 regression tests (from upstream) + { group: "core-regression", runner: "node", file: "test/memory-reflection-issue680-tdd.test.mjs", args: ["--test"] }, + // Issue #606 SDK migration Bug 2 regression tests + { group: "core-regression", runner: "node", file: "test/issue606_sdk-migration.test.mjs" }, + // PR #713 inference regression tests - inferProviderFromBaseURL + model fallback + { group: "core-regression", runner: "node", file: "test/infer-provider-from-baseurl.test.mjs" }, + // Issue #736 recall governance - isRecallUsed() unit tests + { group: "core-regression", runner: "node", file: "test/is-recall-used.test.mjs", args: ["--test"] }, + // Issue #492 agentId validation tests + { group: "core-regression", runner: "node", file: "test/agentid-validation.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/command-reflection-guard.test.mjs", args: ["--test"] }, + // Tier 1 memory counter fix + { group: "core-regression", runner: "node", file: "test/tier1-counters.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/memory-subsession-prompt-hooks.test.mjs", args: ["--test"] }, + // Reflection distiller sub-session must not receive auto-recall/injected blocks + { group: "core-regression", runner: "node", file: "test/reflection-distiller-hook-skip.test.mjs", args: ["--test"] }, + // register() re-registration hardening (scope cache-miss log/handler dedup) { group: "core-regression", runner: "node", file: "test/register-scope-dedup.test.mjs", args: ["--test"] }, - // Reflection distiller sub-run must request raw-run semantics (skip foreign before_prompt_build hooks) + // Reflection distiller sub-run must request raw-run semantics (skip foreign before_prompt_build hooks) { group: "core-regression", runner: "node", file: "test/raw-run-distiller-hooks.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/autocapture-watermark-reset.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/autocapture-internal-session-guard.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/memory-categories-storage-map.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/admission-controller-standalone.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/smart-extractor-admission-controller-injection.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/admission-without-smart-extraction.test.mjs", args: ["--test"] }, -]; - -export function getEntriesForGroup(group) { - if (!CI_TEST_GROUPS.includes(group)) { - throw new Error(`Unknown CI test group: ${group}`); - } - - return CI_TEST_MANIFEST.filter((entry) => entry.group === group); -} + { group: "core-regression", runner: "node", file: "test/autocapture-watermark-reset.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/autocapture-internal-session-guard.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/memory-categories-storage-map.test.mjs", args: ["--test"] }, + // Delete/delete-bulk must synchronously invalidate in-process reflection read caches + { group: "core-regression", runner: "node", file: "test/delete-invalidate-reflection-caches.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/admission-controller-standalone.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/smart-extractor-admission-controller-injection.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/admission-without-smart-extraction.test.mjs", args: ["--test"] }, +]; + +export function getEntriesForGroup(group) { + if (!CI_TEST_GROUPS.includes(group)) { + throw new Error(`Unknown CI test group: ${group}`); + } + + return CI_TEST_MANIFEST.filter((entry) => entry.group === group); +}