From 5f89fb0cec4647e1cfb9c07c03b181d1e9d41fa4 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:18:13 +0200 Subject: [PATCH 01/26] perf(server): bind listener before startup recovery and stop per-task config reloads (#4055) - ServiceContainer.initialize() split into initializeCore() (fast) and runStartupRecovery() (O(workspaces) task/workspace recovery); `xum server` binds its port after core init and runs recovery in the background. - Recovery snapshots config synchronously before its first await so tasks created by clients after listen are never treated as stale `starting` tasks; running-task resume skips tasks that are already streaming. - Startup patch-generation and reported-task cleanup loops reuse one config snapshot instead of re-parsing config.json per task. - TerminalAttentionStore.listPendingOwnerWorkspaceIds scans session dirs with bounded concurrency and short-circuits on the first pending record. - [startup] completion logs switch to warn above SLOW_STARTUP_WARN_THRESHOLD_MS. --- src/cli/server.ts | 27 ++-- src/constants/startup.ts | 7 + .../services/gitPatchArtifactService.test.ts | 100 ++++++++++++- src/node/services/gitPatchArtifactService.ts | 10 +- src/node/services/serviceContainer.test.ts | 53 +++++++ src/node/services/serviceContainer.ts | 90 +++++++++--- src/node/services/taskService.test.ts | 134 ++++++++++++++++++ src/node/services/taskService.ts | 75 ++++++++-- .../services/terminalAttentionStore.test.ts | 62 ++++++++ src/node/services/terminalAttentionStore.ts | 50 +++++-- 10 files changed, 548 insertions(+), 60 deletions(-) create mode 100644 src/constants/startup.ts diff --git a/src/cli/server.ts b/src/cli/server.ts index 9205decaee1..08f236bc29e 100644 --- a/src/cli/server.ts +++ b/src/cli/server.ts @@ -94,11 +94,10 @@ async function main(): Promise { launchProjectPath = null; // Keepalive interval to prevent premature process exit during async initialization. - // During startup, taskService.initialize() may resume running tasks by calling - // sendMessage(), which spawns background AI streams. Between the completion of - // serviceContainer.initialize() and the HTTP server starting to listen, there can - // be a brief moment where no ref'd handles exist, causing Node to exit with code 0. - // This interval ensures the event loop stays alive until the server is listening. + // Between the completion of serviceContainer.initializeCore() and the HTTP server + // starting to listen, there can be a brief moment where no ref'd handles exist, + // causing Node to exit with code 0. This interval ensures the event loop stays alive + // until the server is listening; startup recovery runs after that point. const startupKeepalive = setInterval(() => { // Intentionally empty - keeps event loop alive during startup }, 1000); @@ -115,9 +114,9 @@ async function main(): Promise { } // Early lockfile check: detect an existing server BEFORE initializing services. - // serviceContainer.initialize() resumes queued/running tasks (via TaskService), - // so we must fail fast here to avoid orphaned side effects when another server - // already holds the lock. ServerService.startServer() re-checks as defense-in-depth. + // Task recovery (runStartupRecovery) only runs after startServer() has acquired the + // lock, so this check is a fast-fail nicety that avoids core init work when another + // server already holds the lock. ServerService.startServer() re-checks as the real gate. const xumHome = getXumHome(); const earlyLockfile = new ServerLockfile(xumHome); const existing = await earlyLockfile.read(); @@ -132,7 +131,9 @@ async function main(): Promise { const serviceContainer = new ServiceContainer(stores); // Headless server has no interactive host-key dialog setOpenSSHHostKeyPolicyMode("headless-fallback"); - await serviceContainer.initialize(); + // Only the fast core init gates the listener; workspace/task recovery scales with the + // number of workspaces and runs in the background once the server is accepting connections. + await serviceContainer.initializeCore(); serviceContainer.windowService.setMainWindow(mockWindow); if (ADD_PROJECT_PATH) { @@ -162,6 +163,14 @@ async function main(): Promise { allowHttpOrigin: ALLOW_HTTP_ORIGIN, }); + // Start recovery in the same synchronous continuation as startServer resolving, before any + // further await: runStartupRecovery snapshots config before its first await, and that + // snapshot must predate anything a freshly connected client can create. Recovery is + // best-effort background work; a failure must not take the listening server down. + void serviceContainer.runStartupRecovery().catch((error: unknown) => { + log.error("[startup] Background startup recovery failed", { error }); + }); + // Server is now listening - clear the startup keepalive since httpServer keeps the loop alive clearInterval(startupKeepalive); diff --git a/src/constants/startup.ts b/src/constants/startup.ts new file mode 100644 index 00000000000..5d640bc0908 --- /dev/null +++ b/src/constants/startup.ts @@ -0,0 +1,7 @@ +/** + * Startup passes that take longer than this are logged at warn level so slow deployments + * (thousands of workspaces, large session dirs) surface in logs without enabling debug output. + * Chosen well above a healthy cold start (seconds) but below the point where a health check + * would already have flagged the process. + */ +export const SLOW_STARTUP_WARN_THRESHOLD_MS = 30_000; diff --git a/src/node/services/gitPatchArtifactService.test.ts b/src/node/services/gitPatchArtifactService.test.ts index 2d9b8a68af8..7c07d9a95c8 100644 --- a/src/node/services/gitPatchArtifactService.test.ts +++ b/src/node/services/gitPatchArtifactService.test.ts @@ -1,4 +1,7 @@ -import { describe, expect, it } from "bun:test"; +import * as fsPromises from "node:fs/promises"; +import * as path from "node:path"; + +import { describe, expect, it, spyOn } from "bun:test"; import { buildContinuationProjectArtifacts, @@ -6,6 +9,10 @@ import { upsertProjectArtifact, } from "@/node/services/gitPatchArtifactService"; import { Config } from "@/node/config"; +import { + readSubagentGitPatchArtifact, + upsertSubagentGitPatchArtifact, +} from "@/node/services/subagentGitPatchArtifacts"; import { TestTempDir } from "@/node/services/tools/testHelpers"; describe("upsertProjectArtifact", () => { @@ -79,6 +86,97 @@ describe("GitPatchArtifactService coordination", () => { await Promise.all([apply, refresh]); expect(refreshSettled).toBe(true); }); + + it("resolves task entries from options.config without reloading config.json", async () => { + using tempDir = new TestTempDir("git-patch-artifact-snapshot"); + const config = new Config(tempDir.path); + const projectPath = path.join(tempDir.path, "repo"); + await fsPromises.mkdir(projectPath, { recursive: true }); + const parentId = "parent"; + const childId = "child-reported"; + const freshChildId = "child-fresh"; + const execChild = (id: string) => ({ + path: path.join(projectPath, id), + id, + name: id, + parentWorkspaceId: parentId, + agentId: "exec", + agentType: "exec", + taskStatus: "reported" as const, + }); + await config.editConfig((cfg) => { + cfg.projects.set(projectPath, { + workspaces: [ + { path: projectPath, id: parentId, name: "parent", runtimeConfig: { type: "local" } }, + execChild(childId), + execChild(freshChildId), + ], + }); + return cfg; + }); + // Startup recovery re-visits reported tasks whose artifact already finished; that artifact + // must be left untouched and no generation job should start. + const parentSessionDir = path.join(config.sessionsDir, parentId); + const readyArtifact = await upsertSubagentGitPatchArtifact({ + workspaceId: parentId, + workspaceSessionDir: parentSessionDir, + childTaskId: childId, + updater: () => ({ + childTaskId: childId, + parentWorkspaceId: parentId, + createdAtMs: 1, + updatedAtMs: 2, + status: "ready", + projectArtifacts: [], + readyProjectCount: 0, + failedProjectCount: 0, + skippedProjectCount: 0, + totalCommitCount: 0, + }), + }); + + const snapshot = config.loadConfigOrDefault(); + const service = new GitPatchArtifactService(config); + const loadConfigSpy = spyOn(config, "loadConfigOrDefault"); + const completedChildIds: string[] = []; + let resolveFreshCompletion: (() => void) | undefined; + const freshCompletion = new Promise((resolve) => { + resolveFreshCompletion = resolve; + }); + const onComplete = (completedChildId: string) => { + completedChildIds.push(completedChildId); + if (completedChildId === freshChildId) resolveFreshCompletion?.(); + return Promise.resolve(); + }; + + await service.maybeStartGeneration(parentId, childId, onComplete, { config: snapshot }); + expect(loadConfigSpy).not.toHaveBeenCalled(); + expect(await readSubagentGitPatchArtifact(parentSessionDir, childId)).toEqual(readyArtifact); + + await service.maybeStartGeneration(parentId, childId, onComplete); + expect(loadConfigSpy).toHaveBeenCalledTimes(1); + expect(await readSubagentGitPatchArtifact(parentSessionDir, childId)).toEqual(readyArtifact); + + // The snapshot is authoritative: a child the snapshot does not know about is skipped even + // though config.json now lists it, so no artifact is created for it. + const unknownChildId = "child-unknown"; + await config.editConfig((cfg) => { + cfg.projects.get(projectPath)?.workspaces.push(execChild(unknownChildId)); + return cfg; + }); + loadConfigSpy.mockClear(); + await service.maybeStartGeneration(parentId, unknownChildId, onComplete, { config: snapshot }); + expect(loadConfigSpy).not.toHaveBeenCalled(); + expect(await readSubagentGitPatchArtifact(parentSessionDir, unknownChildId)).toBeNull(); + expect(completedChildIds).toEqual([]); + + // A snapshot child without an artifact still takes the normal generation path (pending + // marker written, background job started), proving the snapshot feeds the real branch. + await service.maybeStartGeneration(parentId, freshChildId, onComplete, { config: snapshot }); + expect(await readSubagentGitPatchArtifact(parentSessionDir, freshChildId)).not.toBeNull(); + await freshCompletion; + expect(completedChildIds).toEqual([freshChildId]); + }); }); describe("buildContinuationProjectArtifacts", () => { diff --git a/src/node/services/gitPatchArtifactService.ts b/src/node/services/gitPatchArtifactService.ts index fc4967cf489..6b9f159affc 100644 --- a/src/node/services/gitPatchArtifactService.ts +++ b/src/node/services/gitPatchArtifactService.ts @@ -1,7 +1,7 @@ import * as path from "node:path"; import assert from "node:assert/strict"; -import type { Config } from "@/node/config"; +import type { Config, ProjectsConfig } from "@/node/config"; import type { SubagentGitPatchArtifact, SubagentGitProjectPatchArtifact, @@ -330,12 +330,14 @@ export class GitPatchArtifactService { * * @param onComplete - called after generation finishes (success *or* failure), * typically used to trigger reported-leaf-task cleanup. + * @param options.config - config snapshot to resolve the child/parent entries from; callers that + * iterate many tasks (startup recovery) pass one instead of reloading config.json per task. */ async maybeStartGeneration( parentWorkspaceId: string, childWorkspaceId: string, onComplete: OnPatchGenerationComplete, - options?: { refreshForContinuation?: boolean } + options?: { refreshForContinuation?: boolean; config?: ProjectsConfig } ): Promise { return await this.withOperationLock(childWorkspaceId, async () => { await this.maybeStartGenerationUnlocked( @@ -351,7 +353,7 @@ export class GitPatchArtifactService { parentWorkspaceId: string, childWorkspaceId: string, onComplete: OnPatchGenerationComplete, - options?: { refreshForContinuation?: boolean } + options?: { refreshForContinuation?: boolean; config?: ProjectsConfig } ): Promise { assert( parentWorkspaceId.length > 0, @@ -371,7 +373,7 @@ export class GitPatchArtifactService { // Write a pending marker before we attempt cleanup, so the reported task workspace isn't deleted // while we're still reading commits from it. const nowMs = Date.now(); - const cfg = this.config.loadConfigOrDefault(); + const cfg = options?.config ?? this.config.loadConfigOrDefault(); const childEntry = findWorkspaceEntry(cfg, childWorkspaceId); if (childEntry?.workspace.kind === "scratch") { diff --git a/src/node/services/serviceContainer.test.ts b/src/node/services/serviceContainer.test.ts index 35dacf73b29..b881862e740 100644 --- a/src/node/services/serviceContainer.test.ts +++ b/src/node/services/serviceContainer.test.ts @@ -110,6 +110,59 @@ describe("ServiceContainer", () => { ); }); + it("initializeCore resolves without waiting for task recovery", async () => { + services = new ServiceContainer(stores); + let releaseTaskInit: (() => void) | undefined; + const taskInitGate = new Promise((resolve) => { + releaseTaskInit = resolve; + }); + const taskInitializeSpy = spyOn(services.taskService, "initialize").mockImplementation( + () => taskInitGate + ); + const workspaceInitializeSpy = spyOn( + services.workspaceService, + "initialize" + ).mockImplementation(() => Promise.resolve()); + const loadConfigSpy = spyOn(config, "loadConfigOrDefault"); + + await services.initializeCore(); + expect(taskInitializeSpy).not.toHaveBeenCalled(); + expect(workspaceInitializeSpy).not.toHaveBeenCalled(); + + const callOrder: string[] = []; + let taskInitCalled: (() => void) | undefined; + const taskInitCalledPromise = new Promise((resolve) => { + taskInitCalled = resolve; + }); + workspaceInitializeSpy.mockImplementation(() => { + callOrder.push("workspace"); + return Promise.resolve(); + }); + taskInitializeSpy.mockImplementation(() => { + callOrder.push("task"); + taskInitCalled?.(); + return taskInitGate; + }); + loadConfigSpy.mockClear(); + + let recoverySettled = false; + const recovery = services.runStartupRecovery().then(() => { + recoverySettled = true; + }); + await taskInitCalledPromise; + expect(callOrder).toEqual(["workspace", "task"]); + expect(recoverySettled).toBe(false); + + // The snapshot handed to TaskService is the one loaded synchronously at recovery start. + const snapshot = loadConfigSpy.mock.results[0]?.value; + expect(snapshot).toBeDefined(); + expect(taskInitializeSpy).toHaveBeenCalledWith(snapshot); + + releaseTaskInit?.(); + await recovery; + expect(recoverySettled).toBe(true); + }); + it("exposes desktopSessionManager in the ORPC context", () => { services = new ServiceContainer(stores); diff --git a/src/node/services/serviceContainer.ts b/src/node/services/serviceContainer.ts index 154aa52b09d..8e0f28af938 100644 --- a/src/node/services/serviceContainer.ts +++ b/src/node/services/serviceContainer.ts @@ -4,6 +4,7 @@ import { DEFAULT_WORKTREE_ARCHIVE_BEHAVIOR } from "@/common/config/worktreeArchi import { log } from "@/node/services/log"; import type { Config, ConfigStores, WorkspaceSessionLocator } from "@/node/config"; import type { FileLeaseManager, ProvidersConfigStore, SecretsStore } from "@/node/config"; +import { SLOW_STARTUP_WARN_THRESHOLD_MS } from "@/constants/startup"; import type { CoreServices } from "@/node/services/coreServices"; import { PTYService } from "@/node/services/ptyService"; import type { TerminalWindowManager } from "@/desktop/terminalWindowManager"; @@ -190,6 +191,11 @@ export class ServiceContainer { public readonly idleDispatcher: IdleDispatcher; public readonly heartbeatService: HeartbeatService; public readonly agentStatusService: AgentStatusService; + // Shared between initializeCore() and runStartupRecovery() so the completion log still + // reports every step and the total wall time from the start of core init. + private startupStartedAt: number | undefined; + private readonly startupStepDurationsMs: Record = {}; + /** * The in-flight (or completed) `dispose()` teardown. Every caller shares it, * so a concurrent or repeated dispose() (the desktop's two before-quit @@ -613,43 +619,78 @@ export class ServiceContainer { } async initialize(): Promise { - const startupStartedAt = Date.now(); - const stepDurationsMs: Record = {}; - const recordStep = async (name: string, fn: () => Promise): Promise => { - const stepStartedAt = Date.now(); - try { - return await fn(); - } finally { - stepDurationsMs[name] = Date.now() - stepStartedAt; - } - }; + await this.initializeCore(); + await this.runStartupRecovery(); + } + + private async recordStartupStep(name: string, fn: () => Promise): Promise { + const stepStartedAt = Date.now(); + try { + return await fn(); + } finally { + this.startupStepDurationsMs[name] = Date.now() - stepStartedAt; + } + } + + /** + * Fast startup steps that request handling depends on. Safe to await before binding the + * server listener; the slow per-workspace recovery pass lives in runStartupRecovery(). + */ + async initializeCore(): Promise { + this.startupStartedAt = Date.now(); log.info("[startup] ServiceContainer.initialize starting"); - await recordStep("extensionMetadata.initialize", () => this.extensionMetadata.initialize()); + await this.recordStartupStep("extensionMetadata.initialize", () => + this.extensionMetadata.initialize() + ); // Initialize telemetry service - await recordStep("telemetryService.initialize", () => this.telemetryService.initialize()); + await this.recordStartupStep("telemetryService.initialize", () => + this.telemetryService.initialize() + ); // Initialize policy service (startup gating) - await recordStep("policyService.initialize", () => this.policyService.initialize()); + await this.recordStartupStep("policyService.initialize", () => this.policyService.initialize()); + + await this.recordStartupStep("experimentsService.initialize", () => + this.experimentsService.initialize() + ); + } + + /** + * Workspace/task restart recovery plus background housekeeping. On large deployments this + * scales with the number of workspaces, so the server entry point runs it after the + * listener is bound instead of on the critical path to accepting connections. + */ + async runStartupRecovery(): Promise { + // Snapshot config synchronously, before the first await. server.ts calls this in the same + // tick that startServer resolves, so a snapshot taken here cannot contain tasks created by + // clients that connect after listen. TaskService's stale-`starting` recovery must only judge + // tasks that were `starting` when the process started, never a task a freshly connected + // client is legitimately starting. + const startupConfig = this.config.loadConfigOrDefault(); - await recordStep("experimentsService.initialize", () => this.experimentsService.initialize()); // Kick off non-task chat restart recovery eagerly; task workspaces recover in TaskService.initialize(). - await recordStep("workspaceService.initialize", () => this.workspaceService.initialize()); - await recordStep("taskService.initialize", () => this.taskService.initialize()); + await this.recordStartupStep("workspaceService.initialize", () => + this.workspaceService.initialize() + ); + await this.recordStartupStep("taskService.initialize", () => + this.taskService.initialize(startupConfig) + ); const idleCompactionStartedAt = Date.now(); // Start idle compaction checker this.idleCompactionService.start(); - stepDurationsMs["idleCompactionService.start"] = Date.now() - idleCompactionStartedAt; + this.startupStepDurationsMs["idleCompactionService.start"] = + Date.now() - idleCompactionStartedAt; const heartbeatStartedAt = Date.now(); this.heartbeatService.start(); - stepDurationsMs["heartbeatService.start"] = Date.now() - heartbeatStartedAt; + this.startupStepDurationsMs["heartbeatService.start"] = Date.now() - heartbeatStartedAt; const agentStatusStartedAt = Date.now(); this.agentStatusService.start(); - stepDurationsMs["agentStatusService.start"] = Date.now() - agentStatusStartedAt; + this.startupStepDurationsMs["agentStatusService.start"] = Date.now() - agentStatusStartedAt; // Dream launch sweep (PRD #3534): consolidate memory for workspaces idle // ≥24h with writes since their last run. Fire-and-forget after the await @@ -673,10 +714,13 @@ export class ServiceContainer { log.warn("Background xum SSH config setup failed", { error }); }); - log.info("[startup] ServiceContainer.initialize completed", { - totalMs: Date.now() - startupStartedAt, - stepDurationsMs, - }); + const totalMs = Date.now() - (this.startupStartedAt ?? Date.now()); + const completedPayload = { totalMs, stepDurationsMs: this.startupStepDurationsMs }; + if (totalMs > SLOW_STARTUP_WARN_THRESHOLD_MS) { + log.warn("[startup] ServiceContainer.initialize completed", completedPayload); + } else { + log.info("[startup] ServiceContainer.initialize completed", completedPayload); + } } /** diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 4dc2b3b77f6..138e7151a8e 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -3438,6 +3438,140 @@ describe("TaskService", () => { expect(findWorkspaceInConfig(config, childTaskId)?.taskPendingGuidance).toBeUndefined(); }); + test("initialize judges stale starting tasks from the startup config snapshot", async () => { + const config = await createTestConfig(rootDir); + const projectPath = path.join(rootDir, "repo"); + const parentWorkspaceId = "parent-snapshot"; + const staleStartingTaskId = "child-stale-starting"; + const freshStartingTaskId = "child-fresh-starting"; + + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "parent", parentWorkspaceId), + projectWorkspace(projectPath, "stale", staleStartingTaskId, { + parentWorkspaceId, + agentId: "exec", + agentType: "exec", + taskStatus: "starting", + taskModelString: "openai:gpt-5.2", + }), + ], + testTaskSettings() + ); + const startupSnapshot = config.loadConfigOrDefault(); + + // A client connected after the listener was bound and is legitimately starting this task. + await config.editConfig((cfg) => { + cfg.projects.get(projectPath)?.workspaces.push( + projectWorkspace(projectPath, "fresh", freshStartingTaskId, { + parentWorkspaceId, + agentId: "exec", + agentType: "exec", + taskStatus: "starting", + taskModelString: "openai:gpt-5.2", + }) + ); + return cfg; + }); + + const isStreaming = mock((workspaceId: string) => workspaceId === staleStartingTaskId); + const { aiService } = createAIServiceMocks(config, { isStreaming }); + const { workspaceService, sendMessage } = createWorkspaceServiceMocks(); + const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); + + await taskService.initialize(startupSnapshot); + + expect(findWorkspaceInConfig(config, staleStartingTaskId)?.taskStatus).toBe("running"); + expect(findWorkspaceInConfig(config, freshStartingTaskId)?.taskStatus).toBe("starting"); + const messagedWorkspaceIds = ( + sendMessage as unknown as { mock: { calls: unknown[][] } } + ).mock.calls.map((call) => call[0]); + expect(messagedWorkspaceIds).not.toContain(freshStartingTaskId); + }); + + test("initialize does not resend the restart nudge to a running task that is already streaming", async () => { + const config = await createTestConfig(rootDir); + const projectPath = path.join(rootDir, "repo"); + const parentWorkspaceId = "parent-restart-streaming"; + const streamingTaskId = "child-running-streaming"; + const idleTaskId = "child-running-idle"; + + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "parent", parentWorkspaceId), + projectWorkspace(projectPath, "streaming", streamingTaskId, { + parentWorkspaceId, + agentId: "exec", + agentType: "exec", + taskStatus: "running", + taskModelString: "openai:gpt-5.2", + }), + projectWorkspace(projectPath, "idle", idleTaskId, { + parentWorkspaceId, + agentId: "exec", + agentType: "exec", + taskStatus: "running", + taskModelString: "openai:gpt-5.2", + }), + ], + testTaskSettings() + ); + + const isStreaming = mock((workspaceId: string) => workspaceId === streamingTaskId); + const { aiService } = createAIServiceMocks(config, { isStreaming }); + const { workspaceService, sendMessage } = createWorkspaceServiceMocks(); + const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); + + await taskService.initialize(); + + const messagedWorkspaceIds = ( + sendMessage as unknown as { mock: { calls: unknown[][] } } + ).mock.calls.map((call) => call[0]); + expect(messagedWorkspaceIds).toContain(idleTaskId); + expect(messagedWorkspaceIds).not.toContain(streamingTaskId); + }); + + test("initialize does not reload config.json per completed-report task", async () => { + const countConfigLoadsDuringInitialize = async (reportedTaskCount: number): Promise => { + const runRootDir = path.join(rootDir, `run-${reportedTaskCount}`); + const config = await createTestConfig(runRootDir); + const projectPath = path.join(runRootDir, "repo"); + const parentWorkspaceId = "parent-reload"; + const reportedTasks = Array.from({ length: reportedTaskCount }, (_unused, index) => + projectWorkspace(projectPath, `reported-${index}`, `child-reported-${index}`, { + parentWorkspaceId, + agentId: "explore", + agentType: "explore", + taskStatus: "reported", + reportedAt: "2026-08-10T00:00:00.000Z", + taskModelString: "openai:gpt-5.2", + }) + ); + await saveWorkspaces( + config, + projectPath, + [projectWorkspace(projectPath, "parent", parentWorkspaceId), ...reportedTasks], + testTaskSettings() + ); + const { taskService } = createTaskServiceHarness(config); + const loadConfigSpy = spyOn(config, "loadConfigOrDefault"); + try { + await taskService.initialize(); + return loadConfigSpy.mock.calls.length; + } finally { + loadConfigSpy.mockRestore(); + } + }; + + const loadsWithTwoTasks = await countConfigLoadsDuringInitialize(2); + const loadsWithSixTasks = await countConfigLoadsDuringInitialize(6); + expect(loadsWithSixTasks).toBe(loadsWithTwoTasks); + }); + test("bare compaction stream-end resumes the pre-compaction parent identity", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 4fbf97844f5..73c2746b3d7 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -10,6 +10,7 @@ import { // Persisted task snapshots stamp the legacy exclusive mirror so downgraded // builds resume tasks in the exclusive posture (see withLegacyPtcExclusiveMirror). import { withLegacyPtcExclusiveMirror } from "@/common/constants/experiments"; +import { SLOW_STARTUP_WARN_THRESHOLD_MS } from "@/constants/startup"; import { raceWithAbortAndTimeout } from "@/node/utils/concurrency/withTimeout"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; import { AsyncMutex } from "@/node/utils/concurrency/asyncMutex"; @@ -2259,9 +2260,14 @@ export class TaskService implements AgentTaskIntegration { return found; } - async initialize(): Promise { + /** + * @param startupConfigSnapshot - config snapshot taken when the process started. When the server + * binds its listener before running recovery, clients may already be creating tasks; judging + * stale-`starting` recovery from this snapshot keeps those fresh tasks out of scope. + */ + async initialize(startupConfigSnapshot?: ProjectsConfig): Promise { const startupStartedAt = Date.now(); - const startupConfig = this.config.loadConfigOrDefault(); + const startupConfig = startupConfigSnapshot ?? this.config.loadConfigOrDefault(); const queuedTaskCountAtStartup = this.listAgentTaskWorkspaces(startupConfig).filter( (task) => task.taskStatus === "queued" && typeof task.id === "string" ).length; @@ -2397,6 +2403,7 @@ export class TaskService implements AgentTaskIntegration { let resumedRunningCount = 0; let skippedRunningDueToActiveDescendants = 0; + let skippedRunningAlreadyStreaming = 0; let failedRunningCount = 0; for (const task of runningTasks) { @@ -2463,6 +2470,14 @@ export class TaskService implements AgentTaskIntegration { continue; } + // Recovery may run after the server is already accepting connections, so a client (or the + // queue drain above) can legitimately have this task streaming by now. Nudging it again + // would inject a spurious "Xum restarted" turn into a live stream. + if (this.aiService.isStreaming(task.id)) { + skippedRunningAlreadyStreaming += 1; + continue; + } + // Best-effort: if xum restarted mid-stream, nudge the agent to continue and report. // Only do this when the task has no blocking running descendants, to avoid duplicate spawns. const hasBlockingActiveDescendants = @@ -2551,10 +2566,13 @@ export class TaskService implements AgentTaskIntegration { for (const task of completedReportTasks) { if (!task.parentWorkspaceId) continue; try { + // Pass the loop snapshot: reloading config.json per reported task made this pass scale + // with (reported tasks x config size) on large deployments. await this.gitPatchArtifactService.maybeStartGeneration( task.parentWorkspaceId, task.id!, - (wsId) => this.requestReportedTaskCleanupRecheck(wsId) + (wsId) => this.requestReportedTaskCleanupRecheck(wsId), + { config } ); } catch (error: unknown) { log.error("Failed to resume subagent git patch generation on startup", { @@ -2591,9 +2609,17 @@ export class TaskService implements AgentTaskIntegration { // Best-effort completed-report ancestor recheck after restart. const cleanupReportedTasksStartedAt = Date.now(); + // Reuse one config snapshot across tasks and reload only after a removal changed the tree. + // A stale snapshot is conservative here: it can only show more workspaces than exist, so the + // structural-leaf gate errs toward skipping. Concurrent removals from patch-generation + // onComplete callbacks are re-evaluated on fresh config by those callbacks. + let cleanupConfig = config; for (const task of completedReportTasks) { if (!task.id) continue; - await this.cleanupReportedLeafTask(task.id); + const removedCount = await this.cleanupReportedLeafTask(task.id, { config: cleanupConfig }); + if (removedCount > 0) { + cleanupConfig = this.config.loadConfigOrDefault(); + } } const cleanupReportedTasksMs = Date.now() - cleanupReportedTasksStartedAt; @@ -2633,8 +2659,9 @@ export class TaskService implements AgentTaskIntegration { await this.schedulePendingTerminalAttentionOwnerDrains(); const terminalAttentionDrainMs = Date.now() - terminalAttentionDrainStartedAt; - log.info("[startup] TaskService.initialize completed", { - totalMs: Date.now() - startupStartedAt, + const totalMs = Date.now() - startupStartedAt; + const completedPayload = { + totalMs, maybeStartQueuedTasksMs, awaitingReportTaskCount: awaitingReportTasks.length, resumedAwaitingReportCount, @@ -2643,6 +2670,7 @@ export class TaskService implements AgentTaskIntegration { runningTaskCount: runningTasks.length, resumedRunningCount, skippedRunningDueToActiveDescendants, + skippedRunningAlreadyStreaming, failedRunningCount, completedReportTaskCount: completedReportTasks.length, patchGenerationRecoveryMs, @@ -2653,7 +2681,12 @@ export class TaskService implements AgentTaskIntegration { pendingTerminalAttentionOwnerWorkspaceCount, terminalAttentionDrainMs, cleanupReportedTasksMs, - }); + }; + if (totalMs > SLOW_STARTUP_WARN_THRESHOLD_MS) { + log.warn("[startup] TaskService.initialize completed", completedPayload); + } else { + log.info("[startup] TaskService.initialize completed", completedPayload); + } } private async hasAcceptedInitialTaskPrompt(workspaceId: string): Promise { @@ -13111,11 +13144,11 @@ export class TaskService implements AgentTaskIntegration { } private async canCleanupReportedTask( - workspaceId: string + workspaceId: string, + config: ProjectsConfig = this.config.loadConfigOrDefault() ): Promise<{ ok: true; parentWorkspaceId: string } | { ok: false; reason: string }> { assert(workspaceId.length > 0, "canCleanupReportedTask: workspaceId must be non-empty"); - const config = this.config.loadConfigOrDefault(); const entry = findWorkspaceEntry(config, workspaceId); if (!entry) { return { ok: false, reason: "workspace_not_found" }; @@ -13179,26 +13212,38 @@ export class TaskService implements AgentTaskIntegration { return { ok: true, parentWorkspaceId }; } - private async cleanupReportedLeafTask(workspaceId: string): Promise { + /** + * @param options.config - config snapshot for the first (depth 0) eligibility check only; every + * later depth re-evaluates the parent on fresh config because a removal just changed the tree. + * @returns the number of workspaces removed. + */ + private async cleanupReportedLeafTask( + workspaceId: string, + options?: { config?: ProjectsConfig } + ): Promise { assert(workspaceId.length > 0, "cleanupReportedLeafTask: workspaceId must be non-empty"); // Lineage reduction: each iteration removes exactly one completed leaf, then re-evaluates // the parent on fresh config. The structural-leaf gate in canCleanupReportedTask ensures // ancestors are only deleted after every child has been pruned. let currentWorkspaceId = workspaceId; + let removedCount = 0; const visited = new Set(); for (let depth = 0; depth < 32; depth++) { if (visited.has(currentWorkspaceId)) { log.error("cleanupReportedLeafTask: possible parentWorkspaceId cycle", { workspaceId: currentWorkspaceId, }); - return; + return removedCount; } visited.add(currentWorkspaceId); - const cleanupEligibility = await this.canCleanupReportedTask(currentWorkspaceId); + const cleanupEligibility = await this.canCleanupReportedTask( + currentWorkspaceId, + depth === 0 ? options?.config : undefined + ); if (!cleanupEligibility.ok) { - return; + return removedCount; } const removeResult = await this.workspaceService.remove(currentWorkspaceId, true); @@ -13207,8 +13252,9 @@ export class TaskService implements AgentTaskIntegration { workspaceId: currentWorkspaceId, error: removeResult.error, }); - return; + return removedCount; } + removedCount += 1; currentWorkspaceId = cleanupEligibility.parentWorkspaceId; } @@ -13216,5 +13262,6 @@ export class TaskService implements AgentTaskIntegration { log.error("cleanupReportedLeafTask: exceeded max parent traversal depth", { workspaceId, }); + return removedCount; } } diff --git a/src/node/services/terminalAttentionStore.test.ts b/src/node/services/terminalAttentionStore.test.ts index 1543d6f575e..0ce742727bb 100644 --- a/src/node/services/terminalAttentionStore.test.ts +++ b/src/node/services/terminalAttentionStore.test.ts @@ -206,4 +206,66 @@ describe("TerminalAttentionStore", () => { expect(await store.listPendingOwnerWorkspaceIds()).toEqual(["owner-b"]); }); + + test("listPendingOwnerWorkspaceIds scans more owners than the concurrency limit", async () => { + const store = new TerminalAttentionStore(makeConfig(rootDir)); + const sessionsDir = path.join(rootDir, "sessions"); + const expectedOwners: string[] = []; + for (let index = 0; index < 40; index++) { + const owner = `owner-${String(index).padStart(2, "0")}`; + switch (index % 4) { + case 0: { + await store.enqueueIfAbsent({ + ownerWorkspaceId: owner, + sourceKind: "agent_task", + sourceId: `task-${index}`, + }); + expectedOwners.push(owner); + break; + } + case 1: { + const created = await store.enqueueIfAbsent({ + ownerWorkspaceId: owner, + sourceKind: "workspace_turn", + sourceId: `wst-${index}`, + }); + expect(created).not.toBeNull(); + await store.markDelivered(owner, created!.id); + break; + } + case 2: { + const created = await store.enqueueIfAbsent({ + ownerWorkspaceId: owner, + sourceKind: "workflow_run", + sourceId: `run-${index}`, + }); + expect(created).not.toBeNull(); + await store.markSuperseded(owner, created!.id); + break; + } + default: { + // Session dir without a terminal-attention folder. + await fsPromises.mkdir(path.join(sessionsDir, owner), { recursive: true }); + } + } + } + // A pending record after superseded/delivered siblings still counts. + const mixedOwner = "owner-mixed"; + const delivered = await store.enqueueIfAbsent({ + ownerWorkspaceId: mixedOwner, + sourceKind: "agent_task", + sourceId: "task-delivered", + }); + await store.markDelivered(mixedOwner, delivered!.id); + await store.enqueueIfAbsent({ + ownerWorkspaceId: mixedOwner, + sourceKind: "agent_task", + sourceId: "task-pending", + }); + expectedOwners.push(mixedOwner); + // Non-directory entries in the sessions dir are ignored. + await fsPromises.writeFile(path.join(sessionsDir, "stray-file.json"), "{}", "utf-8"); + + expect(await store.listPendingOwnerWorkspaceIds()).toEqual([...expectedOwners].sort()); + }); }); diff --git a/src/node/services/terminalAttentionStore.ts b/src/node/services/terminalAttentionStore.ts index ac8388b7128..2e79a395784 100644 --- a/src/node/services/terminalAttentionStore.ts +++ b/src/node/services/terminalAttentionStore.ts @@ -7,6 +7,7 @@ import { z } from "zod"; import assert from "@/common/utils/assert"; import type { WorkspaceSessionLocator } from "@/node/config"; import { log } from "@/node/services/log"; +import { AsyncSemaphore } from "@/node/utils/concurrency/asyncSemaphore"; import { isErrnoWithCode } from "@/node/utils/fs"; /** @@ -16,6 +17,10 @@ import { isErrnoWithCode } from "@/node/utils/fs"; */ export const TERMINAL_ATTENTION_DIR = "terminal-attention"; +// Enough to keep the disk queue busy on the all-ENOENT fast path without opening thousands of +// directory handles at once on network or large session dirs. +const SESSION_DIR_SCAN_CONCURRENCY = 16; + const TERMINAL_ATTENTION_OUTPUT_DELIVERIES = [ "already_injected", "requires_task_await", @@ -226,6 +231,27 @@ export class TerminalAttentionStore { return records; } + /** True as soon as one pending record parses; skips the rest of the directory. */ + private async hasPending(ownerWorkspaceId: string): Promise { + const dir = this.dir(ownerWorkspaceId); + let entries: string[]; + try { + entries = await fsPromises.readdir(dir); + } catch (error) { + if (isErrnoWithCode(error, "ENOENT")) return false; + throw error; + } + for (const entry of entries) { + if (!entry.endsWith(".json")) continue; + const raw = await fsPromises.readFile(path.join(dir, entry), "utf-8").catch(() => null); + if (raw == null) continue; + if (this.parse(raw)?.status === "pending") { + return true; + } + } + return false; + } + async listPendingOwnerWorkspaceIds(): Promise { let entries: Dirent[]; try { @@ -235,15 +261,21 @@ export class TerminalAttentionStore { throw error; } - const ownerWorkspaceIds: string[] = []; - for (const entry of entries) { - if (!entry.isDirectory()) continue; - if ((await this.listPending(entry.name)).length > 0) { - ownerWorkspaceIds.push(entry.name); - } - } - ownerWorkspaceIds.sort(); - return ownerWorkspaceIds; + // Startup scans every session dir (thousands on large deployments); a sequential await per + // directory took minutes to find zero pending records, so overlap the readdir round trips. + const semaphore = new AsyncSemaphore(SESSION_DIR_SCAN_CONCURRENCY); + const results = await Promise.all( + entries.map(async (entry): Promise => { + if (!entry.isDirectory()) return null; + const slot = await semaphore.acquire(); + try { + return (await this.hasPending(entry.name)) ? entry.name : null; + } finally { + slot.release(); + } + }) + ); + return results.filter((name): name is string => name != null).sort(); } async delete(ownerWorkspaceId: string, id: string): Promise { From 314818f42838fc3af1780030b749d8e6b5e28574 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:35:04 +0200 Subject: [PATCH 02/26] fix(server): harden background startup recovery against early clients - Capture the recovery config snapshot before startServer() opens the listener (it accepts requests while awaiting the lockfile/mDNS), and pass it into runStartupRecovery(). - Orphan scratch workdir sweep skips recently touched dirs and re-checks fresh config before deleting, so createScratch during recovery is safe. - Isolate workspace/task recovery failures so the periodic services (idle compaction, heartbeat, agent status) always start. --- src/cli/server.ts | 13 +++-- src/node/services/serviceContainer.test.ts | 30 ++++++++-- src/node/services/serviceContainer.ts | 41 +++++++++----- src/node/services/workspaceService.test.ts | 64 ++++++++++++++++++++++ src/node/services/workspaceService.ts | 30 +++++++--- 5 files changed, 145 insertions(+), 33 deletions(-) diff --git a/src/cli/server.ts b/src/cli/server.ts index 08f236bc29e..11fb167e1d1 100644 --- a/src/cli/server.ts +++ b/src/cli/server.ts @@ -152,6 +152,12 @@ async function main(): Promise { const context = serviceContainer.toORPCContext(); + // Snapshot config before the listener opens: startServer() is already accepting requests while + // it awaits the lockfile and mDNS, so a snapshot taken after it resolves could already contain + // a task a freshly connected client is legitimately starting, and recovery would misjudge it + // as stale. Everything recovery must judge as pre-existing is on disk at this point. + const startupRecoveryConfig = config.loadConfigOrDefault(); + // Start server via ServerService (handles lockfile, mDNS, network URLs) const serverInfo = await serviceContainer.serverService.startServer({ xumHome: serviceContainer.config.rootDir, @@ -163,11 +169,8 @@ async function main(): Promise { allowHttpOrigin: ALLOW_HTTP_ORIGIN, }); - // Start recovery in the same synchronous continuation as startServer resolving, before any - // further await: runStartupRecovery snapshots config before its first await, and that - // snapshot must predate anything a freshly connected client can create. Recovery is - // best-effort background work; a failure must not take the listening server down. - void serviceContainer.runStartupRecovery().catch((error: unknown) => { + // Recovery is best-effort background work; a failure must not take the listening server down. + void serviceContainer.runStartupRecovery(startupRecoveryConfig).catch((error: unknown) => { log.error("[startup] Background startup recovery failed", { error }); }); diff --git a/src/node/services/serviceContainer.test.ts b/src/node/services/serviceContainer.test.ts index b881862e740..72ab2236b1a 100644 --- a/src/node/services/serviceContainer.test.ts +++ b/src/node/services/serviceContainer.test.ts @@ -143,26 +143,44 @@ describe("ServiceContainer", () => { taskInitCalled?.(); return taskInitGate; }); + // The server entry point snapshots config before its listener opens and hands that exact + // snapshot to task recovery; recovery must not substitute a fresh load. + const preListenSnapshot = config.loadConfigOrDefault(); loadConfigSpy.mockClear(); let recoverySettled = false; - const recovery = services.runStartupRecovery().then(() => { + const recovery = services.runStartupRecovery(preListenSnapshot).then(() => { recoverySettled = true; }); await taskInitCalledPromise; expect(callOrder).toEqual(["workspace", "task"]); expect(recoverySettled).toBe(false); - - // The snapshot handed to TaskService is the one loaded synchronously at recovery start. - const snapshot = loadConfigSpy.mock.results[0]?.value; - expect(snapshot).toBeDefined(); - expect(taskInitializeSpy).toHaveBeenCalledWith(snapshot); + expect(taskInitializeSpy).toHaveBeenCalledWith(preListenSnapshot); + expect(loadConfigSpy).not.toHaveBeenCalled(); releaseTaskInit?.(); await recovery; expect(recoverySettled).toBe(true); }); + it("runStartupRecovery starts the periodic services even when task recovery rejects", async () => { + services = new ServiceContainer(stores); + spyOn(services.workspaceService, "initialize").mockImplementation(() => Promise.resolve()); + spyOn(services.taskService, "initialize").mockImplementation(() => + Promise.reject(new Error("terminal-attention store unreadable")) + ); + const idleCompactionStart = spyOn(services.idleCompactionService, "start"); + const heartbeatStart = spyOn(services.heartbeatService, "start"); + const agentStatusStart = spyOn(services.agentStatusService, "start"); + + await services.initializeCore(); + await services.runStartupRecovery(); + + expect(idleCompactionStart).toHaveBeenCalledTimes(1); + expect(heartbeatStart).toHaveBeenCalledTimes(1); + expect(agentStatusStart).toHaveBeenCalledTimes(1); + }); + it("exposes desktopSessionManager in the ORPC context", () => { services = new ServiceContainer(stores); diff --git a/src/node/services/serviceContainer.ts b/src/node/services/serviceContainer.ts index 8e0f28af938..0825924b888 100644 --- a/src/node/services/serviceContainer.ts +++ b/src/node/services/serviceContainer.ts @@ -2,7 +2,7 @@ import * as path from "path"; import { DEFAULT_CODER_ARCHIVE_BEHAVIOR } from "@/common/config/coderArchiveBehavior"; import { DEFAULT_WORKTREE_ARCHIVE_BEHAVIOR } from "@/common/config/worktreeArchiveBehavior"; import { log } from "@/node/services/log"; -import type { Config, ConfigStores, WorkspaceSessionLocator } from "@/node/config"; +import type { Config, ConfigStores, ProjectsConfig, WorkspaceSessionLocator } from "@/node/config"; import type { FileLeaseManager, ProvidersConfigStore, SecretsStore } from "@/node/config"; import { SLOW_STARTUP_WARN_THRESHOLD_MS } from "@/constants/startup"; import type { CoreServices } from "@/node/services/coreServices"; @@ -661,22 +661,33 @@ export class ServiceContainer { * Workspace/task restart recovery plus background housekeeping. On large deployments this * scales with the number of workspaces, so the server entry point runs it after the * listener is bound instead of on the critical path to accepting connections. + * + * @param startupConfig - config snapshot from before the listener opened. TaskService's + * stale-`starting` recovery must only judge tasks that were `starting` when the process + * started, never a task a freshly connected client is legitimately starting, so a caller + * that accepts connections before recovery must capture the snapshot before listening. */ - async runStartupRecovery(): Promise { - // Snapshot config synchronously, before the first await. server.ts calls this in the same - // tick that startServer resolves, so a snapshot taken here cannot contain tasks created by - // clients that connect after listen. TaskService's stale-`starting` recovery must only judge - // tasks that were `starting` when the process started, never a task a freshly connected - // client is legitimately starting. - const startupConfig = this.config.loadConfigOrDefault(); - + async runStartupRecovery( + startupConfig: ProjectsConfig = this.config.loadConfigOrDefault() + ): Promise { + // Recovery is best-effort and runs while the server may already be serving requests: a + // failing recovery step must not skip the periodic services below (startup-time rule: + // never let background housekeeping take the app down). // Kick off non-task chat restart recovery eagerly; task workspaces recover in TaskService.initialize(). - await this.recordStartupStep("workspaceService.initialize", () => - this.workspaceService.initialize() - ); - await this.recordStartupStep("taskService.initialize", () => - this.taskService.initialize(startupConfig) - ); + try { + await this.recordStartupStep("workspaceService.initialize", () => + this.workspaceService.initialize() + ); + } catch (error: unknown) { + log.error("[startup] WorkspaceService recovery failed", { error }); + } + try { + await this.recordStartupStep("taskService.initialize", () => + this.taskService.initialize(startupConfig) + ); + } catch (error: unknown) { + log.error("[startup] TaskService recovery failed", { error }); + } const idleCompactionStartedAt = Date.now(); // Start idle compaction checker diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index a73f238509a..b65b195a5f1 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -8042,6 +8042,70 @@ describe("WorkspaceService initialize", () => { } }); + test("removes stale orphaned scratch workdirs but keeps referenced and recent ones", async () => { + const { config: realConfig, historyService, cleanup } = await createTestHistoryService(); + const scratchDirFor = (id: string) => path.join(realConfig.rootDir, "scratch", id); + const referencedDir = scratchDirFor("referenced-scratch"); + const staleOrphanDir = scratchDirFor("stale-orphan-scratch"); + // A scratch chat created while the sweep runs has a fresh workdir and, briefly, no + // config entry yet (createScratch persists config after mkdir). + const freshOrphanDir = scratchDirFor("fresh-orphan-scratch"); + for (const dir of [referencedDir, staleOrphanDir, freshOrphanDir]) { + await fsPromises.mkdir(dir, { recursive: true }); + } + const staleTime = new Date(Date.now() - 48 * 60 * 60 * 1000); + for (const dir of [referencedDir, staleOrphanDir]) { + await fsPromises.utimes(dir, staleTime, staleTime); + } + await realConfig.editConfig((cfg) => { + cfg.projects.set(SCRATCH_PROJECT_CONFIG_KEY, { + workspaces: [ + { + kind: "scratch", + path: referencedDir, + id: "referenced-scratch", + name: "scratch-referenced-scratch", + runtimeConfig: { type: "local" }, + }, + ], + projectKind: "system", + trusted: true, + }); + return cfg; + }); + + const aiService = { + ...createStreamLifecycleMocks(), + on: mock(() => undefined), + off: mock(() => undefined), + } as unknown as AIService; + const service = createWorkspaceServiceForTest({ + config: realConfig, + historyService, + aiService, + initStateManager: mockInitStateManager as InitStateManager, + }); + const startupAccess = service as unknown as { + startStartupRecovery: (workspaceId: string) => void; + }; + spyOn(startupAccess, "startStartupRecovery").mockImplementation(() => undefined); + + const exists = (dir: string) => + fsPromises.stat(dir).then( + () => true, + () => false + ); + + try { + await service.initialize(); + expect(await exists(referencedDir)).toBe(true); + expect(await exists(freshOrphanDir)).toBe(true); + expect(await exists(staleOrphanDir)).toBe(false); + } finally { + await cleanup(); + } + }); + test("removes stale orphaned session directories but keeps referenced and recent ones", async () => { const { config: realConfig, historyService, cleanup } = await createTestHistoryService(); await realConfig.editConfig((cfg) => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 7b8db4706c7..d4cfb6cf7cb 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4370,19 +4370,27 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return false; } - private async cleanupOrphanScratchWorkdirs(): Promise { - const scratchRoot = this.getScratchRoot(); - await ensurePrivateDir(scratchRoot); - - // Never interpret a config read failure as an empty reference set, because that would - // turn best-effort orphan cleanup into deletion of valid scratch chats. + /** + * Scratch workdirs referenced by config. Throws on unreadable config: never interpret a + * config read failure as an empty reference set, because that would turn best-effort + * orphan cleanup into deletion of valid scratch chats. + */ + private listReferencedScratchPaths(): Set { const config = this.config.loadConfigOrDefault({ throwOnError: true }); - const referencedScratchPaths = new Set( + return new Set( (config.projects.get(SCRATCH_PROJECT_CONFIG_KEY)?.workspaces ?? []) .filter((workspace) => workspace.kind === "scratch") .map((workspace) => path.resolve(workspace.path)) ); + } + + private async cleanupOrphanScratchWorkdirs(): Promise { + const scratchRoot = this.getScratchRoot(); + await ensurePrivateDir(scratchRoot); + + const referencedScratchPaths = this.listReferencedScratchPaths(); + const nowMs = Date.now(); for (const entry of await fsPromises.readdir(scratchRoot, { withFileTypes: true })) { if (!entry.isDirectory()) continue; @@ -4390,6 +4398,14 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { if (referencedScratchPaths.has(candidatePath)) continue; try { + // This sweep can run while the server is already accepting requests, and createScratch + // creates the workdir before persisting its config entry. Same guards as the session + // sweep: skip recently touched directories, then re-check fresh config right before + // deleting so a scratch chat created after the snapshot above is never reaped. + const stat = await fsPromises.stat(candidatePath); + if (nowMs - stat.mtimeMs < ORPHAN_SESSION_DIR_GRACE_MS) continue; + if (this.listReferencedScratchPaths().has(candidatePath)) continue; + await fsPromises.rm(candidatePath, { recursive: true, force: true }); } catch (error: unknown) { log.debug("Failed to clean orphaned scratch workdir", { candidatePath, error }); From 6aa18ab264378a4e7d9c25cce7aa476e93ef77ec Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:13:07 +0200 Subject: [PATCH 03/26] fix(server): make startup recovery respect concurrent client state changes - Stale-starting recovery compare-and-sets: only entries still `starting` are rewritten, so a task_stop that persisted `interrupted` wins. - Running-task resume re-reads the live status before nudging, so a task stopped after the snapshot is not restarted. - WorkspaceService.initialize re-reads config right before scheduling chat startup recovery, skipping workspaces archived or removed since the metadata read. --- src/node/services/taskService.test.ts | 44 ++++++++++++++++++++-- src/node/services/taskService.ts | 17 ++++++++- src/node/services/workspaceService.test.ts | 43 ++++++++++++++++++++- src/node/services/workspaceService.ts | 7 +++- 4 files changed, 105 insertions(+), 6 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 138e7151a8e..0e054ccceb4 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -3443,6 +3443,7 @@ describe("TaskService", () => { const projectPath = path.join(rootDir, "repo"); const parentWorkspaceId = "parent-snapshot"; const staleStartingTaskId = "child-stale-starting"; + const stoppedStartingTaskId = "child-stopped-starting"; const freshStartingTaskId = "child-fresh-starting"; await saveWorkspaces( @@ -3457,14 +3458,23 @@ describe("TaskService", () => { taskStatus: "starting", taskModelString: "openai:gpt-5.2", }), + projectWorkspace(projectPath, "stopped", stoppedStartingTaskId, { + parentWorkspaceId, + agentId: "exec", + agentType: "exec", + taskStatus: "starting", + taskModelString: "openai:gpt-5.2", + }), ], testTaskSettings() ); const startupSnapshot = config.loadConfigOrDefault(); - // A client connected after the listener was bound and is legitimately starting this task. + // A client connected after the listener was bound: it is legitimately starting a new task and + // has stopped one of the snapshot's `starting` tasks before recovery applied its verdict. await config.editConfig((cfg) => { - cfg.projects.get(projectPath)?.workspaces.push( + const workspaces = cfg.projects.get(projectPath)?.workspaces ?? []; + workspaces.push( projectWorkspace(projectPath, "fresh", freshStartingTaskId, { parentWorkspaceId, agentId: "exec", @@ -3473,6 +3483,8 @@ describe("TaskService", () => { taskModelString: "openai:gpt-5.2", }) ); + const stopped = workspaces.find((workspace) => workspace.id === stoppedStartingTaskId); + if (stopped) stopped.taskStatus = "interrupted"; return cfg; }); @@ -3485,10 +3497,12 @@ describe("TaskService", () => { expect(findWorkspaceInConfig(config, staleStartingTaskId)?.taskStatus).toBe("running"); expect(findWorkspaceInConfig(config, freshStartingTaskId)?.taskStatus).toBe("starting"); + expect(findWorkspaceInConfig(config, stoppedStartingTaskId)?.taskStatus).toBe("interrupted"); const messagedWorkspaceIds = ( sendMessage as unknown as { mock: { calls: unknown[][] } } ).mock.calls.map((call) => call[0]); expect(messagedWorkspaceIds).not.toContain(freshStartingTaskId); + expect(messagedWorkspaceIds).not.toContain(stoppedStartingTaskId); }); test("initialize does not resend the restart nudge to a running task that is already streaming", async () => { @@ -3497,6 +3511,7 @@ describe("TaskService", () => { const parentWorkspaceId = "parent-restart-streaming"; const streamingTaskId = "child-running-streaming"; const idleTaskId = "child-running-idle"; + const stoppedTaskId = "child-running-stopped"; await saveWorkspaces( config, @@ -3517,13 +3532,34 @@ describe("TaskService", () => { taskStatus: "running", taskModelString: "openai:gpt-5.2", }), + projectWorkspace(projectPath, "stopped", stoppedTaskId, { + parentWorkspaceId, + agentId: "exec", + agentType: "exec", + taskStatus: "running", + taskModelString: "openai:gpt-5.2", + }), ], testTaskSettings() ); const isStreaming = mock((workspaceId: string) => workspaceId === streamingTaskId); const { aiService } = createAIServiceMocks(config, { isStreaming }); - const { workspaceService, sendMessage } = createWorkspaceServiceMocks(); + // A connected client stops one running task while recovery is resuming the others: the + // persisted `interrupted` status wins over the snapshot, so no restart nudge is sent. + const sendMessage = mock(async (workspaceId: string): Promise> => { + if (workspaceId === idleTaskId) { + await config.editConfig((cfg) => { + const stopped = cfg.projects + .get(projectPath) + ?.workspaces.find((workspace) => workspace.id === stoppedTaskId); + if (stopped) stopped.taskStatus = "interrupted"; + return cfg; + }); + } + return Ok(undefined); + }); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); await taskService.initialize(); @@ -3533,6 +3569,8 @@ describe("TaskService", () => { ).mock.calls.map((call) => call[0]); expect(messagedWorkspaceIds).toContain(idleTaskId); expect(messagedWorkspaceIds).not.toContain(streamingTaskId); + expect(messagedWorkspaceIds).not.toContain(stoppedTaskId); + expect(findWorkspaceInConfig(config, stoppedTaskId)?.taskStatus).toBe("interrupted"); }); test("initialize does not reload config.json per completed-report task", async () => { diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 73c2746b3d7..005bc8855e0 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -2301,7 +2301,10 @@ export class TaskService implements AgentTaskIntegration { const recovery = recoveries.get(task.id); assert(recovery != null, "stale starting task recovery is required"); const entry = findWorkspaceEntry(config, task.id); - if (!entry) continue; + // Compare-and-set: recovery may run while clients are connected, so a task_stop that + // already persisted `interrupted` (or any other transition) must not be overwritten + // with the snapshot-derived status and relaunched by the queue drain below. + if (entry?.workspace.taskStatus !== "starting") continue; entry.workspace.taskStatus = recovery.status; if (recovery.acceptedPrompt) { // The initial prompt is already durable in chat history; clearing taskPrompt makes the @@ -2404,6 +2407,7 @@ export class TaskService implements AgentTaskIntegration { let resumedRunningCount = 0; let skippedRunningDueToActiveDescendants = 0; let skippedRunningAlreadyStreaming = 0; + let skippedRunningNoLongerRunning = 0; let failedRunningCount = 0; for (const task of runningTasks) { @@ -2419,6 +2423,16 @@ export class TaskService implements AgentTaskIntegration { continue; } + // Recovery may run after the server is accepting connections. Re-read the live status so a + // task_stop that persisted `interrupted` since the snapshot is not undone by a restart nudge + // (sendMessage would treat that as a user-driven resume of the stopped task). + const liveStatus = findWorkspaceEntry(this.config.loadConfigOrDefault(), task.id)?.workspace + .taskStatus; + if (liveStatus !== "running") { + skippedRunningNoLongerRunning += 1; + continue; + } + const pendingGuidance = task.taskPendingGuidance ?? []; if (pendingGuidance.length > 0) { // Pending corrections outrank generic restart recovery and must replay even when this task @@ -2671,6 +2685,7 @@ export class TaskService implements AgentTaskIntegration { resumedRunningCount, skippedRunningDueToActiveDescendants, skippedRunningAlreadyStreaming, + skippedRunningNoLongerRunning, failedRunningCount, completedReportTaskCount: completedReportTasks.length, patchGenerationRecoveryMs, diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index b65b195a5f1..11843c1318b 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -7983,10 +7983,51 @@ describe("WorkspaceService initialize", () => { name: "Archived Workspace", archivedAt: "2026-03-20T00:00:00.000Z", }); + // Active when metadata was read, but archived (or removed) by a client before the + // scheduling loop ran: the live config decides, not the stale metadata. + const archivedSinceWorkspace = createFrontendWorkspaceMetadata({ + id: "archived-since-ws", + name: "Archived Since Read", + }); + const removedSinceWorkspace = createFrontendWorkspaceMetadata({ + id: "removed-since-ws", + name: "Removed Since Read", + }); config.getAllWorkspaceMetadata = mock(() => - Promise.resolve([liveWorkspace, taskWorkspace, archivedWorkspace]) + Promise.resolve([ + liveWorkspace, + taskWorkspace, + archivedWorkspace, + archivedSinceWorkspace, + removedSinceWorkspace, + ]) ) as unknown as Config["getAllWorkspaceMetadata"]; + config.loadConfigOrDefault = mock(() => ({ + projects: new Map([ + [ + "/tmp/project", + { + workspaces: [ + { id: "live-ws", name: "live-ws", path: "/tmp/live-ws" }, + { id: "task-ws", name: "task-ws", path: "/tmp/task-ws", taskStatus: "running" }, + { + id: "archived-ws", + name: "archived-ws", + path: "/tmp/archived-ws", + archivedAt: "2026-03-20T00:00:00.000Z", + }, + { + id: "archived-since-ws", + name: "archived-since-ws", + path: "/tmp/archived-since-ws", + archivedAt: "2026-03-21T00:00:00.000Z", + }, + ], + }, + ], + ]), + })) as unknown as Config["loadConfigOrDefault"]; const startupAccess = workspaceService as unknown as { startStartupRecovery: (workspaceId: string) => void; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index d4cfb6cf7cb..c4990609d49 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3178,13 +3178,18 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { let skippedTaskCount = 0; let skippedArchivedCount = 0; + // This can run while the server is already serving clients, and the cleanups above took + // time: re-read config right before the (synchronous) scheduling loop so a workspace archived + // or removed since `allMetadata` was read does not get a hidden recovery stream. + const liveConfig = this.config.loadConfigOrDefault(); for (const metadata of allMetadata) { if (metadata.taskStatus) { skippedTaskCount += 1; continue; } - if (isWorkspaceArchived(metadata.archivedAt, metadata.unarchivedAt)) { + const live = findWorkspaceEntry(liveConfig, metadata.id)?.workspace; + if (live == null || isWorkspaceArchived(live.archivedAt, live.unarchivedAt)) { skippedArchivedCount += 1; continue; } From 12549db6f8677840ed6268672c2cd2819433d420 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:49:03 +0000 Subject: [PATCH 04/26] fix(server): close remaining post-listen recovery races with task_stop and unarchive Startup recovery now runs while clients are connected, so every one-shot read it makes before a mutation can go stale: - The pending-guidance replay, restart nudge, and awaiting_report completion prompt carry an admissionStale probe. A task_stop that lands between the live status read and turn admission now refuses the send instead of resurrecting the stopped task through markInterruptedTaskRunning. - reconcileAgentTaskExecutionIds compare-and-sets its mirror write under the per-handle settlement lock, so a stop that settled the handle during the liveness check keeps its terminal mirror and no stale live registration is installed. - cleanupArchivedDevToolsLogs re-checks the live archive state under the task-tree lock before each deletion. DevToolsService.hasWorkspaceData gates that fresh config read so it only happens for actual deletions, not once per archived workspace. --- src/node/services/devToolsService.ts | 17 +++ src/node/services/taskService.test.ts | 146 +++++++++++++++++++++ src/node/services/taskService.ts | 25 +++- src/node/services/workspaceService.test.ts | 62 ++++++++- src/node/services/workspaceService.ts | 25 +++- src/node/services/workspaceTurnManager.ts | 51 ++++--- 6 files changed, 295 insertions(+), 31 deletions(-) diff --git a/src/node/services/devToolsService.ts b/src/node/services/devToolsService.ts index 5824864ff5c..7f967343110 100644 --- a/src/node/services/devToolsService.ts +++ b/src/node/services/devToolsService.ts @@ -410,6 +410,23 @@ export class DevToolsService extends EventEmitter { this.emit(`update:${workspaceId}`, event); } + /** Whether removeWorkspaceData() has anything to remove: live in-memory state or the on-disk log. */ + async hasWorkspaceData(workspaceId: string): Promise { + assert( + workspaceId.trim().length > 0, + "DevToolsService.hasWorkspaceData requires a workspaceId" + ); + if (this.workspaces.has(workspaceId)) { + return true; + } + try { + await fs.access(this.getSessionFilePath(workspaceId)); + return true; + } catch { + return false; + } + } + private getSessionFilePath(workspaceId: string): string { return path.join(this.config.sessionsDir, workspaceId, "devtools.jsonl"); } diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 0e054ccceb4..489907e6fcf 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -3573,6 +3573,152 @@ describe("TaskService", () => { expect(findWorkspaceInConfig(config, stoppedTaskId)?.taskStatus).toBe("interrupted"); }); + test("startup recovery sends refuse admission once a task_stop lands mid-send", async () => { + const config = await createTestConfig(rootDir); + const projectPath = path.join(rootDir, "repo"); + const parentWorkspaceId = "parent-stop-mid-send"; + const guidanceTaskId = "child-guidance-stopped-mid-send"; + const nudgeTaskId = "child-running-stopped-mid-send"; + const reportTaskId = "child-awaiting-report-stopped-mid-send"; + + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "parent", parentWorkspaceId), + projectWorkspace(projectPath, "guidance", guidanceTaskId, { + parentWorkspaceId, + agentId: "exec", + agentType: "exec", + taskStatus: "running", + taskModelString: "openai:gpt-5.2", + taskPendingGuidance: [ + { id: "guidance-1", message: "First correction", queueDispatchMode: "turn-end" }, + ], + }), + projectWorkspace(projectPath, "nudge", nudgeTaskId, { + parentWorkspaceId, + agentId: "exec", + agentType: "exec", + taskStatus: "running", + taskModelString: "openai:gpt-5.2", + }), + projectWorkspace(projectPath, "report", reportTaskId, { + parentWorkspaceId, + agentId: "explore", + agentType: "explore", + taskStatus: "awaiting_report", + taskModelString: "openai:gpt-5.2", + }), + ], + testTaskSettings() + ); + + // The host re-evaluates internal.admissionStale at enqueue and at the session's admission + // gates. Model a task_stop persisting `interrupted` inside that window: the probe must flip + // from live to stale on the same send, and a stale send is refused (never resumed). + const probeBeforeStop: Record = {}; + const probeAfterStop: Record = {}; + const sendMessage = mock( + async ( + workspaceId: string, + _message: string, + _options: unknown, + internal?: { admissionStale?: () => boolean; onAccepted?: () => Promise | void } + ): Promise> => { + probeBeforeStop[workspaceId] = internal?.admissionStale?.(); + await config.editConfig((cfg) => { + const stopped = cfg.projects + .get(projectPath) + ?.workspaces.find((workspace) => workspace.id === workspaceId); + if (stopped) stopped.taskStatus = "interrupted"; + return cfg; + }); + probeAfterStop[workspaceId] = internal?.admissionStale?.(); + return Err("stopped before admission"); + } + ); + const { aiService } = createAIServiceMocks(config); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); + + await taskService.initialize(); + + for (const taskId of [guidanceTaskId, nudgeTaskId, reportTaskId]) { + expect(probeBeforeStop[taskId]).toBe(false); + expect(probeAfterStop[taskId]).toBe(true); + expect(findWorkspaceInConfig(config, taskId)?.taskStatus).toBe("interrupted"); + } + // A refused guidance replay keeps its durable reservation for the next resume. + expect(findWorkspaceInConfig(config, guidanceTaskId)?.taskPendingGuidance).toHaveLength(1); + }); + + test("initialize does not restore an execution mirror that a stop settled during reconciliation", async () => { + const config = await createTestConfig(rootDir); + const { parentId, projectPath } = await saveLocalParentWorkspace(config, rootDir); + const childTaskId = "child-stopped-during-reconcile"; + const handleId = "wst_stopped_during_reconcile"; + await config.editConfig((cfg) => { + const project = cfg.projects.get(projectPath); + assert(project, "test project must exist"); + project.workspaces.push( + projectWorkspace(projectPath, "child-reconcile", childTaskId, { + parentWorkspaceId: parentId, + agentId: "explore", + agentType: "explore", + taskStatus: "reported", + reportedAt: "2026-08-10T00:00:00.000Z", + taskExecutionId: handleId, + taskExecutionStatus: "running", + }) + ); + return cfg; + }); + const isStreaming = mock((workspaceId: string) => workspaceId === childTaskId); + const { aiService } = createAIServiceMocks(config, { isStreaming }); + const { taskService } = createTaskServiceHarness(config, { aiService }); + const internals = taskService as unknown as { + taskHandleStore: TaskHandleStore; + workspaceTurnManager: WorkspaceTurnManager; + activeWorkspaceTurnHandleByWorkspaceId: Map; + }; + await internals.taskHandleStore.upsertWorkspaceTurn( + workspaceTurnRecord(parentId, childTaskId, handleId, "running", { + turnId: "turn-reconcile", + createdAt: "2026-08-10T00:00:01.000Z", + updatedAt: "2026-08-10T00:00:01.000Z", + }) + ); + + // A task_stop settles the handle while reconciliation is between its liveness check and its + // mirror write: the persisted terminal state must win over the pre-await snapshot. + const manager = internals.workspaceTurnManager; + const originalNormalize = manager.normalizeWorkspaceTurnRecord.bind(manager); + spyOn(manager, "normalizeWorkspaceTurnRecord").mockImplementation(async (record, options) => { + const normalized = await originalNormalize(record, options); + if (normalized?.handleId === handleId) { + await internals.taskHandleStore.upsertWorkspaceTurn({ + ...normalized, + status: "interrupted", + updatedAt: "2026-08-10T00:00:02.000Z", + }); + await config.editConfig((cfg) => { + const child = cfg.projects + .get(projectPath) + ?.workspaces.find((workspace) => workspace.id === childTaskId); + if (child) child.taskExecutionStatus = "interrupted"; + return cfg; + }); + } + return normalized; + }); + + await taskService.initialize(); + + expect(findWorkspaceInConfig(config, childTaskId)?.taskExecutionStatus).toBe("interrupted"); + expect(internals.activeWorkspaceTurnHandleByWorkspaceId.get(childTaskId)).toBeUndefined(); + }); + test("initialize does not reload config.json per completed-report task", async () => { const countConfigLoadsDuringInitialize = async (reportedTaskCount: number): Promise => { const runRootDir = path.join(rootDir, `run-${reportedTaskCount}`); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 005bc8855e0..0ad78c5de3f 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -2425,10 +2425,14 @@ export class TaskService implements AgentTaskIntegration { // Recovery may run after the server is accepting connections. Re-read the live status so a // task_stop that persisted `interrupted` since the snapshot is not undone by a restart nudge - // (sendMessage would treat that as a user-driven resume of the stopped task). - const liveStatus = findWorkspaceEntry(this.config.loadConfigOrDefault(), task.id)?.workspace - .taskStatus; - if (liveStatus !== "running") { + // (sendMessage would treat that as a user-driven resume of the stopped task). The same check + // rides along as the send's admission probe: a stop can also land during the awaits between + // here and turn admission, and a guarded send skips the interrupted-task rescue entirely. + const taskId = task.id; + const admissionStale = (): boolean => + findWorkspaceEntry(this.config.loadConfigOrDefault(), taskId)?.workspace.taskStatus !== + "running"; + if (admissionStale()) { skippedRunningNoLongerRunning += 1; continue; } @@ -2469,6 +2473,7 @@ export class TaskService implements AgentTaskIntegration { { synthetic: true, agentInitiated: true, + admissionStale, onAccepted: clearAcceptedPendingGuidance, } ); @@ -2531,7 +2536,7 @@ export class TaskService implements AgentTaskIntegration { reasoningMode: coerceOpenAIReasoningMode(task.aiSettings?.reasoningMode), experiments: task.taskExperiments, }, - { synthetic: true, agentInitiated: true } + { synthetic: true, agentInitiated: true, admissionStale } ); const durationMs = Date.now() - resumeStartedAt; if (!sendResult.success) { @@ -10471,7 +10476,15 @@ export class TaskService implements AgentTaskIntegration { ? { toolPolicy: [{ regex_match: "^propose_plan$", action: "require" as const }] } : {}), }, - { synthetic: true, agentInitiated: true } + { + synthetic: true, + agentInitiated: true, + // The status check at the top of this method is a one-shot read; a task_stop that lands + // during the awaits since then must refuse admission rather than be resurrected. + admissionStale: () => + findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId)?.workspace + .taskStatus !== "awaiting_report", + } ); const durationMs = Date.now() - startedAt; if (!sendResult.success) { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 11843c1318b..91d90209f4e 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -8250,12 +8250,64 @@ describe("WorkspaceService initialize", () => { name: "Archived Workspace", archivedAt: "2026-03-20T00:00:00.000Z", }); + // Archived when metadata was read, but a client unarchived it (and produced new logs) + // before the sweep reached it: the live config decides. + const unarchivedSinceWorkspace = createFrontendWorkspaceMetadata({ + id: "unarchived-since-ws", + name: "Unarchived Since Read", + archivedAt: "2026-03-20T00:00:00.000Z", + }); + const archivedWithoutDataWorkspace = createFrontendWorkspaceMetadata({ + id: "archived-no-data-ws", + name: "Archived Without DevTools Data", + archivedAt: "2026-03-20T00:00:00.000Z", + }); config.getAllWorkspaceMetadata = mock(() => - Promise.resolve([liveWorkspace, archivedWorkspace]) + Promise.resolve([ + liveWorkspace, + archivedWorkspace, + unarchivedSinceWorkspace, + archivedWithoutDataWorkspace, + ]) ) as unknown as Config["getAllWorkspaceMetadata"]; + config.loadConfigOrDefault = mock(() => ({ + projects: new Map([ + [ + "/tmp/project", + { + workspaces: [ + { id: "live-ws", name: "live-ws", path: "/tmp/live-ws" }, + { + id: "archived-ws", + name: "archived-ws", + path: "/tmp/archived-ws", + archivedAt: "2026-03-20T00:00:00.000Z", + }, + { + id: "unarchived-since-ws", + name: "unarchived-since-ws", + path: "/tmp/unarchived-since-ws", + archivedAt: "2026-03-20T00:00:00.000Z", + unarchivedAt: "2026-03-21T00:00:00.000Z", + }, + { + id: "archived-no-data-ws", + name: "archived-no-data-ws", + path: "/tmp/archived-no-data-ws", + archivedAt: "2026-03-20T00:00:00.000Z", + }, + ], + }, + ], + ]), + })) as unknown as Config["loadConfigOrDefault"]; const removeWorkspaceData = mock(() => Promise.resolve()); - workspaceService.setDevToolsService({ removeWorkspaceData }); + workspaceService.setDevToolsService({ + hasWorkspaceData: (workspaceId: string) => + Promise.resolve(workspaceId !== "archived-no-data-ws"), + removeWorkspaceData, + }); const startupAccess = workspaceService as unknown as { startStartupRecovery: (workspaceId: string) => void; @@ -13662,7 +13714,10 @@ describe("WorkspaceService archive lifecycle hooks", () => { expect(entry?.archivedAt).toBeTruthy(); return Promise.resolve(); }); - workspaceService.setDevToolsService({ removeWorkspaceData }); + workspaceService.setDevToolsService({ + hasWorkspaceData: () => Promise.resolve(true), + removeWorkspaceData, + }); const result = await workspaceService.archive(workspaceId); @@ -13672,6 +13727,7 @@ describe("WorkspaceService archive lifecycle hooks", () => { test("archive() stays successful when DevTools cleanup fails", async () => { workspaceService.setDevToolsService({ + hasWorkspaceData: () => Promise.resolve(true), removeWorkspaceData: mock(() => Promise.reject(new Error("disk error"))), }); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index c4990609d49..5886ffff58a 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -322,6 +322,7 @@ import { } from "@/node/services/taskWorkspaceSeam"; import { findWorkspaceEntry } from "@/node/services/taskUtils"; import type { WorktreeArchiveSnapshotService } from "@/node/services/worktreeArchiveSnapshotService"; +import type { DevToolsService } from "@/node/services/devToolsService"; import { DisposableTempDir } from "@/node/services/tempDir"; import { createBashTool } from "@/node/services/tools/bash"; @@ -634,6 +635,9 @@ interface WorkspaceAgentStatus { url?: string; } type WorkspaceRuntimeStatus = "running" | "stopped" | "unknown" | "unsupported"; + +/** Narrow DevTools cleanup surface used on archive/remove and by the startup sweep. */ +type WorkspaceDevToolsCleanup = Pick; const POST_COMPACTION_METADATA_REFRESH_DEBOUNCE_MS = 100; const DESCENDANT_WORKSPACE_REMOVE_ERROR = @@ -2594,7 +2598,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { private agentTaskIntegration?: AgentTaskIntegration; private workspaceGoalService?: WorkspaceGoalService; /** Narrow DevTools cleanup surface; wired by coreServices when a DevToolsService exists. */ - private devToolsService?: { removeWorkspaceData(workspaceId: string): Promise }; + private devToolsService?: WorkspaceDevToolsCleanup; /** Cancels running /refine passes before removal deletes the session dir; wired post-construction (RefineService is built later). */ private refinePassCanceller?: { cancelInFlightRefinePass(workspaceId: string): Promise }; /** Narrow overrides-cleanup surface; wired by ServiceContainer for stale plugin-key sanitization. */ @@ -2942,7 +2946,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } /** DevTools debug-log cleanup on archive/remove; wired by coreServices. */ - setDevToolsService(service: { removeWorkspaceData(workspaceId: string): Promise }): void { + setDevToolsService(service: WorkspaceDevToolsCleanup): void { this.devToolsService = service; } @@ -4501,10 +4505,25 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return; } + const devToolsService = this.devToolsService; for (const metadata of allMetadata) { if (!isWorkspaceArchived(metadata.archivedAt, metadata.unarchivedAt)) continue; try { - await this.devToolsService.removeWorkspaceData(metadata.id); + // archive() already removed the log for most archived workspaces, so gate the fresh config + // read below on there being something to delete: a per-workspace reload for every archived + // entry would scale this sweep with (archived workspaces x config size). + if (!(await devToolsService.hasWorkspaceData(metadata.id))) continue; + // This sweep can run while the server is serving clients. Unarchive runs under the same + // lock, so re-checking the live config inside it means a workspace unarchived since + // `allMetadata` was read keeps the logs it has produced since. + await this.withTaskTreeLifecycleLock(metadata.id, async () => { + const live = findWorkspaceEntry( + this.config.loadConfigOrDefault(), + metadata.id + )?.workspace; + if (live == null || !isWorkspaceArchived(live.archivedAt, live.unarchivedAt)) return; + await devToolsService.removeWorkspaceData(metadata.id); + }); } catch (error: unknown) { log.debug("Failed to remove DevTools log for archived workspace", { workspaceId: metadata.id, diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index b74f3b9d6ed..592d12e0ff5 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -4830,25 +4830,38 @@ export class WorkspaceTurnManager { continue; } - await this.taskHost.editWorkspaceEntry( - task.id, - (workspace) => { - workspace.taskExecutionId = normalized.handleId; - workspace.taskExecutionStatus = normalized.status; - }, - { allowMissing: true } - ); - await this.taskHost.emitWorkspaceMetadata(task.id); - if (isActiveWorkspaceTurnTaskStatus(normalized.status)) { - this.activeWorkspaceTurnHandleByWorkspaceId.set(task.id, { - handleId: normalized.handleId, - ownerWorkspaceId: normalized.ownerWorkspaceId, - // Fail closed: the restart killed any live stream, so this recovered registration - // exists for settlement ownership, not peer admission. A revival or a fresh - // acceptance re-marks the entry accepted. - accepted: false, - }); - } + const taskId = task.id; + const { handleId, ownerWorkspaceId, status } = normalized; + // This pass can run while clients are connected, and normalizeWorkspaceTurnRecord awaited: + // a task_stop may have settled this handle in the meantime. Settlement persists the record + // and its mirror under the per-handle settlement lock, so compare-and-set under that same + // lock; writing the pre-await snapshot would restore an active taskExecutionStatus (and a + // live registration) for a child that was just stopped. + await this.workspaceTurnSettlementLocks.withLock(handleId, async () => { + const current = await this.taskHandleStore.getWorkspaceTurn(ownerWorkspaceId, handleId); + if (current?.status !== status) { + return; + } + await this.taskHost.editWorkspaceEntry( + taskId, + (workspace) => { + workspace.taskExecutionId = handleId; + workspace.taskExecutionStatus = status; + }, + { allowMissing: true } + ); + await this.taskHost.emitWorkspaceMetadata(taskId); + if (isActiveWorkspaceTurnTaskStatus(status)) { + this.activeWorkspaceTurnHandleByWorkspaceId.set(taskId, { + handleId, + ownerWorkspaceId, + // Fail closed: the restart killed any live stream, so this recovered registration + // exists for settlement ownership, not peer admission. A revival or a fresh + // acceptance re-marks the entry accepted. + accepted: false, + }); + } + }); } catch (error: unknown) { // Startup recovery is best-effort: one read-only/corrupt child must not prevent Xum startup // or block reconciliation of the remaining persistent children. From f0958880845b8ae0bc99d9afef944f4d5a15b471 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:54:13 +0200 Subject: [PATCH 05/26] fix(analytics): wait for the worker to close DuckDB before process exit Now that the graceful SIGINT handler is installed while the startup DuckDB sync may still be running, exiting the process mid-sync tore the worker down inside native DuckDB and aborted the process (Napi::Error -> SIGABRT, 7/8 reproductions in UAT). AnalyticsService.dispose() now waits (bounded by ANALYTICS_WORKER_SHUTDOWN_TIMEOUT_MS) for the worker to exit after posting the shutdown message. --- src/constants/terminationTimeouts.ts | 9 +++++ .../analytics/analyticsService.test.ts | 34 ++++++++++++++++++- .../services/analytics/analyticsService.ts | 32 +++++++++++++---- 3 files changed, 68 insertions(+), 7 deletions(-) diff --git a/src/constants/terminationTimeouts.ts b/src/constants/terminationTimeouts.ts index 0f69546a8a8..b37fa3600dd 100644 --- a/src/constants/terminationTimeouts.ts +++ b/src/constants/terminationTimeouts.ts @@ -22,3 +22,12 @@ export const APP_RUNTIME_DISPOSE_TIMEOUT_MS = 2 * 1000; * bound this fits inside the same 5 s quit budgets. */ export const APP_FIBER_SCOPE_CLOSE_TIMEOUT_MS = 2 * 1000; + +/** + * Bounds how long `AnalyticsService.dispose()` waits for the analytics worker to finish + * in-flight DuckDB work and exit. The worker closes DuckDB only after queued ETL tasks + * complete, so exiting the process before it does tears the thread down mid-query and aborts + * the process (Napi::Error -> SIGABRT). Sized to fit inside the 5 s quit budgets alongside the + * runtime and fiber-scope bounds above. + */ +export const ANALYTICS_WORKER_SHUTDOWN_TIMEOUT_MS = 2 * 1000; diff --git a/src/node/services/analytics/analyticsService.test.ts b/src/node/services/analytics/analyticsService.test.ts index 4d58902f657..597abad9ad5 100644 --- a/src/node/services/analytics/analyticsService.test.ts +++ b/src/node/services/analytics/analyticsService.test.ts @@ -1,6 +1,8 @@ import { describe, expect, test } from "bun:test"; +import { EventEmitter } from "node:events"; +import type { Config } from "@/node/config"; import type { ProviderCacheHitModelRow } from "@/common/orpc/schemas/analytics"; -import { aggregateProviderCacheHitRows } from "./analyticsService"; +import { AnalyticsService, aggregateProviderCacheHitRows } from "./analyticsService"; describe("aggregateProviderCacheHitRows", () => { test("rolls model rows up to providers using weighted token ratios", () => { @@ -89,3 +91,33 @@ describe("aggregateProviderCacheHitRows", () => { ]); }); }); + +describe("AnalyticsService.dispose", () => { + test("waits for the worker to exit after posting shutdown so DuckDB closes before process exit", async () => { + const service = new AnalyticsService({ rootDir: "/tmp/unused" } as unknown as Config); + const fakeWorker = new EventEmitter() as EventEmitter & { + postMessage: (message: unknown) => void; + unref: () => void; + }; + const posted: unknown[] = []; + fakeWorker.postMessage = (message: unknown) => { + posted.push(message); + }; + fakeWorker.unref = () => undefined; + (service as unknown as { worker: unknown }).worker = fakeWorker; + + let disposed = false; + const disposePromise = service.dispose().then(() => { + disposed = true; + }); + await Promise.resolve(); + await Promise.resolve(); + expect(posted).toEqual([{ type: "shutdown" }]); + // The worker is still closing DuckDB: dispose must not resolve yet. + expect(disposed).toBe(false); + + fakeWorker.emit("exit", 0); + await disposePromise; + expect(disposed).toBe(true); + }); +}); diff --git a/src/node/services/analytics/analyticsService.ts b/src/node/services/analytics/analyticsService.ts index 91708bd1b1c..f367a75153e 100644 --- a/src/node/services/analytics/analyticsService.ts +++ b/src/node/services/analytics/analyticsService.ts @@ -26,6 +26,7 @@ import { ensurePrivateDir } from "@/node/utils/fs"; import type { Config } from "@/node/config"; import { PlatformPaths } from "@/common/utils/paths"; import { log } from "@/node/services/log"; +import { ANALYTICS_WORKER_SHUTDOWN_TIMEOUT_MS } from "@/constants/terminationTimeouts"; import type { RawQueryResult } from "./queries"; interface WorkerRequest { @@ -840,13 +841,11 @@ export class AnalyticsService { } dispose(): Promise { - this.disposePromise ??= Promise.resolve().then(() => { - this.disposeInternal(); - }); + this.disposePromise ??= Promise.resolve().then(() => this.disposeInternal()); return this.disposePromise; } - private disposeInternal(): void { + private async disposeInternal(): Promise { this.isDisposed = true; const disposedError = new Error("Analytics service is shutting down"); @@ -865,14 +864,35 @@ export class AnalyticsService { worker.off("error", this.onWorkerError); worker.off("exit", this.onWorkerExit); - // Shut down DuckDB from inside the worker thread first. The worker is - // already unref'd, so process shutdown does not wait for this cleanup. + // Shut down DuckDB from inside the worker thread first. The worker is unref'd, so the + // process would not wait for it on its own; wait (bounded) for it to exit here because + // the worker closes DuckDB only after in-flight ETL finishes, and exiting the process + // mid-sync (e.g. Ctrl-C during the startup sync) tears the thread down inside native + // DuckDB code and aborts the whole process. + const exited = new Promise((resolve) => { + worker.once("exit", () => resolve()); + worker.once("error", () => resolve()); + }); try { worker.postMessage({ type: "shutdown" } satisfies WorkerShutdownMessage); } catch (error) { log.warn("[AnalyticsService] Failed to post graceful shutdown message to analytics worker", { error: getErrorMessage(error), }); + return; + } + let timer: ReturnType | undefined; + const timedOut = new Promise((resolve) => { + timer = setTimeout(() => resolve(true), ANALYTICS_WORKER_SHUTDOWN_TIMEOUT_MS); + }); + try { + if (await Promise.race([exited.then(() => false), timedOut])) { + log.warn("[AnalyticsService] Analytics worker did not exit before the shutdown timeout", { + timeoutMs: ANALYTICS_WORKER_SHUTDOWN_TIMEOUT_MS, + }); + } + } finally { + clearTimeout(timer); } } From 02cfc0a53a68e85f31937101da4c1b75a811d5d6 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:17:24 +0000 Subject: [PATCH 06/26] fix: wait for the analytics worker's real exit; keep a concurrently installed execution handle - AnalyticsService.dispose() now waits for the worker's exit event with no local bound. A 2 s timeout let dispose resolve while an ingest still held DuckDB open, so the caller's process.exit() reproduced the SIGABRT the wait exists to prevent. The outer quit budgets in cli/server.ts and desktop/main.ts already race the whole dispose and own the hard-exit decision. A worker that already exited short-circuits the wait. - reconcileAgentTaskExecutionIds also compare-and-sets the workspace mirror pointer against its snapshot. A client that installs a new handle while reconciliation awaits the old handle's liveness check writes the mirror under a different settlement lock and leaves the old record unchanged, so the record-status check alone let the stale handle overwrite it. --- src/constants/terminationTimeouts.ts | 9 -- .../analytics/analyticsService.test.ts | 26 ++++++ .../services/analytics/analyticsService.ts | 33 ++++---- src/node/services/taskService.test.ts | 83 +++++++++++++++++++ src/node/services/workspaceTurnManager.ts | 28 +++++-- 5 files changed, 145 insertions(+), 34 deletions(-) diff --git a/src/constants/terminationTimeouts.ts b/src/constants/terminationTimeouts.ts index b37fa3600dd..0f69546a8a8 100644 --- a/src/constants/terminationTimeouts.ts +++ b/src/constants/terminationTimeouts.ts @@ -22,12 +22,3 @@ export const APP_RUNTIME_DISPOSE_TIMEOUT_MS = 2 * 1000; * bound this fits inside the same 5 s quit budgets. */ export const APP_FIBER_SCOPE_CLOSE_TIMEOUT_MS = 2 * 1000; - -/** - * Bounds how long `AnalyticsService.dispose()` waits for the analytics worker to finish - * in-flight DuckDB work and exit. The worker closes DuckDB only after queued ETL tasks - * complete, so exiting the process before it does tears the thread down mid-query and aborts - * the process (Napi::Error -> SIGABRT). Sized to fit inside the 5 s quit budgets alongside the - * runtime and fiber-scope bounds above. - */ -export const ANALYTICS_WORKER_SHUTDOWN_TIMEOUT_MS = 2 * 1000; diff --git a/src/node/services/analytics/analyticsService.test.ts b/src/node/services/analytics/analyticsService.test.ts index 597abad9ad5..ed3d348d1ff 100644 --- a/src/node/services/analytics/analyticsService.test.ts +++ b/src/node/services/analytics/analyticsService.test.ts @@ -120,4 +120,30 @@ describe("AnalyticsService.dispose", () => { await disposePromise; expect(disposed).toBe(true); }); + + test("dispose does not wait on a worker that already exited", async () => { + const service = new AnalyticsService({ rootDir: "/tmp/unused" } as unknown as Config); + const fakeWorker = new EventEmitter() as EventEmitter & { + postMessage: (message: unknown) => void; + unref: () => void; + }; + fakeWorker.postMessage = () => undefined; + fakeWorker.unref = () => undefined; + const internals = service as unknown as { + worker: unknown; + onWorkerExit: (code: number) => void; + }; + internals.worker = fakeWorker; + fakeWorker.on("exit", internals.onWorkerExit); + // The worker crashed earlier; nothing is left to close and no exit event will follow. + fakeWorker.emit("exit", 1); + + let disposed = false; + const disposePromise = service.dispose().then(() => { + disposed = true; + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(disposed).toBe(true); + await disposePromise; + }); }); diff --git a/src/node/services/analytics/analyticsService.ts b/src/node/services/analytics/analyticsService.ts index f367a75153e..e6530c26a7a 100644 --- a/src/node/services/analytics/analyticsService.ts +++ b/src/node/services/analytics/analyticsService.ts @@ -26,7 +26,6 @@ import { ensurePrivateDir } from "@/node/utils/fs"; import type { Config } from "@/node/config"; import { PlatformPaths } from "@/common/utils/paths"; import { log } from "@/node/services/log"; -import { ANALYTICS_WORKER_SHUTDOWN_TIMEOUT_MS } from "@/constants/terminationTimeouts"; import type { RawQueryResult } from "./queries"; interface WorkerRequest { @@ -201,6 +200,7 @@ interface AnalyticsFilterInput { export class AnalyticsService { private worker: Worker | null = null; + private workerExited = false; private messageIdCounter = 0; private readonly pendingPromises = new Map< number, @@ -324,6 +324,7 @@ export class AnalyticsService { }; private readonly onWorkerExit = (code: number): void => { + this.workerExited = true; if (code === 0) { return; } @@ -864,14 +865,20 @@ export class AnalyticsService { worker.off("error", this.onWorkerError); worker.off("exit", this.onWorkerExit); + // A worker that already exited (crash or clean stop) has nothing left to close. + if (this.workerExited) { + return; + } + // Shut down DuckDB from inside the worker thread first. The worker is unref'd, so the - // process would not wait for it on its own; wait (bounded) for it to exit here because - // the worker closes DuckDB only after in-flight ETL finishes, and exiting the process - // mid-sync (e.g. Ctrl-C during the startup sync) tears the thread down inside native - // DuckDB code and aborts the whole process. + // process would not wait for it on its own; wait for its actual exit here because the + // worker closes DuckDB only after in-flight ETL finishes, and exiting the process mid-sync + // (e.g. Ctrl-C during the startup sync) tears the thread down inside native DuckDB code and + // aborts the whole process. Deliberately unbounded: an ETL that outlives a local timeout + // would still be mid-query when the caller exits, so the hard-exit decision belongs to the + // outer quit budgets in cli/server.ts and desktop/main.ts, which race the whole dispose. const exited = new Promise((resolve) => { worker.once("exit", () => resolve()); - worker.once("error", () => resolve()); }); try { worker.postMessage({ type: "shutdown" } satisfies WorkerShutdownMessage); @@ -881,19 +888,7 @@ export class AnalyticsService { }); return; } - let timer: ReturnType | undefined; - const timedOut = new Promise((resolve) => { - timer = setTimeout(() => resolve(true), ANALYTICS_WORKER_SHUTDOWN_TIMEOUT_MS); - }); - try { - if (await Promise.race([exited.then(() => false), timedOut])) { - log.warn("[AnalyticsService] Analytics worker did not exit before the shutdown timeout", { - timeoutMs: ANALYTICS_WORKER_SHUTDOWN_TIMEOUT_MS, - }); - } - } finally { - clearTimeout(timer); - } + await exited; } clearWorkspace( diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 489907e6fcf..b21efe2856b 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -3719,6 +3719,89 @@ describe("TaskService", () => { expect(internals.activeWorkspaceTurnHandleByWorkspaceId.get(childTaskId)).toBeUndefined(); }); + test("initialize does not overwrite an execution handle a client installed during reconciliation", async () => { + const config = await createTestConfig(rootDir); + const { parentId, projectPath } = await saveLocalParentWorkspace(config, rootDir); + const childTaskId = "child-replaced-during-reconcile"; + const oldHandleId = "wst_replaced_old"; + const newHandleId = "wst_replaced_new"; + await config.editConfig((cfg) => { + const project = cfg.projects.get(projectPath); + assert(project, "test project must exist"); + project.workspaces.push( + projectWorkspace(projectPath, "child-replaced", childTaskId, { + parentWorkspaceId: parentId, + agentId: "explore", + agentType: "explore", + taskStatus: "reported", + reportedAt: "2026-08-10T00:00:00.000Z", + taskExecutionId: oldHandleId, + taskExecutionStatus: "running", + }) + ); + return cfg; + }); + const isStreaming = mock((workspaceId: string) => workspaceId === childTaskId); + const { aiService } = createAIServiceMocks(config, { isStreaming }); + const { taskService } = createTaskServiceHarness(config, { aiService }); + const internals = taskService as unknown as { + taskHandleStore: TaskHandleStore; + workspaceTurnManager: WorkspaceTurnManager; + activeWorkspaceTurnHandleByWorkspaceId: Map< + string, + { handleId: string; ownerWorkspaceId: string; accepted: boolean } + >; + }; + await internals.taskHandleStore.upsertWorkspaceTurn( + workspaceTurnRecord(parentId, childTaskId, oldHandleId, "running", { + turnId: "turn-replaced-old", + createdAt: "2026-08-10T00:00:01.000Z", + updatedAt: "2026-08-10T00:00:01.000Z", + }) + ); + + // While reconciliation awaits the old handle's liveness check, a client reawakens the child + // under a new handle: it persists the new record, repoints the mirror, and registers the new + // handle under its own settlement lock. The old record itself is untouched. + const manager = internals.workspaceTurnManager; + const originalNormalize = manager.normalizeWorkspaceTurnRecord.bind(manager); + spyOn(manager, "normalizeWorkspaceTurnRecord").mockImplementation(async (record, options) => { + const normalized = await originalNormalize(record, options); + if (normalized?.handleId === oldHandleId) { + await internals.taskHandleStore.upsertWorkspaceTurn( + workspaceTurnRecord(parentId, childTaskId, newHandleId, "running", { + turnId: "turn-replaced-new", + createdAt: "2026-08-10T00:00:03.000Z", + updatedAt: "2026-08-10T00:00:03.000Z", + }) + ); + await config.editConfig((cfg) => { + const child = cfg.projects + .get(projectPath) + ?.workspaces.find((workspace) => workspace.id === childTaskId); + if (child) { + child.taskExecutionId = newHandleId; + child.taskExecutionStatus = "running"; + } + return cfg; + }); + internals.activeWorkspaceTurnHandleByWorkspaceId.set(childTaskId, { + handleId: newHandleId, + ownerWorkspaceId: parentId, + accepted: true, + }); + } + return normalized; + }); + + await taskService.initialize(); + + expect(findWorkspaceInConfig(config, childTaskId)?.taskExecutionId).toBe(newHandleId); + expect(internals.activeWorkspaceTurnHandleByWorkspaceId.get(childTaskId)?.handleId).toBe( + newHandleId + ); + }); + test("initialize does not reload config.json per completed-report task", async () => { const countConfigLoadsDuringInitialize = async (reportedTaskCount: number): Promise => { const runRootDir = path.join(rootDir, `run-${reportedTaskCount}`); diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index 592d12e0ff5..8616b9d144e 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -4832,26 +4832,42 @@ export class WorkspaceTurnManager { const taskId = task.id; const { handleId, ownerWorkspaceId, status } = normalized; - // This pass can run while clients are connected, and normalizeWorkspaceTurnRecord awaited: - // a task_stop may have settled this handle in the meantime. Settlement persists the record - // and its mirror under the per-handle settlement lock, so compare-and-set under that same - // lock; writing the pre-await snapshot would restore an active taskExecutionStatus (and a - // live registration) for a child that was just stopped. + // This pass can run while clients are connected, and normalizeWorkspaceTurnRecord awaited, + // so two things may have changed underneath the snapshot: a task_stop may have settled + // this handle (settlement persists the record and its mirror under the per-handle + // settlement lock, so compare-and-set the record status under that same lock), or a + // client may have installed a different handle for this task, which writes the mirror + // under its own handle's lock and leaves this record untouched, so the mirror pointer + // itself is compare-and-set against the snapshot as well. Writing the stale snapshot in + // either case would restore an active status or registration the live state has moved + // past. await this.workspaceTurnSettlementLocks.withLock(handleId, async () => { const current = await this.taskHandleStore.getWorkspaceTurn(ownerWorkspaceId, handleId); if (current?.status !== status) { return; } + let mirrorUpdated = false; await this.taskHost.editWorkspaceEntry( taskId, (workspace) => { + if (workspace.taskExecutionId !== task.taskExecutionId) { + return; + } workspace.taskExecutionId = handleId; workspace.taskExecutionStatus = status; + mirrorUpdated = true; }, { allowMissing: true } ); + if (!mirrorUpdated) { + return; + } await this.taskHost.emitWorkspaceMetadata(taskId); - if (isActiveWorkspaceTurnTaskStatus(status)) { + const live = this.activeWorkspaceTurnHandleByWorkspaceId.get(taskId); + if ( + isActiveWorkspaceTurnTaskStatus(status) && + (live == null || live.handleId === handleId) + ) { this.activeWorkspaceTurnHandleByWorkspaceId.set(taskId, { handleId, ownerWorkspaceId, From b460a3e68207188ab6525d1103338ad1a3e52ef2 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:58:22 +0200 Subject: [PATCH 07/26] fix(server): make post-listen recovery safe against workflow resumes and removed tasks - interruptTaskRecoveryForInactiveWorkflowOwner leaves a child alone when a workflow resume is admitted for the run's workspace (checked synchronously inside the config edit via the workflow admission registry), so recovery cannot mark a freshly resumed run's child interrupted. - maybeStartGeneration re-reads config before creating a pending artifact for a snapshot child that has no artifact yet, so a task removed since the startup snapshot does not get a new artifact. --- .../services/gitPatchArtifactService.test.ts | 15 +++++ src/node/services/gitPatchArtifactService.ts | 13 +++++ src/node/services/taskService.test.ts | 58 +++++++++++++++++++ src/node/services/taskService.ts | 28 +++++++++ 4 files changed, 114 insertions(+) diff --git a/src/node/services/gitPatchArtifactService.test.ts b/src/node/services/gitPatchArtifactService.test.ts index 7c07d9a95c8..a71d52e605c 100644 --- a/src/node/services/gitPatchArtifactService.test.ts +++ b/src/node/services/gitPatchArtifactService.test.ts @@ -95,6 +95,7 @@ describe("GitPatchArtifactService coordination", () => { const parentId = "parent"; const childId = "child-reported"; const freshChildId = "child-fresh"; + const removedChildId = "child-removed-since-snapshot"; const execChild = (id: string) => ({ path: path.join(projectPath, id), id, @@ -110,6 +111,7 @@ describe("GitPatchArtifactService coordination", () => { { path: projectPath, id: parentId, name: "parent", runtimeConfig: { type: "local" } }, execChild(childId), execChild(freshChildId), + execChild(removedChildId), ], }); return cfg; @@ -170,6 +172,19 @@ describe("GitPatchArtifactService coordination", () => { expect(await readSubagentGitPatchArtifact(parentSessionDir, unknownChildId)).toBeNull(); expect(completedChildIds).toEqual([]); + // A snapshot child that a client removed since the snapshot must not get a fresh pending + // artifact: creating one re-reads config first (an existing artifact would not need this). + await config.editConfig((cfg) => { + const project = cfg.projects.get(projectPath); + if (project) { + project.workspaces = project.workspaces.filter((ws) => ws.id !== removedChildId); + } + return cfg; + }); + await service.maybeStartGeneration(parentId, removedChildId, onComplete, { config: snapshot }); + expect(await readSubagentGitPatchArtifact(parentSessionDir, removedChildId)).toBeNull(); + expect(completedChildIds).toEqual([]); + // A snapshot child without an artifact still takes the normal generation path (pending // marker written, background job started), proving the snapshot feeds the real branch. await service.maybeStartGeneration(parentId, freshChildId, onComplete, { config: snapshot }); diff --git a/src/node/services/gitPatchArtifactService.ts b/src/node/services/gitPatchArtifactService.ts index 6b9f159affc..b551a561cc4 100644 --- a/src/node/services/gitPatchArtifactService.ts +++ b/src/node/services/gitPatchArtifactService.ts @@ -411,6 +411,19 @@ export class GitPatchArtifactService { return; } + if (options?.config != null) { + // The caller's snapshot can predate a client removing this task. The updater below leaves an + // existing artifact untouched, so only a task with no artifact yet would get a fresh pending + // marker; re-read config for exactly those before creating one for a removed task. + const existing = await readSubagentGitPatchArtifact(parentSessionDir, childWorkspaceId); + if ( + existing == null && + findWorkspaceEntry(this.config.loadConfigOrDefault(), childWorkspaceId) == null + ) { + return; + } + } + const pendingProjectArtifacts = buildPendingProjectArtifacts({ projectPath: childEntry.projectPath, projects: childEntry.workspace.projects, diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index b21efe2856b..df6ac0f16f0 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -47,6 +47,7 @@ import { } from "@/node/services/taskHandleStore"; import type { AgentPeerMessageBroker } from "@/node/services/agentPeerMessageBroker"; import { WorkflowRunStore } from "@/node/services/workflows/WorkflowRunStore"; +import { acquireWorkflowArchiveAdmission } from "@/node/services/workflows/workflowArchiveAdmission"; import { log } from "@/node/services/log"; import { recordAgentWorkflowRunReference } from "@/node/services/agentWorkflowRunReferences"; import type { WorkspaceForkParams } from "@/node/runtime/Runtime"; @@ -13696,6 +13697,63 @@ describe("TaskService", () => { expect(findWorkspaceInConfig(config, queuedChildId)?.taskPrompt).toBe("queued work"); }); + test("initialize leaves workflow-owned children alone while their run is being resumed", async () => { + const config = await createTestConfig(rootDir); + + const projectPath = path.join(rootDir, "repo"); + const runtimeConfig = { type: "local" as const }; + const parentId = "parent-workflow-resuming"; + const runningChildId = "child-workflow-resuming-running"; + const workflowRunId = "wfr_interrupted_but_resuming"; + + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "parent", parentId, { runtimeConfig }), + projectWorkspace(projectPath, "running", runningChildId, { + name: "agent_explore_running", + parentWorkspaceId: parentId, + agentId: "explore", + agentType: "explore", + taskStatus: "running", + taskModelString: defaultModel, + runtimeConfig, + workflowTask: { runId: workflowRunId, stepId: "running" }, + }), + ], + testTaskSettings(10, 3) + ); + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: workflowRunId, + workspaceId: parentId, + workflow: { + name: "interrupted", + description: "Interrupted", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return {}; }\n", + args: {}, + now: "2026-05-29T00:00:00.000Z", + }); + await runStore.appendStatus(workflowRunId, "interrupted", "2026-05-29T00:00:01.000Z"); + + const { aiService } = createAIServiceMocks(config, { isStreaming: mock(() => false) }); + const { workspaceService, sendMessage } = createWorkspaceServiceMocks(); + const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); + + // A client resumed the owning run while recovery was reading it as interrupted: the resume + // holds a workflow admission until the run settles, and recovery must neither interrupt the + // child (the resumed run owns it) nor nudge it as an orphaned running task. + using _resumeAdmission = acquireWorkflowArchiveAdmission(parentId); + await taskService.initialize(); + + expect(findWorkspaceInConfig(config, runningChildId)?.taskStatus).toBe("running"); + expect(sendMessage).not.toHaveBeenCalled(); + }); + test("initialize recovers parent tasks after interrupting inactive workflow children", async () => { const config = await createTestConfig(rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 0ad78c5de3f..347a815badd 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -187,6 +187,7 @@ import { isWorkspaceArchived } from "@/common/utils/archive"; import type { ToolPolicy } from "@/common/utils/tools/toolPolicy"; import { CONTEXT_BOUNDARY_KINDS } from "@/common/constants/contextBoundary"; import { WorkflowRunStore } from "@/node/services/workflows/WorkflowRunStore"; +import { hasInProcessWorkflowWork } from "@/node/services/workflows/workflowArchiveAdmission"; import { isActiveWorkspaceTurnTaskStatus, isWorkspaceTurnTaskId, @@ -683,6 +684,8 @@ interface WorkflowTaskOwner { interface InactiveWorkflowTaskOwner { ownerTaskId: string; runId: string; + /** Workspace hosting the run; undefined when the owner has no parent workspace. */ + runWorkspaceId?: string; status?: WorkflowRunStatus; reason: string; } @@ -1546,6 +1549,7 @@ export class TaskService implements AgentTaskIntegration { return { ownerTaskId: owner.taskId, runId: workflowTask.runId, + runWorkspaceId: parentWorkspaceId, status: run.status, reason: `workflow run belongs to ${run.workspaceId}, not ${parentWorkspaceId}`, }; @@ -1556,6 +1560,7 @@ export class TaskService implements AgentTaskIntegration { return { ownerTaskId: owner.taskId, runId: workflowTask.runId, + runWorkspaceId: parentWorkspaceId, status: run.status, reason: `workflow run is ${run.status}`, }; @@ -1563,6 +1568,7 @@ export class TaskService implements AgentTaskIntegration { return { ownerTaskId: owner.taskId, runId: workflowTask.runId, + runWorkspaceId: parentWorkspaceId, reason: `workflow run is unavailable: ${getErrorMessage(error)}`, }; } @@ -1727,10 +1733,23 @@ export class TaskService implements AgentTaskIntegration { let interrupted = false; let transitionedToInterrupted = false; + let resumeInFlight = false; let parentWorkspaceId: string | undefined; await this.editWorkspaceEntry( taskId, (ws) => { + // Startup recovery can run while clients are connected, and the run-store read above + // awaited. A workflow resume admitted since then owns this child's fate: it holds a + // workflow admission (see workflowArchiveAdmission) from its synchronous entry until the + // run settles, so checking it inside this synchronous edit cannot miss an in-flight + // resume. Leave the entry untouched in that case. + if ( + inactiveOwner.runWorkspaceId != null && + hasInProcessWorkflowWork(inactiveOwner.runWorkspaceId) + ) { + resumeInFlight = true; + return; + } const previousStatus = ws.taskStatus; parentWorkspaceId = ws.parentWorkspaceId; interrupted = this.applyInterruptedTaskStatus(ws) === "interrupted"; @@ -1738,6 +1757,15 @@ export class TaskService implements AgentTaskIntegration { }, { allowMissing: true } ); + if (resumeInFlight) { + log.debug("Skipping workflow-owned task recovery; owner workflow is being resumed", { + taskId, + trigger, + ownerTaskId: inactiveOwner.ownerTaskId, + workflowRunId: inactiveOwner.runId, + }); + return true; + } if (transitionedToInterrupted) { this.recordTaskInterrupted(taskId, parentWorkspaceId); } From 3439514f363ba9eb5effa01c9415a40638e37a26 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:13:44 +0200 Subject: [PATCH 08/26] fix(server): key the resume guard by run and confirm cleanup eligibility on live config - Workflow admissions are tracked per run id as well as per workspace; startup recovery skips a stale child only when a resume of its own run is in flight, so unrelated runs in the same workspace no longer shield it. - canCleanupReportedTask treats a caller snapshot as a screen only: any positive verdict is re-evaluated on freshly loaded config before the workspace is removed. --- src/node/services/taskService.test.ts | 79 +++++++++++++++++-- src/node/services/taskService.ts | 53 ++++++++----- .../services/workflows/WorkflowService.ts | 14 ++-- .../workflows/workflowArchiveAdmission.ts | 40 +++++++--- 4 files changed, 139 insertions(+), 47 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index df6ac0f16f0..720abe9ebe9 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -9,7 +9,7 @@ import { TASK_TERMINATION_STOP_STREAM_TIMEOUT_MS, TASK_TERMINATION_WORKSPACE_REMOVE_TIMEOUT_MS, } from "@/constants/terminationTimeouts"; -import { Config, type Workspace as WorkspaceConfigEntry } from "@/node/config"; +import { Config, type ProjectsConfig, type Workspace as WorkspaceConfigEntry } from "@/node/config"; import { HistoryService } from "@/node/services/historyService"; import * as subagentGitPatchArtifacts from "@/node/services/subagentGitPatchArtifacts"; import { @@ -3840,6 +3840,64 @@ describe("TaskService", () => { expect(loadsWithSixTasks).toBe(loadsWithTwoTasks); }); + test("startup cleanup confirms a snapshot-eligible task on fresh config before removing it", async () => { + const config = await createTestConfig(rootDir); + const projectPath = path.join(rootDir, "repo"); + const parentWorkspaceId = "parent-cleanup-live"; + const reactivatedTaskId = "child-workflow-reactivated"; + const staleTaskId = "child-workflow-stale"; + const workflowTask = (stepId: string) => ({ + parentWorkspaceId, + agentId: "exec", + agentType: "exec", + taskStatus: "reported" as const, + reportedAt: "2026-08-10T00:00:00.000Z", + taskModelString: "openai:gpt-5.2", + workflowTask: { runId: "wfr_cleanup_live", stepId }, + }); + + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "parent", parentWorkspaceId), + projectWorkspace(projectPath, "reactivated", reactivatedTaskId, workflowTask("a")), + projectWorkspace(projectPath, "stale", staleTaskId, workflowTask("b")), + ], + testTaskSettings() + ); + const snapshot = config.loadConfigOrDefault(); + // A client reactivated one child after the startup snapshot was taken. + await config.editConfig((cfg) => { + const entry = cfg.projects + .get(projectPath) + ?.workspaces.find((workspace) => workspace.id === reactivatedTaskId); + if (entry) { + entry.taskStatus = "running"; + entry.reportedAt = undefined; + } + return cfg; + }); + + const { workspaceService, remove } = createWorkspaceServiceMocks(); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const internals = taskService as unknown as { + cleanupReportedLeafTask: ( + workspaceId: string, + options?: { config?: ProjectsConfig } + ) => Promise; + }; + + expect(await internals.cleanupReportedLeafTask(reactivatedTaskId, { config: snapshot })).toBe( + 0 + ); + expect(await internals.cleanupReportedLeafTask(staleTaskId, { config: snapshot })).toBe(1); + const removedWorkspaceIds = ( + remove as unknown as { mock: { calls: unknown[][] } } + ).mock.calls.map((call) => call[0]); + expect(removedWorkspaceIds).toEqual([staleTaskId]); + }); + test("bare compaction stream-end resumes the pre-compaction parent identity", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); @@ -13745,13 +13803,22 @@ describe("TaskService", () => { const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); // A client resumed the owning run while recovery was reading it as interrupted: the resume - // holds a workflow admission until the run settles, and recovery must neither interrupt the - // child (the resumed run owns it) nor nudge it as an orphaned running task. - using _resumeAdmission = acquireWorkflowArchiveAdmission(parentId); - await taskService.initialize(); - + // holds a run-keyed workflow admission until the run settles, and recovery must neither + // interrupt the child (the resumed run owns it) nor nudge it as an orphaned running task. + { + using _resumeAdmission = acquireWorkflowArchiveAdmission(parentId, workflowRunId); + await taskService.initialize(); + } expect(findWorkspaceInConfig(config, runningChildId)?.taskStatus).toBe("running"); expect(sendMessage).not.toHaveBeenCalled(); + + // An unrelated run active in the same workspace must not shield the stale child. + { + using _otherRunAdmission = acquireWorkflowArchiveAdmission(parentId, "wfr_unrelated_run"); + await taskService.initialize(); + } + expect(findWorkspaceInConfig(config, runningChildId)?.taskStatus).toBe("interrupted"); + expect(sendMessage).not.toHaveBeenCalled(); }); test("initialize recovers parent tasks after interrupting inactive workflow children", async () => { diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 347a815badd..93eb3691d78 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -187,7 +187,7 @@ import { isWorkspaceArchived } from "@/common/utils/archive"; import type { ToolPolicy } from "@/common/utils/tools/toolPolicy"; import { CONTEXT_BOUNDARY_KINDS } from "@/common/constants/contextBoundary"; import { WorkflowRunStore } from "@/node/services/workflows/WorkflowRunStore"; -import { hasInProcessWorkflowWork } from "@/node/services/workflows/workflowArchiveAdmission"; +import { hasInProcessWorkflowRun } from "@/node/services/workflows/workflowArchiveAdmission"; import { isActiveWorkspaceTurnTaskStatus, isWorkspaceTurnTaskId, @@ -684,8 +684,6 @@ interface WorkflowTaskOwner { interface InactiveWorkflowTaskOwner { ownerTaskId: string; runId: string; - /** Workspace hosting the run; undefined when the owner has no parent workspace. */ - runWorkspaceId?: string; status?: WorkflowRunStatus; reason: string; } @@ -1549,7 +1547,6 @@ export class TaskService implements AgentTaskIntegration { return { ownerTaskId: owner.taskId, runId: workflowTask.runId, - runWorkspaceId: parentWorkspaceId, status: run.status, reason: `workflow run belongs to ${run.workspaceId}, not ${parentWorkspaceId}`, }; @@ -1560,7 +1557,6 @@ export class TaskService implements AgentTaskIntegration { return { ownerTaskId: owner.taskId, runId: workflowTask.runId, - runWorkspaceId: parentWorkspaceId, status: run.status, reason: `workflow run is ${run.status}`, }; @@ -1568,7 +1564,6 @@ export class TaskService implements AgentTaskIntegration { return { ownerTaskId: owner.taskId, runId: workflowTask.runId, - runWorkspaceId: parentWorkspaceId, reason: `workflow run is unavailable: ${getErrorMessage(error)}`, }; } @@ -1739,14 +1734,11 @@ export class TaskService implements AgentTaskIntegration { taskId, (ws) => { // Startup recovery can run while clients are connected, and the run-store read above - // awaited. A workflow resume admitted since then owns this child's fate: it holds a - // workflow admission (see workflowArchiveAdmission) from its synchronous entry until the - // run settles, so checking it inside this synchronous edit cannot miss an in-flight - // resume. Leave the entry untouched in that case. - if ( - inactiveOwner.runWorkspaceId != null && - hasInProcessWorkflowWork(inactiveOwner.runWorkspaceId) - ) { + // awaited. A resume of this run admitted since then owns this child's fate: it holds a + // run-keyed workflow admission (see workflowArchiveAdmission) from its synchronous entry + // until the run settles, so checking it inside this synchronous edit cannot miss an + // in-flight resume, and unrelated runs in the same workspace do not match. + if (hasInProcessWorkflowRun(inactiveOwner.runId)) { resumeInFlight = true; return; } @@ -2656,10 +2648,10 @@ export class TaskService implements AgentTaskIntegration { // Best-effort completed-report ancestor recheck after restart. const cleanupReportedTasksStartedAt = Date.now(); - // Reuse one config snapshot across tasks and reload only after a removal changed the tree. - // A stale snapshot is conservative here: it can only show more workspaces than exist, so the - // structural-leaf gate errs toward skipping. Concurrent removals from patch-generation - // onComplete callbacks are re-evaluated on fresh config by those callbacks. + // Reuse one config snapshot across tasks to screen out the (usual) ineligible majority without + // a config parse each; reload it only after a removal changed the tree. The snapshot never + // decides a deletion: canCleanupReportedTask re-evaluates any positive verdict on fresh config, + // since clients may have reactivated or re-parented a task since the snapshot. let cleanupConfig = config; for (const task of completedReportTasks) { if (!task.id) continue; @@ -13199,9 +13191,27 @@ export class TaskService implements AgentTaskIntegration { }; } + /** + * @param snapshot - optional config snapshot that only screens out ineligible tasks cheaply. + * A positive verdict is always re-evaluated on freshly loaded config, because the snapshot + * can predate a client reactivating or re-parenting the task. + */ private async canCleanupReportedTask( workspaceId: string, - config: ProjectsConfig = this.config.loadConfigOrDefault() + snapshot?: ProjectsConfig + ): Promise<{ ok: true; parentWorkspaceId: string } | { ok: false; reason: string }> { + if (snapshot != null) { + const screened = await this.evaluateReportedTaskCleanup(workspaceId, snapshot); + if (!screened.ok) { + return screened; + } + } + return await this.evaluateReportedTaskCleanup(workspaceId, this.config.loadConfigOrDefault()); + } + + private async evaluateReportedTaskCleanup( + workspaceId: string, + config: ProjectsConfig ): Promise<{ ok: true; parentWorkspaceId: string } | { ok: false; reason: string }> { assert(workspaceId.length > 0, "canCleanupReportedTask: workspaceId must be non-empty"); @@ -13269,8 +13279,9 @@ export class TaskService implements AgentTaskIntegration { } /** - * @param options.config - config snapshot for the first (depth 0) eligibility check only; every - * later depth re-evaluates the parent on fresh config because a removal just changed the tree. + * @param options.config - config snapshot used to screen the first (depth 0) eligibility check + * only; a positive verdict is still confirmed on fresh config, and every later depth + * re-evaluates the parent on fresh config because a removal just changed the tree. * @returns the number of workspaces removed. */ private async cleanupReportedLeafTask( diff --git a/src/node/services/workflows/WorkflowService.ts b/src/node/services/workflows/WorkflowService.ts index affe845de4e..00f9813d933 100644 --- a/src/node/services/workflows/WorkflowService.ts +++ b/src/node/services/workflows/WorkflowService.ts @@ -349,7 +349,7 @@ export class WorkflowService { projectTrusted: boolean; }): Promise { // Archive admission pairing; see resumeRunInBackground. - using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId); + using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId, input.runId); const run = await this.requireRunForWorkspace(input); assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); assertWorkflowRunCanRetryFromCheckpoint(run); @@ -372,7 +372,7 @@ export class WorkflowService { // Archive admission pairing: refuse while the workspace is archiving/archived, and hold // the admission across the method so the archive sink observes a resume that has not yet // durably re-activated its run (see workflowArchiveAdmission). - using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId); + using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId, input.runId); const run = await this.requireRunForWorkspace(input); assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); assertWorkflowRunCanTransition(run.status, "running"); @@ -394,7 +394,7 @@ export class WorkflowService { abortSignal?: AbortSignal; }): Promise { // Archive admission pairing; see resumeRunInBackground. - using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId); + using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId, input.runId); const run = await this.requireRunForWorkspace(input); assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); assertWorkflowRunCanTransition(run.status, "running"); @@ -415,7 +415,7 @@ export class WorkflowService { abortSignal?: AbortSignal; }): Promise { // Archive admission pairing; see resumeRunInBackground. - using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId); + using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId, input.runId); const run = await this.requireRunForWorkspace(input); assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); assertWorkflowRunCanRetryFromCheckpoint(run); @@ -670,7 +670,7 @@ export class WorkflowService { // Registration happens at lease acquisition, while the entry point's archive admission is // still held, so the archive sink observes in-process workflow work continuously from // admission entry to terminal settlement (see workflowArchiveAdmission). - const releaseInProcessWork = registerInProcessWorkflowRun(workspaceId); + const releaseInProcessWork = registerInProcessWorkflowRun(workspaceId, runId); return () => { if (activeWorkflowRunnerAbortControllers.get(runId) === controller) { activeWorkflowRunnerAbortControllers.delete(runId); @@ -1177,7 +1177,7 @@ export async function resumeWorkflowRun( context: WorkflowServiceContext, input: { workspaceId: string; runId: string } ): Promise { - using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId); + using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId, input.runId); const { service, projectTrusted } = await resolveWorkflowContext(context, input.workspaceId); return service.resumeRunInBackground({ ...input, projectTrusted }); } @@ -1186,7 +1186,7 @@ export async function retryWorkflowRunFromCheckpoint( context: WorkflowServiceContext, input: { workspaceId: string; runId: string } ): Promise { - using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId); + using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId, input.runId); const { service, projectTrusted } = await resolveWorkflowContext(context, input.workspaceId, { onBackgroundRunTerminal: (event) => { const name = event.run.workflow.sourcePath ?? event.run.workflow.name; diff --git a/src/node/services/workflows/workflowArchiveAdmission.ts b/src/node/services/workflows/workflowArchiveAdmission.ts index 902d6a593d9..b0fac35f1cd 100644 --- a/src/node/services/workflows/workflowArchiveAdmission.ts +++ b/src/node/services/workflows/workflowArchiveAdmission.ts @@ -17,6 +17,9 @@ import assert from "node:assert/strict"; let admissionGuard: ((workspaceId: string) => string | null) | null = null; const inProcessWorkflowWorkByWorkspace = new Map(); +// Per-run view of the same admissions, for callers that must not confuse unrelated runs in one +// workspace (startup recovery of a terminal run's children while another run is active). +const inProcessWorkflowWorkByRun = new Map(); /** Register the archive-side guard. Returns a refusal message or null when admission is allowed. */ export function setWorkflowArchiveAdmissionGuard( @@ -25,37 +28,43 @@ export function setWorkflowArchiveAdmissionGuard( admissionGuard = guard; } -function incrementInProcessWorkflowWork(workspaceId: string): () => void { - assert(workspaceId.length > 0, "workflowArchiveAdmission: workspaceId is required"); - inProcessWorkflowWorkByWorkspace.set( - workspaceId, - (inProcessWorkflowWorkByWorkspace.get(workspaceId) ?? 0) + 1 - ); +function incrementCount(counts: Map, key: string): () => void { + counts.set(key, (counts.get(key) ?? 0) + 1); let released = false; return () => { if (released) return; released = true; - const remaining = (inProcessWorkflowWorkByWorkspace.get(workspaceId) ?? 1) - 1; + const remaining = (counts.get(key) ?? 1) - 1; if (remaining <= 0) { - inProcessWorkflowWorkByWorkspace.delete(workspaceId); + counts.delete(key); } else { - inProcessWorkflowWorkByWorkspace.set(workspaceId, remaining); + counts.set(key, remaining); } }; } +function incrementInProcessWorkflowWork(workspaceId: string, runId?: string): () => void { + assert(workspaceId.length > 0, "workflowArchiveAdmission: workspaceId is required"); + const releaseWorkspace = incrementCount(inProcessWorkflowWorkByWorkspace, workspaceId); + const releaseRun = runId != null ? incrementCount(inProcessWorkflowWorkByRun, runId) : null; + return () => { + releaseWorkspace(); + releaseRun?.(); + }; +} + /** * Admit a workflow start/resume/retry for this workspace. Throws when an archive gate is * armed or the workspace is archived; otherwise counts the admission as in-process workflow * work until disposed. Entry points hold the admission across the whole method so the * archive sink observes work that has not yet produced a durably active run record. */ -export function acquireWorkflowArchiveAdmission(workspaceId: string): Disposable { +export function acquireWorkflowArchiveAdmission(workspaceId: string, runId?: string): Disposable { const refusal = admissionGuard?.(workspaceId) ?? null; if (refusal != null) { throw new Error(refusal); } - const release = incrementInProcessWorkflowWork(workspaceId); + const release = incrementInProcessWorkflowWork(workspaceId, runId); return { [Symbol.dispose]: release }; } @@ -65,11 +74,16 @@ export function acquireWorkflowArchiveAdmission(workspaceId: string): Disposable * admission is still held), so coverage is continuous from admission entry to terminal * settlement even before the runner durably appends its "running" status. */ -export function registerInProcessWorkflowRun(workspaceId: string): () => void { - return incrementInProcessWorkflowWork(workspaceId); +export function registerInProcessWorkflowRun(workspaceId: string, runId?: string): () => void { + return incrementInProcessWorkflowWork(workspaceId, runId); } /** Whether any workflow admission or in-process runner exists for this workspace. */ export function hasInProcessWorkflowWork(workspaceId: string): boolean { return (inProcessWorkflowWorkByWorkspace.get(workspaceId) ?? 0) > 0; } + +/** Whether an admission (resume/retry) or in-process runner exists for this specific run. */ +export function hasInProcessWorkflowRun(runId: string): boolean { + return (inProcessWorkflowWorkByRun.get(runId) ?? 0) > 0; +} From b14171cabf7e068ef8b4d205e508f5f452bed214 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:47:52 +0200 Subject: [PATCH 09/26] fix(server): decide reported-task cleanup under the lifecycle lock; keep live same-handle registrations - cleanupReportedLeafTask screens with the caller snapshot (no lock, no parse) and then evaluates live eligibility and removes the workspace inside one hold of the task-tree lifecycle lock, so a task reactivated or re-parented between the check and the removal is never deleted. - Execution-id reconciliation only installs a registration when the task has none; a live client turn's registration for the same handle (and its accepted state) is left untouched. --- src/node/services/taskService.ts | 63 +++++++++++------------ src/node/services/workspaceTurnManager.ts | 5 +- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 93eb3691d78..f4ad74922df 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -13191,27 +13191,9 @@ export class TaskService implements AgentTaskIntegration { }; } - /** - * @param snapshot - optional config snapshot that only screens out ineligible tasks cheaply. - * A positive verdict is always re-evaluated on freshly loaded config, because the snapshot - * can predate a client reactivating or re-parenting the task. - */ private async canCleanupReportedTask( workspaceId: string, - snapshot?: ProjectsConfig - ): Promise<{ ok: true; parentWorkspaceId: string } | { ok: false; reason: string }> { - if (snapshot != null) { - const screened = await this.evaluateReportedTaskCleanup(workspaceId, snapshot); - if (!screened.ok) { - return screened; - } - } - return await this.evaluateReportedTaskCleanup(workspaceId, this.config.loadConfigOrDefault()); - } - - private async evaluateReportedTaskCleanup( - workspaceId: string, - config: ProjectsConfig + config: ProjectsConfig = this.config.loadConfigOrDefault() ): Promise<{ ok: true; parentWorkspaceId: string } | { ok: false; reason: string }> { assert(workspaceId.length > 0, "canCleanupReportedTask: workspaceId must be non-empty"); @@ -13279,9 +13261,9 @@ export class TaskService implements AgentTaskIntegration { } /** - * @param options.config - config snapshot used to screen the first (depth 0) eligibility check - * only; a positive verdict is still confirmed on fresh config, and every later depth - * re-evaluates the parent on fresh config because a removal just changed the tree. + * @param options.config - config snapshot used only to screen the first (depth 0) candidate + * cheaply; deletion eligibility is always decided on fresh config under the lifecycle lock, + * and every later depth re-evaluates the parent the same way after a removal changed the tree. * @returns the number of workspaces removed. */ private async cleanupReportedLeafTask( @@ -13305,25 +13287,42 @@ export class TaskService implements AgentTaskIntegration { } visited.add(currentWorkspaceId); - const cleanupEligibility = await this.canCleanupReportedTask( - currentWorkspaceId, - depth === 0 ? options?.config : undefined - ); - if (!cleanupEligibility.ok) { + const targetWorkspaceId = currentWorkspaceId; + if (depth === 0 && options?.config != null) { + // The caller's snapshot only screens: it lets the (usual) ineligible majority return + // without a config parse or a lock acquisition. Deletion is decided on live state below. + const screened = await this.canCleanupReportedTask(targetWorkspaceId, options.config); + if (!screened.ok) { + return removedCount; + } + } + // Live eligibility and removal share one hold of the task-tree lifecycle lock: + // reactivation, re-parenting, and task_stop all mutate under that lock, so a task judged + // eligible here cannot change underneath the removal (the same pattern as task_remove). + const outcome = await this.withTaskTreeLifecycleLock(targetWorkspaceId, async () => { + const cleanupEligibility = await this.canCleanupReportedTask(targetWorkspaceId); + if (!cleanupEligibility.ok) { + return null; + } + const removeResult = await this.workspaceService.removeWhileTaskTreeLocked( + targetWorkspaceId, + true + ); + return { parentWorkspaceId: cleanupEligibility.parentWorkspaceId, removeResult }; + }); + if (outcome == null) { return removedCount; } - - const removeResult = await this.workspaceService.remove(currentWorkspaceId, true); - if (!removeResult.success) { + if (!outcome.removeResult.success) { log.error("Failed to auto-delete completed task workspace", { workspaceId: currentWorkspaceId, - error: removeResult.error, + error: outcome.removeResult.error, }); return removedCount; } removedCount += 1; - currentWorkspaceId = cleanupEligibility.parentWorkspaceId; + currentWorkspaceId = outcome.parentWorkspaceId; } log.error("cleanupReportedLeafTask: exceeded max parent traversal depth", { diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index 8616b9d144e..8644837858f 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -4863,10 +4863,11 @@ export class WorkspaceTurnManager { return; } await this.taskHost.emitWorkspaceMetadata(taskId); - const live = this.activeWorkspaceTurnHandleByWorkspaceId.get(taskId); + // A registration for this same handle that a live client turn already installed (and + // possibly marked accepted) is authoritative; only a missing registration is recovered. if ( isActiveWorkspaceTurnTaskStatus(status) && - (live == null || live.handleId === handleId) + this.activeWorkspaceTurnHandleByWorkspaceId.get(taskId) == null ) { this.activeWorkspaceTurnHandleByWorkspaceId.set(taskId, { handleId, From 2df37acaa31677a4170588ff1d85a36dd19e7a8e Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:06:25 +0200 Subject: [PATCH 10/26] fix(server): cancel startup recovery on dispose; recheck descendants at nudge admission; keep a worker error listener - ServiceContainer.dispose() aborts background startup recovery: recovery stops at its next step/task boundary and never starts the periodic services against disposed dependencies. - The restart nudge's admission probe re-derives the blocking-descendant gate from fresh config, so a descendant spawned after the startup index was captured refuses the send. - AnalyticsService keeps an error listener on the worker while awaiting its exit so a failing DuckDB close cannot crash the shutting-down process. --- src/cli/server.ts | 2 ++ .../services/analytics/analyticsService.ts | 8 ++++++ src/node/services/serviceContainer.test.ts | 27 +++++++++++++++++- src/node/services/serviceContainer.ts | 17 ++++++++++- src/node/services/taskService.ts | 28 +++++++++++++++++-- 5 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/cli/server.ts b/src/cli/server.ts index 11fb167e1d1..ac4b5fd77ec 100644 --- a/src/cli/server.ts +++ b/src/cli/server.ts @@ -170,6 +170,8 @@ async function main(): Promise { }); // Recovery is best-effort background work; a failure must not take the listening server down. + // serviceContainer.dispose() cancels it (recovery stops at its next task/step boundary), so + // shutdown during recovery never starts periodic services against disposed dependencies. void serviceContainer.runStartupRecovery(startupRecoveryConfig).catch((error: unknown) => { log.error("[startup] Background startup recovery failed", { error }); }); diff --git a/src/node/services/analytics/analyticsService.ts b/src/node/services/analytics/analyticsService.ts index e6530c26a7a..63709c8cef1 100644 --- a/src/node/services/analytics/analyticsService.ts +++ b/src/node/services/analytics/analyticsService.ts @@ -877,6 +877,14 @@ export class AnalyticsService { // aborts the whole process. Deliberately unbounded: an ETL that outlives a local timeout // would still be mid-query when the caller exits, so the hard-exit decision belongs to the // outer quit budgets in cli/server.ts and desktop/main.ts, which race the whole dispose. + // Keep an error listener until exit: onWorkerError was detached above, and a worker + // `error` with no listener is rethrown by the parent EventEmitter, which would turn a + // failing DuckDB close into a crash of the shutting-down process. + worker.on("error", (error: Error) => { + log.warn("[AnalyticsService] Analytics worker error during shutdown", { + error: getErrorMessage(error), + }); + }); const exited = new Promise((resolve) => { worker.once("exit", () => resolve()); }); diff --git a/src/node/services/serviceContainer.test.ts b/src/node/services/serviceContainer.test.ts index 72ab2236b1a..d96387e6ed8 100644 --- a/src/node/services/serviceContainer.test.ts +++ b/src/node/services/serviceContainer.test.ts @@ -155,7 +155,7 @@ describe("ServiceContainer", () => { await taskInitCalledPromise; expect(callOrder).toEqual(["workspace", "task"]); expect(recoverySettled).toBe(false); - expect(taskInitializeSpy).toHaveBeenCalledWith(preListenSnapshot); + expect(taskInitializeSpy.mock.calls[0]?.[0]).toBe(preListenSnapshot); expect(loadConfigSpy).not.toHaveBeenCalled(); releaseTaskInit?.(); @@ -163,6 +163,31 @@ describe("ServiceContainer", () => { expect(recoverySettled).toBe(true); }); + it("dispose cancels an in-flight startup recovery before the periodic services start", async () => { + services = new ServiceContainer(stores); + spyOn(services.workspaceService, "initialize").mockImplementation(() => Promise.resolve()); + let releaseTaskInit: (() => void) | undefined; + spyOn(services.taskService, "initialize").mockImplementation( + () => + new Promise((resolve) => { + releaseTaskInit = resolve; + }) + ); + const heartbeatStart = spyOn(services.heartbeatService, "start"); + const idleCompactionStart = spyOn(services.idleCompactionService, "start"); + + await services.initializeCore(); + const recovery = services.runStartupRecovery(); + // Task recovery is mid-flight when the process shuts down. + const disposed = services.dispose(); + releaseTaskInit?.(); + await recovery; + await disposed; + + expect(heartbeatStart).not.toHaveBeenCalled(); + expect(idleCompactionStart).not.toHaveBeenCalled(); + }); + it("runStartupRecovery starts the periodic services even when task recovery rejects", async () => { services = new ServiceContainer(stores); spyOn(services.workspaceService, "initialize").mockImplementation(() => Promise.resolve()); diff --git a/src/node/services/serviceContainer.ts b/src/node/services/serviceContainer.ts index 0825924b888..944312847f9 100644 --- a/src/node/services/serviceContainer.ts +++ b/src/node/services/serviceContainer.ts @@ -195,6 +195,9 @@ export class ServiceContainer { // reports every step and the total wall time from the start of core init. private startupStartedAt: number | undefined; private readonly startupStepDurationsMs: Record = {}; + // Aborted by dispose(): background startup recovery must stop at its next step boundary and + // never start periodic services against services that are being torn down. + private readonly startupRecoveryAbort = new AbortController(); /** * The in-flight (or completed) `dispose()` teardown. Every caller shares it, @@ -670,6 +673,7 @@ export class ServiceContainer { async runStartupRecovery( startupConfig: ProjectsConfig = this.config.loadConfigOrDefault() ): Promise { + const signal = this.startupRecoveryAbort.signal; // Recovery is best-effort and runs while the server may already be serving requests: a // failing recovery step must not skip the periodic services below (startup-time rule: // never let background housekeeping take the app down). @@ -681,13 +685,21 @@ export class ServiceContainer { } catch (error: unknown) { log.error("[startup] WorkspaceService recovery failed", { error }); } + if (signal.aborted) { + log.info("[startup] Startup recovery cancelled by dispose before task recovery"); + return; + } try { await this.recordStartupStep("taskService.initialize", () => - this.taskService.initialize(startupConfig) + this.taskService.initialize(startupConfig, { signal }) ); } catch (error: unknown) { log.error("[startup] TaskService recovery failed", { error }); } + if (signal.aborted) { + log.info("[startup] Startup recovery cancelled by dispose before periodic services"); + return; + } const idleCompactionStartedAt = Date.now(); // Start idle compaction checker @@ -843,6 +855,9 @@ export class ServiceContainer { } private async disposeOnce(): Promise { + // Background startup recovery (server mode) must not start periodic services or keep + // issuing recovery work against the services torn down below. + this.startupRecoveryAbort.abort(); // Must run before any session teardown: AgentSession.dispose() triggers // backgroundProcessManager.cleanup(), which would otherwise erase the persisted // armed-monitor registry records that drive post-restart "monitor lost" wakes. diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index f4ad74922df..224ba5e8f04 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -2285,8 +2285,15 @@ export class TaskService implements AgentTaskIntegration { * binds its listener before running recovery, clients may already be creating tasks; judging * stale-`starting` recovery from this snapshot keeps those fresh tasks out of scope. */ - async initialize(startupConfigSnapshot?: ProjectsConfig): Promise { + async initialize( + startupConfigSnapshot?: ProjectsConfig, + options?: { signal?: AbortSignal } + ): Promise { const startupStartedAt = Date.now(); + // Server-mode recovery runs after the listener is bound and can be cancelled by dispose(); + // the size-dependent loops below check this between tasks so teardown is not delayed by + // (or raced with) recovery work against disposed services. + const cancelled = (): boolean => options?.signal?.aborted === true; const startupConfig = startupConfigSnapshot ?? this.config.loadConfigOrDefault(); const queuedTaskCountAtStartup = this.listAgentTaskWorkspaces(startupConfig).filter( (task) => task.taskStatus === "queued" && typeof task.id === "string" @@ -2456,6 +2463,21 @@ export class TaskService implements AgentTaskIntegration { skippedRunningNoLongerRunning += 1; continue; } + // The restart nudge additionally requires no blocking active descendants; a client can + // spawn one after `taskIndex` was captured, so the probe re-derives that gate from fresh + // config at admission as well (pending guidance replays regardless, see above). + const restartNudgeAdmissionStale = (): boolean => { + const freshConfig = this.config.loadConfigOrDefault(); + if (findWorkspaceEntry(freshConfig, taskId)?.workspace.taskStatus !== "running") { + return true; + } + return ( + this.listBlockingActiveDescendantAgentTaskIdsUsingIndex( + this.buildAgentTaskIndex(freshConfig), + taskId + ).length > 0 + ); + }; const pendingGuidance = task.taskPendingGuidance ?? []; if (pendingGuidance.length > 0) { @@ -2556,7 +2578,7 @@ export class TaskService implements AgentTaskIntegration { reasoningMode: coerceOpenAIReasoningMode(task.aiSettings?.reasoningMode), experiments: task.taskExperiments, }, - { synthetic: true, agentInitiated: true, admissionStale } + { synthetic: true, agentInitiated: true, admissionStale: restartNudgeAdmissionStale } ); const durationMs = Date.now() - resumeStartedAt; if (!sendResult.success) { @@ -2603,6 +2625,7 @@ export class TaskService implements AgentTaskIntegration { const patchGenerationRecoveryStartedAt = Date.now(); for (const task of completedReportTasks) { + if (cancelled()) break; if (!task.parentWorkspaceId) continue; try { // Pass the loop snapshot: reloading config.json per reported task made this pass scale @@ -2654,6 +2677,7 @@ export class TaskService implements AgentTaskIntegration { // since clients may have reactivated or re-parented a task since the snapshot. let cleanupConfig = config; for (const task of completedReportTasks) { + if (cancelled()) break; if (!task.id) continue; const removedCount = await this.cleanupReportedLeafTask(task.id, { config: cleanupConfig }); if (removedCount > 0) { From 63036b7b3b9a897a9898484a14fc6fea14ee9d43 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:21:19 +0200 Subject: [PATCH 11/26] fix(server): recovery cancellation returns early; recovery sends yield to live client state - TaskService.initialize returns at every size-dependent step once startup recovery is cancelled, so no housekeeping or timers start during teardown. - The awaiting_report completion prompt's admission probe re-derives the blocking-descendant gate from fresh config, like the restart nudge. - The restart nudge is idle-only (requireIdle), so a client turn already in preflight suppresses the stale "Xum restarted" instruction instead of queueing it behind the client's turn. --- src/node/services/taskService.ts | 44 ++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 224ba5e8f04..99be3c152c5 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -2291,8 +2291,8 @@ export class TaskService implements AgentTaskIntegration { ): Promise { const startupStartedAt = Date.now(); // Server-mode recovery runs after the listener is bound and can be cancelled by dispose(); - // the size-dependent loops below check this between tasks so teardown is not delayed by - // (or raced with) recovery work against disposed services. + // every size-dependent step below checks this (and returns) so teardown is neither delayed + // by nor raced with recovery work against disposed services. const cancelled = (): boolean => options?.signal?.aborted === true; const startupConfig = startupConfigSnapshot ?? this.config.loadConfigOrDefault(); const queuedTaskCountAtStartup = this.listAgentTaskWorkspaces(startupConfig).filter( @@ -2578,7 +2578,14 @@ export class TaskService implements AgentTaskIntegration { reasoningMode: coerceOpenAIReasoningMode(task.aiSettings?.reasoningMode), experiments: task.taskExperiments, }, - { synthetic: true, agentInitiated: true, admissionStale: restartNudgeAdmissionStale } + { + synthetic: true, + agentInitiated: true, + // Idle-only: a client turn that entered preflight after the isStreaming() check above + // must suppress this stale "Xum restarted" nudge instead of queueing it behind itself. + requireIdle: true, + admissionStale: restartNudgeAdmissionStale, + } ); const durationMs = Date.now() - resumeStartedAt; if (!sendResult.success) { @@ -2625,7 +2632,7 @@ export class TaskService implements AgentTaskIntegration { const patchGenerationRecoveryStartedAt = Date.now(); for (const task of completedReportTasks) { - if (cancelled()) break; + if (cancelled()) return; if (!task.parentWorkspaceId) continue; try { // Pass the loop snapshot: reloading config.json per reported task made this pass scale @@ -2650,6 +2657,7 @@ export class TaskService implements AgentTaskIntegration { // on disk after a restart, there may be no later child stream-end to finalize the pending // parent task tool call. Re-run the deferred parent delivery/finalization pass first so // cleanup rechecks do not stay blocked forever behind a stale input-available partial. + if (cancelled()) return; const bestOfRecoveryStartedAt = Date.now(); const bestOfParentWorkspaceIds = new Set(); for (const task of completedReportTasks) { @@ -2665,6 +2673,7 @@ export class TaskService implements AgentTaskIntegration { bestOfParentWorkspaceIds.add(parentWorkspaceId); } for (const parentWorkspaceId of bestOfParentWorkspaceIds) { + if (cancelled()) return; await this.deliverDeferredBestOfReportsForParent(parentWorkspaceId); } const bestOfRecoveryMs = Date.now() - bestOfRecoveryStartedAt; @@ -2677,7 +2686,7 @@ export class TaskService implements AgentTaskIntegration { // since clients may have reactivated or re-parented a task since the snapshot. let cleanupConfig = config; for (const task of completedReportTasks) { - if (cancelled()) break; + if (cancelled()) return; if (!task.id) continue; const removedCount = await this.cleanupReportedLeafTask(task.id, { config: cleanupConfig }); if (removedCount > 0) { @@ -2685,6 +2694,7 @@ export class TaskService implements AgentTaskIntegration { } } const cleanupReportedTasksMs = Date.now() - cleanupReportedTasksStartedAt; + if (cancelled()) return; // Startup self-heal for leftover workflow task garbage: interrupted-without-report // workflow-owned children of inactive runs (both the ones the prepass above just @@ -2697,6 +2707,7 @@ export class TaskService implements AgentTaskIntegration { log.error("Startup workflow task archive sweep failed", { error }); } + if (cancelled()) return; let queuedTerminalWorkflowRunAttentionCount = 0; try { queuedTerminalWorkflowRunAttentionCount = await this.sweepWorkflowRunTerminalAttention(); @@ -2715,6 +2726,7 @@ export class TaskService implements AgentTaskIntegration { }, WORKFLOW_TERMINAL_ATTENTION_SWEEP_INTERVAL_MS); this.workflowAttentionSweepTimer.unref?.(); } + if (cancelled()) return; const recoveredTerminalWorkspaceTurnNotificationCount = await this.getWorkspaceTurnManager().recoverTerminalWorkspaceTurnAttentionNotifications(); const terminalAttentionDrainStartedAt = Date.now(); @@ -10523,11 +10535,23 @@ export class TaskService implements AgentTaskIntegration { { synthetic: true, agentInitiated: true, - // The status check at the top of this method is a one-shot read; a task_stop that lands - // during the awaits since then must refuse admission rather than be resurrected. - admissionStale: () => - findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId)?.workspace - .taskStatus !== "awaiting_report", + // The status and descendant checks at the top of this method are one-shot reads; a + // task_stop or a newly spawned blocking descendant landing during the awaits since then + // must refuse admission rather than resume the coordinator. + admissionStale: () => { + const freshConfig = this.config.loadConfigOrDefault(); + if ( + findWorkspaceEntry(freshConfig, workspaceId)?.workspace.taskStatus !== "awaiting_report" + ) { + return true; + } + return ( + this.listBlockingActiveDescendantAgentTaskIdsUsingIndex( + this.buildAgentTaskIndex(freshConfig), + workspaceId + ).length > 0 + ); + }, } ); const durationMs = Date.now() - startedAt; From 74890626739483e8a56f40a7127e4ff6c8d85ace Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:07:58 +0200 Subject: [PATCH 12/26] fix(server): confirm reported-task cleanup inside remove()'s lock hold; make the startup completion prompt idle-only - cleanupReportedLeafTask no longer takes the task-tree lifecycle lock itself: the live eligibility recheck runs as a beforeRemove precondition inside WorkspaceService.remove()'s own lock hold, so this path acquires exactly the locks it acquired before the recheck. - The startup awaiting_report completion prompt is idle-only (requireIdle), so a client turn that entered preflight after the isStreaming() probe suppresses the prompt instead of queueing behind it; a busy skip refunds the consumed recovery attempt. - Test harness remove() mock mirrors the beforeRemove precondition; tests assert on config state rather than mock call shapes. --- src/node/services/taskService.test.ts | 96 +++++++++++++++++--- src/node/services/taskService.testHarness.ts | 12 ++- src/node/services/taskService.ts | 78 ++++++++++------ src/node/services/taskWorkspaceSeam.ts | 6 +- src/node/services/workspaceService.ts | 20 +++- 5 files changed, 165 insertions(+), 47 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 720abe9ebe9..0cec442f760 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -3879,7 +3879,9 @@ describe("TaskService", () => { return cfg; }); - const { workspaceService, remove } = createWorkspaceServiceMocks(); + const { workspaceService } = createWorkspaceServiceMocks({ + remove: createConfigBackedRemoveMock(config), + }); const { taskService } = createTaskServiceHarness(config, { workspaceService }); const internals = taskService as unknown as { cleanupReportedLeafTask: ( @@ -3892,10 +3894,8 @@ describe("TaskService", () => { 0 ); expect(await internals.cleanupReportedLeafTask(staleTaskId, { config: snapshot })).toBe(1); - const removedWorkspaceIds = ( - remove as unknown as { mock: { calls: unknown[][] } } - ).mock.calls.map((call) => call[0]); - expect(removedWorkspaceIds).toEqual([staleTaskId]); + expect(findWorkspaceInConfig(config, reactivatedTaskId)?.taskStatus).toBe("running"); + expect(findWorkspaceInConfig(config, staleTaskId)).toBeUndefined(); }); test("bare compaction stream-end resumes the pre-compaction parent identity", async () => { @@ -14467,6 +14467,62 @@ describe("TaskService", () => { ); }); + test("startup completion prompt is idle-only and refunds its attempt when a client turn wins", async () => { + const config = await createTestConfig(rootDir); + const projectPath = path.join(rootDir, "repo"); + const parentId = "parent-idle-only"; + const childId = "child-idle-only"; + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "parent", parentId), + projectWorkspace(projectPath, "child", childId, { + name: "agent_explore_child", + parentWorkspaceId: parentId, + agentType: "explore", + taskStatus: "awaiting_report", + taskRecoveryAttempts: 2, + }), + ], + testTaskSettings(1, 3) + ); + + // A client turn entered preflight after the isStreaming() probe: idle-only sends skip. + const sendMessage = mock((..._args: unknown[]): Promise> => { + const internal = _args[3] as { requireIdle?: boolean } | undefined; + if (internal?.requireIdle === true) { + return Promise.resolve( + Err({ type: "unknown", raw: "Workspace is busy; idle-only send was skipped." }) + ); + } + return Promise.resolve(Ok(undefined)); + }); + const { aiService } = createAIServiceMocks(config); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); + const internals = taskService as unknown as { + promptTaskForRequiredCompletionTool: ( + workspaceId: string, + options?: { reason?: "startup" | "stream_end" | "error" } + ) => Promise; + }; + + await taskService.initialize(); + + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage.mock.calls[0]?.[3]).toMatchObject({ requireIdle: true }); + // The skipped prompt sent nothing, so the recovery budget is untouched. + expect(findWorkspaceInConfig(config, childId)?.taskRecoveryAttempts).toBe(2); + + // Stream-end prompts still queue behind whatever is running. + expect( + await internals.promptTaskForRequiredCompletionTool(childId, { reason: "stream_end" }) + ).toBe(true); + expect(sendMessage.mock.calls[1]?.[3]).not.toMatchObject({ requireIdle: true }); + expect(findWorkspaceInConfig(config, childId)?.taskRecoveryAttempts).toBe(3); + }); + test("initialize uses legacy agentType when modern agentId is unavailable for awaiting_report tasks", async () => { const config = await createTestConfig(rootDir); @@ -22298,6 +22354,26 @@ describe("TaskService", () => { expect(childEntry?.runtimeConfig?.type).toBe("worktree"); }, 20_000); + /** + * A remove() stand-in that mirrors the real one: the under-lock `beforeRemove` precondition + * decides whether the workspace actually leaves the config. + */ + function createConfigBackedRemoveMock(config: Config) { + return mock( + async ( + workspaceId: string, + _force?: boolean, + options?: { beforeRemove?: () => Promise } + ): Promise> => { + if (options?.beforeRemove != null && !(await options.beforeRemove())) { + return Ok(undefined); + } + await removeWorkspaceFromTestConfig(config, workspaceId); + return Ok(undefined); + } + ); + } + async function removeWorkspaceFromTestConfig(config: Config, workspaceId: string): Promise { const cfg = config.loadConfigOrDefault(); let removed = false; @@ -22637,10 +22713,7 @@ describe("TaskService", () => { }); const isStreaming = mock(() => false); - const remove = mock(async (workspaceId: string, _force?: boolean): Promise> => { - await removeWorkspaceFromTestConfig(config, workspaceId); - return Ok(undefined); - }); + const remove = createConfigBackedRemoveMock(config); const { aiService } = createAIServiceMocks(config, { isStreaming }); const { workspaceService } = createWorkspaceServiceMocks({ remove }); const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); @@ -22702,10 +22775,7 @@ describe("TaskService", () => { }); await internal.cleanupReportedLeafTask(childTaskId); - expect(remove.mock.calls).toEqual([ - [childTaskId, true], - [workflowTaskId, true], - ]); + expect(remove.mock.calls.map((call) => call[0])).toEqual([childTaskId, workflowTaskId]); expect(findWorkspaceInConfig(config, childTaskId)).toBeUndefined(); expect(findWorkspaceInConfig(config, workflowTaskId)).toBeUndefined(); }); diff --git a/src/node/services/taskService.testHarness.ts b/src/node/services/taskService.testHarness.ts index c0fd9095add..1d31d04f522 100644 --- a/src/node/services/taskService.testHarness.ts +++ b/src/node/services/taskService.testHarness.ts @@ -310,7 +310,17 @@ export function createWorkspaceServiceMocks(overrides: WorkspaceHostMockOverride remove: overrides.remove ?? overrides.removeWhileTaskTreeLocked ?? - mock((): Promise> => Promise.resolve(Ok(undefined))), + mock( + async ( + _workspaceId: string, + _force?: boolean, + options?: { beforeRemove?: () => Promise } + ): Promise> => { + // Mirror the real remove(): the under-lock precondition decides whether anything happens. + await options?.beforeRemove?.(); + return Ok(undefined); + } + ), updateTitle: overrides.updateTitle ?? mock((): Promise> => Promise.resolve(Ok(undefined))), emitChatEvent: overrides.emitChatEvent ?? mock(() => undefined), diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 99be3c152c5..fb366beb539 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -10535,6 +10535,10 @@ export class TaskService implements AgentTaskIntegration { { synthetic: true, agentInitiated: true, + // Startup recovery runs while clients may be sending to this task: a user turn that + // entered preflight after the isStreaming() check must suppress the prompt instead of + // queueing it behind the user's turn (stream-end re-prompts if that turn ends unreported). + requireIdle: options?.reason === "startup", // The status and descendant checks at the top of this method are one-shot reads; a // task_stop or a newly spawned blocking descendant landing during the awaits since then // must refuse admission rather than resume the coordinator. @@ -10556,6 +10560,24 @@ export class TaskService implements AgentTaskIntegration { ); const durationMs = Date.now() - startedAt; if (!sendResult.success) { + if (isWorkspaceBusyIdleOnlySend(sendResult.error)) { + // A client turn won admission first (only startup sends are idle-only). Nothing was + // sent, so hand the attempt back: a user's turn must not draw down the recovery budget. + await this.editWorkspaceEntry( + workspaceId, + (ws) => { + ws.taskRecoveryAttempts = Math.max(0, (ws.taskRecoveryAttempts ?? 0) - 1); + }, + { allowMissing: true } + ); + log.info("Skipped completion prompt: task became busy", { + workspaceId, + taskName: entry.workspace.name, + completionKind, + reason: options?.reason, + }); + return false; + } log.error("Failed to prompt task for required completion", { workspaceId, taskName: entry.workspace.name, @@ -13310,8 +13332,8 @@ export class TaskService implements AgentTaskIntegration { /** * @param options.config - config snapshot used only to screen the first (depth 0) candidate - * cheaply; deletion eligibility is always decided on fresh config under the lifecycle lock, - * and every later depth re-evaluates the parent the same way after a removal changed the tree. + * cheaply; deletion eligibility is always confirmed on fresh config inside remove()'s + * lifecycle lock, and every later depth re-evaluates the parent after a removal changed the tree. * @returns the number of workspaces removed. */ private async cleanupReportedLeafTask( @@ -13336,41 +13358,41 @@ export class TaskService implements AgentTaskIntegration { visited.add(currentWorkspaceId); const targetWorkspaceId = currentWorkspaceId; - if (depth === 0 && options?.config != null) { - // The caller's snapshot only screens: it lets the (usual) ineligible majority return - // without a config parse or a lock acquisition. Deletion is decided on live state below. - const screened = await this.canCleanupReportedTask(targetWorkspaceId, options.config); - if (!screened.ok) { - return removedCount; - } - } - // Live eligibility and removal share one hold of the task-tree lifecycle lock: - // reactivation, re-parenting, and task_stop all mutate under that lock, so a task judged - // eligible here cannot change underneath the removal (the same pattern as task_remove). - const outcome = await this.withTaskTreeLifecycleLock(targetWorkspaceId, async () => { - const cleanupEligibility = await this.canCleanupReportedTask(targetWorkspaceId); - if (!cleanupEligibility.ok) { - return null; - } - const removeResult = await this.workspaceService.removeWhileTaskTreeLocked( - targetWorkspaceId, - true - ); - return { parentWorkspaceId: cleanupEligibility.parentWorkspaceId, removeResult }; - }); - if (outcome == null) { + // Screen outside any lock (with the caller's snapshot at depth 0) so the (usual) ineligible + // majority returns without a lock acquisition or config parse. + const screened = await this.canCleanupReportedTask( + targetWorkspaceId, + depth === 0 ? options?.config : undefined + ); + if (!screened.ok) { return removedCount; } - if (!outcome.removeResult.success) { + // Deletion is decided on live state inside the task-tree lifecycle lock that remove() + // holds: reactivation, re-parenting, and task_stop all mutate under that lock, so a task + // confirmed eligible there cannot change underneath the removal. remove() is the only lock + // acquisition on this path: runtime callers reach it under the workspace event lock + // (stream-end finalization, cleanup rechecks), nesting event -> task-tree, the inverse of + // the send path's task-tree -> event. That nesting predates the live recheck. + let confirmed = false; + const removeResult = await this.workspaceService.remove(targetWorkspaceId, true, { + beforeRemove: async () => { + confirmed = (await this.canCleanupReportedTask(targetWorkspaceId)).ok; + return confirmed; + }, + }); + if (!removeResult.success) { log.error("Failed to auto-delete completed task workspace", { workspaceId: currentWorkspaceId, - error: outcome.removeResult.error, + error: removeResult.error, }); return removedCount; } + if (!confirmed) { + return removedCount; + } removedCount += 1; - currentWorkspaceId = outcome.parentWorkspaceId; + currentWorkspaceId = screened.parentWorkspaceId; } log.error("cleanupReportedLeafTask: exceeded max parent traversal depth", { diff --git a/src/node/services/taskWorkspaceSeam.ts b/src/node/services/taskWorkspaceSeam.ts index f156763a52e..ecd88101875 100644 --- a/src/node/services/taskWorkspaceSeam.ts +++ b/src/node/services/taskWorkspaceSeam.ts @@ -450,7 +450,11 @@ export interface WorkspaceLifecycleHost { worktreeArchiveBehavior?: WorktreeArchiveBehavior, metadata?: WorkspaceMetadata ): boolean; - remove(workspaceId: string, force?: boolean): Promise>; + remove( + workspaceId: string, + force?: boolean, + options?: { beforeRemove?: () => Promise } + ): Promise>; removeWhileTaskTreeLocked(workspaceId: string, force?: boolean): Promise>; } diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 5886ffff58a..f821b3f7b50 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -5451,10 +5451,22 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { : await integration.withTaskTreeLifecycleLock(workspaceId, operation); } - async remove(workspaceId: string, force = false): Promise> { - return await this.withTaskTreeLifecycleLock(workspaceId, async () => - this.removeUnlocked(workspaceId, force) - ); + /** + * @param options.beforeRemove - evaluated inside the task-tree lifecycle lock; returning false + * turns the call into a no-op. Lets callers that screened eligibility outside the lock confirm + * it against live state within the same lock hold that performs the removal. + */ + async remove( + workspaceId: string, + force = false, + options?: { beforeRemove?: () => Promise } + ): Promise> { + return await this.withTaskTreeLifecycleLock(workspaceId, async () => { + if (options?.beforeRemove != null && !(await options.beforeRemove())) { + return Ok(undefined); + } + return await this.removeUnlocked(workspaceId, force); + }); } /** From dae08c6daf3e90d4381e6a4227d1b4df7ee5736d Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:13:13 +0000 Subject: [PATCH 13/26] refactor(server): run task recovery before the listener binds; keep only housekeeping behind it Split TaskService.initialize into recoverInterruptedTasks (execution-handle reconciliation, stale-starting fixup, queue drain, awaiting_report/running restart prompts; O(active tasks)) and runStartupHousekeeping (patch-artifact recovery, reported-task cleanup, workflow and terminal-attention sweeps; O(workspaces), dispose-cancellable). ServiceContainer.initializeCore now runs task recovery, so the server binds only after it; runStartupHousekeeping runs in the background after listen. With no client able to race task recovery, delete the compare-and-set and admission scaffolding that only existed for post-listen recovery: admissionStale probes, idle-only restart prompts and their attempt refund, the run-keyed workflow admission registry, the settlement-lock CAS in reconcileAgentTaskExecutionIds, and the pre-listen config snapshot parameter. Startup patch-artifact recovery now also skips a snapshot child that a client reactivated (live workspace-turn execution) or that is streaming; the continuation refresh generates its artifact when that execution settles. --- src/cli/server.ts | 32 +- .../services/gitPatchArtifactService.test.ts | 21 + src/node/services/gitPatchArtifactService.ts | 19 +- src/node/services/serviceContainer.test.ts | 123 ++--- src/node/services/serviceContainer.ts | 65 +-- src/node/services/taskService.test.ts | 443 +----------------- src/node/services/taskService.ts | 193 +++----- .../services/workflows/WorkflowService.ts | 14 +- .../workflows/workflowArchiveAdmission.ts | 40 +- src/node/services/workspaceTurnManager.ts | 68 +-- 10 files changed, 269 insertions(+), 749 deletions(-) diff --git a/src/cli/server.ts b/src/cli/server.ts index 94874e417de..fc98192cfca 100644 --- a/src/cli/server.ts +++ b/src/cli/server.ts @@ -95,10 +95,11 @@ async function main(): Promise { launchProjectPath = null; // Keepalive interval to prevent premature process exit during async initialization. - // Between the completion of serviceContainer.initializeCore() and the HTTP server + // During startup, initializeCore() may resume running tasks by calling sendMessage(), + // which spawns background AI streams. Between its completion and the HTTP server // starting to listen, there can be a brief moment where no ref'd handles exist, // causing Node to exit with code 0. This interval ensures the event loop stays alive - // until the server is listening; startup recovery runs after that point. + // until the server is listening. const startupKeepalive = setInterval(() => { // Intentionally empty - keeps event loop alive during startup }, 1000); @@ -115,9 +116,9 @@ async function main(): Promise { } // Early lockfile check: detect an existing server BEFORE initializing services. - // Task recovery (runStartupRecovery) only runs after startServer() has acquired the - // lock, so this check is a fast-fail nicety that avoids core init work when another - // server already holds the lock. ServerService.startServer() re-checks as the real gate. + // initializeCore() resumes queued/running tasks (via TaskService), so we must fail fast + // here to avoid orphaned side effects when another server already holds the lock. + // ServerService.startServer() re-checks as defense-in-depth. const xumHome = getXumHome(); const earlyLockfile = new ServerLockfile(xumHome); const existing = await earlyLockfile.read(); @@ -132,8 +133,9 @@ async function main(): Promise { const serviceContainer = new ServiceContainer(stores); // Headless server has no interactive host-key dialog setOpenSSHHostKeyPolicyMode("headless-fallback"); - // Only the fast core init gates the listener; workspace/task recovery scales with the - // number of workspaces and runs in the background once the server is accepting connections. + // Core init (including agent-task recovery, which must finish before any client can act on + // tasks) gates the listener; the housekeeping that scales with the number of workspaces runs + // in the background once the server is accepting connections. await serviceContainer.initializeCore(); serviceContainer.windowService.setMainWindow(mockWindow); @@ -153,12 +155,6 @@ async function main(): Promise { const context = serviceContainer.toORPCContext(); - // Snapshot config before the listener opens: startServer() is already accepting requests while - // it awaits the lockfile and mDNS, so a snapshot taken after it resolves could already contain - // a task a freshly connected client is legitimately starting, and recovery would misjudge it - // as stale. Everything recovery must judge as pre-existing is on disk at this point. - const startupRecoveryConfig = config.loadConfigOrDefault(); - // Start server via ServerService (handles lockfile, mDNS, network URLs) const serverInfo = await serviceContainer.serverService.startServer({ xumHome: serviceContainer.config.rootDir, @@ -170,11 +166,11 @@ async function main(): Promise { allowHttpOrigin: ALLOW_HTTP_ORIGIN, }); - // Recovery is best-effort background work; a failure must not take the listening server down. - // serviceContainer.dispose() cancels it (recovery stops at its next task/step boundary), so - // shutdown during recovery never starts periodic services against disposed dependencies. - void serviceContainer.runStartupRecovery(startupRecoveryConfig).catch((error: unknown) => { - log.error("[startup] Background startup recovery failed", { error }); + // Housekeeping is best-effort background work; a failure must not take the listening server + // down. serviceContainer.dispose() cancels it at its next task/step boundary, so shutdown + // during housekeeping never starts periodic services against disposed dependencies. + void serviceContainer.runStartupHousekeeping().catch((error: unknown) => { + log.error("[startup] Background startup housekeeping failed", { error }); }); // Server is now listening - clear the startup keepalive since httpServer keeps the loop alive diff --git a/src/node/services/gitPatchArtifactService.test.ts b/src/node/services/gitPatchArtifactService.test.ts index a71d52e605c..a1ba2ecb911 100644 --- a/src/node/services/gitPatchArtifactService.test.ts +++ b/src/node/services/gitPatchArtifactService.test.ts @@ -96,6 +96,7 @@ describe("GitPatchArtifactService coordination", () => { const childId = "child-reported"; const freshChildId = "child-fresh"; const removedChildId = "child-removed-since-snapshot"; + const reactivatedChildId = "child-reactivated-since-snapshot"; const execChild = (id: string) => ({ path: path.join(projectPath, id), id, @@ -112,6 +113,7 @@ describe("GitPatchArtifactService coordination", () => { execChild(childId), execChild(freshChildId), execChild(removedChildId), + execChild(reactivatedChildId), ], }); return cfg; @@ -185,6 +187,25 @@ describe("GitPatchArtifactService coordination", () => { expect(await readSubagentGitPatchArtifact(parentSessionDir, removedChildId)).toBeNull(); expect(completedChildIds).toEqual([]); + // A snapshot child a client reactivated since (live workspace-turn execution) is committing + // again; generating from its checkout now would capture a partial continuation, so it is + // left to the continuation refresh. + await config.editConfig((cfg) => { + const entry = cfg.projects + .get(projectPath) + ?.workspaces.find((ws) => ws.id === reactivatedChildId); + if (entry) { + entry.taskExecutionId = "wst_reactivated"; + entry.taskExecutionStatus = "running"; + } + return cfg; + }); + await service.maybeStartGeneration(parentId, reactivatedChildId, onComplete, { + config: snapshot, + }); + expect(await readSubagentGitPatchArtifact(parentSessionDir, reactivatedChildId)).toBeNull(); + expect(completedChildIds).toEqual([]); + // A snapshot child without an artifact still takes the normal generation path (pending // marker written, background job started), proving the snapshot feeds the real branch. await service.maybeStartGeneration(parentId, freshChildId, onComplete, { config: snapshot }); diff --git a/src/node/services/gitPatchArtifactService.ts b/src/node/services/gitPatchArtifactService.ts index b551a561cc4..9a0ba603845 100644 --- a/src/node/services/gitPatchArtifactService.ts +++ b/src/node/services/gitPatchArtifactService.ts @@ -14,6 +14,7 @@ import { findWorkspaceEntry, } from "@/node/services/taskUtils"; import { log } from "@/node/services/log"; +import { isActiveWorkspaceTurnTaskStatus } from "@/node/services/taskHandleStore"; import { readAgentDefinition } from "@/node/services/agentDefinitions/agentDefinitionsService"; import { resolveAgentInheritanceChain } from "@/node/services/agentDefinitions/resolveAgentInheritanceChain"; import { isExecLikeEditingCapableInResolvedChain } from "@/common/utils/agentTools"; @@ -412,15 +413,17 @@ export class GitPatchArtifactService { } if (options?.config != null) { - // The caller's snapshot can predate a client removing this task. The updater below leaves an - // existing artifact untouched, so only a task with no artifact yet would get a fresh pending - // marker; re-read config for exactly those before creating one for a removed task. + // The caller's snapshot can predate a client removing or reactivating this task. The updater + // below leaves an existing artifact untouched, so only a task with no artifact yet would get + // a fresh pending marker; re-read config for exactly those. A removed task gets no artifact, + // and a reactivated one (live execution handle, so its checkout is changing again) is left + // to the continuation refresh that runs when that execution settles. const existing = await readSubagentGitPatchArtifact(parentSessionDir, childWorkspaceId); - if ( - existing == null && - findWorkspaceEntry(this.config.loadConfigOrDefault(), childWorkspaceId) == null - ) { - return; + if (existing == null) { + const live = findWorkspaceEntry(this.config.loadConfigOrDefault(), childWorkspaceId); + if (live == null || isActiveWorkspaceTurnTaskStatus(live.workspace.taskExecutionStatus)) { + return; + } } } diff --git a/src/node/services/serviceContainer.test.ts b/src/node/services/serviceContainer.test.ts index 0dde4be4b9b..6bec6f6a26d 100644 --- a/src/node/services/serviceContainer.test.ts +++ b/src/node/services/serviceContainer.test.ts @@ -221,88 +221,101 @@ describe("ServiceContainer", () => { ); }); - it("initializeCore resolves without waiting for task recovery", async () => { + it("initializeCore completes task recovery and leaves housekeeping to runStartupHousekeeping", async () => { services = new ServiceContainer(stores); - let releaseTaskInit: (() => void) | undefined; - const taskInitGate = new Promise((resolve) => { - releaseTaskInit = resolve; + const callOrder: string[] = []; + let releaseTaskRecovery: (() => void) | undefined; + let taskRecoveryCalled: (() => void) | undefined; + const taskRecoveryCalledPromise = new Promise((resolve) => { + taskRecoveryCalled = resolve; + }); + const recoverTasksSpy = spyOn( + services.taskService, + "recoverInterruptedTasks" + ).mockImplementation(() => { + callOrder.push("recoverTasks"); + taskRecoveryCalled?.(); + return new Promise((resolve) => { + releaseTaskRecovery = resolve; + }); }); - const taskInitializeSpy = spyOn(services.taskService, "initialize").mockImplementation( - () => taskInitGate - ); const workspaceInitializeSpy = spyOn( services.workspaceService, "initialize" - ).mockImplementation(() => Promise.resolve()); - const loadConfigSpy = spyOn(config, "loadConfigOrDefault"); - - await services.initializeCore(); - expect(taskInitializeSpy).not.toHaveBeenCalled(); - expect(workspaceInitializeSpy).not.toHaveBeenCalled(); - - const callOrder: string[] = []; - let taskInitCalled: (() => void) | undefined; - const taskInitCalledPromise = new Promise((resolve) => { - taskInitCalled = resolve; - }); - workspaceInitializeSpy.mockImplementation(() => { + ).mockImplementation(() => { callOrder.push("workspace"); return Promise.resolve(); }); - taskInitializeSpy.mockImplementation(() => { - callOrder.push("task"); - taskInitCalled?.(); - return taskInitGate; + const taskHousekeepingSpy = spyOn( + services.taskService, + "runStartupHousekeeping" + ).mockImplementation(() => { + callOrder.push("taskHousekeeping"); + return Promise.resolve(); }); - // The server entry point snapshots config before its listener opens and hands that exact - // snapshot to task recovery; recovery must not substitute a fresh load. - const preListenSnapshot = config.loadConfigOrDefault(); - loadConfigSpy.mockClear(); - - let recoverySettled = false; - const recovery = services.runStartupRecovery(preListenSnapshot).then(() => { - recoverySettled = true; + + let coreSettled = false; + const core = services.initializeCore().then(() => { + coreSettled = true; }); - await taskInitCalledPromise; - expect(callOrder).toEqual(["workspace", "task"]); - expect(recoverySettled).toBe(false); - expect(taskInitializeSpy.mock.calls[0]?.[0]).toBe(preListenSnapshot); - expect(loadConfigSpy).not.toHaveBeenCalled(); + await taskRecoveryCalledPromise; + // The listener must wait for task recovery (clients may otherwise race its transitions)... + expect(recoverTasksSpy).toHaveBeenCalledTimes(1); + expect(coreSettled).toBe(false); + releaseTaskRecovery?.(); + await core; + // ...but not for the O(workspaces) housekeeping, which runs only when the caller asks for it. + expect(workspaceInitializeSpy).not.toHaveBeenCalled(); + expect(taskHousekeepingSpy).not.toHaveBeenCalled(); - releaseTaskInit?.(); - await recovery; - expect(recoverySettled).toBe(true); + await services.runStartupHousekeeping(); + expect(callOrder).toEqual(["recoverTasks", "workspace", "taskHousekeeping"]); + expect(taskHousekeepingSpy.mock.calls[0]?.[0]?.signal).toBeInstanceOf(AbortSignal); }); - it("dispose cancels an in-flight startup recovery before the periodic services start", async () => { + it("dispose cancels in-flight startup housekeeping before the periodic services start", async () => { services = new ServiceContainer(stores); - spyOn(services.workspaceService, "initialize").mockImplementation(() => Promise.resolve()); - let releaseTaskInit: (() => void) | undefined; - spyOn(services.taskService, "initialize").mockImplementation( - () => - new Promise((resolve) => { - releaseTaskInit = resolve; - }) + spyOn(services.taskService, "recoverInterruptedTasks").mockImplementation(() => + Promise.resolve() ); + spyOn(services.workspaceService, "initialize").mockImplementation(() => Promise.resolve()); + let releaseTaskHousekeeping: (() => void) | undefined; + let housekeepingSignal: AbortSignal | undefined; + let taskHousekeepingCalled: (() => void) | undefined; + const taskHousekeepingCalledPromise = new Promise((resolve) => { + taskHousekeepingCalled = resolve; + }); + spyOn(services.taskService, "runStartupHousekeeping").mockImplementation((options) => { + housekeepingSignal = options?.signal; + taskHousekeepingCalled?.(); + return new Promise((release) => { + releaseTaskHousekeeping = release; + }); + }); const heartbeatStart = spyOn(services.heartbeatService, "start"); const idleCompactionStart = spyOn(services.idleCompactionService, "start"); await services.initializeCore(); - const recovery = services.runStartupRecovery(); - // Task recovery is mid-flight when the process shuts down. + const housekeeping = services.runStartupHousekeeping(); + await taskHousekeepingCalledPromise; + // Task housekeeping is mid-flight when the process shuts down. const disposed = services.dispose(); - releaseTaskInit?.(); - await recovery; + expect(housekeepingSignal?.aborted).toBe(true); + releaseTaskHousekeeping?.(); + await housekeeping; await disposed; expect(heartbeatStart).not.toHaveBeenCalled(); expect(idleCompactionStart).not.toHaveBeenCalled(); }); - it("runStartupRecovery starts the periodic services even when task recovery rejects", async () => { + it("runStartupHousekeeping starts the periodic services even when task housekeeping rejects", async () => { services = new ServiceContainer(stores); + spyOn(services.taskService, "recoverInterruptedTasks").mockImplementation(() => + Promise.resolve() + ); spyOn(services.workspaceService, "initialize").mockImplementation(() => Promise.resolve()); - spyOn(services.taskService, "initialize").mockImplementation(() => + spyOn(services.taskService, "runStartupHousekeeping").mockImplementation(() => Promise.reject(new Error("terminal-attention store unreadable")) ); const idleCompactionStart = spyOn(services.idleCompactionService, "start"); @@ -310,7 +323,7 @@ describe("ServiceContainer", () => { const agentStatusStart = spyOn(services.agentStatusService, "start"); await services.initializeCore(); - await services.runStartupRecovery(); + await services.runStartupHousekeeping(); expect(idleCompactionStart).toHaveBeenCalledTimes(1); expect(heartbeatStart).toHaveBeenCalledTimes(1); diff --git a/src/node/services/serviceContainer.ts b/src/node/services/serviceContainer.ts index bdad522f7e8..ad61e4cd521 100644 --- a/src/node/services/serviceContainer.ts +++ b/src/node/services/serviceContainer.ts @@ -1,5 +1,5 @@ import { log } from "@/node/services/log"; -import type { Config, ConfigStores, ProjectsConfig, WorkspaceSessionLocator } from "@/node/config"; +import type { Config, ConfigStores, WorkspaceSessionLocator } from "@/node/config"; import type { FileLeaseManager, ProvidersConfigStore, SecretsStore } from "@/node/config"; import { SLOW_STARTUP_WARN_THRESHOLD_MS } from "@/constants/startup"; import type { CoreServices } from "@/node/services/coreServices"; @@ -212,13 +212,13 @@ export class ServiceContainer { public readonly idleDispatcher: IdleDispatcher; public readonly heartbeatService: HeartbeatService; public readonly agentStatusService: AgentStatusService; - // Shared between initializeCore() and runStartupRecovery() so the completion log still + // Shared between initializeCore() and runStartupHousekeeping() so the completion log still // reports every step and the total wall time from the start of core init. private startupStartedAt: number | undefined; private readonly startupStepDurationsMs: Record = {}; - // Aborted by dispose(): background startup recovery must stop at its next step boundary and - // never start periodic services against services that are being torn down. - private readonly startupRecoveryAbort = new AbortController(); + // Aborted by dispose(): background startup housekeeping must stop at its next step boundary + // and never start periodic services against services that are being torn down. + private readonly startupHousekeepingAbort = new AbortController(); /** * The in-flight (or completed) `dispose()` teardown. Every caller shares it, @@ -305,7 +305,7 @@ export class ServiceContainer { async initialize(): Promise { await this.initializeCore(); - await this.runStartupRecovery(); + await this.runStartupHousekeeping(); } private async recordStartupStep(name: string, fn: () => Promise): Promise { @@ -318,8 +318,11 @@ export class ServiceContainer { } /** - * Fast startup steps that request handling depends on. Safe to await before binding the - * server listener; the slow per-workspace recovery pass lives in runStartupRecovery(). + * Everything request handling depends on, plus agent-task restart recovery. The server entry + * point awaits this before binding its listener: task recovery must finish before any client + * can stop, resume, or send to a task (see TaskService.recoverInterruptedTasks), and it is + * bounded by the number of active tasks rather than by deployment size. The per-workspace + * housekeeping lives in runStartupHousekeeping(). */ async initializeCore(): Promise { this.startupStartedAt = Date.now(); @@ -340,26 +343,24 @@ export class ServiceContainer { await this.recordStartupStep("experimentsService.initialize", () => this.experimentsService.initialize() ); + + await this.recordStartupStep("taskService.recoverInterruptedTasks", () => + this.taskService.recoverInterruptedTasks() + ); } /** - * Workspace/task restart recovery plus background housekeeping. On large deployments this - * scales with the number of workspaces, so the server entry point runs it after the - * listener is bound instead of on the critical path to accepting connections. - * - * @param startupConfig - config snapshot from before the listener opened. TaskService's - * stale-`starting` recovery must only judge tasks that were `starting` when the process - * started, never a task a freshly connected client is legitimately starting, so a caller - * that accepts connections before recovery must capture the snapshot before listening. + * Startup housekeeping that scales with the number of workspaces (chat restart retries and + * orphan sweeps, reported-task patch and cleanup passes, terminal-attention sweeps), then the + * periodic services. The server runs it after its listener is bound, so every step re-checks + * live state before mutating; dispose() cancels it at the next step boundary. */ - async runStartupRecovery( - startupConfig: ProjectsConfig = this.config.loadConfigOrDefault() - ): Promise { - const signal = this.startupRecoveryAbort.signal; - // Recovery is best-effort and runs while the server may already be serving requests: a - // failing recovery step must not skip the periodic services below (startup-time rule: - // never let background housekeeping take the app down). - // Kick off non-task chat restart recovery eagerly; task workspaces recover in TaskService.initialize(). + async runStartupHousekeeping(): Promise { + const signal = this.startupHousekeepingAbort.signal; + // Housekeeping is best-effort and may run while the server is already serving requests: a + // failing step must not skip the periodic services below (startup-time rule: never let + // background housekeeping take the app down). + // Kick off non-task chat restart recovery eagerly; task workspaces recover in TaskService. try { await this.recordStartupStep("workspaceService.initialize", () => this.workspaceService.initialize() @@ -368,18 +369,18 @@ export class ServiceContainer { log.error("[startup] WorkspaceService recovery failed", { error }); } if (signal.aborted) { - log.info("[startup] Startup recovery cancelled by dispose before task recovery"); + log.info("[startup] Startup housekeeping cancelled by dispose before task housekeeping"); return; } try { - await this.recordStartupStep("taskService.initialize", () => - this.taskService.initialize(startupConfig, { signal }) + await this.recordStartupStep("taskService.runStartupHousekeeping", () => + this.taskService.runStartupHousekeeping({ signal }) ); } catch (error: unknown) { - log.error("[startup] TaskService recovery failed", { error }); + log.error("[startup] TaskService housekeeping failed", { error }); } if (signal.aborted) { - log.info("[startup] Startup recovery cancelled by dispose before periodic services"); + log.info("[startup] Startup housekeeping cancelled by dispose before periodic services"); return; } @@ -545,9 +546,9 @@ export class ServiceContainer { private async disposeOnce(): Promise { const disposeStartedAt = performance.now(); log.debug("[shutdown] ServiceContainer.dispose starting"); - // Background startup recovery (server mode) must not start periodic services or keep - // issuing recovery work against the services torn down below. - this.startupRecoveryAbort.abort(); + // Background startup housekeeping (server mode) must not start periodic services or keep + // issuing work against the services torn down below. + this.startupHousekeepingAbort.abort(); // Must run before any session teardown: AgentSession.dispose() triggers // backgroundProcessManager.cleanup(), which would otherwise erase the persisted // armed-monitor registry records that drive post-restart "monitor lost" wakes. diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 0cec442f760..e2dfc8a0345 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -47,7 +47,6 @@ import { } from "@/node/services/taskHandleStore"; import type { AgentPeerMessageBroker } from "@/node/services/agentPeerMessageBroker"; import { WorkflowRunStore } from "@/node/services/workflows/WorkflowRunStore"; -import { acquireWorkflowArchiveAdmission } from "@/node/services/workflows/workflowArchiveAdmission"; import { log } from "@/node/services/log"; import { recordAgentWorkflowRunReference } from "@/node/services/agentWorkflowRunReferences"; import type { WorkspaceForkParams } from "@/node/runtime/Runtime"; @@ -3439,80 +3438,12 @@ describe("TaskService", () => { expect(findWorkspaceInConfig(config, childTaskId)?.taskPendingGuidance).toBeUndefined(); }); - test("initialize judges stale starting tasks from the startup config snapshot", async () => { - const config = await createTestConfig(rootDir); - const projectPath = path.join(rootDir, "repo"); - const parentWorkspaceId = "parent-snapshot"; - const staleStartingTaskId = "child-stale-starting"; - const stoppedStartingTaskId = "child-stopped-starting"; - const freshStartingTaskId = "child-fresh-starting"; - - await saveWorkspaces( - config, - projectPath, - [ - projectWorkspace(projectPath, "parent", parentWorkspaceId), - projectWorkspace(projectPath, "stale", staleStartingTaskId, { - parentWorkspaceId, - agentId: "exec", - agentType: "exec", - taskStatus: "starting", - taskModelString: "openai:gpt-5.2", - }), - projectWorkspace(projectPath, "stopped", stoppedStartingTaskId, { - parentWorkspaceId, - agentId: "exec", - agentType: "exec", - taskStatus: "starting", - taskModelString: "openai:gpt-5.2", - }), - ], - testTaskSettings() - ); - const startupSnapshot = config.loadConfigOrDefault(); - - // A client connected after the listener was bound: it is legitimately starting a new task and - // has stopped one of the snapshot's `starting` tasks before recovery applied its verdict. - await config.editConfig((cfg) => { - const workspaces = cfg.projects.get(projectPath)?.workspaces ?? []; - workspaces.push( - projectWorkspace(projectPath, "fresh", freshStartingTaskId, { - parentWorkspaceId, - agentId: "exec", - agentType: "exec", - taskStatus: "starting", - taskModelString: "openai:gpt-5.2", - }) - ); - const stopped = workspaces.find((workspace) => workspace.id === stoppedStartingTaskId); - if (stopped) stopped.taskStatus = "interrupted"; - return cfg; - }); - - const isStreaming = mock((workspaceId: string) => workspaceId === staleStartingTaskId); - const { aiService } = createAIServiceMocks(config, { isStreaming }); - const { workspaceService, sendMessage } = createWorkspaceServiceMocks(); - const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); - - await taskService.initialize(startupSnapshot); - - expect(findWorkspaceInConfig(config, staleStartingTaskId)?.taskStatus).toBe("running"); - expect(findWorkspaceInConfig(config, freshStartingTaskId)?.taskStatus).toBe("starting"); - expect(findWorkspaceInConfig(config, stoppedStartingTaskId)?.taskStatus).toBe("interrupted"); - const messagedWorkspaceIds = ( - sendMessage as unknown as { mock: { calls: unknown[][] } } - ).mock.calls.map((call) => call[0]); - expect(messagedWorkspaceIds).not.toContain(freshStartingTaskId); - expect(messagedWorkspaceIds).not.toContain(stoppedStartingTaskId); - }); - test("initialize does not resend the restart nudge to a running task that is already streaming", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); const parentWorkspaceId = "parent-restart-streaming"; const streamingTaskId = "child-running-streaming"; const idleTaskId = "child-running-idle"; - const stoppedTaskId = "child-running-stopped"; await saveWorkspaces( config, @@ -3533,34 +3464,14 @@ describe("TaskService", () => { taskStatus: "running", taskModelString: "openai:gpt-5.2", }), - projectWorkspace(projectPath, "stopped", stoppedTaskId, { - parentWorkspaceId, - agentId: "exec", - agentType: "exec", - taskStatus: "running", - taskModelString: "openai:gpt-5.2", - }), ], testTaskSettings() ); + // The queue drain can leave a task streaming before the running-task pass reaches it. const isStreaming = mock((workspaceId: string) => workspaceId === streamingTaskId); const { aiService } = createAIServiceMocks(config, { isStreaming }); - // A connected client stops one running task while recovery is resuming the others: the - // persisted `interrupted` status wins over the snapshot, so no restart nudge is sent. - const sendMessage = mock(async (workspaceId: string): Promise> => { - if (workspaceId === idleTaskId) { - await config.editConfig((cfg) => { - const stopped = cfg.projects - .get(projectPath) - ?.workspaces.find((workspace) => workspace.id === stoppedTaskId); - if (stopped) stopped.taskStatus = "interrupted"; - return cfg; - }); - } - return Ok(undefined); - }); - const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + const { workspaceService, sendMessage } = createWorkspaceServiceMocks(); const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); await taskService.initialize(); @@ -3570,237 +3481,57 @@ describe("TaskService", () => { ).mock.calls.map((call) => call[0]); expect(messagedWorkspaceIds).toContain(idleTaskId); expect(messagedWorkspaceIds).not.toContain(streamingTaskId); - expect(messagedWorkspaceIds).not.toContain(stoppedTaskId); - expect(findWorkspaceInConfig(config, stoppedTaskId)?.taskStatus).toBe("interrupted"); }); - test("startup recovery sends refuse admission once a task_stop lands mid-send", async () => { + test("startup phases stay partitioned: recovery resumes tasks, housekeeping prunes reported ones", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); - const parentWorkspaceId = "parent-stop-mid-send"; - const guidanceTaskId = "child-guidance-stopped-mid-send"; - const nudgeTaskId = "child-running-stopped-mid-send"; - const reportTaskId = "child-awaiting-report-stopped-mid-send"; + const parentWorkspaceId = "parent-startup-phases"; + const runningTaskId = "child-running-phases"; + const reportedTaskId = "child-reported-phases"; await saveWorkspaces( config, projectPath, [ projectWorkspace(projectPath, "parent", parentWorkspaceId), - projectWorkspace(projectPath, "guidance", guidanceTaskId, { - parentWorkspaceId, - agentId: "exec", - agentType: "exec", - taskStatus: "running", - taskModelString: "openai:gpt-5.2", - taskPendingGuidance: [ - { id: "guidance-1", message: "First correction", queueDispatchMode: "turn-end" }, - ], - }), - projectWorkspace(projectPath, "nudge", nudgeTaskId, { + projectWorkspace(projectPath, "running", runningTaskId, { parentWorkspaceId, agentId: "exec", agentType: "exec", taskStatus: "running", taskModelString: "openai:gpt-5.2", }), - projectWorkspace(projectPath, "report", reportTaskId, { + projectWorkspace(projectPath, "reported", reportedTaskId, { parentWorkspaceId, agentId: "explore", agentType: "explore", - taskStatus: "awaiting_report", + taskStatus: "reported", + reportedAt: "2026-08-10T00:00:00.000Z", taskModelString: "openai:gpt-5.2", + workflowTask: { runId: "wfr_startup_phases", stepId: "explore" }, }), ], testTaskSettings() ); - // The host re-evaluates internal.admissionStale at enqueue and at the session's admission - // gates. Model a task_stop persisting `interrupted` inside that window: the probe must flip - // from live to stale on the same send, and a stale send is refused (never resumed). - const probeBeforeStop: Record = {}; - const probeAfterStop: Record = {}; - const sendMessage = mock( - async ( - workspaceId: string, - _message: string, - _options: unknown, - internal?: { admissionStale?: () => boolean; onAccepted?: () => Promise | void } - ): Promise> => { - probeBeforeStop[workspaceId] = internal?.admissionStale?.(); - await config.editConfig((cfg) => { - const stopped = cfg.projects - .get(projectPath) - ?.workspaces.find((workspace) => workspace.id === workspaceId); - if (stopped) stopped.taskStatus = "interrupted"; - return cfg; - }); - probeAfterStop[workspaceId] = internal?.admissionStale?.(); - return Err("stopped before admission"); - } - ); - const { aiService } = createAIServiceMocks(config); - const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); - const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); - - await taskService.initialize(); - - for (const taskId of [guidanceTaskId, nudgeTaskId, reportTaskId]) { - expect(probeBeforeStop[taskId]).toBe(false); - expect(probeAfterStop[taskId]).toBe(true); - expect(findWorkspaceInConfig(config, taskId)?.taskStatus).toBe("interrupted"); - } - // A refused guidance replay keeps its durable reservation for the next resume. - expect(findWorkspaceInConfig(config, guidanceTaskId)?.taskPendingGuidance).toHaveLength(1); - }); - - test("initialize does not restore an execution mirror that a stop settled during reconciliation", async () => { - const config = await createTestConfig(rootDir); - const { parentId, projectPath } = await saveLocalParentWorkspace(config, rootDir); - const childTaskId = "child-stopped-during-reconcile"; - const handleId = "wst_stopped_during_reconcile"; - await config.editConfig((cfg) => { - const project = cfg.projects.get(projectPath); - assert(project, "test project must exist"); - project.workspaces.push( - projectWorkspace(projectPath, "child-reconcile", childTaskId, { - parentWorkspaceId: parentId, - agentId: "explore", - agentType: "explore", - taskStatus: "reported", - reportedAt: "2026-08-10T00:00:00.000Z", - taskExecutionId: handleId, - taskExecutionStatus: "running", - }) - ); - return cfg; - }); - const isStreaming = mock((workspaceId: string) => workspaceId === childTaskId); - const { aiService } = createAIServiceMocks(config, { isStreaming }); - const { taskService } = createTaskServiceHarness(config, { aiService }); - const internals = taskService as unknown as { - taskHandleStore: TaskHandleStore; - workspaceTurnManager: WorkspaceTurnManager; - activeWorkspaceTurnHandleByWorkspaceId: Map; - }; - await internals.taskHandleStore.upsertWorkspaceTurn( - workspaceTurnRecord(parentId, childTaskId, handleId, "running", { - turnId: "turn-reconcile", - createdAt: "2026-08-10T00:00:01.000Z", - updatedAt: "2026-08-10T00:00:01.000Z", - }) - ); - - // A task_stop settles the handle while reconciliation is between its liveness check and its - // mirror write: the persisted terminal state must win over the pre-await snapshot. - const manager = internals.workspaceTurnManager; - const originalNormalize = manager.normalizeWorkspaceTurnRecord.bind(manager); - spyOn(manager, "normalizeWorkspaceTurnRecord").mockImplementation(async (record, options) => { - const normalized = await originalNormalize(record, options); - if (normalized?.handleId === handleId) { - await internals.taskHandleStore.upsertWorkspaceTurn({ - ...normalized, - status: "interrupted", - updatedAt: "2026-08-10T00:00:02.000Z", - }); - await config.editConfig((cfg) => { - const child = cfg.projects - .get(projectPath) - ?.workspaces.find((workspace) => workspace.id === childTaskId); - if (child) child.taskExecutionStatus = "interrupted"; - return cfg; - }); - } - return normalized; - }); - - await taskService.initialize(); - - expect(findWorkspaceInConfig(config, childTaskId)?.taskExecutionStatus).toBe("interrupted"); - expect(internals.activeWorkspaceTurnHandleByWorkspaceId.get(childTaskId)).toBeUndefined(); - }); - - test("initialize does not overwrite an execution handle a client installed during reconciliation", async () => { - const config = await createTestConfig(rootDir); - const { parentId, projectPath } = await saveLocalParentWorkspace(config, rootDir); - const childTaskId = "child-replaced-during-reconcile"; - const oldHandleId = "wst_replaced_old"; - const newHandleId = "wst_replaced_new"; - await config.editConfig((cfg) => { - const project = cfg.projects.get(projectPath); - assert(project, "test project must exist"); - project.workspaces.push( - projectWorkspace(projectPath, "child-replaced", childTaskId, { - parentWorkspaceId: parentId, - agentId: "explore", - agentType: "explore", - taskStatus: "reported", - reportedAt: "2026-08-10T00:00:00.000Z", - taskExecutionId: oldHandleId, - taskExecutionStatus: "running", - }) - ); - return cfg; - }); - const isStreaming = mock((workspaceId: string) => workspaceId === childTaskId); - const { aiService } = createAIServiceMocks(config, { isStreaming }); - const { taskService } = createTaskServiceHarness(config, { aiService }); - const internals = taskService as unknown as { - taskHandleStore: TaskHandleStore; - workspaceTurnManager: WorkspaceTurnManager; - activeWorkspaceTurnHandleByWorkspaceId: Map< - string, - { handleId: string; ownerWorkspaceId: string; accepted: boolean } - >; - }; - await internals.taskHandleStore.upsertWorkspaceTurn( - workspaceTurnRecord(parentId, childTaskId, oldHandleId, "running", { - turnId: "turn-replaced-old", - createdAt: "2026-08-10T00:00:01.000Z", - updatedAt: "2026-08-10T00:00:01.000Z", - }) - ); - - // While reconciliation awaits the old handle's liveness check, a client reawakens the child - // under a new handle: it persists the new record, repoints the mirror, and registers the new - // handle under its own settlement lock. The old record itself is untouched. - const manager = internals.workspaceTurnManager; - const originalNormalize = manager.normalizeWorkspaceTurnRecord.bind(manager); - spyOn(manager, "normalizeWorkspaceTurnRecord").mockImplementation(async (record, options) => { - const normalized = await originalNormalize(record, options); - if (normalized?.handleId === oldHandleId) { - await internals.taskHandleStore.upsertWorkspaceTurn( - workspaceTurnRecord(parentId, childTaskId, newHandleId, "running", { - turnId: "turn-replaced-new", - createdAt: "2026-08-10T00:00:03.000Z", - updatedAt: "2026-08-10T00:00:03.000Z", - }) - ); - await config.editConfig((cfg) => { - const child = cfg.projects - .get(projectPath) - ?.workspaces.find((workspace) => workspace.id === childTaskId); - if (child) { - child.taskExecutionId = newHandleId; - child.taskExecutionStatus = "running"; - } - return cfg; - }); - internals.activeWorkspaceTurnHandleByWorkspaceId.set(childTaskId, { - handleId: newHandleId, - ownerWorkspaceId: parentId, - accepted: true, - }); - } - return normalized; - }); + const remove = createConfigBackedRemoveMock(config); + const { workspaceService, sendMessage } = createWorkspaceServiceMocks({ remove }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); - await taskService.initialize(); + // The listener-gating phase only touches active tasks: no O(reported tasks) cleanup here. + await taskService.recoverInterruptedTasks(); + const messagedWorkspaceIds = ( + sendMessage as unknown as { mock: { calls: unknown[][] } } + ).mock.calls.map((call) => call[0]); + expect(messagedWorkspaceIds).toEqual([runningTaskId]); + expect(remove).not.toHaveBeenCalled(); - expect(findWorkspaceInConfig(config, childTaskId)?.taskExecutionId).toBe(newHandleId); - expect(internals.activeWorkspaceTurnHandleByWorkspaceId.get(childTaskId)?.handleId).toBe( - newHandleId - ); + // The background phase never resumes tasks (clients may be acting on them by now). + await taskService.runStartupHousekeeping(); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(remove.mock.calls.map((call) => call[0])).toEqual([reportedTaskId]); + expect(findWorkspaceInConfig(config, reportedTaskId)).toBeUndefined(); }); test("initialize does not reload config.json per completed-report task", async () => { @@ -13755,72 +13486,6 @@ describe("TaskService", () => { expect(findWorkspaceInConfig(config, queuedChildId)?.taskPrompt).toBe("queued work"); }); - test("initialize leaves workflow-owned children alone while their run is being resumed", async () => { - const config = await createTestConfig(rootDir); - - const projectPath = path.join(rootDir, "repo"); - const runtimeConfig = { type: "local" as const }; - const parentId = "parent-workflow-resuming"; - const runningChildId = "child-workflow-resuming-running"; - const workflowRunId = "wfr_interrupted_but_resuming"; - - await saveWorkspaces( - config, - projectPath, - [ - projectWorkspace(projectPath, "parent", parentId, { runtimeConfig }), - projectWorkspace(projectPath, "running", runningChildId, { - name: "agent_explore_running", - parentWorkspaceId: parentId, - agentId: "explore", - agentType: "explore", - taskStatus: "running", - taskModelString: defaultModel, - runtimeConfig, - workflowTask: { runId: workflowRunId, stepId: "running" }, - }), - ], - testTaskSettings(10, 3) - ); - const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); - await runStore.createRun({ - id: workflowRunId, - workspaceId: parentId, - workflow: { - name: "interrupted", - description: "Interrupted", - scope: "built-in", - executable: true, - }, - source: "export default function workflow() { return {}; }\n", - args: {}, - now: "2026-05-29T00:00:00.000Z", - }); - await runStore.appendStatus(workflowRunId, "interrupted", "2026-05-29T00:00:01.000Z"); - - const { aiService } = createAIServiceMocks(config, { isStreaming: mock(() => false) }); - const { workspaceService, sendMessage } = createWorkspaceServiceMocks(); - const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); - - // A client resumed the owning run while recovery was reading it as interrupted: the resume - // holds a run-keyed workflow admission until the run settles, and recovery must neither - // interrupt the child (the resumed run owns it) nor nudge it as an orphaned running task. - { - using _resumeAdmission = acquireWorkflowArchiveAdmission(parentId, workflowRunId); - await taskService.initialize(); - } - expect(findWorkspaceInConfig(config, runningChildId)?.taskStatus).toBe("running"); - expect(sendMessage).not.toHaveBeenCalled(); - - // An unrelated run active in the same workspace must not shield the stale child. - { - using _otherRunAdmission = acquireWorkflowArchiveAdmission(parentId, "wfr_unrelated_run"); - await taskService.initialize(); - } - expect(findWorkspaceInConfig(config, runningChildId)?.taskStatus).toBe("interrupted"); - expect(sendMessage).not.toHaveBeenCalled(); - }); - test("initialize recovers parent tasks after interrupting inactive workflow children", async () => { const config = await createTestConfig(rootDir); @@ -14467,62 +14132,6 @@ describe("TaskService", () => { ); }); - test("startup completion prompt is idle-only and refunds its attempt when a client turn wins", async () => { - const config = await createTestConfig(rootDir); - const projectPath = path.join(rootDir, "repo"); - const parentId = "parent-idle-only"; - const childId = "child-idle-only"; - await saveWorkspaces( - config, - projectPath, - [ - projectWorkspace(projectPath, "parent", parentId), - projectWorkspace(projectPath, "child", childId, { - name: "agent_explore_child", - parentWorkspaceId: parentId, - agentType: "explore", - taskStatus: "awaiting_report", - taskRecoveryAttempts: 2, - }), - ], - testTaskSettings(1, 3) - ); - - // A client turn entered preflight after the isStreaming() probe: idle-only sends skip. - const sendMessage = mock((..._args: unknown[]): Promise> => { - const internal = _args[3] as { requireIdle?: boolean } | undefined; - if (internal?.requireIdle === true) { - return Promise.resolve( - Err({ type: "unknown", raw: "Workspace is busy; idle-only send was skipped." }) - ); - } - return Promise.resolve(Ok(undefined)); - }); - const { aiService } = createAIServiceMocks(config); - const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); - const { taskService } = createTaskServiceHarness(config, { aiService, workspaceService }); - const internals = taskService as unknown as { - promptTaskForRequiredCompletionTool: ( - workspaceId: string, - options?: { reason?: "startup" | "stream_end" | "error" } - ) => Promise; - }; - - await taskService.initialize(); - - expect(sendMessage).toHaveBeenCalledTimes(1); - expect(sendMessage.mock.calls[0]?.[3]).toMatchObject({ requireIdle: true }); - // The skipped prompt sent nothing, so the recovery budget is untouched. - expect(findWorkspaceInConfig(config, childId)?.taskRecoveryAttempts).toBe(2); - - // Stream-end prompts still queue behind whatever is running. - expect( - await internals.promptTaskForRequiredCompletionTool(childId, { reason: "stream_end" }) - ).toBe(true); - expect(sendMessage.mock.calls[1]?.[3]).not.toMatchObject({ requireIdle: true }); - expect(findWorkspaceInConfig(config, childId)?.taskRecoveryAttempts).toBe(3); - }); - test("initialize uses legacy agentType when modern agentId is unavailable for awaiting_report tasks", async () => { const config = await createTestConfig(rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index fb366beb539..0653bfea202 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -187,7 +187,6 @@ import { isWorkspaceArchived } from "@/common/utils/archive"; import type { ToolPolicy } from "@/common/utils/tools/toolPolicy"; import { CONTEXT_BOUNDARY_KINDS } from "@/common/constants/contextBoundary"; import { WorkflowRunStore } from "@/node/services/workflows/WorkflowRunStore"; -import { hasInProcessWorkflowRun } from "@/node/services/workflows/workflowArchiveAdmission"; import { isActiveWorkspaceTurnTaskStatus, isWorkspaceTurnTaskId, @@ -1728,20 +1727,10 @@ export class TaskService implements AgentTaskIntegration { let interrupted = false; let transitionedToInterrupted = false; - let resumeInFlight = false; let parentWorkspaceId: string | undefined; await this.editWorkspaceEntry( taskId, (ws) => { - // Startup recovery can run while clients are connected, and the run-store read above - // awaited. A resume of this run admitted since then owns this child's fate: it holds a - // run-keyed workflow admission (see workflowArchiveAdmission) from its synchronous entry - // until the run settles, so checking it inside this synchronous edit cannot miss an - // in-flight resume, and unrelated runs in the same workspace do not match. - if (hasInProcessWorkflowRun(inactiveOwner.runId)) { - resumeInFlight = true; - return; - } const previousStatus = ws.taskStatus; parentWorkspaceId = ws.parentWorkspaceId; interrupted = this.applyInterruptedTaskStatus(ws) === "interrupted"; @@ -1749,15 +1738,6 @@ export class TaskService implements AgentTaskIntegration { }, { allowMissing: true } ); - if (resumeInFlight) { - log.debug("Skipping workflow-owned task recovery; owner workflow is being resumed", { - taskId, - trigger, - ownerTaskId: inactiveOwner.ownerTaskId, - workflowRunId: inactiveOwner.runId, - }); - return true; - } if (transitionedToInterrupted) { this.recordTaskInterrupted(taskId, parentWorkspaceId); } @@ -2281,25 +2261,31 @@ export class TaskService implements AgentTaskIntegration { } /** - * @param startupConfigSnapshot - config snapshot taken when the process started. When the server - * binds its listener before running recovery, clients may already be creating tasks; judging - * stale-`starting` recovery from this snapshot keeps those fresh tasks out of scope. + * Restart recovery for agent tasks followed by the startup housekeeping passes. The server + * entry point runs the two halves separately around its listener bind (see + * ServiceContainer.initializeCore / runStartupHousekeeping); desktop startup runs them back to back. */ - async initialize( - startupConfigSnapshot?: ProjectsConfig, - options?: { signal?: AbortSignal } - ): Promise { + async initialize(): Promise { + await this.recoverInterruptedTasks(); + await this.runStartupHousekeeping(); + } + + /** + * Re-establishes the tasks that were active when the process last exited: execution-handle + * mirrors, stale `starting` tasks, the queue drain, and the restart prompts for + * `awaiting_report`/`running` tasks. Bounded by the number of active tasks, not by deployment + * size. Must finish before any client can act on tasks: a stop, resume, or send racing these + * transitions would be overwritten or would resurrect a task the client just stopped, so the + * server binds its listener only after this resolves. + */ + async recoverInterruptedTasks(): Promise { const startupStartedAt = Date.now(); - // Server-mode recovery runs after the listener is bound and can be cancelled by dispose(); - // every size-dependent step below checks this (and returns) so teardown is neither delayed - // by nor raced with recovery work against disposed services. - const cancelled = (): boolean => options?.signal?.aborted === true; - const startupConfig = startupConfigSnapshot ?? this.config.loadConfigOrDefault(); + const startupConfig = this.config.loadConfigOrDefault(); const queuedTaskCountAtStartup = this.listAgentTaskWorkspaces(startupConfig).filter( (task) => task.taskStatus === "queued" && typeof task.id === "string" ).length; - log.info("[startup] TaskService.initialize starting", { + log.info("[startup] TaskService.recoverInterruptedTasks starting", { queuedTaskCountAtStartup, }); @@ -2328,10 +2314,7 @@ export class TaskService implements AgentTaskIntegration { const recovery = recoveries.get(task.id); assert(recovery != null, "stale starting task recovery is required"); const entry = findWorkspaceEntry(config, task.id); - // Compare-and-set: recovery may run while clients are connected, so a task_stop that - // already persisted `interrupted` (or any other transition) must not be overwritten - // with the snapshot-derived status and relaunched by the queue drain below. - if (entry?.workspace.taskStatus !== "starting") continue; + if (!entry) continue; entry.workspace.taskStatus = recovery.status; if (recovery.acceptedPrompt) { // The initial prompt is already durable in chat history; clearing taskPrompt makes the @@ -2434,7 +2417,6 @@ export class TaskService implements AgentTaskIntegration { let resumedRunningCount = 0; let skippedRunningDueToActiveDescendants = 0; let skippedRunningAlreadyStreaming = 0; - let skippedRunningNoLongerRunning = 0; let failedRunningCount = 0; for (const task of runningTasks) { @@ -2450,35 +2432,6 @@ export class TaskService implements AgentTaskIntegration { continue; } - // Recovery may run after the server is accepting connections. Re-read the live status so a - // task_stop that persisted `interrupted` since the snapshot is not undone by a restart nudge - // (sendMessage would treat that as a user-driven resume of the stopped task). The same check - // rides along as the send's admission probe: a stop can also land during the awaits between - // here and turn admission, and a guarded send skips the interrupted-task rescue entirely. - const taskId = task.id; - const admissionStale = (): boolean => - findWorkspaceEntry(this.config.loadConfigOrDefault(), taskId)?.workspace.taskStatus !== - "running"; - if (admissionStale()) { - skippedRunningNoLongerRunning += 1; - continue; - } - // The restart nudge additionally requires no blocking active descendants; a client can - // spawn one after `taskIndex` was captured, so the probe re-derives that gate from fresh - // config at admission as well (pending guidance replays regardless, see above). - const restartNudgeAdmissionStale = (): boolean => { - const freshConfig = this.config.loadConfigOrDefault(); - if (findWorkspaceEntry(freshConfig, taskId)?.workspace.taskStatus !== "running") { - return true; - } - return ( - this.listBlockingActiveDescendantAgentTaskIdsUsingIndex( - this.buildAgentTaskIndex(freshConfig), - taskId - ).length > 0 - ); - }; - const pendingGuidance = task.taskPendingGuidance ?? []; if (pendingGuidance.length > 0) { // Pending corrections outrank generic restart recovery and must replay even when this task @@ -2515,7 +2468,6 @@ export class TaskService implements AgentTaskIntegration { { synthetic: true, agentInitiated: true, - admissionStale, onAccepted: clearAcceptedPendingGuidance, } ); @@ -2531,9 +2483,8 @@ export class TaskService implements AgentTaskIntegration { continue; } - // Recovery may run after the server is already accepting connections, so a client (or the - // queue drain above) can legitimately have this task streaming by now. Nudging it again - // would inject a spurious "Xum restarted" turn into a live stream. + // The queue drain above can have launched this task already; nudging it again would queue + // a spurious "Xum restarted" turn behind its first stream. if (this.aiService.isStreaming(task.id)) { skippedRunningAlreadyStreaming += 1; continue; @@ -2578,14 +2529,7 @@ export class TaskService implements AgentTaskIntegration { reasoningMode: coerceOpenAIReasoningMode(task.aiSettings?.reasoningMode), experiments: task.taskExperiments, }, - { - synthetic: true, - agentInitiated: true, - // Idle-only: a client turn that entered preflight after the isStreaming() check above - // must suppress this stale "Xum restarted" nudge instead of queueing it behind itself. - requireIdle: true, - admissionStale: restartNudgeAdmissionStale, - } + { synthetic: true, agentInitiated: true } ); const durationMs = Date.now() - resumeStartedAt; if (!sendResult.success) { @@ -2619,9 +2563,36 @@ export class TaskService implements AgentTaskIntegration { // Startup queue draining already ran before these interruptions freed slots. // Run it once more after recovery prompts so unrelated queued work is not stranded. await this.maybeStartQueuedTasks(); - config = this.config.loadConfigOrDefault(); } + log.info("[startup] TaskService.recoverInterruptedTasks completed", { + totalMs: Date.now() - startupStartedAt, + maybeStartQueuedTasksMs, + awaitingReportTaskCount: awaitingReportTasks.length, + resumedAwaitingReportCount, + skippedAwaitingReportDueToActiveDescendants, + failedAwaitingReportCount, + runningTaskCount: runningTasks.length, + resumedRunningCount, + skippedRunningDueToActiveDescendants, + skippedRunningAlreadyStreaming, + failedRunningCount, + }); + } + + /** + * Startup passes over every reported task and session directory: pending patch-artifact + * generation, reported-task cleanup, workflow garbage sweeps, and terminal-attention delivery. + * Scales with deployment size, so the server runs it after its listener is bound. Clients may + * therefore already be acting: every step re-checks live state before it mutates, and + * `options.signal` (aborted by dispose) stops the pass at the next step boundary so teardown + * neither waits for nor races housekeeping against disposed services. + */ + async runStartupHousekeeping(options?: { signal?: AbortSignal }): Promise { + const startupStartedAt = Date.now(); + const cancelled = (): boolean => options?.signal?.aborted === true; + const config = this.config.loadConfigOrDefault(); + // Restart-safety for git patch artifacts: // - If xum crashed mid-generation, patch artifacts can be left "pending". // - Completed tasks can be stranded in config until cleanup runs again, so restart should @@ -2634,6 +2605,9 @@ export class TaskService implements AgentTaskIntegration { for (const task of completedReportTasks) { if (cancelled()) return; if (!task.parentWorkspaceId) continue; + // A reported task that is streaming again (a client reactivated it) may be committing; + // the continuation refresh that runs when that turn ends generates its artifact. + if (this.aiService.isStreaming(task.id!)) continue; try { // Pass the loop snapshot: reloading config.json per reported task made this pass scale // with (reported tasks x config size) on large deployments. @@ -2737,17 +2711,6 @@ export class TaskService implements AgentTaskIntegration { const totalMs = Date.now() - startupStartedAt; const completedPayload = { totalMs, - maybeStartQueuedTasksMs, - awaitingReportTaskCount: awaitingReportTasks.length, - resumedAwaitingReportCount, - skippedAwaitingReportDueToActiveDescendants, - failedAwaitingReportCount, - runningTaskCount: runningTasks.length, - resumedRunningCount, - skippedRunningDueToActiveDescendants, - skippedRunningAlreadyStreaming, - skippedRunningNoLongerRunning, - failedRunningCount, completedReportTaskCount: completedReportTasks.length, patchGenerationRecoveryMs, bestOfParentRecoveryCount: bestOfParentWorkspaceIds.size, @@ -2759,9 +2722,9 @@ export class TaskService implements AgentTaskIntegration { cleanupReportedTasksMs, }; if (totalMs > SLOW_STARTUP_WARN_THRESHOLD_MS) { - log.warn("[startup] TaskService.initialize completed", completedPayload); + log.warn("[startup] TaskService.runStartupHousekeeping completed", completedPayload); } else { - log.info("[startup] TaskService.initialize completed", completedPayload); + log.info("[startup] TaskService.runStartupHousekeeping completed", completedPayload); } } @@ -10532,52 +10495,10 @@ export class TaskService implements AgentTaskIntegration { ? { toolPolicy: [{ regex_match: "^propose_plan$", action: "require" as const }] } : {}), }, - { - synthetic: true, - agentInitiated: true, - // Startup recovery runs while clients may be sending to this task: a user turn that - // entered preflight after the isStreaming() check must suppress the prompt instead of - // queueing it behind the user's turn (stream-end re-prompts if that turn ends unreported). - requireIdle: options?.reason === "startup", - // The status and descendant checks at the top of this method are one-shot reads; a - // task_stop or a newly spawned blocking descendant landing during the awaits since then - // must refuse admission rather than resume the coordinator. - admissionStale: () => { - const freshConfig = this.config.loadConfigOrDefault(); - if ( - findWorkspaceEntry(freshConfig, workspaceId)?.workspace.taskStatus !== "awaiting_report" - ) { - return true; - } - return ( - this.listBlockingActiveDescendantAgentTaskIdsUsingIndex( - this.buildAgentTaskIndex(freshConfig), - workspaceId - ).length > 0 - ); - }, - } + { synthetic: true, agentInitiated: true } ); const durationMs = Date.now() - startedAt; if (!sendResult.success) { - if (isWorkspaceBusyIdleOnlySend(sendResult.error)) { - // A client turn won admission first (only startup sends are idle-only). Nothing was - // sent, so hand the attempt back: a user's turn must not draw down the recovery budget. - await this.editWorkspaceEntry( - workspaceId, - (ws) => { - ws.taskRecoveryAttempts = Math.max(0, (ws.taskRecoveryAttempts ?? 0) - 1); - }, - { allowMissing: true } - ); - log.info("Skipped completion prompt: task became busy", { - workspaceId, - taskName: entry.workspace.name, - completionKind, - reason: options?.reason, - }); - return false; - } log.error("Failed to prompt task for required completion", { workspaceId, taskName: entry.workspace.name, diff --git a/src/node/services/workflows/WorkflowService.ts b/src/node/services/workflows/WorkflowService.ts index 00f9813d933..affe845de4e 100644 --- a/src/node/services/workflows/WorkflowService.ts +++ b/src/node/services/workflows/WorkflowService.ts @@ -349,7 +349,7 @@ export class WorkflowService { projectTrusted: boolean; }): Promise { // Archive admission pairing; see resumeRunInBackground. - using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId, input.runId); + using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId); const run = await this.requireRunForWorkspace(input); assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); assertWorkflowRunCanRetryFromCheckpoint(run); @@ -372,7 +372,7 @@ export class WorkflowService { // Archive admission pairing: refuse while the workspace is archiving/archived, and hold // the admission across the method so the archive sink observes a resume that has not yet // durably re-activated its run (see workflowArchiveAdmission). - using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId, input.runId); + using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId); const run = await this.requireRunForWorkspace(input); assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); assertWorkflowRunCanTransition(run.status, "running"); @@ -394,7 +394,7 @@ export class WorkflowService { abortSignal?: AbortSignal; }): Promise { // Archive admission pairing; see resumeRunInBackground. - using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId, input.runId); + using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId); const run = await this.requireRunForWorkspace(input); assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); assertWorkflowRunCanTransition(run.status, "running"); @@ -415,7 +415,7 @@ export class WorkflowService { abortSignal?: AbortSignal; }): Promise { // Archive admission pairing; see resumeRunInBackground. - using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId, input.runId); + using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId); const run = await this.requireRunForWorkspace(input); assertRunCanResumeWithCurrentTrust(run, input.projectTrusted); assertWorkflowRunCanRetryFromCheckpoint(run); @@ -670,7 +670,7 @@ export class WorkflowService { // Registration happens at lease acquisition, while the entry point's archive admission is // still held, so the archive sink observes in-process workflow work continuously from // admission entry to terminal settlement (see workflowArchiveAdmission). - const releaseInProcessWork = registerInProcessWorkflowRun(workspaceId, runId); + const releaseInProcessWork = registerInProcessWorkflowRun(workspaceId); return () => { if (activeWorkflowRunnerAbortControllers.get(runId) === controller) { activeWorkflowRunnerAbortControllers.delete(runId); @@ -1177,7 +1177,7 @@ export async function resumeWorkflowRun( context: WorkflowServiceContext, input: { workspaceId: string; runId: string } ): Promise { - using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId, input.runId); + using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId); const { service, projectTrusted } = await resolveWorkflowContext(context, input.workspaceId); return service.resumeRunInBackground({ ...input, projectTrusted }); } @@ -1186,7 +1186,7 @@ export async function retryWorkflowRunFromCheckpoint( context: WorkflowServiceContext, input: { workspaceId: string; runId: string } ): Promise { - using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId, input.runId); + using _archiveAdmission = acquireWorkflowArchiveAdmission(input.workspaceId); const { service, projectTrusted } = await resolveWorkflowContext(context, input.workspaceId, { onBackgroundRunTerminal: (event) => { const name = event.run.workflow.sourcePath ?? event.run.workflow.name; diff --git a/src/node/services/workflows/workflowArchiveAdmission.ts b/src/node/services/workflows/workflowArchiveAdmission.ts index b0fac35f1cd..902d6a593d9 100644 --- a/src/node/services/workflows/workflowArchiveAdmission.ts +++ b/src/node/services/workflows/workflowArchiveAdmission.ts @@ -17,9 +17,6 @@ import assert from "node:assert/strict"; let admissionGuard: ((workspaceId: string) => string | null) | null = null; const inProcessWorkflowWorkByWorkspace = new Map(); -// Per-run view of the same admissions, for callers that must not confuse unrelated runs in one -// workspace (startup recovery of a terminal run's children while another run is active). -const inProcessWorkflowWorkByRun = new Map(); /** Register the archive-side guard. Returns a refusal message or null when admission is allowed. */ export function setWorkflowArchiveAdmissionGuard( @@ -28,43 +25,37 @@ export function setWorkflowArchiveAdmissionGuard( admissionGuard = guard; } -function incrementCount(counts: Map, key: string): () => void { - counts.set(key, (counts.get(key) ?? 0) + 1); +function incrementInProcessWorkflowWork(workspaceId: string): () => void { + assert(workspaceId.length > 0, "workflowArchiveAdmission: workspaceId is required"); + inProcessWorkflowWorkByWorkspace.set( + workspaceId, + (inProcessWorkflowWorkByWorkspace.get(workspaceId) ?? 0) + 1 + ); let released = false; return () => { if (released) return; released = true; - const remaining = (counts.get(key) ?? 1) - 1; + const remaining = (inProcessWorkflowWorkByWorkspace.get(workspaceId) ?? 1) - 1; if (remaining <= 0) { - counts.delete(key); + inProcessWorkflowWorkByWorkspace.delete(workspaceId); } else { - counts.set(key, remaining); + inProcessWorkflowWorkByWorkspace.set(workspaceId, remaining); } }; } -function incrementInProcessWorkflowWork(workspaceId: string, runId?: string): () => void { - assert(workspaceId.length > 0, "workflowArchiveAdmission: workspaceId is required"); - const releaseWorkspace = incrementCount(inProcessWorkflowWorkByWorkspace, workspaceId); - const releaseRun = runId != null ? incrementCount(inProcessWorkflowWorkByRun, runId) : null; - return () => { - releaseWorkspace(); - releaseRun?.(); - }; -} - /** * Admit a workflow start/resume/retry for this workspace. Throws when an archive gate is * armed or the workspace is archived; otherwise counts the admission as in-process workflow * work until disposed. Entry points hold the admission across the whole method so the * archive sink observes work that has not yet produced a durably active run record. */ -export function acquireWorkflowArchiveAdmission(workspaceId: string, runId?: string): Disposable { +export function acquireWorkflowArchiveAdmission(workspaceId: string): Disposable { const refusal = admissionGuard?.(workspaceId) ?? null; if (refusal != null) { throw new Error(refusal); } - const release = incrementInProcessWorkflowWork(workspaceId, runId); + const release = incrementInProcessWorkflowWork(workspaceId); return { [Symbol.dispose]: release }; } @@ -74,16 +65,11 @@ export function acquireWorkflowArchiveAdmission(workspaceId: string, runId?: str * admission is still held), so coverage is continuous from admission entry to terminal * settlement even before the runner durably appends its "running" status. */ -export function registerInProcessWorkflowRun(workspaceId: string, runId?: string): () => void { - return incrementInProcessWorkflowWork(workspaceId, runId); +export function registerInProcessWorkflowRun(workspaceId: string): () => void { + return incrementInProcessWorkflowWork(workspaceId); } /** Whether any workflow admission or in-process runner exists for this workspace. */ export function hasInProcessWorkflowWork(workspaceId: string): boolean { return (inProcessWorkflowWorkByWorkspace.get(workspaceId) ?? 0) > 0; } - -/** Whether an admission (resume/retry) or in-process runner exists for this specific run. */ -export function hasInProcessWorkflowRun(runId: string): boolean { - return (inProcessWorkflowWorkByRun.get(runId) ?? 0) > 0; -} diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index 8644837858f..b74f3b9d6ed 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -4830,55 +4830,25 @@ export class WorkspaceTurnManager { continue; } - const taskId = task.id; - const { handleId, ownerWorkspaceId, status } = normalized; - // This pass can run while clients are connected, and normalizeWorkspaceTurnRecord awaited, - // so two things may have changed underneath the snapshot: a task_stop may have settled - // this handle (settlement persists the record and its mirror under the per-handle - // settlement lock, so compare-and-set the record status under that same lock), or a - // client may have installed a different handle for this task, which writes the mirror - // under its own handle's lock and leaves this record untouched, so the mirror pointer - // itself is compare-and-set against the snapshot as well. Writing the stale snapshot in - // either case would restore an active status or registration the live state has moved - // past. - await this.workspaceTurnSettlementLocks.withLock(handleId, async () => { - const current = await this.taskHandleStore.getWorkspaceTurn(ownerWorkspaceId, handleId); - if (current?.status !== status) { - return; - } - let mirrorUpdated = false; - await this.taskHost.editWorkspaceEntry( - taskId, - (workspace) => { - if (workspace.taskExecutionId !== task.taskExecutionId) { - return; - } - workspace.taskExecutionId = handleId; - workspace.taskExecutionStatus = status; - mirrorUpdated = true; - }, - { allowMissing: true } - ); - if (!mirrorUpdated) { - return; - } - await this.taskHost.emitWorkspaceMetadata(taskId); - // A registration for this same handle that a live client turn already installed (and - // possibly marked accepted) is authoritative; only a missing registration is recovered. - if ( - isActiveWorkspaceTurnTaskStatus(status) && - this.activeWorkspaceTurnHandleByWorkspaceId.get(taskId) == null - ) { - this.activeWorkspaceTurnHandleByWorkspaceId.set(taskId, { - handleId, - ownerWorkspaceId, - // Fail closed: the restart killed any live stream, so this recovered registration - // exists for settlement ownership, not peer admission. A revival or a fresh - // acceptance re-marks the entry accepted. - accepted: false, - }); - } - }); + await this.taskHost.editWorkspaceEntry( + task.id, + (workspace) => { + workspace.taskExecutionId = normalized.handleId; + workspace.taskExecutionStatus = normalized.status; + }, + { allowMissing: true } + ); + await this.taskHost.emitWorkspaceMetadata(task.id); + if (isActiveWorkspaceTurnTaskStatus(normalized.status)) { + this.activeWorkspaceTurnHandleByWorkspaceId.set(task.id, { + handleId: normalized.handleId, + ownerWorkspaceId: normalized.ownerWorkspaceId, + // Fail closed: the restart killed any live stream, so this recovered registration + // exists for settlement ownership, not peer admission. A revival or a fresh + // acceptance re-marks the entry accepted. + accepted: false, + }); + } } catch (error: unknown) { // Startup recovery is best-effort: one read-only/corrupt child must not prevent Xum startup // or block reconciliation of the remaining persistent children. From 98ea055e7166a72c5a08e21ad4d272c0ccf51d59 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:39:46 +0000 Subject: [PATCH 14/26] Finalize pending best-of parent partials before the listener opens Startup best-of recovery rewrites a crashed parent's partial.json after asynchronous report assembly. Run post-listen, a client could start a parent turn in between: commitPartial would commit the old partial and the recovery write would then overwrite the live turn's partial. Move the pass into recoverInterruptedTasks (pre-listen), keeping main's prompts -> best-of -> cleanup order; housekeeping no longer touches parent partials. Cost is one partial read per parent of reported best-of children. --- src/node/services/serviceContainer.ts | 6 +-- src/node/services/taskService.test.ts | 37 ++++++++------ src/node/services/taskService.ts | 73 +++++++++++++++------------ 3 files changed, 65 insertions(+), 51 deletions(-) diff --git a/src/node/services/serviceContainer.ts b/src/node/services/serviceContainer.ts index ad61e4cd521..e25aa560287 100644 --- a/src/node/services/serviceContainer.ts +++ b/src/node/services/serviceContainer.ts @@ -320,9 +320,9 @@ export class ServiceContainer { /** * Everything request handling depends on, plus agent-task restart recovery. The server entry * point awaits this before binding its listener: task recovery must finish before any client - * can stop, resume, or send to a task (see TaskService.recoverInterruptedTasks), and it is - * bounded by the number of active tasks rather than by deployment size. The per-workspace - * housekeeping lives in runStartupHousekeeping(). + * can stop, resume, or send to a task or its parent (see TaskService.recoverInterruptedTasks), + * and it is bounded by the number of active tasks and best-of parents rather than by deployment + * size. The per-workspace housekeeping lives in runStartupHousekeeping(). */ async initializeCore(): Promise { this.startupStartedAt = Date.now(); diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index e2dfc8a0345..deacea41c49 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -20077,7 +20077,7 @@ describe("TaskService", () => { ).toHaveLength(1); }); - test("initialize finalizes ready best-of partials before cleanup rechecks", async () => { + test("pre-listen recovery, not housekeeping, finalizes ready best-of partials before cleanup rechecks", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); @@ -20170,12 +20170,10 @@ describe("TaskService", () => { nowMs: Date.now(), }); - await taskService.initialize(); - - const updatedParentPartial = await partialService.readPartial(parentId); - expect(updatedParentPartial).not.toBeNull(); - if (updatedParentPartial) { - const toolPart = updatedParentPartial.parts.find( + const readParentTaskToolPart = async () => { + const parentPartial = await partialService.readPartial(parentId); + expect(parentPartial).not.toBeNull(); + return parentPartial?.parts.find( (p) => p && typeof p === "object" && @@ -20188,14 +20186,23 @@ describe("TaskService", () => { output?: unknown; } | undefined; - expect(toolPart?.toolName).toBe("task"); - expect(toolPart?.state).toBe("output-available"); - const outputJson = JSON.stringify(toolPart?.output); - expect(outputJson).toContain(childOneId); - expect(outputJson).toContain(childTwoId); - expect(outputJson).toContain("Report from child one"); - expect(outputJson).toContain("Report from child two"); - } + }; + + // Rewriting a parent's partial races any parent turn a client starts once the listener is + // open, so the post-listen housekeeping phase must leave it alone. + await taskService.runStartupHousekeeping(); + expect((await readParentTaskToolPart())?.state).toBe("input-available"); + + await taskService.recoverInterruptedTasks(); + + const toolPart = await readParentTaskToolPart(); + expect(toolPart?.toolName).toBe("task"); + expect(toolPart?.state).toBe("output-available"); + const outputJson = JSON.stringify(toolPart?.output); + expect(outputJson).toContain(childOneId); + expect(outputJson).toContain(childTwoId); + expect(outputJson).toContain("Report from child one"); + expect(outputJson).toContain("Report from child two"); const remainingTaskIds = Array.from(config.loadConfigOrDefault().projects.values()) .flatMap((project) => project.workspaces) diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 0653bfea202..083c3c6d243 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -2272,11 +2272,12 @@ export class TaskService implements AgentTaskIntegration { /** * Re-establishes the tasks that were active when the process last exited: execution-handle - * mirrors, stale `starting` tasks, the queue drain, and the restart prompts for - * `awaiting_report`/`running` tasks. Bounded by the number of active tasks, not by deployment - * size. Must finish before any client can act on tasks: a stop, resume, or send racing these - * transitions would be overwritten or would resurrect a task the client just stopped, so the - * server binds its listener only after this resolves. + * mirrors, stale `starting` tasks, the queue drain, the restart prompts for + * `awaiting_report`/`running` tasks, and pending best-of task tool calls in crashed parents' + * partials. Bounded by the number of active tasks plus one partial read per best-of parent, + * not by deployment size. Must finish before any client can act on tasks or parents: a stop, + * resume, or send racing these transitions would be overwritten or would resurrect a task the + * client just stopped, so the server binds its listener only after this resolves. */ async recoverInterruptedTasks(): Promise { const startupStartedAt = Date.now(); @@ -2565,9 +2566,40 @@ export class TaskService implements AgentTaskIntegration { await this.maybeStartQueuedTasks(); } + // Restart-safety for grouped best-of completion: if child report artifacts already exist + // on disk after a restart, there may be no later child stream-end to finalize the pending + // parent task tool call, and reported-task cleanup stays blocked behind the stale + // input-available partial. Finalizing rewrites the parent's partial.json, which a parent + // turn started by a client would race (commit the old partial, then have its own overwritten), + // so this runs before the listener rather than in housekeeping. Cost: one partial read per + // parent of reported best-of children. + const bestOfRecoveryStartedAt = Date.now(); + const bestOfParentWorkspaceIds = new Set(); + for (const task of this.listAgentTaskWorkspaces(config)) { + const parentWorkspaceId = coerceNonEmptyString(task.parentWorkspaceId); + const taskId = coerceNonEmptyString(task.id); + if (!parentWorkspaceId || !taskId || !hasCompletedAgentReport(task)) { + continue; + } + if ((this.getEffectiveTaskGroup(taskId, task)?.total ?? 1) <= 1) { + continue; + } + bestOfParentWorkspaceIds.add(parentWorkspaceId); + } + for (const parentWorkspaceId of bestOfParentWorkspaceIds) { + // A parent the prompts above resumed is streaming; its own stream-end reruns this pass. + if (this.aiService.isStreaming(parentWorkspaceId)) { + continue; + } + await this.deliverDeferredBestOfReportsForParent(parentWorkspaceId); + } + const bestOfRecoveryMs = Date.now() - bestOfRecoveryStartedAt; + log.info("[startup] TaskService.recoverInterruptedTasks completed", { totalMs: Date.now() - startupStartedAt, maybeStartQueuedTasksMs, + bestOfParentRecoveryCount: bestOfParentWorkspaceIds.size, + bestOfRecoveryMs, awaitingReportTaskCount: awaitingReportTasks.length, resumedAwaitingReportCount, skippedAwaitingReportDueToActiveDescendants, @@ -2627,32 +2659,9 @@ export class TaskService implements AgentTaskIntegration { } const patchGenerationRecoveryMs = Date.now() - patchGenerationRecoveryStartedAt; - // Restart-safety for grouped best-of completion: if child report artifacts already exist - // on disk after a restart, there may be no later child stream-end to finalize the pending - // parent task tool call. Re-run the deferred parent delivery/finalization pass first so - // cleanup rechecks do not stay blocked forever behind a stale input-available partial. - if (cancelled()) return; - const bestOfRecoveryStartedAt = Date.now(); - const bestOfParentWorkspaceIds = new Set(); - for (const task of completedReportTasks) { - const parentWorkspaceId = coerceNonEmptyString(task.parentWorkspaceId); - const taskId = coerceNonEmptyString(task.id); - const bestOf = taskId ? this.getEffectiveTaskGroup(taskId, task) : undefined; - if (!parentWorkspaceId || (bestOf?.total ?? 1) <= 1) { - continue; - } - if (this.aiService.isStreaming(parentWorkspaceId)) { - continue; - } - bestOfParentWorkspaceIds.add(parentWorkspaceId); - } - for (const parentWorkspaceId of bestOfParentWorkspaceIds) { - if (cancelled()) return; - await this.deliverDeferredBestOfReportsForParent(parentWorkspaceId); - } - const bestOfRecoveryMs = Date.now() - bestOfRecoveryStartedAt; - - // Best-effort completed-report ancestor recheck after restart. + // Best-effort completed-report ancestor recheck after restart. Pending best-of parent + // partials were already finalized by recoverInterruptedTasks, so cleanup does not stay + // blocked behind a stale input-available task tool call. const cleanupReportedTasksStartedAt = Date.now(); // Reuse one config snapshot across tasks to screen out the (usual) ineligible majority without // a config parse each; reload it only after a removal changed the tree. The snapshot never @@ -2713,8 +2722,6 @@ export class TaskService implements AgentTaskIntegration { totalMs, completedReportTaskCount: completedReportTasks.length, patchGenerationRecoveryMs, - bestOfParentRecoveryCount: bestOfParentWorkspaceIds.size, - bestOfRecoveryMs, queuedTerminalWorkflowRunAttentionCount, recoveredTerminalWorkspaceTurnNotificationCount, pendingTerminalAttentionOwnerWorkspaceCount, From f8be9321def46ab45e64b7e3ee31f28667f35b5f Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:04:24 +0000 Subject: [PATCH 15/26] Close the remaining post-listen housekeeping races - canCleanupReportedTask rejects a reported task whose execution mirror is starting or running (existing-workspace turn before its stream registers). - Startup patch recovery re-checks live removal/execution state for crash-left pending artifacts too, not only for tasks with no artifact. - Archive disposes the workspace's transient startup-recovery session once archivedAt is durable so it cannot dispatch a hidden stream afterwards. - ServiceContainer.dispose() joins in-flight background housekeeping (bounded by STARTUP_HOUSEKEEPING_JOIN_TIMEOUT_MS) after aborting it, before tearing down the services it uses. --- src/cli/server.ts | 5 +- src/constants/terminationTimeouts.ts | 8 +++ .../services/gitPatchArtifactService.test.ts | 49 +++++++++++++++++++ src/node/services/gitPatchArtifactService.ts | 12 ++--- src/node/services/serviceContainer.test.ts | 9 ++++ src/node/services/serviceContainer.ts | 35 +++++++++++-- src/node/services/taskService.test.ts | 26 +++++++--- src/node/services/taskService.ts | 6 +++ src/node/services/workspaceService.test.ts | 20 ++++++++ src/node/services/workspaceService.ts | 10 ++++ 10 files changed, 162 insertions(+), 18 deletions(-) diff --git a/src/cli/server.ts b/src/cli/server.ts index fc98192cfca..03301b259d3 100644 --- a/src/cli/server.ts +++ b/src/cli/server.ts @@ -167,8 +167,9 @@ async function main(): Promise { }); // Housekeeping is best-effort background work; a failure must not take the listening server - // down. serviceContainer.dispose() cancels it at its next task/step boundary, so shutdown - // during housekeeping never starts periodic services against disposed dependencies. + // down. serviceContainer.dispose() cancels it at its next task/step boundary and waits + // (bounded) for the step in flight, so shutdown during housekeeping never starts periodic + // services against disposed dependencies. void serviceContainer.runStartupHousekeeping().catch((error: unknown) => { log.error("[startup] Background startup housekeeping failed", { error }); }); diff --git a/src/constants/terminationTimeouts.ts b/src/constants/terminationTimeouts.ts index 0f69546a8a8..fff1d5c100f 100644 --- a/src/constants/terminationTimeouts.ts +++ b/src/constants/terminationTimeouts.ts @@ -22,3 +22,11 @@ export const APP_RUNTIME_DISPOSE_TIMEOUT_MS = 2 * 1000; * bound this fits inside the same 5 s quit budgets. */ export const APP_FIBER_SCOPE_CLOSE_TIMEOUT_MS = 2 * 1000; + +/** + * Bounds how long `ServiceContainer.dispose()` waits for in-flight background + * startup housekeeping (server mode) to reach its cancellation checkpoint before + * tearing down the services it uses. Sized to leave the two bounds above room + * inside the same 5 s quit budgets. + */ +export const STARTUP_HOUSEKEEPING_JOIN_TIMEOUT_MS = 500; diff --git a/src/node/services/gitPatchArtifactService.test.ts b/src/node/services/gitPatchArtifactService.test.ts index a1ba2ecb911..0d33a85cf5e 100644 --- a/src/node/services/gitPatchArtifactService.test.ts +++ b/src/node/services/gitPatchArtifactService.test.ts @@ -97,6 +97,7 @@ describe("GitPatchArtifactService coordination", () => { const freshChildId = "child-fresh"; const removedChildId = "child-removed-since-snapshot"; const reactivatedChildId = "child-reactivated-since-snapshot"; + const pendingReactivatedChildId = "child-pending-reactivated-since-snapshot"; const execChild = (id: string) => ({ path: path.join(projectPath, id), id, @@ -114,6 +115,7 @@ describe("GitPatchArtifactService coordination", () => { execChild(freshChildId), execChild(removedChildId), execChild(reactivatedChildId), + execChild(pendingReactivatedChildId), ], }); return cfg; @@ -206,6 +208,53 @@ describe("GitPatchArtifactService coordination", () => { expect(await readSubagentGitPatchArtifact(parentSessionDir, reactivatedChildId)).toBeNull(); expect(completedChildIds).toEqual([]); + // Same for a crash-left pending artifact, the case startup exists to resume: the reactivated + // checkout is not snapshotted and the marker is left for the continuation refresh. + const crashLeftPending = await upsertSubagentGitPatchArtifact({ + workspaceId: parentId, + workspaceSessionDir: parentSessionDir, + childTaskId: pendingReactivatedChildId, + updater: () => ({ + childTaskId: pendingReactivatedChildId, + parentWorkspaceId: parentId, + createdAtMs: 1, + updatedAtMs: 2, + status: "pending", + projectArtifacts: [ + { + projectPath, + projectName: "repo", + storageKey: "repo", + status: "pending", + baseCommitSha: "launch-base", + }, + ], + readyProjectCount: 0, + failedProjectCount: 0, + skippedProjectCount: 0, + totalCommitCount: 0, + }), + }); + await config.editConfig((cfg) => { + const entry = cfg.projects + .get(projectPath) + ?.workspaces.find((ws) => ws.id === pendingReactivatedChildId); + if (entry) { + entry.taskExecutionId = "wst_pending_reactivated"; + entry.taskExecutionStatus = "starting"; + } + return cfg; + }); + await service.maybeStartGeneration(parentId, pendingReactivatedChildId, onComplete, { + config: snapshot, + }); + // Resolves immediately when no job started; a wrongly started job would rewrite the marker. + await service.waitForGeneration(pendingReactivatedChildId); + expect(await readSubagentGitPatchArtifact(parentSessionDir, pendingReactivatedChildId)).toEqual( + crashLeftPending + ); + expect(completedChildIds).toEqual([]); + // A snapshot child without an artifact still takes the normal generation path (pending // marker written, background job started), proving the snapshot feeds the real branch. await service.maybeStartGeneration(parentId, freshChildId, onComplete, { config: snapshot }); diff --git a/src/node/services/gitPatchArtifactService.ts b/src/node/services/gitPatchArtifactService.ts index 9a0ba603845..b18ed58c3fa 100644 --- a/src/node/services/gitPatchArtifactService.ts +++ b/src/node/services/gitPatchArtifactService.ts @@ -413,13 +413,13 @@ export class GitPatchArtifactService { } if (options?.config != null) { - // The caller's snapshot can predate a client removing or reactivating this task. The updater - // below leaves an existing artifact untouched, so only a task with no artifact yet would get - // a fresh pending marker; re-read config for exactly those. A removed task gets no artifact, - // and a reactivated one (live execution handle, so its checkout is changing again) is left - // to the continuation refresh that runs when that execution settles. + // The caller's snapshot can predate a client removing or reactivating this task. Only a + // task with no artifact yet or a crash-left pending one would actually be generated below + // (a completed artifact is left untouched), so re-read config for exactly those. A removed + // task gets no artifact, and a reactivated one (live execution handle, so its checkout is + // changing again) is left to the continuation refresh that runs when that execution settles. const existing = await readSubagentGitPatchArtifact(parentSessionDir, childWorkspaceId); - if (existing == null) { + if (existing == null || existing.status === "pending") { const live = findWorkspaceEntry(this.config.loadConfigOrDefault(), childWorkspaceId); if (live == null || isActiveWorkspaceTurnTaskStatus(live.workspace.taskExecutionStatus)) { return; diff --git a/src/node/services/serviceContainer.test.ts b/src/node/services/serviceContainer.test.ts index 6bec6f6a26d..b0bdbf5a16a 100644 --- a/src/node/services/serviceContainer.test.ts +++ b/src/node/services/serviceContainer.test.ts @@ -294,6 +294,11 @@ describe("ServiceContainer", () => { }); const heartbeatStart = spyOn(services.heartbeatService, "start"); const idleCompactionStart = spyOn(services.idleCompactionService, "start"); + const beginShutdown = spyOn( + (services as unknown as { backgroundProcessManager: { beginShutdown: () => void } }) + .backgroundProcessManager, + "beginShutdown" + ); await services.initializeCore(); const housekeeping = services.runStartupHousekeeping(); @@ -301,10 +306,14 @@ describe("ServiceContainer", () => { // Task housekeeping is mid-flight when the process shuts down. const disposed = services.dispose(); expect(housekeepingSignal?.aborted).toBe(true); + // Teardown of the services housekeeping is using waits for the in-flight step to settle + // (without the join, this first teardown step runs synchronously inside dispose()). + expect(beginShutdown).not.toHaveBeenCalled(); releaseTaskHousekeeping?.(); await housekeeping; await disposed; + expect(beginShutdown).toHaveBeenCalledTimes(1); expect(heartbeatStart).not.toHaveBeenCalled(); expect(idleCompactionStart).not.toHaveBeenCalled(); }); diff --git a/src/node/services/serviceContainer.ts b/src/node/services/serviceContainer.ts index e25aa560287..976d5d6b58b 100644 --- a/src/node/services/serviceContainer.ts +++ b/src/node/services/serviceContainer.ts @@ -2,6 +2,7 @@ import { log } from "@/node/services/log"; import type { Config, ConfigStores, WorkspaceSessionLocator } from "@/node/config"; import type { FileLeaseManager, ProvidersConfigStore, SecretsStore } from "@/node/config"; import { SLOW_STARTUP_WARN_THRESHOLD_MS } from "@/constants/startup"; +import { STARTUP_HOUSEKEEPING_JOIN_TIMEOUT_MS } from "@/constants/terminationTimeouts"; import type { CoreServices } from "@/node/services/coreServices"; import type { TerminalWindowManager } from "@/desktop/terminalWindowManager"; import type { ProjectService } from "@/node/services/projectService"; @@ -58,6 +59,7 @@ import { } from "@/node/services/di/appRuntime"; import { AppLive } from "@/node/services/di/layers/app"; import { shutdownStep } from "@/node/services/shutdownStep"; +import { raceWithAbortAndTimeout } from "@/node/utils/concurrency/withTimeout"; import { AgentBrowserSessionDiscovery, AgentPluginInstall, @@ -219,6 +221,9 @@ export class ServiceContainer { // Aborted by dispose(): background startup housekeeping must stop at its next step boundary // and never start periodic services against services that are being torn down. private readonly startupHousekeepingAbort = new AbortController(); + // Retained so dispose() can wait for the in-flight housekeeping step to settle (bounded) + // before tearing down the services it is using. + private startupHousekeepingSettled: Promise | null = null; /** * The in-flight (or completed) `dispose()` teardown. Every caller shares it, @@ -353,9 +358,19 @@ export class ServiceContainer { * Startup housekeeping that scales with the number of workspaces (chat restart retries and * orphan sweeps, reported-task patch and cleanup passes, terminal-attention sweeps), then the * periodic services. The server runs it after its listener is bound, so every step re-checks - * live state before mutating; dispose() cancels it at the next step boundary. + * live state before mutating; dispose() cancels it at the next step boundary and waits + * (bounded) for the step in flight to settle. */ - async runStartupHousekeeping(): Promise { + runStartupHousekeeping(): Promise { + const housekeeping = this.runStartupHousekeepingSteps(); + this.startupHousekeepingSettled = housekeeping.then( + () => undefined, + () => undefined + ); + return housekeeping; + } + + private async runStartupHousekeepingSteps(): Promise { const signal = this.startupHousekeepingAbort.signal; // Housekeeping is best-effort and may run while the server is already serving requests: a // failing step must not skip the periodic services below (startup-time rule: never let @@ -547,8 +562,22 @@ export class ServiceContainer { const disposeStartedAt = performance.now(); log.debug("[shutdown] ServiceContainer.dispose starting"); // Background startup housekeeping (server mode) must not start periodic services or keep - // issuing work against the services torn down below. + // issuing work against the services torn down below. Its steps only observe the abort at + // their boundaries, so give the one in flight a bounded chance to settle first. this.startupHousekeepingAbort.abort(); + const housekeepingSettled = this.startupHousekeepingSettled; + if (housekeepingSettled != null) { + await shutdownStep("startupHousekeeping.join", async () => { + const joined = await raceWithAbortAndTimeout(housekeepingSettled, { + timeoutMs: STARTUP_HOUSEKEEPING_JOIN_TIMEOUT_MS, + }); + if (joined.kind === "timeout") { + log.warn("[shutdown] startup housekeeping still running; teardown continues", { + timeoutMs: STARTUP_HOUSEKEEPING_JOIN_TIMEOUT_MS, + }); + } + }); + } // Must run before any session teardown: AgentSession.dispose() triggers // backgroundProcessManager.cleanup(), which would otherwise erase the persisted // armed-monitor registry records that drive post-restart "monitor lost" wakes. diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index deacea41c49..6af7a986de5 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -3576,6 +3576,7 @@ describe("TaskService", () => { const projectPath = path.join(rootDir, "repo"); const parentWorkspaceId = "parent-cleanup-live"; const reactivatedTaskId = "child-workflow-reactivated"; + const turnStartingTaskId = "child-workflow-turn-starting"; const staleTaskId = "child-workflow-stale"; const workflowTask = (stepId: string) => ({ parentWorkspaceId, @@ -3593,19 +3594,26 @@ describe("TaskService", () => { [ projectWorkspace(projectPath, "parent", parentWorkspaceId), projectWorkspace(projectPath, "reactivated", reactivatedTaskId, workflowTask("a")), + projectWorkspace(projectPath, "turn-starting", turnStartingTaskId, workflowTask("c")), projectWorkspace(projectPath, "stale", staleTaskId, workflowTask("b")), ], testTaskSettings() ); const snapshot = config.loadConfigOrDefault(); - // A client reactivated one child after the startup snapshot was taken. + // After the startup snapshot, a client reactivated one child, and another got an + // existing-workspace turn whose execution mirror is starting before any stream registers + // (taskStatus stays "reported" on that path). await config.editConfig((cfg) => { - const entry = cfg.projects - .get(projectPath) - ?.workspaces.find((workspace) => workspace.id === reactivatedTaskId); - if (entry) { - entry.taskStatus = "running"; - entry.reportedAt = undefined; + const workspaces = cfg.projects.get(projectPath)?.workspaces ?? []; + const reactivated = workspaces.find((workspace) => workspace.id === reactivatedTaskId); + if (reactivated) { + reactivated.taskStatus = "running"; + reactivated.reportedAt = undefined; + } + const turnStarting = workspaces.find((workspace) => workspace.id === turnStartingTaskId); + if (turnStarting) { + turnStarting.taskExecutionId = "wst_turn_starting"; + turnStarting.taskExecutionStatus = "starting"; } return cfg; }); @@ -3624,8 +3632,12 @@ describe("TaskService", () => { expect(await internals.cleanupReportedLeafTask(reactivatedTaskId, { config: snapshot })).toBe( 0 ); + expect(await internals.cleanupReportedLeafTask(turnStartingTaskId, { config: snapshot })).toBe( + 0 + ); expect(await internals.cleanupReportedLeafTask(staleTaskId, { config: snapshot })).toBe(1); expect(findWorkspaceInConfig(config, reactivatedTaskId)?.taskStatus).toBe("running"); + expect(findWorkspaceInConfig(config, turnStartingTaskId)?.taskExecutionStatus).toBe("starting"); expect(findWorkspaceInConfig(config, staleTaskId)).toBeUndefined(); }); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 083c3c6d243..ed266947153 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -13209,6 +13209,12 @@ export class TaskService implements AgentTaskIntegration { return { ok: false, reason: "task_not_reported" }; } + // A reported task reactivated through an existing-workspace turn keeps taskStatus "reported" + // while its execution mirror is already starting or running, before any stream registers. + if (isActiveWorkspaceTurnTaskStatus(entry.workspace.taskExecutionStatus)) { + return { ok: false, reason: "workspace_turn_active" }; + } + const bestOf = this.getEffectiveTaskGroup(workspaceId, entry.workspace); if (bestOf?.total != null && bestOf.total > 1) { if ( diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 91d90209f4e..371dcf53bbf 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -13605,6 +13605,26 @@ describe("WorkspaceService archive lifecycle hooks", () => { expect(entry?.archivedAt).toBeTruthy(); }); + test("archive() disposes a transient startup-recovery session once archivedAt is durable", async () => { + let editConfigCallsAtDispose = -1; + const dispose = mock(() => { + editConfigCallsAtDispose = editConfigSpy.mock.calls.length; + }); + const access = workspaceService as unknown as { + transientStartupRecoverySessions: Map; + }; + access.transientStartupRecoverySessions.set(workspaceId, { + dispose, + } as unknown as AgentSession); + + const result = await workspaceService.archive(workspaceId); + + expect(result.success).toBe(true); + expect(dispose).toHaveBeenCalledTimes(1); + expect(editConfigCallsAtDispose).toBe(1); + expect(access.transientStartupRecoverySessions.has(workspaceId)).toBe(false); + }); + test("archive() closes workspace terminal sessions on success", async () => { const closeWorkspaceSessions = mock(() => undefined); const terminalService = { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index f821b3f7b50..e52d1b7e69d 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -8643,6 +8643,16 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return config; }); + // Startup housekeeping may still be recovering this chat in a transient session whose + // stream has not started yet, so the stream stop cannot see it. Disposing it once + // archivedAt is durable stops its pending auto-retry or follow-up dispatch; a session the + // scheduling loop creates later sees the archived entry and is never started. + const transientRecoverySession = this.transientStartupRecoverySessions.get(workspaceId); + if (transientRecoverySession) { + this.transientStartupRecoverySessions.delete(workspaceId); + transientRecoverySession.dispose(); + } + if (!needsSnapshotCapture) { try { await this.stopLiveWorkspaceActivityForArchive(workspaceId); From be815d6fe7a2752c57fb59e07d27572d377c59a1 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:19:07 +0000 Subject: [PATCH 16/26] Make best-of partial finalization a compare-and-set Move the deferred best-of finalization back into post-listen housekeeping (its cleanup rechecks scale with reported children and must not gate the listener) and make the write safe wherever it runs: finalization now goes through HistoryService.updatePartialIfMessageIdMatches, which re-reads the partial under the per-workspace lock and writes only while it is still the same message with the task call pending and no stream running. A parent turn started meanwhile, by a client or by a resumed child's task_send_message, commits that partial and writes its own under a new id, so the finalization is dropped instead of overwriting it. The runtime child-report path uses the same primitive. --- src/node/services/historyService.ts | 97 +++++++++++------ src/node/services/serviceContainer.ts | 6 +- src/node/services/taskService.test.ts | 149 ++++++++++++++++++++++---- src/node/services/taskService.ts | 125 ++++++++++++--------- 4 files changed, 267 insertions(+), 110 deletions(-) diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index bdc9a09973f..17b101926f9 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -1868,43 +1868,76 @@ export class HistoryService { * Write a partial message to disk. */ async writePartial(workspaceId: string, message: MuxMessage): Promise> { - return this.fileLocks.withLock(workspaceId, async () => { - try { - // r66: partial flushes ride the cross-process history lock with an - // in-lock removal-tombstone gate — a foreign backend's active stream - // survives the remover's process-local cancellation, and its next - // delta's ensurePrivateDir would otherwise recreate the deleted - // session directory (removal holds this same lock across its - // tombstone+delete critical section). Truncation recovery is skipped: - // partial.json is not part of the archive/chat transaction. - return await this.withHistoryWriteFileLock(workspaceId, async () => { - if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) { - return Err(`workspace ${workspaceId} was removed; refusing partial write`); - } - const workspaceDir = this.getSessionDir(workspaceId); - await ensurePrivateDir(workspaceDir); - const partialPath = this.getPartialPath(workspaceId); - - const partialMessage: MuxMessage = { - ...message, - metadata: { - ...message.metadata, - partial: true, - }, - }; + return this.fileLocks.withLock(workspaceId, () => + this.writePartialUnlocked(workspaceId, message) + ); + } - // Atomic write: writes to temp file then renames, preventing corruption - // if app crashes mid-write (prevents "Unexpected end of JSON input" on read) - await writeFileAtomic(partialPath, JSON.stringify(partialMessage, null, 2)); - return Ok(undefined); - }); - } catch (error) { - const errorMessage = getErrorMessage(error); - return Err(`Failed to write partial: ${errorMessage}`); + /** + * Read-modify-write the partial under the per-workspace lock, only while it still belongs to + * `messageId`. Returns false (writing nothing) when the partial is missing, belongs to another + * message, or `updater` declines. Callers that assembled an update from an earlier read use this + * so a stream started in between (commit of that partial, then a new one under the fresh + * message id) is never resurrected or overwritten. + */ + async updatePartialIfMessageIdMatches( + workspaceId: string, + messageId: string, + updater: (current: MuxMessage) => MuxMessage | null + ): Promise> { + return this.fileLocks.withLock(workspaceId, async () => { + const current = await this.readPartial(workspaceId); + if (current?.id !== messageId) { + return Ok(false); } + const updated = updater(current); + if (updated == null) { + return Ok(false); + } + const writeResult = await this.writePartialUnlocked(workspaceId, updated); + return writeResult.success ? Ok(true) : writeResult; }); } + private async writePartialUnlocked( + workspaceId: string, + message: MuxMessage + ): Promise> { + try { + // r66: partial flushes ride the cross-process history lock with an + // in-lock removal-tombstone gate — a foreign backend's active stream + // survives the remover's process-local cancellation, and its next + // delta's ensurePrivateDir would otherwise recreate the deleted + // session directory (removal holds this same lock across its + // tombstone+delete critical section). Truncation recovery is skipped: + // partial.json is not part of the archive/chat transaction. + return await this.withHistoryWriteFileLock(workspaceId, async () => { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) { + return Err(`workspace ${workspaceId} was removed; refusing partial write`); + } + const workspaceDir = this.getSessionDir(workspaceId); + await ensurePrivateDir(workspaceDir); + const partialPath = this.getPartialPath(workspaceId); + + const partialMessage: MuxMessage = { + ...message, + metadata: { + ...message.metadata, + partial: true, + }, + }; + + // Atomic write: writes to temp file then renames, preventing corruption + // if app crashes mid-write (prevents "Unexpected end of JSON input" on read) + await writeFileAtomic(partialPath, JSON.stringify(partialMessage, null, 2)); + return Ok(undefined); + }); + } catch (error) { + const errorMessage = getErrorMessage(error); + return Err(`Failed to write partial: ${errorMessage}`); + } + } + /** * Delete the partial message file for a workspace. */ diff --git a/src/node/services/serviceContainer.ts b/src/node/services/serviceContainer.ts index 976d5d6b58b..69ade0b9d91 100644 --- a/src/node/services/serviceContainer.ts +++ b/src/node/services/serviceContainer.ts @@ -325,9 +325,9 @@ export class ServiceContainer { /** * Everything request handling depends on, plus agent-task restart recovery. The server entry * point awaits this before binding its listener: task recovery must finish before any client - * can stop, resume, or send to a task or its parent (see TaskService.recoverInterruptedTasks), - * and it is bounded by the number of active tasks and best-of parents rather than by deployment - * size. The per-workspace housekeeping lives in runStartupHousekeeping(). + * can stop, resume, or send to a task (see TaskService.recoverInterruptedTasks), and it is + * bounded by the number of active tasks rather than by deployment size. The per-workspace + * housekeeping lives in runStartupHousekeeping(). */ async initializeCore(): Promise { this.startupStartedAt = Date.now(); diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 6af7a986de5..fdb76f4add4 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -20089,7 +20089,7 @@ describe("TaskService", () => { ).toHaveLength(1); }); - test("pre-listen recovery, not housekeeping, finalizes ready best-of partials before cleanup rechecks", async () => { + test("initialize finalizes ready best-of partials before cleanup rechecks", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); @@ -20182,10 +20182,12 @@ describe("TaskService", () => { nowMs: Date.now(), }); - const readParentTaskToolPart = async () => { - const parentPartial = await partialService.readPartial(parentId); - expect(parentPartial).not.toBeNull(); - return parentPartial?.parts.find( + await taskService.initialize(); + + const updatedParentPartial = await partialService.readPartial(parentId); + expect(updatedParentPartial).not.toBeNull(); + if (updatedParentPartial) { + const toolPart = updatedParentPartial.parts.find( (p) => p && typeof p === "object" && @@ -20198,23 +20200,14 @@ describe("TaskService", () => { output?: unknown; } | undefined; - }; - - // Rewriting a parent's partial races any parent turn a client starts once the listener is - // open, so the post-listen housekeeping phase must leave it alone. - await taskService.runStartupHousekeeping(); - expect((await readParentTaskToolPart())?.state).toBe("input-available"); - - await taskService.recoverInterruptedTasks(); - - const toolPart = await readParentTaskToolPart(); - expect(toolPart?.toolName).toBe("task"); - expect(toolPart?.state).toBe("output-available"); - const outputJson = JSON.stringify(toolPart?.output); - expect(outputJson).toContain(childOneId); - expect(outputJson).toContain(childTwoId); - expect(outputJson).toContain("Report from child one"); - expect(outputJson).toContain("Report from child two"); + expect(toolPart?.toolName).toBe("task"); + expect(toolPart?.state).toBe("output-available"); + const outputJson = JSON.stringify(toolPart?.output); + expect(outputJson).toContain(childOneId); + expect(outputJson).toContain(childTwoId); + expect(outputJson).toContain("Report from child one"); + expect(outputJson).toContain("Report from child two"); + } const remainingTaskIds = Array.from(config.loadConfigOrDefault().projects.values()) .flatMap((project) => project.workspaces) @@ -20225,6 +20218,118 @@ describe("TaskService", () => { expect(remove).not.toHaveBeenCalled(); }); + test("startup best-of finalization never lands on a partial a live parent turn replaced", async () => { + const config = await createTestConfig(rootDir); + + const projectPath = path.join(rootDir, "repo"); + const parentId = "parent-best-of-superseded"; + const childOneId = "child-best-of-superseded-1"; + const childTwoId = "child-best-of-superseded-2"; + const partialTimestamp = Date.now(); + const currentCreatedAt = new Date(partialTimestamp + 60_000).toISOString(); + const bestOf = { groupId: "best-of-superseded", index: 0, total: 2 } as const; + + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "parent", parentId), + projectWorkspace(projectPath, "child-1", childOneId, { + name: "agent_explore_child_1", + parentWorkspaceId: parentId, + agentType: "explore", + taskStatus: "reported", + createdAt: currentCreatedAt, + bestOf, + }), + projectWorkspace(projectPath, "child-2", childTwoId, { + name: "agent_explore_child_2", + parentWorkspaceId: parentId, + agentType: "explore", + taskStatus: "reported", + createdAt: currentCreatedAt, + bestOf: { ...bestOf, index: 1 }, + }), + ], + testTaskSettings() + ); + + const { aiService } = createAIServiceMocks(config); + const { workspaceService } = createWorkspaceServiceMocks(); + const { partialService, taskService } = createTaskServiceHarness(config, { + aiService, + workspaceService, + }); + + const crashPartial = createMuxMessage( + "assistant-parent-best-of-superseded-crash", + "assistant", + "Waiting on best-of subagents…", + { timestamp: partialTimestamp }, + [ + { + type: "dynamic-tool", + toolCallId: "task-best-of-superseded-call", + toolName: "task", + input: { subagent_type: "explore", prompt: "compare options", title: "Best of 2", n: 2 }, + state: "input-available", + }, + ] + ); + expect((await partialService.writePartial(parentId, crashPartial)).success).toBe(true); + const parentSessionDir = path.join(config.sessionsDir, parentId); + for (const [childTaskId, reportMarkdown] of [ + [childOneId, "Report from child one"], + [childTwoId, "Report from child two"], + ] as const) { + await upsertSubagentReportArtifact({ + workspaceId: parentId, + workspaceSessionDir: parentSessionDir, + childTaskId, + parentWorkspaceId: parentId, + ancestorWorkspaceIds: [parentId], + reportMarkdown, + nowMs: Date.now(), + }); + } + + // A parent turn that starts while finalization is still assembling the grouped output + // commits the crash partial and writes its own under a new message id. Finalization reads + // the crash partial once to resolve the group and once more to finalize; the turn lands + // right after that second read. + const liveTurnPartial = createMuxMessage( + "assistant-parent-best-of-superseded-live-turn", + "assistant", + "Working on the follow-up…", + { timestamp: partialTimestamp + 120_000 }, + [] + ); + const originalReadPartial = partialService.readPartial.bind(partialService); + let crashPartialReads = 0; + const readPartialSpy = spyOn(partialService, "readPartial").mockImplementation( + async (workspaceId: string) => { + const current = await originalReadPartial(workspaceId); + if (workspaceId === parentId && current?.id === crashPartial.id) { + crashPartialReads += 1; + if (crashPartialReads === 2) { + await partialService.writePartial(parentId, liveTurnPartial); + } + } + return current; + } + ); + try { + await taskService.runStartupHousekeeping(); + } finally { + readPartialSpy.mockRestore(); + } + + const parentPartial = await partialService.readPartial(parentId); + expect(parentPartial?.id).toBe(liveTurnPartial.id); + expect( + parentPartial?.parts.some((p) => (p as { type?: unknown }).type === "dynamic-tool") + ).toBe(false); + }); test("initialize finalizes ready legacy variants partials", async () => { const parentId = "parent-legacy-variants-initialize"; diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index ed266947153..f9e0a780d00 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -2272,12 +2272,11 @@ export class TaskService implements AgentTaskIntegration { /** * Re-establishes the tasks that were active when the process last exited: execution-handle - * mirrors, stale `starting` tasks, the queue drain, the restart prompts for - * `awaiting_report`/`running` tasks, and pending best-of task tool calls in crashed parents' - * partials. Bounded by the number of active tasks plus one partial read per best-of parent, - * not by deployment size. Must finish before any client can act on tasks or parents: a stop, - * resume, or send racing these transitions would be overwritten or would resurrect a task the - * client just stopped, so the server binds its listener only after this resolves. + * mirrors, stale `starting` tasks, the queue drain, and the restart prompts for + * `awaiting_report`/`running` tasks. Bounded by the number of active tasks, not by deployment + * size. Must finish before any client can act on tasks: a stop, resume, or send racing these + * transitions would be overwritten or would resurrect a task the client just stopped, so the + * server binds its listener only after this resolves. */ async recoverInterruptedTasks(): Promise { const startupStartedAt = Date.now(); @@ -2566,40 +2565,9 @@ export class TaskService implements AgentTaskIntegration { await this.maybeStartQueuedTasks(); } - // Restart-safety for grouped best-of completion: if child report artifacts already exist - // on disk after a restart, there may be no later child stream-end to finalize the pending - // parent task tool call, and reported-task cleanup stays blocked behind the stale - // input-available partial. Finalizing rewrites the parent's partial.json, which a parent - // turn started by a client would race (commit the old partial, then have its own overwritten), - // so this runs before the listener rather than in housekeeping. Cost: one partial read per - // parent of reported best-of children. - const bestOfRecoveryStartedAt = Date.now(); - const bestOfParentWorkspaceIds = new Set(); - for (const task of this.listAgentTaskWorkspaces(config)) { - const parentWorkspaceId = coerceNonEmptyString(task.parentWorkspaceId); - const taskId = coerceNonEmptyString(task.id); - if (!parentWorkspaceId || !taskId || !hasCompletedAgentReport(task)) { - continue; - } - if ((this.getEffectiveTaskGroup(taskId, task)?.total ?? 1) <= 1) { - continue; - } - bestOfParentWorkspaceIds.add(parentWorkspaceId); - } - for (const parentWorkspaceId of bestOfParentWorkspaceIds) { - // A parent the prompts above resumed is streaming; its own stream-end reruns this pass. - if (this.aiService.isStreaming(parentWorkspaceId)) { - continue; - } - await this.deliverDeferredBestOfReportsForParent(parentWorkspaceId); - } - const bestOfRecoveryMs = Date.now() - bestOfRecoveryStartedAt; - log.info("[startup] TaskService.recoverInterruptedTasks completed", { totalMs: Date.now() - startupStartedAt, maybeStartQueuedTasksMs, - bestOfParentRecoveryCount: bestOfParentWorkspaceIds.size, - bestOfRecoveryMs, awaitingReportTaskCount: awaitingReportTasks.length, resumedAwaitingReportCount, skippedAwaitingReportDueToActiveDescendants, @@ -2659,9 +2627,34 @@ export class TaskService implements AgentTaskIntegration { } const patchGenerationRecoveryMs = Date.now() - patchGenerationRecoveryStartedAt; - // Best-effort completed-report ancestor recheck after restart. Pending best-of parent - // partials were already finalized by recoverInterruptedTasks, so cleanup does not stay - // blocked behind a stale input-available task tool call. + // Restart-safety for grouped best-of completion: if child report artifacts already exist + // on disk after a restart, there may be no later child stream-end to finalize the pending + // parent task tool call. Re-run the deferred parent delivery/finalization pass first so + // cleanup rechecks do not stay blocked forever behind a stale input-available partial. + // A parent turn (client or resumed child) may start meanwhile; the finalization write is a + // compare-and-set on the partial's identity, so it never lands on a live turn's partial. + if (cancelled()) return; + const bestOfRecoveryStartedAt = Date.now(); + const bestOfParentWorkspaceIds = new Set(); + for (const task of completedReportTasks) { + const parentWorkspaceId = coerceNonEmptyString(task.parentWorkspaceId); + const taskId = coerceNonEmptyString(task.id); + const bestOf = taskId ? this.getEffectiveTaskGroup(taskId, task) : undefined; + if (!parentWorkspaceId || (bestOf?.total ?? 1) <= 1) { + continue; + } + if (this.aiService.isStreaming(parentWorkspaceId)) { + continue; + } + bestOfParentWorkspaceIds.add(parentWorkspaceId); + } + for (const parentWorkspaceId of bestOfParentWorkspaceIds) { + if (cancelled()) return; + await this.deliverDeferredBestOfReportsForParent(parentWorkspaceId); + } + const bestOfRecoveryMs = Date.now() - bestOfRecoveryStartedAt; + + // Best-effort completed-report ancestor recheck after restart. const cleanupReportedTasksStartedAt = Date.now(); // Reuse one config snapshot across tasks to screen out the (usual) ineligible majority without // a config parse each; reload it only after a removal changed the tree. The snapshot never @@ -2722,6 +2715,8 @@ export class TaskService implements AgentTaskIntegration { totalMs, completedReportTaskCount: completedReportTasks.length, patchGenerationRecoveryMs, + bestOfParentRecoveryCount: bestOfParentWorkspaceIds.size, + bestOfRecoveryMs, queuedTerminalWorkflowRunAttentionCount, recoveredTerminalWorkspaceTurnNotificationCount, pendingTerminalAttentionOwnerWorkspaceCount, @@ -13146,18 +13141,35 @@ export class TaskService implements AgentTaskIntegration { } } - const updated: MuxMessage = { - ...partial, - parts: partial.parts.map((part) => { - if (!isDynamicToolPart(part)) return part; - if (part.toolCallId !== toolCallId) return part; - if (part.toolName !== "task") return part; - if (part.state === "output-available") return part; - return { ...part, state: "output-available" as const, output: finalizedOutput }; - }), - }; - - const writeResult = await this.historyService.writePartial(workspaceId, updated); + // The grouped output above was assembled from disk reads that a parent turn can overtake: + // its commitPartial moves this partial into history and its stream writes a fresh one under a + // new message id. Apply the finalization only while the partial is still this message with + // this call pending, and never under a stream that has already started. + const writeResult = await this.historyService.updatePartialIfMessageIdMatches( + workspaceId, + partial.id, + (current) => { + if (this.aiService.isStreaming(workspaceId)) return null; + const stillPending = current.parts.some( + (part) => + isDynamicToolPart(part) && + part.toolCallId === toolCallId && + part.toolName === "task" && + part.state === "input-available" + ); + if (!stillPending) return null; + return { + ...current, + parts: current.parts.map((part) => { + if (!isDynamicToolPart(part)) return part; + if (part.toolCallId !== toolCallId) return part; + if (part.toolName !== "task") return part; + if (part.state === "output-available") return part; + return { ...part, state: "output-available" as const, output: finalizedOutput }; + }), + }; + } + ); if (!writeResult.success) { log.error("Failed to write finalized task tool output to partial", { workspaceId, @@ -13165,13 +13177,20 @@ export class TaskService implements AgentTaskIntegration { }); return { kind: "failed" }; } + if (!writeResult.data) { + log.debug("tryFinalizePendingTaskToolCallInPartial: partial superseded before finalization", { + workspaceId, + messageId: partial.id, + }); + return { kind: "failed" }; + } this.workspaceService.emit("chat", { workspaceId, message: { type: "tool-call-end", workspaceId, - messageId: updated.id, + messageId: partial.id, toolCallId, toolName: "task", result: finalizedOutput, From cd178f585cc91ed058aa9415031dfdfabbacae15 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:33:59 +0000 Subject: [PATCH 17/26] Stop startup chat recovery on shutdown; follow the live parent in cleanup - WorkspaceService.initialize takes the housekeeping abort signal and skips scheduling recovery once shutdown began; dispose() sweeps the transient startup-recovery sessions after joining housekeeping, so recovery chains that outlive the housekeeping promise stop at their next disposed check instead of dispatching through services being torn down. - cleanupReportedLeafTask continues from the parent the live confirmation saw rather than the snapshot's, so a re-parent between the screen and the lifecycle lock does not strand the new parent. --- src/node/services/serviceContainer.test.ts | 8 +++ src/node/services/serviceContainer.ts | 7 +- src/node/services/taskService.test.ts | 81 ++++++++++++++++++++++ src/node/services/taskService.ts | 13 ++-- src/node/services/workspaceService.test.ts | 46 ++++++++++++ src/node/services/workspaceService.ts | 20 +++++- 6 files changed, 168 insertions(+), 7 deletions(-) diff --git a/src/node/services/serviceContainer.test.ts b/src/node/services/serviceContainer.test.ts index b0bdbf5a16a..b1820bdcfc9 100644 --- a/src/node/services/serviceContainer.test.ts +++ b/src/node/services/serviceContainer.test.ts @@ -270,6 +270,7 @@ describe("ServiceContainer", () => { await services.runStartupHousekeeping(); expect(callOrder).toEqual(["recoverTasks", "workspace", "taskHousekeeping"]); + expect(workspaceInitializeSpy.mock.calls[0]?.[0]?.signal).toBeInstanceOf(AbortSignal); expect(taskHousekeepingSpy.mock.calls[0]?.[0]?.signal).toBeInstanceOf(AbortSignal); }); @@ -299,6 +300,10 @@ describe("ServiceContainer", () => { .backgroundProcessManager, "beginShutdown" ); + const disposeRecoverySessions = spyOn( + services.workspaceService, + "disposeStartupRecoverySessions" + ); await services.initializeCore(); const housekeeping = services.runStartupHousekeeping(); @@ -309,10 +314,13 @@ describe("ServiceContainer", () => { // Teardown of the services housekeeping is using waits for the in-flight step to settle // (without the join, this first teardown step runs synchronously inside dispose()). expect(beginShutdown).not.toHaveBeenCalled(); + expect(disposeRecoverySessions).not.toHaveBeenCalled(); releaseTaskHousekeeping?.(); await housekeeping; await disposed; + // Recovery sessions housekeeping scheduled are stopped once it has settled, before teardown. + expect(disposeRecoverySessions).toHaveBeenCalledTimes(1); expect(beginShutdown).toHaveBeenCalledTimes(1); expect(heartbeatStart).not.toHaveBeenCalled(); expect(idleCompactionStart).not.toHaveBeenCalled(); diff --git a/src/node/services/serviceContainer.ts b/src/node/services/serviceContainer.ts index 69ade0b9d91..f6d4479681c 100644 --- a/src/node/services/serviceContainer.ts +++ b/src/node/services/serviceContainer.ts @@ -378,7 +378,7 @@ export class ServiceContainer { // Kick off non-task chat restart recovery eagerly; task workspaces recover in TaskService. try { await this.recordStartupStep("workspaceService.initialize", () => - this.workspaceService.initialize() + this.workspaceService.initialize({ signal }) ); } catch (error: unknown) { log.error("[startup] WorkspaceService recovery failed", { error }); @@ -578,6 +578,11 @@ export class ServiceContainer { } }); } + // Chat recovery sessions that housekeeping scheduled run past its own promise; stop them + // before the provider/runtime services they would dispatch through go away. + shutdownStep("workspaceService.disposeStartupRecoverySessions", () => + this.workspaceService.disposeStartupRecoverySessions() + ); // Must run before any session teardown: AgentSession.dispose() triggers // backgroundProcessManager.cleanup(), which would otherwise erase the persisted // armed-monitor registry records that drive post-restart "monitor lost" wakes. diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index fdb76f4add4..2550bdc3262 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -3641,6 +3641,87 @@ describe("TaskService", () => { expect(findWorkspaceInConfig(config, staleTaskId)).toBeUndefined(); }); + test("startup cleanup continues from the parent the live confirmation saw after a re-parent", async () => { + const config = await createTestConfig(rootDir); + const projectPath = path.join(rootDir, "repo"); + const rootWorkspaceId = "parent-cleanup-reparent-root"; + const oldParentId = "child-workflow-reparent-old-parent"; + const newParentId = "child-workflow-reparent-new-parent"; + const leafTaskId = "child-workflow-reparent-leaf"; + const workflowTask = (stepId: string, parentWorkspaceId: string) => ({ + parentWorkspaceId, + agentId: "exec", + agentType: "exec", + taskStatus: "reported" as const, + reportedAt: "2026-08-10T00:00:00.000Z", + taskModelString: "openai:gpt-5.2", + workflowTask: { runId: "wfr_cleanup_reparent", stepId }, + }); + + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "root", rootWorkspaceId), + projectWorkspace( + projectPath, + "old-parent", + oldParentId, + workflowTask("old", rootWorkspaceId) + ), + projectWorkspace( + projectPath, + "new-parent", + newParentId, + workflowTask("new", rootWorkspaceId) + ), + projectWorkspace(projectPath, "leaf", leafTaskId, workflowTask("leaf", oldParentId)), + ], + testTaskSettings() + ); + const snapshot = config.loadConfigOrDefault(); + + // A client re-parents the leaf after the snapshot screen, before remove() takes the lifecycle + // lock; the live confirmation inside remove() sees the new parent. + const remove = mock( + async ( + workspaceId: string, + _force?: boolean, + options?: { beforeRemove?: () => Promise } + ): Promise> => { + if (workspaceId === leafTaskId) { + await config.editConfig((cfg) => { + const leaf = cfg.projects + .get(projectPath) + ?.workspaces.find((workspace) => workspace.id === leafTaskId); + if (leaf) leaf.parentWorkspaceId = newParentId; + return cfg; + }); + } + if (options?.beforeRemove != null && !(await options.beforeRemove())) { + return Ok(undefined); + } + await removeWorkspaceFromTestConfig(config, workspaceId); + return Ok(undefined); + } + ); + const { workspaceService } = createWorkspaceServiceMocks({ remove }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const internals = taskService as unknown as { + cleanupReportedLeafTask: ( + workspaceId: string, + options?: { config?: ProjectsConfig } + ) => Promise; + }; + + // Removing the leaf makes its live parent a structural leaf, so cleanup prunes that one + // next; the former parent had nothing removed under it and stays. + expect(await internals.cleanupReportedLeafTask(leafTaskId, { config: snapshot })).toBe(2); + expect(findWorkspaceInConfig(config, leafTaskId)).toBeUndefined(); + expect(findWorkspaceInConfig(config, newParentId)).toBeUndefined(); + expect(findWorkspaceInConfig(config, oldParentId)).toBeDefined(); + }); + test("bare compaction stream-end resumes the pre-compaction parent identity", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index f9e0a780d00..7f6cf8cadb4 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -13326,11 +13326,12 @@ export class TaskService implements AgentTaskIntegration { // acquisition on this path: runtime callers reach it under the workspace event lock // (stream-end finalization, cleanup rechecks), nesting event -> task-tree, the inverse of // the send path's task-tree -> event. That nesting predates the live recheck. - let confirmed = false; + let confirmed: { ok: true; parentWorkspaceId: string } | undefined; const removeResult = await this.workspaceService.remove(targetWorkspaceId, true, { beforeRemove: async () => { - confirmed = (await this.canCleanupReportedTask(targetWorkspaceId)).ok; - return confirmed; + const live = await this.canCleanupReportedTask(targetWorkspaceId); + confirmed = live.ok ? live : undefined; + return live.ok; }, }); if (!removeResult.success) { @@ -13340,12 +13341,14 @@ export class TaskService implements AgentTaskIntegration { }); return removedCount; } - if (!confirmed) { + if (confirmed == null) { return removedCount; } removedCount += 1; - currentWorkspaceId = screened.parentWorkspaceId; + // The removal just made this parent a candidate, so follow the parent the live check saw + // (a client may have re-parented the task since the screen). + currentWorkspaceId = confirmed.parentWorkspaceId; } log.error("cleanupReportedLeafTask: exceeded max parent traversal depth", { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 371dcf53bbf..aa04ec0be79 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -8320,6 +8320,52 @@ describe("WorkspaceService initialize", () => { expect(removeWorkspaceData).toHaveBeenCalledWith("archived-ws"); }); + test("initialize schedules no recovery once shutdown has aborted it", async () => { + config.getAllWorkspaceMetadata = mock(() => + Promise.resolve([createFrontendWorkspaceMetadata({ id: "live-ws", name: "Live Workspace" })]) + ) as unknown as Config["getAllWorkspaceMetadata"]; + config.loadConfigOrDefault = mock(() => ({ + projects: new Map([ + [ + "/tmp/project", + { workspaces: [{ id: "live-ws", name: "live-ws", path: "/tmp/live-ws" }] }, + ], + ]), + })) as unknown as Config["loadConfigOrDefault"]; + const startupAccess = workspaceService as unknown as { + startStartupRecovery: (workspaceId: string) => void; + }; + const startStartupRecoverySpy = spyOn(startupAccess, "startStartupRecovery").mockImplementation( + () => undefined + ); + const shutdown = new AbortController(); + + await workspaceService.initialize({ signal: shutdown.signal }); + expect(startStartupRecoverySpy).toHaveBeenCalledTimes(1); + + shutdown.abort(); + await workspaceService.initialize({ signal: shutdown.signal }); + expect(startStartupRecoverySpy).toHaveBeenCalledTimes(1); + }); + + test("disposeStartupRecoverySessions stops every transient startup-recovery session", () => { + const dispose = mock(() => undefined); + const startupAccess = workspaceService as unknown as { + transientStartupRecoverySessions: Map; + }; + startupAccess.transientStartupRecoverySessions.set("ws-a", { + dispose, + } as unknown as AgentSession); + startupAccess.transientStartupRecoverySessions.set("ws-b", { + dispose, + } as unknown as AgentSession); + + workspaceService.disposeStartupRecoverySessions(); + + expect(dispose).toHaveBeenCalledTimes(2); + expect(startupAccess.transientStartupRecoverySessions.size).toBe(0); + }); + test("disposes transient startup-recovery sessions that go idle", async () => { const dispose = mock(() => undefined); const fakeSession = { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index e52d1b7e69d..3326160df93 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3164,7 +3164,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { * Best-effort startup recovery for non-task chats so restart auto-retry can resume * interrupted turns before the user explicitly opens each workspace. */ - async initialize(): Promise { + async initialize(options?: { signal?: AbortSignal }): Promise { const startupStartedAt = Date.now(); try { @@ -3182,6 +3182,12 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { let skippedTaskCount = 0; let skippedArchivedCount = 0; + // Shutdown may have started during the cleanups above; the (synchronous) loop below must + // not spawn recovery sessions that disposeStartupRecoverySessions() has already swept. + if (options?.signal?.aborted === true) { + log.info("[startup] WorkspaceService.initialize cancelled before scheduling recovery"); + return; + } // This can run while the server is already serving clients, and the cleanups above took // time: re-read config right before the (synchronous) scheduling loop so a workspace archived // or removed since `allMetadata` was read does not get a hidden recovery stream. @@ -3926,6 +3932,18 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } + /** + * Shutdown: stop startup chat recovery that has not started a stream yet. Their recovery chains + * re-check `disposed` before every dispatch, so nothing they still have in flight reaches the + * provider or the services being torn down. + */ + disposeStartupRecoverySessions(): void { + for (const [workspaceId, session] of this.transientStartupRecoverySessions) { + this.transientStartupRecoverySessions.delete(workspaceId); + session.dispose(); + } + } + /** * Run startup recovery without permanently caching a session for every workspace. * Only promote the temporary session if recovery leaves background activity alive. From c81a8dc656920facb05fc100b7c239558b7127f4 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 09:15:00 +0000 Subject: [PATCH 18/26] Serialize commitPartial with the partial compare-and-set commitPartial snapshotted the partial before its locked history writes, so a finalization written between that snapshot and the commit was appended pre-update and then deleted. Mark the workspace while a commit is in flight (the CAS yields) and take the snapshot under the file lock (a CAS mid-write is included). --- src/node/services/historyService.ts | 38 +++++++-- src/node/services/partialService.test.ts | 104 +++++++++++++++++++++++ 2 files changed, 137 insertions(+), 5 deletions(-) diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index 17b101926f9..e54739a47e0 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -1875,10 +1875,11 @@ export class HistoryService { /** * Read-modify-write the partial under the per-workspace lock, only while it still belongs to - * `messageId`. Returns false (writing nothing) when the partial is missing, belongs to another - * message, or `updater` declines. Callers that assembled an update from an earlier read use this - * so a stream started in between (commit of that partial, then a new one under the fresh - * message id) is never resurrected or overwritten. + * `messageId` and no commitPartial is in flight for the workspace. Returns false (writing + * nothing) when the partial is missing, belongs to another message, is being committed, or + * `updater` declines. Callers that assembled an update from an earlier read use this so a + * stream started in between (commit of that partial, then a new one under the fresh message + * id) is never resurrected or overwritten. */ async updatePartialIfMessageIdMatches( workspaceId: string, @@ -1886,6 +1887,9 @@ export class HistoryService { updater: (current: MuxMessage) => MuxMessage | null ): Promise> { return this.fileLocks.withLock(workspaceId, async () => { + if (this.partialCommitsInFlight.has(workspaceId)) { + return Ok(false); + } const current = await this.readPartial(workspaceId); if (current?.id !== messageId) { return Ok(false); @@ -1986,6 +1990,9 @@ export class HistoryService { }); } + // Workspaces with a commitPartial in flight (process-local, see commitPartial). + private readonly partialCommitsInFlight = new Map(); + /** * Commit any existing partial message to chat history and delete partial.json. * @@ -1994,8 +2001,29 @@ export class HistoryService { * - After committing (or if already finalized), partial.json is deleted. */ async commitPartial(workspaceId: string): Promise> { + // commitPartial snapshots the partial before its locked history writes and deletes it last, + // so an updatePartialIfMessageIdMatches landing in between would be appended pre-update and + // then dropped. Mark the commit (process-local) so that CAS yields for its duration; the + // snapshot itself is read under the lock so a CAS already mid-write is included in it. + this.partialCommitsInFlight.set( + workspaceId, + (this.partialCommitsInFlight.get(workspaceId) ?? 0) + 1 + ); + try { + return await this.commitPartialUnguarded(workspaceId); + } finally { + const remaining = (this.partialCommitsInFlight.get(workspaceId) ?? 1) - 1; + if (remaining > 0) { + this.partialCommitsInFlight.set(workspaceId, remaining); + } else { + this.partialCommitsInFlight.delete(workspaceId); + } + } + } + + private async commitPartialUnguarded(workspaceId: string): Promise> { try { - let partial = await this.readPartial(workspaceId); + let partial = await this.fileLocks.withLock(workspaceId, () => this.readPartial(workspaceId)); if (!partial) { return Ok(undefined); } diff --git a/src/node/services/partialService.test.ts b/src/node/services/partialService.test.ts index bc85948cddd..8cdda408f1d 100644 --- a/src/node/services/partialService.test.ts +++ b/src/node/services/partialService.test.ts @@ -69,6 +69,110 @@ describe("HistoryService partial persistence - Error Recovery", () => { expect(deletePartial).toHaveBeenCalledWith(workspaceId); }); + test("updatePartialIfMessageIdMatches yields to a commitPartial already in flight", async () => { + const workspaceId = "test-workspace"; + const partial: MuxMessage = { + id: "msg-1", + role: "assistant", + metadata: { historySequence: 1, timestamp: Date.now(), model: "test-model" }, + parts: [{ type: "text", text: "Hello" }], + }; + expect((await partialService.writePartial(workspaceId, partial)).success).toBe(true); + + // Park the commit after it has snapshotted the partial and before its history writes. + let releaseHistoryRead: (() => void) | undefined; + const historyReadGate = new Promise((resolve) => { + releaseHistoryRead = resolve; + }); + const readHistory = partialService.getHistoryFromLatestBoundary.bind(partialService); + const historySpy = spyOn(partialService, "getHistoryFromLatestBoundary").mockImplementation( + async (targetWorkspaceId: string) => { + await historyReadGate; + return readHistory(targetWorkspaceId); + } + ); + try { + const commit = partialService.commitPartial(workspaceId); + + const cas = await partialService.updatePartialIfMessageIdMatches( + workspaceId, + "msg-1", + (current) => ({ + ...current, + parts: [...current.parts, { type: "text", text: " (finalized)" }], + }) + ); + expect(cas).toEqual(Ok(false)); + + releaseHistoryRead?.(); + expect((await commit).success).toBe(true); + } finally { + historySpy.mockRestore(); + } + + // The commit landed exactly what it snapshotted, and the CAS did not write anything. + const committed = await partialService.getLastMessages(workspaceId, 1); + expect(committed.success && committed.data[0]?.parts).toEqual(partial.parts); + expect(await partialService.readPartial(workspaceId)).toBeNull(); + }); + + test("commitPartial snapshots the partial after an in-flight updatePartialIfMessageIdMatches lands", async () => { + const workspaceId = "test-workspace"; + const partial: MuxMessage = { + id: "msg-1", + role: "assistant", + metadata: { historySequence: 1, timestamp: Date.now(), model: "test-model" }, + parts: [{ type: "text", text: "Hello" }], + }; + expect((await partialService.writePartial(workspaceId, partial)).success).toBe(true); + const finalizedParts = [...partial.parts, { type: "text" as const, text: " (finalized)" }]; + + // Park the CAS inside its critical section (after it took the lock, before it writes). + let releaseCasRead: (() => void) | undefined; + const casReadGate = new Promise((resolve) => { + releaseCasRead = resolve; + }); + let casReadReached: (() => void) | undefined; + const casReadReachedGate = new Promise((resolve) => { + casReadReached = resolve; + }); + const readPartial = partialService.readPartial.bind(partialService); + let readCalls = 0; + const readSpy = spyOn(partialService, "readPartial").mockImplementation( + async (targetWorkspaceId: string) => { + if (readCalls++ === 0) { + casReadReached?.(); + await casReadGate; + } + return readPartial(targetWorkspaceId); + } + ); + try { + const cas = partialService.updatePartialIfMessageIdMatches( + workspaceId, + "msg-1", + (current) => ({ + ...current, + parts: finalizedParts, + }) + ); + await casReadReachedGate; + + const commit = partialService.commitPartial(workspaceId); + releaseCasRead?.(); + + expect(await cas).toEqual(Ok(true)); + expect((await commit).success).toBe(true); + } finally { + readSpy.mockRestore(); + } + + // The commit waited for the CAS and committed the finalized partial, not its pre-update state. + const committed = await partialService.getLastMessages(workspaceId, 1); + expect(committed.success && committed.data[0]?.parts).toEqual(finalizedParts); + expect(await partialService.readPartial(workspaceId)).toBeNull(); + }); + test("commitPartial should update existing placeholder when errored partial has more parts", async () => { const workspaceId = "test-workspace"; const erroredPartial: MuxMessage = { From 1cb415c4ee7ad334c5ee7d0ae2e7987adb4c9967 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 09:23:02 +0000 Subject: [PATCH 19/26] Keep beginShutdown ahead of the recovery-session sweep Disposing a transient startup-recovery session runs the workspace cleanup that erases persisted armed-monitor records unless the shutdown latch is already set. Restore beginShutdown() as the first teardown step, before the housekeeping join and the session sweep. --- src/node/services/di/appRuntime.ts | 2 ++ src/node/services/serviceContainer.test.ts | 8 +++++--- src/node/services/serviceContainer.ts | 13 +++++++------ 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/node/services/di/appRuntime.ts b/src/node/services/di/appRuntime.ts index 97266b2cd99..515076a9458 100644 --- a/src/node/services/di/appRuntime.ts +++ b/src/node/services/di/appRuntime.ts @@ -97,6 +97,8 @@ * * 1. `backgroundProcessManager.beginShutdown()` — first; the latch that keeps * persisted armed-monitor records from being erased by session teardown. + * Then (server mode) abort and bounded-join the background startup + * housekeeping and dispose the transient chat-recovery sessions it scheduled. * 2. `closeScopeBounded(appFiberScope, APP_FIBER_SCOPE_CLOSE_TIMEOUT_MS)`. * 3. The explicit sequence verbatim (`desktopBridgeServer.stop()` … * `terminateAll()` … `timelineService.flush()` last), each step timed as a diff --git a/src/node/services/serviceContainer.test.ts b/src/node/services/serviceContainer.test.ts index b1820bdcfc9..0c9b144179d 100644 --- a/src/node/services/serviceContainer.test.ts +++ b/src/node/services/serviceContainer.test.ts @@ -311,9 +311,11 @@ describe("ServiceContainer", () => { // Task housekeeping is mid-flight when the process shuts down. const disposed = services.dispose(); expect(housekeepingSignal?.aborted).toBe(true); - // Teardown of the services housekeeping is using waits for the in-flight step to settle - // (without the join, this first teardown step runs synchronously inside dispose()). - expect(beginShutdown).not.toHaveBeenCalled(); + // The armed-monitor latch is set synchronously, ahead of the join, so no session disposed + // during shutdown can erase registry records. Teardown of the services housekeeping is using + // waits for the in-flight step to settle (without the join, the recovery-session sweep would + // also run synchronously inside dispose()). + expect(beginShutdown).toHaveBeenCalledTimes(1); expect(disposeRecoverySessions).not.toHaveBeenCalled(); releaseTaskHousekeeping?.(); await housekeeping; diff --git a/src/node/services/serviceContainer.ts b/src/node/services/serviceContainer.ts index f6d4479681c..f2a01f4b57b 100644 --- a/src/node/services/serviceContainer.ts +++ b/src/node/services/serviceContainer.ts @@ -561,6 +561,13 @@ export class ServiceContainer { private async disposeOnce(): Promise { const disposeStartedAt = performance.now(); log.debug("[shutdown] ServiceContainer.dispose starting"); + // Must run before any session teardown, including the recovery-session sweep below: + // AgentSession.dispose() triggers backgroundProcessManager.cleanup(), which would otherwise + // erase the persisted armed-monitor registry records that drive post-restart "monitor lost" + // wakes. + shutdownStep("backgroundProcessManager.beginShutdown", () => + this.backgroundProcessManager.beginShutdown() + ); // Background startup housekeeping (server mode) must not start periodic services or keep // issuing work against the services torn down below. Its steps only observe the abort at // their boundaries, so give the one in flight a bounded chance to settle first. @@ -583,12 +590,6 @@ export class ServiceContainer { shutdownStep("workspaceService.disposeStartupRecoverySessions", () => this.workspaceService.disposeStartupRecoverySessions() ); - // Must run before any session teardown: AgentSession.dispose() triggers - // backgroundProcessManager.cleanup(), which would otherwise erase the persisted - // armed-monitor registry records that drive post-restart "monitor lost" wakes. - shutdownStep("backgroundProcessManager.beginShutdown", () => - this.backgroundProcessManager.beginShutdown() - ); // Interrupt and await the runtime's supervised fibers while every dependency // they might touch during finalization is still alive. Fixed here (before // the explicit teardown) so later occupants do not re-derive the position; From fdce45f6f00144cd3608e54f9a7d3475b11b0a72 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:26:28 +0000 Subject: [PATCH 20/26] Make commitPartial one cross-process transaction The in-flight mark only serialized the partial compare-and-set with commits from this process; a foreign backend could still snapshot, append, and delete the partial around the CAS write. Run the commit (snapshot, history append/update/delete, partial delete) under the workspace mutex and the cross-process history write lock, and hold the same pair across the CAS read and write. Extract the lock-held bodies of appendToHistory, updateHistory, deleteMessage, and getHistoryFromLatestBoundary so the transaction never re-enters a lock. --- src/node/services/historyService.ts | 563 ++++++++++++----------- src/node/services/partialService.test.ts | 239 ++++++---- 2 files changed, 433 insertions(+), 369 deletions(-) diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index e54739a47e0..66ba349e3c4 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -1637,58 +1637,63 @@ export class HistoryService { * that only needs the active compaction epoch. */ async getHistoryFromLatestBoundary(workspaceId: string, skip = 0): Promise> { - const operation = async (): Promise> => { - // One-time lazy migration: seal any pre-boundary prefix left in chat.jsonl - // by older builds so this read (and every later one) stays O(active epoch). - await this.ensureSealedHistoryRotatedUnlocked(workspaceId); - - const chatPath = this.getChatHistoryPath(workspaceId); - const archivePath = this.getChatArchivePath(workspaceId); - - // Try the requested boundary in chat.jsonl, falling back to less-skipped boundaries. - let chatBoundaryCount = 0; - let chatFallbackOffset: number | null = null; - for (let s = skip; s >= 0; s--) { - const offset = await this.findLastBoundaryByteOffset(chatPath, s); - if (offset !== null) { - if (s === skip) { - return Ok(await this.readHistoryFromOffset(chatPath, offset)); - } - // chat.jsonl has fewer boundaries than requested; remember its oldest - // boundary as a fallback and keep counting into the archive. - chatBoundaryCount = s + 1; - chatFallbackOffset = offset; - break; - } - } + try { + return await this.withRecoveredHistoryLock(workspaceId, () => + this.getHistoryFromLatestBoundaryUnlocked(workspaceId, skip) + ); + } catch (error) { + const message = getErrorMessage(error); + return Err(`Failed to read history from boundary: ${message}`); + } + } - // Boundaries older than chat.jsonl live in the sealed archive. A window that - // starts at an archive boundary spans the archive tail plus all of chat.jsonl. - for (let s = skip - chatBoundaryCount; s >= 0; s--) { - const offset = await this.findLastBoundaryByteOffset(archivePath, s); - if (offset !== null) { - const archived = await this.readHistoryFromOffset(archivePath, offset); - const active = await this.readChatHistory(workspaceId); - return Ok([...archived, ...active]); + private async getHistoryFromLatestBoundaryUnlocked( + workspaceId: string, + skip: number + ): Promise> { + // One-time lazy migration: seal any pre-boundary prefix left in chat.jsonl + // by older builds so this read (and every later one) stays O(active epoch). + await this.ensureSealedHistoryRotatedUnlocked(workspaceId); + + const chatPath = this.getChatHistoryPath(workspaceId); + const archivePath = this.getChatArchivePath(workspaceId); + + // Try the requested boundary in chat.jsonl, falling back to less-skipped boundaries. + let chatBoundaryCount = 0; + let chatFallbackOffset: number | null = null; + for (let s = skip; s >= 0; s--) { + const offset = await this.findLastBoundaryByteOffset(chatPath, s); + if (offset !== null) { + if (s === skip) { + return Ok(await this.readHistoryFromOffset(chatPath, offset)); } + // chat.jsonl has fewer boundaries than requested; remember its oldest + // boundary as a fallback and keep counting into the archive. + chatBoundaryCount = s + 1; + chatFallbackOffset = offset; + break; } + } - if (chatFallbackOffset !== null) { - return Ok(await this.readHistoryFromOffset(chatPath, chatFallbackOffset)); + // Boundaries older than chat.jsonl live in the sealed archive. A window that + // starts at an archive boundary spans the archive tail plus all of chat.jsonl. + for (let s = skip - chatBoundaryCount; s >= 0; s--) { + const offset = await this.findLastBoundaryByteOffset(archivePath, s); + if (offset !== null) { + const archived = await this.readHistoryFromOffset(archivePath, offset); + const active = await this.readChatHistory(workspaceId); + return Ok([...archived, ...active]); } + } - // No boundaries at all — workspace is uncompacted, full read is the only option - const archived = await this.readArchivedHistory(workspaceId); - const active = await this.readChatHistory(workspaceId); - return Ok([...archived, ...active]); - }; - - try { - return await this.withRecoveredHistoryLock(workspaceId, operation); - } catch (error) { - const message = getErrorMessage(error); - return Err(`Failed to read history from boundary: ${message}`); + if (chatFallbackOffset !== null) { + return Ok(await this.readHistoryFromOffset(chatPath, chatFallbackOffset)); } + + // No boundaries at all — workspace is uncompacted, full read is the only option + const archived = await this.readArchivedHistory(workspaceId); + const active = await this.readChatHistory(workspaceId); + return Ok([...archived, ...active]); } // ── Sealed-history rotation ───────────────────────────────────────────── @@ -1874,33 +1879,38 @@ export class HistoryService { } /** - * Read-modify-write the partial under the per-workspace lock, only while it still belongs to - * `messageId` and no commitPartial is in flight for the workspace. Returns false (writing - * nothing) when the partial is missing, belongs to another message, is being committed, or - * `updater` declines. Callers that assembled an update from an earlier read use this so a - * stream started in between (commit of that partial, then a new one under the fresh message - * id) is never resurrected or overwritten. + * Read-modify-write the partial while it still belongs to `messageId`. Returns false (writing + * nothing) when the partial is missing, belongs to another message, or `updater` declines. + * Callers that assembled an update from an earlier read use this so a stream started in + * between (commit of that partial, then a new one under the fresh message id) is never + * resurrected or overwritten. The read and the write happen under the workspace mutex AND the + * cross-process history write lock, the same pair commitPartial holds for its whole + * transaction, so neither this process nor a foreign backend can commit the partial between + * them. */ async updatePartialIfMessageIdMatches( workspaceId: string, messageId: string, updater: (current: MuxMessage) => MuxMessage | null ): Promise> { - return this.fileLocks.withLock(workspaceId, async () => { - if (this.partialCommitsInFlight.has(workspaceId)) { - return Ok(false); - } - const current = await this.readPartial(workspaceId); - if (current?.id !== messageId) { - return Ok(false); - } - const updated = updater(current); - if (updated == null) { - return Ok(false); - } - const writeResult = await this.writePartialUnlocked(workspaceId, updated); - return writeResult.success ? Ok(true) : writeResult; - }); + try { + return await this.fileLocks.withLock(workspaceId, () => + this.withHistoryWriteFileLock(workspaceId, async () => { + const current = await this.readPartial(workspaceId); + if (current?.id !== messageId) { + return Ok(false); + } + const updated = updater(current); + if (updated == null) { + return Ok(false); + } + const writeResult = await this.writePartialUnderWriteLock(workspaceId, updated); + return writeResult.success ? Ok(true) : writeResult; + }) + ); + } catch (error) { + return Err(`Failed to update partial: ${getErrorMessage(error)}`); + } } private async writePartialUnlocked( @@ -1915,50 +1925,59 @@ export class HistoryService { // session directory (removal holds this same lock across its // tombstone+delete critical section). Truncation recovery is skipped: // partial.json is not part of the archive/chat transaction. - return await this.withHistoryWriteFileLock(workspaceId, async () => { - if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) { - return Err(`workspace ${workspaceId} was removed; refusing partial write`); - } - const workspaceDir = this.getSessionDir(workspaceId); - await ensurePrivateDir(workspaceDir); - const partialPath = this.getPartialPath(workspaceId); - - const partialMessage: MuxMessage = { - ...message, - metadata: { - ...message.metadata, - partial: true, - }, - }; - - // Atomic write: writes to temp file then renames, preventing corruption - // if app crashes mid-write (prevents "Unexpected end of JSON input" on read) - await writeFileAtomic(partialPath, JSON.stringify(partialMessage, null, 2)); - return Ok(undefined); - }); + return await this.withHistoryWriteFileLock(workspaceId, () => + this.writePartialUnderWriteLock(workspaceId, message) + ); } catch (error) { const errorMessage = getErrorMessage(error); return Err(`Failed to write partial: ${errorMessage}`); } } + private async writePartialUnderWriteLock( + workspaceId: string, + message: MuxMessage + ): Promise> { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) { + return Err(`workspace ${workspaceId} was removed; refusing partial write`); + } + const workspaceDir = this.getSessionDir(workspaceId); + await ensurePrivateDir(workspaceDir); + const partialPath = this.getPartialPath(workspaceId); + + const partialMessage: MuxMessage = { + ...message, + metadata: { + ...message.metadata, + partial: true, + }, + }; + + // Atomic write: writes to temp file then renames, preventing corruption + // if app crashes mid-write (prevents "Unexpected end of JSON input" on read) + await writeFileAtomic(partialPath, JSON.stringify(partialMessage, null, 2)); + return Ok(undefined); + } + /** * Delete the partial message file for a workspace. */ async deletePartial(workspaceId: string): Promise> { - return this.fileLocks.withLock(workspaceId, async () => { - try { - const partialPath = this.getPartialPath(workspaceId); - await fs.unlink(partialPath); + return this.fileLocks.withLock(workspaceId, () => this.deletePartialUnlocked(workspaceId)); + } + + private async deletePartialUnlocked(workspaceId: string): Promise> { + try { + const partialPath = this.getPartialPath(workspaceId); + await fs.unlink(partialPath); + return Ok(undefined); + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { return Ok(undefined); - } catch (error) { - if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { - return Ok(undefined); - } - const errorMessage = getErrorMessage(error); - return Err(`Failed to delete partial: ${errorMessage}`); } - }); + const errorMessage = getErrorMessage(error); + return Err(`Failed to delete partial: ${errorMessage}`); + } } /** @@ -1990,40 +2009,33 @@ export class HistoryService { }); } - // Workspaces with a commitPartial in flight (process-local, see commitPartial). - private readonly partialCommitsInFlight = new Map(); - /** * Commit any existing partial message to chat history and delete partial.json. * * This is idempotent: * - If the partial has already been finalized in history, it is not committed again. * - After committing (or if already finalized), partial.json is deleted. + * + * Snapshot, history append/update/delete, and partial delete run as ONE transaction under the + * workspace mutex and the cross-process history write lock: a partial write landing between + * the snapshot and the delete (updatePartialIfMessageIdMatches here, or a foreign backend's + * commit of the same partial) would otherwise be appended pre-update and then dropped, or + * resurrect an already committed partial. */ async commitPartial(workspaceId: string): Promise> { - // commitPartial snapshots the partial before its locked history writes and deletes it last, - // so an updatePartialIfMessageIdMatches landing in between would be appended pre-update and - // then dropped. Mark the commit (process-local) so that CAS yields for its duration; the - // snapshot itself is read under the lock so a CAS already mid-write is included in it. - this.partialCommitsInFlight.set( - workspaceId, - (this.partialCommitsInFlight.get(workspaceId) ?? 0) + 1 - ); - try { - return await this.commitPartialUnguarded(workspaceId); - } finally { - const remaining = (this.partialCommitsInFlight.get(workspaceId) ?? 1) - 1; - if (remaining > 0) { - this.partialCommitsInFlight.set(workspaceId, remaining); - } else { - this.partialCommitsInFlight.delete(workspaceId); - } + // Lock-free probe: most stream starts have no partial to commit, and a reader observes either + // the old or the new atomically written file, never a torn one. + if ((await this.readPartial(workspaceId)) == null) { + return Ok(undefined); } + return this.withRecoveredHistoryWriteResultLock(workspaceId, "Failed to commit partial", () => + this.commitPartialUnderWriteLock(workspaceId) + ); } - private async commitPartialUnguarded(workspaceId: string): Promise> { + private async commitPartialUnderWriteLock(workspaceId: string): Promise> { try { - let partial = await this.fileLocks.withLock(workspaceId, () => this.readPartial(workspaceId)); + let partial = await this.readPartial(workspaceId); if (!partial) { return Ok(undefined); } @@ -2041,7 +2053,7 @@ export class HistoryService { return Err("Partial message has no historySequence"); } - const historyResult = await this.getHistoryFromLatestBoundary(workspaceId); + const historyResult = await this.getHistoryFromLatestBoundaryUnlocked(workspaceId, 0); if (!historyResult.success) { return Err(`Failed to read history: ${historyResult.error}`); } @@ -2076,7 +2088,7 @@ export class HistoryService { partialSeq, maxExistingSequence, }); - return this.deletePartial(workspaceId); + return this.deletePartialUnlocked(workspaceId); } const shouldCommit = @@ -2094,18 +2106,18 @@ export class HistoryService { if (shouldCommit) { if (existingMessage) { - const updateResult = await this.updateHistory(workspaceId, partial); + const updateResult = await this.updateHistoryUnderWriteLock(workspaceId, partial); if (!updateResult.success) { return updateResult; } } else { - const appendResult = await this.appendToHistory(workspaceId, partial); + const appendResult = await this.appendToHistoryUnderWriteLock(workspaceId, partial); if (!appendResult.success) { return appendResult; } } } else if (shouldDeleteErroredPlaceholder) { - const deleteMessageResult = await this.deleteMessage(workspaceId, partial.id); + const deleteMessageResult = await this.deleteMessageUnderWriteLock(workspaceId, partial.id); if ( !deleteMessageResult.success && !deleteMessageResult.error.includes("not found in history") @@ -2114,7 +2126,7 @@ export class HistoryService { } } - return this.deletePartial(workspaceId); + return this.deletePartialUnlocked(workspaceId); } catch (error) { const errorMessage = getErrorMessage(error); return Err(`Failed to commit partial: ${errorMessage}`); @@ -2364,22 +2376,25 @@ export class HistoryService { } async appendToHistory(workspaceId: string, message: MuxMessage): Promise> { - return this.withRecoveredHistoryWriteResultLock( - workspaceId, - "Failed to append history", - async () => { - await this.refreshSequenceCounterUnderWriteLock(workspaceId); - const result = await this._appendToHistoryUnlocked(workspaceId, message); - if (result.success) { - // A new durable boundary seals the previous epoch — rotate it out of - // chat.jsonl so subsequent reads/rewrites stay O(active epoch). - await this.rotateAfterBoundaryWriteUnlocked(workspaceId, message); - } - return result; - } + return this.withRecoveredHistoryWriteResultLock(workspaceId, "Failed to append history", () => + this.appendToHistoryUnderWriteLock(workspaceId, message) ); } + private async appendToHistoryUnderWriteLock( + workspaceId: string, + message: MuxMessage + ): Promise> { + await this.refreshSequenceCounterUnderWriteLock(workspaceId); + const result = await this._appendToHistoryUnlocked(workspaceId, message); + if (result.success) { + // A new durable boundary seals the previous epoch — rotate it out of + // chat.jsonl so subsequent reads/rewrites stay O(active epoch). + await this.rotateAfterBoundaryWriteUnlocked(workspaceId, message); + } + return result; + } + /** * Append several messages as ONE durable write (a single JSONL append). * Family-message delivery persists its payload row(s) and the trigger's @@ -2497,80 +2512,83 @@ export class HistoryService { * never in the sealed archive. */ async updateHistory(workspaceId: string, message: MuxMessage): Promise> { - return this.withRecoveredHistoryWriteResultLock( - workspaceId, - "Failed to update history", - async () => { - try { - const historyPath = this.getChatHistoryPath(workspaceId); + return this.withRecoveredHistoryWriteResultLock(workspaceId, "Failed to update history", () => + this.updateHistoryUnderWriteLock(workspaceId, message) + ); + } - // Read the active epoch — structural rewrite requires full file content - const messages = await this.readChatHistory(workspaceId); - const targetSequence = message.metadata?.historySequence; + private async updateHistoryUnderWriteLock( + workspaceId: string, + message: MuxMessage + ): Promise> { + try { + const historyPath = this.getChatHistoryPath(workspaceId); - if (targetSequence === undefined) { - return Err("Cannot update message without historySequence"); - } + // Read the active epoch — structural rewrite requires full file content + const messages = await this.readChatHistory(workspaceId); + const targetSequence = message.metadata?.historySequence; - assert( - isNonNegativeInteger(targetSequence), - "updateHistory requires historySequence to be a non-negative integer" - ); + if (targetSequence === undefined) { + return Err("Cannot update message without historySequence"); + } - // Find and replace the message with matching historySequence - let found = false; - let persistedMessage: MuxMessage | undefined; - for (let i = 0; i < messages.length; i++) { - if (messages[i].metadata?.historySequence === targetSequence) { - const existingMessage = messages[i]; - assert(existingMessage, "updateHistory matched message must exist"); - - // Preserve compaction boundary metadata during late in-place rewrites. - // Compaction may update an assistant row first, then a late stream rewrite can - // update that same historySequence and accidentally drop compaction markers. - const preservedCompactionMetadata = getCompactionMetadataToPreserve( - workspaceId, - existingMessage, - message - ); + assert( + isNonNegativeInteger(targetSequence), + "updateHistory requires historySequence to be a non-negative integer" + ); - // Preserve the historySequence, update everything else. - messages[i] = { - ...message, - metadata: { - ...message.metadata, - ...(preservedCompactionMetadata ?? {}), - historySequence: targetSequence, - }, - }; - persistedMessage = messages[i]; - found = true; - break; - } - } + // Find and replace the message with matching historySequence + let found = false; + let persistedMessage: MuxMessage | undefined; + for (let i = 0; i < messages.length; i++) { + if (messages[i].metadata?.historySequence === targetSequence) { + const existingMessage = messages[i]; + assert(existingMessage, "updateHistory matched message must exist"); + + // Preserve compaction boundary metadata during late in-place rewrites. + // Compaction may update an assistant row first, then a late stream rewrite can + // update that same historySequence and accidentally drop compaction markers. + const preservedCompactionMetadata = getCompactionMetadataToPreserve( + workspaceId, + existingMessage, + message + ); - if (!found || !persistedMessage) { - return Err(`No message found with historySequence ${targetSequence}`); - } + // Preserve the historySequence, update everything else. + messages[i] = { + ...message, + metadata: { + ...message.metadata, + ...(preservedCompactionMetadata ?? {}), + historySequence: targetSequence, + }, + }; + persistedMessage = messages[i]; + found = true; + break; + } + } - // Rewrite entire file - const historyEntries = this.serializeHistoryEntries(messages, workspaceId); + if (!found || !persistedMessage) { + return Err(`No message found with historySequence ${targetSequence}`); + } - // Atomic write prevents corruption if app crashes mid-write - await writeFileAtomic(historyPath, historyEntries); + // Rewrite entire file + const historyEntries = this.serializeHistoryEntries(messages, workspaceId); - // Compaction updates the streamed summary row in-place with boundary - // metadata — seal the previous epoch once that lands. Check the persisted - // row (not the incoming message) so preserved boundary metadata counts. - await this.rotateAfterBoundaryWriteUnlocked(workspaceId, persistedMessage); + // Atomic write prevents corruption if app crashes mid-write + await writeFileAtomic(historyPath, historyEntries); - return Ok(undefined); - } catch (error) { - const message = getErrorMessage(error); - return Err(`Failed to update history: ${message}`); - } - } - ); + // Compaction updates the streamed summary row in-place with boundary + // metadata — seal the previous epoch once that lands. Check the persisted + // row (not the incoming message) so preserved boundary metadata counts. + await this.rotateAfterBoundaryWriteUnlocked(workspaceId, persistedMessage); + + return Ok(undefined); + } catch (error) { + const message = getErrorMessage(error); + return Err(`Failed to update history: ${message}`); + } } /** @@ -2759,83 +2777,86 @@ export class HistoryService { * messages may already have been appended. */ async deleteMessage(workspaceId: string, messageId: string): Promise> { - return this.withRecoveredHistoryWriteResultLock( - workspaceId, - "Failed to delete message", - async () => { - try { - // Structural rewrite requires full file content - const messages = await this.readChatHistory(workspaceId); - const filteredMessages = messages.filter((msg) => msg.id !== messageId); - - if (filteredMessages.length === messages.length) { - // Not in the active epoch — the row may live in the sealed archive - // (rare: cleanup paths almost always target recent rows). - const archiveMessages = await this.readArchivedHistory(workspaceId); - const filteredArchive = archiveMessages.filter((msg) => msg.id !== messageId); - if (filteredArchive.length === archiveMessages.length) { - return Err(`Message with ID ${messageId} not found in history`); - } + return this.withRecoveredHistoryWriteResultLock(workspaceId, "Failed to delete message", () => + this.deleteMessageUnderWriteLock(workspaceId, messageId) + ); + } - // Archived rows are strictly older than active rows, so deleting one - // can never affect the sequence counter. - await writeFileAtomic( - this.getChatArchivePath(workspaceId), - this.serializeHistoryEntries(filteredArchive, workspaceId) - ); - return Ok(undefined); - } + private async deleteMessageUnderWriteLock( + workspaceId: string, + messageId: string + ): Promise> { + try { + // Structural rewrite requires full file content + const messages = await this.readChatHistory(workspaceId); + const filteredMessages = messages.filter((msg) => msg.id !== messageId); + + if (filteredMessages.length === messages.length) { + // Not in the active epoch — the row may live in the sealed archive + // (rare: cleanup paths almost always target recent rows). + const archiveMessages = await this.readArchivedHistory(workspaceId); + const filteredArchive = archiveMessages.filter((msg) => msg.id !== messageId); + if (filteredArchive.length === archiveMessages.length) { + return Err(`Message with ID ${messageId} not found in history`); + } - const historyPath = this.getChatHistoryPath(workspaceId); - const historyEntries = this.serializeHistoryEntries(filteredMessages, workspaceId); + // Archived rows are strictly older than active rows, so deleting one + // can never affect the sequence counter. + await writeFileAtomic( + this.getChatArchivePath(workspaceId), + this.serializeHistoryEntries(filteredArchive, workspaceId) + ); + return Ok(undefined); + } - // Atomic write prevents corruption if app crashes mid-write - await writeFileAtomic(historyPath, historyEntries); + const historyPath = this.getChatHistoryPath(workspaceId); + const historyEntries = this.serializeHistoryEntries(filteredMessages, workspaceId); - // Keep the in-memory sequence counter monotonic. It's okay to reuse deleted sequence - // numbers on restart, but we must not regress within a running process. - const maxSeq = filteredMessages.reduce((max, msg) => { - const seq = msg.metadata?.historySequence; - if (seq === undefined) { - return max; - } + // Atomic write prevents corruption if app crashes mid-write + await writeFileAtomic(historyPath, historyEntries); - if (!isNonNegativeInteger(seq)) { - log.warn( - "Ignoring malformed persisted historySequence while updating sequence counter after delete", - { - workspaceId, - messageId: msg.id, - historySequence: seq, - } - ); - return max; - } + // Keep the in-memory sequence counter monotonic. It's okay to reuse deleted sequence + // numbers on restart, but we must not regress within a running process. + const maxSeq = filteredMessages.reduce((max, msg) => { + const seq = msg.metadata?.historySequence; + if (seq === undefined) { + return max; + } - return seq > max ? seq : max; - }, -1); - // Sealed archive rows keep their sequences across active-file deletes. - // Without this floor, deleting the last sequenced active row in a fresh - // process would cache a counter below archived rows and reuse their - // historySequence values on the next append. - const archiveMaxSeq = await this.getArchiveTailMaxSequence(workspaceId); - const nextSeq = Math.max(maxSeq, archiveMaxSeq) + 1; - assert( - isNonNegativeInteger(nextSeq), - "next history sequence counter after delete must be a non-negative integer" + if (!isNonNegativeInteger(seq)) { + log.warn( + "Ignoring malformed persisted historySequence while updating sequence counter after delete", + { + workspaceId, + messageId: msg.id, + historySequence: seq, + } ); - const currentCounter = this.sequenceCounters.get(workspaceId); - if (currentCounter === undefined || currentCounter < nextSeq) { - this.sequenceCounters.set(workspaceId, nextSeq); - } - - return Ok(undefined); - } catch (error) { - const message = getErrorMessage(error); - return Err(`Failed to delete message: ${message}`); + return max; } + + return seq > max ? seq : max; + }, -1); + // Sealed archive rows keep their sequences across active-file deletes. + // Without this floor, deleting the last sequenced active row in a fresh + // process would cache a counter below archived rows and reuse their + // historySequence values on the next append. + const archiveMaxSeq = await this.getArchiveTailMaxSequence(workspaceId); + const nextSeq = Math.max(maxSeq, archiveMaxSeq) + 1; + assert( + isNonNegativeInteger(nextSeq), + "next history sequence counter after delete must be a non-negative integer" + ); + const currentCounter = this.sequenceCounters.get(workspaceId); + if (currentCounter === undefined || currentCounter < nextSeq) { + this.sequenceCounters.set(workspaceId, nextSeq); } - ); + + return Ok(undefined); + } catch (error) { + const message = getErrorMessage(error); + return Err(`Failed to delete message: ${message}`); + } } /** diff --git a/src/node/services/partialService.test.ts b/src/node/services/partialService.test.ts index 8cdda408f1d..7fcf7df0685 100644 --- a/src/node/services/partialService.test.ts +++ b/src/node/services/partialService.test.ts @@ -1,5 +1,4 @@ import * as path from "path"; -/* eslint-disable @typescript-eslint/unbound-method */ import { describe, test, expect, beforeEach, afterEach, mock, spyOn } from "bun:test"; import type { HistoryService } from "./historyService"; import type { Config } from "@/node/config"; @@ -7,6 +6,8 @@ import { createMuxMessage, type MuxMessage } from "@/common/types/message"; import { Ok } from "@/common/types/result"; import { createTestHistoryService } from "./testHistoryService"; import * as fs from "fs/promises"; +import { acquireProcessFileLock } from "@/node/utils/concurrency/fileLock"; +import { historyWriteLockPath } from "@/node/services/workspaceRemoval"; describe("HistoryService partial persistence - Error Recovery", () => { let partialService: HistoryService; @@ -39,37 +40,23 @@ describe("HistoryService partial persistence - Error Recovery", () => { ], }; - // Mock readPartial to return errored partial - partialService.readPartial = mock(() => Promise.resolve(erroredPartial)); + expect((await partialService.writePartial(workspaceId, erroredPartial)).success).toBe(true); - // Mock deletePartial - partialService.deletePartial = mock(() => Promise.resolve(Ok(undefined))); - - // Spy on partialService methods to verify calls - const appendSpy = spyOn(partialService, "appendToHistory"); - - // Call commitPartial const result = await partialService.commitPartial(workspaceId); - - // Should succeed expect(result.success).toBe(true); - // Should have called appendToHistory with cleaned metadata (no error/errorType) - expect(appendSpy).toHaveBeenCalledTimes(1); - const appendedMessage = appendSpy.mock.calls[0][1]; - - expect(appendedMessage.id).toBe("msg-1"); - expect(appendedMessage.parts).toEqual(erroredPartial.parts); - expect(appendedMessage.metadata?.error).toBeUndefined(); - expect(appendedMessage.metadata?.errorType).toBeUndefined(); - expect(appendedMessage.metadata?.historySequence).toBe(1); - - // Should have deleted the partial after committing - const deletePartial = partialService.deletePartial as ReturnType; - expect(deletePartial).toHaveBeenCalledWith(workspaceId); + // Committed with cleaned metadata (no error/errorType), then the partial is deleted. + const committed = await partialService.getHistoryFromLatestBoundary(workspaceId); + expect(committed.success && committed.data.map((m) => m.id)).toEqual(["msg-1"]); + const appendedMessage = committed.success ? committed.data[0] : undefined; + expect(appendedMessage?.parts).toEqual(erroredPartial.parts); + expect(appendedMessage?.metadata?.error).toBeUndefined(); + expect(appendedMessage?.metadata?.errorType).toBeUndefined(); + expect(appendedMessage?.metadata?.historySequence).toBe(1); + expect(await partialService.readPartial(workspaceId)).toBeNull(); }); - test("updatePartialIfMessageIdMatches yields to a commitPartial already in flight", async () => { + test("updatePartialIfMessageIdMatches waits out a commitPartial transaction and declines", async () => { const workspaceId = "test-workspace"; const partial: MuxMessage = { id: "msg-1", @@ -79,22 +66,32 @@ describe("HistoryService partial persistence - Error Recovery", () => { }; expect((await partialService.writePartial(workspaceId, partial)).success).toBe(true); - // Park the commit after it has snapshotted the partial and before its history writes. - let releaseHistoryRead: (() => void) | undefined; - const historyReadGate = new Promise((resolve) => { - releaseHistoryRead = resolve; + // Park the commit inside its transaction: call 1 is its lock-free probe, call 2 the snapshot + // it takes once it holds both locks. + let releaseCommitRead: (() => void) | undefined; + const commitReadGate = new Promise((resolve) => { + releaseCommitRead = resolve; }); - const readHistory = partialService.getHistoryFromLatestBoundary.bind(partialService); - const historySpy = spyOn(partialService, "getHistoryFromLatestBoundary").mockImplementation( + let commitReadReached: (() => void) | undefined; + const commitReadReachedGate = new Promise((resolve) => { + commitReadReached = resolve; + }); + const readPartial = partialService.readPartial.bind(partialService); + let readCalls = 0; + const readSpy = spyOn(partialService, "readPartial").mockImplementation( async (targetWorkspaceId: string) => { - await historyReadGate; - return readHistory(targetWorkspaceId); + if (++readCalls === 2) { + commitReadReached?.(); + await commitReadGate; + } + return readPartial(targetWorkspaceId); } ); try { const commit = partialService.commitPartial(workspaceId); + await commitReadReachedGate; - const cas = await partialService.updatePartialIfMessageIdMatches( + const cas = partialService.updatePartialIfMessageIdMatches( workspaceId, "msg-1", (current) => ({ @@ -102,12 +99,19 @@ describe("HistoryService partial persistence - Error Recovery", () => { parts: [...current.parts, { type: "text", text: " (finalized)" }], }) ); - expect(cas).toEqual(Ok(false)); - - releaseHistoryRead?.(); + const sentinel = Symbol("still-pending"); + expect( + await Promise.race([ + cas, + new Promise((resolve) => setTimeout(() => resolve(sentinel), 100)), + ]) + ).toBe(sentinel); + + releaseCommitRead?.(); expect((await commit).success).toBe(true); + expect(await cas).toEqual(Ok(false)); } finally { - historySpy.mockRestore(); + readSpy.mockRestore(); } // The commit landed exactly what it snapshotted, and the CAS did not write anything. @@ -127,7 +131,7 @@ describe("HistoryService partial persistence - Error Recovery", () => { expect((await partialService.writePartial(workspaceId, partial)).success).toBe(true); const finalizedParts = [...partial.parts, { type: "text" as const, text: " (finalized)" }]; - // Park the CAS inside its critical section (after it took the lock, before it writes). + // Park the CAS inside its critical section (after it took both locks, before it writes). let releaseCasRead: (() => void) | undefined; const casReadGate = new Promise((resolve) => { releaseCasRead = resolve; @@ -210,38 +214,21 @@ describe("HistoryService partial persistence - Error Recovery", () => { parts: [], // Empty placeholder }; - // Mock readPartial to return errored partial - partialService.readPartial = mock(() => Promise.resolve(erroredPartial)); - - // Mock deletePartial - partialService.deletePartial = mock(() => Promise.resolve(Ok(undefined))); - - // Seed existing placeholder into history so getHistoryFromLatestBoundary finds it + // Seed the existing placeholder into history so the commit finds it by historySequence. await partialService.appendToHistory(workspaceId, existingPlaceholder); + expect((await partialService.writePartial(workspaceId, erroredPartial)).success).toBe(true); - // Spy on partialService methods AFTER seeding to verify only commitPartial calls - const appendSpy = spyOn(partialService, "appendToHistory"); - const updateSpy = spyOn(partialService, "updateHistory"); - - // Call commitPartial const result = await partialService.commitPartial(workspaceId); - - // Should succeed expect(result.success).toBe(true); - // Should have called updateHistory (not append) with cleaned metadata - expect(updateSpy).toHaveBeenCalledTimes(1); - expect(appendSpy).not.toHaveBeenCalled(); - - const updatedMessage = updateSpy.mock.calls[0][1]; - - expect(updatedMessage.parts).toEqual(erroredPartial.parts); - expect(updatedMessage.metadata?.error).toBeUndefined(); - expect(updatedMessage.metadata?.errorType).toBeUndefined(); - - // Should have deleted the partial after updating - const deletePartial = partialService.deletePartial as ReturnType; - expect(deletePartial).toHaveBeenCalledWith(workspaceId); + // The placeholder row is updated in place (no second row) with cleaned metadata. + const committed = await partialService.getHistoryFromLatestBoundary(workspaceId); + expect(committed.success && committed.data.map((m) => m.id)).toEqual(["msg-1"]); + const updatedMessage = committed.success ? committed.data[0] : undefined; + expect(updatedMessage?.parts).toEqual(erroredPartial.parts); + expect(updatedMessage?.metadata?.error).toBeUndefined(); + expect(updatedMessage?.metadata?.errorType).toBeUndefined(); + expect(await partialService.readPartial(workspaceId)).toBeNull(); }); test("commitPartial should skip tool-only incomplete partials", async () => { @@ -268,21 +255,15 @@ describe("HistoryService partial persistence - Error Recovery", () => { ], }; - partialService.readPartial = mock(() => Promise.resolve(toolOnlyPartial)); - partialService.deletePartial = mock(() => Promise.resolve(Ok(undefined))); - - // Spy on partialService methods to verify calls - const appendSpy = spyOn(partialService, "appendToHistory"); - const updateSpy = spyOn(partialService, "updateHistory"); + expect((await partialService.writePartial(workspaceId, toolOnlyPartial)).success).toBe(true); const result = await partialService.commitPartial(workspaceId); expect(result.success).toBe(true); - expect(appendSpy).not.toHaveBeenCalled(); - expect(updateSpy).not.toHaveBeenCalled(); - - const deletePartial = partialService.deletePartial as ReturnType; - expect(deletePartial).toHaveBeenCalledWith(workspaceId); + // Nothing committed, partial still cleaned up. + const committed = await partialService.getHistoryFromLatestBoundary(workspaceId); + expect(committed.success && committed.data).toEqual([]); + expect(await partialService.readPartial(workspaceId)).toBeNull(); }); test("commitPartial should skip empty errored partial", async () => { const workspaceId = "test-workspace"; @@ -300,27 +281,15 @@ describe("HistoryService partial persistence - Error Recovery", () => { parts: [], // Empty - no content accumulated before error }; - // Mock readPartial to return empty errored partial - partialService.readPartial = mock(() => Promise.resolve(emptyErrorPartial)); - - // Mock deletePartial - partialService.deletePartial = mock(() => Promise.resolve(Ok(undefined))); - - // Spy on partialService methods to verify calls - const appendSpy = spyOn(partialService, "appendToHistory"); + expect((await partialService.writePartial(workspaceId, emptyErrorPartial)).success).toBe(true); - // Call commitPartial const result = await partialService.commitPartial(workspaceId); - - // Should succeed expect(result.success).toBe(true); - // Should NOT call appendToHistory for empty message (no value to preserve) - expect(appendSpy).not.toHaveBeenCalled(); - - // Should still delete the partial (cleanup) - const deletePartial = partialService.deletePartial as ReturnType; - expect(deletePartial).toHaveBeenCalledWith(workspaceId); + // No value to preserve: nothing committed, partial still cleaned up. + const committed = await partialService.getHistoryFromLatestBoundary(workspaceId); + expect(committed.success && committed.data).toEqual([]); + expect(await partialService.readPartial(workspaceId)).toBeNull(); }); test("commitPartial deletes a blank assistant placeholder after an empty errored partial", async () => { @@ -353,11 +322,8 @@ describe("HistoryService partial persistence - Error Recovery", () => { } satisfies MuxMessage) ); - const deleteMessageSpy = spyOn(partialService, "deleteMessage"); - const result = await partialService.commitPartial(workspaceId); expect(result.success).toBe(true); - expect(deleteMessageSpy).toHaveBeenCalledWith(workspaceId, "msg-1"); const historyResult = await partialService.getHistoryFromLatestBoundary(workspaceId); expect(historyResult.success).toBe(true); @@ -415,6 +381,83 @@ describe("HistoryService partial persistence - Error Recovery", () => { }); }); +describe("HistoryService partial persistence - Foreign backend", () => { + let config: Config; + let partialService: HistoryService; + let cleanup: () => Promise; + const workspaceId = "foreign-ws"; + const partial: MuxMessage = { + id: "msg-1", + role: "assistant", + metadata: { historySequence: 1, timestamp: 1, model: "test-model", partial: true }, + parts: [{ type: "text", text: "Hello" }], + }; + const stillPending = Symbol("still-pending"); + + beforeEach(async () => { + ({ config, historyService: partialService, cleanup } = await createTestHistoryService()); + expect((await partialService.writePartial(workspaceId, partial)).success).toBe(true); + }); + + afterEach(async () => { + await cleanup(); + }); + + // Another backend (XUM_ALLOW_MULTIPLE_INSTANCES=1) holds the session-dir write lock. + const holdForeignLock = () => + acquireProcessFileLock({ + lockPath: historyWriteLockPath(config.rootDir, workspaceId), + timeoutMs: 5_000, + label: "test foreign backend", + }); + const overwritePartialOnDisk = (message: MuxMessage) => + fs.writeFile( + path.join(config.sessionsDir, workspaceId, "partial.json"), + JSON.stringify(message) + ); + const raceWithTimeout = (promise: Promise) => + Promise.race([promise, new Promise((resolve) => setTimeout(() => resolve(stillPending), 250))]); + + test("commitPartial snapshots the partial only once it holds the cross-process lock", async () => { + const foreign = await holdForeignLock(); + const commit = partialService.commitPartial(workspaceId); + expect(await raceWithTimeout(commit)).toBe(stillPending); + + // The foreign holder finalizes the partial before releasing the lock. + const finalizedParts = [...partial.parts, { type: "text" as const, text: " (finalized)" }]; + await overwritePartialOnDisk({ ...partial, parts: finalizedParts }); + await foreign[Symbol.asyncDispose](); + + expect((await commit).success).toBe(true); + const committed = await partialService.getLastMessages(workspaceId, 1); + expect(committed.success && committed.data[0]?.parts).toEqual(finalizedParts); + expect(await partialService.readPartial(workspaceId)).toBeNull(); + }); + + test("updatePartialIfMessageIdMatches reads the partial only once it holds the cross-process lock", async () => { + const foreign = await holdForeignLock(); + const cas = partialService.updatePartialIfMessageIdMatches(workspaceId, "msg-1", (current) => ({ + ...current, + parts: [...current.parts, { type: "text", text: " (finalized)" }], + })); + expect(await raceWithTimeout(cas)).toBe(stillPending); + + // The foreign holder committed msg-1 and started a new stream under msg-2. + const foreignPartial: MuxMessage = { + ...partial, + id: "msg-2", + parts: [{ type: "text", text: "Next" }], + }; + await overwritePartialOnDisk(foreignPartial); + await foreign[Symbol.asyncDispose](); + + expect(await cas).toEqual(Ok(false)); + const onDisk = await partialService.readPartial(workspaceId); + expect(onDisk?.id).toBe("msg-2"); + expect(onDisk?.parts).toEqual(foreignPartial.parts); + }); +}); + describe("HistoryService partial persistence - Legacy compatibility", () => { let config: Config; let partialService: HistoryService; From 2169beb7e7762f4f7f4b7b0abb1de034ec18f581 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:26:29 +0000 Subject: [PATCH 21/26] Re-check archived state before startup recovery dispatches Archive only disposes transient startup-recovery sessions; a recovery scheduled on a session a client had already created kept running and could start a hidden stream through the internal send path, which bypasses WorkspaceService.sendMessage's archived guard. Re-read the durable archived state right before the pending follow-up and startup auto-retry dispatches. --- ...gentSession.continueMessageAgentId.test.ts | 42 +++++++++++++++- .../agentSession.startupAutoRetry.test.ts | 49 +++++++++++++++++++ src/node/services/agentSession.ts | 38 ++++++++++++++ 3 files changed, 127 insertions(+), 2 deletions(-) diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index a0e0db679a8..a2786e89208 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -150,7 +150,7 @@ describe("AgentSession continue-message agentId fallback", () => { historyCleanup = undefined; }); - const createSession = async (messages: MuxMessage[] = []) => { + const createSession = async (messages: MuxMessage[] = [], config = createConfig()) => { const { historyService, cleanup } = await createTestHistoryService(); historyCleanup = cleanup; for (const message of messages) { @@ -159,7 +159,7 @@ describe("AgentSession continue-message agentId fallback", () => { const session = new AgentSession({ workspaceId: "ws", - config: createConfig(), + config, historyService, aiService: createAiService(), initStateManager: createInitStateManager(), @@ -289,6 +289,44 @@ describe("AgentSession continue-message agentId fallback", () => { expect(dispatchedOptions?.strictAgentResolution).toBe(true); }); + test("dispatchPendingFollowUp leaves the follow-up pending when the workspace is archived on disk", async () => { + const archivedConfig = { + ...createConfig(), + loadConfigOrDefault: () => ({ + projects: new Map([ + [ + "/tmp", + { + workspaces: [{ id: "ws", path: "/tmp/ws", archivedAt: "2026-01-01T00:00:00.000Z" }], + }, + ], + ]), + }), + } as unknown as Config; + const { historyService, internals } = await createSession( + [ + compactionSummaryMessage("summary-archived", { + text: "resume after compaction", + model: "openai:gpt-4o", + agentId: "exec", + }), + ], + archivedConfig + ); + internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + + const dispatched = await internals.dispatchPendingFollowUp(); + + expect(dispatched).toBe(false); + expect(internals.sendMessage).not.toHaveBeenCalled(); + // Still pending for the next startup after an unarchive. + const lastMessages = await historyService.getLastMessages("ws", 1); + expect(lastMessages.success && lastMessages.data[0]?.metadata?.muxMetadata).toMatchObject({ + type: "compaction-summary", + pendingFollowUp: { text: "resume after compaction" }, + }); + }); + test("dispatchPendingFollowUp skips idle-only follow-ups when queued user input exists", async () => { const { session, historyService, internals } = await createSession([ compactionSummaryMessage("summary-idle-only", idleFollowUp()), diff --git a/src/node/services/agentSession.startupAutoRetry.test.ts b/src/node/services/agentSession.startupAutoRetry.test.ts index 424892c4b6a..c05b0334b14 100644 --- a/src/node/services/agentSession.startupAutoRetry.test.ts +++ b/src/node/services/agentSession.startupAutoRetry.test.ts @@ -141,6 +141,55 @@ describe("AgentSession startup auto-retry recovery", () => { session.dispose(); }); + test("startup auto-retry does not dispatch once the workspace is archived on disk", async () => { + const workspaceId = "startup-retry-archived"; + const { session, config, historyService, events, cleanup } = + await createSessionBundle(workspaceId); + cleanups.push(cleanup); + + const appendResult = await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-1", "user", "Interrupted before the archive", { + timestamp: Date.now(), + }) + ); + expect(appendResult.success).toBe(true); + + // The archive lands while the check is still reading history (a regular session the client + // created is not disposed by archive, so only the durable state can stop the dispatch). + const getLastMessages = historyService.getLastMessages.bind(historyService); + spyOn(historyService, "getLastMessages").mockImplementation(async (id, count) => { + const result = await getLastMessages(id, count); + await config.editConfig((cfg) => { + cfg.projects.set("/tmp/project", { + workspaces: [ + { + id: workspaceId, + path: `/tmp/project/${workspaceId}`, + name: workspaceId, + archivedAt: new Date().toISOString(), + }, + ], + }); + return cfg; + }); + return result; + }); + + const privateSession = session as unknown as { + startupAutoRetryCheckPromise: Promise | null; + startupAutoRetryCheckScheduled: boolean; + }; + session.ensureStartupAutoRetryCheck(); + await privateSession.startupAutoRetryCheckPromise; + + expect(events.some((event) => event.type === "auto-retry-scheduled")).toBe(false); + // Completed rather than deferred: nothing reruns the check for an archived workspace. + expect(privateSession.startupAutoRetryCheckScheduled).toBe(true); + + session.dispose(); + }); + test("visible completed subagent report cards do not schedule startup auto-retry", async () => { const workspaceId = "startup-retry-subagent-report-card"; const { session, historyService, events, cleanup } = await createSessionBundle(workspaceId); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 7c360101acd..c6562468958 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -47,6 +47,8 @@ import { } from "@/common/orpc/schemas"; import { ToolPolicySchema } from "@/common/orpc/schemas/stream"; import { normalizeAgentId, resolvePersistedAgentIdCandidates } from "@/common/utils/agentIds"; +import { isWorkspaceArchived } from "@/common/utils/archive"; +import { findWorkspaceEntry } from "@/node/services/taskUtils"; import { buildStreamErrorEventData, createStreamErrorMessage, @@ -2312,6 +2314,12 @@ export class AgentSession { this.startupAutoRetryDeferredRetryDelayMs = 0; return "deferred"; } + if (this.isWorkspaceArchivedOnDisk()) { + log.debug("Startup auto-retry skipped: workspace is archived", { + workspaceId: this.workspaceId, + }); + return "completed"; + } await this.handleStreamFailureForAutoRetry({ type: "unknown", message: "startup_interrupted_stream", @@ -4554,6 +4562,26 @@ export class AgentSession { return followUp; } + /** + * Startup recovery dispatches through this session's internal send path, which bypasses + * WorkspaceService.sendMessage's archived guard, so an archive that lands while a recovery + * step awaits disk I/O would otherwise start a hidden stream. Re-read the durable state right + * before dispatching: dispose() reaches only transient recovery sessions, not a session a + * client had already created when housekeeping scheduled the recovery on it. + */ + private isWorkspaceArchivedOnDisk(): boolean { + try { + const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), this.workspaceId); + return ( + entry != null && + isWorkspaceArchived(entry.workspace.archivedAt, entry.workspace.unarchivedAt) + ); + } catch { + // Partial Config mocks (see getCompactionResolverInputs); a real config never throws here. + return false; + } + } + /** * Layers for auto-compaction resolution. Defensive reads: tests construct * sessions with partial Config mocks, so missing methods degrade to empty @@ -7377,6 +7405,16 @@ export class AgentSession { options.muxMetadata = metadata; } + // Leave the follow-up pending on the summary: it dispatches on the next startup after an + // unarchive instead of running hidden now. + if (this.isWorkspaceArchivedOnDisk()) { + log.debug("Pending follow-up skipped: workspace is archived", { + workspaceId: this.workspaceId, + summaryMessageId: lastMessage.id, + }); + return false; + } + // The compaction summary is now the source of truth for the next live resume // request. Pre-arm retry state from the reconstructed follow-up so failures // before stream startup do not fall back to the already-completed compact turn. From 3a74713dbb6b604eb70fca4f74b92307eb60ac86 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:19:00 +0000 Subject: [PATCH 22/26] Stop internal dispatch on archive during backoff and on shutdown retryActiveStream re-reads the durable archived state before resuming, so a workspace archived while a retry timer counts down abandons the retry instead of starting a hidden stream. WorkspaceService.beginShutdown (formerly disposeStartupRecoverySessions) still disposes transient recovery sessions and now also calls AgentSession.beginShutdown on every regular session: promoted recovery sessions and client-created sessions that housekeeping scheduled recovery on cancel their retry timer and refuse further internal dispatch without abandoning a live stream's partial. --- .../agentSession.startupAutoRetry.test.ts | 83 ++++++++++++++++++- src/node/services/agentSession.ts | 28 ++++++- src/node/services/serviceContainer.test.ts | 5 +- src/node/services/serviceContainer.ts | 4 +- src/node/services/workspaceService.test.ts | 14 +++- src/node/services/workspaceService.ts | 14 ++-- 6 files changed, 129 insertions(+), 19 deletions(-) diff --git a/src/node/services/agentSession.startupAutoRetry.test.ts b/src/node/services/agentSession.startupAutoRetry.test.ts index c05b0334b14..b636ea8687d 100644 --- a/src/node/services/agentSession.startupAutoRetry.test.ts +++ b/src/node/services/agentSession.startupAutoRetry.test.ts @@ -50,7 +50,10 @@ interface SessionBundle { cleanup: () => Promise; } -async function createSessionBundle(workspaceId: string): Promise { +async function createSessionBundle( + workspaceId: string, + aiServiceOverrides?: Partial +): Promise { const workspaceMetadata: WorkspaceMetadata = { id: workspaceId, name: workspaceId, @@ -69,6 +72,7 @@ async function createSessionBundle(workspaceId: string): Promise workspaceId, aiServiceOverrides: { getWorkspaceMetadata: mock(() => Promise.resolve(Ok(workspaceMetadata))), + ...aiServiceOverrides, }, initStateManagerOverrides: { replayInit: mock(() => Promise.resolve()), @@ -190,6 +194,83 @@ describe("AgentSession startup auto-retry recovery", () => { session.dispose(); }); + test("auto-retry abandons instead of resuming once the workspace is archived on disk", async () => { + const workspaceId = "startup-retry-archived-before-timer"; + const { session, config, events, cleanup } = await createSessionBundle(workspaceId); + cleanups.push(cleanup); + + const privateSession = session as unknown as RetryableSessionForTests; + privateSession.lastAutoRetryResumeRequest = { + options: { model: "anthropic:claude-sonnet-4-5", agentId: "exec" }, + }; + const resumeStreamMock = mock((_options: SendMessageOptions) => + Promise.resolve({ success: true as const, data: { started: true } }) + ); + privateSession.resumeStream = resumeStreamMock; + + // The archive lands during the backoff countdown; only the durable state can stop the timer. + await config.editConfig((cfg) => { + cfg.projects.set("/tmp/project", { + workspaces: [ + { + id: workspaceId, + path: `/tmp/project/${workspaceId}`, + name: workspaceId, + archivedAt: new Date().toISOString(), + }, + ], + }); + return cfg; + }); + + await privateSession.retryActiveStream(); + + expect(resumeStreamMock).not.toHaveBeenCalled(); + const abandonedReasons = events + .filter( + (event): event is Extract => + event.type === "auto-retry-abandoned" + ) + .map((event) => event.reason); + expect(abandonedReasons).toEqual(["workspace_archived"]); + + session.dispose(); + }); + + test("beginShutdown cancels the pending retry and stops re-arming or streaming", async () => { + const workspaceId = "startup-retry-shutdown"; + const streamMessage = mock(() => Promise.resolve(Ok(createStartedTurnHandle("assistant-1")))); + const { session, events, cleanup } = await createSessionBundle(workspaceId, { + streamMessage: streamMessage as unknown as AgentSessionAIService["streamMessage"], + }); + cleanups.push(cleanup); + + const privateSession = session as unknown as { + handleStreamFailureForAutoRetry: (error: { type: string; message?: string }) => Promise; + }; + await privateSession.handleStreamFailureForAutoRetry({ type: "unknown", message: "boom" }); + expect(session.shouldRetainAfterStartupRecovery()).toBe(true); + const scheduledBefore = events.filter((event) => event.type === "auto-retry-scheduled").length; + expect(scheduledBefore).toBe(1); + + session.beginShutdown(); + + expect(session.shouldRetainAfterStartupRecovery()).toBe(false); + await privateSession.handleStreamFailureForAutoRetry({ type: "unknown", message: "boom" }); + expect(events.filter((event) => event.type === "auto-retry-scheduled").length).toBe( + scheduledBefore + ); + // Not disposed: the session still answers, but nothing reaches the provider. + const sendResult = await session.sendMessage("hello", { + model: "anthropic:claude-sonnet-4-5", + agentId: "exec", + }); + expect(sendResult.success).toBe(true); + expect(streamMessage).not.toHaveBeenCalled(); + + session.dispose(); + }); + test("visible completed subagent report cards do not schedule startup auto-retry", async () => { const workspaceId = "startup-retry-subagent-report-card"; const { session, historyService, events, cleanup } = await createSessionBundle(workspaceId); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index c6562468958..b03d3e280e6 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -672,6 +672,7 @@ export class AgentSession { private readonly initListeners: Array<{ event: string; handler: (...args: unknown[]) => void }> = []; private disposed = false; + private shuttingDown = false; private turnPhase: TurnPhase = TurnPhase.IDLE; /** Edit-flow admission reservations currently holding busy-ness (see isBusy, r32). */ private editAdmissionDepth = 0; @@ -971,6 +972,16 @@ export class AgentSession { eventSpine.emit("session.start", { workspaceId: this.workspaceId }); } + /** + * Process shutdown for a session that stays alive through teardown (a live stream's partial must + * survive for the next startup's recovery, so dispose() and its abandonPartial are wrong here). + * Stops the retry timer and makes every internal dispatch boundary below bail like `disposed`. + */ + beginShutdown(): void { + this.shuttingDown = true; + this.retryManager.cancel(); + } + dispose(): void { if (this.disposed) { return; @@ -1269,6 +1280,9 @@ export class AgentSession { typeof error.type === "string" && error.type.length > 0, "handleStreamFailureForAutoRetry requires a non-empty error.type" ); + if (this.shuttingDown) { + return; + } // Load persisted preference before scheduling retries so an on-disk opt-out is // honored even when the first failure happens before startup recovery runs. @@ -1316,6 +1330,12 @@ export class AgentSession { return; } + // Archive only interrupts a live stream; a backoff timer armed before it keeps ticking. + if (this.isWorkspaceArchivedOnDisk()) { + this.emitRetryEvent({ type: "auto-retry-abandoned", reason: "workspace_archived" }); + return; + } + const result = await this.resumeStream(request.options, { agentInitiated: request.agentInitiated === true ? true : undefined, goalKind: request.goalKind, @@ -2419,7 +2439,7 @@ export class AgentSession { } async runStartupRecovery(): Promise { - if (this.disposed) { + if (this.disposed || this.shuttingDown) { return; } @@ -2458,7 +2478,7 @@ export class AgentSession { } let deferredAttempts = 0; - while (!this.disposed) { + while (!this.disposed && !this.shuttingDown) { let outcome: StartupAutoRetryCheckOutcome; try { outcome = await this.scheduleStartupAutoRetryIfNeeded(); @@ -5009,7 +5029,7 @@ export class AgentSession { ): Promise> { const isStartupAbortRequested = (): boolean => abortSignal?.aborted === true; - if (this.disposed || isStartupAbortRequested()) { + if (this.disposed || this.shuttingDown || isStartupAbortRequested()) { return Ok(undefined); } @@ -7130,7 +7150,7 @@ export class AgentSession { * proof of dispatch (no history rewrite needed). */ private async dispatchPendingFollowUp(summaryMessageId?: string): Promise { - if (this.disposed) { + if (this.disposed || this.shuttingDown) { return false; } diff --git a/src/node/services/serviceContainer.test.ts b/src/node/services/serviceContainer.test.ts index 0c9b144179d..2f0d4995bc6 100644 --- a/src/node/services/serviceContainer.test.ts +++ b/src/node/services/serviceContainer.test.ts @@ -300,10 +300,7 @@ describe("ServiceContainer", () => { .backgroundProcessManager, "beginShutdown" ); - const disposeRecoverySessions = spyOn( - services.workspaceService, - "disposeStartupRecoverySessions" - ); + const disposeRecoverySessions = spyOn(services.workspaceService, "beginShutdown"); await services.initializeCore(); const housekeeping = services.runStartupHousekeeping(); diff --git a/src/node/services/serviceContainer.ts b/src/node/services/serviceContainer.ts index f2a01f4b57b..836be57d269 100644 --- a/src/node/services/serviceContainer.ts +++ b/src/node/services/serviceContainer.ts @@ -587,9 +587,7 @@ export class ServiceContainer { } // Chat recovery sessions that housekeeping scheduled run past its own promise; stop them // before the provider/runtime services they would dispatch through go away. - shutdownStep("workspaceService.disposeStartupRecoverySessions", () => - this.workspaceService.disposeStartupRecoverySessions() - ); + shutdownStep("workspaceService.beginShutdown", () => this.workspaceService.beginShutdown()); // Interrupt and await the runtime's supervised fibers while every dependency // they might touch during finalization is still alive. Fixed here (before // the explicit teardown) so later occupants do not re-derive the position; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index aa04ec0be79..113cc847b15 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -8348,10 +8348,12 @@ describe("WorkspaceService initialize", () => { expect(startStartupRecoverySpy).toHaveBeenCalledTimes(1); }); - test("disposeStartupRecoverySessions stops every transient startup-recovery session", () => { + test("beginShutdown disposes transient recovery sessions and halts the rest", () => { const dispose = mock(() => undefined); + const beginShutdown = mock(() => undefined); const startupAccess = workspaceService as unknown as { transientStartupRecoverySessions: Map; + sessions: Map; }; startupAccess.transientStartupRecoverySessions.set("ws-a", { dispose, @@ -8359,11 +8361,19 @@ describe("WorkspaceService initialize", () => { startupAccess.transientStartupRecoverySessions.set("ws-b", { dispose, } as unknown as AgentSession); + // A recovery session promoted with a retry pending, or a client-created session that + // housekeeping scheduled recovery on: it may own a live stream, so it is not disposed. + startupAccess.sessions.set("ws-promoted", { + dispose, + beginShutdown, + } as unknown as AgentSession); - workspaceService.disposeStartupRecoverySessions(); + workspaceService.beginShutdown(); expect(dispose).toHaveBeenCalledTimes(2); expect(startupAccess.transientStartupRecoverySessions.size).toBe(0); + expect(beginShutdown).toHaveBeenCalledTimes(1); + startupAccess.sessions.delete("ws-promoted"); }); test("disposes transient startup-recovery sessions that go idle", async () => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 3326160df93..2c76e74741c 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3183,7 +3183,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { let skippedArchivedCount = 0; // Shutdown may have started during the cleanups above; the (synchronous) loop below must - // not spawn recovery sessions that disposeStartupRecoverySessions() has already swept. + // not spawn recovery sessions that beginShutdown() has already swept. if (options?.signal?.aborted === true) { log.info("[startup] WorkspaceService.initialize cancelled before scheduling recovery"); return; @@ -3933,15 +3933,19 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } /** - * Shutdown: stop startup chat recovery that has not started a stream yet. Their recovery chains - * re-check `disposed` before every dispatch, so nothing they still have in flight reaches the - * provider or the services being torn down. + * Shutdown: stop startup chat recovery before the services it dispatches through go away. + * Transient recovery sessions are disposed outright. Sessions that outlived that sweep (promoted + * because recovery left a retry pending, or client-created ones housekeeping scheduled recovery + * on) may own a live stream whose partial the next startup needs, so they only stop dispatching. */ - disposeStartupRecoverySessions(): void { + beginShutdown(): void { for (const [workspaceId, session] of this.transientStartupRecoverySessions) { this.transientStartupRecoverySessions.delete(workspaceId); session.dispose(); } + for (const session of this.sessions.values()) { + session.beginShutdown(); + } } /** From 5cebf008cfe20172928887d07e8a903ac58d2f80 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:32:50 +0000 Subject: [PATCH 23/26] Latch sessions against dispatch before the housekeeping join A startup-recovery chain observes neither the housekeeping abort nor the bounded join, so a recovery that was one await away from dispatching could start a provider stream during the join window. Set the session shutdown latch right after the abort, before waiting on housekeeping to settle. --- src/node/services/di/appRuntime.ts | 6 ++++-- src/node/services/serviceContainer.test.ts | 10 ++++------ src/node/services/serviceContainer.ts | 7 ++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/node/services/di/appRuntime.ts b/src/node/services/di/appRuntime.ts index 515076a9458..b191c676c09 100644 --- a/src/node/services/di/appRuntime.ts +++ b/src/node/services/di/appRuntime.ts @@ -97,8 +97,10 @@ * * 1. `backgroundProcessManager.beginShutdown()` — first; the latch that keeps * persisted armed-monitor records from being erased by session teardown. - * Then (server mode) abort and bounded-join the background startup - * housekeeping and dispose the transient chat-recovery sessions it scheduled. + * Then (server mode) abort the background startup housekeeping, latch every + * chat session against further dispatch (`workspaceService.beginShutdown()`, + * which also disposes the transient chat-recovery sessions housekeeping + * scheduled), and only then bounded-join the housekeeping. * 2. `closeScopeBounded(appFiberScope, APP_FIBER_SCOPE_CLOSE_TIMEOUT_MS)`. * 3. The explicit sequence verbatim (`desktopBridgeServer.stop()` … * `terminateAll()` … `timelineService.flush()` last), each step timed as a diff --git a/src/node/services/serviceContainer.test.ts b/src/node/services/serviceContainer.test.ts index 2f0d4995bc6..ccc5604d0dc 100644 --- a/src/node/services/serviceContainer.test.ts +++ b/src/node/services/serviceContainer.test.ts @@ -308,17 +308,15 @@ describe("ServiceContainer", () => { // Task housekeeping is mid-flight when the process shuts down. const disposed = services.dispose(); expect(housekeepingSignal?.aborted).toBe(true); - // The armed-monitor latch is set synchronously, ahead of the join, so no session disposed - // during shutdown can erase registry records. Teardown of the services housekeeping is using - // waits for the in-flight step to settle (without the join, the recovery-session sweep would - // also run synchronously inside dispose()). + // Both latches are set synchronously, ahead of the join: no session disposed during shutdown + // can erase registry records, and no recovery chain still running under the join can start a + // stream. Only then does teardown wait for the in-flight housekeeping step to settle. expect(beginShutdown).toHaveBeenCalledTimes(1); - expect(disposeRecoverySessions).not.toHaveBeenCalled(); + expect(disposeRecoverySessions).toHaveBeenCalledTimes(1); releaseTaskHousekeeping?.(); await housekeeping; await disposed; - // Recovery sessions housekeeping scheduled are stopped once it has settled, before teardown. expect(disposeRecoverySessions).toHaveBeenCalledTimes(1); expect(beginShutdown).toHaveBeenCalledTimes(1); expect(heartbeatStart).not.toHaveBeenCalled(); diff --git a/src/node/services/serviceContainer.ts b/src/node/services/serviceContainer.ts index 836be57d269..c7518ce9e36 100644 --- a/src/node/services/serviceContainer.ts +++ b/src/node/services/serviceContainer.ts @@ -572,6 +572,10 @@ export class ServiceContainer { // issuing work against the services torn down below. Its steps only observe the abort at // their boundaries, so give the one in flight a bounded chance to settle first. this.startupHousekeepingAbort.abort(); + // Chat recovery that housekeeping scheduled runs past its own promise and observes neither the + // abort nor the join, so latch every session before the wait: nothing may start a stream inside + // it, and nothing may dispatch through the provider/runtime services torn down below. + shutdownStep("workspaceService.beginShutdown", () => this.workspaceService.beginShutdown()); const housekeepingSettled = this.startupHousekeepingSettled; if (housekeepingSettled != null) { await shutdownStep("startupHousekeeping.join", async () => { @@ -585,9 +589,6 @@ export class ServiceContainer { } }); } - // Chat recovery sessions that housekeeping scheduled run past its own promise; stop them - // before the provider/runtime services they would dispatch through go away. - shutdownStep("workspaceService.beginShutdown", () => this.workspaceService.beginShutdown()); // Interrupt and await the runtime's supervised fibers while every dependency // they might touch during finalization is still alive. Fixed here (before // the explicit teardown) so later occupants do not re-derive the position; From 1d4d77ecd98d52f55c03b40b20123964a963f920 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 18:26:53 +0000 Subject: [PATCH 24/26] Honor the shutdown latch at every pre-stream checkpoint streamWithHistory re-checked only the per-turn abortSignal after its awaits, and internal retries carry none, so a retry that was mid-preparation when shutdown began could still reach the provider. Fold disposed/shuttingDown into the checkpoint predicate so every boundary before streamMessage honors it. --- .../agentSession.startupAutoRetry.test.ts | 26 +++++++++++++++++++ src/node/services/agentSession.ts | 18 ++++++++----- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/node/services/agentSession.startupAutoRetry.test.ts b/src/node/services/agentSession.startupAutoRetry.test.ts index b636ea8687d..9d851168a4a 100644 --- a/src/node/services/agentSession.startupAutoRetry.test.ts +++ b/src/node/services/agentSession.startupAutoRetry.test.ts @@ -271,6 +271,32 @@ describe("AgentSession startup auto-retry recovery", () => { session.dispose(); }); + test("beginShutdown during pre-stream awaits stops the stream before the provider", async () => { + const workspaceId = "startup-retry-shutdown-mid-prepare"; + const streamMessage = mock(() => Promise.resolve(Ok(createStartedTurnHandle("assistant-1")))); + const { session, historyService, cleanup } = await createSessionBundle(workspaceId, { + streamMessage: streamMessage as unknown as AgentSessionAIService["streamMessage"], + }); + cleanups.push(cleanup); + + // Shutdown lands while the stream start is already past its entry check, awaiting disk I/O. + const commitPartial = historyService.commitPartial.bind(historyService); + spyOn(historyService, "commitPartial").mockImplementation(async (id) => { + const result = await commitPartial(id); + session.beginShutdown(); + return result; + }); + + const sendResult = await session.sendMessage("hello", { + model: "anthropic:claude-sonnet-4-5", + agentId: "exec", + }); + expect(sendResult.success).toBe(true); + expect(streamMessage).not.toHaveBeenCalled(); + + session.dispose(); + }); + test("visible completed subagent report cards do not schedule startup auto-retry", async () => { const workspaceId = "startup-retry-subagent-report-card"; const { session, historyService, events, cleanup } = await createSessionBundle(workspaceId); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index f75de4fdb45..bec96ad9a6e 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -5023,9 +5023,13 @@ export class AgentSession { // up its replacement's holder. Absent for internal retry paths. activeTurnThinkingOverride?: ActiveTurnThinkingOverride ): Promise> { - const isStartupAbortRequested = (): boolean => abortSignal?.aborted === true; + // Re-read at every pre-stream checkpoint below: dispose or shutdown can land while a + // recovery-initiated stream (which carries no abortSignal) awaits commitPartial, file-change + // detection, or history reads, and must not reach the provider afterwards. + const isStreamStartAborted = (): boolean => + this.disposed || this.shuttingDown || abortSignal?.aborted === true; - if (this.disposed || this.shuttingDown || isStartupAbortRequested()) { + if (isStreamStartAborted()) { return Ok(undefined); } @@ -5054,7 +5058,7 @@ export class AgentSession { ); } - if (isStartupAbortRequested()) { + if (isStreamStartAborted()) { return Ok(undefined); } @@ -5067,7 +5071,7 @@ export class AgentSession { // abort or append failure therefore re-detects the same change (nothing is // dropped), while a successful append cannot produce a duplicate row. const fileChangeDetection = await this.fileChangeTracker.getChangedAttachments(); - if (isStartupAbortRequested()) { + if (isStreamStartAborted()) { return Ok(undefined); } if (fileChangeDetection.attachments.length > 0) { @@ -5084,7 +5088,7 @@ export class AgentSession { } const historyResult = await this.historyService.getHistoryFromLatestBoundary(this.workspaceId); - if (isStartupAbortRequested()) { + if (isStreamStartAborted()) { return Ok(undefined); } @@ -5145,7 +5149,7 @@ export class AgentSession { options ); - if (isStartupAbortRequested()) { + if (isStreamStartAborted()) { return Ok(undefined); } @@ -5154,7 +5158,7 @@ export class AgentSession { disablePostCompactionAttachments === true ? null : await this.getPostCompactionAttachmentsIfNeeded(this.isRlmCompactionEnabled(options)); - if (isStartupAbortRequested()) { + if (isStreamStartAborted()) { return Ok(undefined); } From 433e2bda4f25afd8df3b5cf71d91a20b7c06ae07 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 19:15:29 +0000 Subject: [PATCH 25/26] Defer best-of finalization while a sibling executes again A reported best-of child that a client reawakened keeps its previous report artifact while its continuation runs, and the grouped builder only checked that every artifact exists. Revalidate each non-reporting sibling's execution and stream state before assembling the grouped output, and treat such a sibling as recoverable for the synthetic fallback so the parent waits for the new report. --- src/node/services/taskService.test.ts | 101 ++++++++++++++++++++++++++ src/node/services/taskService.ts | 31 +++++++- 2 files changed, 131 insertions(+), 1 deletion(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 2550bdc3262..29715309dd6 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -20299,6 +20299,107 @@ describe("TaskService", () => { expect(remove).not.toHaveBeenCalled(); }); + test("startup best-of finalization defers while a reawakened sibling is executing again", async () => { + const config = await createTestConfig(rootDir); + + const projectPath = path.join(rootDir, "repo"); + const parentId = "parent-best-of-reawakened-sibling"; + const childOneId = "child-best-of-reawakened-sibling-1"; + const childTwoId = "child-best-of-reawakened-sibling-2"; + const partialTimestamp = Date.now(); + const currentCreatedAt = new Date(partialTimestamp + 60_000).toISOString(); + const bestOf = { + groupId: "best-of-reawakened-sibling", + index: 0, + total: 2, + } as const; + + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "parent", parentId), + { + path: path.join(projectPath, "child-1"), + id: childOneId, + name: "agent_explore_child_1", + parentWorkspaceId: parentId, + agentType: "explore", + taskStatus: "reported", + createdAt: currentCreatedAt, + bestOf, + }, + { + // Reported, then reawakened by a client: its old artifact is about to be replaced. + path: path.join(projectPath, "child-2"), + id: childTwoId, + name: "agent_explore_child_2", + parentWorkspaceId: parentId, + agentType: "explore", + taskStatus: "reported", + taskExecutionStatus: "running", + taskExecutionId: "wst_reawakened_child_2", + createdAt: currentCreatedAt, + bestOf: { ...bestOf, index: 1 }, + }, + ], + testTaskSettings() + ); + + const { aiService } = createAIServiceMocks(config); + const { workspaceService } = createWorkspaceServiceMocks(); + const { partialService, taskService } = createTaskServiceHarness(config, { + aiService, + workspaceService, + }); + + const parentPartial = createMuxMessage( + "assistant-parent-best-of-reawakened-sibling", + "assistant", + "Waiting on best-of subagents…", + { timestamp: partialTimestamp }, + [ + { + type: "dynamic-tool", + toolCallId: "task-best-of-reawakened-sibling-call", + toolName: "task", + input: { + subagent_type: "explore", + prompt: "compare options", + title: "Best of 2", + n: 2, + }, + state: "input-available", + }, + ] + ); + expect((await partialService.writePartial(parentId, parentPartial)).success).toBe(true); + + const parentSessionDir = path.join(config.sessionsDir, parentId); + for (const [childTaskId, reportMarkdown] of [ + [childOneId, "Report from child one"], + [childTwoId, "Report from child two (pre-continuation)"], + ] as const) { + await upsertSubagentReportArtifact({ + workspaceId: parentId, + workspaceSessionDir: parentSessionDir, + childTaskId, + parentWorkspaceId: parentId, + ancestorWorkspaceIds: [parentId], + reportMarkdown, + nowMs: Date.now(), + }); + } + + await taskService.runStartupHousekeeping(); + + const updatedParentPartial = await partialService.readPartial(parentId); + expect(updatedParentPartial).not.toBeNull(); + const toolPart = updatedParentPartial?.parts.find( + (part) => isDynamicToolPart(part) && part.toolName === "task" + ); + expect(toolPart && "state" in toolPart ? toolPart.state : undefined).toBe("input-available"); + }); test("startup best-of finalization never lands on a partial a live parent turn replaced", async () => { const config = await createTestConfig(rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 7f6cf8cadb4..9e4ef453f68 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -12659,6 +12659,7 @@ export class TaskService implements AgentTaskIntegration { agentId?: string; agentType?: string; taskStatus?: WorkspaceConfigEntry["taskStatus"]; + taskExecutionStatus?: WorkspaceConfigEntry["taskExecutionStatus"]; }> { const cfg = this.config.loadConfigOrDefault(); const siblings: Array<{ @@ -12667,6 +12668,7 @@ export class TaskService implements AgentTaskIntegration { agentId?: string; agentType?: string; taskStatus?: WorkspaceConfigEntry["taskStatus"]; + taskExecutionStatus?: WorkspaceConfigEntry["taskExecutionStatus"]; }> = []; for (const project of cfg.projects.values()) { @@ -12691,6 +12693,7 @@ export class TaskService implements AgentTaskIntegration { agentId: coerceNonEmptyString(workspace.agentId), agentType: coerceNonEmptyString(workspace.agentType), taskStatus: workspace.taskStatus, + taskExecutionStatus: workspace.taskExecutionStatus, }); } } @@ -12715,10 +12718,27 @@ export class TaskService implements AgentTaskIntegration { return siblings; } + /** + * A reported sibling that a client reawakened (workspace-turn continuation) or that is streaming + * still owns its previous report artifact. Finalizing the parent on that artifact would freeze the + * grouped result on a report the child is replacing, so its group waits for the new report. + */ + private isBestOfSiblingExecutingAgain(sibling: { + taskId: string; + taskExecutionStatus?: WorkspaceConfigEntry["taskExecutionStatus"]; + }): boolean { + return ( + isActiveWorkspaceTurnTaskStatus(sibling.taskExecutionStatus) || + this.aiService.isStreaming(sibling.taskId) + ); + } + private async buildBestOfCompletedTaskToolOutput(params: { parentWorkspaceId: string; groupId: string; total: number; + /** The sibling whose report is being delivered right now; its execution is live by construction. */ + reportingTaskId?: string; }): Promise | null> { const siblings = this.listBestOfSiblingTasks({ parentWorkspaceId: params.parentWorkspaceId, @@ -12754,6 +12774,13 @@ export class TaskService implements AgentTaskIntegration { }> = []; for (const sibling of siblings) { + if ( + sibling.taskId !== params.reportingTaskId && + this.isBestOfSiblingExecutingAgain(sibling) + ) { + return null; + } + const artifact = await readSubagentReportArtifact(parentSessionDir, sibling.taskId); if (!artifact) { return null; @@ -12856,7 +12883,8 @@ export class TaskService implements AgentTaskIntegration { sibling.taskStatus === "queued" || sibling.taskStatus === "starting" || sibling.taskStatus === "running" || - sibling.taskStatus === "awaiting_report" + sibling.taskStatus === "awaiting_report" || + this.isBestOfSiblingExecutingAgain(sibling) ); }); if (hasRecoverableSibling) { @@ -13132,6 +13160,7 @@ export class TaskService implements AgentTaskIntegration { parentWorkspaceId: workspaceId, groupId: bestOf.groupId, total: bestOf.total, + reportingTaskId: childWorkspaceId, }); if (!groupedOutput) { return { kind: "not_ready" }; From b795316c1ecbcfea7be6d53e9620ff123b1b28db Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 3 Sep 2026 19:39:46 +0000 Subject: [PATCH 26/26] Close the remaining dispatch windows around removal, shutdown, and best-of assembly Removal holds turn admission on the workspace's sessions for its whole duration (released on failure), so startup recovery resuming from a disk read cannot start work against a checkout being deleted. sendMessage refuses at the pre-persist gate once the shutdown latch is set, so no follow-up row can read as a dispatched turn on the next startup. Best-of assembly re-reads sibling state after its artifact reads so a reactivation during those awaits cannot ship the stale report it already collected. --- .../agentSession.startupAutoRetry.test.ts | 9 ++- src/node/services/agentSession.ts | 9 +++ src/node/services/taskService.ts | 16 ++++ src/node/services/workspaceService.test.ts | 81 +++++++++++++++++++ src/node/services/workspaceService.ts | 13 +++ 5 files changed, 125 insertions(+), 3 deletions(-) diff --git a/src/node/services/agentSession.startupAutoRetry.test.ts b/src/node/services/agentSession.startupAutoRetry.test.ts index 9d851168a4a..42ee1ea7b5b 100644 --- a/src/node/services/agentSession.startupAutoRetry.test.ts +++ b/src/node/services/agentSession.startupAutoRetry.test.ts @@ -240,7 +240,7 @@ describe("AgentSession startup auto-retry recovery", () => { test("beginShutdown cancels the pending retry and stops re-arming or streaming", async () => { const workspaceId = "startup-retry-shutdown"; const streamMessage = mock(() => Promise.resolve(Ok(createStartedTurnHandle("assistant-1")))); - const { session, events, cleanup } = await createSessionBundle(workspaceId, { + const { session, historyService, events, cleanup } = await createSessionBundle(workspaceId, { streamMessage: streamMessage as unknown as AgentSessionAIService["streamMessage"], }); cleanups.push(cleanup); @@ -260,13 +260,16 @@ describe("AgentSession startup auto-retry recovery", () => { expect(events.filter((event) => event.type === "auto-retry-scheduled").length).toBe( scheduledBefore ); - // Not disposed: the session still answers, but nothing reaches the provider. + // Not disposed, but sends are refused before any row lands: a follow-up row persisted here + // would read as a dispatched turn on the next startup while its stream never ran. const sendResult = await session.sendMessage("hello", { model: "anthropic:claude-sonnet-4-5", agentId: "exec", }); - expect(sendResult.success).toBe(true); + expect(sendResult.success).toBe(false); expect(streamMessage).not.toHaveBeenCalled(); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success ? history.data : ["unexpected"]).toHaveLength(0); session.dispose(); }); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index bec96ad9a6e..4ea8bac14ab 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -516,6 +516,7 @@ export async function clearProviderConfigFixableAbandonMarkers( */ export const CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE = "Workspace history is being cleared or reset. Please wait and try again."; +const SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE = "Xum is shutting down; the message was not sent."; const STARTUP_AUTO_RETRY_HISTORY_FAILURE_BASE_DELAY_MS = 1_000; const STARTUP_AUTO_RETRY_HISTORY_FAILURE_MAX_DELAY_MS = 30_000; @@ -3485,6 +3486,9 @@ export class AgentSession { if (this.turnAdmissionBlocks > 0 || isAdmissionStale()) { return Err(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); } + if (this.shuttingDown) { + return Err(createUnknownSendMessageError(SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE)); + } // Idle (or preempted to idle) now: hold busy-ness from here until the // turn phase takes over, so concurrent sends queue instead of racing the @@ -3812,6 +3816,11 @@ export class AgentSession { if (this.turnAdmissionBlocks > 0 || isAdmissionStale()) { return Err(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); } + // Still pre-persist: a row appended now would read as a dispatched turn on the next startup + // while streamWithHistory's own latch check keeps its stream from ever running. + if (this.shuttingDown) { + return Err(createUnknownSendMessageError(SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE)); + } // Persist snapshots only when this turn will be sent immediately. // On on-send compaction paths, snapshots are deferred with the follow-up turn. diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 9e4ef453f68..b4d5ab7706d 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -12799,6 +12799,22 @@ export class TaskService implements AgentTaskIntegration { }); } + // The artifact reads above awaited disk I/O, during which a client can reawaken a sibling + // whose old report is already in `reports`; re-read live state once more before handing out. + const liveSiblings = this.listBestOfSiblingTasks({ + parentWorkspaceId: params.parentWorkspaceId, + groupId: params.groupId, + }); + if ( + liveSiblings.length !== siblings.length || + liveSiblings.some( + (sibling) => + sibling.taskId !== params.reportingTaskId && this.isBestOfSiblingExecutingAgain(sibling) + ) + ) { + return null; + } + const output = { status: "completed" as const, taskIds: siblings.map((sibling) => sibling.taskId), diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index a692f804ca5..66bc71107af 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -16862,6 +16862,87 @@ describe("WorkspaceService init cancellation", () => { await fsPromises.rm(tempRoot, { recursive: true, force: true }); } }); + test("remove() holds turn admission on the session until removal settles", async () => { + const workspaceId = "ws-remove-holds-admission"; + const projectPath = "/tmp/proj"; + + let releases = 0; + let releasesWhenRuntimeDeleted = -1; + const deleteWorkspaceMock = mock(() => { + releasesWhenRuntimeDeleted = releases; + return Promise.resolve({ success: false as const, error: "dirty" }); + }); + const createRuntimeSpy = spyOn(runtimeFactory, "createRuntime").mockReturnValue({ + deleteWorkspace: deleteWorkspaceMock, + } as unknown as ReturnType); + + const tempRoot = await fsPromises.mkdtemp(path.join(tmpdir(), "mux-ws-remove-hold-")); + try { + const mockAIService = { + ...createStreamLifecycleMocks(), + isStreaming: mock(() => false), + stopStream: mock(() => Promise.resolve({ success: true as const, data: undefined })), + getWorkspaceMetadata: mock(() => + Promise.resolve( + Ok({ + id: workspaceId, + name: "ws", + projectPath, + projectName: "proj", + runtimeConfig: { type: "local" }, + }) + ) + ), + // eslint-disable-next-line @typescript-eslint/no-empty-function + on: mock(() => {}), + // eslint-disable-next-line @typescript-eslint/no-empty-function + off: mock(() => {}), + } as unknown as AIService; + const mockConfig: MockWorkspaceConfig = { + rootDir: path.join(tempRoot, "root"), + srcDir: "/tmp/src", + sessionsDir: tempRoot, + removeWorkspace: mock(() => Promise.resolve()), + findWorkspace: mock(() => ({ projectPath, workspacePath: "/tmp/proj/ws" })), + loadConfigOrDefault: mock(() => ({ projects: new Map() })), + }; + const workspaceService = new WorkspaceService( + mockConfig as Config, + historyService, + mockAIService, + mockInitStateManager as InitStateManager, + mockExtensionMetadataService as ExtensionMetadataService, + mockBackgroundProcessManager as BackgroundProcessManager + ); + + // A session whose startup recovery may be one await away from dispatching. + const holdTurnAdmission = mock(() => ({ + [Symbol.dispose]: () => { + releases += 1; + }, + })); + const dispose = mock(() => undefined); + (workspaceService as unknown as { sessions: Map }).sessions.set( + workspaceId, + { + holdTurnAdmission, + dispose, + } as unknown as AgentSession + ); + + const result = await workspaceService.remove(workspaceId, false); + expect(result.success).toBe(false); + expect(holdTurnAdmission).toHaveBeenCalledTimes(1); + // Held across the runtime deletion, released once the failed removal settles so the + // still-configured workspace stays usable. + expect(releasesWhenRuntimeDeleted).toBe(0); + expect(releases).toBe(1); + expect(dispose).not.toHaveBeenCalled(); + } finally { + createRuntimeSpy.mockRestore(); + await fsPromises.rm(tempRoot, { recursive: true, force: true }); + } + }); test("remove() calls runtime.deleteWorkspace when force=true", async () => { const workspaceId = "ws-remove-runtime-delete"; const projectPath = "/tmp/proj"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 5f0a72a0d6e..3e24599c358 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -5536,6 +5536,16 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } const persistedWorkspace = this.config.findWorkspace(workspaceId); + // Startup recovery or a queued dispatch can be one await from starting a stream that the single + // stopStream() below cannot see, and the entry stays present and unarchived until removal has + // finished with the runtime. Hold admission for the whole removal: success disposes the session, + // failure releases the hold so the workspace stays usable. + const admissionHolds = [ + this.sessions.get(workspaceId), + this.transientStartupRecoverySessions.get(workspaceId), + ] + .filter((session): session is AgentSession => session != null) + .map((session) => session.holdTurnAdmission()); // Try to remove from runtime (filesystem) try { @@ -6213,6 +6223,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { const message = getErrorMessage(error); return Err(`Failed to remove workspace: ${message}`); } finally { + for (const hold of admissionHolds) { + hold[Symbol.dispose](); + } this.removingWorkspaces.delete(workspaceId); } }