diff --git a/.ai/contexts/session-cache.md b/.ai/contexts/session-cache.md index 0857b93e..8207ca1b 100644 --- a/.ai/contexts/session-cache.md +++ b/.ai/contexts/session-cache.md @@ -374,6 +374,25 @@ untouched. cannot grow unboundedly across host-list edits. Attach now exists off this data (issue #221, below); capacity tiers and a liveness badge in the UI (#218, #212) still don't. + - **Remote hosts — meta.json sidecars (issue #244).** A subagent's agent + type lives in a sidecar `agent-.meta.json` next to its transcript, + read by `readSubagentMeta()` (`read-session-file.js`). `LIST_COMMAND`'s + projects `find` matches `*.jsonl` **or** `*.meta.json`, and + `isSafeMirrorRelPath` (`remote-hosts.js`) — not `isSafeRelPath` — gates + both in `parseInventory` and in `remote-mirror.js`'s inventory filter and + fetch queue, so the sidecar rides the same `scp` path as its transcript + and lands in the same mirrored directory (no path-layout code needed: + `fetchOne` already preserves the full relative path). `isSafeRelPath` + itself is untouched on purpose — `remote-watch.js` still imports it + directly, so a `.meta.json` write on the host is never classified as + project activity; the sidecar only ever arrives on the next inventory + refresh. Inside `syncMirror`, transcripts are sorted ahead of sidecars + before the per-cycle budget (`MAX_CYCLE_FILES`/`MAX_CYCLE_BYTES`, #238) is + applied, so a flood of tiny sidecars can never push a transcript out of a + full cycle. A sidecar that arrives (or leaves) on its own — the transcript + itself unchanged — is reported to the indexer under its **transcript's** + rel path, not its own, because `readSubagentMeta()` in the transcript's + row is what actually needs re-deriving. ## Remote hosts — tmux attach (issue #221) diff --git a/remote-hosts.js b/remote-hosts.js index b6562cee..3440885b 100644 --- a/remote-hosts.js +++ b/remote-hosts.js @@ -81,6 +81,20 @@ function isSafeRelPath(rel) { return SAFE_REL_RE.test(rel); } +// see .ai/contexts/session-cache.md ("Remote hosts — meta.json sidecars") +function isSafeMetaRelPath(rel) { + if (typeof rel !== 'string' || rel.length === 0 || rel.length > 512) return false; + if (!rel.endsWith('.meta.json')) return false; + if (rel.includes('..')) return false; + if (rel.split('/').some(seg => seg === '.')) return false; + return SAFE_REL_RE.test(rel); +} + +// see .ai/contexts/session-cache.md ("Remote hosts — meta.json sidecars") +function isSafeMirrorRelPath(rel) { + return isSafeRelPath(rel) || isSafeMetaRelPath(rel); +} + function topFolderOf(rel) { const i = rel.indexOf('/'); return i < 0 ? null : rel.slice(0, i); @@ -101,5 +115,7 @@ module.exports = { mirrorProjectsDirFor, manifestPathFor, isSafeRelPath, + isSafeMetaRelPath, + isSafeMirrorRelPath, topFolderOf, }; diff --git a/remote-mirror.js b/remote-mirror.js index d3f4ebea..2a6cb1cc 100644 --- a/remote-mirror.js +++ b/remote-mirror.js @@ -3,7 +3,7 @@ const fs = require('fs'); const path = require('path'); -const { isSafeRelPath, topFolderOf } = require('./remote-hosts'); +const { isSafeMirrorRelPath, topFolderOf } = require('./remote-hosts'); const MAX_INVENTORY_ENTRIES = 20_000; // Per-file ceiling: scp is bounded in time, never in bytes. @@ -54,7 +54,7 @@ async function syncMirror({ alias, transport, projectsDir, manifestPath, log }) const want = new Map(); for (const entry of files) { - if (!entry || !isSafeRelPath(entry.rel)) continue; + if (!entry || !isSafeMirrorRelPath(entry.rel)) continue; if (!topFolderOf(entry.rel)) continue; // a transcript must live under a project folder want.set(entry.rel, { size: Number(entry.size) || 0, mtimeMs: Number(entry.mtimeMs) || 0 }); } @@ -67,7 +67,13 @@ async function syncMirror({ alias, transport, projectsDir, manifestPath, log }) let cycleFull = false; let deferredFiles = 0; let deferredBytes = 0; - for (const [rel, meta] of want) { + // see .ai/contexts/session-cache.md ("Remote hosts — meta.json sidecars") + const byFetchPriority = [...want.entries()].sort((a, b) => { + const aMeta = a[0].endsWith('.meta.json') ? 1 : 0; + const bMeta = b[0].endsWith('.meta.json') ? 1 : 0; + return aMeta - bMeta; + }); + for (const [rel, meta] of byFetchPriority) { // The inventory already carries the size; scp is bounded in time only, so // this is the only place a single oversized transcript can be refused // before it lands. See .ai/contexts/session-cache.md, "Remote hosts". @@ -139,16 +145,19 @@ async function syncMirror({ alias, transport, projectsDir, manifestPath, log }) writeManifest(manifestPath, nextFiles); - // see .ai/contexts/session-cache.md ("Remote hosts file-level rescan") + // see .ai/contexts/session-cache.md ("Remote hosts file-level rescan" and + // "Remote hosts — meta.json sidecars") const changedFolders = new Set(); const changedFilesByFolder = new Map(); const markChanged = (rel) => { const folder = topFolderOf(rel); if (!folder) return; changedFolders.add(folder); + // see .ai/contexts/session-cache.md ("Remote hosts — meta.json sidecars") + const targetRel = rel.endsWith('.meta.json') ? rel.slice(0, -'.meta.json'.length) + '.jsonl' : rel; let set = changedFilesByFolder.get(folder); if (!set) { set = new Set(); changedFilesByFolder.set(folder, set); } - set.add(rel.slice(folder.length + 1)); + set.add(targetRel.slice(folder.length + 1)); }; for (const rel of fetched) markChanged(rel); if (failed.length === 0) { diff --git a/remote-transport.js b/remote-transport.js index bf5c977e..90a6cda2 100644 --- a/remote-transport.js +++ b/remote-transport.js @@ -3,7 +3,7 @@ const fs = require('fs'); const path = require('path'); -const { isSafeRelPath } = require('./remote-hosts'); +const { isSafeMirrorRelPath } = require('./remote-hosts'); const REMOTE_PROJECTS_REL = '.claude/projects'; const REMOTE_SESSIONS_REL = '.claude/sessions'; @@ -21,9 +21,10 @@ const SSH_BASE_OPTS = [ '-o', `ConnectTimeout=${DEFAULT_CONNECT_TIMEOUT_S}`, ]; -// see .ai/contexts/session-cache.md ("Remote SSH hosts (issue #211)") +// see .ai/contexts/session-cache.md ("Remote SSH hosts (issue #211)" and +// "Remote hosts — meta.json sidecars") const LIST_COMMAND = - `find ${REMOTE_PROJECTS_REL} -type f -name '*.jsonl' -printf '%T@\\t%s\\t%P\\n' || exit $?; ` + + `find ${REMOTE_PROJECTS_REL} -type f \\( -name '*.jsonl' -o -name '*.meta.json' \\) -printf '%T@\\t%s\\t%P\\n' || exit $?; ` + `printf '\\001SWITCHBOARD-SESSIONS\\001\\n'; ` + `find ${REMOTE_SESSIONS_REL} -maxdepth 1 -type f -name '[0-9]*.json' 2>/dev/null | LC_ALL=C sort | ` + `head -n ${MAX_SESSION_DESCRIPTORS} | while IFS= read -r f; do head -c ${MAX_SESSION_DESCRIPTOR_BYTES} "$f"; printf '\\n'; done`; @@ -38,7 +39,7 @@ function parseInventory(stdout) { const size = Number.parseInt(parts[1], 10); const rel = parts.slice(2).join('\t').replace(/\r$/, ''); if (!Number.isFinite(mtime) || !Number.isFinite(size)) continue; - if (!isSafeRelPath(rel)) continue; + if (!isSafeMirrorRelPath(rel)) continue; out.push({ rel, size, mtimeMs: Math.round(mtime * 1000) }); } return out; @@ -177,7 +178,7 @@ function createSshTransport(opts = {}) { const destPath = path.join(destRoot, rel); fs.mkdirSync(path.dirname(destPath), { recursive: true }); const tmpPath = destPath + '.part'; - // Deliberately unquoted; isSafeRelPath is the guard. see .ai/contexts/session-cache.md ("Remote SSH hosts") + // Deliberately unquoted; isSafeMirrorRelPath is the guard. see .ai/contexts/session-cache.md ("Remote SSH hosts") const remote = `${alias}:${REMOTE_PROJECTS_REL}/${rel}`; const res = await run('scp', [...SSH_BASE_OPTS, '-p', '-q', remote, tmpPath], { timeoutMs: fetchTimeoutMs, @@ -201,7 +202,7 @@ function createSshTransport(opts = {}) { async function fetchFiles(alias, rels, destRoot) { const fetched = []; const failed = []; - const queue = rels.filter(isSafeRelPath); + const queue = rels.filter(isSafeMirrorRelPath); let cursor = 0; const workers = Array.from({ length: Math.min(concurrency, queue.length) }, async () => { diff --git a/test/remote-hosts.test.js b/test/remote-hosts.test.js index 841c7652..58d81099 100644 --- a/test/remote-hosts.test.js +++ b/test/remote-hosts.test.js @@ -9,7 +9,8 @@ const assert = require('node:assert/strict'); const { isValidAlias, joinFolderKey, parseFolderKey, isRemoteFolder, - normalizeHosts, enabledHosts, normalizeRefreshMs, isSafeRelPath, topFolderOf, + normalizeHosts, enabledHosts, normalizeRefreshMs, isSafeRelPath, + isSafeMetaRelPath, isSafeMirrorRelPath, topFolderOf, MIN_REFRESH_MS, } = require('../remote-hosts'); const { encodeProjectPath } = require('../encode-project-path'); @@ -87,3 +88,25 @@ test('isSafeRelPath is the only guard between remote output and an scp argument' assert.equal(topFolderOf('bare.jsonl'), null); assert.equal(topFolderOf('-srv-x/abc.jsonl'), '-srv-x'); }); + +// issue #244: readSubagentMeta()'s sidecar needs its own safety gate, kept +// apart from isSafeRelPath so remote-watch.js's activity classification (which +// imports isSafeRelPath directly) keeps treating a sidecar write as a no-op — +// see .ai/contexts/session-cache.md ("Remote hosts — meta.json sidecars"). +test('isSafeMetaRelPath admits only a well-formed .meta.json sidecar path', () => { + assert.equal(isSafeMetaRelPath('-srv-x/uuid/subagents/agent-1.meta.json'), true); + assert.equal(isSafeMetaRelPath('-srv-x/uuid/subagents/agent-1.jsonl'), false); + assert.equal(isSafeMetaRelPath('../../etc/passwd.meta.json'), false); + assert.equal(isSafeMetaRelPath('/abs/path.meta.json'), false); + assert.equal(isSafeMetaRelPath("a/x';id;'.meta.json"), false); +}); + +test('isSafeMirrorRelPath admits both a transcript and its sidecar; isSafeRelPath stays .jsonl-only', () => { + assert.equal(isSafeMirrorRelPath('-srv-x/uuid/subagents/agent-1.jsonl'), true); + assert.equal(isSafeMirrorRelPath('-srv-x/uuid/subagents/agent-1.meta.json'), true); + assert.equal(isSafeMirrorRelPath('-srv-x/notes.txt'), false); + // The regression this guards: widening isSafeRelPath itself would make + // remote-watch.js's parseWatchLine() treat a sidecar write as project + // activity, which issue #244 explicitly rules out. + assert.equal(isSafeRelPath('-srv-x/uuid/subagents/agent-1.meta.json'), false); +}); diff --git a/test/remote-mirror.test.js b/test/remote-mirror.test.js index e1baf09e..41d5173d 100644 --- a/test/remote-mirror.test.js +++ b/test/remote-mirror.test.js @@ -11,6 +11,7 @@ const os = require('os'); const path = require('path'); const { syncMirror, readManifest, MAX_CYCLE_FILES, MAX_CYCLE_BYTES } = require('../remote-mirror'); +const { readSubagentMeta } = require('../read-session-file'); function tmp(name) { return fs.mkdtempSync(path.join(os.tmpdir(), 'switchboard-' + name + '-')); @@ -447,3 +448,123 @@ test('a host with no sessions dir completes the cycle with zero descriptors', as assert.ok(fs.existsSync(path.join(projectsDir, '-srv-a', 'a.jsonl'))); } finally { fs.rmSync(dir, { recursive: true, force: true }); } }); + +// issue #244: readSubagentMeta() (read-session-file.js) is the real consumer +// of the mirrored sidecar — it derives the sidecar path from the jsonl path by +// suffix substitution, so the two must land in the same mirrored directory. +// See .ai/contexts/session-cache.md ("Remote hosts — meta.json sidecars"). +test('a subagent .meta.json sidecar is mirrored next to its transcript, and readSubagentMeta finds it', async () => { + const dir = tmp('mirror-meta-sidecar'); + try { + const projectsDir = path.join(dir, 'projects'); + const manifestPath = path.join(dir, 'inventory.json'); + const jsonlRel = '-srv-x/parent-uuid/subagents/agent-1.jsonl'; + const metaRel = '-srv-x/parent-uuid/subagents/agent-1.meta.json'; + const t = fakeTransport({ + [jsonlRel]: { content: line('/srv/x'), mtimeMs: 1000 }, + [metaRel]: { content: JSON.stringify({ agentType: 'Explore', description: 'find things' }), mtimeMs: 1000 }, + }); + + const r = await syncMirror({ alias: 'vps', transport: t, projectsDir, manifestPath }); + + assert.equal(r.total, 2, 'both the transcript and its sidecar are in the inventory'); + assert.equal(r.fetched, 2); + const mirroredJsonl = path.join(projectsDir, ...jsonlRel.split('/')); + const mirroredMeta = path.join(projectsDir, ...metaRel.split('/')); + assert.ok(fs.existsSync(mirroredJsonl)); + assert.ok(fs.existsSync(mirroredMeta)); + assert.deepEqual(readSubagentMeta(mirroredJsonl), { agentType: 'Explore', description: 'find things' }); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +}); + +// The sidecar's own rel path is meaningless to the file-subset rescan (it +// carries no session row of its own); it must be reported under its +// transcript's rel path instead, so a sidecar arriving alone still triggers a +// re-derive of the row that needs it. +test('a sidecar-only change is reported to the indexer under its transcript rel path', async () => { + const dir = tmp('mirror-meta-changed'); + try { + const projectsDir = path.join(dir, 'projects'); + const manifestPath = path.join(dir, 'inventory.json'); + const jsonlRel = '-srv-x/parent-uuid/subagents/agent-1.jsonl'; + const metaRel = '-srv-x/parent-uuid/subagents/agent-1.meta.json'; + // First cycle: transcript only, no sidecar yet on the host. + const files = { [jsonlRel]: { content: line('/srv/x'), mtimeMs: 1000 } }; + await syncMirror({ alias: 'vps', transport: fakeTransport(files), projectsDir, manifestPath }); + + // Second cycle: the sidecar shows up; the transcript itself is unchanged. + files[metaRel] = { content: JSON.stringify({ agentType: 'Explore' }), mtimeMs: 2000 }; + const r = await syncMirror({ alias: 'vps', transport: fakeTransport(files), projectsDir, manifestPath }); + + assert.equal(r.fetched, 1, 'only the sidecar is new'); + assert.ok(r.changedFolders.has('-srv-x')); + const files1 = r.changedFilesByFolder.get('-srv-x'); + assert.ok(files1 && files1.has('parent-uuid/subagents/agent-1.jsonl'), + 'the transcript, not the sidecar, must be the file the indexer re-derives'); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +}); + +// issue #244 acceptance: no sidecar on the host must never be an error. +test('a subagent transcript with no sidecar on the host still mirrors cleanly', async () => { + const dir = tmp('mirror-meta-absent'); + try { + const projectsDir = path.join(dir, 'projects'); + const manifestPath = path.join(dir, 'inventory.json'); + const jsonlRel = '-srv-x/parent-uuid/subagents/agent-1.jsonl'; + const t = fakeTransport({ [jsonlRel]: { content: line('/srv/x'), mtimeMs: 1000 } }); + + const r = await syncMirror({ alias: 'vps', transport: t, projectsDir, manifestPath }); + + assert.equal(r.total, 1); + const mirroredJsonl = path.join(projectsDir, ...jsonlRel.split('/')); + assert.ok(fs.existsSync(mirroredJsonl)); + assert.equal(readSubagentMeta(mirroredJsonl), null, 'no sidecar on disk, no error, just null'); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +}); + +// issue #244: sidecars are tiny but must never starve a transcript out of a +// full cycle. Fillers are all .jsonl transcripts (bigger consumers of the +// same per-cycle file quota) placed ahead of the sidecar in host `find` +// order — the priority sort must still put every transcript ahead of every +// sidecar regardless of listing order. +test('transcripts keep priority over .meta.json sidecars when a cycle is full', async () => { + const dir = tmp('mirror-meta-priority'); + try { + const projectsDir = path.join(dir, 'projects'); + const manifestPath = path.join(dir, 'manifest.json'); + const metaRel = '-srv-a/parent/subagents/agent-1.meta.json'; + const entries = [ + { rel: metaRel, size: 10, mtimeMs: 1 }, + ...Array.from({ length: MAX_CYCLE_FILES }, (_, i) => ({ + rel: `-srv-a/f${String(i).padStart(4, '0')}.jsonl`, size: 10, mtimeMs: 1, + })), + ]; + const asked = []; + const transport = { + listFiles: async () => ({ files: entries, sessions: [] }), + fetchFiles: async (_alias, rels, destRoot) => { + asked.push(...rels); + for (const rel of rels) { + const p = path.join(destRoot, rel); + fs.mkdirSync(path.dirname(p), { recursive: true }); + fs.writeFileSync(p, 'x'); + } + return { fetched: rels, failed: [] }; + }, + }; + const warned = []; + const r = await syncMirror({ + alias: 'vps', transport, projectsDir, manifestPath, + log: { warn: (m) => warned.push(m), info() {}, error() {} }, + }); + + assert.equal(r.fetched, MAX_CYCLE_FILES, 'the cycle is exactly full of transcripts'); + assert.ok(!asked.includes(metaRel), 'the sidecar was deferred even though it was listed first'); + assert.ok(warned.some(m => m.includes('deferred'))); + + asked.length = 0; + const second = await syncMirror({ alias: 'vps', transport, projectsDir, manifestPath, log: { warn: () => {}, info() {}, error() {} } }); + assert.equal(second.fetched, 1, 'the deferred sidecar is picked up once the transcripts are unchanged'); + assert.deepEqual(asked, [metaRel], 'the deferred sidecar is picked up once the transcripts are unchanged'); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +}); diff --git a/test/remote-transport.test.js b/test/remote-transport.test.js index d246f319..bc3c1442 100644 --- a/test/remote-transport.test.js +++ b/test/remote-transport.test.js @@ -55,6 +55,23 @@ test('parseInventory keeps well-formed lines and drops everything else', () => { ]); }); +// issue #244: readSubagentMeta() needs the sidecar mirrored next to its +// transcript. parseInventory is the first gate the sidecar has to clear — +// see .ai/contexts/session-cache.md ("Remote hosts — meta.json sidecars"). +test('parseInventory keeps a .meta.json sidecar alongside its transcript', () => { + const out = [ + '1757200000.0\t10\t-srv-x/uuid/subagents/agent-1.jsonl', + '1757200000.0\t42\t-srv-x/uuid/subagents/agent-1.meta.json', + '1757200000.0\t10\t../escape.meta.json', + '1757200000.0\t10\t-srv-x/notes.meta.txt', + ].join('\n') + '\n'; + + assert.deepEqual(parseInventory(out), [ + { rel: '-srv-x/uuid/subagents/agent-1.jsonl', size: 10, mtimeMs: 1757200000000 }, + { rel: '-srv-x/uuid/subagents/agent-1.meta.json', size: 42, mtimeMs: 1757200000000 }, + ]); +}); + test('listFiles spawns one bounded ssh with the alias as an operand, never as a shell string', async () => { const spawn = spawnRecorder((child) => { child.stdout.push('1757200000.0\t9\t-srv-a/a.jsonl\n'); @@ -89,13 +106,18 @@ test('listFiles spawns one bounded ssh with the alias as an operand, never as a // feature and reading a '*.key' secret file dropped in the same directory. test('LIST_COMMAND is pinned exactly — any widening of the sessions glob must fail this test', () => { const expected = - `find .claude/projects -type f -name '*.jsonl' -printf '%T@\\t%s\\t%P\\n' || exit $?; ` + + `find .claude/projects -type f \\( -name '*.jsonl' -o -name '*.meta.json' \\) -printf '%T@\\t%s\\t%P\\n' || exit $?; ` + `printf '\\001SWITCHBOARD-SESSIONS\\001\\n'; ` + `find .claude/sessions -maxdepth 1 -type f -name '[0-9]*.json' 2>/dev/null | LC_ALL=C sort | ` + `head -n ${MAX_SESSION_DESCRIPTORS} | while IFS= read -r f; do head -c ${MAX_SESSION_DESCRIPTOR_BYTES} "$f"; printf '\\n'; done`; assert.equal(LIST_COMMAND, expected); }); +// issue #244: the projects find must list both the transcript and its sidecar. +test('LIST_COMMAND lists .meta.json sidecars alongside .jsonl transcripts', () => { + assert.ok(LIST_COMMAND.includes("-name '*.jsonl' -o -name '*.meta.json'")); +}); + test('LIST_COMMAND can never match a .key file, independent of exact wording', () => { assert.ok(!LIST_COMMAND.includes('.key')); }); @@ -332,6 +354,25 @@ test('a failed scp reports the file instead of leaving a partial one behind', as } finally { fs.rmSync(dir, { recursive: true, force: true }); } }); +test('fetchFiles accepts a .meta.json sidecar rel path, not just .jsonl', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'switchboard-scp-meta-')); + try { + const spawn = spawnRecorder((child, cmd, args) => { + fs.writeFileSync(args[args.length - 1], '{"agentType":"Explore"}', 'utf8'); + child.stdout.push(null); + child.emit('close', 0); + }); + const t = createSshTransport({ spawn }); + + const r = await t.fetchFiles('vps', ['-srv-a/uuid/subagents/agent-1.meta.json'], dir); + + assert.deepEqual(r.fetched, ['-srv-a/uuid/subagents/agent-1.meta.json']); + assert.deepEqual(r.failed, []); + const final = path.join(dir, '-srv-a', 'uuid', 'subagents', 'agent-1.meta.json'); + assert.equal(fs.readFileSync(final, 'utf8'), '{"agentType":"Explore"}'); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +}); + test('an unsafe rel path never reaches scp', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'switchboard-scp-evil-')); try {