(remote): push liveness over a persistent watch channel

.ai/contexts/session-cache.md

@@ -103,12 +103,23 @@ or deleted from here.
   and every renderer id — it is not worth doing before a second host exists.
   **If you ever see two sessions fighting over a star or a title, this is why.**
 
-- **The mirror is pulled, never watched.** `fs.watch` cannot cross SSH
+- **The mirror is pulled on a timer, floored at 60 s, as the reconciliation
+  path — it never goes away.** `fs.watch` cannot cross SSH
   (inotify/FSEvents/ReadDirectoryChangesW are kernel-local), and the local
   watcher at `main.js` `startProjectsWatcher()` is deliberately not pointed at the
-  mirror — it would fire on our own `scp` writes, not on remote activity. A timer
-  drives it instead, floored at 60 s: a tighter loop costs latency and VPS CPU for
-  a "where is my session at" use case that does not need it.
+  mirror — it would fire on our own `scp` writes, not on remote activity.
+  Issue #240 adds a push channel alongside it (below) so a live host is not
+  stale for up to 5 minutes; the pull remains the ground truth and the only
+  path for a host with no push channel (see below).
+
+### Remote hosts — watch channel (issue #240)
+
+`remote-watch.js` keeps one long-lived `ssh -tt … inotifywait` child per
+declared alias and calls `remoteIndexer.refreshHostNow(alias)` (the periodic
+cycle's per-host entry point) on a coalesced "this host changed" signal —
+never on every line, and never in place of the periodic pull. Full rationale,
+the exact remote command, and the mutation proofs are in
+`.work-files/switchboard/remote-watch-report.md`.
 
 - **One `ssh` inventory, then only the deltas.** `remote-transport.js` runs
   `find .claude/projects -type f -name '*.jsonl' -printf '%T@	%s	%P

main.js

@@ -70,7 +70,7 @@
 }
 
 // Shell profiles → shell-profiles.js
 const { discoverShellProfiles, getShellProfiles, resolveShell, isWindows, isWslShell, windowsToWslPath, shellArgs, quoteArgvForShell } = require('./shell-profiles');
 const { startScheduler } = require('./schedule-runner');
 const { encodeProjectPath } = require('./encode-project-path');
 const { isSensitivePath, isAllowedMemoryPath: _isAllowedMemoryPath, resolveAllowedMemoryPath: _resolveAllowedMemoryPath, isKnownProjectRoot: _isKnownProjectRoot } = require('./ipc-path-validator');
@@ -453,16 +453,17 @@
     isInitialScanComplete, setInitialScanComplete,
   },
 });
 const { readSessionFile, readFolderFromFilesystem, refreshFolder, reconcileCacheFromFilesystem,
         buildProjectsFromCache, notifyRendererProjectsChanged, sendStatus, populateCacheViaWorker,
         scanFoldersViaWorker, setRemoteRoots, resolveFolderDir } = sessionCache;
 const { resolveJsonlPath, enumerateSessionFiles } = require('./read-session-file');
 
 // --- Remote SSH hosts (observation only) — see .ai/contexts/session-cache.md ---
-const { isRemoteFolder, parseFolderKey, joinFolderKey } = require('./remote-hosts');
+const { isRemoteFolder, parseFolderKey, joinFolderKey, enabledHosts } = require('./remote-hosts');
 const REMOTE_READ_ONLY = 'remote sessions are read-only — this build observes them, it does not attach to them';
 const { createSshTransport } = require('./remote-transport');
 const { createRemoteIndexer } = require('./remote-index');
+const { createRemoteWatcher } = require('./remote-watch');
 
 const remoteTransport = createSshTransport({ log });
 const remoteIndexer = createRemoteIndexer({
@@ -478,6 +479,22 @@
   log,
 });
 
+// see .ai/contexts/session-cache.md ("Remote hosts — watch channel")
+const remoteWatcher = createRemoteWatcher({ log });
+let watchedAliases = new Set();
+function onRemoteWatchEvent(alias) { remoteIndexer.refreshHostNow(alias).catch(() => {}); }
+function syncRemoteWatchers() {
+  const declared = enabledHosts((getSetting('global') || {}).remoteHosts);
+  const wanted = new Set(declared.map(h => h.alias));
+  for (const alias of watchedAliases) {
+    if (!wanted.has(alias)) remoteWatcher.stop(alias);
+  }
+  for (const host of declared) {
+    if (!remoteWatcher.isRunning(host.alias)) remoteWatcher.start(host.alias, onRemoteWatchEvent);
+  }
+  watchedAliases = wanted;
+}
+
 // see .ai/contexts/session-cache.md ("Remote hosts — tmux attach")
 const remoteAttachAdapter = createTmuxAttachAdapter({
   spawnPty: (file, args, ptyOpts) => spawnPty(file, args, { ...ptyOpts, cwd: os.homedir(), env: cleanPtyEnv }),
@@ -1476,6 +1493,7 @@
 ipcMain.handle('remote-hosts-apply', () => {
   try {
     const running = remoteIndexer.restart();
+    syncRemoteWatchers();
     return { ok: true, running };
   } catch (err) {
     return { ok: false, error: err.message };
@@ -2097,7 +2115,7 @@
   // WSL profiles only work for plain terminals — Claude CLI sessions need the
   // Windows shell because session data lives on the Windows filesystem.
   const requestedProfile = resolveShell(effectiveProfileId);
   const useWslProfile = isWslShell(requestedProfile.path) && isPlainTerminal;
   const shellProfile = (isWslShell(requestedProfile.path) && !isPlainTerminal)
     ? resolveShell('auto')
     : requestedProfile;
@@ -2649,6 +2667,7 @@
     startProjectsWatcher();
     // No declared host => no timer and no ssh call. see .ai/contexts/session-cache.md ("Remote SSH hosts")
     remoteIndexer.start();
+    syncRemoteWatchers();
     cliSessionState.ensureWatching();
     // Remove IDE lock files left behind by a crashed instance whose PID was
     // reused (the function only unlinks locks matching our own pid).
@@ -2790,6 +2809,7 @@
   cliSessionState.stop();
   // Stops the timer and SIGKILLs any ssh/scp still in flight.
   remoteIndexer.dispose();
+  remoteWatcher.stopAll();
 
   // Kill all PTY processes on quit
   for (const [id, session] of activeSessions) {

remote-index.js

@@ -17,6 +17,12 @@ const NOOP_LOG = { info() {}, warn() {}, error() {} };
 // see .ai/contexts/session-cache.md ("Remote hosts backoff")
 const MAX_BACKOFF_MS = 30 * 60 * 1000;
 
+// see .ai/contexts/session-cache.md ("Remote hosts backoff")
+function backoffDelayMs(failures, intervalMs) {
+  if (failures <= 0) return 0;
+  return Math.min(intervalMs * Math.pow(2, failures - 1), MAX_BACKOFF_MS);
+}
+
 /**
  * Periodic mirror + index of every declared SSH host.
  * see .ai/contexts/session-cache.md ("Remote SSH hosts")
@@ -47,6 +53,7 @@ function createRemoteIndexer(ctx) {
   const remoteSessions = new Map(); // alias -> sessions array, from the same ssh cycle as the inventory
   const remoteSessionsAt = new Map(); // alias -> epoch ms of the last cycle that did not throw
   const hostBackoff = new Map(); // alias -> { failures, lastError, nextAttemptAt }
+  const hostInFlight = new Set();
 
   function backoffState(alias) {
     let s = hostBackoff.get(alias);
@@ -57,12 +64,6 @@ function createRemoteIndexer(ctx) {
     return s;
   }
 
-  // see .ai/contexts/session-cache.md ("Remote hosts backoff")
-  function backoffDelayMs(failures, intervalMs) {
-    if (failures <= 0) return 0;
-    return Math.min(intervalMs * Math.pow(2, failures - 1), MAX_BACKOFF_MS);
-  }
-
   function onHostSuccess(alias) {
     const state = backoffState(alias);
     if (state.failures > 0) {
@@ -211,6 +212,7 @@ function createRemoteIndexer(ctx) {
     try {
       for (const host of list) {
         if (stopped) break;
+        if (hostInFlight.has(host.alias)) continue;
         const state = backoffState(host.alias);
         if (now() < state.nextAttemptAt) continue; // still backing off: no attempt, no log, no ssh
         try {
@@ -231,6 +233,32 @@ function createRemoteIndexer(ctx) {
     return { skipped: false, hosts: list.length, changed, errors };
   }
 
+  // see .ai/contexts/session-cache.md ("Remote hosts — watch channel")
+  async function refreshHostNow(alias) {
+    if (stopped || inFlight || hostInFlight.has(alias)) return { skipped: true };
+    const host = hosts().find(h => h.alias === alias);
+    if (!host) return { skipped: true };
+    const state = backoffState(alias);
+    if (now() < state.nextAttemptAt) return { skipped: true };
+    const intervalMs = normalizeRefreshMs(ctx.getRefreshMs ? ctx.getRefreshMs() : undefined);
+    hostInFlight.add(alias);
+    let changed = false;
+    let error = null;
+    try {
+      changed = await refreshHost(host);
+      onHostSuccess(alias);
+      remoteSessionsAt.set(alias, now());
+    } catch (err) {
+      remoteSessions.set(alias, []);
+      onHostFailure(alias, err, intervalMs);
+      error = err.message;
+    } finally {
+      hostInFlight.delete(alias);
+    }
+    if (changed && ctx.notify) ctx.notify();
+    return { skipped: false, changed, error };
+  }
+
   function start() {
     stopped = false;
     const list = hosts();
@@ -276,11 +304,11 @@ function createRemoteIndexer(ctx) {
   }
 
   return {
-    start, stop, dispose, restart, refreshNow,
+    start, stop, dispose, restart, refreshNow, refreshHostNow,
     isRunning: () => timer !== null,
     getRemoteSessions,
     getRemoteHostState,
   };
 }
 
-module.exports = { createRemoteIndexer };
+module.exports = { createRemoteIndexer, backoffDelayMs };

remote-watch.js

@@ -0,0 +1,189 @@
+// see .ai/contexts/session-cache.md ("Remote hosts — watch channel")
+'use strict';
+
+const { REMOTE_PROJECTS_REL, REMOTE_SESSIONS_REL } = require('./remote-transport');
+const { isSafeRelPath } = require('./remote-hosts');
+const { backoffDelayMs } = require('./remote-index');
+
+const NOOP_LOG = { info() {}, warn() {}, error() {} };
+const NO_INOTIFYWAIT_MARKER = 'SWITCHBOARD-NO-INOTIFYWAIT';
+const NO_INOTIFYWAIT_EXIT_CODE = 44;
+const SESSION_FILE_RE = /^[0-9]+\.json$/;
+const PROJECTS_EVENTS = 'modify,close_write,create,moved_to';
+const SESSIONS_EVENTS = 'close_write,create,delete,moved_to';
+const COALESCE_MS = 15000;
+const RESTART_BASE_MS = 5000;
+const HEALTHY_MS = 30000;
+
+function buildWatchCommand() {
+  return `if ! command -v inotifywait >/dev/null 2>&1; then echo ${NO_INOTIFYWAIT_MARKER}; exit ${NO_INOTIFYWAIT_EXIT_CODE}; fi; ` +
+    `mkdir -p '${REMOTE_PROJECTS_REL}' '${REMOTE_SESSIONS_REL}'; ` +
+    `inotifywait -m -r -e ${PROJECTS_EVENTS} --format 'P|%w%f' '${REMOTE_PROJECTS_REL}' & p=$!; ` +
+    `inotifywait -m -e ${SESSIONS_EVENTS} --format 'S|%w%f' '${REMOTE_SESSIONS_REL}' & s=$!; ` +
+    `wait $p $s`;
+}
+
+function buildSshArgs(alias) {
+  return ['-tt', '-o', 'BatchMode=yes', alias, buildWatchCommand()];
+}
+
+function parseWatchLine(line) {
+  if (typeof line !== 'string' || line.length < 3 || line[1] !== '|') return null;
+  const kind = line[0];
+  const raw = line.slice(2);
+  if (kind === 'P') {
+    const prefix = REMOTE_PROJECTS_REL + '/';
+    if (!raw.startsWith(prefix)) return null;
+    const rel = raw.slice(prefix.length);
+    return isSafeRelPath(rel) ? { kind: 'project', rel } : null;
+  }
+  if (kind === 'S') {
+    const prefix = REMOTE_SESSIONS_REL + '/';
+    if (!raw.startsWith(prefix)) return null;
+    const rel = raw.slice(prefix.length);
+    return SESSION_FILE_RE.test(rel) ? { kind: 'session', rel } : null;
+  }
+  return null;
+}
+
+function createRemoteWatcher(opts = {}) {
+  const spawnFn = opts.spawn || require('child_process').spawn;
+  const log = opts.log || NOOP_LOG;
+  const setT = (opts.timers && opts.timers.setTimeout) || setTimeout;
+  const clearT = (opts.timers && opts.timers.clearTimeout) || clearTimeout;
+
+  const states = new Map();
+
+  function killChild(s) {
+    if (s.restartTimer) { clearT(s.restartTimer); s.restartTimer = null; }
+    if (s.child) {
+      const child = s.child;
+      s.child = null;
+      try { child.kill(); } catch {}
+    }
+  }
+
+  function emitCoalesced(s, kind) {
+    if (s.cooldown[kind]) { s.pending[kind] = true; return; }
+    s.onEvent(s.alias, kind);
+    s.cooldown[kind] = true;
+    const t = setT(() => {
+      s.cooldown[kind] = false;
+      if (s.pending[kind]) { s.pending[kind] = false; emitCoalesced(s, kind); }
+    }, COALESCE_MS);
+    if (t && t.unref) t.unref();
+  }
+
+  function handleLine(s, rawLine) {
+    const line = rawLine.replace(/\r$/, '');
+    if (!line) return;
+    if (line.includes(NO_INOTIFYWAIT_MARKER)) {
+      s.unwatchable = true;
+      log.warn(`[remote-watch:${s.alias}] inotifywait is not installed on this host — ` +
+        'watch channel disabled, periodic refresh still covers it');
+      killChild(s);
+      return;
+    }
+    const parsed = parseWatchLine(line);
+    if (!parsed) return;
+    emitCoalesced(s, parsed.kind);
+  }
+
+  function onData(s, chunk) {
+    s.buf += chunk;
+    let idx;
+    while ((idx = s.buf.indexOf('\n')) !== -1) {
+      const line = s.buf.slice(0, idx);
+      s.buf = s.buf.slice(idx + 1);
+      handleLine(s, line);
+    }
+  }
+
+  function scheduleRestart(s) {
+    const delay = backoffDelayMs(s.failures, RESTART_BASE_MS);
+    s.restartTimer = setT(() => { s.restartTimer = null; spawnChild(s); }, delay);
+    if (s.restartTimer && s.restartTimer.unref) s.restartTimer.unref();
+  }
+
+  function onExit(s) {
+    s.child = null;
+    if (s.stopped || s.unwatchable) return;
+    s.failures = (Date.now() - s.spawnedAt) < HEALTHY_MS ? s.failures + 1 : 0;
+    scheduleRestart(s);
+  }
+
+  function spawnChild(s) {
+    if (s.stopped || s.unwatchable) return;
+    let child;
+    try {
+      child = spawnFn('ssh', buildSshArgs(s.alias), { windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
+    } catch (err) {
+      log.warn(`[remote-watch:${s.alias}] spawn failed: ${err.message}`);
+      s.failures += 1;
+      scheduleRestart(s);
+      return;
+    }
+    s.child = child;
+    s.buf = '';
+    s.spawnedAt = Date.now();
+    if (child.stdout) {
+      child.stdout.setEncoding('utf8');
+      child.stdout.on('data', (chunk) => onData(s, chunk));
+    }
+    if (child.stderr) child.stderr.on('data', () => {});
+    child.on('error', () => {});
+    child.on('close', () => onExit(s));
+  }
+
+  function getState(alias) {
+    let s = states.get(alias);
+    if (!s) {
+      s = {
+        alias, child: null, buf: '', stopped: true, unwatchable: false,
+        failures: 0, spawnedAt: 0, restartTimer: null, onEvent: null,
+        cooldown: { project: false, session: false },
+        pending: { project: false, session: false },
+      };
+      states.set(alias, s);
+    }
+    return s;
+  }
+
+  function start(alias, onEvent) {
+    if (typeof alias !== 'string' || !alias || typeof onEvent !== 'function') return;
+    const s = getState(alias);
+    if (!s.stopped && !s.unwatchable) return;
+    s.stopped = false;
+    s.unwatchable = false;
+    s.failures = 0;
+    s.onEvent = onEvent;
+    spawnChild(s);
+  }
+
+  function stop(alias) {
+    const s = states.get(alias);
+    if (!s) return;
+    s.stopped = true;
+    killChild(s);
+  }
+
+  function stopAll() {
+    for (const alias of [...states.keys()]) stop(alias);
+  }
+
+  function isRunning(alias) {
+    const s = states.get(alias);
+    return !!(s && !s.stopped && !s.unwatchable);
+  }
+
+  return { start, stop, stopAll, isRunning };
+}
+
+module.exports = {
+  createRemoteWatcher,
+  parseWatchLine,
+  buildWatchCommand,
+  buildSshArgs,
+  NO_INOTIFYWAIT_MARKER,
+  NO_INOTIFYWAIT_EXIT_CODE,
+};