feat(hub,cli,web): fleet runner version governance (skew, self-upgrade, soft-fail reopen)

cli/src/agent/sessionFactory.test.ts

@@ -1,4 +1,4 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 import type { Session } from '@/api/types'
 
 const {
@@ -29,7 +29,8 @@ vi.mock('@/api/api', () => ({
 }))
 
 vi.mock('@/runner/controlClient', () => ({
-    notifyRunnerSessionStarted: notifyRunnerSessionStartedMock
+    notifyRunnerSessionStarted: notifyRunnerSessionStartedMock,
+    getInstalledCliMtimeMs: () => 1_700_000_000_000,
 }))
 
 vi.mock('@/persistence', () => ({
@@ -55,6 +56,7 @@ import {
     bootstrapExistingSession,
     bootstrapLazySession,
     bootstrapSession,
+    buildMachineMetadata,
     buildSessionMetadata
 } from './sessionFactory'
 
@@ -363,3 +365,40 @@ describe('bootstrapSession HAPI_SESSION_ID export', () => {
         expect(process.env[HAPI_SESSION_ID_ENV]).toBe('hub-session-42')
     })
 })
+
+describe('buildMachineMetadata runner-only capabilities', () => {
+    const originalSupervised = process.env.HAPI_RUNNER_SUPERVISED
+
+    afterEach(() => {
+        if (originalSupervised === undefined) {
+            delete process.env.HAPI_RUNNER_SUPERVISED
+        } else {
+            process.env.HAPI_RUNNER_SUPERVISED = originalSupervised
+        }
+    })
+
+    it('omits machine RPC capabilities for terminal bootstrap metadata', () => {
+        delete process.env.HAPI_RUNNER_SUPERVISED
+        const metadata = buildMachineMetadata()
+        expect(metadata.capabilities).toBeUndefined()
+        expect(metadata.startedCliMtimeMs).toBeUndefined()
+        expect(metadata.installedCliMtimeMs).toBeUndefined()
+        expect(metadata.supervisedRestart).toBeUndefined()
+    })
+
+    it('advertises capabilities and supervisedRestart only for asRunner', () => {
+        process.env.HAPI_RUNNER_SUPERVISED = '1'
+        const metadata = buildMachineMetadata({ asRunner: true, startedCliMtimeMs: 42 })
+        expect(metadata.capabilities).toEqual(expect.arrayContaining(['cursor-chat-store-status', 'stop-runner']))
+        expect(metadata.startedCliMtimeMs).toBe(42)
+        expect(metadata.installedCliMtimeMs).toBe(1_700_000_000_000)
+        expect(metadata.supervisedRestart).toBe(true)
+    })
+
+    it('always sends supervisedRestart boolean for asRunner so sticky true can clear', () => {
+        delete process.env.HAPI_RUNNER_SUPERVISED
+        const metadata = buildMachineMetadata({ asRunner: true })
+        expect(metadata.capabilities).toEqual(expect.arrayContaining(['stop-runner']))
+        expect(metadata.supervisedRestart).toBe(false)
+    })
+})

cli/src/agent/sessionFactory.ts

@@ -5,13 +5,14 @@ import { resolve } from 'node:path'
 import { ApiClient } from '@/api/api'
 import type { ApiSessionClient } from '@/api/apiSession'
 import type { AgentState, MachineMetadata, Metadata, Session } from '@/api/types'
-import { notifyRunnerSessionStarted } from '@/runner/controlClient'
+import { getInstalledCliMtimeMs, notifyRunnerSessionStarted } from '@/runner/controlClient'
 import { readSettings } from '@/persistence'
 import { configuration } from '@/configuration'
 import { logger } from '@/ui/logger'
 import { runtimePath } from '@/projectPath'
 import { getInvokedCwd } from '@/utils/invokedCwd'
 import { readWorktreeEnv } from '@/utils/worktreeEnv'
+import { CURRENT_MACHINE_CAPABILITIES } from '@hapi/protocol/runnerCapabilities'
 import { exportHapiSessionEnv } from '@/agent/hapiSessionEnv'
 import packageJson from '../../package.json'
 
@@ -41,15 +42,38 @@ export type SessionBootstrapResult = {
     workingDirectory: string
 }
 
-export function buildMachineMetadata(options?: { workspaceRoots?: string[] }): MachineMetadata {
-    return {
+export function buildMachineMetadata(options?: {
+    workspaceRoots?: string[]
+    startedCliMtimeMs?: number
+    /**
+     * Only the long-lived runner daemon may advertise machine RPC capabilities
+     * and CLI mtimes. Terminal/lazy/existing session bootstraps must omit this
+     * so a newer CLI session cannot paint an old connected runner as current
+     * (#1108 bot Major).
+     */
+    asRunner?: boolean
+}): MachineMetadata {
+    const installedCliMtimeMs = getInstalledCliMtimeMs()
+    const startedCliMtimeMs = options?.startedCliMtimeMs ?? installedCliMtimeMs
+    const base: MachineMetadata = {
         host: process.env.HAPI_HOSTNAME || os.hostname(),
         platform: os.platform(),
         happyCliVersion: packageJson.version,
         homeDir: os.homedir(),
         happyHomeDir: configuration.happyHomeDir,
         happyLibDir: runtimePath(),
-        workspaceRoots: options?.workspaceRoots
+        workspaceRoots: options?.workspaceRoots,
+    }
+    if (!options?.asRunner) {
+        return base
+    }
+    return {
+        ...base,
+        capabilities: [...CURRENT_MACHINE_CAPABILITIES],
+        ...(typeof startedCliMtimeMs === 'number' ? { startedCliMtimeMs } : {}),
+        ...(typeof installedCliMtimeMs === 'number' ? { installedCliMtimeMs } : {}),
+        // Always boolean so hub merge can clear a prior true on unsupervised restart.
+        supervisedRestart: process.env.HAPI_RUNNER_SUPERVISED === '1',
     }
 }
 

cli/src/api/apiMachine.ts

@@ -24,6 +24,7 @@ import { RPC_METHODS } from '@hapi/protocol/rpcMethods'
 import { RUNNER_CAPABILITIES } from '@hapi/protocol'
 import type { RunnerState, Machine, MachineMetadata } from './types'
 import { RunnerStateSchema, MachineMetadataSchema } from './types'
+import { getInstalledCliMtimeMs } from '@/runner/controlClient'
 import { backoff } from '@/utils/time'
 import { getInvokedCwd } from '@/utils/invokedCwd'
 import { RpcHandlerManager } from './rpc/RpcHandlerManager'
@@ -656,6 +657,19 @@ export class ApiMachineClient {
                 time: Date.now(),
                 health: collectMachineHealth()
             })
+            const installedCliMtimeMs = getInstalledCliMtimeMs()
+            if (
+                typeof installedCliMtimeMs === 'number'
+                && this.machine.metadata
+                && this.machine.metadata.installedCliMtimeMs !== installedCliMtimeMs
+            ) {
+                void this.updateMachineMetadata((current) => ({
+                    ...(current ?? this.machine.metadata!),
+                    installedCliMtimeMs,
+                })).catch((error) => {
+                    logger.debug('[API MACHINE] Failed to refresh installedCliMtimeMs', error)
+                })
+            }
         }
         // Prime CPU sampling so the first heartbeat already includes CPU %.
         collectMachineHealth()

cli/src/runner/run.ts

@@ -1121,7 +1121,11 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}):
     const machine = await withRetry(
       () => api.getOrCreateMachine({
         machineId,
-        metadata: buildMachineMetadata({ workspaceRoots }),
+        metadata: buildMachineMetadata({
+            workspaceRoots,
+            startedCliMtimeMs: startedWithCliMtimeMs,
+            asRunner: true,
+        }),
         runnerState: initialRunnerState
       }),
       {

docs/guide/deployment.md

@@ -136,7 +136,9 @@ npm install -g pm2
 
 # Start hub and runner
 pm2 start "hapi hub --relay" --name hapi-hub
-pm2 start "hapi runner start-sync" --name hapi-runner
+# HAPI_RUNNER_SUPERVISED=1 lets the web Restart button stop the runner knowing
+# pm2 will cold-start it again (unsupervised stop would leave the host offline).
+HAPI_RUNNER_SUPERVISED=1 pm2 start "hapi runner start-sync" --name hapi-runner
 
 # View status and logs
 pm2 status
@@ -196,6 +198,11 @@ Create plist files for automatic startup on macOS.
         <string>runner</string>
         <string>start-sync</string>
     </array>
+    <key>EnvironmentVariables</key>
+    <dict>
+        <key>HAPI_RUNNER_SUPERVISED</key>
+        <string>1</string>
+    </dict>
     <key>RunAtLoad</key>
     <true/>
     <key>KeepAlive</key>
@@ -259,6 +266,9 @@ After=network.target hapi-hub.service
 [Service]
 Type=simple
 KillMode=process
+# Advertise supervisedRestart so the web UI Restart button may stop-runner
+# knowing systemd will cold-start the unit again.
+Environment=HAPI_RUNNER_SUPERVISED=1
 ExecStart=/usr/local/bin/hapi runner start-sync
 Restart=always
 RestartSec=5

docs/guide/installation.md

@@ -324,7 +324,11 @@ Use `--workspace-root <path>` to restrict which directories the runner can brows
 hapi runner start --workspace-root ~/projects --workspace-root ~/work
 ```
 
-For running the hub and runner as persistent background services (pm2, launchd, systemd), see [Deployment](./deployment.md).
+For running the hub and runner as persistent background services (pm2, launchd, systemd), see [Deployment](./deployment.md). Supervised installs should set `HAPI_RUNNER_SUPERVISED=1` on the runner process (systemd `Environment=` / pm2 `--env`) so the web **Restart** control can safely stop-runner knowing the supervisor will cold-start it.
+
+### Multi-machine hubs
+
+You can run **one hub** and **runners on many machines** (each machine installs its own CLI). When you upgrade the hub, upgrade the HAPI CLI on every machine that parents sessions. After the CLI binary on disk changes, that machine’s runner normally **self-restarts** via version handoff (unless `HAPI_DISABLE_VERSION_HANDOFF=1`). Until a runner reports the capabilities the hub requires, the web UI shows a **Runner out of date** banner (minimizable / snoozeable) with the host name and upgrade steps. The banner’s per-host **Restart** is only an escape hatch when handoff is stuck or disabled — the hub never downloads or installs packages on remotes.
 
 ## Security notes
 

hub/src/store/machines.test.ts

@@ -49,6 +49,85 @@ describe('mergeMachineMetadata', () => {
     it('returns undefined when the merge is a no-op', () => {
         expect(mergeMachineMetadata({ host: 'a' }, { host: 'a' })).toBeUndefined()
     })
+
+    it('clears omitted runner ads when clearOmittedRunnerAds is set', () => {
+        const merged = mergeMachineMetadata(
+            {
+                host: 'box',
+                capabilities: ['stop-runner'],
+                supervisedRestart: true,
+                startedCliMtimeMs: 1,
+                installedCliMtimeMs: 2,
+                displayName: 'keep-me',
+            },
+            { host: 'box', supervisedRestart: false },
+            { clearOmittedRunnerAds: true },
+        )
+        expect(merged).toEqual({
+            host: 'box',
+            supervisedRestart: false,
+            displayName: 'keep-me',
+        })
+    })
+
+    it('keeps sticky runner ads without clearOmittedRunnerAds (terminal bootstrap)', () => {
+        const merged = mergeMachineMetadata(
+            { host: 'box', capabilities: ['stop-runner'], supervisedRestart: true },
+            { host: 'box' },
+        )
+        expect(merged).toBeUndefined()
+    })
+})
+
+describe('runner metadata ad clear on re-registration', () => {
+    it('drops sticky supervisedRestart and capabilities when runner re-registers without them', () => {
+        const store = new Store(':memory:')
+        store.machines.getOrCreateMachine(
+            'machine-1',
+            {
+                host: 'box',
+                capabilities: ['stop-runner'],
+                supervisedRestart: true,
+                startedCliMtimeMs: 10,
+            },
+            { status: 'running', pid: 1 },
+            'ns',
+        )
+
+        const refreshed = store.machines.getOrCreateMachine(
+            'machine-1',
+            { host: 'box', supervisedRestart: false },
+            { status: 'running', pid: 2 },
+            'ns',
+        )
+
+        expect(refreshed.metadata).toEqual({ host: 'box', supervisedRestart: false })
+        expect(refreshed.metadata).not.toHaveProperty('capabilities')
+        expect(refreshed.metadata).not.toHaveProperty('startedCliMtimeMs')
+    })
+
+    it('does not clear runner ads on terminal-only metadata refresh (no runnerState)', () => {
+        const store = new Store(':memory:')
+        store.machines.getOrCreateMachine(
+            'machine-1',
+            { host: 'box', capabilities: ['stop-runner'], supervisedRestart: true },
+            { status: 'running', pid: 1 },
+            'ns',
+        )
+
+        const refreshed = store.machines.getOrCreateMachine(
+            'machine-1',
+            { host: 'box' },
+            null,
+            'ns',
+        )
+
+        expect(refreshed.metadata).toEqual({
+            host: 'box',
+            capabilities: ['stop-runner'],
+            supervisedRestart: true,
+        })
+    })
 })
 
 describe('runner capabilities backfill', () => {

hub/src/store/machines.ts

@@ -44,10 +44,33 @@ function isPlainObject(value: unknown): value is Record<string, unknown> {
 // machine-owned fields over the stored ones so registration doubles as a
 // refresh; hub-side fields the CLI never sends (e.g. displayName) survive.
 // Returns undefined when the merge would not change anything.
-export function mergeMachineMetadata(stored: unknown, incoming: unknown): Record<string, unknown> | undefined {
+//
+// When `clearOmittedRunnerAds` is set (full runner daemon registration with
+// runnerState), runner-advertised keys omitted from incoming are deleted so
+// rollback / unsupervised restart cannot leave sticky capabilities or
+// supervisedRestart:true (#1108 bot Major).
+export const RUNNER_ADVERTISED_METADATA_KEYS = [
+    'capabilities',
+    'supervisedRestart',
+    'startedCliMtimeMs',
+    'installedCliMtimeMs',
+] as const
+
+export function mergeMachineMetadata(
+    stored: unknown,
+    incoming: unknown,
+    options?: { clearOmittedRunnerAds?: boolean },
+): Record<string, unknown> | undefined {
     if (!isPlainObject(incoming)) return undefined
     const base = isPlainObject(stored) ? stored : {}
-    const merged = { ...base, ...incoming }
+    const merged: Record<string, unknown> = { ...base, ...incoming }
+    if (options?.clearOmittedRunnerAds) {
+        for (const key of RUNNER_ADVERTISED_METADATA_KEYS) {
+            if (!(key in incoming)) {
+                delete merged[key]
+            }
+        }
+    }
     return JSON.stringify(merged) === JSON.stringify(base) ? undefined : merged
 }
 
@@ -82,7 +105,11 @@ export function getOrCreateMachine(
         if (stored.namespace !== namespace) {
             throw new Error('Machine namespace mismatch')
         }
-        const merged = mergeMachineMetadata(stored.metadata, metadata)
+        const merged = mergeMachineMetadata(stored.metadata, metadata, {
+            // Full runner registration (with runnerState) owns the skew ads —
+            // omit means clear, so rollback cannot leave sticky supervisedRestart.
+            clearOmittedRunnerAds: runnerState !== null && runnerState !== undefined,
+        })
         let current = stored
         if (merged !== undefined) {
             db.prepare(`

hub/src/sync/rpcGateway.ts

@@ -279,6 +279,10 @@ export class RpcGateway {
         return CursorChatStoreStatusSchema.parse(result)
     }
 
+    async stopRunner(machineId: string): Promise<void> {
+        await this.machineRpc(machineId, RPC_METHODS.StopRunner, {})
+    }
+
     async getGitStatus(sessionId: string, cwd?: string): Promise<RpcCommandResponse> {
         return await this.sessionRpc(sessionId, RPC_METHODS.GitStatus, { cwd }) as RpcCommandResponse
     }