From 0fc31ae109929550ec61a1f96dcd8fc19eb0ea03 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 11 Aug 2026 14:52:32 -0700 Subject: [PATCH 1/3] feat(web): expose Prometheus metrics for the web process The web process had no instrumentation of any kind, so heap usage, GC duration, and event loop lag were invisible for the process that serves all application traffic. Diagnosing a recent GC-thrash incident required reading cgroup files inside the container and inferring the rest from trace spans. Mirror the backend's prom-client setup and serve it on its own port (WEB_METRICS_PORT, default 3070) rather than as a Next.js route, so scraping bypasses app middleware and is not reachable through the ingress. Also adds nodejs_heap_size_limit_bytes, which prom-client's default metrics omit. Without the ceiling, heap usage alone cannot distinguish a busy process from one pinned at its limit running back-to-back full mark-compacts. Co-Authored-By: Claude Opus 5 (1M context) --- packages/shared/src/env.server.ts | 3 ++ packages/web/package.json | 1 + packages/web/src/instrumentation.ts | 5 +++ packages/web/src/metricsServer.ts | 50 +++++++++++++++++++++++++++++ packages/web/src/promClient.test.ts | 47 +++++++++++++++++++++++++++ packages/web/src/promClient.ts | 21 ++++++++++++ yarn.lock | 1 + 7 files changed, 128 insertions(+) create mode 100644 packages/web/src/metricsServer.ts create mode 100644 packages/web/src/promClient.test.ts create mode 100644 packages/web/src/promClient.ts diff --git a/packages/shared/src/env.server.ts b/packages/shared/src/env.server.ts index 2edd9050c..52e4f1d25 100644 --- a/packages/shared/src/env.server.ts +++ b/packages/shared/src/env.server.ts @@ -174,6 +174,9 @@ const options = { WORKER_API_URL: z.string().url().default("http://localhost:3060"), + // Port the web process serves its Prometheus metrics on. + WEB_METRICS_PORT: numberSchema.default(3070), + // Auth AUTH_SECRET: z.string(), AUTH_URL: z.string().url(), diff --git a/packages/web/package.json b/packages/web/package.json index f3e8d2715..91b7a0ad6 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -173,6 +173,7 @@ "posthog-js": "^1.369.0", "posthog-node": "^5.24.15", "pretty-bytes": "^6.1.1", + "prom-client": "^15.1.3", "psl": "^1.15.0", "react": "19.2.4", "react-day-picker": "^9.14.0", diff --git a/packages/web/src/instrumentation.ts b/packages/web/src/instrumentation.ts index 7bea4bbf0..e64926611 100644 --- a/packages/web/src/instrumentation.ts +++ b/packages/web/src/instrumentation.ts @@ -9,6 +9,11 @@ export async function register() { await import('./sentry.edge.config'); } + if (process.env.NEXT_RUNTIME === 'nodejs') { + const { startMetricsServer } = await import('./metricsServer'); + startMetricsServer(); + } + if (process.env.NEXT_RUNTIME === 'nodejs') { const { initialize } = await import('./initialize'); await initialize(); diff --git a/packages/web/src/metricsServer.ts b/packages/web/src/metricsServer.ts new file mode 100644 index 000000000..4755bbed6 --- /dev/null +++ b/packages/web/src/metricsServer.ts @@ -0,0 +1,50 @@ +import { createLogger, env } from '@sourcebot/shared'; +import { createServer, Server } from 'node:http'; +import { registry } from './promClient'; + +const logger = createLogger('web-metrics-server'); + +/** + * Serves the web process' Prometheus metrics on its own port, rather than as a + * Next.js route, so that scraping doesn't pass through the app's middleware or + * get exposed publicly through the ingress. + */ +export const startMetricsServer = (): Server | undefined => { + // Guard against a missing port: `listen(undefined)` binds a random one, which + // would leave the scrape target silently broken instead of loudly absent. + const port = Number(env.WEB_METRICS_PORT); + if (!Number.isInteger(port) || port <= 0) { + logger.error(`Invalid WEB_METRICS_PORT '${env.WEB_METRICS_PORT}'; metrics server not started.`); + return undefined; + } + + const server = createServer(async (req, res) => { + if (req.url !== '/metrics') { + res.writeHead(404); + res.end(); + return; + } + + try { + const metrics = await registry.metrics(); + res.writeHead(200, { 'Content-Type': registry.contentType }); + res.end(metrics); + } catch (error) { + logger.error(`Failed to collect metrics: ${error}`); + res.writeHead(500); + res.end(); + } + }); + + // Metrics must never take down the web server, so swallow listen failures + // (a port collision, most likely) instead of letting the 'error' event throw. + server.on('error', (error) => { + logger.error(`Metrics server error: ${error}`); + }); + + server.listen(port, () => { + logger.info(`Web metrics server listening on port ${port}`); + }); + + return server; +}; diff --git a/packages/web/src/promClient.test.ts b/packages/web/src/promClient.test.ts new file mode 100644 index 000000000..eb571555a --- /dev/null +++ b/packages/web/src/promClient.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest'; +import { registry } from './promClient'; + +const metricNames = (output: string): Set => { + return new Set( + output + .split('\n') + .filter(line => line.length > 0 && !line.startsWith('#')) + .map(line => line.split(/[ {]/)[0]) + ); +}; + +describe('web promClient', () => { + it('exposes the metrics needed to diagnose heap pressure', async () => { + const names = metricNames(await registry.metrics()); + + expect(names).toContain('nodejs_heap_size_limit_bytes'); + expect(names).toContain('nodejs_heap_size_used_bytes'); + expect(names).toContain('nodejs_eventloop_lag_p99_seconds'); + }); + + it('registers the gc duration histogram', () => { + // Asserted via the registry rather than the rendered output: the histogram + // emits no series until a garbage collection has actually been observed. + expect(registry.getSingleMetric('nodejs_gc_duration_seconds')).toBeDefined(); + }); + + it('reports a plausible heap size limit', async () => { + const output = await registry.metrics(); + const line = output.split('\n').find(l => l.startsWith('nodejs_heap_size_limit_bytes ')); + + expect(line).toBeDefined(); + + const limit = Number(line!.split(' ')[1]); + expect(Number.isFinite(limit)).toBe(true); + // Any real V8 heap limit is well above 100MB and well below 100GB. + expect(limit).toBeGreaterThan(100 * 1024 * 1024); + expect(limit).toBeLessThan(100 * 1024 * 1024 * 1024); + }); + + it('can be collected repeatedly', async () => { + const first = await registry.metrics(); + const second = await registry.metrics(); + + expect(metricNames(first)).toEqual(metricNames(second)); + }); +}); diff --git a/packages/web/src/promClient.ts b/packages/web/src/promClient.ts new file mode 100644 index 000000000..294159d31 --- /dev/null +++ b/packages/web/src/promClient.ts @@ -0,0 +1,21 @@ +import client, { Gauge, Registry } from 'prom-client'; +import { getHeapStatistics } from 'node:v8'; + +export const registry = new Registry(); + +// `collectDefaultMetrics` reports heap usage but not the ceiling it's measured +// against, and usage alone can't distinguish "busy" from "out of room". Without +// the limit there's no way to tell whether V8 is doing cheap incremental +// collections or is pinned at its ceiling running full mark-compacts. +const heapSizeLimit = new Gauge({ + name: 'nodejs_heap_size_limit_bytes', + help: 'V8 heap size limit in bytes', + collect() { + this.set(getHeapStatistics().heap_size_limit); + }, +}); +registry.registerMetric(heapSizeLimit); + +client.collectDefaultMetrics({ + register: registry, +}); diff --git a/yarn.lock b/yarn.lock index 9f0096f20..084538d10 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9291,6 +9291,7 @@ __metadata: posthog-js: "npm:^1.369.0" posthog-node: "npm:^5.24.15" pretty-bytes: "npm:^6.1.1" + prom-client: "npm:^15.1.3" psl: "npm:^1.15.0" raw-loader: "npm:^4.0.2" react: "npm:19.2.4" From b4441390551c80d7d2e6b80ba55bfebfb4d4ef99 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 11 Aug 2026 15:01:46 -0700 Subject: [PATCH 2/3] docs: add changelog entry for web process metrics Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9b5bb986..ca5a1e25a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added a manually triggered cloud image release workflow for isolated internal deployments. [#1566](https://github.com/sourcebot-dev/sourcebot/pull/1566) +- Added Prometheus metrics for the web process (heap, garbage collection, and event loop lag), served on `WEB_METRICS_PORT` (default `3070`). [#1570](https://github.com/sourcebot-dev/sourcebot/pull/1570) ## [5.1.6] - 2026-08-10 From 10d20474dfea98b0b9a272c489d8aba914d5d2f4 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 11 Aug 2026 15:10:34 -0700 Subject: [PATCH 3/3] fix changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca5a1e25a..a53d9e78f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added a manually triggered cloud image release workflow for isolated internal deployments. [#1566](https://github.com/sourcebot-dev/sourcebot/pull/1566) -- Added Prometheus metrics for the web process (heap, garbage collection, and event loop lag), served on `WEB_METRICS_PORT` (default `3070`). [#1570](https://github.com/sourcebot-dev/sourcebot/pull/1570) +- Added Prometheus metrics for the web process, served on `WEB_METRICS_PORT` (default `3070`). [#1570](https://github.com/sourcebot-dev/sourcebot/pull/1570) ## [5.1.6] - 2026-08-10