feat(rpc): export retried RPC errors and ingestion restarts as metrics

Errors that used to stop evm-dump or restart real-time ingestion are now
retried, so a dump stuck on a persistently failing endpoint no longer shows
up as restarts. Counting retried errors by kind, per endpoint, keeps that
condition alertable. In the data service the RPC clients live in worker
threads, so their counts are read on each scrape and the final counts of
closed workers are kept.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Author: mo4islona

common/changes/@subsquid/evm-data-service/fix-evm-rpc-retry-transient-errors-metrics_2026-09-29-14-14.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/evm-data-service",
+      "comment": "Report retried RPC errors from the worker threads",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@subsquid/evm-data-service"
+}

common/changes/@subsquid/evm-rpc/fix-evm-rpc-retry-transient-errors-metrics_2026-09-29-14-14.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/evm-rpc",
+      "comment": "Classify retried errors as rate_limit, transient, internal or no_result for metrics",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@subsquid/evm-rpc"
+}

common/changes/@subsquid/rpc-client/fix-evm-rpc-retry-transient-errors-metrics_2026-09-29-14-14.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/rpc-client",
+      "comment": "Count retried errors by kind: `getRetryKind()` and `retriedErrors` in `getMetrics()`",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@subsquid/rpc-client"
+}

common/changes/@subsquid/util-internal-data-service/fix-evm-rpc-retry-transient-errors-metrics_2026-09-29-14-14.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/util-internal-data-service",
+      "comment": "Export `sqd_hotblocks_ingestion_restarts_total` and `sqd_chain_rpc_retried_errors_total`, with `RpcMetricsCollector` to read RPC clients in worker threads",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@subsquid/util-internal-data-service"
+}

common/changes/@subsquid/util-internal-dump-cli/fix-evm-rpc-retry-transient-errors-metrics_2026-09-29-14-14.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/util-internal-dump-cli",
+      "comment": "Export `sqd_chain_rpc_retried_errors_total` by kind",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@subsquid/util-internal-dump-cli"
+}

evm/evm-data-service/src/data-source/setup.ts

@@ -1,6 +1,7 @@
 import {type CallFrameValidationMode, Rpc, EvmRpcDataSource, EvmRpcClient} from '@subsquid/evm-rpc'
 import type {Block, DataSource} from '@subsquid/util-internal-data-service'
 import {createLogger} from '@subsquid/logger'
+import type {RpcMetrics} from '@subsquid/rpc-client'
 import {Mapping} from './mapping'
 
 
@@ -36,7 +37,13 @@ export interface DataSourceOptions {
 }
 
 
-export function createDataSource(options: DataSourceOptions): DataSource<Block> {
+export interface DataSourceSetup {
+    source: DataSource<Block>
+    getRpcMetrics(): RpcMetrics
+}
+
+
+export function createDataSource(options: DataSourceOptions): DataSourceSetup {
     let httpRpcClient = new EvmRpcClient({
         url: options.httpRpc,
         maxBatchCallSize: options.httpRpcMaxBatchCallSize,
@@ -78,8 +85,12 @@ export function createDataSource(options: DataSourceOptions): DataSource<Block>
         strideSize: options.httpRpcStrideSize,
         strideConcurrency: options.httpRpcStrideConcurrency
     })
-    return new Mapping(rpcSource, {
+    let source = new Mapping(rpcSource, {
         withTraces: options.withTraces,
         withStateDiffs: options.withStatediffs,
     })
+    return {
+        source,
+        getRpcMetrics: () => httpRpcClient.getMetrics()
+    }
 }

evm/evm-data-service/src/data-source/worker-client.ts

@@ -1,3 +1,4 @@
+import type {RpcMetrics} from '@subsquid/rpc-client'
 import {Block, BlockRef, BlockStream, DataSource, StreamRequest} from '@subsquid/util-internal-data-service'
 import {Client, createWorker} from '@subsquid/util-internal-worker-thread'
 import {DataSourceOptions} from './setup'
@@ -33,6 +34,10 @@ export class WorkerClient implements DataSource<Block> {
         return this.stream('getStream', req)
     }
 
+    getRpcMetrics(): Promise<RpcMetrics> {
+        return this.worker.call('getRpcMetrics', [])
+    }
+
     private async *stream(method: string, req: StreamRequest): BlockStream<Block> {
         let stream: BlockStream<Block> = await this.worker.call(method, [req])
         yield* stream

evm/evm-data-service/src/data-source/worker.ts

@@ -3,7 +3,7 @@ import {getServer, getServerArguments} from '@subsquid/util-internal-worker-thre
 import {createDataSource} from './setup'
 
 
-const source = createDataSource(getServerArguments())
+const {source, getRpcMetrics} = createDataSource(getServerArguments())
 
 
 getServer()
@@ -15,4 +15,5 @@ getServer()
     .def('getStream', (req: StreamRequest) => {
         return source.getStream(req)
     })
+    .def('getRpcMetrics', () => getRpcMetrics())
     .start()

evm/evm-data-service/src/main.ts

@@ -6,6 +6,7 @@ import {
     type Block,
     type BlockStream,
     type DataSource,
+    RpcMetricsCollector,
     runDataService,
     type StreamRequest
 } from '@subsquid/util-internal-data-service'
@@ -122,7 +123,9 @@ runProgram(async () => {
         useGasUsedForReceiptsRoot: args.useGasUsedForReceiptsRoot
     }
 
+    let rpcMetrics = new RpcMetricsCollector()
     let mainWorker = new WorkerClient(dataSourceOptions)
+    rpcMetrics.add(mainWorker)
     let service: Awaited<ReturnType<typeof runDataService>> | undefined
     let dataSource: DataSource<Block> = {
         getHead() {
@@ -133,11 +136,13 @@ runProgram(async () => {
         },
         async *getFinalizedStream(req: StreamRequest): BlockStream<Block> {
             let worker = new WorkerClient(dataSourceOptions)
+            rpcMetrics.add(worker)
             service?.metrics.incActiveWorkers()
             try {
                 yield* worker.getFinalizedStream(req)
             } finally {
                 service?.metrics.decActiveWorkers()
+                await rpcMetrics.remove(worker)
                 worker.close()
             }
         },
@@ -152,6 +157,7 @@ runProgram(async () => {
         port: args.port,
         autoAdjustFinalizedHead: args.autoAdjustFinalizedHead
     })
+    service.metrics.setRpcMetricsSource(() => rpcMetrics.collect())
 
     log.info(`listening on port ${service.port}`)
     return waitForInterruption(service)

evm/evm-rpc/src/rpc-client.ts

@@ -1,4 +1,4 @@
-import { RpcClient, RpcError, RpcClientOptions } from '@subsquid/rpc-client'
+import { RetryError, RpcClient, RpcError, RpcClientOptions } from '@subsquid/rpc-client'
 import { HttpError } from '@subsquid/http-client'
 
 export interface EvmRpcClientOptions extends RpcClientOptions {
@@ -48,6 +48,23 @@ export class EvmRpcClient extends RpcClient {
         return false
     }
 
+    getRetryKind(err: Error): string {
+        if (err instanceof RpcError) {
+            if (this.isRpcRateLimitError(err)) return 'rate_limit'
+            if (this.isRpcTransientError(err)) return 'transient'
+            let kind = super.getRetryKind(err)
+            if (kind != 'other') return kind
+            if (this.isRpcInternalError(err)) return 'internal'
+        }
+        if (err instanceof HttpError && err.response.status === 500) {
+            return this.retryInternalServerErrors ? 'internal' : 'transient'
+        }
+        // Thrown here when the endpoint had no usable answer yet: a null or an
+        // error text in place of a result, or a block height it does not serve yet.
+        if (err instanceof RetryError) return 'no_result'
+        return super.getRetryKind(err)
+    }
+
     isRpcInternalError(err: RpcError): boolean {
         return (
             err.code === -32000 || // generic "catch-all" code