Adds tracker check

api/_common/browser.js

@@ -0,0 +1,34 @@
+import puppeteer from 'puppeteer-core';
+import chromium from '@sparticuz/chromium';
+
+// Serverless launch args, minus the ones that turn off same-origin rules
+const UNSAFE_ARGS = ['--disable-web-security', '--allow-running-insecure-content'];
+const ARGS = chromium.args.filter((arg) => !UNSAFE_ARGS.includes(arg));
+
+// Pick Chromium: CHROME_PATH, puppeteer's copy when self-hosted, or the serverless build
+const chromiumPath = async () => {
+  if (process.env.CHROME_PATH) return process.env.CHROME_PATH;
+  if (process.env.WC_SERVER) return (await import('puppeteer')).default.executablePath();
+  return chromium.executablePath();
+};
+
+// Launch headless Chromium with our default args
+export const launchBrowser = async (options = {}) =>
+  puppeteer.launch({
+    args: ARGS,
+    executablePath: await chromiumPath(),
+    headless: true,
+    acceptInsecureCerts: true,
+    ignoreDefaultArgs: ['--disable-extensions'],
+    ...options,
+  });
+
+// Kill the browser, then let puppeteer tidy up its temp profile
+export const closeBrowser = async (browser) => {
+  browser.process()?.kill('SIGKILL');
+  await browser.close().catch(() => {});
+};
+
+// True when the error means there's no Chromium binary on this host
+export const isBrowserMissing = (error) =>
+  /ENOENT|Browser was not found|Could not find Chromium/i.test(error.message);

api/_common/parse-target.js

@@ -1,3 +1,4 @@
+import net from 'net';
 import psl from 'psl';
 
 // Wrap bare IPv6 in brackets for URL parsing (2+ colons = IPv6)
@@ -30,6 +31,6 @@ export const parseTarget = (input) => {
 };
 
 // Reduce a hostname to its registrable domain, ignoring subdomains
-export const baseDomain = (host) => psl.parse(host)?.domain || host;
+export const baseDomain = (host) => (net.isIP(host) ? host : psl.parse(host)?.domain || host);
 
 export default parseTarget;

api/screenshot.js

@@ -1,11 +1,10 @@
-import puppeteer from 'puppeteer-core';
-import chromium from '@sparticuz/chromium';
 import { randomUUID } from 'crypto';
 import { execFile } from 'child_process';
 import { promises as fs } from 'fs';
 import path from 'path';
 import middleware from './_common/middleware.js';
 import { createLogger } from './_common/logger.js';
+import { launchBrowser, closeBrowser, isBrowserMissing } from './_common/browser.js';
 
 const log = createLogger('screenshot');
 
@@ -45,14 +44,7 @@ const directChromiumScreenshot = async (url) => {
 const puppeteerScreenshot = async (targetUrl) => {
   let browser = null;
   try {
-    browser = await puppeteer.launch({
-      args: [...chromium.args, '--no-sandbox'],
-      defaultViewport: { width: 800, height: 600 },
-      executablePath: process.env.CHROME_PATH || (await chromium.executablePath()),
-      headless: true,
-      acceptInsecureCerts: true,
-      ignoreDefaultArgs: ['--disable-extensions'],
-    });
+    browser = await launchBrowser({ defaultViewport: { width: 800, height: 600 } });
     const page = await browser.newPage();
     await page.emulateMediaFeatures([{ name: 'prefers-color-scheme', value: 'dark' }]);
     page.setDefaultNavigationTimeout(8000);
@@ -65,7 +57,7 @@ const puppeteerScreenshot = async (targetUrl) => {
     const buffer = await page.screenshot();
     return buffer.toString('base64');
   } finally {
-    if (browser) await browser.close().catch(() => {});
+    if (browser) await closeBrowser(browser);
   }
 };
 
@@ -86,7 +78,7 @@ const screenshotHandler = async (targetUrl) => {
   try {
     return { image: await puppeteerScreenshot(targetUrl) };
   } catch (error) {
-    if (/ENOENT|Browser was not found|Could not find Chromium/i.test(error.message)) {
+    if (isBrowserMissing(error)) {
       return { skipped: error.message };
     }
     log.error(`puppeteer screenshot failed: ${error.message}`);

api/tech-stack.js

@@ -1,5 +1,6 @@
 import chromium from '@sparticuz/chromium';
 import middleware from './_common/middleware.js';
+import { isBrowserMissing } from './_common/browser.js';
 
 // Wappalyzer reads CHROMIUM_BIN at module load, so we must resolve
 // the path before importing it (hence the dynamic import in the handler)
@@ -34,7 +35,7 @@ const techStackHandler = async (url) => {
     }
     return results;
   } catch (error) {
-    if (/ENOENT|Browser was not found|Could not find Chromium/i.test(error.message)) {
+    if (isBrowserMissing(error)) {
       return { skipped: error.message };
     }
     throw new Error(error.message);

api/trackers.js

@@ -0,0 +1,144 @@
+import path from 'path';
+import { readFile } from 'fs/promises';
+import trackerdb from '@ghostery/trackerdb';
+import { FiltersEngine, Request } from '@ghostery/adblocker';
+import middleware from './_common/middleware.js';
+import { httpGet, UA } from './_common/http.js';
+import { createLogger } from './_common/logger.js';
+import { baseDomain } from './_common/parse-target.js';
+import { launchBrowser, closeBrowser, isBrowserMissing } from './_common/browser.js';
+
+const log = createLogger('trackers');
+const loadTrackerDB = trackerdb.default || trackerdb;
+const ENGINE_PATH = path.join(
+  process.cwd(),
+  'node_modules/@ghostery/trackerdb/dist/trackerdb.engine',
+);
+const TIMEOUT = 10000;
+const IDLE = { concurrency: 2, timeout: 5000 };
+const MAX_REQUESTS = 1000;
+const EASYPRIVACY_URL = 'https://easylist.to/easylist/easyprivacy.txt';
+const EASYPRIVACY_TTL = 6 * 60 * 60 * 1000;
+let trackerDB = null;
+let easyPrivacy = { expires: 0, filters: null };
+
+// Load Ghostery's tracker database, once per instance
+const getTrackerDB = async () => {
+  trackerDB ??= await loadTrackerDB(await readFile(ENGINE_PATH));
+  return trackerDB;
+};
+
+// Fetch the EasyPrivacy filters, cached for a few hours, falling back to the last copy (if any)
+const getEasyPrivacy = async () => {
+  if (Date.now() < easyPrivacy.expires) return easyPrivacy.filters;
+  try {
+    const { data } = await httpGet(EASYPRIVACY_URL, { timeout: 5000 });
+    if (!String(data).includes('! Title: EasyPrivacy')) throw new Error('Unexpected response');
+    easyPrivacy = { expires: Date.now() + EASYPRIVACY_TTL, filters: FiltersEngine.parse(data) };
+  } catch (error) {
+    log.warn(`EasyPrivacy unavailable: ${error.message}`);
+  }
+  return easyPrivacy.filters;
+};
+
+// True for requests that load the page itself, rather than what it loads
+const isPage = (page, req) => req.isNavigationRequest() && req.frame() === page.mainFrame();
+
+// True for bot check pages (like Cloudflare's or AWS WAF's), which reload once passed
+const isChallenge = (res) => res.status() === 202 || res.headers()['cf-mitigated'] === 'challenge';
+
+// Wait for a bot check to pass and the real page to arrive, else keep the check's response
+const passChallenge = (page, response) => {
+  const isRealPage = (res) => isPage(page, res.request()) && res.ok() && !isChallenge(res);
+  return page.waitForResponse(isRealPage, { timeout: TIMEOUT }).catch(() => response);
+};
+
+// Open the page and record the requests it makes, until the network goes quiet
+const loadPage = async (url) => {
+  const browser = await launchBrowser();
+  try {
+    const page = await browser.newPage();
+    const visit = { pageUrl: url, requests: new Map() };
+    page.on('request', (req) => {
+      if (!req.url().startsWith('http')) return;
+      if (isPage(page, req)) {
+        visit.pageUrl = req.url();
+        visit.requests.clear();
+      } else if (visit.requests.size < MAX_REQUESTS) {
+        visit.requests.set(req.url(), req.resourceType());
+      }
+    });
+    page.on('dialog', (dialog) => dialog.dismiss().catch(() => {}));
+    await page.setUserAgent({ userAgent: UA });
+    let response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: TIMEOUT });
+    if (isChallenge(response)) response = await passChallenge(page, response);
+    await page.waitForNetworkIdle(IDLE).catch(() => {});
+    return { ...visit, response };
+  } finally {
+    await closeBrowser(browser);
+  }
+};
+
+// Add an item to a list, unless it's already there
+const addOnce = (list, item) => list.includes(item) || list.push(item);
+
+// Shape a TrackerDB match into the fields the card shows
+const toTracker = ({ pattern, category, organization }) => ({
+  name: pattern.name,
+  category: category.name,
+  company: organization?.name || pattern.name,
+  country: organization?.country || null,
+  privacyPolicy: organization?.privacy_policy_url || null,
+  hosts: [],
+  requests: 0,
+});
+
+// Group requests by the tracker they match, then list unnamed tracking and unknown hosts apart
+const findTrackers = (pageUrl, requests, db, filters) => {
+  const site = baseDomain(new URL(pageUrl).hostname);
+  const trackers = {};
+  const otherTrackers = [];
+  const unknown = [];
+  for (const [url, type] of requests) {
+    const { hostname } = new URL(url);
+    const thirdParty = baseDomain(hostname) !== site;
+    const request = { url, type, sourceUrl: pageUrl };
+    const matches = db.matchUrl(request, { getDomainMetadata: thirdParty });
+    for (const match of matches) {
+      const tracker = (trackers[match.pattern.key] ??= toTracker(match));
+      tracker.requests++;
+      addOnce(tracker.hosts, hostname);
+    }
+    if (matches.length) continue;
+    if (filters?.match(Request.fromRawDetails(request)).match) addOnce(otherTrackers, hostname);
+    else if (thirdParty) addOnce(unknown, hostname);
+  }
+  return {
+    trackers: Object.values(trackers).sort(
+      (a, b) => a.category.localeCompare(b.category) || a.name.localeCompare(b.name),
+    ),
+    otherTrackers: otherTrackers.sort(),
+    unknown: unknown.filter((host) => !otherTrackers.includes(host)).sort(),
+  };
+};
+
+// List the trackers and other third parties a page loads
+const trackersHandler = async (url) => {
+  const db = await getTrackerDB();
+  try {
+    const [filters, { response, pageUrl, requests }] = await Promise.all([
+      getEasyPrivacy(),
+      loadPage(url),
+    ]);
+    if (!response.ok() || isChallenge(response)) {
+      return { error: `Page returned HTTP ${response.status()}, so couldn't be checked` };
+    }
+    return { requests: requests.size, ...findTrackers(pageUrl, requests, db, filters) };
+  } catch (error) {
+    if (isBrowserMissing(error)) return { skipped: error.message };
+    throw error;
+  }
+};
+
+export const handler = middleware(trackersHandler);
+export default handler;

netlify.toml

@@ -18,6 +18,10 @@
   # GOOGLE_CLOUD_API_KEY=''     # Google Cloud API key, for the quality and Safe Browsing checks
   # SHODAN_API_KEY=''           # Shodan API key, for host names, server info and vulnerabilities
 
+# Ship the tracker database with the function that reads it
+[functions.trackers]
+  included_files = ["node_modules/@ghostery/trackerdb/dist/trackerdb.engine"]
+
 # Rewrite the /api/* path to the lambda functions
 [[redirects]]
   from = "/api/*"

package.json

@@ -31,6 +31,8 @@
     "@fortawesome/free-regular-svg-icons": "^7.3.1",
     "@fortawesome/free-solid-svg-icons": "^7.3.1",
     "@fortawesome/svelte-fontawesome": "^0.2.4",
+    "@ghostery/adblocker": "2.17.3",
+    "@ghostery/trackerdb": "^1.0.951",
     "@sparticuz/chromium": "^148.0.0",
     "@types/react": "^19.3.0",
     "@types/react-dom": "^19.3.0",

public/resources/openapi-spec.yml

@@ -2992,6 +2992,87 @@ paths:
           $ref: '#/components/responses/Unauthorized'
         '429':
           $ref: '#/components/responses/TooManyRequests'
+  /trackers:
+    get:
+      summary: List the trackers and other third parties a page loads, via TrackerDB and EasyPrivacy
+      tags:
+        - Client-Side Information
+      parameters:
+        - name: url
+          in: query
+          required: true
+          description: The URL to fetch results about
+          schema:
+            type: string
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  requests:
+                    type: integer
+                    description: Unique requests the page made while loading
+                    example: 170
+                  trackers:
+                    type: array
+                    description: Known third parties, grouped by service, sorted by category
+                    items:
+                      type: object
+                      properties:
+                        name:
+                          type: string
+                          example: 'Google Analytics'
+                        category:
+                          type: string
+                          description: What the service does, as classified by TrackerDB
+                          example: 'Site Analytics'
+                        company:
+                          type: string
+                          description: The organisation behind the service
+                          example: 'Google'
+                        country:
+                          type: string
+                          nullable: true
+                          description: Two-letter code of the country the company is based in
+                          example: 'US'
+                        privacyPolicy:
+                          type: string
+                          nullable: true
+                          description: Link to the company's privacy policy
+                          example: 'https://policies.google.com/privacy'
+                        hosts:
+                          type: array
+                          description: Hostnames the page contacted for this service
+                          items:
+                            type: string
+                          example: ['www.google-analytics.com']
+                        requests:
+                          type: integer
+                          description: Number of requests matched to this service
+                          example: 3
+                  otherTrackers:
+                    type: array
+                    description: Hosts not named by TrackerDB, but with requests flagged by EasyPrivacy
+                    items:
+                      type: string
+                    example: ['a1.api.bbc.co.uk']
+                  unknown:
+                    type: array
+                    description: Third-party hosts not recognised by either list
+                    items:
+                      type: string
+                    example: ['cdn.example.net']
+        '204':
+          $ref: '#/components/responses/Skipped'
+        '500':
+          $ref: '#/components/responses/Error'
+        '400':
+          $ref: '#/components/responses/MissingParam'
+        '429':
+          $ref: '#/components/responses/TooManyRequests'
   /txt-records:
     get:
       summary: Retrieve the TXT records for a specified domain

src/client/analysis/registry.ts

@@ -13,6 +13,7 @@ import breaches from './rules/breaches';
 import blockLists from './rules/block-lists';
 import firewall from './rules/firewall';
 import cookies from './rules/cookies';
+import trackers from './rules/trackers';
 import headers from './rules/headers';
 import ports from './rules/ports';
 import mailConfig from './rules/mail-config';
@@ -40,6 +41,7 @@ export const analyzers: Partial<Record<CheckId, Analyzer>> = {
   'block-lists': blockLists,
   firewall,
   cookies,
+  trackers,
   headers,
   ports,
   'mail-config': mailConfig,