[core] much better referer handling

packages/chrome/src/proxy/Controller.ts

@@ -56,6 +56,7 @@ export class Controller {
 						? new URL(data.rawClientUrl)
 						: undefined,
 					rawReferrer: data.rawReferrer,
+					rawReferrerPolicy: data.rawReferrerPolicy,
 					method: data.method,
 					initialHeaders: headers,
 					body: data.body,

packages/chrome/src/proxy/scramjet.ts

@@ -347,7 +347,6 @@ export function createFetchHandler(controller: Controller) {
 		contexts.push(frameContext);
 
 		const initHeaders = htmlcontext.headers ?? [];
-		const history = htmlcontext.history ?? [];
 
 		const injected = `
 			$injectLoad({
@@ -360,7 +359,7 @@ export function createFetchHandler(controller: Controller) {
 				codecDecode: ${codecDecode.toString()},
 				prefix: "${controller.prefix.href}",
 				initHeaders: ${JSON.stringify(initHeaders)},
-				history: ${JSON.stringify(history)},
+				referrer: ${JSON.stringify(htmlcontext.referrer)},
 			});
 			document.querySelectorAll("script[scramjet-injected]").forEach(script => script.remove());
 		`;

packages/inject/src/context.ts

@@ -171,7 +171,9 @@ export class ExecutionContextWrapper {
 					codecDecode: this.init.codecDecode,
 					// TODO: what should be the behavior here? is inheriting correct?
 					initHeaders: this.init.initHeaders,
-					history: this.init.history,
+					// a frame hooked before the proxy serves it anything keeps the
+					// browser's referrer, not this document's
+					referrer: undefined,
 				});
 
 				return context.client;
@@ -186,7 +188,7 @@ export class ExecutionContextWrapper {
 				});
 			},
 			initHeaders: this.init.initHeaders,
-			history: this.init.history,
+			referrer: this.init.referrer,
 		});
 		this.client.hook();
 	}

packages/inject/src/index.ts

@@ -24,7 +24,7 @@ function $injectLoad(init: InjectScramjetInit) {
 		const existing = (globalThis as any)[SCRAMJETCLIENT] as ScramjetClient;
 		existing.syncDocumentInit({
 			initHeaders: init.initHeaders,
-			history: init.history,
+			referrer: init.referrer,
 			cookies: init.cookies,
 		});
 		return;

packages/inject/src/types.d.ts

@@ -1,7 +1,4 @@
-import {
-	ScramjetInterface,
-	TrackedHistoryState,
-} from "@mercuryworkshop/scramjet/bundled";
+import { ScramjetInterface } from "@mercuryworkshop/scramjet/bundled";
 import type { RawHeaders } from "@mercuryworkshop/proxy-transports";
 import type { ThemeDefinition } from "../../chrome/src/themes";
 export type FrameSequence = number[];
@@ -133,5 +130,6 @@ export type InjectScramjetInit = {
 	codecEncode: ScramjetInterface["codecEncode"];
 	codecDecode: ScramjetInterface["codecDecode"];
 	initHeaders: RawHeaders;
-	history: TrackedHistoryState[];
+	/** document.referrer, for a document the proxy served. */
+	referrer?: string;
 };

packages/scramjet/packages/controller/src/index.ts

@@ -19,7 +19,6 @@ import {
 	type ScramjetContext,
 	type ScramjetFlags,
 	type ScramjetInterface,
-	type TrackedHistoryState,
 	Plugin,
 	unrewriteUrl,
 } from "@mercuryworkshop/scramjet";
@@ -321,6 +320,7 @@ export class Controller {
 						: undefined,
 					rawUrl: new URL(data.rawUrl),
 					rawReferrer: data.rawReferrer,
+					rawReferrerPolicy: data.rawReferrerPolicy,
 					rawDestination: data.destination,
 					method: data.method,
 					mode: data.mode,
@@ -707,7 +707,7 @@ function yieldGetInjectScripts(
 						codecEncode: ${codecEncode.toString()},
 						codecDecode: ${codecDecode.toString()},
 						initHeaders: ${JSON.stringify(htmlcontext.headers ?? [])},
-						history: ${JSON.stringify(htmlcontext.history ?? [])},
+						referrer: ${JSON.stringify(htmlcontext.referrer)},
 					})
 				`)
 			),

packages/scramjet/packages/controller/src/inject.ts

@@ -22,7 +22,6 @@ import {
 	type CookieSyncOptions,
 	type ScramjetConfig,
 	type ScramjetContext,
-	type TrackedHistoryState,
 } from "@mercuryworkshop/scramjet";
 
 const MessagePort_postMessage = MessagePort.prototype.postMessage;
@@ -181,14 +180,15 @@ type Init = {
 	codecEncode: (input: string) => string;
 	codecDecode: (input: string) => string;
 	initHeaders: RawHeaders;
-	history: TrackedHistoryState[];
+	/** document.referrer, for a document the proxy served. */
+	referrer?: string;
 };
 
 export function load(init: Init) {
 	if (SCRAMJETCLIENT in globalThis) {
 		((globalThis as any)[SCRAMJETCLIENT] as ScramjetClient).syncDocumentInit({
 			initHeaders: init.initHeaders,
-			history: init.history,
+			referrer: init.referrer,
 			cookies: init.cookies,
 		});
 		return;
@@ -343,11 +343,14 @@ class ExecutionContextWrapper {
 				const context = new ExecutionContextWrapper(frameself, {
 					...this.init,
 					cookies: this.cookieJar.dump(),
+					// a frame hooked before the proxy serves it anything keeps the
+					// browser's referrer, not this document's
+					referrer: undefined,
 				});
 				return context.client;
 			},
 			initHeaders: this.init.initHeaders,
-			history: this.init.history,
+			referrer: this.init.referrer,
 		});
 		const frameInitContext = {
 			window: this.global.window,

packages/scramjet/packages/controller/src/sw.ts

@@ -176,6 +176,7 @@ export async function route(event: FetchEvent): Promise<Response> {
 			{
 				rawUrl: event.request.url,
 				rawReferrer: event.request.referrer,
+				rawReferrerPolicy: event.request.referrerPolicy,
 				destination: event.request.destination,
 				mode: event.request.mode,
 				referrer: event.request.referrer,

packages/scramjet/packages/controller/src/types.d.ts

@@ -10,6 +10,7 @@ export type BodyType =
 export type TransferRequest = {
 	rawUrl: string;
 	rawReferrer: string | null;
+	rawReferrerPolicy?: ReferrerPolicy;
 	destination: RequestDestination;
 	mode: RequestMode;
 	referrer: string;

packages/scramjet/packages/core/src/client/client.ts

@@ -11,6 +11,8 @@ import { createWrapFn } from "@client/shared/wrap";
 import { LifecycleHooks } from "@client/events";
 import {
 	frozenBaseUrl,
+	MAX_REFERRER_LENGTH,
+	referrerFallback,
 	rewriteUrl,
 	RewriteUrlOptions,
 	unrewriteUrl,
@@ -29,11 +31,7 @@ import { AttributeLayer, HTML_NAMESPACE } from "./attributes";
 import { TextLayer } from "./text";
 import { ScramjetConfig } from "@/types";
 import { Tap } from "@/Tap";
-import {
-	type CookieSyncEntry,
-	type CookieSyncOptions,
-	TrackedHistoryState,
-} from "@/fetch";
+import { type CookieSyncEntry, type CookieSyncOptions } from "@/fetch";
 import { AnyFunction } from "@/types";
 import {
 	AsyncFunction_prototype,
@@ -107,7 +105,8 @@ export type ScramjetClientInit = {
 	shouldBlockMessageEvent?: (ev: MessageEvent) => boolean;
 	hookSubcontext: (self: Self, frame?: HTMLIFrameElement) => ScramjetClient;
 	initHeaders: RawHeaders;
-	history: TrackedHistoryState[];
+	/** document.referrer, for a document the proxy served. */
+	referrer?: string;
 };
 
 export type ProxyCtx<
@@ -408,7 +407,12 @@ export class ScramjetClient {
 
 	initHeaders: ScramjetHeaders;
 
-	history: TrackedHistoryState[];
+	/**
+	 * document.referrer as the proxy worked it out when it served the
+	 * document. Unset for one it never served (an initial about:blank, a
+	 * srcdoc, a worker), whose referrer the browser keeps.
+	 */
+	documentReferrer?: string;
 
 	/** Assigned by {@link SingletonBox.registerClient}. */
 	id: string;
@@ -556,7 +560,7 @@ export class ScramjetClient {
 		this.context = init.context;
 		if (init.initHeaders)
 			this.initHeaders = ScramjetHeaders.fromRawHeaders(init.initHeaders);
-		this.history = init.history;
+		this.documentReferrer = init.referrer;
 		this.context.hooks = {
 			rewriter: this.hooks.rewriter,
 		};
@@ -585,6 +589,22 @@ export class ScramjetClient {
 			get origin() {
 				return client.url;
 			},
+			get referrerFallback(): string | undefined {
+				// the length first, which spares every short URL the unrewrite
+				const href = client.global.location.href;
+				// a srcdoc document's referrer is the document it is in
+				// https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer
+				if (href === "about:srcdoc") {
+					const parent = client.parentFrame();
+
+					return typeof parent === "object"
+						? parent.meta.referrerFallback
+						: undefined;
+				}
+				if (href.length <= MAX_REFERRER_LENGTH) return undefined;
+
+				return referrerFallback(href, client.url);
+			},
 			/**
 			 * https://html.spec.whatwg.org/multipage/urls-and-fetching.html#document-base-url -
 			 * the frozen base URL of the first base element with an href: that
@@ -624,34 +644,6 @@ export class ScramjetClient {
 
 				return parent.frameName();
 			},
-			get referrerPolicy(): string | undefined {
-				if (client.initHeaders && client.initHeaders.has("referrer-policy")) {
-					return client.initHeaders.get("referrer-policy");
-				}
-				if (!iswindow) return "";
-				// TODO: need to nullify the actual meta tag so it still sends unsafe-url
-				const nDoc = new client.native.Document(client.global.document);
-				// only the last match counts, so look for it list by list from the
-				// back. Indexed, and not `drain`: a NodeList is a live platform
-				// collection, and spreading it ran the page-replaceable
-				// iteration protocol over what decides the referrer policy
-				let last: Element | undefined;
-				for (const selector of drain([
-					"meta[http-equiv='referrer-policy']",
-					"meta[name='referrer-policy']",
-					"meta[name='referrer']",
-				])) {
-					const list = nDoc.querySelectorAll(selector);
-					last = list[list.length - 1];
-					if (last) break;
-				}
-				if (last) {
-					const nLast = new client.native.HTMLMetaElement(last);
-					return nLast.getAttribute("content");
-				}
-
-				return "";
-			},
 		};
 		this.locationProxy = createLocationProxy(this, global);
 
@@ -661,11 +653,11 @@ export class ScramjetClient {
 	/** Apply document injection init when a client was already installed (e.g. early contentWindow). */
 	syncDocumentInit(init: {
 		initHeaders: RawHeaders;
-		history: TrackedHistoryState[];
+		referrer?: string;
 		cookies?: string;
 	}) {
 		this.initHeaders = ScramjetHeaders.fromRawHeaders(init.initHeaders);
-		this.history = init.history;
+		this.documentReferrer = init.referrer;
 		if (init.cookies !== undefined) {
 			this.context.cookieJar.load(init.cookies);
 		}

packages/scramjet/packages/core/src/client/dom/document.ts

@@ -9,7 +9,7 @@ import {
 	String_toLowerCase,
 	_URL,
 } from "@/shared/snapshot";
-import { createReferrerString } from "@/fetch/util";
+import { SCRAMJETCLIENT } from "@/symbols";
 import { openWindowSteps } from "@client/helpers";
 import { Arguments, Returns, Type } from "@client/webidl";
 import { rewriteAttributeSelectors } from "@client/selectors";
@@ -241,18 +241,29 @@ export default function (client: ScramjetClient, self: Self) {
 		get referrer(): string {
 			// a document with no browsing context has no referrer, whatever the
 			// live one's history says
-			if (!super.defaultView) return "";
-
-			if (!client.history) return "";
-			if (client.history.length < 2) return "";
-			const lastState = client.history[client.history.length - 2];
-			const referrerURL = new _URL(lastState.url);
+			const view = super.defaultView;
+			if (!view) return "";
+
+			// the proxy worked it out when it served the document, and the
+			// document may be another frame's
+			const owner: ScramjetClient | undefined = (view as any)[SCRAMJETCLIENT];
+			const served = (owner ?? client).documentReferrer;
+			if (served !== undefined) return served;
+
+			// one it never served - an initial about:blank, a srcdoc, one that
+			// was written - has the browser's, which points into the proxy
+			const native = super.referrer;
+			if (String_startsWith(native, client.context.prefix.href)) {
+				return client.unrewriteUrl(native);
+			}
+			// or is only the proxy's origin, as a srcdoc's is. the document it
+			// came from is the one this one inherited its origin from
+			if (native === client.context.prefix.origin + "/") {
+				const origin = (owner ?? client).siteOrigin;
+				return origin && origin !== "null" ? origin + "/" : "";
+			}
 
-			return createReferrerString(
-				referrerURL,
-				client.url,
-				lastState.refererPolicy
-			);
+			return "";
 		}
 	});
 

packages/scramjet/packages/core/src/client/shared/incumbency.ts

@@ -51,18 +51,18 @@ export function incumbentClient(client: ScramjetClient): ScramjetClient | null {
 
 /**
  * How many frames an `Intercept` member's body sits above its caller: the body
- * itself, `invoke`, `attemptToCallHandler` and the proxy's `apply` - and the
- * trampoline, with `debugTrampolines` on.
+ * itself, the slot's body and dispatch, `invoke`, the proxy's `apply` - and
+ * the trampoline, with `debugTrampolines` on.
  */
 export const interceptDepth = (client: ScramjetClient) =>
-	client.flagEnabled("debugTrampolines") ? 5 : 4;
+	client.flagEnabled("debugTrampolines") ? 9 : 8;
 
 /**
- * The same for a `client.Proxy` handler: the handler, the proxy's `apply`, and
- * the trampoline between them with `debugTrampolines` on.
+ * The same for a `client.Proxy` handler: the handler, the slot's dispatch and
+ * layer, and the proxy's `apply` - plus its debug trampoline when enabled.
  */
 const proxyDepth = (client: ScramjetClient) =>
-	client.flagEnabled("debugTrampolines") ? 3 : 2;
+	client.flagEnabled("debugTrampolines") ? 6 : 5;
 
 /**
  * The incumbent for the member whose body called this, `depth` frames above