diff --git a/CHANGELOG.md b/CHANGELOG.md index cc6c0feb9..d41e2daa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,22 @@ All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased [major] + +> Development of this release was supported by [User Rights](https://www.user-rights.org). + +### Changed + +- **Breaking:** Set `navigator.language` and `navigator.languages` in the full DOM fetcher strictly from the tags in `@opentermsarchive/engine.fetcher.language` through the stealth evasions registered at browser launch rather than a per-page CDP override + +### Removed + +- **Breaking:** Drop support for quality factors (`;q=…`) in `@opentermsarchive/engine.fetcher.language`; previously accepted values such as `en-IE,en-GB;q=0.9,en;q=0.8` now throw at launch. Replace with a plain comma-separated priority list, e.g. `en-IE,en-GB,en` + +### Fixed + +- Reactivate the `puppeteer-extra` stealth plugin in the full DOM fetcher, which had been silently inert since v10.3.1 because it was registered after `puppeteer.launch()`, restoring removal of `navigator.webdriver` and `HeadlessChrome` from the user agent + ## 12.0.2 - 2026-05-26 > Development of this release was supported by [User Rights](https://www.user-rights.org). diff --git a/package-lock.json b/package-lock.json index 1dc3be610..9a691ae50 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,7 +50,7 @@ "morgan": "^1.10.1", "node-fetch": "^3.3.2", "octokit": "^5.0.3", - "puppeteer": "^24.22.0", + "puppeteer": "^25.1.0", "puppeteer-extra": "^3.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2", "simple-git": "^3.28.0", @@ -110,7 +110,6 @@ "resolved": "https://registry.npmjs.org/@accordproject/concerto-cto/-/concerto-cto-4.1.3.tgz", "integrity": "sha512-GNQMBwHiubNP1w4u0eE3Zu97fUTQHJ2ahMAj3F3/X7gLPbXGYYFQGag4hk9Z58UW/bRks4cgxtUJsWhaFQrFQw==", "license": "Apache-2.0", - "peer": true, "dependencies": { "@accordproject/concerto-metamodel": "^3.13.0", "@accordproject/concerto-util": "4.1.3", @@ -128,7 +127,6 @@ "resolved": "https://registry.npmjs.org/@accordproject/concerto-util/-/concerto-util-4.1.3.tgz", "integrity": "sha512-5Y1fSTqvjB7QOUpy1ivR+Ih7TtrYCGomyrRruWzIP4Xq2he86u2YhKm4aBjrN3SpQRHX6BnGl/GGX0yTTRwVNg==", "license": "Apache-2.0", - "peer": true, "dependencies": { "@supercharge/promise-pool": "1.7.0", "colors": "1.4.0", @@ -145,7 +143,6 @@ "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "license": "MIT", - "peer": true, "dependencies": { "ms": "2.1.2" }, @@ -162,15 +159,13 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@accordproject/concerto-core/node_modules/yaml": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", "license": "ISC", - "peer": true, "bin": { "yaml": "bin.mjs" }, @@ -210,7 +205,6 @@ "resolved": "https://registry.npmjs.org/@accordproject/concerto-metamodel/-/concerto-metamodel-3.13.1.tgz", "integrity": "sha512-af8mpDuHGSfBH+VlVRwBqhiBVrLI0srRslT44Q4hkZJJHHnxU5K5U2wznGamazP1FMXLnOwXiFG4ZMudAll7Vw==", "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=14", "npm": ">=6" @@ -831,6 +825,7 @@ "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", + "dev": true, "license": "MIT", "dependencies": { "@babel/helper-validator-identifier": "^7.27.1", @@ -845,6 +840,7 @@ "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", + "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -1207,6 +1203,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=20.19.0" }, @@ -1253,6 +1250,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=20.19.0" } @@ -1571,6 +1569,7 @@ "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", "devOptional": true, "license": "MIT", + "peer": true, "engines": { "node": "^14.21.3 || >=16" }, @@ -1724,6 +1723,7 @@ "resolved": "https://registry.npmjs.org/@octokit/core/-/core-7.0.4.tgz", "integrity": "sha512-jOT8V1Ba5BdC79sKrRWDdMT5l1R+XNHTPR6CPWzUP2EcfAcvIHZWF0eAbmRcpOOP5gVIwnqNg0C4nvh6Abc3OA==", "license": "MIT", + "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.1", @@ -2037,55 +2037,29 @@ } }, "node_modules/@puppeteer/browsers": { - "version": "2.10.10", - "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.10.10.tgz", - "integrity": "sha512-3ZG500+ZeLql8rE0hjfhkycJjDj0pI/btEh3L9IkWUYcOrgP0xCNRq3HbtbqOPbvDhFaAWD88pDFtlLv8ns8gA==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-3.0.4.tgz", + "integrity": "sha512-HGM8iAmGTf+Y7t0373szVbTmt3d7vPkYL/1bpOkOFO0YUYLgSeuYBCzESklogNPvOBnZ/MRD5f07OkpqH1trtA==", "license": "Apache-2.0", "dependencies": { - "debug": "^4.4.3", - "extract-zip": "^2.0.1", - "progress": "^2.0.3", - "proxy-agent": "^6.5.0", - "semver": "^7.7.2", - "tar-fs": "^3.1.0", + "modern-tar": "^0.7.6", "yargs": "^17.7.2" }, "bin": { - "browsers": "lib/cjs/main-cli.js" + "browsers": "lib/main-cli.js" }, "engines": { - "node": ">=18" - } - }, - "node_modules/@puppeteer/browsers/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" + "node": ">=22.12.0" }, - "engines": { - "node": ">=6.0" + "peerDependencies": { + "proxy-agent": ">=8.0.1" }, "peerDependenciesMeta": { - "supports-color": { + "proxy-agent": { "optional": true } } }, - "node_modules/@puppeteer/browsers/node_modules/semver": { - "version": "7.7.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", - "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/@rtsao/scc": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@rtsao/scc/-/scc-1.1.0.tgz", @@ -2149,12 +2123,6 @@ "node": ">=8" } }, - "node_modules/@tootallnate/quickjs-emscripten": { - "version": "0.23.0", - "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", - "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", - "license": "MIT" - }, "node_modules/@types/aws-lambda": { "version": "8.10.152", "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.152.tgz", @@ -2224,7 +2192,8 @@ "version": "20.7.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.7.0.tgz", "integrity": "sha512-zI22/pJW2wUZOVyguFaUL1HABdmSVxpXrzIqkjsHmyUjNhPoWM1CKfvVuXfetHhIok4RY573cqS0mZ1SJEnoTg==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/triple-beam": { "version": "1.3.5", @@ -2247,16 +2216,6 @@ "@types/webidl-conversions": "*" } }, - "node_modules/@types/yauzl": { - "version": "2.10.3", - "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", - "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", - "license": "MIT", - "optional": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@typescript-eslint/types": { "version": "8.44.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.44.0.tgz", @@ -2315,6 +2274,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2351,6 +2311,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -2384,7 +2345,6 @@ "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3" }, @@ -2612,18 +2572,6 @@ "dev": true, "license": "MIT" }, - "node_modules/ast-types": { - "version": "0.13.4", - "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", - "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", - "license": "MIT", - "dependencies": { - "tslib": "^2.0.1" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -2696,83 +2644,6 @@ "license": "Apache-2.0", "optional": true }, - "node_modules/bare-fs": { - "version": "4.4.4", - "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.4.4.tgz", - "integrity": "sha512-Q8yxM1eLhJfuM7KXVP3zjhBvtMJCYRByoTT+wHXjpdMELv0xICFJX+1w4c7csa+WZEOsq4ItJ4RGwvzid6m/dw==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "bare-events": "^2.5.4", - "bare-path": "^3.0.0", - "bare-stream": "^2.6.4", - "bare-url": "^2.2.2", - "fast-fifo": "^1.3.2" - }, - "engines": { - "bare": ">=1.16.0" - }, - "peerDependencies": { - "bare-buffer": "*" - }, - "peerDependenciesMeta": { - "bare-buffer": { - "optional": true - } - } - }, - "node_modules/bare-os": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.6.2.tgz", - "integrity": "sha512-T+V1+1srU2qYNBmJCXZkUY5vQ0B4FSlL3QDROnKQYOqeiQR8UbjNHlPa+TIbM4cuidiN9GaTaOZgSEgsvPbh5A==", - "license": "Apache-2.0", - "optional": true, - "engines": { - "bare": ">=1.14.0" - } - }, - "node_modules/bare-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", - "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "bare-os": "^3.0.1" - } - }, - "node_modules/bare-stream": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.7.0.tgz", - "integrity": "sha512-oyXQNicV1y8nc2aKffH+BUHFRXmx6VrPzlnaEvMhram0nPBrKcEdcyBg5r08D0i8VxngHFAiVyn1QKXpSG0B8A==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "streamx": "^2.21.0" - }, - "peerDependencies": { - "bare-buffer": "*", - "bare-events": "*" - }, - "peerDependenciesMeta": { - "bare-buffer": { - "optional": true - }, - "bare-events": { - "optional": true - } - } - }, - "node_modules/bare-url": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.2.2.tgz", - "integrity": "sha512-g+ueNGKkrjMazDG3elZO1pNs3HY5+mMmOet1jtKyhOaCnkLzitxf26z7hoAEkDNgdNmnc1KIlt/dw6Po6xZMpA==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "bare-path": "^3.0.0" - } - }, "node_modules/base64-js": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", @@ -2797,15 +2668,6 @@ "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "license": "MIT" }, - "node_modules/basic-ftp": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", - "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", - "license": "MIT", - "engines": { - "node": ">=10.0.0" - } - }, "node_modules/before-after-hook": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-4.0.0.tgz", @@ -3049,6 +2911,7 @@ "resolved": "https://registry.npmjs.org/chai/-/chai-6.0.1.tgz", "integrity": "sha512-/JOoU2//6p5vCXh00FpNgtlw0LjvhGttaWc+y7wpW9yjBm3ys0dI8tSKZxIOgNruz5J0RleccatSIC3uxEZP0g==", "license": "MIT", + "peer": true, "engines": { "node": ">=18" } @@ -3145,14 +3008,17 @@ } }, "node_modules/chromium-bidi": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-8.0.0.tgz", - "integrity": "sha512-d1VmE0FD7lxZQHzcDUCKZSNRtRwISXDsdg4HjdTR5+Ll5nQ/vzU12JeNmupD6VWffrPSlrnGhEWlLESKH3VO+g==", + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-16.0.1.tgz", + "integrity": "sha512-J63PGu/9PpeCwLIcKYyzWP6yaVL5pxuBc0shlYCYM8BaAkmlwiQboXO1iNbOgSDbVklEyYFfNEcHD8oOAWacUA==", "license": "Apache-2.0", "dependencies": { "mitt": "^3.0.1", "zod": "^3.24.1" }, + "engines": { + "node": ">=20.19.0 <22.0.0 || >=22.12.0" + }, "peerDependencies": { "devtools-protocol": "*" } @@ -3508,7 +3374,9 @@ "version": "9.0.0", "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.0.tgz", "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", + "dev": true, "license": "MIT", + "peer": true, "dependencies": { "env-paths": "^2.2.1", "import-fresh": "^3.3.0", @@ -3768,8 +3636,7 @@ "version": "1.11.13", "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.13.tgz", "integrity": "sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/debug": { "version": "4.3.7", @@ -3875,20 +3742,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/degenerator": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", - "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", - "license": "MIT", - "dependencies": { - "ast-types": "^0.13.4", - "escodegen": "^2.1.0", - "esprima": "^4.0.1" - }, - "engines": { - "node": ">= 14" - } - }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -3926,10 +3779,11 @@ } }, "node_modules/devtools-protocol": { - "version": "0.0.1495869", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1495869.tgz", - "integrity": "sha512-i+bkd9UYFis40RcnkW7XrOprCujXRAHg62IVh/Ah3G8MmNXpCGt1m0dTFhSdx/AVs8XEMbdOGRwdkR1Bcta8AA==", - "license": "BSD-3-Clause" + "version": "0.0.1624250", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1624250.tgz", + "integrity": "sha512-YFAat/lOiIk0ARmBweG+ygrEcbZrq5B9urRyUoeQKp53MlidHXE2TmTbxKcaXoQj7u/aX+jebDO4BW55rs0WwA==", + "license": "BSD-3-Clause", + "peer": true }, "node_modules/dezalgo": { "version": "1.0.4", @@ -4186,15 +4040,6 @@ "node": ">=0.10.0" } }, - "node_modules/end-of-stream": { - "version": "1.4.5", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", - "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", - "license": "MIT", - "dependencies": { - "once": "^1.4.0" - } - }, "node_modules/entities": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", @@ -4211,6 +4056,7 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -4220,6 +4066,7 @@ "version": "1.3.4", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", + "dev": true, "license": "MIT", "dependencies": { "is-arrayish": "^0.2.1" @@ -4394,33 +4241,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/escodegen": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", - "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", - "license": "BSD-2-Clause", - "dependencies": { - "esprima": "^4.0.1", - "estraverse": "^5.2.0", - "esutils": "^2.0.2" - }, - "bin": { - "escodegen": "bin/escodegen.js", - "esgenerate": "bin/esgenerate.js" - }, - "engines": { - "node": ">=6.0" - }, - "optionalDependencies": { - "source-map": "~0.6.1" - } - }, "node_modules/eslint": { "version": "8.57.1", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -4562,6 +4389,7 @@ "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.32.0.tgz", "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "license": "MIT", + "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -4917,19 +4745,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/esquery": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", @@ -5058,26 +4873,6 @@ } } }, - "node_modules/extract-zip": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", - "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", - "license": "BSD-2-Clause", - "dependencies": { - "debug": "^4.1.1", - "get-stream": "^5.1.0", - "yauzl": "^2.10.0" - }, - "bin": { - "extract-zip": "cli.js" - }, - "engines": { - "node": ">= 10.17.0" - }, - "optionalDependencies": { - "@types/yauzl": "^2.9.1" - } - }, "node_modules/fast-content-type-parse": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/fast-content-type-parse/-/fast-content-type-parse-3.0.0.tgz", @@ -5178,15 +4973,6 @@ "reusify": "^1.0.4" } }, - "node_modules/fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", - "license": "MIT", - "dependencies": { - "pend": "~1.2.0" - } - }, "node_modules/fecha": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", @@ -5597,21 +5383,6 @@ "node": ">= 0.4" } }, - "node_modules/get-stream": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", - "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", - "license": "MIT", - "dependencies": { - "pump": "^3.0.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/get-symbol-description": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.1.0.tgz", @@ -5629,29 +5400,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/get-uri": { - "version": "6.0.5", - "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz", - "integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==", - "license": "MIT", - "dependencies": { - "basic-ftp": "^5.0.2", - "data-uri-to-buffer": "^6.0.2", - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/get-uri/node_modules/data-uri-to-buffer": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", - "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, "node_modules/git-hooks-list": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/git-hooks-list/-/git-hooks-list-1.0.3.tgz", @@ -6193,6 +5941,7 @@ "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.0.1.tgz", "integrity": "sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA==", "license": "MIT", + "optional": true, "engines": { "node": ">= 12" } @@ -6243,6 +5992,7 @@ "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true, "license": "MIT" }, "node_modules/is-async-function": { @@ -6788,6 +6538,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, "license": "MIT" }, "node_modules/js-yaml": { @@ -7044,6 +6795,7 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true, "license": "MIT" }, "node_modules/json-schema-traverse": { @@ -7213,6 +6965,18 @@ "node": ">= 0.8.0" } }, + "node_modules/lilconfig": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", + "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", + "license": "MIT", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antonk52" + } + }, "node_modules/line-column": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/line-column/-/line-column-1.0.2.tgz", @@ -7227,6 +6991,7 @@ "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true, "license": "MIT" }, "node_modules/linkify-it": { @@ -7786,6 +7551,15 @@ "node": ">=8" } }, + "node_modules/modern-tar": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/modern-tar/-/modern-tar-0.7.6.tgz", + "integrity": "sha512-sweCIVXzx1aIGTCdzcMlSZt1h8k5Tmk08VNAuRk3IU28XamGiOH5ypi11g6De2CH7PhYqSSnGy2A/EFhbWnVKg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/mongodb": { "version": "6.19.0", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.19.0.tgz", @@ -7952,15 +7726,6 @@ "node": ">= 0.6" } }, - "node_modules/netmask": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", - "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/nock": { "version": "14.0.10", "resolved": "https://registry.npmjs.org/nock/-/nock-14.0.10.tgz", @@ -8361,38 +8126,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/pac-proxy-agent": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", - "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", - "license": "MIT", - "dependencies": { - "@tootallnate/quickjs-emscripten": "^0.23.0", - "agent-base": "^7.1.2", - "debug": "^4.3.4", - "get-uri": "^6.0.1", - "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.6", - "pac-resolver": "^7.0.1", - "socks-proxy-agent": "^8.0.5" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/pac-resolver": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", - "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", - "license": "MIT", - "dependencies": { - "degenerator": "^5.0.0", - "netmask": "^2.0.2" - }, - "engines": { - "node": ">= 14" - } - }, "node_modules/package-json-from-dist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", @@ -8430,6 +8163,7 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.0.0", @@ -8645,12 +8379,6 @@ "node": ">=0.10" } }, - "node_modules/pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", - "license": "MIT" - }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -8707,15 +8435,6 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "license": "MIT" }, - "node_modules/progress": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/propagate": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz", @@ -8739,50 +8458,12 @@ "node": ">= 0.10" } }, - "node_modules/proxy-agent": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", - "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "^4.3.4", - "http-proxy-agent": "^7.0.1", - "https-proxy-agent": "^7.0.6", - "lru-cache": "^7.14.1", - "pac-proxy-agent": "^7.1.0", - "proxy-from-env": "^1.1.0", - "socks-proxy-agent": "^8.0.5" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/proxy-agent/node_modules/lru-cache": { - "version": "7.18.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", - "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", "license": "MIT" }, - "node_modules/pump": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", - "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", - "license": "MIT", - "dependencies": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -8802,59 +8483,42 @@ } }, "node_modules/puppeteer": { - "version": "24.22.0", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-24.22.0.tgz", - "integrity": "sha512-QabGIvu7F0hAMiKGHZCIRHMb6UoH0QAJA2OaqxEU2tL5noXPrxUcotg2l3ttOA4p1PFnVIGkr6PXRAWlM2evVQ==", + "version": "25.1.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-25.1.0.tgz", + "integrity": "sha512-7L6/0JM7XStK99lIL4xQySyNEXNfII6pk0BxkI5kKBTOhR7AsoQiv067YTsE/rIXxQiq9ajlO4WcqBjS/FWK1A==", "hasInstallScript": true, "license": "Apache-2.0", + "peer": true, "dependencies": { - "@puppeteer/browsers": "2.10.10", - "chromium-bidi": "8.0.0", - "cosmiconfig": "^9.0.0", - "devtools-protocol": "0.0.1495869", - "puppeteer-core": "24.22.0", - "typed-query-selector": "^2.12.0" + "@puppeteer/browsers": "3.0.4", + "chromium-bidi": "16.0.1", + "devtools-protocol": "0.0.1624250", + "lilconfig": "^3.1.3", + "puppeteer-core": "25.1.0", + "typed-query-selector": "^2.12.2" }, "bin": { - "puppeteer": "lib/cjs/puppeteer/node/cli.js" + "puppeteer": "lib/puppeteer/node/cli.js" }, "engines": { - "node": ">=18" + "node": ">=22.12.0" } }, "node_modules/puppeteer-core": { - "version": "24.22.0", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.22.0.tgz", - "integrity": "sha512-oUeWlIg0pMz8YM5pu0uqakM+cCyYyXkHBxx9di9OUELu9X9+AYrNGGRLK9tNME3WfN3JGGqQIH3b4/E9LGek/w==", + "version": "25.1.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-25.1.0.tgz", + "integrity": "sha512-jKzy5y4WG6uNuFbTWgW1D7mqoT9o0nllc/6a1DGF775T1mPmgw3scdFEtEq67yVFikavQmbYq6NLfbTfxHSlqQ==", "license": "Apache-2.0", "dependencies": { - "@puppeteer/browsers": "2.10.10", - "chromium-bidi": "8.0.0", - "debug": "^4.4.3", - "devtools-protocol": "0.0.1495869", - "typed-query-selector": "^2.12.0", - "webdriver-bidi-protocol": "0.2.11", - "ws": "^8.18.3" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/puppeteer-core/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" + "@puppeteer/browsers": "3.0.4", + "chromium-bidi": "16.0.1", + "devtools-protocol": "0.0.1624250", + "typed-query-selector": "^2.12.2", + "webdriver-bidi-protocol": "0.4.2", + "ws": "^8.21.0" }, "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } + "node": ">=22.12.0" } }, "node_modules/puppeteer-extra": { @@ -8862,6 +8526,7 @@ "resolved": "https://registry.npmjs.org/puppeteer-extra/-/puppeteer-extra-3.3.6.tgz", "integrity": "sha512-rsLBE/6mMxAjlLd06LuGacrukP2bqbzKCLzV1vrhHFavqQE/taQ2UXv3H5P0Ls7nsrASa+6x3bDbXHpqMwq+7A==", "license": "MIT", + "peer": true, "dependencies": { "@types/debug": "^4.1.0", "debug": "^4.1.1", @@ -9252,8 +8917,7 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.4.1.tgz", "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/rimraf": { "version": "3.0.2", @@ -9470,7 +9134,6 @@ "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.3.3.tgz", "integrity": "sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA==", "license": "MIT", - "peer": true, "dependencies": { "@types/json-schema": "^7.0.9", "ajv": "^8.9.0", @@ -9490,7 +9153,6 @@ "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", "license": "MIT", - "peer": true, "dependencies": { "ajv": "^8.0.0" }, @@ -9857,39 +9519,12 @@ "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", "license": "MIT", + "optional": true, "engines": { "node": ">= 6.0.0", "npm": ">= 3.0.0" } }, - "node_modules/socks": { - "version": "2.8.7", - "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", - "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", - "license": "MIT", - "dependencies": { - "ip-address": "^10.0.1", - "smart-buffer": "^4.2.0" - }, - "engines": { - "node": ">= 10.0.0", - "npm": ">= 3.0.0" - } - }, - "node_modules/socks-proxy-agent": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", - "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "^4.3.4", - "socks": "^2.8.3" - }, - "engines": { - "node": ">= 14" - } - }, "node_modules/sort-object-keys": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/sort-object-keys/-/sort-object-keys-1.1.3.tgz", @@ -9913,16 +9548,6 @@ "sort-package-json": "cli.js" } }, - "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "license": "BSD-3-Clause", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -10385,20 +10010,6 @@ "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", "license": "MIT" }, - "node_modules/tar-fs": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.1.tgz", - "integrity": "sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg==", - "license": "MIT", - "dependencies": { - "pump": "^3.0.0", - "tar-stream": "^3.1.5" - }, - "optionalDependencies": { - "bare-fs": "^4.0.1", - "bare-path": "^3.0.0" - } - }, "node_modules/tar-stream": { "version": "3.1.7", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", @@ -10576,12 +10187,6 @@ "json5": "lib/cli.js" } }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" - }, "node_modules/turndown": { "version": "7.2.1", "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.1.tgz", @@ -10719,16 +10324,16 @@ } }, "node_modules/typed-query-selector": { - "version": "2.12.0", - "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.0.tgz", - "integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==", + "version": "2.12.2", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.2.tgz", + "integrity": "sha512-EOPFbyIub4ngnEdqi2yOcNeDLaX/0jcE1JoAXQDDMIthap7FoN795lc/SHfIq2d416VufXpM8z/lD+WRm2gfOQ==", "license": "MIT" }, "node_modules/typescript": { "version": "5.9.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.2.tgz", "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "peer": true, "bin": { @@ -10865,7 +10470,6 @@ "https://github.com/sponsors/ctavan" ], "license": "MIT", - "peer": true, "bin": { "uuid": "dist/esm/bin/uuid" } @@ -10910,9 +10514,9 @@ } }, "node_modules/webdriver-bidi-protocol": { - "version": "0.2.11", - "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.2.11.tgz", - "integrity": "sha512-Y9E1/oi4XMxcR8AT0ZC4OvYntl34SPgwjmELH+owjBr0korAX4jKgZULBWILGCVGdVCQ0dodTToIETozhG8zvA==", + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.2.tgz", + "integrity": "sha512-VSV+fzfChirL3e7jay2yUC7B4HQCGtEWEg/MSSQbK+qWbqeGlRLlXTzPpYr3XGUvbpDHumWZBJxgesg4N7dbtA==", "license": "Apache-2.0" }, "node_modules/webidl-conversions": { @@ -11294,9 +10898,9 @@ "license": "ISC" }, "node_modules/ws": { - "version": "8.18.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", - "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", "license": "MIT", "engines": { "node": ">=10.0.0" @@ -11439,25 +11043,6 @@ "node": ">=8" } }, - "node_modules/yauzl": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", - "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", - "license": "MIT", - "dependencies": { - "buffer-crc32": "~0.2.3", - "fd-slicer": "~1.1.0" - } - }, - "node_modules/yauzl/node_modules/buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "license": "MIT", - "engines": { - "node": "*" - } - }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index 919b80527..a5d0dd0cd 100644 --- a/package.json +++ b/package.json @@ -92,7 +92,7 @@ "morgan": "^1.10.1", "node-fetch": "^3.3.2", "octokit": "^5.0.3", - "puppeteer": "^24.22.0", + "puppeteer": "^25.1.0", "puppeteer-extra": "^3.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2", "simple-git": "^3.28.0", diff --git a/scripts/declarations/validate/index.mocha.js b/scripts/declarations/validate/index.mocha.js index 3962fb41b..4d0b59e70 100644 --- a/scripts/declarations/validate/index.mocha.js +++ b/scripts/declarations/validate/index.mocha.js @@ -51,9 +51,11 @@ export default async options => { const filePath = path.join(declarationsPath, `${serviceId}.json`); const historyFilePath = path.join(declarationsPath, `${serviceId}.history.json`); - before(launchHeadlessBrowser); + if (!schemaOnly) { + before(() => launchHeadlessBrowser(config.get('@opentermsarchive/engine.fetcher.language'))); - after(stopHeadlessBrowser); + after(stopHeadlessBrowser); + } context(serviceId, () => { before(function () { diff --git a/src/archivist/fetcher/fullDomFetcher.js b/src/archivist/fetcher/fullDomFetcher.js index bef0e6afc..45f9d040f 100644 --- a/src/archivist/fetcher/fullDomFetcher.js +++ b/src/archivist/fetcher/fullDomFetcher.js @@ -1,13 +1,31 @@ import puppeteer from 'puppeteer-extra'; import stealthPlugin from 'puppeteer-extra-plugin-stealth'; +import navigatorLanguages from 'puppeteer-extra-plugin-stealth/evasions/navigator.languages/index.js'; +import userAgentOverride from 'puppeteer-extra-plugin-stealth/evasions/user-agent-override/index.js'; import { resolveProxyConfiguration, extractProxyCredentials } from './proxyUtils.js'; let browser; -export default async function fetch(url, cssSelectors, config) { - puppeteer.use(stealthPlugin({ locale: config.language })); +export function parseLanguage(value) { + if (typeof value !== 'string') { + throw new Error(`Fetcher language must be a string; received ${JSON.stringify(value)}. Pass the value of "@opentermsarchive/engine.fetcher.language" explicitly.`); + } + + if (value.toLowerCase().includes(';q=')) { + throw new Error(`Quality factors are not supported in fetcher language configuration; received "${value}". Provide a comma-separated list of BCP 47 tags in priority order, for example "en-IE,en-GB,en".`); + } + + const languages = value.split(',').map(tag => tag.trim()).filter(Boolean); + + if (!languages.length) { + throw new Error(`Fetcher language must contain at least one tag; received "${value}".`); + } + return { locale: languages.join(','), languages }; +} + +export default async function fetch(url, cssSelectors, config) { if (!browser) { throw new Error('The headless browser should be controlled manually with "launchHeadlessBrowser" and "stopHeadlessBrowser".'); } @@ -21,7 +39,7 @@ export default async function fetch(url, cssSelectors, config) { page = await context.newPage(); client = await page.createCDPSession(); - await configurePage(page, client, config); + await configurePage(page, config); const selectors = [].concat(cssSelectors).filter(Boolean); @@ -97,14 +115,24 @@ export default async function fetch(url, cssSelectors, config) { /** * Launches a headless browser instance using Puppeteer if one is not already running. Returns the existing browser instance if one is already running, otherwise creates and returns a new instance. * @function launchHeadlessBrowser - * @returns {Promise} The Puppeteer browser instance. + * @param {string} language Accept-Language header value applied to the browser context + * @returns {Promise} The Puppeteer browser instance. * @async */ -export async function launchHeadlessBrowser() { +export async function launchHeadlessBrowser(language) { if (browser) { return browser; } + const { locale, languages } = parseLanguage(language); + const stealth = stealthPlugin(); + + stealth.enabledEvasions.delete('user-agent-override'); + stealth.enabledEvasions.delete('navigator.languages'); + puppeteer.use(stealth); + puppeteer.use(userAgentOverride({ locale })); + puppeteer.use(navigatorLanguages({ languages })); + const options = { args: [], headless: !process.env.OTA_ENGINE_FETCHER_NO_HEADLESS, @@ -156,16 +184,9 @@ function isValidHttpStatus(status) { return (status >= 200 && status < 300) || status === 304; } -async function configurePage(page, client, config) { +async function configurePage(page, config) { await page.setViewport({ width: 1920, height: 1080 }); // Realistic viewport to avoid detection based on default Puppeteer dimensions (800x600) await page.setDefaultNavigationTimeout(config.navigationTimeout); - await page.setExtraHTTPHeaders({ 'Accept-Language': config.language }); - - // Use CDP to ensure browser language is set correctly (see https://zirkelc.dev/posts/puppeteer-language-experiment) - await client.send('Network.setUserAgentOverride', { - userAgent: await browser.userAgent(), - acceptLanguage: config.language, - }); if (browser.proxyCredentials?.username && browser.proxyCredentials?.password) { await page.authenticate(browser.proxyCredentials); diff --git a/src/archivist/fetcher/fullDomFetcher.test.js b/src/archivist/fetcher/fullDomFetcher.test.js index 3737eb4fa..eb916ff2a 100644 --- a/src/archivist/fetcher/fullDomFetcher.test.js +++ b/src/archivist/fetcher/fullDomFetcher.test.js @@ -6,7 +6,7 @@ import { fileURLToPath } from 'url'; import { expect, use } from 'chai'; import chaiAsPromised from 'chai-as-promised'; -import fetch, { launchHeadlessBrowser, stopHeadlessBrowser } from './fullDomFetcher.js'; +import fetch, { launchHeadlessBrowser, stopHeadlessBrowser, parseLanguage } from './fullDomFetcher.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -16,6 +16,8 @@ use(chaiAsPromised); const dynamicHTML = 'Dynamic Page'; const delayedContentHTML = 'Delayed Content
'; +const langEchoHTML = ''; +const stealthProbeHTML = ''; describe('Full DOM Fetcher', function () { this.timeout(60000); @@ -24,7 +26,7 @@ describe('Full DOM Fetcher', function () { let expectedPDFContent; before(async () => { - await launchHeadlessBrowser(); + await launchHeadlessBrowser('en'); temporaryServer = http.createServer((request, response) => { if (request.url === '/dynamic') { @@ -33,6 +35,17 @@ describe('Full DOM Fetcher', function () { if (request.url === '/delayed-content') { response.writeHead(200, { 'Content-Type': 'text/html' }).write(delayedContentHTML); } + if (request.url === '/lang-header') { + const acceptLanguage = request.headers['accept-language'] || ''; + + response.writeHead(200, { 'Content-Type': 'text/html' }).write(``); + } + if (request.url === '/lang-echo') { + response.writeHead(200, { 'Content-Type': 'text/html' }).write(langEchoHTML); + } + if (request.url === '/stealth-probe') { + response.writeHead(200, { 'Content-Type': 'text/html' }).write(stealthProbeHTML); + } if (request.url === '/terms.pdf') { expectedPDFContent = fs.readFileSync(path.resolve(__dirname, '../../../test/fixtures/terms.pdf')); response.writeHead(200, { 'Content-Type': 'application/pdf' }).write(expectedPDFContent); @@ -54,12 +67,12 @@ describe('Full DOM Fetcher', function () { await stopHeadlessBrowser(); await expect(fetch('http://example.com', [], { navigationTimeout: 5000, waitForElementsTimeout: 5000, language: 'en' })) .to.be.rejectedWith('The headless browser should be controlled manually'); - await launchHeadlessBrowser(); + await launchHeadlessBrowser('en'); }); it('reuses existing browser instance', async () => { - const browser1 = await launchHeadlessBrowser(); - const browser2 = await launchHeadlessBrowser(); + const browser1 = await launchHeadlessBrowser('en'); + const browser2 = await launchHeadlessBrowser('en'); expect(browser1).to.equal(browser2); }); @@ -81,6 +94,12 @@ describe('Full DOM Fetcher', function () { await expect(fetch(url, ['.non-existent'], { ...config, navigationTimeout: timeout })).to.be.rejectedWith(`Timed out after ${timeout / 1000} seconds when trying to fetch '${url}'`); }); + it('sends the configured language as Accept-Language header', async () => { + const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/lang-header`, [], config); + + expect(result.content).to.match(/data-accept-language="en"/); + }); + context('when a DOM element exists but its content is loaded asynchronously', () => { it('waits for the element content to be fully loaded', async () => { const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/delayed-content`, ['.content'], config); @@ -118,4 +137,145 @@ describe('Full DOM Fetcher', function () { }); }); }); + + describe('Language configuration', () => { + context('with a regional locale such as fr-FR,fr', () => { + const language = 'fr-FR,fr'; + const config = { navigationTimeout: 5000, waitForElementsTimeout: 5000, language }; + + before(async () => { + await stopHeadlessBrowser(); + await launchHeadlessBrowser(language); + }); + + it('exposes the primary tag through navigator.language', async () => { + const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/lang-echo`, [], config); + + expect(result.content).to.match(/data-language="fr-FR"/); + }); + + it('exposes every tag through navigator.languages', async () => { + const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/lang-echo`, [], config); + + expect(result.content).to.match(/data-languages="fr-FR,fr"/); + }); + + it('sends every configured tag as Accept-Language header', async () => { + const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/lang-header`, [], config); + + expect(result.content).to.match(/data-accept-language="fr-FR,fr[^"]*"/); + }); + }); + + context('with multiple comma-separated tags such as en-IE,en-GB,en', () => { + const language = 'en-IE,en-GB,en'; + const config = { navigationTimeout: 5000, waitForElementsTimeout: 5000, language }; + + before(async () => { + await stopHeadlessBrowser(); + await launchHeadlessBrowser(language); + }); + + it('derives Accept-Language quality factors from tag order', async () => { + const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/lang-header`, [], config); + + expect(result.content).to.match(/data-accept-language="en-IE,en-GB;q=0\.9,en;q=0\.8"/); + }); + }); + }); + + describe('Stealth evasions', () => { + // These assertions guard against a class of regressions where the puppeteer-extra-plugin-stealth fails to be registered before puppeteer.launch(): if it is registered later, puppeteer-extra never binds its onPageCreated hooks and all evasions stay inactive, leaving navigator.webdriver === true and HeadlessChrome in the UA. + const config = { navigationTimeout: 5000, waitForElementsTimeout: 5000, language: 'en' }; + let content; + + before(async () => { + ({ content } = await fetch(`http://127.0.0.1:${SERVER_PORT}/stealth-probe`, [], config)); + }); + + it('hides navigator.webdriver', () => { + expect(content).to.match(/data-webdriver="false"/); + }); + + it('removes HeadlessChrome from the user agent', () => { + expect(content).not.to.match(/HeadlessChrome/); + }); + + it('uses a realistic viewport instead of Puppeteer default', () => { + expect(content).to.match(/data-viewport-width="1920"/); + expect(content).to.match(/data-viewport-height="1080"/); + }); + + it('exposes a non-empty navigator.plugins list', () => { + const match = content.match(/data-plugin-count="(\d+)"/); + + expect(match).to.not.be.null; + expect(Number(match[1])).to.be.greaterThan(0); + }); + + it('hides headless WebGL vendor and renderer signature', () => { + expect(content).to.not.match(/data-webgl-vendor="[^"]*Google[^"]*"/); + expect(content).to.not.match(/data-webgl-renderer="[^"]*(?:SwiftShader|ANGLE)[^"]*"/); + }); + }); +}); + +describe('parseLanguage', () => { + context('with a single tag', () => { + it('returns the tag as both locale and a singleton languages array', () => { + expect(parseLanguage('en')).to.deep.equal({ locale: 'en', languages: ['en'] }); + }); + + it('preserves the region of a regional tag', () => { + expect(parseLanguage('fr-FR')).to.deep.equal({ locale: 'fr-FR', languages: ['fr-FR'] }); + }); + }); + + context('with a comma-separated priority list', () => { + it('splits every tag into the languages array and rejoins them as locale', () => { + expect(parseLanguage('en-IE,en-GB,en')).to.deep.equal({ locale: 'en-IE,en-GB,en', languages: [ 'en-IE', 'en-GB', 'en' ] }); + }); + + it('trims whitespace surrounding each tag', () => { + expect(parseLanguage('fr-FR, fr')).to.deep.equal({ locale: 'fr-FR,fr', languages: [ 'fr-FR', 'fr' ] }); + }); + + it('drops empty entries produced by consecutive commas', () => { + expect(parseLanguage('en,,fr')).to.deep.equal({ locale: 'en,fr', languages: [ 'en', 'fr' ] }); + }); + + it('drops a trailing comma', () => { + expect(parseLanguage('en,fr,')).to.deep.equal({ locale: 'en,fr', languages: [ 'en', 'fr' ] }); + }); + }); + + context('with quality factors', () => { + it('throws for a priority list carrying quality factors', () => { + expect(() => parseLanguage('en-IE,en-GB;q=0.9,en;q=0.8')).to.throw('Quality factors are not supported'); + }); + + it('throws for a single tag carrying a quality factor', () => { + expect(() => parseLanguage('en;q=1.0')).to.throw('Quality factors are not supported'); + }); + + it('throws regardless of quality-factor letter case', () => { + expect(() => parseLanguage('en-GB;Q=0.9,en')).to.throw('Quality factors are not supported'); + }); + }); + + context('with invalid input', () => { + it('throws when the value is not a string', () => { + expect(() => parseLanguage(undefined)).to.throw('must be a string'); + expect(() => parseLanguage(42)).to.throw('must be a string'); + }); + + it('throws when the value is empty or blank', () => { + expect(() => parseLanguage('')).to.throw('must contain at least one tag'); + expect(() => parseLanguage(' ')).to.throw('must contain at least one tag'); + }); + + it('throws when the value contains only separators', () => { + expect(() => parseLanguage(',,')).to.throw('must contain at least one tag'); + }); + }); }); diff --git a/src/archivist/fetcher/index.js b/src/archivist/fetcher/index.js index 81cbc1515..f9a32436c 100644 --- a/src/archivist/fetcher/index.js +++ b/src/archivist/fetcher/index.js @@ -21,7 +21,7 @@ export const FETCHER_TYPES = { * @param {string|Array} [params.cssSelectors] List of CSS selectors to await when loading the resource in a headless browser. Can be a CSS selector or an array of CSS selectors. Only relevant when `executeClientScripts` is enabled * @param {object} [params.config] Fetcher configuration * @param {number} [params.config.navigationTimeout] Maximum time (in milliseconds) to wait before considering the fetch failed - * @param {string} [params.config.language] Language (in [ISO 639-1 format](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)) to be passed in request headers + * @param {string} [params.config.language] Accept-Language value applied to the browser context. Accepts a comma-separated list of [BCP 47](https://www.rfc-editor.org/rfc/rfc5646) language tags in priority order, without quality factors (for example `fr`, `en-US`, or `en-IE,en-GB,en`). The tag list drives `navigator.language` and `navigator.languages` in the headless browser, and the browser derives the `Accept-Language` HTTP header from the tag order * @param {number} [params.config.waitForElementsTimeout] Maximum time (in milliseconds) to wait for selectors to exist on page before considering the fetch failed. Only relevant when `executeClientScripts` is enabled * @returns {Promise<{ mimeType: string, content: string | Buffer, fetcher: string }>} Promise containing the fetched resource's MIME type, content, and fetcher type * @throws {FetchDocumentError} When the fetch operation fails diff --git a/src/archivist/fetcher/index.test.js b/src/archivist/fetcher/index.test.js index 004df98ea..22145ddd8 100644 --- a/src/archivist/fetcher/index.test.js +++ b/src/archivist/fetcher/index.test.js @@ -21,7 +21,7 @@ const termsWithOtherCharsetHTML = '