From 3ca6504b67c17294f08e561ba1628c5da5d0fcc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Biron?= Date: Thu, 2 Jul 2026 10:31:00 +0200 Subject: [PATCH 1/6] Add normalizeWhitespace built-in filter --- src/archivist/extract/exposedFilters.js | 15 +++++++ src/archivist/extract/exposedFilters.test.js | 44 +++++++++++++++++++- src/archivist/services/index.test.js | 2 +- 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/archivist/extract/exposedFilters.js b/src/archivist/extract/exposedFilters.js index 9f327f43f..fc04a75cc 100644 --- a/src/archivist/extract/exposedFilters.js +++ b/src/archivist/extract/exposedFilters.js @@ -26,3 +26,18 @@ export function removeQueryParams(webPageDOM, paramsToRemove = []) { } } } + +const SPACE_SEPARATORS = /\p{Zs}/gu; + +export function normalizeWhitespace(webPageDOM) { + const walker = webPageDOM.createTreeWalker(webPageDOM.body, webPageDOM.defaultView.NodeFilter.SHOW_TEXT); + + for (let node = walker.nextNode(); node; node = walker.nextNode()) { + const original = node.nodeValue; + const normalized = original.replace(SPACE_SEPARATORS, ' '); + + if (normalized !== original) { + node.nodeValue = normalized; + } + } +} diff --git a/src/archivist/extract/exposedFilters.test.js b/src/archivist/extract/exposedFilters.test.js index e61ffb53a..dbf1a8d13 100644 --- a/src/archivist/extract/exposedFilters.test.js +++ b/src/archivist/extract/exposedFilters.test.js @@ -1,7 +1,7 @@ import { expect } from 'chai'; import createWebPageDOM from './dom.js'; -import { removeQueryParams } from './exposedFilters.js'; +import { normalizeWhitespace, removeQueryParams } from './exposedFilters.js'; describe('exposedFilters', () => { let webPageDOM; @@ -265,4 +265,46 @@ fetch(trackingUrl); }); }); }); + + describe('#normalizeWhitespace', () => { + let element; + + afterEach(() => { + element.remove(); + }); + + it('replaces Unicode space separators with a regular space', () => { + element = webPageDOM.createElement('p'); + element.textContent = 'a\u00A0b\u202Fc\u2009d\u3000e'; + webPageDOM.body.appendChild(element); + + normalizeWhitespace(webPageDOM); + + expect(element.textContent).to.equal('a b c d e'); + }); + + it('normalizes text across nested elements', () => { + element = webPageDOM.createElement('div'); + element.innerHTML = 'first\u00A0partsecond\u00A0part'; + webPageDOM.body.appendChild(element); + + normalizeWhitespace(webPageDOM); + + expect(element.textContent).to.equal('first partsecond part'); + }); + + it('does not alter element attributes or URLs, only text content', () => { + const href = 'https://example.com/page?label=gen173__nr-1'; + + element = webPageDOM.createElement('a'); + element.setAttribute('href', href); + element.textContent = 'read\u00A0the\u00A0policy'; + webPageDOM.body.appendChild(element); + + normalizeWhitespace(webPageDOM); + + expect(element.getAttribute('href')).to.equal(href); + expect(element.textContent).to.equal('read the policy'); + }); + }); }); diff --git a/src/archivist/services/index.test.js b/src/archivist/services/index.test.js index 968b01e38..abaae74b8 100644 --- a/src/archivist/services/index.test.js +++ b/src/archivist/services/index.test.js @@ -431,7 +431,7 @@ describe('Services', () => { expect(secondSourceDocument.filters).to.be.an('array'); expect(secondSourceDocument.filters).to.have.length(2); expect(secondSourceDocument.filters[0]).to.be.a('function'); - expect(secondSourceDocument.filters[0].name).to.equal('removeQueryParams'); + expect(secondSourceDocument.filters[0].name).to.equal(realFilterNames[0]); expect(secondSourceDocument.filters[1]).to.be.a('function'); expect(secondSourceDocument.filters[1].name).to.equal('removePrintButton'); }); From 5ee1f7971b6b80cf68c3d289680805dc03456380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Biron?= Date: Thu, 2 Jul 2026 10:31:00 +0200 Subject: [PATCH 2/6] Add changelog entry --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cdfc85ca..e920b4e07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased [minor] + +### Added + +- Add `normalizeWhitespace` built-in filter that replaces Unicode space separators (non-breaking space, narrow no-break space, ...) with a regular space; see more in the [built-in filters documentation](https://docs.opentermsarchive.org/terms/reference/built-in-filters/) + ## 15.0.0 - 2026-07-07 > Development of this release was supported by the [NGI0 Commons Fund](https://nlnet.nl/project/Modular-OTA/), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://www.ngi.eu) programme, under the aegis of DG CNECT under grant agreement N°101069594. From 23937b3ecc9b27cbef6c5d4a294c39543b2ac256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Biron?= Date: Mon, 6 Jul 2026 11:03:13 +0200 Subject: [PATCH 3/6] Improve function name Co-authored-by: Matti Schneider --- CHANGELOG.md | 2 +- src/archivist/extract/exposedFilters.js | 2 +- src/archivist/extract/exposedFilters.test.js | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e920b4e07..03cbe2ffb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All changes that impact users of this module are documented in this file, in the ### Added -- Add `normalizeWhitespace` built-in filter that replaces Unicode space separators (non-breaking space, narrow no-break space, ...) with a regular space; see more in the [built-in filters documentation](https://docs.opentermsarchive.org/terms/reference/built-in-filters/) +- Add `convertSpacesToStandard` built-in filter that replaces Unicode space separators (non-breaking space, narrow no-break space, ...) with a regular space; see more in the [built-in filters documentation](https://docs.opentermsarchive.org/terms/reference/built-in-filters/) ## 15.0.0 - 2026-07-07 diff --git a/src/archivist/extract/exposedFilters.js b/src/archivist/extract/exposedFilters.js index fc04a75cc..208b81583 100644 --- a/src/archivist/extract/exposedFilters.js +++ b/src/archivist/extract/exposedFilters.js @@ -29,7 +29,7 @@ export function removeQueryParams(webPageDOM, paramsToRemove = []) { const SPACE_SEPARATORS = /\p{Zs}/gu; -export function normalizeWhitespace(webPageDOM) { +export function convertSpacesToStandard(webPageDOM) { const walker = webPageDOM.createTreeWalker(webPageDOM.body, webPageDOM.defaultView.NodeFilter.SHOW_TEXT); for (let node = walker.nextNode(); node; node = walker.nextNode()) { diff --git a/src/archivist/extract/exposedFilters.test.js b/src/archivist/extract/exposedFilters.test.js index dbf1a8d13..b74a50a86 100644 --- a/src/archivist/extract/exposedFilters.test.js +++ b/src/archivist/extract/exposedFilters.test.js @@ -1,7 +1,7 @@ import { expect } from 'chai'; import createWebPageDOM from './dom.js'; -import { normalizeWhitespace, removeQueryParams } from './exposedFilters.js'; +import { convertSpacesToStandard, removeQueryParams } from './exposedFilters.js'; describe('exposedFilters', () => { let webPageDOM; @@ -266,7 +266,7 @@ fetch(trackingUrl); }); }); - describe('#normalizeWhitespace', () => { + describe('#convertSpacesToStandard', () => { let element; afterEach(() => { @@ -278,7 +278,7 @@ fetch(trackingUrl); element.textContent = 'a\u00A0b\u202Fc\u2009d\u3000e'; webPageDOM.body.appendChild(element); - normalizeWhitespace(webPageDOM); + convertSpacesToStandard(webPageDOM); expect(element.textContent).to.equal('a b c d e'); }); @@ -288,7 +288,7 @@ fetch(trackingUrl); element.innerHTML = 'first\u00A0partsecond\u00A0part'; webPageDOM.body.appendChild(element); - normalizeWhitespace(webPageDOM); + convertSpacesToStandard(webPageDOM); expect(element.textContent).to.equal('first partsecond part'); }); @@ -301,7 +301,7 @@ fetch(trackingUrl); element.textContent = 'read\u00A0the\u00A0policy'; webPageDOM.body.appendChild(element); - normalizeWhitespace(webPageDOM); + convertSpacesToStandard(webPageDOM); expect(element.getAttribute('href')).to.equal(href); expect(element.textContent).to.equal('read the policy'); From f83855200214123afabfb4986da27b3a0e3f89e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Biron?= Date: Mon, 6 Jul 2026 11:06:01 +0200 Subject: [PATCH 4/6] Improve and simplify tests --- src/archivist/extract/exposedFilters.test.js | 60 +++++++++++--------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/src/archivist/extract/exposedFilters.test.js b/src/archivist/extract/exposedFilters.test.js index b74a50a86..af7135ab4 100644 --- a/src/archivist/extract/exposedFilters.test.js +++ b/src/archivist/extract/exposedFilters.test.js @@ -267,44 +267,50 @@ fetch(trackingUrl); }); describe('#convertSpacesToStandard', () => { - let element; + describe('with Unicode space separators in text', () => { + let element; - afterEach(() => { - element.remove(); - }); + before(() => { + element = webPageDOM.createElement('p'); + element.textContent = 'a\u00A0b\u202Fc\u2009d\u3000e'; + webPageDOM.body.appendChild(element); - it('replaces Unicode space separators with a regular space', () => { - element = webPageDOM.createElement('p'); - element.textContent = 'a\u00A0b\u202Fc\u2009d\u3000e'; - webPageDOM.body.appendChild(element); + convertSpacesToStandard(webPageDOM); + }); - convertSpacesToStandard(webPageDOM); + after(() => { + element.remove(); + }); - expect(element.textContent).to.equal('a b c d e'); + it('replaces them with a regular space', () => { + expect(element.textContent).to.equal('a b c d e'); + }); }); - it('normalizes text across nested elements', () => { - element = webPageDOM.createElement('div'); - element.innerHTML = 'first\u00A0partsecond\u00A0part'; - webPageDOM.body.appendChild(element); - - convertSpacesToStandard(webPageDOM); + describe('with Unicode space separators in an attribute', () => { + let element; + const className = 'label\u00A0primary'; - expect(element.textContent).to.equal('first partsecond part'); - }); + before(() => { + element = webPageDOM.createElement('a'); + element.setAttribute('class', className); + element.textContent = 'read\u00A0the\u00A0policy'; + webPageDOM.body.appendChild(element); - it('does not alter element attributes or URLs, only text content', () => { - const href = 'https://example.com/page?label=gen173__nr-1'; + convertSpacesToStandard(webPageDOM); + }); - element = webPageDOM.createElement('a'); - element.setAttribute('href', href); - element.textContent = 'read\u00A0the\u00A0policy'; - webPageDOM.body.appendChild(element); + after(() => { + element.remove(); + }); - convertSpacesToStandard(webPageDOM); + it('leaves attribute values untouched', () => { + expect(element.getAttribute('class')).to.equal(className); + }); - expect(element.getAttribute('href')).to.equal(href); - expect(element.textContent).to.equal('read the policy'); + it('replaces them in the text content', () => { + expect(element.textContent).to.equal('read the policy'); + }); }); }); }); From ee1098b13e52800a1245a79294df3918f0e92167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Biron?= Date: Tue, 7 Jul 2026 11:07:45 +0200 Subject: [PATCH 5/6] Add funder --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03cbe2ffb..157cfe6e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ All changes that impact users of this module are documented in this file, in the ## Unreleased [minor] +> Development of this release was supported by [User Rights](https://www.user-rights.org). + ### Added - Add `convertSpacesToStandard` built-in filter that replaces Unicode space separators (non-breaking space, narrow no-break space, ...) with a regular space; see more in the [built-in filters documentation](https://docs.opentermsarchive.org/terms/reference/built-in-filters/) From a2f081913114a3274dc90016283820d8f68be3d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Biron?= Date: Tue, 7 Jul 2026 14:12:09 +0200 Subject: [PATCH 6/6] Fix flaky tests on Windows CI --- src/archivist/fetcher/fullDomFetcher.test.js | 2 +- src/archivist/index.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/archivist/fetcher/fullDomFetcher.test.js b/src/archivist/fetcher/fullDomFetcher.test.js index eb916ff2a..d2174b8c6 100644 --- a/src/archivist/fetcher/fullDomFetcher.test.js +++ b/src/archivist/fetcher/fullDomFetcher.test.js @@ -79,7 +79,7 @@ describe('Full DOM Fetcher', function () { }); describe('#fetch', () => { - const config = { navigationTimeout: 1000, waitForElementsTimeout: 1000, language: 'en' }; + const config = { navigationTimeout: 5000, waitForElementsTimeout: 5000, language: 'en' }; it('waits for dynamically injected elements to appear in the DOM', async () => { const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/dynamic`, ['.dynamic'], config); diff --git a/src/archivist/index.test.js b/src/archivist/index.test.js index f805b11fb..fa2ad996e 100644 --- a/src/archivist/index.test.js +++ b/src/archivist/index.test.js @@ -35,7 +35,7 @@ function resetGitRepositories() { } describe('Archivist', function () { - this.timeout(10000); + this.timeout(30000); const SERVICE_A_ID = 'service·A'; const SERVICE_A_TYPE = 'Terms of Service';