Add normalize whitespace built-in filter

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased [minor]
+
+> Development of this release was supported by [User Rights](https://www.user-rights.org).
+
+### Added
+
+- Add `convertSpacesToStandard` built-in filter that replaces Unicode space separators (non-breaking space, narrow no-break space, ...) with a regular space; see more in the [built-in filters documentation](https://docs.opentermsarchive.org/terms/reference/built-in-filters/)
+
 ## 15.0.0 - 2026-07-07
 
 > Development of this release was supported by the [NGI0 Commons Fund](https://nlnet.nl/project/Modular-OTA/), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://www.ngi.eu) programme, under the aegis of DG CNECT under grant agreement N°101069594.

src/archivist/extract/exposedFilters.js

@@ -26,3 +26,18 @@ export function removeQueryParams(webPageDOM, paramsToRemove = []) {
     }
   }
 }
+
+const SPACE_SEPARATORS = /\p{Zs}/gu;
+
+export function convertSpacesToStandard(webPageDOM) {
+  const walker = webPageDOM.createTreeWalker(webPageDOM.body, webPageDOM.defaultView.NodeFilter.SHOW_TEXT);
+
+  for (let node = walker.nextNode(); node; node = walker.nextNode()) {
+    const original = node.nodeValue;
+    const normalized = original.replace(SPACE_SEPARATORS, ' ');
+
+    if (normalized !== original) {
+      node.nodeValue = normalized;
+    }
+  }
+}

src/archivist/extract/exposedFilters.test.js

@@ -1,7 +1,7 @@
 import { expect } from 'chai';
 
 import createWebPageDOM from './dom.js';
-import { removeQueryParams } from './exposedFilters.js';
+import { convertSpacesToStandard, removeQueryParams } from './exposedFilters.js';
 
 describe('exposedFilters', () => {
   let webPageDOM;
@@ -265,4 +265,52 @@ fetch(trackingUrl);
       });
     });
   });
+
+  describe('#convertSpacesToStandard', () => {
+    describe('with Unicode space separators in text', () => {
+      let element;
+
+      before(() => {
+        element = webPageDOM.createElement('p');
+        element.textContent = 'a\u00A0b\u202Fc\u2009d\u3000e';
+        webPageDOM.body.appendChild(element);
+
+        convertSpacesToStandard(webPageDOM);
+      });
+
+      after(() => {
+        element.remove();
+      });
+
+      it('replaces them with a regular space', () => {
+        expect(element.textContent).to.equal('a b c d e');
+      });
+    });
+
+    describe('with Unicode space separators in an attribute', () => {
+      let element;
+      const className = 'label\u00A0primary';
+
+      before(() => {
+        element = webPageDOM.createElement('a');
+        element.setAttribute('class', className);
+        element.textContent = 'read\u00A0the\u00A0policy';
+        webPageDOM.body.appendChild(element);
+
+        convertSpacesToStandard(webPageDOM);
+      });
+
+      after(() => {
+        element.remove();
+      });
+
+      it('leaves attribute values untouched', () => {
+        expect(element.getAttribute('class')).to.equal(className);
+      });
+
+      it('replaces them in the text content', () => {
+        expect(element.textContent).to.equal('read the policy');
+      });
+    });
+  });
 });

src/archivist/fetcher/fullDomFetcher.test.js

@@ -79,7 +79,7 @@ describe('Full DOM Fetcher', function () {
   });
 
   describe('#fetch', () => {
-    const config = { navigationTimeout: 1000, waitForElementsTimeout: 1000, language: 'en' };
+    const config = { navigationTimeout: 5000, waitForElementsTimeout: 5000, language: 'en' };
 
     it('waits for dynamically injected elements to appear in the DOM', async () => {
       const result = await fetch(`http://127.0.0.1:${SERVER_PORT}/dynamic`, ['.dynamic'], config);

src/archivist/index.test.js

@@ -35,7 +35,7 @@ function resetGitRepositories() {
 }
 
 describe('Archivist', function () {
-  this.timeout(10000);
+  this.timeout(30000);
 
   const SERVICE_A_ID = 'service·A';
   const SERVICE_A_TYPE = 'Terms of Service';

src/archivist/services/index.test.js

@@ -431,7 +431,7 @@ describe('Services', () => {
         expect(secondSourceDocument.filters).to.be.an('array');
         expect(secondSourceDocument.filters).to.have.length(2);
         expect(secondSourceDocument.filters[0]).to.be.a('function');
-        expect(secondSourceDocument.filters[0].name).to.equal('removeQueryParams');
+        expect(secondSourceDocument.filters[0].name).to.equal(realFilterNames[0]);
         expect(secondSourceDocument.filters[1]).to.be.a('function');
         expect(secondSourceDocument.filters[1].name).to.equal('removePrintButton');
       });