Add version endpoints to the Collection API

CHANGELOG.md

@@ -2,6 +2,26 @@
 
 All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased [major]
+
+> Development of this release was supported by the [NGI0 Commons Fund](https://nlnet.nl/project/Modular-OTA/), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://www.ngi.eu) programme, under the aegis of DG CNECT under grant agreement N°101069594.
+
+### Added
+
+- Add paginated `GET /versions`, `GET /versions/{serviceId}` and `GET /versions/{serviceId}/{termsType}` endpoints to the Collection API to list versions
+- Add `GET /version/{versionId}` and `GET /version/{serviceId}/{termsType}/latest` endpoints to the Collection API to retrieve a single version
+- Expose `serviceId`, `termsType`, `isFirstRecord`, `isTechnicalUpgrade`, source `fetchUrls`, navigation `links` and diff statistics (`additions` and `deletions`) in Collection API version responses
+
+### Changed
+
+- **Breaking:** Return the `GET /service/{serviceId}` not-found response as JSON instead of plain text
+
+### Fixed
+
+- Reject version lookups whose service ID, terms type or record ID would be parsed as a Git option, which previously let an unauthenticated caller overwrite arbitrary files through the Collection API
+- Return `404` instead of a server error when a version lookup contains path separators or relative path segments; the error previously exposed the repository filesystem location
+- Return a generic message instead of internal error details in Collection API `500` responses
+
 ## 14.1.0 - 2026-06-29
 
 > Development of this release was supported by [the Research Chair in Content Moderation](https://regulation-tech.cnam.fr/) at the Conservatoire National des Arts et Métiers.

src/archivist/recorder/repositories/git/dataMapper.js

@@ -30,6 +30,10 @@ const MULTIPLE_SOURCE_DOCUMENTS_PREFIX = 'This version was recorded after extrac
 
 export const COMMIT_MESSAGE_PREFIXES_REGEXP = new RegExp(`^(${Object.values(COMMIT_MESSAGE_PREFIXES).join('|')})`);
 
+export function isTechnicalUpgrade(message) {
+  return message.startsWith(COMMIT_MESSAGE_PREFIXES.technicalUpgrade) || message.startsWith(COMMIT_MESSAGE_PREFIXES.deprecated_refilter);
+}
+
 export function toPersistence(record, snapshotIdentiferTemplate) {
   const { serviceId, termsType, documentId, snapshotIds = [], mimeType, metadata } = record;
 

src/archivist/recorder/repositories/git/git.js

@@ -22,7 +22,7 @@ export default class Git {
 
     this.git = simpleGit(this.path, {
       trimmed: true,
-      maxConcurrentProcesses: 1,
+      maxConcurrentProcesses: 1, // Concurrent runs on the same repository race the index and the commit-graph and can corrupt them
     });
 
     await this.git.init();
@@ -106,7 +106,8 @@ export default class Git {
 
       return commits;
     } catch (error) {
-      if (/unknown revision or path not in the working tree|does not have any commits yet/.test(error.message)) {
+      // `bad object` is raised for a well-formed but absent object ID; like an unknown revision, it means "no match" rather than a hard failure
+      if (/unknown revision or path not in the working tree|does not have any commits yet|bad object/.test(error.message)) {
         return [];
       }
 
@@ -132,15 +133,24 @@ export default class Git {
     await fs.rm(path.join(this.path, '.git', 'objects', 'info', 'commit-graph.lock'), { force: true }); // Remove a leftover commit-graph lock from a previous `commit-graph write` that was killed mid-write (e.g. the process was terminated during a deploy or restart). The commit-graph is a disposable cache rebuilt by `writeCommitGraph`, so clearing a stale lock is safe and prevents every subsequent run from failing.
     await this.git.reset('hard');
 
-    return this.git.clean('f', '-d');
+    return this.git.clean('f', '-d'); // Force-remove untracked files (`f`) and untracked directories (`-d`)
   }
 
   getFullHash(shortHash) {
-    return this.git.show([ shortHash, '--pretty=%H', '-s' ]);
+    return this.git.show([
+      shortHash,
+      '--pretty=%H', // Print the full 40-character commit hash
+      '-s', // Suppress the diff output, only the formatted hash is wanted
+    ]);
   }
 
   restore(path, commit) {
-    return this.git.raw([ 'restore', '-s', commit, '--', path ]);
+    return this.git.raw([
+      'restore',
+      '-s', commit, // Take the file contents from this specific commit rather than from the index
+      '--', // Everything after is a pathspec, not a revision or option
+      path,
+    ]);
   }
 
   async destroyHistory() {
@@ -154,14 +164,86 @@ export default class Git {
   }
 
   async listFiles(path) {
-    return (await this.git.raw([ 'ls-files', path ])).split('\n');
+    return (await this.git.raw([ 'ls-files', '--', path ])).split('\n'); // Everything after "--" is a pathspec, not a revision or option
   }
 
   async writeCommitGraph() {
-    await this.git.raw([ 'commit-graph', 'write', '--reachable', '--changed-paths' ]);
+    await this.git.raw([
+      'commit-graph',
+      'write',
+      '--reachable', // Cover every commit reachable from the refs, so the whole history is indexed
+      '--changed-paths', // Also store the changed-path Bloom filters that speed up path-limited log/diff
+    ]);
   }
 
   async updateCommitGraph() {
-    await this.git.raw([ 'commit-graph', 'write', '--reachable', '--changed-paths', '--append' ]);
+    await this.git.raw([
+      'commit-graph',
+      'write',
+      '--reachable',
+      '--changed-paths',
+      '--append', // Extend the existing commit-graph instead of rewriting it in full
+    ]);
+  }
+
+  async listPathRevisions(pathFilter) {
+    let output;
+
+    try {
+      // Ordering and technical-upgrade filtering are done by the caller in memory, so `--author-date-order`/`--grep`/`--name-only` are deliberately omitted to keep this walk lean
+      output = await this.git.raw([
+        'log',
+        '--no-merges',
+        '--format=%H%x09%at%x09%s', // Tab-separated hash, author date and subject: the minimum needed to order versions and detect technical upgrades, with no diff or message body loaded
+        '--', // Everything after is a pathspec, never a revision or an option
+        pathFilter,
+      ]);
+    } catch (error) {
+      if (/unknown revision or path not in the working tree|does not have any commits yet/.test(error.message)) {
+        return [];
+      }
+
+      throw error;
+    }
+
+    if (!output) {
+      return [];
+    }
+
+    return output.trim().split('\n').filter(Boolean).map(line => {
+      const [ hash, timestamp, ...subjectParts ] = line.split('\t');
+
+      return { hash, timestamp: parseInt(timestamp, 10), subject: subjectParts.join('\t') };
+    });
+  }
+
+  async getDiffStats(commitHash) {
+    const output = await this.git.raw([
+      'show',
+      '--numstat', // Report added/deleted line counts per file as tab-separated numbers, instead of a textual diff
+      '--format=', // Drop the commit header so the output holds only the numstat lines
+      commitHash,
+    ]);
+
+    let additions = 0;
+    let deletions = 0;
+
+    for (const line of output.trim().split('\n')) {
+      if (!line) {
+        continue;
+      }
+
+      const [ added, deleted ] = line.split('\t');
+
+      // Binary files show '-' for additions/deletions
+      if (added !== '-') {
+        additions += parseInt(added, 10);
+      }
+      if (deleted !== '-') {
+        deletions += parseInt(deleted, 10);
+      }
+    }
+
+    return { additions, deletions };
   }
 }

src/archivist/recorder/repositories/git/index.js

@@ -15,6 +15,25 @@ import Git from './git.js';
 
 const fs = fsApi.promises;
 
+const RECORD_ID_REGEXP = /^[0-9a-f]{7,40}$/i; // Git commit SHA 7 (abbreviated) to 40 (full) hexadecimal characters. Prevent value such as `--output=…` to be parsed as a command-line option
+
+const CONTROL_CHARACTERS_REGEXP = /\p{Cc}/u; // Matches any Unicode "control" character: the C0 range (U+0000 to U+001F), DEL (U+007F) and the C1 range (U+0080 to U+009F), i.e. 65 non-printable characters including NUL. The `u` flag is required for the `\p{...}` property escape to be recognised, otherwise the pattern would match the literal text `p{Cc}`. Legitimate service IDs, terms types and document IDs never contain these, and NUL in particular can truncate a value once it reaches git or the filesystem, so any segment holding one is rejected.
+
+// Keeps hostile values from reaching git, where a pathspec that resolves outside the repository (such as `../foo/*`) aborts with an error that exposes the repository location.
+function isPlainPathSegment(segment) {
+  return segment.length > 0
+    && segment !== '.'
+    && segment !== '..'
+    && !segment.includes('/')
+    && !segment.includes('\\')
+    && !CONTROL_CHARACTERS_REGEXP.test(segment);
+}
+
+function canMatchRecordFilePath(...pathSegments) {
+  // A non-string segment means "not provided" (`undefined`, or `false` for an absent document ID) and constrains nothing
+  return pathSegments.every(segment => typeof segment !== 'string' || isPlainPathSegment(segment));
+}
+
 export default class GitRepository extends RepositoryInterface {
   constructor({ path, author, publish, snapshotIdentiferTemplate }) {
     super();
@@ -64,6 +83,10 @@ export default class GitRepository extends RepositoryInterface {
   }
 
   async findLatest(serviceId, termsType, documentId) {
+    if (!canMatchRecordFilePath(serviceId, termsType, documentId)) {
+      return null;
+    }
+
     const matchingFilesPaths = await this.git.listFiles(DataMapper.generateFilePath(serviceId, termsType, documentId));
 
     if (!matchingFilesPaths.length) {
@@ -76,35 +99,95 @@ export default class GitRepository extends RepositoryInterface {
   }
 
   async findByDate(serviceId, termsType, date, documentId) {
+    if (!canMatchRecordFilePath(serviceId, termsType, documentId)) {
+      return null;
+    }
+
     const filePath = DataMapper.generateFilePath(serviceId, termsType, documentId);
-    const commit = await this.git.getCommit([ `--until=${date?.toISOString()}`, filePath ]);
+    const commit = await this.git.getCommit([ `--until=${date?.toISOString()}`, '--', filePath ]);
 
     return this.#toDomain(commit);
   }
 
   async findById(recordId) {
-    const commit = await this.git.getCommit([recordId]);
+    if (!RECORD_ID_REGEXP.test(recordId)) {
+      return null;
+    }
+
+    const commit = await this.git.getCommit([ '--end-of-options', recordId ]); // `--end-of-options` forces git to treat `recordId` as a revision, never as an option: a second line of defence that keeps the lookup safe from argument injection even if the format guard above is ever relaxed
 
     return this.#toDomain(commit);
   }
 
+  async findMetadataById(recordId) {
+    if (!RECORD_ID_REGEXP.test(recordId)) {
+      return null;
+    }
+
+    const commit = await this.git.getCommit([ '--end-of-options', recordId ]); // `--end-of-options` forces git to treat `recordId` as a revision, never as an option: a second line of defence that keeps the lookup safe from argument injection even if the format guard above is ever relaxed
+
+    return this.#toDomain(commit, { deferContentLoading: true });
+  }
+
   async findAll({ limit, offset, includeTechnicalUpgrades = true } = {}) {
     return Promise.all((await this.#getCommits({ limit, offset, includeTechnicalUpgrades })).map(commit => this.#toDomain(commit, { deferContentLoading: true })));
   }
 
+  async findByServiceAndTermsType(serviceId, termsType, { limit, offset, includeTechnicalUpgrades = true } = {}) {
+    if (!canMatchRecordFilePath(serviceId, termsType)) {
+      return [];
+    }
+
+    const pathPattern = DataMapper.generateFilePath(serviceId, termsType);
+
+    return Promise.all((await this.#getCommits({ pathFilter: pathPattern, limit, offset, includeTechnicalUpgrades })).map(commit => this.#toDomain(commit, { deferContentLoading: true })));
+  }
+
   async findByService(serviceId, { limit, offset, includeTechnicalUpgrades = true } = {}) {
+    if (!canMatchRecordFilePath(serviceId)) {
+      return [];
+    }
+
     const pathPattern = DataMapper.generateFilePath(serviceId);
 
     return Promise.all((await this.#getCommits({ pathFilter: pathPattern, limit, offset, includeTechnicalUpgrades })).map(commit => this.#toDomain(commit, { deferContentLoading: true })));
   }
 
-  async findByServiceAndTermsType(serviceId, termsType, { limit, offset, includeTechnicalUpgrades = true } = {}) {
+  async getNavigationIds(serviceId, termsType, versionId, { includeTechnicalUpgrades = true } = {}) {
+    if (!canMatchRecordFilePath(serviceId, termsType)) {
+      return { first: null, prev: null, next: null, last: null };
+    }
+
     const pathPattern = DataMapper.generateFilePath(serviceId, termsType);
+    let revisions = await this.git.listPathRevisions(pathPattern); // single lean walk of the terms history
 
-    return Promise.all((await this.#getCommits({ pathFilter: pathPattern, limit, offset, includeTechnicalUpgrades })).map(commit => this.#toDomain(commit, { deferContentLoading: true })));
+    if (!includeTechnicalUpgrades) {
+      revisions = revisions.filter(revision => !DataMapper.isTechnicalUpgrade(revision.subject));
+    }
+
+    // Deterministic total order: most recent first, commit SHA as a stable tiebreaker for versions sharing the same fetch date (git stores second precision).
+    // prev/next are then adjacent entries in this single order, so navigation always round-trips, unlike the previous chronological/topological mix.
+    revisions.sort((a, b) => b.timestamp - a.timestamp || (a.hash < b.hash ? -1 : 1));
+
+    const index = revisions.findIndex(revision => revision.hash === versionId);
+
+    if (index === -1) {
+      return { first: null, prev: null, next: null, last: null };
+    }
+
+    return {
+      last: revisions[0].hash,
+      first: revisions[revisions.length - 1].hash,
+      next: index > 0 ? revisions[index - 1].hash : null,
+      prev: index < revisions.length - 1 ? revisions[index + 1].hash : null,
+    };
   }
 
   async count(serviceId, termsType) {
+    if (!canMatchRecordFilePath(serviceId, termsType)) {
+      return 0;
+    }
+
     const grepOptions = Object.values(DataMapper.COMMIT_MESSAGE_PREFIXES).map(prefix => `--grep=${prefix}`);
     const pathOptions = [];
 
@@ -158,6 +241,10 @@ export default class GitRepository extends RepositoryInterface {
     record.content = pdfBuffer;
   }
 
+  getDiffStats(recordId) {
+    return this.git.getDiffStats(recordId);
+  }
+
   async #getCommits({ pathFilter, reverse = false, limit, offset, includeTechnicalUpgrades = true } = {}) {
     const prefixes = includeTechnicalUpgrades
       ? DataMapper.COMMIT_MESSAGE_PREFIXES