diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 309fac34f..096f1c84b 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -35,4 +35,21 @@ ignore = [ # ever built with the `mysql` feature, or any other consumer of rsa enters # the build, at which point it becomes a real reachable advisory. "RUSTSEC-2023-0071", # rsa Marvin attack (no fix; not linked in our build) + + # lru 0.12.5 (use-after-free in pop()). Reachable via aws-sdk-s3 + # (reverted from the upgrade that pulled in lru 0.12.5). The + # lockfile currently has 0.12.5 (via aws-sdk-s3) AND 0.16.3 (via + # alloy). The RUSTSEC-2026-0253 advisory covers both ranges, and + # this ignore is still load-bearing for 0.12.5 — a future alloy + # release that bumps 0.16.x to a fixed version would let this + # ignore retire for the 0.16 range. REMOVE only when BOTH + # lockfile versions are fixed (or removed). + "RUSTSEC-2026-0253", + # + # Round-3 P2 (reviewer 2): the previous version of this file + # ignored RUSTSEC-2026-0258 for h2 0.4.13, but the lockfile + # already has h2 0.4.18 (verified 2026-08-31: cargo tree -p h2 + # shows 0.4.18) — the fix is in. The ignore was stale and would + # have silently re-accepted the advisory if h2 dropped back to + # 0.4.13. Removed. ] diff --git a/.env.example b/.env.example index 81c60824d..552968517 100644 --- a/.env.example +++ b/.env.example @@ -305,6 +305,20 @@ GITLAWB_TRUSTED_PROXY= # Enable automatic background sync from known peers GITLAWB_AUTO_SYNC=false +# ── Reconciliation sweep ───────────────────────────────────────────────── +# Periodic durability sweep: re-derives the public pin set and the withheld-blob +# recovery set each hour and fills gaps so a dropped replication job never means +# data loss. Defaults to true; set to false to disable the sweep even when a pin +# backend (IPFS/Pinata) is configured. +# +# Phase-capability matrix: +# - Public pin repair: IPFS-only, Pinata-only, or both (requires the +# respective backend to be configured). +# - Encrypted recovery repair: requires local IPFS (GITLAWB_IPFS_API). +# Pinata-only nodes reconcile public pins only; encrypted recovery +# reconciliation is not performed. +GITLAWB_RECONCILIATION_SWEEP=true + # ── iCaptcha proof-of-intelligence gate ─────────────────────────────────── # Optional gate on create_repo + register: require callers to present an # iCaptcha proof (X-ICaptcha-Proof header) earned at icaptcha.gitlawb.com. diff --git a/Cargo.lock b/Cargo.lock index 3f29b0767..d202730d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3468,6 +3468,7 @@ dependencies = [ "mockito", "multiaddr", "prometheus", + "rand 0.8.6", "reqwest", "serde", "serde_json", diff --git a/README.md b/README.md index 3a092bf21..c4ad07875 100644 --- a/README.md +++ b/README.md @@ -395,6 +395,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. Defaults to `false` during the staged rollout below. | | `GITLAWB_ENFORCE_OWNER_PUSH` | Require the authenticated pusher to be the repo owner on `git-receive-pack`. **Defaults to `true`.** A `did:key` signature is authentication, not authorization — anyone can mint a key and sign — so with this off every signed caller may push to every repository, private ones included. Delegated and CI keys count as non-owners: a UCAN `git/push` capability is verified but not yet honored for authorization, so they cannot push while this is on. Set `false` only for a rolling upgrade; see [`docs/RUN-A-NODE.md`](docs/RUN-A-NODE.md). | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | +| `GITLAWB_RECONCILIATION_SWEEP` | Enable the hourly durability sweep that re-pins/backstops missing objects (default `true`; disabled when no IPFS/Pinata backend is configured). Public pin repair runs against any configured backend (IPFS, Pinata, or both). Encrypted recovery repair requires local IPFS (`GITLAWB_IPFS_API`); Pinata-only nodes reconcile public pins only. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | | `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 3153600000 (100 years), since the node derives deadlines from this value and a larger one cannot be represented. | | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | diff --git a/crates/gitlawb-attest/src/attestation.rs b/crates/gitlawb-attest/src/attestation.rs index 0e29e3a90..88ab2fdae 100644 --- a/crates/gitlawb-attest/src/attestation.rs +++ b/crates/gitlawb-attest/src/attestation.rs @@ -483,6 +483,37 @@ mod tests { assert!(matches!(err, Error::Signature(_))); } + /// The identity-point forgery must be rejected: the shared attestation + /// verifier is a cert-bound provenance gate, so accepting the weak-key + /// signature would let anyone mint a forged attestation that verifies. + /// Strict verification rejects small-order public keys and R (the identity + /// point here), which ordinary verification does not. + #[test] + fn verify_rejects_identity_point_forgery() { + let cert_hash = sample_cert_hash(); + let mut att = dummy_attestation(&fresh(), cert_hash); + + // Public key A = identity point (0,1); signature R = identity, S = 0. + // The equation `[S]B = R + [k]A` then holds for any k and any message. + let identity = [ + 1u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, + ]; + let mut buf = Vec::with_capacity(34); + buf.extend_from_slice(&ED25519_MULTICODEC); + buf.extend_from_slice(&identity); + att.signer = format!( + "did:key:{}", + multibase::encode(multibase::Base::Base58Btc, &buf) + ); + let mut sig = [0u8; 64]; + sig[..32].copy_from_slice(&identity); + att.sig = B64U.encode(sig); + + let err = att.verify_signature(cert_hash).unwrap_err(); + assert!(matches!(err, Error::Signature(_))); + } + /// A payload that happens to contain a `cert_hash` field of its own does /// not interfere with the outer binding: the attestation envelope's /// `cert_hash` is the only field consulted by `verify_signature`, and the diff --git a/crates/gitlawb-core/src/identity.rs b/crates/gitlawb-core/src/identity.rs index beef4d1bc..ca87f8ecc 100644 --- a/crates/gitlawb-core/src/identity.rs +++ b/crates/gitlawb-core/src/identity.rs @@ -77,6 +77,14 @@ impl Keypair { } /// Verify an Ed25519 signature. +/// +/// Strict verification: rejects small-order `R` and small-order public keys +/// (the identity point, and any point of low order). Ordinary `verify` accepts +/// a signature forged with the identity point as the public key plus +/// `R = identity, S = 0`, which verifies for *any* message. `identity::verify` +/// is the shared primitive behind HTTP request authentication, UCANs, and +/// certificates, so weak-key acceptance is an authentication bypass, not a +/// malleability nuance. pub fn verify(verifying_key: &VerifyingKey, msg: &[u8], sig_bytes: &[u8; 64]) -> Result<()> { let sig = Signature::from_bytes(sig_bytes); verifying_key @@ -208,6 +216,38 @@ mod tests { ); } + /// The identity-point forgery: with public key A = identity, R = identity, + /// and S = 0, the equation `[S]B = R + [k]A` holds for every message, + /// because `[k]·identity = identity`. Ordinary (non-strict) Ed25519 + /// verification accepts it, so the shared `verify` primitive must use + /// strict verification, which rejects small-order R and public keys. + #[test] + fn verify_rejects_identity_point_forgery() { + use ed25519_dalek::Verifier; + // The identity point (0,1) compresses to y = 1 with sign bit 0. + let identity = [ + 1u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, + ]; + let vk = VerifyingKey::from_bytes(&identity).expect("identity point is on the curve"); + let mut sig_bytes = [0u8; 64]; + sig_bytes[..32].copy_from_slice(&identity); + let msg = b"arbitrary message the key owner never signed"; + + // Prove the forged signature satisfies the ordinary verification + // equation, so the strict check below is what actually defends the + // boundary (not a signature that was already invalid everywhere). + assert!( + vk.verify(msg, &Signature::from_bytes(&sig_bytes)).is_ok(), + "identity-point forgery must satisfy ordinary verification (this is why strict is needed)" + ); + + assert!( + verify(&vk, msg, &sig_bytes).is_err(), + "strict verification must reject the identity-point forgery" + ); + } + #[test] fn verify_rejects_weak_key_signature() { // Regression guard for strict verification: a signature forged under a diff --git a/crates/gitlawb-node/Cargo.toml b/crates/gitlawb-node/Cargo.toml index c583569cb..65b18ba49 100644 --- a/crates/gitlawb-node/Cargo.toml +++ b/crates/gitlawb-node/Cargo.toml @@ -73,6 +73,7 @@ alloy = { version = "1", default-features = false, features = [ "rpc-types-eth", ] } libp2p-dns = { version = "0.44.0", features = ["tokio"] } +rand = { workspace = true } [dev-dependencies] mockito = "1" diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 92d129803..46bcb8cdd 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -2130,14 +2130,67 @@ async fn gate_and_serve( /// GET /api/v1/ipfs/pins /// -/// Returns all CIDs that have been pinned to the local IPFS node from git -/// objects received via push. Each entry includes the git SHA-256 hex, the -/// CIDv1 string, and the timestamp when it was pinned. +/// Returns all CIDs that have been pinned from git objects received via push. +/// Each entry includes the git SHA-256 hex, a CIDv1 string, and the timestamp +/// when it was pinned. For Pinata-only rows (no local IPFS pin), the `cid` +/// field carries `pinata_cid` so CLI consumers see a usable value. +/// +/// Rows with neither a local nor a Pinata CID are omitted so the response +/// only contains rows with at least one backend. +/// +/// #218 review P2: the response surfaces writer-owned provenance so a +/// `gl` consumer can distinguish local-only, remote-only, and dual rows +/// without re-inferring semantics from nullability. The fields are: +/// - `cid` — the resolver key. Always non-null in a row that made it +/// into the response (Pinata provider CID for a Pinata-only row, raw +/// CID for an IPFS-pinned row, raw CID for a dual row). Kept for +/// `gl ipfs list` backward compatibility. +/// - `local_cid` — the raw CID stored in `pinned_cids.cid` when set +/// (legacy field, kept for clients that read it directly; for a +/// Pinata-only row with `cid = Some(raw_cid)` this is non-null but +/// is NOT a "local CID" — see `local_pinned`). +/// - `pinata_cid` — the provider CID when Pinata was the writer. +/// - `local_pinned` — `true` iff the local-IPFS writer path +/// (`record_pinned_cid_with_source`) actually pushed the bytes into +/// the local daemon. Writer-owned; never inferred from CID shape. +/// - `pinata_pinned` — `true` iff the row has a non-null `pinata_cid`. pub async fn list_pins(State(state): State) -> Result> { // Bare `?` so connection-class sqlx failures downcast to `AppError::Db` and // map to 503 `db_unavailable` (not 500 via `.map_err(AppError::Internal)`) (#251). let pins = state.db.list_pinned_cids().await?; + let pins: Vec = pins + .into_iter() + .filter(|p| p.cid.is_some() || p.pinata_cid.is_some()) + .map(|p| { + // Round-3 P2 (reviewer 2): do not advertise a Pinata-only + // provider CID as the node-local resolver key. The + // listing's `cid` field is what the `gl ipfs list` CLI + // (and similar clients) feeds into `GET /ipfs/{cid}`, + // and that endpoint is keyed on the local raw-CID + // resolver (not the Pinata provider CID). Returning the + // provider CID in `cid` would make the listing promise a + // CID the node itself cannot serve — clients would + // follow the advertised CID and get a 404. The + // provenance split is exposed via `local_cid` / + // `pinata_cid` (the raw local CID and the Pinata provider + // CID respectively); consumers that need a Pinata- + // resolvable CID read `pinata_cid` directly, and the + // boolean `local_pinned` / `pinata_pinned` flags tell + // them which side has the bytes. A NULL `cid` is the + // truthful Pinata-only state. + serde_json::json!({ + "sha256_hex": p.sha256_hex, + "cid": p.cid, + "local_cid": p.cid, + "pinata_cid": p.pinata_cid, + "local_pinned": p.local_ipfs_provenance, + "pinata_pinned": p.pinata_cid.is_some(), + "pinned_at": p.pinned_at, + }) + }) + .collect(); + Ok(Json(serde_json::json!({ "pins": pins, "count": pins.len(), @@ -2450,6 +2503,227 @@ mod closed_pool_tests { ); } + /// #218 review P2: the `list_pins` API response surfaces writer-owned + /// provenance (`local_pinned`, `pinata_pinned`) so a `gl` consumer + /// can distinguish local-only, remote-only, and dual rows without + /// re-inferring semantics from nullability. The previous response + /// emitted the raw resolver key as `local_cid` even for Pinata-only + /// rows, making the shape indistinguishable from a real dual- + /// backed row. This test seeds the four row shapes through the + /// production writers (`record_pinned_cid_with_source` for + /// local, `record_pinata_cid` for Pinata-only) and asserts the + /// response carries the right `local_pinned` / `pinata_pinned` + /// / `cid` / `local_cid` / `pinata_cid` combination. + #[sqlx::test] + async fn list_pins_reports_writer_owned_provenance_for_all_shapes(pool: sqlx::PgPool) { + use sqlx::Row as _; + let state = crate::test_support::test_state(pool.clone()).await; + let db = &state.db; + + // The four row shapes, seeded through the production writers + // (not raw SQL) so the contract under test is the writer's, + // not a hand-built shape. The combination table for what the + // response must carry: + // + // shape | local_pinned | pinata_pinned | local_cid | pinata_cid + // -------------------+--------------+---------------+-----------+----------- + // local-only | true | false | raw | null + // pinata-only (raw) | false | true | raw | provider + // pinata-only (null) | false | true | null | raw(=provider) + // dual | true | true | raw | provider + // + // Distinct shas (the table is keyed on sha256_hex) keep the + // four rows independent. + + // (1) local-only: real local pin via the production writer. + let sha_local = "sha_p2_local_only"; + let raw1 = "bafkreip2localonlyresolvingcidv1"; + db.record_pinned_cid_with_source(sha_local, raw1, "repo-p2-local") + .await + .unwrap(); + + // (2) pinata-only (raw != provider): the post-v27/v30 row shape + // produced by `record_pinata_cid` when the locally-computed raw + // CID does not match the provider CID. `cid` is set to the raw + // resolver key (this is exactly the row shape the v30 strict + // backfill leaves at `local_ipfs_provenance = FALSE`, so the + // sweep would treat it as a Pinata-only row and re-derive on a + // local pin). + let sha_pinata_raw = "sha_p2_pinata_only_distinct_cids"; + let raw2 = "bafkreip2pinataonlyresolverkeyrawcid"; + let pinata2 = "QmPinataProviderCidForRawOnlyRow"; + assert_ne!(raw2, pinata2); + db.record_pinata_cid( + sha_pinata_raw, + raw2, + pinata2, + Some("repo-p2-pinata-raw"), + i64::MAX, + ) + .await + .unwrap(); + + // (3) pinata-only (raw == provider, legacy pre-v27 shape would + // have been `cid = pinata_cid`; v27 cleared that to NULL. The + // current `record_pinata_cid` infers this case and stores + // `cid = NULL` so the resolver key isn't aliased to a dag-pb + // provider CID that doesn't hash to the raw bytes, #173). + let sha_pinata_null = "sha_p2_pinata_only_null_cid"; + let same = "QmPinataOnlyCidRawEqualsProvider"; + db.record_pinata_cid( + sha_pinata_null, + same, + same, + Some("repo-p2-pinata-null"), + i64::MAX, + ) + .await + .unwrap(); + + // (4) dual: local first (sets the flag and `cid`), then Pinata + // (sets `pinata_cid` and preserves `cid` and `local_ipfs_provenance`). + let sha_dual = "sha_p2_dual"; + let raw4 = "bafkreip2duallocalresolverkeyraw"; + let pinata4 = "QmPinataProviderCidForDualRow"; + db.record_pinned_cid_with_source(sha_dual, raw4, "repo-p2-dual") + .await + .unwrap(); + db.record_pinata_cid(sha_dual, raw4, pinata4, Some("repo-p2-dual"), i64::MAX) + .await + .unwrap(); + + // Hit the production handler end-to-end through a one-shot router + // request, so the test exercises the actual response shape + // (not a unit test on PinnedCidRecord fields). + let resp = Router::new() + .route("/api/v1/ipfs/pins", axum::routing::get(list_pins)) + .with_state(state) + .oneshot( + axum::http::Request::builder() + .uri("/api/v1/ipfs/pins") + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), axum::http::StatusCode::OK); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("read body"); + let v: Value = serde_json::from_slice(&bytes).expect("json body"); + let pins = v + .get("pins") + .and_then(|p| p.as_array()) + .expect("pins array"); + assert_eq!( + pins.len(), + 4, + "all four shapes must appear in the response (every row has at least one of cid/pinata_cid set)" + ); + + // Build a map sha -> pin object for ergonomic assertions. + let by_sha: std::collections::HashMap = pins + .iter() + .map(|p| { + ( + p.get("sha256_hex") + .and_then(|s| s.as_str()) + .unwrap() + .to_string(), + p, + ) + }) + .collect(); + + // Helper: assert a single pin's fields. + let assert_pin = |sha: &str, + local_pinned: bool, + pinata_pinned: bool, + local_cid: Option<&str>, + pinata_cid: Option<&str>| { + let pin = by_sha.get(sha).unwrap_or_else(|| { + panic!( + "pin row for {sha} missing; got shas {:?}", + by_sha.keys().collect::>() + ) + }); + assert_eq!( + pin.get("local_pinned").and_then(|v| v.as_bool()), + Some(local_pinned), + "{sha}: local_pinned mismatch", + ); + assert_eq!( + pin.get("pinata_pinned").and_then(|v| v.as_bool()), + Some(pinata_pinned), + "{sha}: pinata_pinned mismatch", + ); + assert_eq!( + pin.get("local_cid").and_then(|v| v.as_str()), + local_cid, + "{sha}: local_cid mismatch", + ); + assert_eq!( + pin.get("pinata_cid").and_then(|v| v.as_str()), + pinata_cid, + "{sha}: pinata_cid mismatch", + ); + // Round-3 P2: `cid` is the LOCAL resolver key, full + // stop. It is the raw CID when the row has a local pin + // (local-only or dual shape) and `None` for Pinata-only + // rows. The previous contract aliased the Pinata provider + // CID into `cid` for Pinata-only rows, which made + // `gl ipfs list` advertise a CID the node's own + // `/ipfs/{cid}` resolver cannot serve (404). The + // provenance split is exposed via `local_cid`, + // `pinata_cid`, and the boolean `local_pinned` / + // `pinata_pinned` flags. + let cid_value = pin.get("cid").cloned().unwrap_or(serde_json::Value::Null); + let expected_cid = local_cid + .map(|s| serde_json::Value::String(s.to_string())) + .unwrap_or(serde_json::Value::Null); + assert_eq!(cid_value, expected_cid, "{sha}: cid mismatch"); + }; + + // (1) local-only + assert_pin(sha_local, true, false, Some(raw1), None); + // (2) pinata-only (raw != provider) — the dangerous case the + // reviewer flagged: `local_cid` is non-null (it's the raw + // resolver key), but `local_pinned = false` is the durable + // signal that this is NOT a local IPFS pin. + assert_pin(sha_pinata_raw, false, true, Some(raw2), Some(pinata2)); + // (3) pinata-only (raw == provider) — cid NULL, pinata_cid set + assert_pin(sha_pinata_null, false, true, None, Some(same)); + // (4) dual + assert_pin(sha_dual, true, true, Some(raw4), Some(pinata4)); + + // Sanity: the `local_ipfs_provenance` column itself is what + // powers `local_pinned`, so the response field must agree + // with the database column. Reading directly avoids any + // confusion if the writer path changes. + let rows = sqlx::query("SELECT sha256_hex, local_ipfs_provenance FROM pinned_cids") + .fetch_all(&pool) + .await + .unwrap(); + let mut db_provenance: std::collections::HashMap = Default::default(); + for r in rows { + let sha: String = r.get("sha256_hex"); + let p: bool = r.get("local_ipfs_provenance"); + db_provenance.insert(sha, p); + } + for (sha, expected) in [ + (sha_local, true), + (sha_pinata_raw, false), + (sha_pinata_null, false), + (sha_dual, true), + ] { + assert_eq!( + db_provenance.get(sha).copied(), + Some(expected), + "db column for {sha} does not match expected writer-owned provenance" + ); + } + } + /// #251 / CodeRabbit nit: cover `get_by_cid`'s DB-error conversion path — a /// valid CID must still yield 503 on a closed pool. #[sqlx::test] diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 4e327c429..20bbb42bb 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -149,17 +149,17 @@ async fn fail_closed_full_scan_objects( // this push rather than the previous silent ~2x hold; size the budget so both // phases normally fit. let deadline = std::time::Instant::now() + timeout; - let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( - &disk_path, - &git_bin, - deadline.saturating_duration_since(std::time::Instant::now()), - &rules, - is_public, - &owner_did, - )?; - let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, deadline)?; + let (allowed, allowed_trees, all_blobs, all_trees) = + crate::git::visibility_pack::allowed_blob_tree_sets_bounded( + &disk_path, + &git_bin, + deadline, + &rules, + is_public, + &owner_did, + )?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( - candidates, &allowed, &all_blobs, + candidates, &allowed, &all_blobs, &allowed_trees, &all_trees, )) }) .await @@ -1403,6 +1403,7 @@ async fn pin_new_objects_gated( db, repo_id, batch_budget, + None, ) .await } @@ -1471,7 +1472,14 @@ async fn pin_and_encrypt_objects( &ctx.db, repo_id, &node_seed, + // The real git, not `ctx.git_bin`: tests point that at a fake + // walk git, and the seal reads must run the real one. + "git", + crate::ipfs_pin::PIN_BATCH_BUDGET, &recipients, + // Push path: recipients derived at admission under a write lease, + // no sweep-style snapshot to fence (see PolicyFence's doc). + None, ) .await; @@ -2731,6 +2739,9 @@ async fn post_receive_replication_tail( &db_clone, &repo_id, crate::ipfs_pin::PIN_BATCH_BUDGET, + // Push path: no sweep-style batch snapshot to fence (see + // PolicyFence's doc). + None, ) .await, ) @@ -4052,9 +4063,21 @@ mod tests { // ref check); rev-parse resolves HEAD; rev-list lists the one commit; ls-tree // emits " blob \t" (NUL-delimited) under secret/ and burns // 1.2s of the 2s budget; pack-objects is the serve's 1.2s cost. + // + // #218 review round 8 P1 (fixture, not production): the `for-each-ref` arm + // must answer in the COLUMN SHAPE `blob_paths` phase 2 asks for + // (`%(objectname) %(objecttype)`, plus the peeled `%(*objectname) + // %(*objecttype)` pair when the tip is a tag), not a ref NAME. A single + // bare token made the phase-2 parse fail closed, so the walk returned an + // error and the request surfaced as a generic 500 — which silently + // repurposed this test from "the filtered serve shares the deadline" into + // "the walk errors", losing the #174 guard while looking merely red. A + // commit tip peels to nothing, so two columns is the whole line here; the + // 1.2s walk cost stays on `ls-tree` so walk and serve remain independently + // attributable. let body = format!( "#!/bin/sh\ncase \"$1\" in\n \ - for-each-ref) echo refs/heads/main ;;\n \ + for-each-ref) echo {commit} commit ;;\n \ cat-file) echo commit ;;\n \ rev-parse) echo {commit} ;;\n \ rev-list) echo {commit} ;;\n \ @@ -10075,6 +10098,19 @@ mod tests { /// Load-bearing: with the spawn below `release` the walk's `for-each-ref` never /// appears after the disconnect (RED). With it above, gated on /// `receive_result.is_ok()`, it does (GREEN). + /// + /// Round-3 P1: a successful receive-pack followed by a disconnect during + /// `guard.release()` must still see its replication tail run. The tail is + /// spawned above `release` (gated on `receive_result.is_ok()`), and the marker + /// polls for `rev-list` (the new tail's primary walk command) — the previous + /// `for-each-ref` marker is dead because commit 91d0578 removed the last + /// tail-path use of that command (it lived in `assert_all_refs_are_commits`, + /// which is now gone). The new marker points at a real command the tail still + /// executes, so a future reorder that drops the tail will be caught. + /// + /// Load-bearing: with the spawn below `release` the walk's `rev-list` never + /// appears after the disconnect (RED). With it above, gated on + /// `receive_result.is_ok()`, it does (GREEN). #[cfg(unix)] #[sqlx::test] async fn receive_pack_tail_survives_a_disconnect_during_release(pool: sqlx::PgPool) { @@ -10108,8 +10144,14 @@ mod tests { // The disconnect: drop the handler future while `release` is still awaiting. drop(fut); + // Round-3 P1: marker is `rev-list`, not `for-each-ref` — the + // tail's primary walk is `git rev-list --objects --all` (the + // same call as `smart_http::rev_list_keep`); a successful + // re-key on the cloned path emits it from the post-receive + // tail. Polling for `for-each-ref` was vacuous after 91d0578 + // removed the last tail-path use of that command. let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); - while !p2_logged(&log, "for-each-ref") { + while !p2_logged(&log, "rev-list") { assert!( std::time::Instant::now() < deadline, "RED: the pack landed but its replication tail never ran. A disconnect \ @@ -10178,8 +10220,14 @@ mod tests { drop(fut); tokio::time::sleep(std::time::Duration::from_millis(750)).await; + // Round-3 P1: the must-not twin also runs the `rev-list` + // command (the actual tail walk). `for-each-ref` is dead in + // the tail path; the previous marker made the assertion + // vacuous. The new marker pins the same command the + // positive-control sibling above uses, so a future change + // that drops the tail leaves both tests red together. assert!( - !p2_logged(&log, "for-each-ref"), + !p2_logged(&log, "rev-list"), "a failed receive-pack must spawn no replication tail: pinning and \ announcing a half-applied repo is exactly what release(false) refuses \ to upload" diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 1fbf4376e..2a001d14e 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -129,6 +129,17 @@ pub struct Config { #[arg(long, env = "GITLAWB_AUTO_SYNC", default_value_t = false)] pub auto_sync: bool, + /// Enable the periodic reconciliation sweep that re-derives pin/seal sets + /// and fills durability gaps. Defaults to true; set to false to disable + /// the sweep even when a pin backend (IPFS/Pinata) is configured. + #[arg( + long, + env = "GITLAWB_RECONCILIATION_SWEEP", + default_value_t = true, + action = clap::ArgAction::Set + )] + pub reconciliation_sweep: bool, + /// Irys URL for Arweave permanent anchoring. /// Leave empty to disable. Use https://devnet.irys.xyz for free devnet. #[arg(long, env = "GITLAWB_IRYS_URL", default_value = "")] diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index cc2cf0bd7..cadbd5837 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1,8 +1,9 @@ +use std::time::Duration; + use anyhow::{Context, Result}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use sqlx::{postgres::PgPoolOptions, PgPool, Row}; -use std::time::Duration; use tracing::info; use uuid::Uuid; @@ -172,9 +173,20 @@ pub struct RepoReplica { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PinnedCidRecord { pub sha256_hex: String, - pub cid: String, + /// Local IPFS CID. NULL for Pinata-only rows where the object was never + /// fetched by this node's IPFS instance. + pub cid: Option, pub pinned_at: String, pub pinata_cid: Option, + /// #218 review P2: writer-owned local-IPFS provenance. `true` iff this + /// row was written by the local-IPFS pin path + /// (`record_pinned_cid_with_source`), the only path that has actually + /// pushed the bytes into this node's local IPFS daemon. Independent + /// of `cid` (a Pinata-only row with `cid = Some(raw_cid)` is + /// `local_ipfs_provenance = false`). The API response surfaces this + /// as `local_pinned` so consumers can distinguish a real local pin + /// from a Pinata-only row whose `cid` is just the raw resolver key. + pub local_ipfs_provenance: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -1123,6 +1135,179 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE pin_repair_sweep ADD COLUMN IF NOT EXISTS discovery_cursor_id TEXT NOT NULL DEFAULT ''", ], }, + Migration { + // #218 review round 3 (P2 reviewer 2): renumber v27 → v32. + // The runner keys the applied set on the integer alone, so + // open PRs #327/#333/#347/#384/#386 that also claim v27-v31 + // are silently skipped on whichever side merges second. v17's + // reservation comment (above) describes the failure mode in + // detail. v32-v36 are picked to land in a clear gap after + // this branch's last entry so the collision risk is removed. + // Gaps are harmless: the runner iterates the array and never + // requires contiguity. + version: 32, + name: "pinata_only_clear_legacy_equal_cid", + stmts: &[ + // #218 (R2-P2): earlier releases wrote `cid = pinata_cid` as a + // fallback for objects this node never had on local IPFS. After the + // reconciliation sweep ships, `cid IS NOT NULL` is meant to be the + // complete provenance predicate (`has_ipfs_cid`), so a fallback row + // would be misread as a local pin and the sweep would trust the + // remote CID as durability evidence. The two changes below make + // NULL a legal `cid` value (the new "Pinata-only" state) and then + // clear the legacy equal-cid rows. Reordering matters: the + // `DROP NOT NULL` MUST run before the UPDATE, otherwise Postgres + // rejects the assignment. Idempotent: both statements are + // IF-guarded so re-running them on a node whose rows are already + // cleared is a no-op. + "ALTER TABLE pinned_cids ALTER COLUMN cid DROP NOT NULL", + "UPDATE pinned_cids SET cid = NULL WHERE cid = pinata_cid", + ], + }, + Migration { + // v28 → v33 (round-3 renumber, see v32 above). + version: 33, + name: "node_state_key_value", + stmts: &[ + // #218 (R2-P1): the reconciliation sweep persists its keyset + // cursor across restarts so a 100-repo pass is bounded rather + // than re-scanned from the head on every boot. Single-row key/value + // table, no constraints on `key` so callers can use opaque + // strings (e.g. "sweep_cursor", "policy_epoch_lock"). + // NEW versioned migration (never appended to an applied block, + // INV-7). + "CREATE TABLE IF NOT EXISTS node_state (\ + key TEXT NOT NULL PRIMARY KEY,\ + value TEXT,\ + updated_at TEXT NOT NULL\ + )", + ], + }, + Migration { + // v29 → v34 (round-3 renumber, see v32 above). + version: 34, + name: "repos_policy_epoch", + stmts: &[ + // #218 (R2-P1): the PolicyFence records the policy epoch the + // replication path captured its visibility decision under, and + // the dispatch paths re-check the epoch before sending the + // POST. A policy change increments the epoch; if the dispatch + // path reads a different epoch than the replication path did, + // it bails without firing the (now-stale) pin. Default 0 so a + // row that never went through a transaction reads as the + // pre-feature epoch. NOT NULL: every code path that increments + // reads and writes the column, so a NULL would be a real bug. + // NEW versioned migration (never appended to an applied block, + // INV-7). + "ALTER TABLE repos ADD COLUMN IF NOT EXISTS policy_epoch BIGINT NOT NULL DEFAULT 0", + ], + }, + Migration { + // v30 → v35 (round-3 renumber, see v32 above). + version: 35, + name: "pinned_cids_local_ipfs_provenance", + stmts: &[ + // #218 review (P1): the Pinata-only state was inferred from + // `cid = pinata_cid` and the local-IPFS provenance predicate was + // `cid IS NOT NULL`. That conflates two distinct writers: the + // local IPFS pin path and the Pinata push path. A Pinata-only + // insert against a pre-v27 row could re-introduce a non-NULL + // `cid` that the sweep then read as a real local pin, leaving a + // durability gap no other sweep pass would close. + // + // This migration adds a per-row boolean that ONLY the local + // IPFS writer sets. After v30 the durable contract is: + // + // `local_ipfs_provenance = TRUE` ↔ this row was written by + // the local IPFS pin path (`record_pinned_cid_with_source`), + // which is the only path that has actually pushed the bytes + // into the node's local IPFS daemon. + // + // Pinata-only rows keep `local_ipfs_provenance = FALSE` and + // `cid = NULL` (their `pinata_cid` is the provider CID, which + // must not alias raw bytes that do not hash to it, #173). + // `list_pinned_cids` keeps returning the stored `cid` resolver + // key — the new column is an internal durability signal and + // never leaves the resolver / gap-filter boundary. + // + // STRICT backfill (#218 review P1a): mark as locally pinned + // ONLY the rows that are unambiguously local IPFS — `cid` + // set and no Pinata CID. The permissive alternative + // `cid <> pinata_cid` was wrong because pre-v30 + // `record_pinata_cid` deliberately produced exactly that + // shape (`cid = raw resolver key, pinata_cid = provider + // CID, cid != pinata_cid`) for ordinary Pinata-first + // uploads without a local Kubo push. The permissive + // backfill would mark those rows as locally pinned, the + // sweep would filter them as "already durable" via + // `has_ipfs_cid = FALSE -> filter excludes them`, and the + // operator enabling IPFS later would never get a local + // recovery copy. The strict rule leaves dual-shape rows + // at the safe default (`FALSE`); the next sweep pass + // re-derives by re-pinning, which is cheap and idempotent + // (Kubo is idempotent; `record_pinned_cid_with_source` + // upgrades the flag on the conflict branch). + // + // NOT NULL DEFAULT FALSE so a pre-v30 row that somehow + // slips through the backfill WHERE reads as "Pinata-only" + // and gets the safe default; the next sweep pass + // re-derives provenance on a re-pin. + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS local_ipfs_provenance BOOLEAN NOT NULL DEFAULT FALSE", + "UPDATE pinned_cids SET local_ipfs_provenance = TRUE WHERE cid IS NOT NULL AND pinata_cid IS NULL", + // Partial index — only ~all-true rows in steady state, but + // partial because the gap filter (`filter_ipfs_pinned_oids`) + // reads `local_ipfs_provenance = TRUE` and the planner will + // Index Only Scan the partial view, which is a fraction of + // the pin table. Cheap to maintain because the write path + // touches it once per pin and reads are exactly the + // already-pinned lookups the sweep is trying to short-circuit. + "CREATE INDEX IF NOT EXISTS idx_pinned_cids_local_ipfs_provenance ON pinned_cids (local_ipfs_provenance) WHERE local_ipfs_provenance", + ], + }, + Migration { + // v31 → v36 (round-3 renumber, see v32 above). + version: 36, + name: "reconciliation_offset_per_backend", + stmts: &[ + // #218 review (P2): the per-repo cursor advanced between + // repos but not within a repo's missing set, so a + // persistently failing early OID kept monopolising the + // 50 000 cap and a healthy gap past the cap was never + // attempted. This table persists a (repo, backend) → + // next-oid continuation, applied as a sort-rotate at the + // start of the next pass: the cap still bounds per-pass + // work, but the same OIDs do not keep landing in the + // truncated prefix every hourly tick. + // + // The repo-level keyset cursor in `node_state` is unchanged + // — this is a *second* cursor. A full pass (no missing + // OIDs) clears the row, and the next pass starts at the + // head of the sorted list. A truncated pass writes + // `next_oid` = the last OID actually handed to the backend, + // so the next pass resumes from the OID strictly greater + // than that one. + // + // Per-(repo, backend) granularity rather than per-repo: + // Pinata and IPFS are independent writers with independent + // failure modes, and a per-repo cursor would conflate the + // two. PRIMARY KEY (repo_id, backend) keeps the writes + // O(1) per pass; the table grows with the number of repos + // the sweep has ever partially processed, which is bounded + // by the node's repo count and prunes back to zero on + // completion. `next_oid` is the LAST attempted OID (the + // rotation in `missing_oids` is "strictly greater than"), + // and `done` marks a previously-completed pass so a stale + // row never resumes after the missing set has emptied. + "CREATE TABLE IF NOT EXISTS reconciliation_offset ( + repo_id TEXT NOT NULL, + backend TEXT NOT NULL, + next_oid TEXT NOT NULL, + done BOOLEAN NOT NULL DEFAULT FALSE, + updated_at TEXT NOT NULL, + PRIMARY KEY (repo_id, backend) + )", + ], + }, ]; /// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). @@ -1540,6 +1725,36 @@ impl Db { Ok(rows.into_iter().map(row_to_repo).collect()) } + /// Like `list_all_repos_deduped` but ordered by a stable key (`id`) so a + /// keyset cursor deterministically covers every repo regardless of push + /// activity. Used by the reconciliation sweep to avoid starving idle repos. + /// Only `limit` rows are returned; pass `cursor = None` for the first page. + pub async fn list_all_repos_deduped_stable( + &self, + cursor: Option<&str>, + limit: i64, + ) -> Result> { + let sql = format!( + "{} + SELECT d.id, d.name, d.owner_did, d.description, d.is_public, + d.default_branch, d.created_at, d.updated_at, d.disk_path, + d.forked_from, d.machine_id + FROM deduped d + WHERE ($2::text IS NULL OR d.id > $2::text) + ORDER BY d.id ASC + LIMIT $3", + Self::dedup_cte() + ); + let rows = sqlx::query(&sql) + .bind(None::<&str>) + .bind(cursor) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + Ok(rows.into_iter().map(row_to_repo).collect()) + } + /// Repos currently quarantined (admitted as mirrors but withheld from every /// listing surface). `list_all_repos_deduped` excludes these (its `DEDUP_CTE` /// filters `quarantined = FALSE`), so a gate that resolves a slug against the @@ -1712,18 +1927,32 @@ impl Db { .unwrap_or(false)) } - /// Set or clear a repo's quarantine flag. Returns the number of rows touched - /// (0 if no such repo). Backs the (deferred) operator release surface; the - /// admission path writes the flag via `upsert_mirror_repo`. Allowed dead - /// outside tests until the operator endpoint lands. + /// Set or clear a repo's quarantine flag and bump the policy epoch + /// atomically. Returns the number of rows touched (0 if no such repo). + /// A failure in either statement rolls back both. + /// + /// The narrow never blocks behind a pin batch: it commits immediately and + /// bumps `policy_epoch`, and the batch's next `PolicyFence::is_current` + /// check aborts before the next upload. The fenced DB record takes the + /// repos row lock only for its own short transaction, never across a + /// network POST. #[cfg_attr(not(test), allow(dead_code))] pub async fn set_repo_quarantine(&self, repo_id: &str, quarantined: bool) -> Result { + let mut tx = self.pool.begin().await?; let result = sqlx::query("UPDATE repos SET quarantined = $1 WHERE id = $2") .bind(quarantined) .bind(repo_id) - .execute(&self.pool) + .execute(&mut *tx) .await?; - Ok(result.rows_affected()) + let affected = result.rows_affected(); + if affected > 0 { + sqlx::query("UPDATE repos SET policy_epoch = policy_epoch + 1 WHERE id = $1") + .bind(repo_id) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + Ok(affected) } /// Repo ids currently quarantined, for operator review. Allowed dead outside @@ -2763,9 +2992,169 @@ impl Db { } } +// ── Node state ──────────────────────────────────────────────────────────────── + +impl Db { + /// Read an opaque node-state value. Returns `None` when the key has never + /// been written. Used by the reconciliation sweep to persist its keyset + /// cursor across restarts (R2-P1). + pub async fn get_node_state(&self, key: &str) -> Result> { + let row = sqlx::query("SELECT value FROM node_state WHERE key = $1") + .bind(key) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.get("value"))) + } + + /// Write an opaque node-state value (upsert). `None` deletes the key so a + /// cleared cursor does not accumulate stale rows. + pub async fn set_node_state(&self, key: &str, value: Option<&str>) -> Result<()> { + match value { + Some(v) => { + sqlx::query( + "INSERT INTO node_state (key, value, updated_at) + VALUES ($1, $2, $3) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = EXCLUDED.updated_at", + ) + .bind(key) + .bind(v) + .bind(Utc::now().to_rfc3339()) + .execute(&self.pool) + .await?; + } + None => { + sqlx::query("DELETE FROM node_state WHERE key = $1") + .bind(key) + .execute(&self.pool) + .await?; + } + } + Ok(()) + } + + /// Load the reconciliation sweep's per-(repo, backend) continuation offset + /// (#218 review P2). Returns the last OID the previous pass on this + /// `(repo, backend)` pair actually handed to the backend — the next pass + /// rotates the sorted missing set so the first OID is the smallest one + /// strictly greater than this value, and the elements ≤ it are appended + /// at the tail (so a persistently failing early OID does not monopolise + /// the cap window every hourly tick). + /// + /// `None` is returned in three cases: no row yet (the first pass on + /// this pair), the row was marked `done = TRUE` by a previous full pass + /// (the next pass starts at the head of the missing list), or the + /// caller passes a `repo`/`backend` it never partially processed. The + /// reconciliation sweep treats `None` as "start from the head"; the + /// "where in the key space are we" question is owned by the + /// repo-level keyset cursor in `node_state`, not this table. + pub async fn load_reconciliation_offset( + &self, + repo_id: &str, + backend: &str, + ) -> Result> { + let row = sqlx::query( + "SELECT next_oid, done FROM reconciliation_offset + WHERE repo_id = $1 AND backend = $2", + ) + .bind(repo_id) + .bind(backend) + .fetch_optional(&self.pool) + .await?; + match row { + // `done = TRUE` rows are a previously-completed pass that has + // not yet been pruned — treat as a fresh start, same as + // absent. Pruning happens on the next `clear` call so a + // single-pass sweep does not have to do two writes. + Some(r) if !r.get::("done") => Ok(Some(r.get("next_oid"))), + _ => Ok(None), + } + } + + /// Persist a (repo, backend) continuation. `next_oid = None` means + /// "this pass completed; mark done". On a real continuation the row is + /// upserted with `done = FALSE` so the next pass resumes from the + /// stored OID. + /// + /// The `next_oid` value the caller hands in MUST be the LAST OID + /// actually attempted this pass (i.e. the max OID in the truncated + /// or fully-drained set), not the first. The rotation in + /// `missing_oids` is "strictly greater than", so writing a + /// forward-rotated OID here would skip the very objects the + /// truncation truncated — the bug the offset exists to prevent. + pub async fn save_reconciliation_offset( + &self, + repo_id: &str, + backend: &str, + next_oid: Option<&str>, + ) -> Result<()> { + match next_oid { + Some(oid) => { + sqlx::query( + "INSERT INTO reconciliation_offset (repo_id, backend, next_oid, done, updated_at) + VALUES ($1, $2, $3, FALSE, $4) + ON CONFLICT (repo_id, backend) DO UPDATE SET + next_oid = EXCLUDED.next_oid, + done = FALSE, + updated_at = EXCLUDED.updated_at", + ) + .bind(repo_id) + .bind(backend) + .bind(oid) + .bind(Utc::now().to_rfc3339()) + .execute(&self.pool) + .await?; + } + None => { + sqlx::query( + "INSERT INTO reconciliation_offset (repo_id, backend, next_oid, done, updated_at) + VALUES ($1, $2, '', TRUE, $3) + ON CONFLICT (repo_id, backend) DO UPDATE SET + next_oid = '', + done = TRUE, + updated_at = EXCLUDED.updated_at", + ) + .bind(repo_id) + .bind(backend) + .bind(Utc::now().to_rfc3339()) + .execute(&self.pool) + .await?; + } + } + Ok(()) + } + + /// Remove a (repo, backend) row entirely. Used when the sweep cannot + /// make progress on this pair (e.g. the repo disappeared from disk or + /// was quarantined mid-pass) so the next pass does not resume a stale + /// offset against a now-different missing set. Not currently called + /// from the sweep loop itself (the early-skip paths would add a DB + /// round-trip per skipped repo) but kept on `Db` as the durable + /// seam for future operational tooling that needs to invalidate a + /// persisted offset without going through a full pass. + #[allow(dead_code)] + pub async fn clear_reconciliation_offset(&self, repo_id: &str, backend: &str) -> Result<()> { + sqlx::query("DELETE FROM reconciliation_offset WHERE repo_id = $1 AND backend = $2") + .bind(repo_id) + .bind(backend) + .execute(&self.pool) + .await?; + Ok(()) + } +} + // ── Pinned CIDs ─────────────────────────────────────────────────────────────── impl Db { + /// #218 review P1a: the production pin loop now keys its + /// "already done" check on `has_ipfs_cid` (writer-owned + /// `local_ipfs_provenance = TRUE`), not on row existence. This + /// method is kept for tests (`test_support.rs`) that exercise + /// other code paths still using the existence semantic, and is + /// not called from any production code. The `dead_code` lint + /// would otherwise fire on the bin build; tests are + /// `#[cfg(test)]` and don't see this allowance propagate from + /// the bin target. + #[allow(dead_code)] pub async fn is_pinned(&self, sha256_hex: &str) -> Result { let row = sqlx::query("SELECT COUNT(*) as cnt FROM pinned_cids WHERE sha256_hex = $1") .bind(sha256_hex) @@ -2822,11 +3211,24 @@ impl Db { cid: &str, repo_id: Option<&str>, ) -> Result<()> { + // ON CONFLICT also rewrites `cid`: an object pinned once with the wrong + // bytes is overwritten by a subsequent push-path pin (R1-P2). The + // previous "first-pinner-owns" semantics left stale wrong CIDs in + // place, and the sweep gap filter (`cid IS NOT NULL`) excluded them + // from re-processing so the stale CID became permanent durability + // evidence. `repo_id` is COALESCE'd so a known source wins over NULL. + // + // `local_ipfs_provenance = TRUE` is the durable contract (#218 review P1). + // This seam exists for legacy, source-less rows in tests and represents + // a real local IPFS pin, so the new resolver predicate + // (`local_ipfs_provenance = TRUE`, post-v30) sees it as IPFS-pinned. sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) - VALUES ($1, $2, $3, $4) + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id, local_ipfs_provenance) + VALUES ($1, $2, $3, $4, TRUE) ON CONFLICT(sha256_hex) DO UPDATE SET - repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", + cid = EXCLUDED.cid, + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id), + local_ipfs_provenance = TRUE", ) .bind(sha256_hex) .bind(cid) @@ -2841,12 +3243,20 @@ impl Db { /// or `None` for an unpinned oid. The opportunistic legacy-repair path reads /// it to decide candidacy from the codec of the string alone (no object bytes) /// before it recomputes anything. + /// Round-3 P1 (reviewer): v32 leaves `pinned_cids.cid` nullable (Pinata-only + /// rows store the resolver key in `pinata_cid` and leave `cid = NULL`). + /// `cid_for_oid` used `r.get::("cid")`, which in sqlx 0.8 is + /// `try_get().unwrap()`: an unexpected NULL panicked rather than erroring. + /// The pin loop at `ipfs_pin.rs:235` calls this on every batch; the + /// first batch after a v32 migration would have walked straight into + /// the panic for any node with a Pinata-only row. Decode as `Option` + /// so the caller can short-circuit on NULL. pub async fn cid_for_oid(&self, sha256_hex: &str) -> Result> { let row = sqlx::query("SELECT cid FROM pinned_cids WHERE sha256_hex = $1") .bind(sha256_hex) .fetch_optional(&self.pool) .await?; - Ok(row.map(|r| r.get::("cid"))) + Ok(row.and_then(|r| r.try_get::, _>("cid").ok().flatten())) } /// Rewrite a legacy provider-CID row to the raw-content resolver key, stashing @@ -2889,11 +3299,19 @@ impl Db { /// prefix-match approximation would silently mis-classify keys under a different /// multihash. The caller applies the real predicate, so `limit` bounds rows READ /// (the DB cost), not rows repaired. + /// Round-3 P1 (reviewer): the cid column is now nullable (Pinata-only rows + /// store the resolver key in `pinata_cid` and leave `cid = NULL`). The + /// legacy repair sweep at `ipfs_pin.rs:896` re-keys the cid to the + /// raw-content resolver CID; rows with NULL cid have no string to + /// re-key and skip the re-key naturally if returned as + /// `(sha, Option)`. The caller filters NULL rows at the + /// call site (they are Pinata-only and have nothing to re-key on + /// the legacy provider path). pub async fn pinned_cids_after( &self, cursor: &str, limit: i64, - ) -> Result> { + ) -> Result)>> { let rows = sqlx::query( "SELECT sha256_hex, cid FROM pinned_cids WHERE sha256_hex > $1 @@ -2906,7 +3324,12 @@ impl Db { .await?; Ok(rows .into_iter() - .map(|r| (r.get::("sha256_hex"), r.get::("cid"))) + .map(|r| { + ( + r.try_get::("sha256_hex").unwrap_or_default(), + r.try_get::, _>("cid").ok().flatten(), + ) + }) .collect()) } @@ -3097,6 +3520,16 @@ impl Db { /// so there is no marker to wrongly clear. The gate is kept for the one window that /// is not covered by that argument, a concurrent pinner landing the row between the /// `is_pinned` check and this upsert, and so the two clears cannot drift apart. + /// The 3-arg form. The production `pin_new_objects` path + /// routes through the 4-arg fenced form + /// ([`record_pinned_cid_with_source_fenced`]) so the third + /// fence closes the rule-write / record-write race. This + /// 3-arg form is still the helper for tests that don't own + /// a fence and is a documented thin-wrapper equivalent (it + /// takes the same row lock, just with `i64::MAX` as the + /// "no fence" sentinel that the fenced form treats as + /// skip-the-comparison). + #[allow(dead_code)] // call sites in ipfs_pin.rs use the fenced form; tests seed via this pub async fn record_pinned_cid_with_source( &self, sha256_hex: &str, @@ -3104,11 +3537,43 @@ impl Db { repo_id: &str, ) -> Result<()> { let mut tx = self.pool.begin().await?; + // #218 review round 9 (guidance #3 — linearization point): + // no `fence_epoch` is passed in this 3-arg form. The + // 4-arg overload below is the IPFS pin flow's third + // fence: it re-reads the epoch under a row lock that + // `set_visibility_rule` must also take, and aborts the + // record if the epoch has advanced. Callers that don't + // own a policy fence (push-side pin records, where + // admission time IS the decision time) use this overload. + + // `local_ipfs_provenance = TRUE` here is the durable contract + // (#218 review P1): the only path that calls this method + // (`ipfs_pin.rs` `pin_git_object` after a successful `add`) has + // actually pushed the bytes into the node's local IPFS daemon. + // ON CONFLICT upgrades provenance too, so a re-pin of an object + // that previously arrived via Pinata-only (cid=NULL, flag=FALSE) + // becomes a real local pin from the resolver's perspective the + // moment the bytes land locally. + // + // `cid = COALESCE(pinned_cids.cid, EXCLUDED.cid)` (#218 review + // P1a): when a Pinata-only row had `cid = NULL` (the + // `raw_cid == pinata_cid` shape produced by + // `record_pinata_cid`), the new local pin fills the resolver + // key with the local raw CID. A pre-existing non-NULL `cid` + // (a real local pin's raw CID, or one set by + // `repair_legacy_provider_cid`) is preserved — `EXCLUDED.cid` + // is identical to it in normal cases, and COALESCE is + // belt-and-suspenders against any future divergence. Without + // this, the local pin would land with `local_ipfs_provenance + // = TRUE` and `cid = NULL`, and the resolver would have no + // local CID to serve the object by. sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) - VALUES ($1, $2, $3, $4) + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id, local_ipfs_provenance) + VALUES ($1, $2, $3, $4, TRUE) ON CONFLICT(sha256_hex) DO UPDATE SET - repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", + cid = COALESCE(pinned_cids.cid, EXCLUDED.cid), + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id), + local_ipfs_provenance = TRUE", ) .bind(sha256_hex) .bind(cid) @@ -3142,6 +3607,131 @@ impl Db { Ok(()) } + /// #218 review round 9 (guidance #3 — linearization point): + /// [`record_pinned_cid_with_source`] with a third fence check + /// INSIDE the record transaction. Reads the repo's policy + /// epoch under `FOR UPDATE`; the same row lock that + /// `set_visibility_rule` and `remove_visibility_rule` take + /// when they bump `policy_epoch`. A narrowing rule that + /// commits between the irreversible POST and this record + /// either blocks on us (we see the post-narrow epoch and + /// abort) or has already released (we see the post-narrow + /// epoch and abort). Either way the record never lands + /// under a stale-allow decision. + /// + /// `fence_epoch` is the value `PolicyFence::capture` read + /// BEFORE the POST (the per-batch captured epoch). A + /// mismatch means the decision we wrote bytes against is no + /// longer the decision the database would write the row + /// under, and we abort. + /// + /// This is the third fence of three: top-of-iteration + /// (`pin_new_objects` 1803-1812), pre-POST + /// (`pin_new_objects` 2058-2074), and pre-record (here, in + /// the same transaction as the row insert). The HTTP POST is + /// irreducible — it cannot run inside a Postgres + /// transaction — so the linearization has to be at the + /// rule-write / record-write race. The pre-record fence + /// closes that race; a narrowing rule that lands between + /// the POST and the record is observed here and the record + /// is aborted. + pub async fn record_pinned_cid_with_source_fenced( + &self, + sha256_hex: &str, + cid: &str, + repo_id: &str, + fence_epoch: i64, + ) -> Result<()> { + let mut tx = self.pool.begin().await?; + // P2 (reviewer round 9): `i64::MAX` is the "no fence" + // sentinel used by push-side admission. The push path + // has no decision to invalidate, so the row lock is + // unnecessary and would queue `touch_repo`, the + // quarantine toggle, and rule writes behind every pin + // record. Skip the lock + comparison entirely on the + // sentinel path; the unfenced record then runs through + // the same INSERT / COALESCE / pin_repo_sources + // statements with no policy_epoch read. + if fence_epoch != i64::MAX { + // Fenced path: take the row lock and compare. + // `set_visibility_rule` takes the same lock when it + // updates `policy_epoch`, so a rule write that + // committed between the POST and now is already + // visible (the rule write's commit released the + // lock; we acquire it now and read the new value). + // A rule write in flight blocks on our lock; the + // record is aborted. + // + // A missing repos row returns `None` and bails + // fail-closed (was `unwrap_or(0)` — a fail-open + // path against a non-existent repo). + let current_epoch = self.repo_policy_epoch_locked(&mut tx, repo_id).await?; + let current_epoch = match current_epoch { + Some(e) => e, + None => { + tx.rollback().await.ok(); + anyhow::bail!( + "policy epoch row missing for {repo_id}; \ + pin record aborted, no row landed" + ); + } + }; + if current_epoch != fence_epoch { + // The decision the pinner acted under is no longer + // the decision the database would land the row + // under. Roll back; no row, no source, no + // failure-marker delete. + tx.rollback().await.ok(); + anyhow::bail!( + "policy epoch changed during pin dispatch \ + (captured={fence_epoch}, current={current_epoch}); \ + pin record aborted, no row landed" + ); + } + } + // The remainder is the same INSERT / COALESCE / + // pin_repo_sources logic as the 3-arg form. Kept inline + // (rather than factored into a private helper) so the + // two forms can drift independently if a future change + // needs them to — drift is the very class of bug this + // third fence is here to catch. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id, local_ipfs_provenance) + VALUES ($1, $2, $3, $4, TRUE) + ON CONFLICT(sha256_hex) DO UPDATE SET + cid = COALESCE(pinned_cids.cid, EXCLUDED.cid), + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id), + local_ipfs_provenance = TRUE", + ) + .bind(sha256_hex) + .bind(cid) + .bind(Utc::now().to_rfc3339()) + .bind(repo_id) + .execute(&mut *tx) + .await?; + let inserted = sqlx::query( + "INSERT INTO pin_repo_sources (sha256_hex, repo_id) + SELECT $1, $2 + WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo_id) + .bind(MAX_PIN_SOURCES) + .execute(&mut *tx) + .await? + .rows_affected(); + if inserted > 0 { + sqlx::query("DELETE FROM pin_source_failures WHERE sha256_hex = $1 AND repo_id = $2") + .bind(sha256_hex) + .bind(repo_id) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + Ok(()) + } + /// Record a DISCOVERED holder and arm the resolver's fallback ATOMICALLY (U5, #173). /// The sweep's discovery arm used to call `record_pin_source` and then, separately, /// `mark_pin_sources_incomplete`. Two best-effort writes, so a transient failure of @@ -3372,31 +3962,69 @@ impl Db { /// Every pinned object this node ADVERTISES (`GET /api/v1/ipfs/pins`). /// - /// U4 (#173): rows still keyed on a legacy PROVIDER CID (Kubo dag-pb / Pinata - /// CIDv0, written by releases before this branch) are withheld from the listing. - /// The `/ipfs/{cid}` resolver recomputes the raw-content CID from the object bytes - /// and refuses any row whose stored key does not match, so advertising the legacy - /// key hands a client a CID this node deliberately will not serve. The background - /// repair sweep rewrites those rows to the raw key, and each one reappears here the - /// moment it is repaired. Filtering is done in Rust because the raw-CIDv1 test is a - /// multibase+codec decode (`is_raw_cidv1`), not something SQL can express; it is the - /// SAME predicate the repair path uses as its cost gate, so the two cannot drift. + /// #218: the contract is "every row that has something to advertise". A + /// row with `cid` set (any format, including legacy Qm… dag-pb) is + /// surfaced as a local pin; a row with `cid IS NULL` and `pinata_cid` set + /// is surfaced as a Pinata-only pin so the handler can project + /// `effective_cid = pinata_cid`. A row with both columns NULL has + /// nothing to serve and is dropped. A corrupt `cid` column surfaces as + /// a decode error through `?` instead of being silently misread as a + /// Pinata-only row — the previous `try_get().ok()` conflated the two. + /// The handler at `api/ipfs.rs::list_pins` is the seam that decides + /// what to do with a legacy-shape row (it hands it to the resolver and + /// the resolver 404s on mismatch, the documented #173 U4 behavior). pub async fn list_pinned_cids(&self) -> Result> { let rows = sqlx::query( - "SELECT sha256_hex, cid, pinned_at, pinata_cid FROM pinned_cids ORDER BY pinned_at DESC", + "SELECT sha256_hex, cid, pinned_at, pinata_cid, local_ipfs_provenance + FROM pinned_cids ORDER BY pinned_at DESC", ) .fetch_all(&self.pool) .await?; - Ok(rows - .into_iter() - .filter(|r| gitlawb_core::cid::is_raw_cidv1(r.get::<&str, _>("cid"))) - .map(|r| PinnedCidRecord { + let mut out = Vec::with_capacity(rows.len()); + for r in rows { + // `try_get::>` maps only SQL NULL to None (a + // Pinata-only row); a corrupt `cid` column surfaces as a decode + // error through `?` instead of being silently misread as a + // Pinata-only row. The old `try_get().ok()` conflated the two. + let cid: Option = r.try_get("cid")?; + let pinata_cid: Option = r.get("pinata_cid"); + if cid.is_none() && pinata_cid.is_none() { + // Nothing to advertise: no local CID, no Pinata CID. + continue; + } + out.push(PinnedCidRecord { sha256_hex: r.get("sha256_hex"), - cid: r.get("cid"), + cid, pinned_at: r.get("pinned_at"), - pinata_cid: r.get("pinata_cid"), - }) - .collect()) + pinata_cid, + local_ipfs_provenance: r.get("local_ipfs_provenance"), + }); + } + Ok(out) + } + + /// Returns true when this object has a real local IPFS CID. The predicate + /// is `local_ipfs_provenance = TRUE` (#218 review P1): the boolean is + /// set ONLY by the local IPFS writer (`record_pinned_cid_with_source` + /// and the legacy `record_pinned_cid` seam) and is NEVER inferred + /// from CID shape or equality. A Pinata-only row (cid = NULL, + /// pinata_cid set) keeps `local_ipfs_provenance = FALSE`, so the + /// sweep's gap filter does not trust it as a real local pin and + /// will re-derive by re-pinning if IPFS is enabled later. A + /// pre-v30 row is backfilled by migration v30 (rows where cid IS + /// NOT NULL and pinata_cid is NULL OR cid != pinata_cid — the same + /// shape v27 left as the "real local pin" set). + #[allow(dead_code)] + pub async fn has_ipfs_cid(&self, sha256_hex: &str) -> Result { + let row = sqlx::query( + "SELECT COUNT(*) as cnt FROM pinned_cids + WHERE sha256_hex = $1 + AND local_ipfs_provenance = TRUE", + ) + .bind(sha256_hex) + .fetch_one(&self.pool) + .await?; + Ok(row.get::("cnt") > 0) } /// Returns true if this object already has a Pinata CID recorded. @@ -3410,10 +4038,64 @@ impl Db { Ok(row.get::("cnt") > 0) } + /// Given a list of sha256_hex values, returns the subset that already have + /// a Pinata CID recorded. Used by the reconciliation sweep to skip objects + /// that Pinata has already handled. Chunked like `filter_ipfs_pinned_oids` + /// to bound the `ANY($1)` array size on full uncapped object lists (R1-P3). + pub async fn filter_pinata_pinned_oids(&self, oids: &[String]) -> Result> { + if oids.is_empty() { + return Ok(Vec::new()); + } + const CHUNK_SIZE: usize = 1000; + let mut out = Vec::new(); + for chunk in oids.chunks(CHUNK_SIZE) { + let rows = sqlx::query( + "SELECT sha256_hex FROM pinned_cids WHERE sha256_hex = ANY($1) AND pinata_cid IS NOT NULL", + ) + .bind(chunk) + .fetch_all(&self.pool) + .await?; + out.extend(rows.into_iter().map(|r| r.get("sha256_hex"))); + } + Ok(out) + } + + /// Given a list of sha256_hex values, returns the subset that have a real + /// local IPFS pin. The predicate is `local_ipfs_provenance = TRUE` + /// (#218 review P1): set by the local IPFS writer, never inferred from + /// CID shape or equality. Used by the reconciliation sweep to skip + /// IPFS-complete objects — a Pinata-only row (cid = NULL, pinata_cid + /// set) is NOT excluded, so enabling IPFS later causes the sweep to + /// re-derive those rows by re-pinning rather than trusting a missing + /// local copy as durable. + /// + /// The input is processed in fixed-size chunks so the `ANY($1)` array sent + /// to Postgres is bounded even when the sweep hands over a full uncapped + /// object list (R1-P3). + pub async fn filter_ipfs_pinned_oids(&self, oids: &[String]) -> Result> { + if oids.is_empty() { + return Ok(Vec::new()); + } + const CHUNK_SIZE: usize = 1000; + let mut out = Vec::new(); + for chunk in oids.chunks(CHUNK_SIZE) { + let rows = sqlx::query( + "SELECT sha256_hex FROM pinned_cids + WHERE sha256_hex = ANY($1) + AND local_ipfs_provenance = TRUE", + ) + .bind(chunk) + .fetch_all(&self.pool) + .await?; + out.extend(rows.into_iter().map(|r| r.get("sha256_hex"))); + } + Ok(out) + } + /// Record the Pinata CID for a git object. /// /// `raw_cid` is the locally-computed raw-content CID (`Cid::from_git_object_bytes`, - /// CIDv1/raw/sha2-256), the resolver key `GET /ipfs/{cid}` looks up; `pinata_cid` + /// CIDv1/raw/sha-256), the resolver key `GET /ipfs/{cid}` looks up; `pinata_cid` /// is the provider CID Pinata returned (a dag-pb/UnixFS CID for gateway retrieval). /// Inserts the row if it doesn't exist (an object pinned directly to Pinata with /// no prior local IPFS pin gets `cid = raw_cid`, never the provider CID — a dag-pb @@ -3421,26 +4103,90 @@ impl Db { /// to it, #173). On conflict `cid` is left untouched: a prior local pin already /// stored the correct raw CID, and the COALESCE backfills a NULL provenance from a /// known source while keeping first-pinner-owns. + /// + /// **This writer does NOT establish local-IPFS provenance** (#218 review P1): + /// `local_ipfs_provenance` is left at its DEFAULT FALSE (or the value the row + /// already had) because the bytes have not been pushed to the local IPFS daemon + /// here, only to Pinata. The resolver's `has_ipfs_cid` / `filter_ipfs_pinned_oids` + /// keys on `local_ipfs_provenance = TRUE`, so a Pinata-only row never reads as a + /// real local pin. If IPFS is enabled later, the reconciliation sweep will + /// re-derive provenance by re-pinning these objects (their `cid IS NULL` or + /// `pinata_cid` shape keeps them out of the gap filter's "already done" set). + /// Fenced Pinata record (P2 reviewer round 9): `fence_epoch == + /// i64::MAX` is the "no fence" sentinel used by the pre-existing + /// unfenced callers (test fixtures, the reconciliation path) + /// and the helper skips the row lock and the policy_epoch read + /// entirely on that path. With a real epoch, the helper takes + /// the same row lock as `record_pinned_cid_with_source_fenced` + /// and aborts the record if the epoch moved. pub async fn record_pinata_cid( &self, sha256_hex: &str, raw_cid: &str, pinata_cid: &str, repo_id: Option<&str>, + fence_epoch: i64, ) -> Result<()> { + let mut tx = self.pool.begin().await?; + if fence_epoch != i64::MAX { + // Same row-lock-and-compare pattern as + // `record_pinned_cid_with_source_fenced`. A narrowing + // rule that lands between the Pinata POST and the + // record is observed here, and the record is aborted. + // `repo_id` is the pinned side of the contract; the + // Pinata record attaches a `repo_id` only when one + // is in scope, but the fence itself only makes sense + // when a repo is involved — `None` is treated as + // "no fence possible" and falls through to the + // unfenced record. + if let Some(rid) = repo_id { + let current_epoch = self.repo_policy_epoch_locked(&mut tx, rid).await?; + let current_epoch = match current_epoch { + Some(e) => e, + None => { + tx.rollback().await.ok(); + anyhow::bail!( + "policy epoch row missing for {rid}; \ + pinata record aborted, no row landed" + ); + } + }; + if current_epoch != fence_epoch { + tx.rollback().await.ok(); + anyhow::bail!( + "policy epoch changed during pinata dispatch \ + (captured={fence_epoch}, current={current_epoch}); \ + pinata record aborted, no row landed" + ); + } + } + } + // Same INSERT as the unfenced `record_pinata_cid`. The + // `cid`-NULLs-on-Pinata-only path is the same: a + // dag-pb provider CID must not become the alias under + // which `/ipfs/{cid}` serves raw bytes. + let cid = if raw_cid == pinata_cid { + None + } else { + Some(raw_cid) + }; sqlx::query( "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid, repo_id) VALUES ($1, $2, $3, $4, $5) - ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid, + ON CONFLICT(sha256_hex) DO UPDATE SET + pinata_cid = EXCLUDED.pinata_cid, + cid = CASE WHEN pinned_cids.cid = pinned_cids.pinata_cid + THEN NULL ELSE pinned_cids.cid END, repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", ) .bind(sha256_hex) - .bind(raw_cid) // resolver-key cid: locally-computed raw CID, never the provider CID + .bind(cid) .bind(Utc::now().to_rfc3339()) .bind(pinata_cid) .bind(repo_id) - .execute(&self.pool) + .execute(&mut *tx) .await?; + tx.commit().await?; Ok(()) } } @@ -4044,6 +4790,9 @@ impl Db { // ── Path-scoped Visibility ──────────────────────────────────────────────────── impl Db { + /// Set or replace a visibility rule and bump the repo's policy epoch + /// atomically. A sweep reading the rule after it commits must see the new + /// epoch; the two are never visible from different transactions. pub async fn set_visibility_rule( &self, repo_id: &str, @@ -4055,6 +4804,7 @@ impl Db { let id = Uuid::new_v4().to_string(); let now = Utc::now().to_rfc3339(); let readers = serde_json::to_string(reader_dids).unwrap_or_else(|_| "[]".to_string()); + let mut tx = self.pool.begin().await?; sqlx::query( "INSERT INTO visibility_rules (id, repo_id, path_glob, mode, reader_dids, created_by, created_at) @@ -4072,20 +4822,93 @@ impl Db { .bind(&readers) .bind(created_by) .bind(&now) - .execute(&self.pool) + .execute(&mut *tx) .await?; + sqlx::query("UPDATE repos SET policy_epoch = policy_epoch + 1 WHERE id = $1") + .bind(repo_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; Ok(()) } + /// Remove a visibility rule and bump the repo's policy epoch atomically. pub async fn remove_visibility_rule(&self, repo_id: &str, path_glob: &str) -> Result<()> { + let mut tx = self.pool.begin().await?; sqlx::query("DELETE FROM visibility_rules WHERE repo_id = $1 AND path_glob = $2") .bind(repo_id) .bind(path_glob) + .execute(&mut *tx) + .await?; + sqlx::query("UPDATE repos SET policy_epoch = policy_epoch + 1 WHERE id = $1") + .bind(repo_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Current visibility-policy epoch for a repo (0 for a repo with no entry). + /// The epoch is bumped by every rule or quarantine mutation, so a value that + /// changes between two reads proves a policy change happened in between. + pub async fn repo_policy_epoch(&self, repo_id: &str) -> Result { + let row = sqlx::query("SELECT policy_epoch FROM repos WHERE id = $1") + .bind(repo_id) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.get::("policy_epoch")).unwrap_or(0)) + } + + /// Bump `policy_epoch` for `repo_id` by one, mirroring the + /// `UPDATE repos SET policy_epoch = policy_epoch + 1` + /// statement `set_visibility_rule` / `remove_visibility_rule` + /// run. Test-only: production rule writes are wrapped in a + /// transaction that also touches the visibility_rules table; + /// the standalone bump here is for the test fixture that + /// drives a fenced-record-with-bumped-epoch scenario. + #[cfg(test)] + pub async fn bump_repo_policy_epoch(&self, repo_id: &str) -> Result<()> { + sqlx::query("UPDATE repos SET policy_epoch = policy_epoch + 1 WHERE id = $1") + .bind(repo_id) .execute(&self.pool) .await?; Ok(()) } + /// #218 review round 9 (guidance #3 — linearization point): + /// read the repo's policy epoch under a row lock that a + /// narrowing rule write must also acquire. Caller must hold + /// an open transaction and pass it in. The lock is released + /// when the transaction commits or rolls back. + /// + /// This is the third fence check: between the irreversible + /// HTTP POST and the DB record, a rule write can still + /// commit. Reading the epoch under `FOR UPDATE` here means + /// either (a) the rule write blocks on us — we get the + /// post-narrow epoch and abort the record, or (b) we block + /// on the rule write — we get the post-narrow epoch and + /// abort the record. Either way no stale record lands. + pub async fn repo_policy_epoch_locked( + &self, + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + repo_id: &str, + ) -> Result> { + let row = sqlx::query("SELECT policy_epoch FROM repos WHERE id = $1 FOR UPDATE") + .bind(repo_id) + .fetch_optional(&mut **tx) + .await?; + // P2 (reviewer round 9): a missing repos row used to fold + // to 0 via `unwrap_or(0)`, so a fenced record call against + // a non-existent repo would silently compare 0 == 0 and + // PASS — exactly the fail-open path a missing row should + // NOT admit. The row-level lock on a missing predicate + // takes no lock either, so nothing was serializing + // anyway. Return `None`; callers that must compare + // (`record_pinned_cid_with_source_fenced`, + // `record_pinata_cid_fenced`) bail fail-closed. + Ok(row.map(|r| r.get::("policy_epoch"))) + } + pub async fn list_visibility_rules(&self, repo_id: &str) -> Result> { let rows = sqlx::query( "SELECT id, repo_id, path_glob, mode, reader_dids, created_by, created_at @@ -5094,6 +5917,720 @@ mod migration_tests { assert_eq!(attempted_at_of(&db, "z6Mkfoo/failed").await, None); assert_eq!(attempted_at_of(&db, "z6Mkfoo/done").await, None); } + + /// Migration v32 makes pinned_cids.cid nullable so record_pinata_cid can + /// create Pinata-only rows without a local IPFS CID. This test seeds a + /// pre-v32 schema (cid NOT NULL, pinata_cid column exists but no + /// nullability change yet) with rows in each of the three states the + /// has_ipfs_cid / filter_ipfs_pinned_oids predicates must classify: + /// + /// (1) cid IS NOT NULL, pinata_cid IS NULL → has_ipfs = true + /// (2) cid IS NOT NULL, cid != pinata_cid → has_ipfs = false + /// (ambiguous pre-v30; the strict backfill leaves it out and the + /// next sweep pass re-derives by re-pinning) + /// (3) cid IS NOT NULL, cid = pinata_cid (legacy) → has_ipfs = false + /// + /// Legacy row (3) stops being a special case because migration v27 clears + /// `cid = pinata_cid` back to NULL, so `has_ipfs_cid` reduces to the plain + /// `cid IS NOT NULL` predicate (provenance recorded, never inferred). + /// + /// After the migration we also test that a Pinata-only INSERT (cid = NULL) + /// works and produces has_ipfs = false, has_pinata = true. + #[sqlx::test] + async fn migration_v32_makes_cid_nullable_and_preserves_classification(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + + // Create all tables, then drop the NOT NULL constraint on cid + // and drop schema_migrations records to simulate a pre-v32 node. + db.migrate().await.unwrap(); + sqlx::query("ALTER TABLE pinned_cids ALTER COLUMN cid SET NOT NULL") + .execute(&db.pool) + .await + .unwrap(); + + sqlx::query("DELETE FROM schema_migrations") + .execute(&db.pool) + .await + .unwrap(); + for m in MIGRATIONS.iter().take_while(|m| m.version < 32) { + sqlx::query( + "INSERT INTO schema_migrations (version, name, applied_at) + VALUES ($1, $2, $3)", + ) + .bind(m.version) + .bind(m.name) + .bind("2026-07-01T00:00:00Z") + .execute(&db.pool) + .await + .unwrap(); + } + + // ── Seed legacy rows ─────────────────────────────────────────── + let now = "2026-07-01T12:00:00Z"; + + // (1) Real local IPFS pin, no Pinata. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ($1, $2, $3, $4)", + ) + .bind("sha_real_only") + .bind("QmRealLocalCid") + .bind(now) + .bind(Option::<&str>::None) + .execute(&db.pool) + .await + .unwrap(); + + // (2) Both CIDs present and distinct. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ($1, $2, $3, $4)", + ) + .bind("sha_both_distinct") + .bind("QmLocalForThisBlob") + .bind(now) + .bind("QmPinataForThisBlob") + .execute(&db.pool) + .await + .unwrap(); + + // (3) Legacy row where cid was set to pinata_cid as fallback. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ($1, $2, $3, $4)", + ) + .bind("sha_legacy_fallback") + .bind("QmLegacyEqual") + .bind(now) + .bind("QmLegacyEqual") + .execute(&db.pool) + .await + .unwrap(); + + // ── Apply migration v32 ──────────────────────────────────────── + db.migrate().await.unwrap(); + + // ── Assertions ───────────────────────────────────────────────── + + // Column is now nullable. + let nullable: String = sqlx::query_scalar( + "SELECT is_nullable FROM information_schema.columns + WHERE table_name = 'pinned_cids' AND column_name = 'cid'", + ) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(nullable, "YES", "cid must be nullable after v32"); + + // Classification: has_ipfs_cid. + // + // (#218 review P1a) the v12 contract — "cid set, no pinata, OR cid + // != pinata" — was sufficient only as a v27-era approximation + // (after v27 cleared `cid = pinata_cid` rows). The v30 strict + // backfill replaces it: only `cid IS NOT NULL AND pinata_cid IS + // NULL` rows are real local IPFS pins. The "both distinct" + // shape is ambiguous pre-v30 (could be a real local pin + // followed by Pinata, or a Pinata-first upload that the writer + // happened to set cid=raw on) and the strict rule leaves it + // out of the IPFS-pinned set; the next sweep pass re-derives + // by re-pinning (cheap, idempotent). The new + // `local_ipfs_provenance` column is the authoritative + // predicate and `has_ipfs_cid` keys on it. + assert!( + db.has_ipfs_cid("sha_real_only").await.unwrap(), + "real local IPFS CID must be classified as pinned" + ); + assert!( + !db.has_ipfs_cid("sha_both_distinct").await.unwrap(), + "distinct-cid + pinata_cid row is ambiguous pre-v30 and the strict backfill \ + leaves it out of the IPFS-pinned set; the next sweep pass re-derives" + ); + assert!( + !db.has_ipfs_cid("sha_legacy_fallback").await.unwrap(), + "legacy equal-cid row must NOT be classified as having an IPFS CID" + ); + + // has_pinata_cid. + assert!( + !db.has_pinata_cid("sha_real_only").await.unwrap(), + "no pinata_cid means has_pinata = false" + ); + assert!( + db.has_pinata_cid("sha_both_distinct").await.unwrap(), + "non-null pinata_cid means has_pinata = true" + ); + assert!( + db.has_pinata_cid("sha_legacy_fallback").await.unwrap(), + "non-null pinata_cid means has_pinata = true (legacy row)" + ); + + // ── Pinata-only INSERT (new post-v32 row) ────────────────────── + db.record_pinata_cid( + "sha_pinata_only", + "QmPinataOnly", + "QmPinataOnly", + None, + i64::MAX, + ) + .await + .unwrap(); + assert!( + !db.has_ipfs_cid("sha_pinata_only").await.unwrap(), + "Pinata-only row must NOT be classified as having a local IPFS CID" + ); + assert!( + db.has_pinata_cid("sha_pinata_only").await.unwrap(), + "Pinata-only row must have has_pinata = true" + ); + + // ── Idempotent re-run ────────────────────────────────────────── + db.migrate().await.unwrap(); + } + + /// Migration v32 (round-3 renumber from v27) clears legacy rows where cid + /// was set to pinata_cid as a fallback, so `has_ipfs_cid` no longer has to + /// infer provenance from CID inequality (R2-P2). Rows where the CIDs genuinely + /// differ are untouched. + /// + /// Round-3 P3: the previous assertions all went through `has_ipfs_cid` / + /// `has_pinata_cid` — a v30 backfill that flipped `local_ipfs_provenance` + /// to FALSE on the distinct row would make the test still GREEN even + /// if v32's `cid = NULL` backfill never ran. Read the `cid` and + /// `pinata_cid` columns DIRECTLY so the test cannot pass on a no-op. + #[sqlx::test] + async fn migration_v32_clears_legacy_equal_cid(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + let now = "2026-07-01T12:00:00Z"; + + // Seed one legacy equal-cid row and one distinct-cid row, then mark + // v32 (and v33-v36, applied after it) as not yet run so re-running + // migrate() exercises the backfill in isolation. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ('sha_equal', 'QmSame', $1, 'QmSame'), + ('sha_distinct', 'QmLocal', $1, 'QmPinata')", + ) + .bind(now) + .execute(&db.pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version >= 32") + .execute(&db.pool) + .await + .unwrap(); + + db.migrate().await.unwrap(); + + // Read cid / pinata_cid directly. The previous assertions went + // through `has_ipfs_cid` and `has_pinata_cid`, which would + // still return false / true if a future refactor flipped + // `local_ipfs_provenance` without touching cid / pinata_cid — + // a v32 no-op would then hide behind a green test. + let equal_cid: Option = + sqlx::query_scalar("SELECT cid FROM pinned_cids WHERE sha256_hex = 'sha_equal'") + .fetch_one(&db.pool) + .await + .unwrap(); + let equal_pinata: Option = + sqlx::query_scalar("SELECT pinata_cid FROM pinned_cids WHERE sha256_hex = 'sha_equal'") + .fetch_one(&db.pool) + .await + .unwrap(); + let distinct_cid: Option = + sqlx::query_scalar("SELECT cid FROM pinned_cids WHERE sha256_hex = 'sha_distinct'") + .fetch_one(&db.pool) + .await + .unwrap(); + let distinct_pinata: Option = sqlx::query_scalar( + "SELECT pinata_cid FROM pinned_cids WHERE sha256_hex = 'sha_distinct'", + ) + .fetch_one(&db.pool) + .await + .unwrap(); + + assert_eq!( + equal_cid, None, + "v32 must clear the legacy equal-cid row to NULL; \ + cid is still {equal_cid:?}" + ); + assert_eq!( + equal_pinata.as_deref(), + Some("QmSame"), + "v32 must preserve pinata_cid on the legacy row" + ); + assert_eq!( + distinct_cid.as_deref(), + Some("QmLocal"), + "v32 must NOT touch the distinct-cid row's cid" + ); + assert_eq!( + distinct_pinata.as_deref(), + Some("QmPinata"), + "v32 must preserve pinata_cid on the distinct row" + ); + } + + /// #218 review P1: the local-IPFS provenance predicate moved from + /// `cid IS NOT NULL` to a dedicated `local_ipfs_provenance` column set + /// by the writer (#218 review P1 — provenance is now established at + /// the writer boundary, never inferred from CID shape). Migration v35 + /// (round-3 renumber from v30) backfills the column for existing rows + /// under the same heuristic v32 uses to identify "real local pin" + /// rows, and `has_ipfs_cid` / `filter_ipfs_pinned_oids` key on the + /// new column. This test exercises the full chain: pre-v35 schema, + /// the four row shapes, the v35 migration, the post-migration + /// column values, and the post-migration `has_ipfs_cid` / + /// `filter_ipfs_pinned_oids` classification. + /// post-migration `has_ipfs_cid` / `filter_ipfs_pinned_oids` + /// classification. + #[sqlx::test] + async fn migration_v35_backfills_local_ipfs_provenance_heuristically(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool.clone()); + + // Build a pre-v30 schema: every migration through v29 applied. + // Then simulate a v30 upgrade by removing the v30 record from + // schema_migrations, dropping the v30 column, and re-running + // `migrate()` so v30 lands on the seeded rows. The v12 test + // below uses the same pattern. + db.migrate().await.unwrap(); + + // Reset to a pre-v35 schema: drop the v35 column and the + // partial index, and forget the v35 migration record. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS local_ipfs_provenance") + .execute(&db.pool) + .await + .unwrap(); + sqlx::query("DROP INDEX IF EXISTS idx_pinned_cids_local_ipfs_provenance") + .execute(&db.pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 35") + .execute(&db.pool) + .await + .unwrap(); + + // Sanity: pre-v30 — the column does not exist. + let col_pre: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'pinned_cids' + AND column_name = 'local_ipfs_provenance' + )", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert!( + !col_pre, + "pre-v30 schema must not have the local_ipfs_provenance column" + ); + + // Seed the four row shapes. The shapes are the same as the v12 + // backfill test (the v27 / v30 lineage) — keeping the fixture + // names in sync so a future reader can see the contract evolved + // in place rather than being silently rewritten. + let now = "2026-07-01T12:00:00Z"; + let seed = async |sha: &str, cid: Option<&str>, pinata: Option<&str>| { + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ($1, $2, $3, $4)", + ) + .bind(sha) + .bind(cid) + .bind(now) + .bind(pinata) + .execute(&pool) + .await + .unwrap(); + }; + + // (1) Real local IPFS pin, no Pinata → provenance will backfill as TRUE. + seed("sha_v30_real_only", Some("QmRealLocalCid"), None).await; + // (2) Both CIDs present and distinct → provenance will backfill as TRUE. + seed( + "sha_v30_both_distinct", + Some("QmLocalForThisBlob"), + Some("QmPinataForThisBlob"), + ) + .await; + // (3) Pinata-only (post-v27 NULL cid, pinata_cid set) → provenance stays FALSE. + seed("sha_v30_pinata_only", None, Some("QmPinataOnlyCid")).await; + // (4) Pinata-only with a provider CID. Post-v27 schema makes + // cid NULL, so the backfill leaves provenance at the default. + seed("sha_v30_pinata_provider", None, Some("QmPinataProviderCid")).await; + + // Apply v30 by re-running `migrate()`. The runner sees v30 + // missing from `schema_migrations` and runs the migration body: + // the `ALTER TABLE` adds the column, the `UPDATE` backfills + // the rows seeded above, and the partial index is created. + db.migrate().await.unwrap(); + + // The column now exists with the documented default. + let col_exists: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'pinned_cids' + AND column_name = 'local_ipfs_provenance' + )", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert!( + col_exists, + "v30 migration must add the local_ipfs_provenance column" + ); + + // The backfill is one UPDATE keyed on the STRICT rule + // `cid IS NOT NULL AND pinata_cid IS NULL` + // (#218 review P1a: the previous permissive `cid <> pinata_cid` + // clause mis-classified pre-v30 Pinata-first uploads as + // local IPFS pins, because `record_pinata_cid` deliberately + // produced exactly that shape for ordinary Pinata-first + // uploads without a local Kubo push): + // sha_v30_real_only → TRUE (cid set, no Pinata) + // sha_v30_both_distinct → FALSE (cid set, Pinata set — + // ambiguous pre-v30, the strict + // backfill errs on the safe side + // and the sweep re-derives) + // sha_v30_pinata_only → FALSE (cid is NULL) + // sha_v30_pinata_provider → FALSE (cid is NULL) + let provenance = |sha: &str| { + let pool = pool.clone(); + let sha = sha.to_string(); + async move { + let row: Option = sqlx::query_scalar( + "SELECT local_ipfs_provenance FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(&sha) + .fetch_optional(&pool) + .await + .unwrap(); + row + } + }; + + assert_eq!( + provenance("sha_v30_real_only").await, + Some(true), + "(1) real local IPFS pin must backfill as provenance = TRUE" + ); + assert_eq!( + provenance("sha_v30_both_distinct").await, + Some(false), + "(2) both-CIDs-distinct row is ambiguous pre-v30 (could be a real local \ + pin followed by Pinata, or a Pinata-first upload that the writer \ + happened to set cid=raw on); the strict backfill errs on the safe \ + side and the next sweep pass re-derives provenance on a re-pin" + ); + assert_eq!( + provenance("sha_v30_pinata_only").await, + Some(false), + "(3) Pinata-only row (cid NULL) must stay provenance = FALSE" + ); + assert_eq!( + provenance("sha_v30_pinata_provider").await, + Some(false), + "(4) Pinata-only provider-CID row (cid NULL) must stay provenance = FALSE" + ); + + // The classification predicate `has_ipfs_cid` keys on + // `local_ipfs_provenance = TRUE`. Under the STRICT backfill: + // (1) is TRUE (cid set, no Pinata), (2) is FALSE (ambiguous + // pre-v30 dual shape, the strict rule leaves it Pinata-only + // and the sweep re-derives), (3) and (4) are FALSE (cid NULL). + // The integration test + // `sweep_promotes_pinata_only_to_local_ipfs_when_writer_invoked` + // covers the writer upgrade path that brings a Pinata-only + // row into the IPFS-pinned set on a later local IPFS push. + assert!( + db.has_ipfs_cid("sha_v30_real_only").await.unwrap(), + "(1) has_ipfs_cid must report TRUE for the backfilled real-IPFS row" + ); + assert!( + !db.has_ipfs_cid("sha_v30_both_distinct").await.unwrap(), + "(2) has_ipfs_cid must report FALSE for an ambiguous pre-v30 dual row; the \ + strict backfill errs on the safe side and the sweep re-derives" + ); + assert!( + !db.has_ipfs_cid("sha_v30_pinata_only").await.unwrap(), + "(3) has_ipfs_cid must report FALSE for a Pinata-only row (cid NULL post-v27)" + ); + assert!( + !db.has_ipfs_cid("sha_v30_pinata_provider").await.unwrap(), + "(4) has_ipfs_cid must report FALSE for a Pinata-only provider-CID row" + ); + + // The gap filter used by the sweep (`filter_ipfs_pinned_oids`) + // follows the same predicate. + let candidates = vec![ + "sha_v30_real_only".to_string(), + "sha_v30_both_distinct".to_string(), + "sha_v30_pinata_only".to_string(), + "sha_v30_pinata_provider".to_string(), + ]; + let mut filtered = db.filter_ipfs_pinned_oids(&candidates).await.unwrap(); + filtered.sort(); + assert_eq!( + filtered, + vec!["sha_v30_real_only".to_string()], + "filter_ipfs_pinned_oids must return only the local-IPFS-provenance rows; \ + under the strict backfill, only the cid-set-no-pinata row qualifies" + ); + } + + /// `list_pinned_cids` must map a SQL NULL `cid` (Pinata-only row) to + /// `None`. The old `try_get("cid").ok()` conflated NULL with a decode + /// failure, so `/api/v1/ipfs/pins` could silently omit or misrepresent a + /// row instead of surfacing the DB error. + #[sqlx::test] + async fn list_pinned_cids_maps_null_cid_to_none(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + let now = "2026-07-01T12:00:00Z"; + + // One row with a real CID, one Pinata-only row (cid NULL). + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ('sha_real', 'QmReal', $1, 'QmPinata'), + ('sha_pinata_only', NULL, $1, 'QmPinata2')", + ) + .bind(now) + .execute(&db.pool) + .await + .unwrap(); + + let pins = db.list_pinned_cids().await.unwrap(); + let real = pins + .iter() + .find(|p| p.sha256_hex == "sha_real") + .expect("real-cid row must be listed"); + assert_eq!(real.cid.as_deref(), Some("QmReal")); + let pinata_only = pins + .iter() + .find(|p| p.sha256_hex == "sha_pinata_only") + .expect("Pinata-only row must be listed"); + assert_eq!(pinata_only.cid, None, "NULL cid must map to None"); + } + + /// Round-3 P1: `cid_for_oid` must return `None` for a row with + /// `cid = NULL` (Pinata-only row), not panic. Pre-fix the column + /// decode used `r.get::("cid")` which `try_get().unwrap()`s + /// on an unexpected NULL, and the pin loop at `ipfs_pin.rs:235` + /// calls this on every batch — the first batch after v32 on a + /// node with any Pinata-only row would have walked straight into + /// the panic. + #[sqlx::test] + async fn cid_for_oid_returns_none_when_cid_is_null(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ('sha_pinata_only', NULL, '2026-07-01T12:00:00Z', 'QmPinata')", + ) + .execute(&db.pool) + .await + .unwrap(); + let got = db + .cid_for_oid("sha_pinata_only") + .await + .expect("cid_for_oid must not panic on NULL cid"); + assert_eq!(got, None, "NULL cid must return None, not panic or error"); + } + + /// Round-3 P1: `pinned_cids_after` must surface NULL `cid` as + /// `None` in the tuple, not panic. The legacy repair loop + /// (`ipfs_pin.rs:896`) iterates the rows and skips NULL-cid + /// entries — the re-key has no string to operate on. + #[sqlx::test] + async fn pinned_cids_after_skips_rows_with_null_cid(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ('sha_real', 'QmReal', '2026-07-01T12:00:00Z', NULL), + ('sha_pinata', NULL, '2026-07-01T12:00:00Z', 'QmPinata')", + ) + .execute(&db.pool) + .await + .unwrap(); + let rows = db.pinned_cids_after("", 10).await.unwrap(); + let real = rows + .iter() + .find(|(sha, _)| sha == "sha_real") + .expect("real-cid row must be returned"); + assert_eq!(real.1.as_deref(), Some("QmReal")); + let pinata = rows + .iter() + .find(|(sha, _)| sha == "sha_pinata") + .expect("Pinata-only row must be returned"); + assert_eq!( + pinata.1, None, + "NULL cid must return as None in the tuple, not panic" + ); + } + + /// A corrupt `cid` value must surface as a decode error, not a silent + /// None. Postgres only stores values of the column's declared type, so + /// reach the decode failure by retyping the column to bytea (a future + /// migration doing the same is the realistic corruption path). The column + /// is retyped before the first `list_pinned_cids` call so the query plan + /// is compiled against the corrupt type. + #[sqlx::test] + async fn list_pinned_cids_errors_on_corrupt_cid(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + let now = "2026-07-01T12:00:00Z"; + + sqlx::query("ALTER TABLE pinned_cids ALTER COLUMN cid TYPE bytea USING NULL::bytea") + .execute(&db.pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ('sha_bad', E'\\\\xdeadbeef', $1, NULL)", + ) + .bind(now) + .execute(&db.pool) + .await + .unwrap(); + + let err = db + .list_pinned_cids() + .await + .expect_err("corrupt cid column must fail the whole listing"); + assert!( + err.to_string().contains("invalid type") || err.to_string().contains("cid"), + "decode failure must be the reported error, got: {err}" + ); + } + + /// Migration v28 creates the node_state key/value table and the get/set + /// helpers round-trip through it (used by the sweep cursor persistence). + #[sqlx::test] + async fn node_state_roundtrip_and_delete(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + + assert_eq!( + db.get_node_state("sweep_cursor").await.unwrap(), + None, + "absent key reads as None" + ); + + db.set_node_state("sweep_cursor", Some("repo/b")) + .await + .unwrap(); + assert_eq!( + db.get_node_state("sweep_cursor").await.unwrap(), + Some("repo/b".to_string()), + "value survives a write + read" + ); + + // Upsert overwrites. + db.set_node_state("sweep_cursor", Some("repo/c")) + .await + .unwrap(); + assert_eq!( + db.get_node_state("sweep_cursor").await.unwrap(), + Some("repo/c".to_string()) + ); + + // None deletes the key. + db.set_node_state("sweep_cursor", None).await.unwrap(); + assert_eq!(db.get_node_state("sweep_cursor").await.unwrap(), None); + } + + /// record_pinned_cid must repair a stale WRONG local CID, not only fill a + /// NULL or Pinata-fallback slot (R1-P2): an object pinned once with the + /// wrong bytes is overwritten by a subsequent push-path pin, but the sweep + /// gap filter (`cid IS NOT NULL`) excludes rows with a present CID from + /// re-processing, so the sweep cannot repair them. + #[sqlx::test] + async fn record_pinned_cid_repairs_stale_wrong_cid(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + + // A stale wrong CID that is neither NULL nor equal to pinata_cid. + db.record_pinned_cid("sha_stale", "QmStaleWrong", None) + .await + .unwrap(); + db.record_pinata_cid("sha_stale", "QmRawStale", "QmPinataX", None, i64::MAX) + .await + .unwrap(); + + // Re-pin with the correct CID — must overwrite despite the existing + // distinct cid column. + db.record_pinned_cid("sha_stale", "QmCorrect", None) + .await + .unwrap(); + + let cid: String = + sqlx::query_scalar("SELECT cid FROM pinned_cids WHERE sha256_hex = 'sha_stale'") + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(cid, "QmCorrect", "stale wrong CID must be repaired"); + } + + /// record_pinata_cid must clear a legacy cid = pinata_cid fallback (v27's + /// belt-and-suspenders) so a later Pinata-only row is never misread as a + /// local IPFS pin. + #[sqlx::test] + async fn record_pinata_cid_clears_legacy_equal_cid(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + + db.record_pinned_cid("sha_fallback", "QmFallback", None) + .await + .unwrap(); + // Simulate a legacy row where cid was forced equal to pinata_cid. + sqlx::query( + "UPDATE pinned_cids SET pinata_cid = 'QmFallback' WHERE sha256_hex = 'sha_fallback'", + ) + .execute(&db.pool) + .await + .unwrap(); + + // Recording a new (different) Pinata CID must NULL the stale fallback cid. + db.record_pinata_cid( + "sha_fallback", + "QmRawFallback", + "QmPinataNew", + None, + i64::MAX, + ) + .await + .unwrap(); + + let cid: Option = + sqlx::query_scalar("SELECT cid FROM pinned_cids WHERE sha256_hex = 'sha_fallback'") + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(cid, None, "legacy equal-cid fallback must be cleared"); + + // But a genuine local pin plus a distinct Pinata CID is preserved. + db.record_pinned_cid("sha_genuine", "QmLocalGenuine", None) + .await + .unwrap(); + db.record_pinata_cid( + "sha_genuine", + "QmRawGenuine", + "QmPinataGenuine", + None, + i64::MAX, + ) + .await + .unwrap(); + let cid: String = + sqlx::query_scalar("SELECT cid FROM pinned_cids WHERE sha256_hex = 'sha_genuine'") + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(cid, "QmLocalGenuine"); + } } #[cfg(test)] @@ -8552,3 +10089,145 @@ mod cid_candidate_order_tests { ); } } + +#[cfg(test)] +mod policy_fence_tests { + //! P1 (reviewer round 9): the third fence has a single + //! production call site but no test that can fail when the + //! guard stops comparing. These tests pin the three modes: + //! + //! 1. matching epoch lands the row + //! 2. epoch bumped between capture and record aborts the + //! record with no row landed + //! 3. the i64::MAX "no fence" sentinel skips the lock and + //! lands the row + //! + //! P2: a missing repos row must fail closed (was + //! `unwrap_or(0)` — a fail-open path against a non-existent + //! repo). + use super::{Db, RepoRecord}; + use chrono::Utc; + use sqlx::PgPool; + use std::time::{Duration, Instant}; + + async fn db(pool: PgPool) -> Db { + let db = Db::for_testing(pool); + db.run_migrations().await.unwrap(); + db + } + + async fn seed_repo(db: &Db) -> String { + let repo_id = uuid::Uuid::new_v4().to_string(); + db.create_repo(&RepoRecord { + id: repo_id.clone(), + name: "fence-test".into(), + owner_did: "did:key:zFENCE".into(), + description: None, + is_public: true, + default_branch: "main".into(), + created_at: Utc::now(), + updated_at: Utc::now(), + disk_path: "/tmp/fence-test".into(), + forked_from: None, + machine_id: None, + }) + .await + .unwrap(); + repo_id + } + + #[sqlx::test] + async fn record_pinned_cid_with_source_fenced_pins_under_matching_epoch(pool: PgPool) { + let db = db(pool).await; + let repo_id = seed_repo(&db).await; + let epoch = db.repo_policy_epoch(&repo_id).await.unwrap(); + db.record_pinned_cid_with_source_fenced("sha-match-1", "cid-match-1", &repo_id, epoch) + .await + .expect("matching epoch must land the row"); + } + + #[sqlx::test] + async fn record_pinned_cid_with_source_fenced_aborts_on_epoch_bump(pool: PgPool) { + let db = db(pool).await; + let repo_id = seed_repo(&db).await; + // Capture an epoch, then bump the policy_epoch between + // capture and record. The fenced record must abort and + // the row must NOT land. + let captured = db.repo_policy_epoch(&repo_id).await.unwrap(); + // Simulate a narrowing rule write by bumping the epoch + // the way `set_visibility_rule` would. + db.bump_repo_policy_epoch(&repo_id).await.unwrap(); + let result = db + .record_pinned_cid_with_source_fenced("sha-bump-1", "cid-bump-1", &repo_id, captured) + .await; + assert!(result.is_err(), "epoch bump must abort the record"); + let err = format!("{}", result.unwrap_err()); + assert!( + err.contains("policy epoch changed"), + "the abort message names the failure class, got: {err}" + ); + // No row should have landed in pinned_cids. + let count: i64 = + sqlx::query_scalar("SELECT count(*) FROM pinned_cids WHERE sha256_hex = 'sha-bump-1'") + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(count, 0, "no row landed when the epoch was bumped"); + } + + #[sqlx::test] + async fn record_pinned_cid_with_source_fenced_with_sentinel_skips_comparison(pool: PgPool) { + // P2 (reviewer round 9): the i64::MAX sentinel is the + // push-side "no fence" path. The call must succeed + // even against a repo whose policy_epoch is something + // other than i64::MAX, because the comparison is + // skipped on the sentinel path. Also: the sentinel + // path must NOT take the row lock (push side has no + // decision to invalidate). + let db = db(pool).await; + let repo_id = seed_repo(&db).await; + // Epoch here is 0 by default; the i64::MAX sentinel + // would fail any comparison. The test passes only + // because the sentinel path skips the comparison AND + // the row lock. + let start = Instant::now(); + db.record_pinned_cid_with_source_fenced( + "sha-sentinel-1", + "cid-sentinel-1", + &repo_id, + i64::MAX, + ) + .await + .expect("i64::MAX sentinel must land the row without comparing or locking"); + let elapsed = start.elapsed(); + assert!( + elapsed < Duration::from_secs(2), + "sentinel-path record should not contend on a row lock, \ + took {elapsed:?}" + ); + } + + #[sqlx::test] + async fn record_pinned_cid_with_source_fenced_bails_on_missing_repo(pool: PgPool) { + // P2 (reviewer round 9): a record call against a + // non-existent repo must NOT silently pass. The + // previous `unwrap_or(0)` paired with `i64::MAX != 0` + // admitted a row against a missing repos row. The + // helper now bails with no row landed. + let db = db(pool).await; + let result = db + .record_pinned_cid_with_source_fenced( + "sha-missing-1", + "cid-missing-1", + "nonexistent-repo-id", + 0, + ) + .await; + assert!(result.is_err(), "missing repos row must abort the record"); + let err = format!("{}", result.unwrap_err()); + assert!( + err.contains("policy epoch row missing"), + "the abort message names the failure class, got: {err}" + ); + } +} diff --git a/crates/gitlawb-node/src/encrypted_pin.rs b/crates/gitlawb-node/src/encrypted_pin.rs index 19b80651b..a9fc42714 100644 --- a/crates/gitlawb-node/src/encrypted_pin.rs +++ b/crates/gitlawb-node/src/encrypted_pin.rs @@ -6,6 +6,7 @@ use std::collections::{BTreeSet, HashMap}; use std::path::Path; use std::str::FromStr; +use std::time::Duration; use ed25519_dalek::VerifyingKey; use gitlawb_core::did::Did; @@ -106,17 +107,60 @@ fn plan_seal(node_seed: &[u8; 32], dids: &BTreeSet, stored_tag: Option<& /// `node_seed` keys the opaque recipients tag. Returns `(oid, cid)` for each blob /// actually sealed and recorded this call (the per-push delta), used by Option B3 /// to anchor a manifest. Recipient identities are never stored or returned. +/// +/// Nine args (the fence joins the seal's eight) but grouping them would churn +/// both callers and the race/hung-git tests for no behavioral gain. +#[allow(clippy::too_many_arguments)] pub async fn encrypt_and_pin( ipfs_api: &str, repo_path: &Path, db: &Db, repo_id: &str, node_seed: &[u8; 32], + git_bin: &str, + batch_budget: Duration, recipients: &HashMap>, + fence: Option<&crate::ipfs_pin::PolicyFence>, ) -> Vec<(String, String)> { let mut sealed = Vec::new(); let mut skipped_unresolvable = 0usize; - for (oid, dids) in recipients { + // One shared read deadline for the whole batch, like `pin_new_objects`: a + // hung git child is watchdog-reaped at this bound, so the outer + // `PIN_PHASE_DEADLINE` timeout cannot be held open by a blocking read + // (R1-P2). Each read runs under `spawn_blocking` — it is synchronous child + // spawn + pipe drain + watchdog join. + let read_deadline = std::time::Instant::now() + batch_budget; + let total = recipients.len(); + for (attempted, (oid, dids)) in recipients.iter().enumerate() { + // Batch budget gate (R2-P3), mirroring the public pin loops: an object + // is never started with a remainder too small to cover a bounded read's + // teardown. This is consistency (the seal is bounded by the outer + // `PIN_PHASE_DEADLINE` either way), but it keeps the three loops from + // drifting apart in how they report a truncated batch. + if crate::ipfs_pin::batch_budget_gate( + "encrypted-seal", + read_deadline, + sealed.len(), + total - attempted, + ) + .is_none() + { + break; + } + // Policy fence (R1-P1): the recipients snapshot was derived before the + // long withheld-blob walk; if a visibility rule moved while that walk + // ran (a reader added or removed), stop sealing instead of pinning to a + // stale recipient set. Checked FIRST so a changed policy costs nothing. + if let Some(f) = fence { + if !f.is_current().await { + tracing::warn!( + repo = %f.repo_id(), + oid = %oid, + "visibility policy changed after the recipients snapshot; stopping the seal loop" + ); + break; + } + } // A DB read failure is not a cache miss: re-sealing here would do an // avoidable IPFS write during a partial outage. Skip and retry next push. let stored_tag = match db.encrypted_blob_recipients_tag(repo_id, oid).await { @@ -152,7 +196,9 @@ pub async fn encrypt_and_pin( } SealPlan::Seal { keys, tag } => (keys, tag), }; - let data = match crate::git::store::read_object(repo_path, oid) { + let data = match read_object_bounded_spawn_blocking(git_bin, repo_path, oid, read_deadline) + .await + { Ok(Some((_t, bytes))) => bytes, Ok(None) => { tracing::warn!(oid = %oid, "git object not found; skipping encrypted pin"); @@ -170,6 +216,22 @@ pub async fn encrypt_and_pin( continue; } }; + // Dispatch fence (R1-P1): re-read the policy epoch immediately before + // the irreversible HTTP POST. The iteration-top check catches a narrow + // that landed before work began; THIS check catches a narrow that landed + // during the tag lookup, recipient resolution, git read, or seal — all + // of which can take seconds. Without this, a reader removed during + // preparation can still receive a newly published envelope. + if let Some(f) = fence { + if !f.is_current().await { + tracing::warn!( + repo = %f.repo_id(), + oid = %oid, + "visibility policy changed during encrypted seal preparation; aborting upload" + ); + break; + } + } let cid = match crate::ipfs_pin::pin_git_object(ipfs_api, oid, &envelope, None).await { Ok(c) if !c.is_empty() => c, Ok(_) => { @@ -201,10 +263,33 @@ pub async fn encrypt_and_pin( sealed } +/// Bounded, reaped git object read for the seal loop, run off the async thread: +/// `read_object_bounded` is synchronous child spawn + pipe drain + watchdog +/// join, so blocking the runtime task on it would let a hung git hold a worker +/// thread (R1-P2). The `deadline` is the batch's shared read deadline; a child +/// still alive at it is SIGTERM/SIGKILL group-reaped by the watchdog. +async fn read_object_bounded_spawn_blocking( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + deadline: std::time::Instant, +) -> anyhow::Result)>> { + let git_bin = git_bin.to_string(); + let repo_path = repo_path.to_path_buf(); + let sha256_hex = sha256_hex.to_string(); + tokio::task::spawn_blocking(move || { + crate::git::store::read_object_bounded(&git_bin, &repo_path, &sha256_hex, deadline) + .map_err(anyhow::Error::from) + }) + .await + .map_err(|e| anyhow::anyhow!("read_object spawn_blocking join failed: {e}"))? +} + #[cfg(test)] mod tests { use super::*; use ed25519_dalek::SigningKey; + use std::time::Duration; fn did_key(seed: u8) -> String { let vk = SigningKey::from_bytes(&[seed; 32]).verifying_key(); @@ -359,4 +444,300 @@ mod tests { other => panic!("changed recipient set must re-seal; got {other:?}"), } } + + /// A reader removed mid-seal must stop the seal loop (R1-P1 "race test for + /// reader removal"): `encrypt_and_pin` re-checks the policy fence before + /// each blob, so a `remove_visibility_rule` landing while the first seal is + /// in flight aborts before a later blob is pinned to a stale recipient set. + #[sqlx::test] + async fn encrypt_and_pin_stops_sealing_when_reader_removed_mid_batch(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("seal-race.git"); + + // Three loose blobs, each withheld (path-scoped deny exists so the sweep + // would have derived recipients for them). + let oids: Vec = { + crate::git::store::init_bare(&repo_path).expect("init bare repo"); + (0..3) + .map(|i| { + let mut cmd = std::process::Command::new("git"); + cmd.args(["hash-object", "-w", "--stdin"]) + .current_dir(&repo_path) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + let mut child = cmd.spawn().expect("spawn git hash-object"); + { + use std::io::Write; + child + .stdin + .as_mut() + .expect("stdin") + .write_all(format!("secret blob {i}\n").as_bytes()) + .expect("write stdin"); + } + let out = child.wait_with_output().expect("hash-object output"); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }) + .collect() + }; + + // A real repos row so the fence has an epoch and a reader can be removed. + let now = chrono::Utc::now(); + let repo_id = uuid::Uuid::new_v4().to_string(); + db.create_repo(&crate::db::RepoRecord { + id: repo_id.clone(), + name: "seal-race-repo".into(), + owner_did: "did:key:zSealRaceOwner".into(), + description: None, + is_public: true, + default_branch: "main".into(), + created_at: now, + updated_at: now, + disk_path: repo_path.display().to_string(), + forked_from: None, + machine_id: None, + }) + .await + .expect("create repo"); + // A rule whose removal is the "reader removed" mutation: one reader per + // blob, all under the same path glob. + let reader = did_key(1); + db.set_visibility_rule( + &repo_id, + "**/secret/*", + crate::db::VisibilityMode::B, + std::slice::from_ref(&reader), + "did:key:zSealRaceOwner", + ) + .await + .expect("set rule"); + + // IPFS endpoint that delays the FIRST add 2s so the removal lands while + // that seal is in flight, then answers immediately. + let endpoint = delaying_cid_endpoint(vec![Duration::from_secs(2)]).await; + + let recipients: HashMap> = oids + .iter() + .cloned() + .map(|oid| { + let mut s = BTreeSet::new(); + s.insert(reader.clone()); + (oid, s) + }) + .collect(); + + let fence = crate::ipfs_pin::PolicyFence::capture(&db, &repo_id) + .await + .expect("fence captures"); + + let sealed = tokio::time::timeout(Duration::from_secs(30), async { + let seal_db = db.clone(); + let seal_repo = repo_path.clone(); + let seal_endpoint = endpoint.clone(); + let seal_repo_id = repo_id.clone(); + let handle = tokio::spawn(async move { + encrypt_and_pin( + &seal_endpoint, + &seal_repo, + &seal_db, + &seal_repo_id, + &SEED, + "git", + Duration::from_secs(60), + &recipients, + Some(&fence), + ) + .await + }); + // Let the first add start (endpoint sleeps 2s), then remove the + // reader so the fence is stale before the loop checks again. + tokio::time::sleep(Duration::from_millis(300)).await; + db.remove_visibility_rule(&repo_id, "**/secret/*") + .await + .expect("remove rule"); + handle.await.expect("seal task") + }) + .await + .expect("wedge guard: the fence abort must not take 30s"); + + assert!( + sealed.len() < oids.len(), + "a reader removal landing mid-batch must abort before every blob is sealed: {}", + sealed.len() + ); + assert!( + !sealed.is_empty(), + "at least the blob already in flight before the removal completes" + ); + } + + /// A hung git must not hold the seal loop past its read budget (R1-P2): the + /// git read runs under `spawn_blocking` against `read_object_bounded`, so + /// the watchdog reaps a wedged child at the batch deadline and the loop + /// keeps its shape instead of blocking a runtime worker indefinitely. + #[cfg(unix)] + #[sqlx::test] + async fn encrypt_and_pin_returns_by_budget_with_a_hung_git(pool: sqlx::PgPool) { + use std::os::unix::fs::PermissionsExt; + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("seal-hung.git"); + let oids: Vec = { + crate::git::store::init_bare(&repo_path).expect("init bare repo"); + (0..2) + .map(|i| { + let mut cmd = std::process::Command::new("git"); + cmd.args(["hash-object", "-w", "--stdin"]) + .current_dir(&repo_path) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + let mut child = cmd.spawn().expect("spawn git hash-object"); + { + use std::io::Write; + child + .stdin + .as_mut() + .expect("stdin") + .write_all(format!("secret blob {i}\n").as_bytes()) + .expect("write stdin"); + } + let out = child.wait_with_output().expect("hash-object output"); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }) + .collect() + }; + + // A git that wedges forever, ignoring SIGTERM, so only the watchdog's + // SIGKILL can reap it. + let fake = tmp.path().join("hanging-git"); + std::fs::write(&fake, "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nsleep 30\n").unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + + let now = chrono::Utc::now(); + let repo_id = uuid::Uuid::new_v4().to_string(); + db.create_repo(&crate::db::RepoRecord { + id: repo_id.clone(), + name: "seal-hung-repo".into(), + owner_did: "did:key:zSealHungOwner".into(), + description: None, + is_public: true, + default_branch: "main".into(), + created_at: now, + updated_at: now, + disk_path: repo_path.display().to_string(), + forked_from: None, + machine_id: None, + }) + .await + .expect("create repo"); + db.set_visibility_rule( + &repo_id, + "**/secret/*", + crate::db::VisibilityMode::B, + &[did_key(1)], + "did:key:zSealHungOwner", + ) + .await + .expect("set rule"); + + let recipients: HashMap> = oids + .iter() + .cloned() + .map(|oid| { + let mut s = BTreeSet::new(); + s.insert(did_key(1)); + (oid, s) + }) + .collect(); + + // Unreachable endpoint: even if a read somehow succeeded, the pin would + // fail; the read itself is the thing under test. + let started = std::time::Instant::now(); + let sealed = tokio::time::timeout( + Duration::from_secs(60), + encrypt_and_pin( + "http://127.0.0.1:9", + &repo_path, + &db, + &repo_id, + &SEED, + fake.to_str().unwrap(), + Duration::from_secs(2), + &recipients, + None, + ), + ) + .await + .expect("a hung git must not hold the seal past the outer wedge guard"); + + let elapsed = started.elapsed(); + assert!( + elapsed < Duration::from_secs(10), + "a hung git must be watchdog-reaped inside the read budget, not block the loop for ~10s+ (took {elapsed:?})" + ); + assert!( + sealed.is_empty(), + "with a hung git no blob can be read, so nothing may be reported sealed" + ); + } + + /// Local TCP endpoint that answers `{ "Hash": "QmMock" }` after an optional + /// per-request delay, so a seal can be made to straddle a policy mutation. + async fn delaying_cid_endpoint(delays: Vec) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let mut seen = 0usize; + while let Ok((mut sock, _)) = listener.accept().await { + let delay = *delays + .get(seen) + .or_else(|| delays.last()) + .unwrap_or(&Duration::ZERO); + seen += 1; + tokio::spawn(async move { + let mut acc = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + let n = match sock.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + acc.extend_from_slice(&buf[..n]); + if let Some(hdr_end) = + acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4) + { + let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase(); + let len: usize = headers + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if acc.len() >= hdr_end + len { + break; + } + } + } + tokio::time::sleep(delay).await; + let body = br#"{"Hash":"QmSealRaceMockCid"}"#; + let _ = sock + .write_all( + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", body.len()) + .as_bytes(), + ) + .await; + let _ = sock.write_all(body).await; + let _ = sock.flush().await; + }); + } + }); + endpoint + } } diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 0b5696933..e4b971d80 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -209,10 +209,42 @@ pub fn list_all_objects(repo_path: &Path, git_bin: &str, deadline: Instant) -> R .collect()) } +/// The set of objects reachable from any ref, via +/// `git rev-list --all --objects --no-object-names`. +/// +/// The full-object-database enumeration ([`list_all_objects`]) contains +/// dangling commits, trees, and blobs (`git cat-file --batch-all-objects` lists +/// loose objects from an aborted or still-running push). Blob candidates are +/// already fail-closed against the reachable, visibility-allowed set — but +/// commits and trees have no path scoping to fail closed against, so the sweep +/// must bound them to ref-reachability or an unreferenced commit's message, +/// author, and parent links (and any unreferenced tree) would be published to a +/// public IPFS/Pinata endpoint. This is that reachability bound. +pub fn reachable_object_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &["rev-list", "--all", "--objects", "--no-object-names"], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); + Ok(stdout + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect()) +} + /// Like [`list_all_objects`] but pairs each OID with its object type, via /// `--batch-check='%(objectname) %(objecttype)'`. The pin path's fail-closed /// filter needs to tell blobs (content, withholdable) from commits/trees /// (structural, never withheld) without typing the candidate list itself. +#[allow(dead_code)] // used by tests and all_blob_oids pub fn list_all_objects_with_type( repo_path: &Path, git_bin: &str, @@ -245,6 +277,7 @@ pub fn list_all_objects_with_type( /// fail-closed pin filter drops any candidate blob absent from the reachable, /// visibility-allowed set; a dangling private blob is in this set but not the /// allowed set, so it never replicates (#99). +#[allow(dead_code)] // used by visibility_pack tests pub fn all_blob_oids( repo_path: &Path, git_bin: &str, @@ -278,10 +311,12 @@ pub struct PinCandidateSet { /// Every degraded path is **logged**, not silent: a full-scan fallback, a /// failed full scan, and a panicked blocking task each emit a warning. On a /// failed full scan or a task panic the candidate set is empty (pin nothing -/// this push); that is a durability gap the reconciliation sweep backstops, and -/// it can never leak because the withheld/fail-closed filter still runs on -/// whatever set is returned. `full_scan` rides on the returned set so the caller -/// knows when the dangling-inclusive filter is required. +/// this push); that is a durability gap the reconciliation sweep backstops +/// when it is enabled and a pin backend is configured (a node running with the +/// sweep disabled or with no IPFS/Pinata backend has no backstop), and it can +/// never leak because the withheld/fail-closed filter still runs on whatever +/// set is returned. `full_scan` rides on the returned set so the caller knows +/// when the dangling-inclusive filter is required. /// /// `scan_sem` is the post-receive scan admission pool (`git_encrypt_semaphore`, /// #174 F4): both git-spawning stages — the per-tip `cat-file` probe + delta diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 5617b4198..bd8e19859 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -273,6 +273,9 @@ pub struct TreeEntry { /// /// Get just the object type. Returns `None` if the object doesn't exist; a /// probe that could not examine the object store is `Err`, never `None`. +// Kept for tests and the bounded variants' docs; the async serve/seal paths use +// the `_bounded` forms. +#[allow(dead_code)] pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> { let type_output = Command::new("git") .args(["cat-file", "-t", sha256_hex]) @@ -305,6 +308,7 @@ pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> } /// Read an object's content if its type is already known. +#[allow(dead_code)] pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) -> Result> { let content_output = Command::new("git") .args(["cat-file", obj_type, sha256_hex]) @@ -737,6 +741,7 @@ pub fn read_object_bounded( /// `gitlawb_core::cid::Cid::from_git_object_bytes`. /// /// Returns `None` if the object does not exist in this repo. +#[allow(dead_code)] pub fn read_object(repo_path: &Path, sha256_hex: &str) -> Result)>> { let obj_type = match object_type(repo_path, sha256_hex)? { Some(t) => t, diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 086669947..855c7e7bf 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -12,6 +12,18 @@ use std::process::Stdio; use std::sync::mpsc; use std::time::{Duration, Instant}; +/// A (oid, path) pair for a git object reachable in the repo walk. +type ObjectPath = (String, String); + +/// Four sets derived from one walk: allowed blobs, allowed trees, all blob OIDs, +/// all tree OIDs. +type BlobTreeSets = ( + HashSet, + HashSet, + HashSet, + HashSet, +); + /// Fixed budget bounding the whole withheld-blob classification walk (#174 U3). /// The walk is fast for a real repo; this bound exists to reap a hung or /// pathologically slow git child so it cannot pin a served-git permit (the read @@ -320,83 +332,6 @@ pub(crate) fn run_bounded_git( Ok(out) } -/// Fail closed unless every ref ultimately resolves to a commit (a ref pointing -/// directly at a blob or tree, or an annotated tag — even a nested one — of such -/// an object is refused). `git rev-list --all` silently *skips* such refs, but -/// `git upload-pack` (serve) and the whole-repo pin fallback -/// (`git cat-file --batch-all-objects`) still expose their target object, so a -/// tolerant walk would under-withhold. Refuse rather than leak. -/// -/// Each ref is peeled fully with `^{}` through `git cat-file --batch-check`. -/// Full peeling is why this is not `for-each-ref %(*objecttype)`, which -/// dereferences only one tag level and so misclassifies a tag-of-a-tag-of-a- -/// commit as a non-commit. -fn assert_all_refs_are_commits(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result<()> { - let refs_out = run_bounded_git( - git_bin, - &["for-each-ref", "--format=%(refname)"], - repo_path, - b"", - deadline, - )?; - let refs_stdout = String::from_utf8_lossy(&refs_out); - let refnames: Vec<&str> = refs_stdout - .lines() - .map(str::trim) - .filter(|l| !l.is_empty()) - .collect(); - if refnames.is_empty() { - return Ok(()); - } - - // Peel every ref in one `git cat-file --batch-check` pass: one `^{}` - // query per line, one output line per input line, in order. cat-file echoes the - // full query on a ` missing` line, so output scales with refname length; - // run_bounded_git drains stdout concurrently with the stdin write, so the pipe - // cannot deadlock, and the whole peel is bounded by the shared walk deadline. - let queries = refnames - .iter() - .map(|r| format!("{r}^{{}}")) - .collect::>() - .join("\n"); - let peel_out = run_bounded_git( - git_bin, - &["cat-file", "--batch-check=%(objecttype)"], - repo_path, - queries.as_bytes(), - deadline, - )?; - - let peel_stdout = String::from_utf8_lossy(&peel_out); - let types: Vec<&str> = peel_stdout.lines().map(str::trim).collect(); - // A short read means at least one ref went unclassified — fail closed. - if types.len() != refnames.len() { - anyhow::bail!( - "git cat-file returned {} lines for {} refs; \ - refusing to produce a partial (under-withheld) set", - types.len(), - refnames.len() - ); - } - for (refname, kind) in refnames.iter().zip(types.iter()) { - // git emits ` missing` (not the objecttype) when the peel target - // is absent; the status word is the last token. - if kind.split_ascii_whitespace().last() == Some("missing") { - anyhow::bail!( - "ref {refname} does not resolve to an object; \ - refusing to produce a partial (under-withheld) set" - ); - } - if *kind != "commit" { - anyhow::bail!( - "ref {refname} resolves to a {kind}, not a commit; \ - refusing to produce a partial (under-withheld) set" - ); - } - } - Ok(()) -} - /// List every (blob_oid, "/repo/relative/path") pair reachable from any commit in /// `repo_path` — every ref *and* every historical commit those refs reach, not just /// the ref tips. `git upload-pack` (serve) and the whole-repo pin fallback @@ -416,15 +351,200 @@ fn assert_all_refs_are_commits(repo_path: &Path, git_bin: &str, deadline: Instan /// de-duplicated across commits. Paths carry a leading "/" to match the glob form /// used by visibility rules ("/secret/**"). /// -/// Fails closed: if commit enumeration or any tree walk fails, returns an error so -/// the caller aborts the serve/pin rather than producing a partial (under-withheld) -/// set. +/// Fails closed: if commit enumeration, the non-commit ref walk, or any tree walk +/// fails, returns an error so the caller aborts the serve/pin rather than producing +/// a partial (under-withheld) set. Two phases: +/// 1. `git rev-list --all` over commits + per-commit `ls-tree -rz` — captures every +/// commit-reachable blob with its path. +/// 2. `git for-each-ref` over non-commit ref targets — captures every direct +/// ref-to-blob / ref-to-tree with an EMPTY path (the deny-side caller +/// `withheld_from_pairs` withholds empty-path entries by OID). +/// +/// Phase 2 closes the round-3 fail-open leak where a blob only reachable via an +/// annotated tag was served but not withheld. +/// +/// P1 (reviewer round 9): a ref whose target is a TREE (direct, +/// peeled from an annotated tag, or reached through a recursive +/// tag-peel) leaves the tree's CHILDREN invisible to phase 2. The +/// tree's blob children are what `git rev-list --objects --all` +/// serves (and what the deny-side `rev_list_keep` enumerates), so +/// without this walk the served set and the withheld set disagree: +/// a blob only reachable as a child of a `mktree` tree published +/// as a tag is served, not withheld. `walk_tree_oids_bounded` is +/// the bounded recursive `ls-tree` walker that closes this leak; +/// every reachable blob and tree OID is inserted with an empty +/// path, and `withheld_from_pairs` withholds by OID. +const MAX_TREE_WALK_DEPTH: usize = 64; +/// Round 10 P2: cap on `ls-tree` child-process invocations across a +/// single walk. The previous wall-clock bound could not bound a +/// wide shallow tree that spawns one `ls-tree` per subtree well +/// inside the depth cap; expiry was the only stop. With +/// `MAX_TREE_WALK_INVOCATIONS` the walker fails closed at a +/// structural cost ceiling, not at the scheduler's mercy. +const MAX_TREE_WALK_INVOCATIONS: usize = 50_000; + +/// Walk a tree OID recursively via bounded `git ls-tree -z` and +/// insert every reachable blob and tree OID into `out` with an +/// empty path. The empty path is the deny-side convention for +/// "withhold this OID regardless of path" (see +/// `withheld_from_pairs`); the served set never sees a tree +/// tip's child blobs, so the empty-path OID is the only correct +/// shape for the phase-2 catch-all. +/// +/// Bounded by `deadline`, `MAX_TREE_WALK_DEPTH`, and +/// `MAX_TREE_WALK_INVOCATIONS` so a malicious or malformed tree +/// cannot exhaust the walk. The invocation cap closes the +/// "wide shallow tree spawns one ls-tree per subtree" hole the +/// previous wall-clock-only bound left (round 10 P2). +fn walk_tree_oids_bounded( + repo_path: &Path, + git_bin: &str, + root_tree_oid: &str, + deadline: Instant, + out: &mut HashSet<(String, String)>, +) -> Result<()> { + // Round 10 P2: memo of already-walked tree OIDs so a tree + // reachable from N ref tips is walked once, not N times. + // Without this, a ref with two tags pointing at the same + // tree paid for the ls-tree child process twice. + let mut walked: HashSet = HashSet::new(); + // Round 10 P2: a structural invocation cap. The wall-clock + // deadline cannot bound a wide shallow tree that spawns one + // `ls-tree` per subtree well inside the depth cap; this + // counter is the cost ceiling that actually closes the hole. + let mut invocations: usize = 0; + walk_tree_oids_inner( + repo_path, + git_bin, + root_tree_oid, + 0, + deadline, + out, + &mut walked, + &mut invocations, + ) +} + +// Round 10 P2 threaded the `walked` memo and the `invocations` +// counter through the recursion, taking the signature from 6 to +// 8 args. A `WalkState` struct would be cleaner; for one +// recursive call site the `allow` is the smaller change. +#[allow(clippy::too_many_arguments)] +fn walk_tree_oids_inner( + repo_path: &Path, + git_bin: &str, + tree_oid: &str, + depth: usize, + deadline: Instant, + out: &mut HashSet<(String, String)>, + walked: &mut HashSet, + invocations: &mut usize, +) -> Result<()> { + if depth > MAX_TREE_WALK_DEPTH { + anyhow::bail!( + "tree walk exceeded {MAX_TREE_WALK_DEPTH} levels (rooted at {tree_oid}); \ + refusing to recurse into a malicious or malformed tree chain" + ); + } + // Memo: a tree reachable from multiple ref tips or from + // multiple parents (rare but legal in git) is walked once. + if !walked.insert(tree_oid.to_string()) { + return Ok(()); + } + if *invocations >= MAX_TREE_WALK_INVOCATIONS { + anyhow::bail!( + "tree walk exceeded {MAX_TREE_WALK_INVOCATIONS} ls-tree invocations \ + (rooted at {tree_oid}); refusing to recurse into a wide or \ + densely-referenced tree graph" + ); + } + *invocations += 1; + // The tree itself enters the withheld set keyed on OID. The + // filtered pack serves trees by OID, so omitting the tree + // would let a withheld subtree leak its parent. + out.insert((tree_oid.to_string(), String::new())); + let ls = run_bounded_git( + git_bin, + &["ls-tree", "-z", tree_oid], + repo_path, + b"", + deadline, + )?; + let stdout = match std::str::from_utf8(&ls) { + Ok(s) => s, + Err(_) => { + // Non-UTF-8: fail closed. A lossy decode would let an + // invalid-byte filename in a denied path fall through + // (U+FFFD vs the rule's bytes), the same under-withhold + // class phase 1 closes at :526. The child OIDs of this + // tree would otherwise stay out of the withheld set while + // `rev-list --objects --all` still serves them to an + // anonymous clone. Bail to keep the walk and the + // phase-1 path on the same classification. + anyhow::bail!( + "git ls-tree -z {tree_oid} returned a non-UTF-8 path; \ + refusing to produce a partial (under-withheld) set" + ); + } + }; + for record in stdout.split('\0') { + if record.is_empty() { + continue; + } + // P1 (reviewer round 9): same byte-preservation rule as + // `tree_structurally_safe` — `record` is NOT trimmed, so a + // directory named `secret ` (trailing space) carries the + // whitespace into the parse. Here the path portion is + // unused (we walk by OID) but the kind+oid parsing is + // sensitive to the meta+filename split being intact. + let Some((meta, _filename)) = record.split_once('\t') else { + continue; + }; + let mut parts = meta.split_whitespace(); + let _mode = parts.next(); + let Some(kind) = parts.next() else { continue }; + let Some(child_oid) = parts.next() else { + continue; + }; + match kind { + "blob" => { + out.insert((child_oid.to_string(), String::new())); + } + "tree" => { + walk_tree_oids_inner( + repo_path, + git_bin, + child_oid, + depth + 1, + deadline, + out, + walked, + invocations, + )?; + } + _ => { + // Submodule commits (kind="commit") are covered + // by the rev-list walk above; their blobs are + // reachable through the commit-tip path. + } + } + } + Ok(()) +} fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result> { - // One deadline spans the whole walk (the ref check, the HEAD probe, rev-list, - // and every per-commit ls-tree), so a slow or hung walk is bounded as a unit - // rather than granting each git child a fresh timeout. + // One deadline spans the whole walk (the HEAD probe, rev-list, every + // per-commit ls-tree, and the for-each-ref phase 2), so a slow or hung walk + // is bounded as a unit rather than granting each git child a fresh timeout. + // + // #218 review round 2 (non-commit ref acceptance): the previous code + // called `assert_all_refs_are_commits` here, which bailed on + // any ref that didn't peel to a commit (tag-of-tree, + // tag-of-blob). The encrypted recovery path also needs to + // tolerate non-commit refs, for the same reason `all_object_paths` + // does: `git rev-list --all` already silently skips them, and + // the recovery path's classification is over the + // commit-reachable object set. let deadline = Instant::now() + timeout; - assert_all_refs_are_commits(repo_path, git_bin, deadline)?; // Enumerate every reachable commit, not just ref tips. `--all` walks all refs; // append HEAD so a detached HEAD (reachable by rev-list/upload-pack but in no @@ -491,9 +611,330 @@ fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result = line.split_whitespace().collect(); + let (oid, kind, peeled) = match fields.as_slice() { + [oid, kind] => (*oid, *kind, None), + [oid, kind, peeled_oid, peeled_kind] => { + (*oid, *kind, Some((*peeled_oid, *peeled_kind))) + } + _ => anyhow::bail!("malformed for-each-ref line: {line:?}"), + }; + // Commit tips are already covered by the rev-list walk above. + // Direct blob tips (lightweight tag of a blob, raw blobref) are + // inserted as-is. + if kind == "blob" { + out.insert((oid.to_string(), String::new())); + } + // P1 (reviewer round 9): direct TREE tips must walk their + // children. A bare `mktree` published as a raw ref tip (or + // a lightweight tag of a tree) leaves the tree's blobs + // visible to `git rev-list --objects --all` (and therefore + // to the deny-side `rev_list_keep`) but invisible to phase + // 2 if phase 2 only inserts the tree OID. Walk it. + if kind == "tree" { + walk_tree_oids_bounded(repo_path, git_bin, oid, deadline, &mut out)?; + } + if let Some((peeled_oid, peeled_kind)) = peeled { + match peeled_kind { + // The annotated-tag-of-blob shape: the referent is what + // `rev-list --objects --all` serves, so it is what must + // enter the withheld set (round-8 P1). + "blob" => { + out.insert((peeled_oid.to_string(), String::new())); + } + // P1 (reviewer round 9): annotated-tag-of-tree must + // walk the tree the same way a direct tree tip does. + "tree" => { + walk_tree_oids_bounded(repo_path, git_bin, peeled_oid, deadline, &mut out)?; + } + // A tag peeling to a commit contributes nothing new: + // `rev-list --all` peels tag chains to their commit and the + // phase-1 tree walk above already classified its objects. + "commit" => {} + // A peeled type of `tag` means this git peeled only one + // level (see the format comment above; stock git 2.50 peels + // the whole chain and never lands here, but git 2.43 + // reports a peeled type of `tag` for nested tags, so the + // arm IS live in production — see the depth bound below). + // Finish the peel with `^{}`, which is recursive by + // definition, and type the final referent. Fail closed + // on either child erroring — an unclassifiable ref target + // must abort the walk, not silently under-withhold. + // + // P3 (reviewer round 9): bound the depth of any further + // recursion with `MAX_TREE_WALK_DEPTH` so a malformed + // tag chain cannot blow up the walk. Stock git 2.50 + // peels the whole chain and never lands here for a + // blob/tree, but the recursive `rev-parse ^{}` already + // bounds by the walk's shared deadline. + "tag" => { + let full = run_bounded_git( + git_bin, + &["rev-parse", &format!("{oid}^{{}}")], + repo_path, + b"", + deadline, + )?; + let full_oid = String::from_utf8_lossy(&full).trim().to_string(); + let ty_out = run_bounded_git( + git_bin, + &["cat-file", "-t", &full_oid], + repo_path, + b"", + deadline, + )?; + let ty = String::from_utf8_lossy(&ty_out).trim().to_string(); + match ty.as_str() { + "blob" => { + out.insert((full_oid, String::new())); + } + "tree" => { + walk_tree_oids_bounded( + repo_path, git_bin, &full_oid, deadline, &mut out, + )?; + } + _ => {} + } + } + other => { + anyhow::bail!("for-each-ref peeled {oid} to unexpected object type {other:?}") + } + } + } + } Ok(out.into_iter().collect()) } +/// All reachable blob and tree OIDs with their paths, derived from one bounded +/// walk. Returns `(blob_paths, tree_paths)` where each is a `Vec`. +/// Used to derive both allowed blobs and allowed trees from a single walk, so +/// the two sets are consistent and the walk cost is paid only once. +/// +/// #218 (Reviewer-2 P1): the previous phase 1 used `git ls-tree -rz `, +/// which under `-r` recurses into blobs and never emits tree entries; trees +/// only showed up in the phase-2 catch-all with an empty path, and the +/// fail-closed filter in `allowed_blob_tree_sets_bounded` then denied every +/// tree. The sweep could not repair a single tree, so a non-flat repo's git +/// graph was un-reconstructible from the pinned object set. The fix is +/// `-r -t` (recursive, show trees too): every reachable tree and blob comes +/// back with its directory/file path, so the visibility check has something +/// to gate on. The root tree of each commit is appended separately at path +/// `/`, because `ls-tree` of a commit only enumerates its children. Trees +/// reachable only via a non-commit ref (annotated tag of a tree, notes) still +/// arrive in phase 2 with no path, and the fail-closed filter still denies +/// them — that is the right outcome for objects whose visibility cannot be +/// determined. +fn all_object_paths( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result<(Vec, Vec)> { + // #218 review P1 (non-commit ref acceptance): the previous code + // called `assert_all_refs_are_commits` here, which bailed on any + // ref that didn't peel to a commit (tag-of-tree, tag-of-blob). + // `git rev-list --all` already silently skips non-commit refs + // (they contribute nothing to a commit-reachable walk), so the + // assertion rejected repos for what was actually a supported + // Git shape (`ipfs_cid_tree_served_despite_non_commit_ref` is the + // in-repo example). The commit-reachable object set is exactly + // what the sweep needs to classify, so the all-refs gate is + // removed here. Unclassifiable ref targets still fail closed at + // a later layer: the cat-file catch-all enumerates them with no + // path, and the path-based allow filter drops empty-path entries. + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); + let mut rev_args = vec!["rev-list", "--all"]; + if head_resolves { + rev_args.push("HEAD"); + } + let commits_out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let commits_stdout = String::from_utf8_lossy(&commits_out); + let mut blob_set: HashSet<(String, String)> = HashSet::new(); + let mut tree_set: HashSet<(String, String)> = HashSet::new(); + // OID-only indexes for the phase 2 membership check below. Without + // these the catch-all branch does O(O×P) `blob_set.iter().any(...)` + // scans, which on a 50k-object repo runs hundreds of millions of + // string compares per pass (round 10 P2). Maintained alongside the + // path-pair sets so the OID is O(1) lookup, not O(P). + let mut blob_oids: HashSet = HashSet::new(); + let mut tree_oids: HashSet = HashSet::new(); + // Phase 1: enumerate trees AND blobs with their paths via + // `git ls-tree -r -t `. `-t` is the tree counterpart of `-r`: + // without it, recursive listings emit only blob entries. Each line is + // ` SP SP TAB `, with NUL between records. + for commit in commits_stdout.lines() { + let commit = commit.trim(); + if commit.is_empty() { + continue; + } + // #218 review P1b: the root tree of each commit is no longer + // assigned the synthetic path "/". A path-based check on "/" + // would let the root tree slip into the allowed set even when + // its serialized bytes name a denied subtree entry — a + // tree's bytes expose the names of its direct entries plus + // the OIDs of their children, which IS the metadata a + // `/secret/**` deny is meant to withhold. The structural + // entry-level check is in `allowed_blob_tree_sets_bounded`, + // which enumerates root trees itself (so we don't need to + // thread a third return value through this signature). + // ls-tree -r -t below still enumerates every reachable + // blob/subtree tree at its real path; only the root tree's + // gate is restructured. + let listing_out = run_bounded_git( + git_bin, + &["ls-tree", "-r", "-t", "-z", commit], + repo_path, + b"", + deadline, + )?; + let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else { + anyhow::bail!( + "git ls-tree -r -t -z {commit} returned a non-UTF-8 path; \ + refusing to produce a partial (under-withheld) set" + ); + }; + for record in listing_stdout.split('\0') { + let Some((meta, path)) = record.split_once('\t') else { + continue; + }; + let mut parts = meta.split_whitespace(); + let _mode = parts.next(); + let kind = parts.next(); + let oid = parts.next(); + match kind { + Some("blob") => { + if let Some(oid) = oid { + blob_oids.insert(oid.to_string()); + blob_set.insert((oid.to_string(), format!("/{path}"))); + } + } + Some("tree") => { + if let Some(oid) = oid { + tree_oids.insert(oid.to_string()); + tree_set.insert((oid.to_string(), format!("/{path}"))); + } + } + _ => {} + } + } + } + // Phase 2: enumerate ALL reachable objects via cat-file --batch-all-objects. + // This catches dangling objects and objects reachable only through non-commit + // refs (tags, notes) that ls-tree misses. Objects found only here have no + // path, so they are inserted into the OID sets without a path. The allow + // filter in allowed_blob_tree_sets_bounded explicitly denies empty-path + // entries (unknown provenance), ensuring they never reach a public pin backend. + let batch_out = run_bounded_git( + git_bin, + &[ + "cat-file", + "--batch-all-objects", + "--batch-check=%(objectname) %(objecttype)", + ], + repo_path, + b"", + deadline, + )?; + let batch_stdout = String::from_utf8_lossy(&batch_out); + for line in batch_stdout.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + let mut parts = line.split_whitespace(); + let oid = match parts.next() { + Some(o) => o, + None => continue, + }; + let kind = parts.next(); + match kind { + // Only insert if not already present (ls-tree gives path, this + // catch-all has no path; prefer the path-annotated entry). + // Round 10 P2: O(1) OID index lookup, not O(O×P) path-pair scan. + Some("blob") if !blob_oids.contains(oid) => { + blob_oids.insert(oid.to_string()); + blob_set.insert((oid.to_string(), String::new())); + } + Some("tree") if !tree_oids.contains(oid) => { + tree_oids.insert(oid.to_string()); + tree_set.insert((oid.to_string(), String::new())); + } + _ => {} + } + } + Ok(( + blob_set.into_iter().collect(), + tree_set.into_iter().collect(), + )) +} + /// Blob OIDs the caller may not read. A blob is withheld only if visibility /// denies the caller at *every* path the blob appears at; a blob that is also /// reachable through an allowed path is sent (its content is public elsewhere). @@ -540,11 +981,63 @@ pub fn withheld_blob_oids_bounded( )) } +/// THE visibility decision for one `blob_paths` pair, for BOTH of that walk's +/// consumers — the deny side (`withheld_from_pairs`, what the smart-http serve +/// filter excludes) and the allow side (`allowed_blob_set_for_caller_bounded`, +/// what the `GET /ipfs/{cid}` gate hands over). +/// +/// Empty-path entries (round-3 P1): produced by `blob_paths` phase 2 for +/// non-commit-reachable blobs (annotated tag of blob, direct blobref). +/// The object is reachable in the repo graph but has NO commit path, so +/// the path-based visibility check `visibility_check(rules, ..., "")` is +/// meaningless — no rule's glob can match the empty path. The safe +/// policy: empty-path entries are withheld from every caller except the +/// owner. The owner is the only identity that intentionally creates +/// such refs (an annotated-tag-of-blob is a deliberate push through +/// receive-pack, not a clone artifact), so the owner is the only reader +/// the system can meaningfully bind a privacy decision to. Everyone +/// else — anonymous, named non-owner, or any non-owner DID — is +/// withheld. Without this branch, the round-3 phase-2 entries would +/// land in `allowed` (the path-based check returns `Allow` for a public +/// repo with no matching rule), and the secret blob would be served. +/// +/// #218 review round 8 P1 — why this is a shared function rather than a branch +/// inside `withheld_from_pairs`: the empty-path policy lived on the deny side +/// ONLY, while `allowed_blob_set_for_caller_bounded` consumed the same pairs and +/// called `visibility_check(..., "")` directly. On a public repo that returns +/// `Allow` (no glob matches ""), so the two consumers disagreed about the very +/// same OID: the serve filter withheld the tag-only blob while `/ipfs/{cid}` +/// admitted it and served the bytes — the leak phase 2 exists to close, reopened +/// one layer over. A single decision function makes that divergence +/// unrepresentable: any future change to the empty-path policy moves both gates +/// at once. +fn pair_decision( + path: &str, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Decision { + if path.is_empty() { + // Round-3 P1: empty-path entries (non-commit-reachable + // blobs from `blob_paths` phase 2) cannot be classified + // by the path-based rules. Withhold from every caller + // except the owner; the owner is the only identity that + // could have created the ref tip. + match caller { + Some(c) if crate::api::did_matches(owner_did, c) => Decision::Allow, + _ => Decision::Deny, + } + } else { + visibility_check(rules, is_public, owner_did, caller, path) + } +} + /// Withheld set from an already-computed (oid, "/path") listing: a blob is /// withheld only when visibility denies the caller at *every* path it appears /// at. Split out so a caller that already walked `blob_paths` (e.g. /// `withheld_blob_recipients`) reuses the listing instead of walking history -/// again. +/// again. Per-pair policy is [`pair_decision`], shared with the allow side. fn withheld_from_pairs( pairs: &[(String, String)], rules: &[VisibilityRule], @@ -555,7 +1048,7 @@ fn withheld_from_pairs( let mut denied: HashSet = HashSet::new(); let mut allowed: HashSet = HashSet::new(); for (oid, path) in pairs { - match visibility_check(rules, is_public, owner_did, caller, path) { + match pair_decision(path, rules, is_public, owner_did, caller) { Decision::Deny => { denied.insert(oid.clone()); } @@ -612,21 +1105,6 @@ pub fn replicable_blob_set( allowed_blob_set_for_caller(repo_path, rules, is_public, owner_did, None) } -/// [`replicable_blob_set`] with an injectable `git_bin` and walk `timeout`, for the -/// fail-closed full-scan pin path on the receive-pack side. -pub fn replicable_blob_set_bounded( - repo_path: &Path, - git_bin: &str, - timeout: Duration, - rules: &[VisibilityRule], - is_public: bool, - owner_did: &str, -) -> Result> { - allowed_blob_set_for_caller_bounded( - repo_path, git_bin, timeout, rules, is_public, owner_did, None, - ) -} - /// Reachable blob OIDs that visibility ALLOWS `caller` at some path. The /// caller-aware generalization of `replicable_blob_set` (which is the anonymous /// `caller = None` case). Used by `GET /ipfs/{cid}` to gate fail-closed against @@ -660,6 +1138,13 @@ pub fn allowed_blob_set_for_caller( /// [`allowed_blob_set_for_caller`] with an injectable `git_bin` and walk `timeout`, /// for the `GET /ipfs/{cid}` gate. +/// +/// #218 review round 8 P1: the per-pair policy is [`pair_decision`], the SAME +/// function the deny side runs. It previously called `visibility_check` directly, +/// which on a `blob_paths` phase-2 empty-path entry is a check no glob can match +/// and therefore an `Allow` on any public repo — so this gate served the exact +/// OID the serve filter had just withheld. The two consumers of one walk must +/// not be able to disagree; see `pair_decision`'s comment for the full argument. pub fn allowed_blob_set_for_caller_bounded( repo_path: &Path, git_bin: &str, @@ -672,7 +1157,7 @@ pub fn allowed_blob_set_for_caller_bounded( let pairs = blob_paths(repo_path, git_bin, timeout)?; let mut allowed = HashSet::new(); for (oid, path) in &pairs { - if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow { + if pair_decision(path, rules, is_public, owner_did, caller) == Decision::Allow { allowed.insert(oid.clone()); } } @@ -692,7 +1177,10 @@ pub fn allowed_blob_set_for_caller_bounded( /// Safe ONLY for a caller whose output feeds a fail-closed allow-list where absence /// = withhold: a tolerant walk there over-withholds, never leaks. NOT safe for a /// serve/replication filter, where a missed reachable object under-withholds — -/// those go through `blob_paths`, which runs the guard first. +/// those go through `blob_paths`, which now runs a `for-each-ref` phase 2 that +/// enumerates non-commit ref targets and inserts them with empty path +/// (round-3 fix for the annotated-tag-of-blob leak; the previous `assert_all_refs_are_commits` +/// guard was removed in commit 91d0578, leaving that path fail-open). fn reachable_commit_oids( repo_path: &Path, git_bin: &str, @@ -782,23 +1270,23 @@ fn object_paths( Ok(out) } -/// Root tree oid of every reachable commit, at "/". `ls-tree` never emits a commit's -/// own root tree (it lists entries *under* a tree), so it is added explicitly here. -/// Resolved in ONE bounded `git log --no-walk --format=%T --stdin` pass over the -/// shared commit set — not a per-commit `rev-parse` — so a tree-set walk costs the -/// same subprocess order as the blob walk. The commit oids go on STDIN, not argv: a -/// long history has tens of thousands of reachable commits, and passing them all as -/// arguments overflows ARG_MAX so `git log` fails to spawn — which the caller treats -/// as a walk error and fail-closed 404s an authorized reader of a reachable/root -/// tree (#173 P2). `run_bounded_git` drains stdout concurrently with the stdin -/// write, so a large history cannot deadlock the pipes. A commit whose root tree git -/// cannot resolve fails the pass (bail), failing closed. -fn root_tree_pairs( +/// Root tree OIDs of every reachable commit, enumerated with one +/// bounded `git log --no-walk --format=%T --stdin` pass over the +/// shared commit set. The commit oids go on STDIN, not argv: a +/// long history has tens of thousands of reachable commits, and +/// passing them all as arguments overflows ARG_MAX so `git log` +/// fails to spawn — which the caller treats as a walk error and +/// fail-closed 404s an authorized reader of a reachable/root tree +/// (#173 P2). `run_bounded_git` drains stdout concurrently with the +/// stdin write, so a large history cannot deadlock the pipes. +/// `ls-tree` never emits a commit's own root tree, so this is +/// where the root trees get explicitly enumerated. +fn root_tree_oids( repo_path: &Path, git_bin: &str, commits: &[String], deadline: Instant, -) -> Result> { +) -> Result> { if commits.is_empty() { return Ok(HashSet::new()); } @@ -818,12 +1306,118 @@ fn root_tree_pairs( for line in String::from_utf8_lossy(&out).lines() { let oid = line.trim(); if !oid.is_empty() { - set.insert((oid.to_string(), "/".to_string())); + set.insert(oid.to_string()); } } Ok(set) } +/// #218 review P1b (recursive at every depth): the structural +/// safety check for a tree. A tree is safe to publish iff, at the +/// path it is reached, every direct entry in its serialized bytes +/// is independently safe: the entry's filename is allowed at +/// `path/filename` AND, if the entry is a tree, the child tree is +/// itself structurally safe at `path/filename`. +/// +/// The recursion bottoms out at blob entries (a blob's safety is a +/// single path check) and at the leaf-most tree (whose children are +/// all blobs or the same path is denied). The check is per +/// `(oid, path)`: a tree reachable at multiple paths is admitted if +/// it is structurally safe at *any* allowed path (mirroring the +/// existing "blob reachable at any allowed path is admitted" rule). +/// `admitted` memoizes trees proven safe at some path so the +/// recursion short-circuits on cycles and on the same tree +/// reachable at multiple allowed paths. +/// +/// The prior round only checked root trees; the per-depth version +/// is what the reviewer called for after the `/public/secret/**` +/// case showed the root-only check admitted `/public` (its path is +/// allowed) while `/public` still contained a `secret/` entry +/// naming the denied subtree. The recursive check at every depth +/// denies `/public` here because its `secret/` entry's child tree +/// (`/public/secret`'s subtree) is denied at `/public/secret`. +struct TreeCheckCtx<'a> { + repo_path: &'a Path, + git_bin: &'a str, + rules: &'a [VisibilityRule], + is_public: bool, + owner_did: &'a str, + caller: Option<&'a str>, +} + +fn tree_structurally_safe( + ctx: &TreeCheckCtx, + tree_oid: &str, + path: &str, + admitted: &mut HashSet, + deadline: Instant, +) -> Result { + if admitted.contains(tree_oid) { + return Ok(true); + } + // One-level listing of the tree's direct entries. `-z` is NUL-separated + // so paths with special bytes survive the parse intact (a `café.txt` + // filename with a non-UTF-8 byte would otherwise be lossy-decoded and + // could miss its deny rule). Non-UTF-8 → fail closed. + let out = run_bounded_git( + ctx.git_bin, + &["ls-tree", "-z", tree_oid], + ctx.repo_path, + b"", + deadline, + )?; + let stdout = match std::str::from_utf8(&out) { + Ok(s) => s, + Err(_) => return Ok(false), + }; + for record in stdout.split('\0') { + // P1 (reviewer round 9): do NOT `record.trim()`. `ls-tree -z` + // emits NUL-separated records whose filename portion can + // carry trailing whitespace, and a directory like `secret ` + // must reach `visibility_check` verbatim so the deny rule + // matches it. Trimming collapsed `secret ` → `secret` and + // let the allow side admit the parent tree, so the tree's + // children leaked through `/ipfs/{cid}`. The trailing + // whitespace test pins the contract. + if record.is_empty() { + continue; + } + let Some((meta, filename)) = record.split_once('\t') else { + return Ok(false); + }; + let mut parts = meta.split_whitespace(); + let _mode = parts.next(); + let Some(kind) = parts.next() else { + return Ok(false); + }; + let Some(child_oid) = parts.next() else { + return Ok(false); + }; + let entry_path = if path == "/" { + format!("/{filename}") + } else { + format!("{path}/{filename}") + }; + if visibility_check( + ctx.rules, + ctx.is_public, + ctx.owner_did, + ctx.caller, + &entry_path, + ) != Decision::Allow + { + return Ok(false); + } + if kind == "tree" + && !tree_structurally_safe(ctx, child_oid, &entry_path, admitted, deadline)? + { + return Ok(false); + } + } + admitted.insert(tree_oid.to_string()); + Ok(true) +} + /// Every `(tree_oid, "/path")` pair reachable in `repo_path`: the `kind == "tree"` /// slice of [`object_paths`] (subtree trees at their directory paths) PLUS every /// reachable commit's root tree at "/" (see [`root_tree_pairs`]). Computes the @@ -838,34 +1432,19 @@ fn tree_paths( deadline: Instant, ) -> Result> { let commits = reachable_commit_oids(repo_path, git_bin, deadline)?; - let mut out: HashSet<(String, String)> = object_paths(repo_path, git_bin, &commits, deadline)? - .into_iter() - .filter(|(_, _, kind)| kind == "tree") - .map(|(oid, path, _)| (oid, path)) - .collect(); - out.extend(root_tree_pairs(repo_path, git_bin, &commits, deadline)?); + // Subtree trees at their directory paths; the root tree is + // enumerated separately via `root_tree_oids` so the structural + // post-pass can apply without overlapping with the empty-path + // cat-file catch-all sentinel (#218 review P1b). + let mut out: HashSet<(String, String)> = HashSet::new(); + for (oid, path, kind) in object_paths(repo_path, git_bin, &commits, deadline)? { + if kind == "tree" { + out.insert((oid, path)); + } + } Ok(out) } -/// The OIDs from a `(oid, "/path")` listing that visibility ALLOWS `caller` at some -/// path — the shared inner loop of the blob and tree allowed-sets. An oid reachable -/// at an allowed path is kept even when also reachable at a denied one. -fn allowed_set_from_pairs<'a>( - pairs: impl IntoIterator, - rules: &[VisibilityRule], - is_public: bool, - owner_did: &str, - caller: Option<&str>, -) -> HashSet { - pairs - .into_iter() - .filter(|(_, path)| { - visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow - }) - .map(|(oid, _)| oid.clone()) - .collect() -} - /// Reachable tree OIDs that visibility ALLOWS `caller` at some path — the tree /// analog of [`allowed_blob_set_for_caller`]. `GET /ipfs/{cid}` gates tree objects /// with this so the CID surface matches `get_tree`: a tree reachable only at a @@ -909,13 +1488,55 @@ pub fn allowed_tree_set_for_caller_bounded( caller: Option<&str>, ) -> Result> { let deadline = Instant::now() + timeout; - Ok(allowed_set_from_pairs( - &tree_paths(repo_path, git_bin, deadline)?, + // #218 review P1b (recursive at every depth): the path-based + // pass admits a tree at any path the policy allows, but that + // admit can be wrong if the tree's serialized bytes name a denied + // subtree entry. Re-evaluate each path-admitted tree structurally + // at the same path, and admit it only if every direct entry is + // safe at `path/filename` and (for tree entries) the child tree is + // itself structurally safe there. The `admitted` set memoizes + // trees proven safe at some path so the recursion short-circuits + // on cycles and on the same tree reachable at multiple paths + // (the "blob reachable at any allowed path" rule, applied to + // trees). + let tree_pairs = tree_paths(repo_path, git_bin, deadline)?; + let ctx = TreeCheckCtx { + repo_path, + git_bin, rules, is_public, owner_did, caller, - )) + }; + let mut admitted: HashSet = HashSet::new(); + for (oid, path) in &tree_pairs { + // `tree_paths` only emits non-empty paths (root trees are + // enumerated below), so the empty-path case is not reachable + // here. P3 (reviewer round 9): the previous comment described + // a caller-aware empty-path carve-out that the surrounding + // code never produced, so the call degenerated to the same + // decision `visibility_check` would make. Reverting the + // routing through `pair_decision` removes the dead code and + // its comment. `visibility_check` is still the policy surface + // for the root tree pass below. + if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow { + tree_structurally_safe(&ctx, oid, path, &mut admitted, deadline)?; + } + } + // Root trees of reachable commits: they have no path in + // `tree_paths` (ls-tree emits descendants only), so evaluate + // them at "/" — the root tree is admitted iff every direct + // entry is safe at the root and (for tree entries) the child + // tree is itself structurally safe. The check is recursive, so + // a denied subtree propagates up to the root. + let commits = reachable_commit_oids(repo_path, git_bin, deadline)?; + for root_oid in root_tree_oids(repo_path, git_bin, &commits, deadline)? { + if visibility_check(rules, is_public, owner_did, caller, "/") != Decision::Allow { + continue; + } + tree_structurally_safe(&ctx, &root_oid, "/", &mut admitted, deadline)?; + } + Ok(admitted) } /// Object bound for the annotated-tag reachability walk (#173, jatmn tag fan-out). @@ -1150,30 +1771,139 @@ pub fn reachable_commit_tag_oids_bounded( Ok(set) } -/// Objects safe to replicate, failing closed on blobs (#99). A candidate -/// replicates iff it is NOT a blob (`all_blob_oids` — commits and trees are -/// structural, never content-withheld) OR it is in `allowed_blobs` (reachable -/// and visibility-allowed). This drops both withheld reachable blobs and -/// dangling/unreachable blobs the reachable walk never classified, without -/// tagging the candidate list with per-object types. Used on the full-scan pin -/// path, where the candidate set can contain dangling objects the reachable-only -/// withheld set cannot cover; the delta path keeps `replicable_objects`. -pub fn replicable_objects_fail_closed( - candidates: Vec, - allowed_blobs: &HashSet, - all_blob_oids: &HashSet, -) -> Vec { - candidates - .into_iter() - .filter(|oid| !all_blob_oids.contains(oid) || allowed_blobs.contains(oid)) - .collect() -} - -/// For every blob withheld from anonymous, the DIDs allowed to read it: the -/// owner plus any reader DID that `visibility_check` Allows at some path the -/// blob appears at. Least-privilege: a reader of one private subtree is not a -/// recipient of a blob that only lives in another. -#[cfg(test)] +/// Both the allowed blob set and the allowed tree set, derived from ONE bounded +/// walk so the two are consistent and the walk cost is paid only once. Returns +/// `(allowed_blobs, allowed_trees, all_blob_oids, all_tree_oids)`. +/// +/// #218 review P1b: enumerate every reachable commit's root tree OID so +/// the structural entry-level check (`structurally_safe_root_tree`) can +/// be applied to each. One `git rev-list --all` followed by one +/// `git rev-parse ^{tree}` per commit, both bounded by +/// `deadline`; the cost is small (root-tree OIDs are tiny, one git +/// invocation per commit) and pays for itself the first time the +/// root tree would otherwise leak a denied subtree's name. +/// A blob or tree is "allowed" if visibility permits it at *some* reachable +/// path; a tree reachable at both an allowed and denied path is allowed (its +/// metadata is public elsewhere). Commits and tags are not classified here — +/// the caller decides per type whether the allow-set applies. +pub fn allowed_blob_tree_sets_bounded( + repo_path: &Path, + git_bin: &str, + deadline: Instant, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, +) -> Result { + let (blob_pairs, tree_pairs) = all_object_paths(repo_path, git_bin, deadline)?; + let all_blob_oids: HashSet = blob_pairs.iter().map(|(oid, _)| oid.clone()).collect(); + let all_tree_oids: HashSet = tree_pairs.iter().map(|(oid, _)| oid.clone()).collect(); + let mut allowed_blobs = HashSet::new(); + for (oid, path) in &blob_pairs { + // #218 review round 9 (guidance #1): the empty-path + // decision is now in `pair_decision` so this consumer and + // `withheld_from_pairs` / `allowed_blob_set_for_caller_bounded` + // cannot disagree. For this caller (the sweep's anonymous + // allow-set), `pair_decision("", ..., None)` is Deny — + // identical to the previous `!path.is_empty()` skip, but + // the path is now annotated with the explicit + // "unclassifiable → deny" reasoning rather than a silent + // skip. If the policy is ever relaxed (e.g. to allow + // owner-only paths), it lands in one place. + if pair_decision(path, rules, is_public, owner_did, None) == Decision::Allow { + allowed_blobs.insert(oid.clone()); + } + } + // #218 review P1b (recursive at every depth): the path-based pass + // admits a tree at any path the policy allows, but a tree's + // serialized bytes name its direct entries plus their OIDs — so a + // path-admitted tree whose entries point at a denied subtree + // would leak that subtree's existence. Re-evaluate each + // path-admitted tree structurally at the same path: admit it + // only if every direct entry is safe at `path/filename` and + // (for tree entries) the child tree is itself structurally safe + // there. The `admitted` set memoizes trees proven safe at some + // path so the recursion short-circuits on cycles and on the + // same tree reachable at multiple allowed paths. + let ctx = TreeCheckCtx { + repo_path, + git_bin, + rules, + is_public, + owner_did, + caller: None, + }; + let mut allowed_trees: HashSet = HashSet::new(); + for (oid, path) in &tree_pairs { + // #218 review round 9 (guidance #1): route through + // `pair_decision` so this tree allow-set and the blob + // allow-set above share the empty-path policy. The + // structural check (`tree_structurally_safe`) only runs + // for trees the path-based decision admits; an + // unclassifiable empty-path tree is not in `allowed_trees` + // for an anonymous caller (the sweep's policy). + if pair_decision(path, rules, is_public, owner_did, None) != Decision::Allow { + continue; + } + if tree_structurally_safe(&ctx, oid, path, &mut allowed_trees, deadline)? { + allowed_trees.insert(oid.clone()); + } + } + // Root trees of reachable commits: they have no path in + // `tree_pairs` (ls-tree emits descendants only), so evaluate + // them at "/" — the root tree is admitted iff every direct + // entry is safe at the root and (for tree entries) the child + // tree is itself structurally safe. The check is recursive, so + // a denied subtree propagates up to the root. + let commits = reachable_commit_oids(repo_path, git_bin, deadline)?; + for root_oid in root_tree_oids(repo_path, git_bin, &commits, deadline)? { + if visibility_check(rules, is_public, owner_did, None, "/") != Decision::Allow { + continue; + } + if tree_structurally_safe(&ctx, &root_oid, "/", &mut allowed_trees, deadline)? { + allowed_trees.insert(root_oid); + } + } + + Ok((allowed_blobs, allowed_trees, all_blob_oids, all_tree_oids)) +} + +/// Objects safe to replicate, failing closed on blobs (#99) and denied trees +/// (#172). A candidate replicates iff: +/// - it is a commit (structural metadata, always safe), OR +/// - it is a blob AND is in `allowed_blobs` (reachable and visibility-allowed), OR +/// - it is a tree AND is in `allowed_trees` (reachable and visibility-allowed). +/// +/// This drops withheld blobs, withheld trees, and dangling/unreachable objects. +/// Used on the full-scan pin path, where the candidate set can contain objects +/// the reachable-only withheld set cannot cover; the delta path keeps +/// `replicable_objects`. +pub fn replicable_objects_fail_closed( + candidates: Vec, + allowed_blobs: &HashSet, + all_blob_oids: &HashSet, + allowed_trees: &HashSet, + all_tree_oids: &HashSet, +) -> Vec { + candidates + .into_iter() + .filter(|oid| { + if all_blob_oids.contains(oid) { + // Blobs: fail closed — only allowed blobs pass. + allowed_blobs.contains(oid) + } else if all_tree_oids.contains(oid) { + // Trees: fail closed — only allowed trees pass (#172). + // A denied tree exposes child filenames and blob OIDs even + // though the secret content itself is excluded. + allowed_trees.contains(oid) + } else { + // Commits/tags: structural metadata, always safe. + true + } + }) + .collect() +} + +#[cfg(test)] pub fn withheld_blob_recipients( repo_path: &Path, rules: &[VisibilityRule], @@ -1213,7 +1943,11 @@ pub fn withheld_blob_recipients_bounded( let entry = out.entry(oid.clone()).or_default(); entry.insert(owner_did.to_string()); for did in &candidates { - if visibility_check(rules, is_public, owner_did, Some(did), path) == Decision::Allow { + // Same shared per-pair policy as the deny/allow gates (round-8 P1): + // an empty-path (phase-2, unclassifiable) blob grants a recovery + // copy to the owner only, never to a rule's named reader whose + // grant was written against a path this object does not have. + if pair_decision(path, rules, is_public, owner_did, Some(did)) == Decision::Allow { entry.insert(did.clone()); } } @@ -1535,6 +2269,29 @@ esac\n"; } } + /// Write `bytes` to the bare repo's object store and return + /// the resulting loose blob OID. Used by the consumer matrix + /// test to give each ref shape its OWN blob so a missing + /// phase-2 arm is observable (sharing one blob across all + /// three shapes meant every consumer was green under every + /// combination of arms, P2 reviewer round 9). + fn make_blob(bare: &Path, bytes: &[u8]) -> String { + use std::io::Write; + use std::process::Stdio; + let out = Command::new("git") + .args(["hash-object", "-w", "--stdin"]) + .current_dir(bare) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .and_then(|mut c| { + c.stdin.take().unwrap().write_all(bytes)?; + c.wait_with_output() + }) + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + const OWNER: &str = "did:key:zOwner"; /// Build a bare repo with public/a.txt and secret/b.txt at one commit. @@ -1837,13 +2594,24 @@ esac\n"; let reader = "did:key:z6MkReader"; let rules = [rule("/secret/**", &[reader])]; - // anon: the withheld /secret tree is excluded; root ("/") and /public are in. + // anon: the withheld /secret subtree tree is excluded (#172). The + // root tree is ALSO excluded (#218 review P1b): its serialized + // bytes name the `/secret` entry and the OID of its child + // subtree, so publishing it would leak the same metadata the + // `/secret/**` deny is meant to withhold. The structural + // entry-level check in `allowed_tree_set_for_caller_bounded` + // gates the root tree on every direct entry being safe. + // `/public` (allowed path) is still in. let anon = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, None).unwrap(); assert!( !anon.contains(&secret_tree), "withheld /secret subtree tree excluded for anon" ); - assert!(anon.contains(&root_tree), "root tree included (path /)"); + assert!( + !anon.contains(&root_tree), + "root tree excluded for anon: its serialized bytes name /secret and the \ + secret subtree OID, which is the metadata the /secret/** deny must withhold" + ); assert!(anon.contains(&public_tree), "/public subtree tree included"); // listed reader: sees the /secret tree (caller-aware, not a blanket deny). @@ -2057,19 +2825,26 @@ esac\n"; let commits = reachable_commit_oids(&bare, "git", Instant::now() + WALK_TIMEOUT).unwrap(); assert_eq!(commits.len(), N, "all {N} commits reachable"); - // Call root_tree_pairs directly (private, same module) under a liveness - // watchdog, then assert it returned every distinct root tree. + // Call root_tree_oids directly (private, same module) under a + // liveness watchdog, then assert it returned every distinct + // root tree. (#218 review P1b: the previous shape returned + // `(oid, "/")` pairs so the path-based filter would admit + // root trees on the synthetic "/". The new shape is a plain + // oid set; the structural post-pass in + // `allowed_tree_set_for_caller_bounded` and + // `allowed_blob_tree_sets_bounded` is what actually admits + // them.) let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { let _ = tx.send( - root_tree_pairs(&bare, "git", &commits, Instant::now() + WALK_TIMEOUT) + root_tree_oids(&bare, "git", &commits, Instant::now() + WALK_TIMEOUT) .map(|s| s.len()), ); }); match rx.recv_timeout(std::time::Duration::from_secs(30)) { Ok(Ok(len)) => assert_eq!(len, N, "every distinct root tree returned"), - Ok(Err(e)) => panic!("root_tree_pairs errored: {e}"), - Err(_) => panic!("root_tree_pairs did not return within 30s"), + Ok(Err(e)) => panic!("root_tree_oids errored: {e}"), + Err(_) => panic!("root_tree_oids did not return within 30s"), } } @@ -2393,6 +3168,8 @@ esac\n"; .into_iter() .map(String::from) .collect(); + let allowed_trees: HashSet = HashSet::new(); + let all_trees: HashSet = HashSet::new(); let candidates = vec![ "commit1".to_string(), "tree1".to_string(), @@ -2400,7 +3177,13 @@ esac\n"; "b_secret".to_string(), "b_dangling".to_string(), ]; - let got = replicable_objects_fail_closed(candidates, &allowed, &all_blobs); + let got = replicable_objects_fail_closed( + candidates, + &allowed, + &all_blobs, + &allowed_trees, + &all_trees, + ); assert_eq!( got, vec![ @@ -2483,7 +3266,15 @@ esac\n"; // Full-scan candidate set includes the dangling blob; fail-closed drops it. let candidates = vec![dangling_oid.clone(), public_oid.clone()]; - let replicable = replicable_objects_fail_closed(candidates, &allowed, &all_blobs); + let allowed_trees: HashSet = HashSet::new(); + let all_trees: HashSet = HashSet::new(); + let replicable = replicable_objects_fail_closed( + candidates, + &allowed, + &all_blobs, + &allowed_trees, + &all_trees, + ); assert!( !replicable.contains(&dangling_oid), "#99: a dangling private blob must not replicate" @@ -3040,18 +3831,434 @@ esac\n"; ); } + /// #218 review round 10 (P1): `walk_tree_oids_inner` previously + /// returned `Ok(())` on a non-UTF-8 `ls-tree -z` listing, + /// inserting only the tree OID and skipping the child blob/tree + /// OIDs. A direct tree ref (or an annotated tag peeling to a + /// tree) is valid Git input; `git rev-list --objects --all` + /// still enumerates the tree and every descendant. The + /// keep-side therefore removed the tree from the served set but + /// passed the child blob OIDs to `pack-objects`, exposing their + /// bytes to an anonymous clone. Phase 1 already bails on the + /// same input at `:526`; this test pins the walk on the same + /// fail-closed outcome for direct tree refs and peeled + /// tag-of-tree refs (the two non-commit shapes that round 9 + /// added tolerance for). + #[cfg(unix)] + #[test] + fn fails_closed_on_non_utf8_tree_tip() { + use std::os::unix::ffi::OsStrExt; + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + std::fs::create_dir_all(&work).unwrap(); + let run = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + // Hash a blob, then index it at a path whose directory byte is invalid UTF-8. + let blob_oid = { + let out = Command::new("git") + .args(["hash-object", "-w", "--stdin"]) + .current_dir(&work) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .and_then(|mut c| { + use std::io::Write; + c.stdin.take().unwrap().write_all(b"TOP SECRET\n")?; + c.wait_with_output() + }) + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let mut bad_path = std::ffi::OsString::from("s"); + bad_path.push(std::ffi::OsStr::from_bytes(&[0xFF])); + bad_path.push("cret/b.txt"); + let cacheinfo = { + let mut s = std::ffi::OsString::from(format!("100644,{blob_oid},")); + s.push(&bad_path); + s + }; + assert!( + Command::new("git") + .arg("update-index") + .arg("--add") + .arg("--cacheinfo") + .arg(&cacheinfo) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git update-index failed" + ); + // Direct tree ref: a ref whose target is the TREE OID, not a commit. + // `git update-ref refs/tags/direct-tree ` writes that. + let tree_oid = String::from_utf8_lossy( + &Command::new("git") + .args(["write-tree"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .trim() + .to_string(); + run(&["update-ref", "refs/tags/direct-tree", &tree_oid], &work); + // Peeled tag-of-tree: an annotated tag whose target is the same tree. + // The walker has to peel the tag before it reaches the tree. + run( + &["tag", "-a", "-m", "tagged", "tag-of-tree", &tree_oid], + &work, + ); + // Push both to the bare clone so the walker exercises the + // post-clone refs. + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + + // Direct-tree case: the ref's target is the tree OID. The + // walker must fail closed (Err) so the keep-side withholds + // the whole subtree by name, never serving the child blob. + let rules = [rule("/s\u{fffd}cret/**", &[])]; + let direct = withheld_blob_oids(&bare, &rules, true, OWNER, None); + assert!( + direct.is_err(), + "a direct-tree ref with a non-UTF-8 child must fail closed (Err), \ + not return Ok with a partial withheld set (review round 10 P1)" + ); + + // Peeled-tag case: the direct-tree ref is deleted first so this + // call walks ONLY the annotated tag ref. Otherwise both calls + // would share one clone, the walk would fail closed on the direct + // ref first, and the peel arm would never run in either call — + // a regression in the tag-of-tree shape would not be caught + // (review round 11 P3). With only the tag ref left, an Err + // proves the walker peeled the tag to the tree and hit the + // non-UTF-8 child through that path. + run(&["update-ref", "-d", "refs/tags/direct-tree"], &bare); + let peeled = withheld_blob_oids(&bare, &rules, true, OWNER, None); + assert!( + peeled.is_err(), + "a peeled annotated-tag-of-tree with a non-UTF-8 child must \ + fail closed (Err), not return Ok with a partial withheld set" + ); + } + + /// #218 review round 9 (guidance #2 — preserve Git path bytes): + /// a path with a TRAILING SPACE is a real, valid Git shape + /// (`git` stores raw bytes, no POSIX/NTFS rule applies). The + /// visibility pipeline must see the bytes verbatim: a `/secret/**` + /// rule has to match a path of `secret /f.txt` (the parent + /// directory is `secret ` with one trailing space, not the + /// directory `secret` followed by `/f.txt`). + /// + /// Pre-fix regression: any `record.trim()` on the `ls-tree -z` + /// field would have stripped the trailing space and let the blob + /// leak. The current parser at `blob_paths` does NOT `.trim()` + /// the path — the test pins that invariant at the cargo-test + /// level so a future refactor that reintroduces a trim fails + /// the suite, not the production walk. + #[cfg(unix)] + #[test] + fn withholds_secret_blob_at_path_with_trailing_space() { + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + // Create a parent directory whose name has a trailing space. + // `git` permits this; some filesystems do too on Linux. + std::fs::create_dir_all(&work).unwrap(); + std::fs::create_dir_all(work.join("secret ")).unwrap(); + std::fs::write(work.join("public.txt"), b"public\n").unwrap(); + std::fs::write( + work.join("secret /f.txt"), + b"TOP SECRET (trailing-space path)\n", + ) + .unwrap(); + let run = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "init"], &work); + let oid = |path: &str| { + let out = Command::new("git") + .args(["rev-parse", &format!("HEAD:{path}")]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let secret_oid = oid("secret /f.txt"); + let public_oid = oid("public.txt"); + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + + // The rule matches the trailing-space parent (a normal + // /secret/** won't catch it). Use the explicit pattern + // that includes the space. + let rules = [rule("/secret /**", &[])]; + let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, None).unwrap(); + assert!( + withheld.contains(&secret_oid), + "secret blob at a path with a trailing-space parent directory must be withheld \ + (the rule was /secret /** with the literal trailing space)" + ); + assert!( + !withheld.contains(&public_oid), + "public blob must NOT be withheld" + ); + } + + /// #218 review round 9 (guidance #2 — preserve Git path bytes): + /// `ls-tree -z` is NUL-delimited, and the record's + /// `\t\0` shape has no leading whitespace + /// outside the field separator. If a future parser + /// inadvertently eats leading whitespace from the field + /// (e.g. a `path.trim_start()`), a path beginning with a space + /// would be re-shaped into the same one with the space gone + /// — a quiet leak class symmetric with the trailing-space + /// case above. + /// + /// The contract: leading whitespace in the field is part of + /// the filename (rare but possible; the field is bytes, not a + /// POSIX path) and must be preserved. + /// + /// The test creates a *directory* whose name has a leading + /// space (` secret/`), then a file at ` secret/f.txt`. The + /// leading space is inside a directory name, not at the + /// top-level (where the `git update-index --cacheinfo` + /// path-separator would eat it). The full path is + /// `/ secret/f.txt` and the rule is `/ secret/**` with a + /// literal leading space. A `path.trim_start()` on the + /// post-`/` portion would collapse this to `/secret/**` + /// and the rule would no longer match. + #[cfg(unix)] + #[test] + fn withholds_secret_blob_at_path_with_leading_space() { + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + std::fs::create_dir_all(&work).unwrap(); + // A directory with a leading space in its name. The + // working tree on Linux permits this; some shells don't, + // so we materialise via `std::fs` not via a shell glob. + std::fs::create_dir_all(work.join(" secret")).unwrap(); + std::fs::write(work.join("public.txt"), b"public\n").unwrap(); + std::fs::write( + work.join(" secret").join("f.txt"), + b"TOP SECRET (leading-space dir)\n", + ) + .unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "init"], &work); + let oid = |path: &str| { + let out = Command::new("git") + .args(["rev-parse", &format!("HEAD:{path}")]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let secret_oid = oid(" secret/f.txt"); + let public_oid = oid("public.txt"); + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + + // The rule matches the leading-space directory (a normal + // /secret/** won't catch it). Use the explicit pattern + // that includes the space. + let rules = [rule("/ secret/**", &[])]; + let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, None).unwrap(); + assert!( + withheld.contains(&secret_oid), + "secret blob at a path inside a LEADING-SPACE directory must be withheld \ + (the rule was / secret/** with the literal leading space); a trim_start() on the \ + post-/ portion would leak it" + ); + assert!( + !withheld.contains(&public_oid), + "public blob must NOT be withheld" + ); + } + + /// Write a blob into `bare`'s object store that NO commit reaches, and + /// return its OID. + /// + /// #218 review round 8 P2 (why this exists): the phase-2 ref tests used to + /// hang the ref on `fixture()`'s `secret` blob, which is COMMITTED at + /// `secret/b.txt`. Phase 1's per-commit `ls-tree` therefore already yielded + /// `(secret, "/secret/b.txt")`, the `/secret/**` rule already denied it, and + /// the `withheld.contains(&secret)` assertion passed with phase 2 deleted + /// outright — the tests bound nothing. A blob written straight to the object + /// store is absent from every commit's tree, so phase 1 cannot see it and the + /// ONLY way it reaches the withheld set is the `for-each-ref` phase. That is + /// also the exact shape of the leak: `rev_list_keep`'s + /// `git rev-list --objects --all` DOES follow a ref to such a blob (verified + /// against stock git), so an under-withheld one ships in the clone pack. + /// + /// `hash-object -w --stdin` is the minimal way to produce it; committing the + /// content and then orphaning the commit reaches the same state by a longer + /// route, and would additionally leave the content in a reflog the walk does + /// not read. + #[cfg(test)] + fn orphan_blob(bare: &Path, content: &str) -> String { + use std::io::Write; + use std::process::Stdio; + let mut child = Command::new("git") + .args(["hash-object", "-w", "--stdin"]) + .current_dir(bare) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(content.as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "git hash-object failed"); + let oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + assert!(!oid.is_empty(), "git hash-object produced no OID"); + oid + } + + #[test] + fn skips_a_ref_pointing_at_a_blob() { + // #218 review round 1: a ref pointing at a blob is a valid Git + // shape (tag-of-blob, blobref). The pre-fix + // `assert_all_refs_are_commits` guard bailed on this and + // failed the whole walk closed; round 1 drops the guard. + // #218 review round 3 P1: the round-1 fix was correct for + // the allow-list sweep (empty paths dropped at + // visibility_pack.rs:1396, :1423) but it left the deny-set + // path fail-OPEN — a blob only reachable via a non-commit + // ref tip would be served (the `git rev-list --objects --all` + // enumeration in `smart_http::rev_list_keep` includes + // non-commit targets) but NOT withheld (the deny set comes + // from `blob_paths`, which only walks commits). Round 3 + // adds a `for-each-ref` phase 2 to `blob_paths` that + // enumerates non-commit ref targets and inserts them with + // empty path; the deny-side caller withholds empty-path + // entries by OID. + // + // #218 review round 8 P2 (non-vacuity): the blob under test is + // `orphan_blob`'s, reachable ONLY through the ref written below — + // no commit's tree names it, so phase 1 contributes nothing for it + // and the assertion binds phase 2 alone. Verified by mutation: + // neutralizing the phase-2 filter turns this RED. + let (_td, bare, _secret, _public) = fixture(); + let orphan = orphan_blob(&bare, "TOP SECRET, ref-only\n"); + std::fs::write(bare.join("refs/heads/blobref"), format!("{orphan}\n")).unwrap(); + let rules = [rule("/secret/**", &[])]; + let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, None) + .expect("a ref pointing at a non-commit object no longer fails the whole walk"); + assert!( + withheld.contains(&orphan), + "a blob reachable ONLY via a direct ref-to-blob must be withheld — no \ + commit path names it, so nothing but the for-each-ref phase can put it \ + in the deny set, while `git rev-list --objects --all` already serves it" + ); + } + + /// #218 review round 8 P1 (the allow side of the same pair): the + /// `GET /ipfs/{cid}` gate consumes the SAME `blob_paths` listing through + /// `allowed_blob_set_for_caller_bounded`. Before the shared `pair_decision`, + /// that consumer ran `visibility_check(..., "")` on a phase-2 entry, and on a + /// public repo no glob matches the empty path so the answer was `Allow` — the + /// serve filter withheld the OID while the IPFS gate handed the bytes over. + /// The two sides must agree: an anonymous caller is denied, the owner is not. #[test] - fn fails_closed_when_a_ref_cannot_be_traversed() { - let (_td, bare, secret, _public) = fixture(); - // Point a ref at a blob (a valid object that is not tree-ish). `ls-tree -r` - // fails on it; that must propagate as Err rather than silently dropping the - // ref and under-withholding. - std::fs::write(bare.join("refs/heads/blobref"), format!("{secret}\n")).unwrap(); + fn ref_only_blob_is_denied_on_the_allow_side_too() { + let (_td, bare, _secret, _public) = fixture(); + let orphan = orphan_blob(&bare, "TOP SECRET, ref-only, allow side\n"); + std::fs::write(bare.join("refs/heads/blobref"), format!("{orphan}\n")).unwrap(); + // A PUBLIC repo with a rule that cannot match an empty path: the + // pre-fix path-based check returned Allow here. let rules = [rule("/secret/**", &[])]; - let result = withheld_blob_oids(&bare, &rules, true, OWNER, None); + + let anon = allowed_blob_set_for_caller(&bare, &rules, true, OWNER, None).unwrap(); assert!( - result.is_err(), - "a ref that cannot be traversed must fail closed (Err)" + !anon.contains(&orphan), + "the allow side must NOT admit a ref-only blob to an anonymous caller — \ + the serve filter withholds this exact OID, and a disagreement means \ + `GET /ipfs/{{cid}}` serves what the clone pack refused" + ); + + let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, None).unwrap(); + assert!( + withheld.contains(&orphan) && !anon.contains(&orphan), + "deny side and allow side must reach the SAME verdict for one OID" + ); + + // The owner is the one identity the empty-path policy admits, so the + // test also proves the shared decision is owner-only rather than + // deny-everything (which would pass the assertion above vacuously). + let owner_set = allowed_blob_set_for_caller(&bare, &rules, true, OWNER, Some(OWNER)) + .expect("owner walk must succeed"); + assert!( + owner_set.contains(&orphan), + "the owner — the only identity that could have created the ref tip — \ + must still be able to read a ref-only blob" ); } @@ -3087,10 +4294,30 @@ esac\n"; } #[test] - fn fails_closed_on_annotated_tag_of_a_blob() { - let (_td, bare, secret, _public) = fixture(); - // An annotated tag whose target peels to a blob is not a commit; the - // guard must fail closed rather than skip the ref. + fn skips_an_annotated_tag_of_a_blob() { + // #218 review round 1: an annotated tag of a blob is a + // valid Git shape (pushable through receive-pack). The + // pre-fix `assert_all_refs_are_commits` guard bailed on + // this and failed the whole walk closed; round 1 drops + // the guard. #218 review round 3 P1: same shape as + // `skips_a_ref_pointing_at_a_blob` — the deny set must + // include the blob. The annotated tag `blobtag` peels to + // the blob, not a commit; `git rev-list --all` skips the + // tag; the phase-2 `for-each-ref` in `blob_paths` + // enumerates the tag and inserts the referent with an + // empty path; the deny-side caller withholds by OID. + // + // #218 review round 8 P1 (peeling) + P2 (non-vacuity): this is the + // shape the ref walk MISSED before the peeled atoms were added. + // `%(objecttype)` of an annotated tag is `tag`, so the blob/tree arms + // never saw the referent and the tag contributed nothing; the OLD test + // passed anyway only because its blob was also committed at + // `secret/b.txt` and phase 1 withheld it. The blob here is + // `orphan_blob`'s — reachable through the tag and nothing else — so + // the assertion now fails without BOTH the phase and its peel. + // Verified by mutation: neutralizing the phase-2 filter turns this RED. + let (_td, bare, _secret, _public) = fixture(); + let orphan = orphan_blob(&bare, "TOP SECRET, tag-only\n"); let run = |args: &[&str]| { assert!( Command::new("git") @@ -3104,13 +4331,96 @@ esac\n"; }; run(&["config", "user.email", "t@t"]); run(&["config", "user.name", "t"]); - run(&["tag", "-a", "-m", "blobtag", "blobtag", &secret]); + run(&["tag", "-a", "-m", "blobtag", "blobtag", &orphan]); let rules = [rule("/secret/**", &[])]; - let result = withheld_blob_oids(&bare, &rules, true, OWNER, None); + let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, None) + .expect("an annotated tag of a blob no longer fails the whole walk"); assert!( - result.is_err(), - "an annotated tag of a blob must fail closed (Err)" + withheld.contains(&orphan), + "a blob reachable only via an ANNOTATED tag must be withheld — the ref's \ + own object type is `tag`, so only the peeled referent puts it in the deny \ + set, while `git rev-list --objects --all` (which DOES peel tags) serves it" + ); + } + + /// #218 review round 8 P1: the peel must survive a NESTED annotated tag + /// (tag -> tag -> blob). Stock git's `%(*objectname)` peels the whole chain, + /// so this covers the shipped behavior; the fake-git twin below covers a git + /// that peels only one level. + #[test] + fn skips_a_nested_annotated_tag_of_a_blob() { + let (_td, bare, _secret, _public) = fixture(); + let orphan = orphan_blob(&bare, "TOP SECRET, nested-tag-only\n"); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&bare) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + run(&["tag", "-a", "-m", "inner", "blobtag-inner", &orphan]); + run(&["tag", "-a", "-m", "outer", "blobtag-outer", "blobtag-inner"]); + // Only the outer tag stays a ref, so the blob is reachable exclusively + // through a two-level tag chain. + run(&["tag", "-d", "blobtag-inner"]); + + let rules = [rule("/secret/**", &[])]; + let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, None) + .expect("a nested annotated tag of a blob must not fail the walk closed"); + assert!( + withheld.contains(&orphan), + "a blob behind a tag-of-a-tag must be withheld: `rev-list --objects --all` \ + peels the whole chain and serves it, so the deny set has to as well" + ); + } + + /// #218 review round 8 P1: drives the one-level-peel fallback that stock git + /// (2.50) never reaches. A fake git answers `for-each-ref` with a peeled type + /// of `tag` — the shape `push_delta.rs`'s ref-type guard documents — and the + /// walk must finish the peel via `rev-parse ^{}` + `cat-file -t` and + /// withhold the final blob, rather than bail and 500 the clone. + #[cfg(unix)] + #[test] + fn peels_a_tag_whose_peeled_target_is_still_a_tag() { + let tmp = TempDir::new().unwrap(); + let outer = "1111111111111111111111111111111111111111"; + let inner = "2222222222222222222222222222222222222222"; + let blob = "3333333333333333333333333333333333333333"; + // rev-parse: HEAD probe must FAIL (exit 1) so the walk skips it, but the + // `^{}` full peel must answer with the blob. `rev-list` lists no commits, + // so phase 1 contributes nothing and the OID can only arrive via phase 2. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n \ + rev-parse) case \"$2\" in --verify) exit 1 ;; *) echo {blob} ;; esac ;;\n \ + rev-list) : ;;\n \ + for-each-ref) echo {outer} tag {inner} tag ;;\n \ + cat-file) echo blob ;;\n \ + *) : ;;\nesac\nexit 0\n" + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let rules = [rule("/secret/**", &[])]; + let withheld = withheld_blob_oids_bounded( + tmp.path(), + &git_bin, + Duration::from_secs(10), + &rules, + true, + OWNER, + None, + ) + .expect("a still-a-tag peel must be resolved, not bailed on"); + assert!( + withheld.contains(blob), + "under a git that peels only one level, the walk must finish the peel \ + itself and withhold the final blob" ); } @@ -3222,4 +4532,339 @@ esac\n"; "a blob also reachable via an allowed path must not be withheld" ); } + + /// #218 review round 9 (guidance #1 — single fail-closed + /// classification contract): for every non-commit ref shape the + /// parser can produce, the FOUR consumers of `(oid, path)` — + /// smart-HTTP deny set (`withheld_blob_oids_bounded`), + /// `/ipfs/{cid}` allow set + /// (`allowed_blob_set_for_caller_bounded`), reconciliation + /// object set (`allowed_blob_tree_sets_bounded`), and encrypted + /// recovery (`withheld_blob_recipients_bounded`) — must agree + /// on the OID's classification for every caller identity. + /// Drift between consumers is a leak. + /// + /// The matrix is the canonical record: a regression in + /// `pair_decision`, a re-introduced `!path.is_empty()` skip + /// guard, or a wire-shape mismatch between the consumers fails + /// one row of the table at the cargo-test level, with a name + /// that points at the offending consumer. + #[test] + fn ref_classification_is_consistent_across_consumers() { + // Build a single bare repo with one secret blob reachable + // through every non-commit ref shape the parser produces: + // * direct blob ref (lightweight tag of a blob) + // * direct tree ref (lightweight tag of a tree) + // * annotated tag of a blob + // * annotated tag of a tree + // * nested tag (tag-of-tag-of-blob) + // The blob OID and tree OID are distinct so the consumers + // can disambiguate. The blob is NOT committed anywhere, so + // phase 1 (`rev-list --all` + `ls-tree`) does not see it — + // every entry in the `(oid, path)` set arrives through + // phase 2 (`for-each-ref`) with an empty path. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + std::fs::create_dir_all(&work).unwrap(); + // Init the bare first so the orphaned blobs can be written + // into its object store before any commit exists. + run(&["init", "-q", "--bare", bare.to_str().unwrap()], td.path()); + // An annotated tag is a tag OBJECT, with a tagger header, + // and `git tag -a` refuses to create one without a + // configured user.email/user.name — even on a bare repo. + // The bare is where the test's refs live, so set the + // tagger identity there directly. + run(&["config", "user.email", "t@t"], &bare); + run(&["config", "user.name", "t"], &bare); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + + // P2 (reviewer round 9): each ref shape must carry its + // OWN blob, not all share one OID. Sharing one blob + // meant deleting any single phase-2 classification arm + // left every consumer green. The tree here is reused + // because both the direct-tree and annotated-tree + // branches share a `mktree`, but each leaf blob is + // unique to its ref shape. + // + // Direct blob ref: hash-object, then update-ref to a ref + // tip that points at the loose blob (not a commit). + // `withheld_blob_oids` walks the BARE repo, so the ref + // must be created on the bare — `update-ref` on the work + // tree would put it in a refs file the walk never reads. + let direct_blob = make_blob(&bare, b"DIRECT BLOB\n"); + run( + &["update-ref", "refs/tags/direct-blob", &direct_blob], + &bare, + ); + + // Direct tree ref: a tree object, then a ref tip pointing + // at the tree. `git mktree` materialises the tree. The + // tree contains `direct_blob` as its single entry; the + // tree's OID is the only thing the ref points at, so the + // blob is reachable ONLY through the tree walk. + let tree_oid = { + use std::io::Write; + use std::process::Stdio; + let out = Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .and_then(|mut c| { + c.stdin + .take() + .unwrap() + .write_all(format!("100644 blob {direct_blob}\ttree-blob\n").as_bytes())?; + c.wait_with_output() + }) + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + run(&["update-ref", "refs/tags/direct-tree", &tree_oid], &bare); + + // Annotated tag of a blob: each annotated tag wraps its + // OWN blob so a missing peel-arm is observable. + let annotated_blob = make_blob(&bare, b"ANNOTATED BLOB\n"); + run( + &[ + "tag", + "-a", + "-m", + "annotated-blob", + "tagged-blob", + &annotated_blob, + ], + &bare, + ); + + // Annotated tag of a tree: the tree's children are + // `annotated_blob` (NOT `direct_blob`), so a missing + // tree-walk arm under the annotated-tag-of-tree path + // would let `annotated_blob` leak. + let annotated_tree = { + use std::io::Write; + use std::process::Stdio; + let out = Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .and_then(|mut c| { + c.stdin.take().unwrap().write_all( + format!("100644 blob {annotated_blob}\ttree-blob\n").as_bytes(), + )?; + c.wait_with_output() + }) + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + run( + &[ + "tag", + "-a", + "-m", + "annotated-tree", + "tagged-tree", + &annotated_tree, + ], + &bare, + ); + + // Nested tag (tag-of-tag-of-blob): a tag of a tag, with + // its own unique blob so a missing recursive-peel arm + // is observable. + let nested_blob = make_blob(&bare, b"NESTED BLOB\n"); + run( + &[ + "tag", + "-a", + "-m", + "nested-blob", + "tagged-nested-blob", + &nested_blob, + ], + &bare, + ); + run( + &["tag", "-a", "-m", "outer", "outer", "tagged-nested-blob"], + &bare, + ); + + // P2 (reviewer round 9): the rule carries a reader DID + // so the encrypted-recovery consumer 4 can observe a + // non-owner recipient. The previous `caller.unwrap_or("??")` + // always matched against `"??"`, which is not in + // `reader_dids`, so the assertion held for any + // implementation. With a real reader DID in the rule + // and `caller = Some(reader)`, the assertion now + // exercises the actual contract. + const READER: &str = "did:key:z6MkReaderrrrrrrrrrrrrrrrrrrrrrrrr"; + let rules = [rule("/secret/**", &[READER])]; + + // P2 (reviewer round 9): consumer 4 (encrypted-recovery + // recipients) is a single invariant — "owner is in the + // recipients, reader/anon is not" — that does NOT vary + // per ref shape. Hoist it OUT of the per-shape loop so + // deleting a phase-2 arm (which would only fail + // consumer 1 for one of the three ref shapes) cannot + // make consumer 4 silently pass. + // + // Run all four consumers, but only consumer 1 runs + // once per ref shape; consumers 2, 3, 4 run once per + // caller. Consumers 2 and 3 use the direct_blob (the + // simplest unclassifiable target); consumer 4 uses the + // direct_blob so the assertion targets one specific + // OID and the recipient map is unambiguous. + for caller in [None, Some(READER), Some(OWNER)] { + let label = format!("caller={caller:?}"); + + // 1. Smart-HTTP deny set: per ref shape, the OID + // must be withheld iff the caller is not the + // owner. The cross-shape assertion: each shape + // independently withholds (or admits, for the + // owner) its OWN OID, so a missing peel-arm + // would let a different shape's blob through. + let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, caller).unwrap(); + for (label_inner, oid) in [ + ("direct-blob", &direct_blob), + ("annotated-blob", &annotated_blob), + ("nested-tag-blob", &nested_blob), + ] { + let in_withheld = withheld.contains(oid); + let expected = !matches!(caller, Some(c) if c == OWNER); + assert_eq!( + in_withheld, expected, + "[{label}] smart-HTP deny for {label_inner}: expected withheld={expected}, got {in_withheld}" + ); + } + // Direct tree and annotated tree: the smart-HTP deny + // set names this function "blob OIDs" but in fact + // `blob_paths` enumerates ANY non-commit ref target, + // so a tree is in the set the same way a blob is. The + // `pair_decision` empty-path policy applies uniformly: + // withheld for non-owners, Allow for the owner. The + // structural consumer `allowed_blob_tree_sets_bounded` + // is the one that splits blobs and trees — the smart + // HTP gate treats both as opaque withheld OIDs. + let expected = !matches!(caller, Some(c) if c == OWNER); + assert_eq!( + withheld.contains(&tree_oid), + expected, + "[{label}] smart-HTP deny for the unclassifiable tree: expected withheld={expected}" + ); + assert_eq!( + withheld.contains(&annotated_tree), + expected, + "[{label}] smart-HTP deny for the annotated tag's tree: expected withheld={expected}" + ); + + // 2. /ipfs/{cid} allow set: the direct_blob must be + // in the allow set iff the caller is the owner + // (owner-only carve-out for unclassifiable ref + // targets). This consumer is caller-invariant; + // hoist the shape variation out of the per-shape + // loop so a regression in only this consumer is + // visible to the test. + let allowed = allowed_blob_set_for_caller(&bare, &rules, true, OWNER, caller).unwrap(); + let expected = matches!(caller, Some(c) if c == OWNER); + assert_eq!( + allowed.contains(&direct_blob), + expected, + "[{label}] /ipfs/{{cid}} allow set for the unclassifiable blob: \ + expected in set = {expected}" + ); + + // 3. Reconciliation object set: same allow-set shape + // as /ipfs/{cid} (with caller = None baked in), so + // the unclassifiable blob is DENIED — the sweep + // never pins it. This is the cross-consumer + // assertion: the /ipfs/{cid} gate and the sweep + // agree on what the anonymous allow set contains. + use std::time::Instant; + let (rec_allowed_blobs, _rec_allowed_trees, _, _) = allowed_blob_tree_sets_bounded( + &bare, + "git", + Instant::now() + WALK_TIMEOUT, + &rules, + true, + OWNER, + ) + .unwrap(); + assert!( + !rec_allowed_blobs.contains(&direct_blob), + "[{label}] reconciliation allow-set (caller = None) must NOT include \ + the unclassifiable blob; the sweep never pins an empty-path blob to anon" + ); + } + + // Consumer 4: encrypted-recovery recipients. The owner + // sees `direct_blob` in the recipients (the owner + // encrypts+pins for self). The anon caller must NOT + // see it. The reader caller is in `reader_dids` and + // also must NOT see it (the rule's allow shape is + // "owner only for the unclassifiable ref" — the + // reader DID is irrelevant to the empty-path decision; + // the previous `caller.unwrap_or("??")` always passed + // because `"??"` is not in any list, so the assertion + // was vacuous). Use `Some(reader)` to exercise the + // actual contract. + let recipients = withheld_blob_recipients(&bare, &rules, true, OWNER).unwrap(); + + // P2 (reviewer round 9): the recipient set is + // CALLER-INVARIANT — it enumerates every identity + // (owner + every rule's reader DID) that the + // path-decision allows for THIS OID. Whether a given + // caller can decrypt the seal is a separate question + // answered at seal time (the seal checks membership in + // the recipient set). The test asserts the recipient + // set shape, not the seal-time check, because the + // seal is a separate code path tested elsewhere. + // + // The contract the test pins: + // - The owner is in the recipient set (the owner + // encrypts+pins for self). + // - The empty-string anon sentinel is NOT in the + // recipient set (anon does not decrypt anything + // from the seal; the empty path is owner-only). + // - A reader DID listed on a rule whose path does + // not match the empty path is NOT in the recipient + // set (the pair_decision empty-path allow shape + // is owner-only; the reader is on a path-scoped + // rule that does not match the empty path, so the + // seal cannot leak `direct_blob` to the reader + // through the empty path). + let direct_recipients = recipients.get(&direct_blob).cloned().unwrap_or_default(); + assert!( + direct_recipients.contains(OWNER), + "owner must be in encrypted-recovery recipients for direct_blob; \ + got {direct_recipients:?}" + ); + assert!( + !direct_recipients.iter().any(|d| d.is_empty()), + "the empty-string anon sentinel must not be a recipient of direct_blob; \ + got {direct_recipients:?}" + ); + assert!( + !direct_recipients.iter().any(|d| d == READER), + "a reader DID on a path-scoped rule that does not match the empty path must \ + not be a recipient of direct_blob (empty-path allow shape is owner-only); \ + got {direct_recipients:?}" + ); + } } diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 5d4579a3b..dd0ca0408 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -909,6 +909,12 @@ async fn sweep_pass( // Advance FIRST: every path below this line may skip the row, and none of them // may wedge the walk (scenario 7). last = sha.clone(); + // Round-3 P1: skip Pinata-only rows whose local cid is NULL. + // The legacy repair walk re-keys `cid` from a provider CID to + // the raw resolver CID; a row with no local cid has no string + // to re-key, and skipping is the natural behavior. The cursor + // still advances so the walk does not loop on this row. + let Some(stored) = stored else { continue }; // Same cost gate as the skip-path repair: a canonical raw CIDv1 key is already // the resolver key, so it reads no bytes and resolves no repo. if gitlawb_core::cid::is_raw_cidv1(&stored) { @@ -1434,6 +1440,84 @@ fn note_legacy_repair_read() { /// have to be documented, validated, and kept meaningful. pub const PIN_BATCH_BUDGET: Duration = Duration::from_secs(120); +/// A captured per-repo visibility-policy epoch that fences a pin batch. +/// +/// The reconciliation sweep reads the epoch immediately before dispatching a +/// pin loop and passes a fence in; the loop re-reads the epoch before every +/// upload and aborts the batch the moment it moves. A visibility narrow that +/// lands mid-batch (a rule made private, a repo quarantined) must not let the +/// remaining pre-authorized objects still go to a public content-addressed +/// backend — the narrow is a policy change, and dispatching against the stale +/// snapshot is the exact irreversible-publication class this fence exists for +/// (R1-P1). `None` (the push path) means "no fence": the push derives its own +/// object list at admission and holds a write lease, so no sweep-style batch +/// snapshot crosses the dispatch boundary. +/// +/// The fence deliberately does NOT hold any lock across the batch: a +/// visibility narrow (rule insert/remove, quarantine set) must commit +/// immediately, and the batch aborts on its next `is_current` check. Holding +/// a per-repo mutex from capture through drop would invert this — the narrow +/// would block behind the background sweep and every object in the batch +/// would still be sealed/posted to the reader being removed. The accepted +/// residual is a single in-flight object: a narrow that commits between the +/// pre-POST `is_current` check and the HTTP POST cannot be recalled, but the +/// next iteration aborts and the fenced DB record (row-locked against the +/// narrow's epoch bump) refuses to land the raced row as durable. +/// Multi-process / multi-node narrows are ordered by the same epoch column; +/// no in-process registry is involved. +#[derive(Clone)] +pub struct PolicyFence { + db: crate::db::Db, + repo_id: String, + epoch: i64, +} + +impl PolicyFence { + /// Capture the current policy epoch for `repo_id`. A read failure is a + /// skip, not a retry-with-zero: the caller must not dispatch a batch it + /// cannot fence (fail closed on a stale allow). + pub async fn capture(db: &crate::db::Db, repo_id: &str) -> Option { + match db.repo_policy_epoch(repo_id).await { + Ok(epoch) => Some(PolicyFence { + db: db.clone(), + repo_id: repo_id.to_string(), + epoch, + }), + Err(e) => { + tracing::warn!(repo = %repo_id, err = %e, "policy-epoch read failed; not fencing pin batch"); + None + } + } + } + + /// Whether the repo's policy epoch is unchanged since capture. A read + /// failure is treated as "changed": never dispatch on a policy we cannot + /// prove current. + pub async fn is_current(&self) -> bool { + match self.db.repo_policy_epoch(&self.repo_id).await { + Ok(epoch) => epoch == self.epoch, + Err(_) => false, + } + } + + /// The epoch value captured at `capture` time. Exposed so the + /// pinner can pass it to + /// `Db::record_pinned_cid_with_source_fenced` — the third + /// fence in the same transaction as the row insert. + /// Returning the field directly (rather than a `Option`) + /// matches the contract: a `PolicyFence` always has a + /// captured epoch; `is_current()` reports whether it still + /// matches. + pub fn captured_epoch(&self) -> i64 { + self.epoch + } + + /// The repo this fence guards, for log correlation. + pub fn repo_id(&self) -> &str { + &self.repo_id + } +} + /// The smallest remainder worth starting a bounded git read (or an add) with. /// /// A 1ms remainder otherwise buys a child spawned already past its deadline, which @@ -1530,6 +1614,15 @@ pub async fn pin_git_object( // Kubo returns newline-delimited JSON; we only care about the last object // (there's typically just one for a single-file add). + // + // The response MUST carry a real `Hash`: a misconfigured `GITLAWB_IPFS_API` + // (proxy returning HTML, health check on the wrong port, truncated gateway) + // can otherwise answer 2xx with no JSON, and falling back to the locally + // computed `expected_cid` would record a row for bytes the backend never + // stored. The reconciliation sweep trusts `pinned_cids` rows as durability + // evidence, so a silent false positive at pin time becomes a permanent blind + // spot for the backstop. A missing `Hash` fails the pin rather than recording + // a phantom row (mirrors Pinata's `data.cid` check). let body = resp .text() .await @@ -1541,8 +1634,26 @@ pub async fn pin_git_object( let v: serde_json::Value = serde_json::from_str(line).ok()?; v["Hash"].as_str().map(|s| s.to_string()) }) - .next_back() - .unwrap_or(expected_cid.clone()); + .next_back(); + let cid = match cid { + Some(cid) => { + if cid != expected_cid { + tracing::warn!( + sha256 = %sha256_hex, + returned = %cid, + expected = %expected_cid, + "IPFS returned a different CID than computed locally (Kubo chunking may differ); recording the backend's answer" + ); + } + cid + } + None => { + return Err(anyhow::anyhow!( + "IPFS /api/v0/add returned 2xx without a Hash field; refusing to record \ + a CID the backend never acknowledged (misconfigured GITLAWB_IPFS_API?)" + )); + } + }; tracing::debug!(sha256 = %sha256_hex, %cid, "pinned git object to IPFS"); Ok(cid) @@ -1620,8 +1731,8 @@ pub(crate) fn batch_budget_gate( /// than [`PIN_READ_FLOOR`] left. It is a gate, not a hard ceiling, since a started /// iteration still runs to completion; /// - the git read: `store::read_object_bounded` runs under `spawn_blocking` against the -/// ABSOLUTE batch deadline (not the loop-top remainder, which the `is_pinned` round-trip -/// sitting between the two would push past it), with SIGTERM-then-SIGKILL +/// ABSOLUTE batch deadline (not the loop-top remainder, which the `has_ipfs_cid` +/// round-trip sitting between the two would push past it), with SIGTERM-then-SIGKILL /// process-group teardown, so a hung `git cat-file` costs this batch its remaining /// budget plus one watchdog teardown instead of holding the permit for the child's /// whole lifetime and blocking a runtime worker while it does; @@ -1665,10 +1776,11 @@ pub(crate) fn batch_budget_gate( /// # Truncation semantics /// /// A batch stopped at the deadline leaves its remaining objects unpinned, and -/// nothing sweeps them up afterwards. There is no reconciliation pass over -/// `pinned_cids`; recovery is opportunistic, happening only if some later push -/// on the repo takes the full-scan fallback (`push_delta::list_all_objects`) and -/// re-derives the whole object set, which then re-offers the skipped OIDs. +/// nothing sweeps them up afterwards on the push path; recovery is opportunistic +/// (a later full-scan push re-offers the skipped OIDs). The reconciliation +/// sweep is the systematic backstop: when it is enabled and a pin backend is +/// configured, it re-derives the public object set each pass and fills any +/// remaining gap. /// /// The twin in `pinata.rs` is back at parity on everything that bounds or repairs an /// object: it runs the same shared budget gate at the top of every iteration, the same @@ -1699,6 +1811,7 @@ pub async fn pin_new_objects( db: &crate::db::Db, repo_id: &str, batch_budget: Duration, + fence: Option<&PolicyFence>, ) -> Vec<(String, String)> { if ipfs_api.is_empty() { return vec![]; @@ -1709,6 +1822,19 @@ pub async fn pin_new_objects( let mut pinned = Vec::new(); for (attempted, sha) in object_list.into_iter().enumerate() { + // Policy fence (R1-P1): a visibility narrow that lands after the caller + // built this batch must abort it before the next irreversible upload. + // Checked FIRST so a changed policy costs nothing beyond the read. + if let Some(f) = fence { + if !f.is_current().await { + tracing::warn!( + repo = %f.repo_id, + unattempted = total - attempted, + "visibility policy changed mid-batch; stopping the pin loop" + ); + break; + } + } // Top of the iteration, before any of this object's work: an object is // never started with a remainder too small to cover a bounded read's // teardown. Consumed as a guard only: the read below runs against the @@ -1717,19 +1843,35 @@ pub async fn pin_new_objects( if batch_budget_gate("IPFS", deadline, pinned.len(), total - attempted).is_none() { break; } - // Skip if already pinned, but first backfill provenance if the existing - // pin has none. A legacy pin (recorded before repo_id existed, #173, jatmn) - // is skipped here before record_pinned_cid ever runs, so its NULL provenance - // would never resolve to one repo and known CIDs keep hitting the scan. The - // backfill only sets repo_id (AND repo_id IS NULL guard preserves - // first-pinner-owns) and never re-pins the bytes: the object is already on IPFS. - // Every DB call from here to the end of the iteration is bounded by the - // ABSOLUTE batch deadline (F3, #173): the loop runs under a global pin permit - // and a bare await parked it for the whole stall. The elapsed arm is mapped per - // site below, never as a blanket "existing error arm": a timeout cancels the - // client future but not the statement Postgres is running, so it reports an - // UNKNOWN outcome, not a failed write. - match db_bounded(deadline, db.is_pinned(&sha)).await { + // Skip if the object is ALREADY a real local IPFS pin, but first + // backfill provenance if the existing pin has none. A legacy pin + // (recorded before repo_id existed, #173, jatmn) is skipped here + // before record_pinned_cid ever runs, so its NULL provenance would + // never resolve to one repo and known CIDs keep hitting the scan. + // The backfill only sets repo_id (AND repo_id IS NULL guard + // preserves first-pinner-owns) and never re-pins the bytes: the + // object is already on IPFS. + // + // #218 review P1a: this check keys on `has_ipfs_cid` (writer-owned + // `local_ipfs_provenance = TRUE`), NOT on row existence + // (`is_pinned`). A Pinata-only row is `is_pinned = true` but + // `has_ipfs_cid = false`: the bytes never reached the local IPFS + // daemon, only Pinata, and we MUST fall through to the local + // writer path so a real local pin lands. Using `is_pinned` here + // was the gap that made the Pinata-only → local-IPFS repair + // path inert: every sweep pass re-entered this arm, recorded + // the source, and continued without ever calling + // `pin_git_object` or `record_pinned_cid_with_source`. The flag + // stayed FALSE forever. + // + // Every DB call from here to the end of the iteration is bounded + // by the ABSOLUTE batch deadline (F3, #173): the loop runs under + // a global pin permit and a bare await parked it for the whole + // stall. The elapsed arm is mapped per site below, never as a + // blanket "existing error arm": a timeout cancels the client + // future but not the statement Postgres is running, so it + // reports an UNKNOWN outcome, not a failed write. + match db_bounded(deadline, db.has_ipfs_cid(&sha)).await { Ok(true) => { // Elapsed here is free to skip: these are reads, so a late server-side // completion costs nothing, and the backfill's own `AND repo_id IS NULL` @@ -1839,7 +1981,7 @@ pub async fn pin_new_objects( } Ok(false) => {} Err(e) => { - tracing::warn!(sha = %sha, err = %e, "DB error checking pinned status"); + tracing::warn!(sha = %sha, err = %e, "DB error checking IPFS pinned status"); continue; } } @@ -1853,7 +1995,7 @@ pub async fn pin_new_objects( // own deadline regardless. // // The read runs against the ABSOLUTE batch deadline, not against the remainder - // measured at the top of the iteration: the `is_pinned` round-trip above sits + // measured at the top of the iteration: the `has_ipfs_cid` round-trip above sits // between the two, so `Instant::now() + budget_left` would land past `deadline` // by however long the DB took, and under a saturated pool that is the dominant // term. A slow DB check must not push the read's own bound out. @@ -1938,6 +2080,24 @@ pub async fn pin_new_objects( break; }; + // Dispatch fence (R1-P1): re-read the policy epoch immediately before + // the irreversible HTTP POST. The iteration-top check catches a narrow + // that landed before work began; THIS check catches a narrow that landed + // during the has_ipfs_cid round-trip or the bounded Git read — both of + // which can take seconds and during which a quarantine or rule change may + // have committed. Without this, stale plaintext can start uploading + // under authorization that is no longer current. + if let Some(f) = fence { + if !f.is_current().await { + tracing::warn!( + repo = %f.repo_id, + unattempted = total - attempted, + "visibility policy changed during preparation; aborting IPFS upload" + ); + break; + } + } + // Pin to IPFS match pin_git_object(ipfs_api, &sha, &data, Some(add_timeout)).await { Ok(cid) if !cid.is_empty() => { @@ -1987,7 +2147,31 @@ pub async fn pin_new_objects( // per-object failure. match db_bounded( db_record_deadline(deadline), - retry_db_record(|| db.record_pinned_cid_with_source(&sha, &raw_cid, repo_id)), + retry_db_record(|| { + // #218 review round 9 (guidance #3 — + // linearization): always go through the + // fenced form. The fence is either + // captured (sweep / public-pin path: the + // third fence is the linearization point + // that closes the rule-write / + // record-write race) or absent + // (push-side admission where the + // decision is made at request time — + // we pass `i64::MAX` as a sentinel that + // the fenced form treats as "no fence + // check"). The 3-arg + // `record_pinned_cid_with_source` is + // still available for tests that don't + // own a fence, but the production + // pinner routes through here. + let fence_epoch = fence.map(|f| f.captured_epoch()).unwrap_or(i64::MAX); + db.record_pinned_cid_with_source_fenced( + &sha, + &raw_cid, + repo_id, + fence_epoch, + ) + }), ) .await { @@ -2200,7 +2384,7 @@ mod tests { endpoint } - /// A sleeping-but-live endpoint. Answers `200` with an empty body after + /// A sleeping-but-live endpoint. Answers `200` with a JSON `Hash` after /// `delays[i]` for the i-th request it accepts (the last entry repeats), so /// a test can make one add slow and the next fast. Drains the full request, /// headers plus the declared `Content-Length` body, before sleeping: exactly @@ -2208,8 +2392,9 @@ mod tests { /// a write failure on the client and turn a slow-but-healthy add into a /// different failure shape. /// - /// An empty body is a successful pin: `pin_git_object` falls back to the CID - /// it computed from the bytes when the response carries no `Hash`. + /// The response carries a real `Hash` because `pin_git_object` now refuses + /// to record a CID a 2xx body did not actually acknowledge: a successful + /// pin needs `{"Hash":"..."}`, not an empty body. async fn delaying_endpoint(delays: Vec) -> String { use tokio::io::{AsyncReadExt, AsyncWriteExt}; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -2246,9 +2431,14 @@ mod tests { } } tokio::time::sleep(delay).await; + let body = b"{\"Hash\":\"QmDelayMockCid\"}"; let _ = sock - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .write_all( + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", body.len()) + .as_bytes(), + ) .await; + let _ = sock.write_all(body).await; let _ = sock.flush().await; }); } @@ -2335,6 +2525,74 @@ mod tests { ); } + /// The misconfigured-`GITLAWB_IPFS_API` false positive (P3): a 2xx response + /// that carries no `Hash` field (proxy returning HTML, health check on the + /// wrong port, truncated gateway) must FAIL the pin, not fall back to the + /// locally computed `expected_cid`. Falling back records a `pinned_cids` + /// row for bytes the backend never stored, and the reconciliation sweep + /// trusts rows as durability evidence — so the false positive becomes a + /// permanent blind spot for the backstop. A missing `Hash` must surface as + /// an explicit error, never a successful pin. + #[tokio::test] + async fn pin_git_object_rejects_a_2xx_without_a_hash_field() { + let endpoint = empty_ok_endpoint().await; + let inner = tokio::time::timeout( + Duration::from_secs(30), + pin_git_object(&endpoint, "deadbeef", b"some object bytes\n", None), + ) + .await + .expect("wedge guard: an immediate empty 200 cannot take 30s"); + let err = inner.expect_err( + "a 2xx without a Hash field must not surface as a successful pin \ + (would record a phantom pinned_cids row the sweep then trusts)", + ); + assert!( + err.to_string().contains("without a Hash field"), + "the error must name the missing Hash so operators diagnose the endpoint: {err:#}" + ); + } + + /// A 200 that answers with an empty body and no `Hash` — the exact shape of + /// a proxy or health-check endpoint mistaken for a Kubo API. + async fn empty_ok_endpoint() -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + tokio::spawn(async move { + let mut acc = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + let n = match sock.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + acc.extend_from_slice(&buf[..n]); + if let Some(hdr_end) = + acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4) + { + let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase(); + let len: usize = headers + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if acc.len() >= hdr_end + len { + break; + } + } + } + let _ = sock + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .await; + let _ = sock.flush().await; + }); + } + }); + endpoint + } + /// The second unhardened sink, reached from `sync.rs`. Same shape as above. #[tokio::test] async fn cat_against_silent_endpoint_errors_within_its_own_timeout() { @@ -2382,6 +2640,7 @@ mod tests { &db, "repo-batch-budget", Duration::from_millis(5500), + None, ), ) .await @@ -2449,6 +2708,7 @@ mod tests { &db, "repo-batch-continues", Duration::from_secs(90), + None, ), ) .await @@ -2487,6 +2747,7 @@ mod tests { &db, "repo-batch-rejects", Duration::from_secs(60), + None, ), ) .await @@ -2584,6 +2845,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(2), + None, ), ) .await @@ -2671,6 +2933,7 @@ mod tests { &db, "repo-merge-test", Duration::from_millis(1500), + None, ), ) .await @@ -2744,6 +3007,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(60), + None, ), ) .await @@ -2818,6 +3082,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(60), + None, ), ) .await @@ -2886,6 +3151,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(60), + None, ), ) .await @@ -3120,6 +3386,7 @@ mod tests { &db, "repo-stalled-db", Duration::from_millis(1500), + None, ), ) .await @@ -3190,6 +3457,7 @@ mod tests { &db, "repo-skip-stalled", Duration::from_millis(1500), + None, ), ) .await @@ -3256,6 +3524,7 @@ mod tests { &db, "repo-multi-stalled", Duration::from_millis(1500), + None, ), ) .await @@ -3323,6 +3592,7 @@ mod tests { &db, "repo-spent-budget", Duration::from_millis(2000), + None, ), ); let (pinned, ()) = tokio::join!(pin, locker); @@ -3396,6 +3666,7 @@ mod tests { &db, "repo-definite-error", Duration::from_millis(1200), + None, ), ); let (pinned, ()) = tokio::join!(pin, commit); @@ -3477,6 +3748,7 @@ mod tests { &db, "repo-marker-floor", Duration::from_millis(1500), + None, ), ); let (pinned, ()) = tokio::join!(pin, controller); @@ -3581,6 +3853,7 @@ mod tests { &db, "repo-repair-stalled", Duration::from_millis(2200), + None, ), ); let (pinned, mut cids_lock) = tokio::join!(pin, controller); diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 66bfa0961..f2125cf68 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -16,6 +16,7 @@ mod operator; mod p2p; mod pinata; mod rate_limit; +mod reconciliation; mod server; mod state; mod sync; @@ -642,6 +643,29 @@ async fn main() -> Result<()> { info!("auto-sync worker started"); } + // Periodic reconciliation sweep: re-derives pin/seal sets and fills gaps + // so a dropped replication job never means data loss. + { + let db = state.db.clone(); + let config = Arc::clone(&state.config); + let http_client = Arc::clone(&state.http_client); + let node_keypair = Arc::clone(&state.node_keypair); + let node_did = state.node_did.clone(); + let pin_sem = Arc::clone(&state.pin_semaphore); + let shutdown_rx = state.subscribe_shutdown(); + if reconciliation::spawn( + db, + config, + http_client, + node_keypair, + node_did, + pin_sem, + shutdown_rx, + ) { + info!("reconciliation sweep worker started"); + } + } + // On-chain operator setup: verify stake + spawn heartbeat loop if !state.config.contract_node_staking.is_empty() && !state.config.operator_private_key.is_empty() diff --git a/crates/gitlawb-node/src/metrics.rs b/crates/gitlawb-node/src/metrics.rs index c95ef1d18..85c98f488 100644 --- a/crates/gitlawb-node/src/metrics.rs +++ b/crates/gitlawb-node/src/metrics.rs @@ -15,6 +15,9 @@ //! `gitlawb_pack_size_bytes` //! * a single `gitlawb_info{version, did}` gauge = 1, for joins/dashboards //! * currently-connected peer count — `gitlawb_peers_connected` +//! * reconciliation sweep gaps found and filled — +//! `gitlawb_reconciliation_gaps_found_total` / +//! `gitlawb_reconciliation_gaps_filled_total` //! //! All metrics live in a single process-wide registry initialized by //! [`init`]. Increment helpers (`record_push`, `record_auth_failure`, ...) @@ -33,8 +36,8 @@ use std::sync::OnceLock; use prometheus::{ - Encoder, Histogram, HistogramOpts, IntCounterVec, IntGauge, IntGaugeVec, Opts, Registry, - TextEncoder, + Encoder, Histogram, HistogramOpts, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Opts, + Registry, TextEncoder, }; /// The single, process-wide metrics registry. Initialized by [`init`]. @@ -51,6 +54,8 @@ static SYNC_PROCESSED: OnceLock = OnceLock::new(); static WEBHOOK_DELIVERIES: OnceLock = OnceLock::new(); static PACK_SIZE: OnceLock = OnceLock::new(); static PEERS_CONNECTED: OnceLock = OnceLock::new(); +static RECONCILIATION_GAPS_FOUND: OnceLock = OnceLock::new(); +static RECONCILIATION_GAPS_FILLED: OnceLock = OnceLock::new(); /// One-time initializer. Builds the registry, registers every metric, /// and sets the constant `gitlawb_info` gauge. Idempotent — calling @@ -202,6 +207,30 @@ fn init_inner(version: &str, node_did: &str) { .set(peers_connected) .expect("set PEERS_CONNECTED once"); + let gaps_found = IntCounter::with_opts(Opts::new( + "gitlawb_reconciliation_gaps_found_total", + "Total reconciliation sweep gaps detected (objects that should be pinned but are not)", + )) + .expect("gitlawb_reconciliation_gaps_found_total definition"); + registry + .register(Box::new(gaps_found.clone())) + .expect("register gitlawb_reconciliation_gaps_found_total"); + RECONCILIATION_GAPS_FOUND + .set(gaps_found) + .expect("set RECONCILIATION_GAPS_FOUND once"); + + let gaps_filled = IntCounter::with_opts(Opts::new( + "gitlawb_reconciliation_gaps_filled_total", + "Total reconciliation sweep gaps successfully filled (objects pinned by the sweep)", + )) + .expect("gitlawb_reconciliation_gaps_filled_total definition"); + registry + .register(Box::new(gaps_filled.clone())) + .expect("register gitlawb_reconciliation_gaps_filled_total"); + RECONCILIATION_GAPS_FILLED + .set(gaps_filled) + .expect("set RECONCILIATION_GAPS_FILLED once"); + REGISTRY .set(registry) .expect("set REGISTRY once (init must be called exactly once)"); @@ -284,6 +313,20 @@ pub fn set_peers_connected(count: i64) { } } +/// Record reconciliation sweep gaps found (objects that should be pinned but are not). +pub fn record_reconciliation_gaps_found(count: u64) { + if let Some(c) = RECONCILIATION_GAPS_FOUND.get() { + c.inc_by(count); + } +} + +/// Record reconciliation sweep gaps filled (objects successfully pinned by the sweep). +pub fn record_reconciliation_gaps_filled(count: u64) { + if let Some(c) = RECONCILIATION_GAPS_FILLED.get() { + c.inc_by(count); + } +} + /// Encode the registry as the standard Prometheus text exposition format. /// Returns an error if `init` was never called. pub fn encode() -> Result { @@ -321,6 +364,8 @@ mod tests { .expect("PUSHES set after init") .with_label_values(&["alice/repo"]) .inc(); + record_reconciliation_gaps_found(7); + record_reconciliation_gaps_filled(3); let body = encode().expect("encode should succeed after init"); assert!( @@ -335,6 +380,14 @@ mod tests { body.contains("gitlawb_pushes_total{repo=\"alice/repo\"} 1"), "expected the incremented counter to be visible in: {body}" ); + assert!( + body.contains("gitlawb_reconciliation_gaps_found_total 7"), + "expected the reconciliation gaps-found counter to be visible in: {body}" + ); + assert!( + body.contains("gitlawb_reconciliation_gaps_filled_total 3"), + "expected the reconciliation gaps-filled counter to be visible in: {body}" + ); } /// #192 F4: `init` is idempotent and safe to call repeatedly. The panic that diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 14f1d5824..180904023 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -169,6 +169,7 @@ pub async fn pin_new_objects( db: &crate::db::Db, repo_id: &str, batch_budget: Duration, + fence: Option<&crate::ipfs_pin::PolicyFence>, ) -> Vec<(String, String)> { if jwt.is_empty() { return vec![]; @@ -179,6 +180,18 @@ pub async fn pin_new_objects( let mut pinned = Vec::new(); for (attempted, sha) in object_list.into_iter().enumerate() { + // Policy fence (R1-P1): a visibility narrow that lands after the caller + // built this batch must abort it before the next irreversible upload. + if let Some(f) = fence { + if !f.is_current().await { + tracing::warn!( + repo = %f.repo_id(), + unattempted = total - attempted, + "visibility policy changed mid-batch; stopping the Pinata pin loop" + ); + break; + } + } // Top of the iteration, before any of this object's work: an object is never // started with a remainder too small to cover a bounded read's teardown. The // gate is shared with the IPFS loop so the two cannot drift apart in how they @@ -394,6 +407,21 @@ pub async fn pin_new_objects( } }; + // Dispatch fence (R1-P1): re-read the policy epoch immediately before + // the irreversible HTTP POST. The iteration-top check catches a narrow + // that landed before work began; THIS check catches a narrow that landed + // during the has_pinata_cid round-trip or the bounded Git read. + if let Some(f) = fence { + if !f.is_current().await { + tracing::warn!( + repo = %f.repo_id(), + unattempted = total - attempted, + "visibility policy changed during preparation; aborting Pinata upload" + ); + break; + } + } + match pin_object(client, upload_url, jwt, &sha, &data).await { Ok(cid) if !cid.is_empty() => { // The resolver key (`pinned_cids.cid`) must be the locally-computed @@ -420,15 +448,55 @@ pub async fn pin_new_objects( // future is cancelled. The warn names the arm through the error's own // Display and the site keeps its existing behavior: the pair is still // returned, and the row may or may not exist. - if let Err(e) = crate::ipfs_pin::db_bounded( + // Round 10 P2: a closed/failed DB write means the + // (sha, cid) pair is not durable; we suppress the + // `pinned.push` for this sha so the reconcile cannot + // count a Pinata gap as filled when no row exists. + // The autocommit timeout case (the `BoundedDbError` + // other than `Elapsed`) is genuinely unknown, so we + // keep the previous single-statement reasoning and + // push in that case. The source-record failure arms + // are also a hard failure (multi-statement + // transaction never committed) and suppress the push + // in the same way. + let mut db_record_durable = false; + match crate::ipfs_pin::db_bounded( crate::ipfs_pin::db_record_deadline(deadline), crate::ipfs_pin::retry_db_record(|| { - db.record_pinata_cid(&sha, &raw_cid, &cid, Some(repo_id)) + // P2 (reviewer round 9): Pinata's POST is + // irreversible exactly like IPFS's, so + // route the record through the fenced + // variant when a fence is in scope. The + // `i64::MAX` sentinel tells the helper + // to skip the lock + comparison (the + // unfenced caller path). + let fence_epoch = fence.map(|f| f.captured_epoch()).unwrap_or(i64::MAX); + db.record_pinata_cid(&sha, &raw_cid, &cid, Some(repo_id), fence_epoch) }), ) .await { - tracing::warn!(sha = %sha, err = %e, "failed to record pinata_cid in DB"); + Ok(()) => db_record_durable = true, + Err(crate::ipfs_pin::BoundedDbError::Elapsed) => { + // Autocommit: statement may have committed. + // Treat as unknown-but-persisted, keep the + // old behavior (push the pair). + tracing::warn!( + sha = %sha, + "record_pinata_cid deadline elapsed for autocommit upsert; \ + the row may or may not exist (the previous comment's reasoning)" + ); + db_record_durable = true; + } + Err(e) => { + tracing::warn!( + sha = %sha, + err = %e, + "failed to record pinata_cid in DB; suppressing the (sha, cid) push \ + so the reconcile cannot count this gap as filled" + ); + // db_record_durable stays false → push suppressed + } } // F1 (#173 round 8): also record the first pinner in pin_repo_sources. // U3: an exhausted retry marks the set incomplete so the resolver keeps @@ -444,9 +512,9 @@ pub async fn pin_new_objects( // wraps `record_pin_source`, an explicit transaction, so a timed-out // call definitely never committed and the source is definitely // missing. Mark the set incomplete rather than leaving it incomplete - // and unmarked. Note the contrast with `record_pinata_cid` a few - // lines up: that one is a single autocommit statement, so its - // timeout genuinely is an unknown outcome and it is warn-only. + // and unmarked. Round 10 P2: also suppress the (sha, cid) push + // because the durable record set is now incomplete; the next pass + // re-offers the gap. Err(e @ crate::ipfs_pin::BoundedDbError::Elapsed) => { tracing::warn!( sha = %sha, @@ -463,6 +531,7 @@ pub async fn pin_new_objects( { tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); } + db_record_durable = false; } Err(e) => { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); @@ -474,9 +543,12 @@ pub async fn pin_new_objects( { tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); } + db_record_durable = false; } } - pinned.push((sha, cid)); + if db_record_durable { + pinned.push((sha, cid)); + } } Ok(_) => {} Err(e) => { @@ -687,6 +759,7 @@ mod tests { &db, "repo-merge-test", Duration::from_millis(5500), + None, ), ) .await @@ -778,6 +851,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(2), + None, ), ) .await @@ -867,6 +941,7 @@ mod tests { "repo-git-timeout", // Generous, so a call that ends on time ended on `git_timeout`. Duration::from_secs(60), + None, ), ) .await @@ -968,6 +1043,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(60), + None, ), ) .await @@ -1042,6 +1118,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(60), + None, ), ) .await @@ -1093,6 +1170,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(60), + None, ), ) .await @@ -1122,6 +1200,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(60), + None, ), ) .await @@ -1175,6 +1254,7 @@ mod tests { &db, "repo-merge-test", Duration::from_secs(60), + None, ), ) .await @@ -1280,6 +1360,7 @@ mod tests { &db, "repo-pinata-stalled", Duration::from_millis(1500), + None, ), ) .await @@ -1336,9 +1417,15 @@ mod tests { // bytes. let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"pinata skip seed").to_string(); - db.record_pinata_cid(&sha, &raw_cid, "QmSeedProviderCid", Some("repo-seed")) - .await - .unwrap(); + db.record_pinata_cid( + &sha, + &raw_cid, + "QmSeedProviderCid", + Some("repo-seed"), + i64::MAX, + ) + .await + .unwrap(); db.record_pin_source(&sha, "repo-seed").await.unwrap(); let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let endpoint = delaying_pinata_endpoint( @@ -1370,6 +1457,7 @@ mod tests { &db, "repo-pinata-skip-stalled", Duration::from_millis(1500), + None, ), ) .await @@ -1470,6 +1558,7 @@ mod tests { &db, "repo-pinata-spent-budget", Duration::from_millis(2000), + None, ), ); let (pinned, ()) = tokio::join!(pin, locker); @@ -1555,6 +1644,7 @@ mod tests { &db, "repo-pinata-post-upload", Duration::from_millis(1500), + None, ), ) .await @@ -1568,10 +1658,18 @@ mod tests { drop(lock); upload.assert_async().await; + // Round 10 P2: a successful upload with the post-upload + // source record timing out no longer returns the (sha, cid) + // pair. The reconcile contract is "filled iff durable": when + // any post-upload DB write fails, the push is suppressed + // and the gap is re-offered. Prior to the round 10 fix, the + // push fired regardless of the source-record outcome, and + // the next pass would re-offer the same gap. assert_eq!( pinned.len(), - 1, - "the upload succeeded, so this lane still returns the pair: {pinned:?}" + 0, + "record_pin_source timed out (pin_repo_sources locked); the push must be \ + suppressed so the reconcile does not count a non-durable pair as filled: {pinned:?}" ); assert!( elapsed < Duration::from_secs(8), diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs new file mode 100644 index 000000000..67c176275 --- /dev/null +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -0,0 +1,3520 @@ +use rand::Rng; +use std::collections::HashSet; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::watch; + +use crate::config::Config; +use crate::db::Db; + +/// How often to run a sweep pass. +const SWEEP_INTERVAL_SECS: u64 = 3600; + +/// Maximum repos to process per pass — prevents the sweep from becoming +/// the O(repos) amplification the admission-control work exists to prevent. +const REPOS_PER_PASS: usize = 100; + +/// Maximum objects to pin per backend per repo in a single pass — prevents one +/// large repo from monopolizing the blocking pool or the hourly budget. Applied +/// after filtering out already-pinned objects so the cap reflects actual work. +const MAX_OBJECTS_PER_REPO: usize = 50_000; + +/// Per-repo deadline for the blocking git scan (list_all_objects + visibility +/// filter). A pathological repo that stalls past this is skipped for the pass. +const REPO_SCAN_DEADLINE: Duration = Duration::from_secs(300); + +/// Per-repo deadline for the pinning phase (IPFS + Pinata uploads). An +/// unavailable backend that stalls per-object must not hold the sweep for +/// the entire backlog; this bounds the wall time of each pinning PHASE. +/// +/// The phases do NOT share one budget (R2-P3): the scan, the mid-scan +/// visibility re-filter, the per-backend pin-boundary authorization +/// re-derivation, the withheld-blob walk, and each pin/seal phase each get +/// their own `REPO_SCAN_DEADLINE` / `PIN_PHASE_DEADLINE`. A repo's worst case +/// is therefore ADDITIVE, up to ~30min in pathological conditions (scan 5m + +/// mid-scan re-filter 5m + authz re-derivation 5m + withheld walk 5m + public +/// pin 5m + encrypted seal 5m), not bounded at a single deadline. That is a +/// deliberate trade: starving a later phase of the budget the scan consumed +/// would silently disable the authorization check or the recovery-copy seal +/// for exactly the large repos the sweep exists for. The sweep runs hourly +/// and each phase is still individually bounded, so a pathological repo delays +/// other repos by at most that phase, not the hour. +const PIN_PHASE_DEADLINE: Duration = Duration::from_secs(300); + +/// node_state key under which the sweep's keyset cursor is persisted across +/// restarts (R2-P1). +const CURSOR_KEY: &str = "reconciliation_sweep_cursor"; + +/// Log message emitted when the Irys anchor call fails after a successful +/// seal. The contract is one-shot: `plan_seal` returns `SkipUnchanged` on +/// every subsequent pass once the recipients tag matches, so a failed +/// anchor here is permanent until a withheld change forces a new seal. +/// Factored to a const so the test +/// `encrypted_manifest_anchor_log_does_not_promise_retry` can pin the +/// "no retry promised" property at the cargo-test level. +const ENCRYPTED_MANIFEST_ANCHOR_FAILED_MSG: &str = + "encrypted manifest anchor failed; this seal will NOT be \ + retried on a later pass (plan_seal returns SkipUnchanged \ + when the recipients tag is stable). A subsequent withheld \ + change forces a new seal and re-anchors the manifest."; + +/// Per-backend continuation-cursor progress, expressed as an +/// effect-side state machine (#218 review round 9, guidance #4). +/// The tri-state `Option>` is the wire form; this +/// enum is the documented shape the closure maps from. +/// +/// The contract: a cursor must reflect EFFECT, not plan. A +/// pre-dispatch failure (fence capture failed, refilter returned +/// `None`, dispatch produced an empty `to_pin`) cannot look like +/// work completed — the unattempted prefix must retry at the +/// head of the next pass. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ProgressState { + /// No work was attempted this pass (fence capture failed, + /// refilter returned `None`, dispatch produced an empty + /// `to_pin`). The cursor is preserved — the unattempted + /// prefix retries at the head of the next pass. + Idle, + /// A subset of the cap was dispatched. The next pass + /// rotates past `last_dispatched`, retrying everything + /// beyond. + Advanced { last_dispatched: String }, + /// The missing set was empty. The cursor is cleared + /// (a future pass sees a fresh start). + Drained, +} + +impl ProgressState { + /// Map the three states to the wire form: `Some(value)` for + /// a write, `None` for "leave the row alone". + /// + /// The function [`next_offset_write`] is the same logic in + /// callable form; this method exists so a future caller + /// that has a `ProgressState` in hand (rather than the + /// inputs to `next_offset_write`) can convert without + /// re-deriving the decision. + pub(crate) fn to_wire(&self) -> Option> { + match self { + ProgressState::Idle => None, + ProgressState::Advanced { last_dispatched } => Some(Some(last_dispatched.clone())), + ProgressState::Drained => Some(None), + } + } +} + +/// The next-offset decision. Called by `run_pass` at the +/// cursor-write site and exposed at module scope so a test can +/// drive it with known `(scan_ok, had_work, dispatched)` triples +/// and assert that the returned `ProgressState` (and its +/// `to_wire()`) is what the cursor-write site will land. P2 +/// (reviewer round 9): the previous test never called the +/// closure, so the wire-form test pinned itself to its own +/// arm-by-arm reproduction. Now there is one encoding and the +/// test calls the function under test. +pub(crate) fn next_offset_write( + scan_ok: bool, + had_work: bool, + dispatched: Option, +) -> ProgressState { + if !scan_ok { + ProgressState::Idle + } else if let Some(last) = dispatched { + ProgressState::Advanced { + last_dispatched: last, + } + } else if !had_work { + ProgressState::Drained + } else { + ProgressState::Idle + } +} + +/// Whether the sweep should spawn given the current configuration. +/// Extracted for testing — test both directions independently. +fn should_spawn(config: &Config) -> bool { + if !config.reconciliation_sweep { + return false; + } + !config.ipfs_api.is_empty() || !config.pinata_jwt.is_empty() +} + +/// Spawn the periodic reconciliation sweep background task. +/// No-op when neither IPFS nor Pinata is configured, or when +/// `reconciliation_sweep` is disabled. Returns `true` when the worker was +/// actually spawned so the caller can gate its own "worker started" logging. +pub fn spawn( + db: Arc, + config: Arc, + http_client: Arc, + node_keypair: Arc, + node_did: gitlawb_core::did::Did, + pin_sem: Arc, + mut shutdown_rx: watch::Receiver, +) -> bool { + if !should_spawn(&config) { + tracing::info!( + "reconciliation sweep: disabled or neither IPFS nor Pinata configured, skipping spawn" + ); + return false; + } + + tokio::spawn(async move { + let node_seed = *node_keypair.to_seed(); + // Resume from the persisted cursor (R2-P1): a node restart must not + // re-walk every repo, and the cursor is only ever advanced after a + // batch completes, so an interrupted pass resumes where it stopped. + let mut cursor: Option = match db.get_node_state(CURSOR_KEY).await { + Ok(v) => v, + Err(e) => { + tracing::warn!(err = %e, "failed to load reconciliation sweep cursor from node_state; starting from scratch"); + None + } + }; + + // First pass: random delay to desynchronize sweep starts across nodes + // on a rolling restart (R1-P3). Subsequent passes use the fixed interval. + // Generate the delay before the async block to avoid Send issues with thread_rng. + let initial_delay = Duration::from_millis(rand::thread_rng().gen_range(0..60000)); + let mut first_pass = true; + + loop { + // On first pass, wait for the initial random delay before starting + if first_pass { + tracing::debug!( + delay_ms = initial_delay.as_millis() as u64, + "reconciliation sweep: waiting initial jitter delay" + ); + tokio::select! { + _ = tokio::time::sleep(initial_delay) => {} + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + tracing::info!("reconciliation sweep: shutdown signal received during initial delay, exiting"); + return; + } + } + } + first_pass = false; + } + + let start = std::time::Instant::now(); + match run_pass( + &db, + &config, + &http_client, + &node_seed, + &node_did, + &pin_sem, + REPO_SCAN_DEADLINE, + &mut cursor, + &mut shutdown_rx, + ) + .await + { + Ok((count, gaps, filled)) => { + tracing::info!( + repos = count, + gaps_found = gaps, + gaps_filled = filled, + elapsed_ms = start.elapsed().as_millis() as u64, + "reconciliation sweep pass complete" + ); + } + Err(e) => { + tracing::warn!(err = %e, "reconciliation sweep pass failed"); + } + } + + if *shutdown_rx.borrow() { + tracing::info!("reconciliation sweep: shutdown signal received, exiting"); + return; + } + + tokio::select! { + _ = tokio::time::sleep(std::time::Duration::from_secs(SWEEP_INTERVAL_SECS)) => {} + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + tracing::info!("reconciliation sweep: shutdown signal received, exiting"); + return; + } + } + } + } + }); + + true +} + +/// Re-derive the *allowed* public-object set from fresh rules and intersect it +/// with the scanned object list. Returns `None` when the re-derivation failed +/// (caller skips the repo). This is the path-scoped-visibility re-filter that +/// runs against rules re-fetched after the git scan, so a narrowing made +/// mid-scan is honored before anything is pinned. +/// +/// The caller hands an absolute `deadline`; the whole re-derivation +/// (replicable_blob_set_bounded + all_blob_oids) runs against the remaining +/// budget rather than granting each git child a fresh timeout. The mid-scan +/// re-filter and each pin-boundary re-derivation each get their OWN fresh +/// `REPO_SCAN_DEADLINE` (R2-P1) so a scan that exhausts its own budget cannot +/// disable the authorization-at-dispatch recheck — the read phase is additive +/// with the pin phases, documented at `PIN_PHASE_DEADLINE`. +async fn refilter_public_objects( + disk: &std::path::Path, + rules: &[crate::db::VisibilityRule], + is_public: bool, + owner_did: &str, + object_list: Vec, + deadline: Instant, +) -> Option> { + let disk_clone = disk.to_path_buf(); + let rules_clone = rules.to_vec(); + let owner_clone = owner_did.to_string(); + + match tokio::time::timeout( + deadline.saturating_duration_since(Instant::now()), + tokio::task::spawn_blocking(move || -> anyhow::Result> { + // The shared deadline spans this whole re-filter + // (allowed_blob_tree_sets_bounded), so a slow walk is bounded as a + // unit rather than granting each git child a fresh timeout. + let (allowed, allowed_trees, all_blobs, all_trees) = + crate::git::visibility_pack::allowed_blob_tree_sets_bounded( + &disk_clone, + "git", + deadline, + &rules_clone, + is_public, + &owner_clone, + )?; + Ok(crate::git::visibility_pack::replicable_objects_fail_closed( + object_list, + &allowed, + &all_blobs, + &allowed_trees, + &all_trees, + )) + }), + ) + .await + { + Ok(Ok(Ok(list))) => Some(list), + Ok(Ok(Err(e))) => { + tracing::warn!(err = %e, "visibility re-derivation failed"); + None + } + Ok(Err(e)) => { + tracing::warn!(err = %e, "visibility re-derivation task panicked"); + None + } + Err(_) => { + tracing::warn!("visibility re-derivation deadline exceeded"); + None + } + } +} +// Test-only fault injection for the PIN-BOUNDARY re-derivation (#218 review +// round 8 P2). The "nothing was dispatched" branch of the continuation write is +// reached when a stage between the missing-set query and the backend call +// declines — a `PolicyFence` capture that fails, a quarantine recheck that says +// skip, a re-derivation that errors. Every one of those is a DB or git failure +// on a repo the test has just built healthy, and no fixture can produce one from +// the outside: the mid-scan re-filter runs first on the same rules and the same +// budget, so anything that would starve the pin-boundary call has already made +// the sweep `continue` well before the offset write. Rather than assert the +// contract at a lower layer than the one that owns it (the sweep loop's call +// site), the boundary gets an explicit seam. +// +// Thread-local because `#[sqlx::test]` drives each test on its own +// current-thread runtime, so the flag cannot race across tests. +#[cfg(test)] +thread_local! { + static FAIL_PIN_BOUNDARY_REDERIVE: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// Force (or release) the pin-boundary re-derivation failure. Test-only. +#[cfg(test)] +fn set_fail_pin_boundary_rederive(on: bool) { + FAIL_PIN_BOUNDARY_REDERIVE.with(|c| c.set(on)); +} + +/// [`refilter_public_objects`] at the pin boundary — the last authorization +/// stage before an irreversible public pin, and the one stage whose failure the +/// continuation write has to distinguish from "nothing to do". Identical to the +/// mid-scan call except for the test seam above. +async fn pin_boundary_refilter( + disk: &std::path::Path, + rules: &[crate::db::VisibilityRule], + is_public: bool, + owner_did: &str, + object_list: Vec, + deadline: Instant, +) -> Option> { + #[cfg(test)] + if FAIL_PIN_BOUNDARY_REDERIVE.with(|c| c.get()) { + return None; + } + refilter_public_objects(disk, rules, is_public, owner_did, object_list, deadline).await +} + +/// Re-check quarantine AND root visibility immediately before an irreversible +/// public pin (R1-P1). Returns the fresh repo row plus fresh rules, or `None` +/// when the pin must be skipped. DB failures are treated as skip (never pin on +/// a stale allow), so one repo's failure does not abort the pass. +async fn recheck_public_pin( + db: &Db, + repo_id: &str, + repo_slug: &str, +) -> Option<(crate::db::RepoRecord, Vec)> { + match db.is_repo_quarantined(repo_id).await { + Ok(true) => { + tracing::warn!(repo = %repo_slug, "repo quarantined, skipping pin"); + return None; + } + Ok(false) => {} + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "quarantine recheck failed, skipping pin"); + return None; + } + } + let rules = match db.list_visibility_rules(repo_id).await { + Ok(r) => r, + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "visibility rules re-fetch failed, skipping pin"); + return None; + } + }; + let fresh = match db.get_repo_by_id(repo_id).await { + Ok(Some(r)) => r, + Ok(None) => { + tracing::warn!(repo = %repo_slug, "repo disappeared from DB, skipping pin"); + return None; + } + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "repo re-fetch failed, skipping pin"); + return None; + } + }; + if !crate::visibility::listable_at_root(&rules, fresh.is_public, &fresh.owner_did, None) { + tracing::warn!(repo = %repo_slug, "visibility narrowed, skipping pin"); + return None; + } + Some((fresh, rules)) +} + +/// Compute the deterministic missing set: `all` minus `done`, sorted so two +/// passes over the same data yield the same pin order. Not capped here — the +/// caller applies the cap and logs a truncation warning. +/// +/// `start_after` is the per-(repo, backend) continuation offset (#218 +/// review P2). When `Some`, the sorted missing set is ROTATED so the +/// first OID is the smallest one strictly greater than `start_after`, +/// and every OID ≤ `start_after` is appended at the tail. The set as a +/// whole is unchanged; only the attempt order changes. Without the +/// rotation, a persistently failing early OID (e.g. one the local IPFS +/// daemon refuses for a transient-but-recurring reason) keeps landing +/// at the start of the sort and dominates the 50 000 cap every +/// hourly tick, so a healthy gap past the cap is never attempted. +/// With the rotation, the cap still bounds per-pass work but advances +/// fairly across passes: failed OIDs retried at the tail of the +/// next pass, the healthy gap moves into the cap window. +/// +/// `start_after = None` preserves the pre-P2 deterministic head-first +/// order, which is what a fresh (repo, backend) or a `done = TRUE` +/// pair does. +fn missing_oids(all: &[String], done: &[String], start_after: Option<&str>) -> Vec { + let done_set: HashSet<&str> = done.iter().map(|s| s.as_str()).collect(); + let mut missing: Vec = all + .iter() + .filter(|s| !done_set.contains(s.as_str())) + .cloned() + .collect(); + missing.sort(); + let Some(start) = start_after else { + return missing; + }; + // Find the rotation point: the first OID strictly greater than + // `start`. OIDs ≤ start (typically: previously truncated, possibly + // failing) move to the tail so the cap window sees fresh ground. + // `partition_point` is the standard-library rotation seam: it + // returns the index of the first element for which the predicate + // is false, which is exactly the first `oid > start` after a sort. + let split = missing.partition_point(|oid| oid.as_str() <= start); + if split == 0 || split >= missing.len() { + // Either nothing has been attempted yet (split == 0) or every + // missing OID is ≤ start (the offset is past the end, which + // should not happen on a well-formed pass but the rotation + // would lose data — return sorted order as-is). + return missing; + } + let mut rotated = Vec::with_capacity(missing.len()); + rotated.extend(missing[split..].iter().cloned()); + rotated.extend(missing[..split].iter().cloned()); + rotated +} + +/// Cap a missing set, logging once when it was truncated. +fn cap_missing(v: Vec, repo_slug: &str, backend: &str) -> Vec { + if v.len() > MAX_OBJECTS_PER_REPO { + tracing::warn!( + repo = %repo_slug, + backend, + cap = MAX_OBJECTS_PER_REPO, + "per-repo missing cap reached, truncating" + ); + let mut v = v; + v.truncate(MAX_OBJECTS_PER_REPO); + v + } else { + v + } +} + +/// Run one sweep pass. Returns `(repos_scanned, gaps_found, gaps_filled)`. +/// +/// `repos_scanned` counts every repo actually visited this pass (mirror rows +/// and hard skips excluded, and the loop stops counting the moment a shutdown +/// signal breaks the batch), so the returned value never overreports work that +/// a mid-pass shutdown prevented (R1-P3). +/// +/// Nine args but grouping them would churn every test caller for no behavioral +/// gain; the pins each arg names are independently documented at their use. +/// `rederive_budget` is the budget each authorization-at-dispatch +/// re-derivation runs against: the mid-scan re-filter and each pin-boundary +/// re-derivation compute their OWN fresh `Instant::now() + rederive_budget` +/// (R2-P1), so a scan that exhausts `REPO_SCAN_DEADLINE` cannot starve the +/// visibility recheck that runs right before anything is pinned. Plumbed +/// through the signature (rather than read as a module const) so the call-site +/// wiring is testable. +#[allow(clippy::too_many_arguments)] +async fn run_pass( + db: &Db, + config: &Config, + http_client: &reqwest::Client, + node_seed: &[u8; 32], + node_did: &gitlawb_core::did::Did, + pin_sem: &Arc, + rederive_budget: Duration, + cursor: &mut Option, + shutdown_rx: &mut watch::Receiver, +) -> anyhow::Result<(usize, usize, usize)> { + // Keyset pagination over repos ordered by immutable id so the cursor is + // robust against insertions, deletions, or updated_at shifts. The LIMIT + // is pushed into the SQL query so the hourly pass does not allocate, + // transfer, or deduplicate every repo on every sweep. + // + // Fetch one EXTRA row as a lookahead (R1-P2): `batch.len() < REPOS_PER_PASS` + // is a wrong "final page" proxy when the key space ends on an exact multiple + // of the page size — that batch LOOKS full, yet no row follows. With a + // lookahead row present, the batch is full for real (more remain); without + // it, the batch is the terminal page even at exactly REPOS_PER_PASS rows. + let fetched = db + .list_all_repos_deduped_stable(cursor.as_deref(), REPOS_PER_PASS as i64 + 1) + .await?; + let has_more = fetched.len() > REPOS_PER_PASS; + let batch: Vec<_> = fetched.into_iter().take(REPOS_PER_PASS).collect(); + + if batch.is_empty() { + // Covered everything: clear the persisted cursor so the next pass + // starts a fresh cycle instead of wedging on a stale key. + *cursor = None; + db.set_node_state(CURSOR_KEY, None).await?; + return Ok((0, 0, 0)); + } + + // Advance the in-memory cursor now so the next page in this run continues + // after this batch; the PERSISTED cursor is only moved once the batch fully + // completes below, so an interrupted batch is re-walked on restart. + let batch_last = batch.last().unwrap().id.clone(); + *cursor = Some(batch_last.clone()); + + let mut total_gaps_found = 0usize; + let mut total_gaps_filled = 0usize; + let mut repos_scanned = 0usize; + let mut batch_completed = true; + + for repo in &batch { + if *shutdown_rx.borrow() { + tracing::info!("reconciliation sweep: shutdown signal received mid-pass, exiting"); + batch_completed = false; + break; + } + + let repo_slug = format!( + "{}/{}", + crate::db::normalize_owner_key(&repo.owner_did), + repo.name + ); + + // Mirror rows carry a slash-form id written only by upsert_mirror_repo; + // they hardcode is_public = true and replicate no visibility rules, so a + // sweep over one would irreversibly publish content that the canonical + // gate never admitted (R2-P1). Skip them — the canonical row (if any) + // is swept under its own id. + if repo.id.contains('/') { + tracing::debug!(repo = %repo_slug, "mirror row (no canonical repo), skipping sweep"); + continue; + } + + let disk = PathBuf::from(&repo.disk_path); + if !disk.exists() { + tracing::warn!(repo = %repo_slug, "disk path missing, skipping"); + continue; + } + + // Counted only once the repo has a real chance of work: mirror rows and + // missing-disk rows are hard skips and never count as scanned (R1-P3). + repos_scanned += 1; + + // Cheap quarantine pre-check BEFORE the expensive git scan (R1-P3): + // a repo quarantined since admission should not burn a full scan just + // to be told to skip. + match db.is_repo_quarantined(&repo.id).await { + Ok(true) => { + tracing::warn!(repo = %repo_slug, "repo quarantined, skipping"); + continue; + } + Ok(false) => {} + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "quarantine check failed, skipping"); + continue; + } + } + + let rules = match db.list_visibility_rules(&repo.id).await { + Ok(r) => r, + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "visibility rules fetch failed, skipping"); + continue; + } + }; + + if !crate::visibility::listable_at_root(&rules, repo.is_public, &repo.owner_did, None) { + continue; + } + + // ── Full git scan (bounded) ───────────────────────────────────── + // One absolute deadline spans the whole scan. The mandatory visibility + // re-filter below runs against its OWN fresh budget (`authz_deadline`), + // NOT this spent deadline (R2-P1): a scan that legitimately consumes + // its whole budget would otherwise compute a zero remaining duration + // for the re-filter, time out immediately, and abort the repo + // iteration — permanently skipping exactly the large repos the sweep + // exists for. The pin-boundary re-derivations use the same fresh- + // budget pattern per backend arm, so no later authorization stage can + // be starved by the read phase's consumption. + let scan_deadline = Instant::now() + REPO_SCAN_DEADLINE; + let disk_clone = disk.clone(); + let owner_clone = repo.owner_did.clone(); + let rules_clone = rules.clone(); + let is_public = repo.is_public; + + let object_list = tokio::time::timeout( + scan_deadline.saturating_duration_since(Instant::now()), + tokio::task::spawn_blocking(move || -> anyhow::Result> { + let all_objs = + crate::git::push_delta::list_all_objects(&disk_clone, "git", scan_deadline)?; + let (allowed, allowed_trees, all_blobs, all_trees) = + crate::git::visibility_pack::allowed_blob_tree_sets_bounded( + &disk_clone, + "git", + scan_deadline, + &rules_clone, + is_public, + &owner_clone, + )?; + // Fail closed for blobs and denied trees (#172): the + // batch-all-objects enumeration carries dangling commits/trees + // from an aborted push, which have no path scoping to fail + // closed against. Requiring membership in the reachable object + // set keeps their messages, authors, parent links, and + // tree/file-name metadata off public pin backends (R2). + let reachable = crate::git::push_delta::reachable_object_oids( + &disk_clone, + "git", + scan_deadline, + )?; + Ok(crate::git::visibility_pack::replicable_objects_fail_closed( + all_objs, + &allowed, + &all_blobs, + &allowed_trees, + &all_trees, + ) + .into_iter() + .filter(|oid| reachable.contains(oid)) + .collect()) + }), + ) + .await; + + let object_list: Vec = match object_list { + Ok(Ok(Ok(list))) => list, + Ok(Ok(Err(e))) => { + tracing::warn!(repo = %repo_slug, err = %e, "full-scan failed, skipping"); + continue; + } + Ok(Err(e)) => { + tracing::warn!(repo = %repo_slug, err = %e, "full-scan task panicked, skipping"); + continue; + } + Err(_) => { + tracing::warn!(repo = %repo_slug, "full-scan deadline exceeded, skipping"); + continue; + } + }; + + // #218 review round 10 (P1): a path-scoped repo whose only + // reachable object is a direct blob/tree ref yields an + // empty public list (the anonymous public classifier removes + // the only object from the served set) while + // `withheld_blob_recipients_bounded` still assigns that + // object to the owner recovery set below. Skipping the + // whole repo here would suppress encrypted recovery too, + // and a lost/failed encrypted copy would never be + // repaired. Track empty-public-work as a flag and run + // the public phase conditionally; encrypted phase 2 runs + // regardless. + let has_public_work = !object_list.is_empty(); + + // Backend enable flags live outside the `if has_public_work` + // block because phase 2 (encrypted) consults `ipfs_enabled` + // and `_pin_permit` regardless of public-work state. Pulling + // them out of the inner block is a round 10 P1 follow-up: + // before that, an empty public list (which made + // `has_public_work` false) left these names out of scope + // for phase 2 and the build failed. + let ipfs_enabled = !config.ipfs_api.is_empty(); + let pinata_enabled = !config.pinata_jwt.is_empty(); + // `_pin_permit` is set by the public phase when it had gaps + // to pin; phase 2 reuses it. When `has_public_work` is false + // the public phase never ran, so the permit starts as + // `None` and phase 2 acquires a fresh one. + let mut _pin_permit: Option = None; + + // Fresh budget for the authorization-at-dispatch re-derivations (R1/R2): + // the scan may have legitimately consumed its whole `scan_deadline`, and + // reusing that deadline here would compute a zero remaining duration, + // return None, and turn an empty `to_pin` into a permanent hourly skip + // for exactly the large/slow repos the sweep exists for. This deadline is + // deliberately NOT shared with the scan. The mid-scan re-filter and each + // backend arm each re-derive against their OWN fresh budget (R2-P1): the + // IPFS arm re-derives first, and if two stages shared one budget a large + // repo that consumed it on an earlier walk would leave the later stage + // silently skipped every pass — empty `to_pin` behind a warn. + + // ── Phase 1: Public-object pinning (IPFS + Pinata) ──────────────── + // Gated on `has_public_work` (round 10 P1) so an empty + // post-scan public set skips phase 1 but still reaches phase 2 + // below. The flag is computed pre-refilter, so it governs only + // the post-scan empty case. The two `continue`s inside this + // block are unchanged and intentional: a FAILED recheck or a + // FAILED refilter (`None`) is a fail-closed skip of the whole + // repo iteration, while a SUCCESSFUL refilter that yields an + // empty list falls through — every downstream stage no-ops on + // empty missing sets and phase 2 still runs. + if has_public_work { + // Re-check quarantine AND visibility right now (fresh rules + repo row), + // then re-derive the allowed set from those fresh rules so a path-scoped + // narrowing made mid-scan is honored before anything is pinned. + let (fresh_repo, fresh_rules) = match recheck_public_pin(db, &repo.id, &repo_slug).await + { + Some(v) => v, + None => continue, + }; + + // Visibility may have narrowed mid-scan with a path-scoped deny. + // Recompute the allowed set from fresh rules and intersect it with the + // existing object_list. Runs against its OWN fresh `authz_deadline`, NOT + // the spent `scan_deadline` (R2-P1): the scan may have consumed the whole + // read budget, and a reused deadline computes a zero remaining duration, + // times out immediately, and aborts the repo iteration before the pin + // phases ever run — permanently skipping exactly the large repos the + // durability backstop exists for. The pin-boundary re-derivations below + // use the same fresh-budget pattern per backend arm. + let authz_deadline = Instant::now() + rederive_budget; + let refiltered = refilter_public_objects( + &disk, + &fresh_rules, + fresh_repo.is_public, + &fresh_repo.owner_did, + object_list, + authz_deadline, + ) + .await; + let Some(object_list) = refiltered else { + tracing::warn!(repo = %repo_slug, "fresh-visibility re-filter failed, skipping"); + continue; + }; + if object_list.is_empty() { + // #218 review round 10 (P1): a mid-pass visibility + // narrowing can leave the public set empty while + // withheld recipients are still non-empty. Nothing is + // skipped here: the offset loads, missing-set filters, + // and dispatch arms below all no-op on empty lists, and + // `next_offset_write(true, false, None)` resolves to + // `Drained`, clearing the per-backend cursor (correct: + // nothing is outstanding). Phase 2 then runs. + tracing::debug!(repo = %repo_slug, "refiltered public set is empty; encrypted recovery still runs"); + } + + // `ipfs_enabled` and `pinata_enabled` are declared outside + // the `if has_public_work` block (see above) so phase 2 + // can read them when the public list is empty. + + // Per-(repo, backend) continuation offset (#218 review P2): loaded + // here so the same offset is read once, used to rotate the + // missing set, and then the loop below writes the new offset + // back. A DB error on the load is treated as "start from the + // head" — the worst case is one pass at the old sort order, + // not a stalled sweep — so a corrupt row never blocks the + // per-hour gap-fill. + let ipfs_offset = if ipfs_enabled { + match db.load_reconciliation_offset(&repo.id, "IPFS").await { + Ok(v) => v, + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "load_reconciliation_offset(IPFS) failed, starting from head"); + None + } + } + } else { + None + }; + let pinata_offset = if pinata_enabled { + match db.load_reconciliation_offset(&repo.id, "PINATA").await { + Ok(v) => v, + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "load_reconciliation_offset(PINATA) failed, starting from head"); + None + } + } + } else { + None + }; + + // IPFS-missing set. A filter DB error skips only the IPFS gap-fill and + // lets the Pinata path still run (R1-P3), instead of dropping the repo. + // + // `*_scan_ok` records whether the missing set is a TRUTHFUL answer + // (#218 review round 8 P2). An empty set means two opposite things: "every + // object is already pinned" (the happy path, which should mark the + // continuation done) or "the filter query failed and we know nothing" + // (which must leave the stored continuation exactly where it was). Writing + // a done marker for the second case discards a resume point that a capped + // pass paid for, so the two are tracked apart. + let mut ipfs_scan_ok = ipfs_enabled; + let ipfs_missing: Vec = if ipfs_enabled { + match db.filter_ipfs_pinned_oids(&object_list).await { + Ok(already) => cap_missing( + missing_oids(&object_list, &already, ipfs_offset.as_deref()), + &repo_slug, + "IPFS", + ), + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "filter_ipfs_pinned_oids failed, IPFS gap-fill skipped this pass"); + ipfs_scan_ok = false; + Vec::new() + } + } + } else { + Vec::new() + }; + + let mut pinata_scan_ok = pinata_enabled; + let pinata_missing: Vec = if pinata_enabled { + match db.filter_pinata_pinned_oids(&object_list).await { + Ok(already) => cap_missing( + missing_oids(&object_list, &already, pinata_offset.as_deref()), + &repo_slug, + "Pinata", + ), + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "filter_pinata_pinned_oids failed, Pinata gap-fill skipped this pass"); + pinata_scan_ok = false; + Vec::new() + } + } + } else { + Vec::new() + }; + + // Whether this pass had gap-fill work to do at all, captured before the + // missing sets are moved into the pin loops below. This is NOT the + // continuation value — see `ipfs_dispatched` / `pinata_dispatched`. + let ipfs_had_work = !ipfs_missing.is_empty(); + let pinata_had_work = !pinata_missing.is_empty(); + + // The last OID each backend actually DISPATCHED — handed to + // `pin_new_objects` — or `None` if this pass dispatched nothing. + // + // #218 review round 8 P2: the continuation used to be captured here, from + // `missing.last()`, BEFORE the pin permit, both `PolicyFence` captures and + // both pin loops, and was then written unconditionally. Every stage between + // capture and dispatch can legitimately produce nothing — a fence capture + // that fails, a quarantine/visibility recheck that says skip, a + // pin-boundary re-derivation that errors — and each of those is a + // TRANSIENT failure. Advancing the continuation past OIDs that were never + // attempted rotates that whole unattempted prefix to the BACK of the next + // pass's order, behind the entire backlog. For an at-cap repo (the only + // kind the continuation exists for) the backlog never drains inside one + // cap window, so those objects are not merely retried later — they are + // starved indefinitely, which is exactly the durability hole this sweep is + // the backstop for. The offset therefore moves only for work that was + // really dispatched; a pass that dispatched nothing leaves the stored + // resume point untouched and retries the same prefix next tick. + let mut ipfs_dispatched: Option = None; + let mut pinata_dispatched: Option = None; + + // Count UNIQUE missing objects across both backends (R1-P3): an object + // absent from both must not be counted twice. + let mut gap_union: HashSet<&str> = HashSet::new(); + gap_union.extend(ipfs_missing.iter().map(|s| s.as_str())); + gap_union.extend(pinata_missing.iter().map(|s| s.as_str())); + let repo_gaps = gap_union.len(); + if repo_gaps > 0 { + total_gaps_found += repo_gaps; + crate::metrics::record_reconciliation_gaps_found(repo_gaps as u64); + } + + // Re-validate quarantine + visibility IMMEDIATELY before each backend + // pin (R1-P1) and re-derive the allowed set from the rules read at that + // moment, intersecting it with the to-pin list (R2-P1): for + // content-addressed public pins a stale allow is effectively + // irreversible, and the pin itself takes time. A path-scoped deny that + // landed after the mid-scan refilter (which only checks root listability) + // is honored here because the candidates are intersected with the set + // allowed under the fresh rules, not just root-gated. Each backend runs + // under a PolicyFence captured at ITS dispatch boundary, so a narrow that + // lands mid-batch aborts the remaining uploads (R1-P1). + // + // Acquire the same global pin permit the push path holds (R2-P2): the + // sweep's pin loops must not bypass `max_concurrent_pin_tasks`. Acquired + // only when there is actual pin work; the scan above holds no permit. + // The permit is held across the public pin loops AND the encrypted seal + // below (which also writes to IPFS) and dropped at the end of this repo's + // iteration. + // Reassign the outer `_pin_permit` (declared before this + // block so phase 2 can read it even when the public phase + // did not run) instead of shadowing with `let`. + _pin_permit = if !ipfs_missing.is_empty() || !pinata_missing.is_empty() { + let permit = pin_sem.clone().acquire_owned().await?; + Some(permit) + } else { + None + }; + let ipfs_fence = if ipfs_enabled && !ipfs_missing.is_empty() { + crate::ipfs_pin::PolicyFence::capture(db, &repo.id).await + } else { + None + }; + let pinata_fence = if pinata_enabled && !pinata_missing.is_empty() { + crate::ipfs_pin::PolicyFence::capture(db, &repo.id).await + } else { + None + }; + + let pinned_ipfs: Vec<(String, String)> = if ipfs_enabled && !ipfs_missing.is_empty() { + match ipfs_fence { + None => { + tracing::warn!(repo = %repo_slug, "IPFS policy-epoch capture failed, skipping"); + Vec::new() + } + Some(fence) => match recheck_public_pin(db, &repo.id, &repo_slug).await { + None => Vec::new(), + Some((fresh_repo, fresh_rules)) => { + let to_pin = match pin_boundary_refilter( + &disk, + &fresh_rules, + fresh_repo.is_public, + &fresh_repo.owner_did, + ipfs_missing, + Instant::now() + rederive_budget, + ) + .await + { + Some(list) => list, + None => { + tracing::warn!(repo = %repo_slug, "IPFS pin-boundary re-derivation failed, skipping"); + Vec::new() + } + }; + if to_pin.is_empty() { + Vec::new() + } else { + // Dispatch boundary (round-8 P2): from here the OIDs + // in `to_pin` really are handed to the backend, so + // the continuation may advance to the last of them. + // Recorded BEFORE the call so a pin phase that times + // out mid-batch still counts as dispatched — those + // objects were attempted, and re-attempting them + // ahead of the rest of the backlog is the starvation + // the rotation exists to avoid. It is `to_pin`'s last + // element, not the missing set's: an OID the + // pin-boundary re-derivation dropped was never + // offered to the backend. + ipfs_dispatched = to_pin.last().cloned(); + match tokio::time::timeout( + PIN_PHASE_DEADLINE, + crate::ipfs_pin::pin_new_objects( + &config.ipfs_api, + &disk, + "git", + Duration::from_secs(config.git_service_timeout_secs), + to_pin, + db, + &repo.id, + crate::ipfs_pin::PIN_BATCH_BUDGET, + Some(&fence), + ), + ) + .await + { + Ok(v) => v, + Err(_) => { + tracing::warn!(repo = %repo_slug, "IPFS pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + Vec::new() + } + } + } + } + }, + } + } else { + Vec::new() + }; + + let pinned_pinata: Vec<(String, String)> = if pinata_enabled + && !pinata_missing.is_empty() + { + match pinata_fence { + None => { + tracing::warn!(repo = %repo_slug, "Pinata policy-epoch capture failed, skipping"); + Vec::new() + } + Some(fence) => match recheck_public_pin(db, &repo.id, &repo_slug).await { + None => Vec::new(), + Some((fresh_repo, fresh_rules)) => { + // Own budget (R2-P1): the IPFS arm above may have + // consumed the whole shared deadline, and a reused + // spent deadline here would silently skip Pinata every + // pass for exactly the large repos this sweep exists + // for. + let to_pin = match pin_boundary_refilter( + &disk, + &fresh_rules, + fresh_repo.is_public, + &fresh_repo.owner_did, + pinata_missing, + Instant::now() + rederive_budget, + ) + .await + { + Some(list) => list, + None => { + tracing::warn!(repo = %repo_slug, "Pinata pin-boundary re-derivation failed, skipping"); + Vec::new() + } + }; + if to_pin.is_empty() { + Vec::new() + } else { + // Dispatch boundary (round-8 P2); see the IPFS arm + // above for why this is recorded here rather than + // from the missing set before the fence. + pinata_dispatched = to_pin.last().cloned(); + match tokio::time::timeout( + PIN_PHASE_DEADLINE, + crate::pinata::pin_new_objects( + http_client, + &config.pinata_upload_url, + &config.pinata_jwt, + &disk, + "git", + Duration::from_secs(config.git_service_timeout_secs), + to_pin, + db, + &repo.id, + crate::ipfs_pin::PIN_BATCH_BUDGET, + Some(&fence), + ), + ) + .await + { + Ok(v) => v, + Err(_) => { + tracing::warn!(repo = %repo_slug, "Pinata pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + Vec::new() + } + } + } + } + }, + } + } else { + Vec::new() + }; + + // `pin_new_objects` returns only objects whose DB record was written + // (R1-P3), so a backend that uploaded bytes but failed to persist is + // not counted as "filled". Count UNIQUE objects across both backends + // (R2-P3): `gaps_found` is the union of missing OIDs, so an object + // pinned to BOTH backends must not count twice against that union. + let mut filled_union: HashSet<&String> = HashSet::new(); + filled_union.extend(pinned_ipfs.iter().map(|(sha, _)| sha)); + filled_union.extend(pinned_pinata.iter().map(|(sha, _)| sha)); + let repo_filled = filled_union.len(); + if repo_filled > 0 { + total_gaps_filled += repo_filled; + crate::metrics::record_reconciliation_gaps_filled(repo_filled as u64); + + tracing::info!( + repo = %repo_slug, + ipfs = pinned_ipfs.len(), + pinata = pinned_pinata.len(), + total = repo_filled, + "reconciliation sweep filled public-object gaps" + ); + } + + // Persist the per-(repo, backend) continuation offset (#218 review + // P2). The offset is the last DISPATCHED OID per backend — for a + // non-truncated pass this is the OID at the tail of the missing + // set, for a truncated pass it is the OID at the cap edge. The + // next pass's `missing_oids` rotates the sorted set so the first + // OID is strictly greater than this value, and the previously + // attempted tail is retried at the end of the next pass — so a + // persistent early failure does not monopolise the cap window. + // + // Three outcomes, and the round-8 P2 fix is that they are three + // rather than two (see `ipfs_dispatched` above for the starvation + // this prevents): + // * work dispatched -> advance to the last dispatched OID. + // * nothing missing, and the missing-set query SUCCEEDED + // -> `None`, which marks the row done and + // starts the next pass at the head. + // * nothing dispatched from a non-empty missing set, or a failed + // missing-set query + // -> write NOTHING. The stored resume + // point is the only record of how far a + // capped pass got; a transient fence, + // recheck or re-derivation failure must + // not be allowed to erase or advance it. + // + // A DB error on the write is logged but does NOT abort the pass: a + // missed offset write means the next pass starts at the head + // (the worst case is one pass at the old sort order). + // + // `next_offset_write` returns a `ProgressState` directly + // (the documented shape), and the write site converts to + // the wire form via `to_wire`. One encoding — the enum + // is no longer a parallel implementation of the same + // logic. P2 (reviewer round 9): the previous code held + // `Option>` in the closure and the + // `ProgressState` enum on the side, with the two only + // cross-checked in a test that never called the closure. + // Now there is one mapping. + // + // The three states: + // - `Idle`: no work was attempted this pass (fence + // capture failed, refilter returned `None`, dispatch + // produced an empty `to_pin`). The cursor is + // preserved — the unattempted prefix retries at the + // head of the next pass. + // - `Advanced { last_dispatched }`: a subset of the cap + // was dispatched. The next pass rotates past + // `last_dispatched`, retrying everything beyond. + // - `Drained`: the missing set was empty. The cursor is + // cleared (a future pass sees a fresh start). + // + // The two backends' cursors are independent: a drained + // IPFS missing set clears the IPFS offset but does NOT + // touch the Pinata offset, and vice versa. The write + // site persists each backend's state without sharing. + + if ipfs_enabled { + let next_wire = + next_offset_write(ipfs_scan_ok, ipfs_had_work, ipfs_dispatched).to_wire(); + if let Some(next) = next_wire { + if let Err(e) = db + .save_reconciliation_offset(&repo.id, "IPFS", next.as_deref()) + .await + { + tracing::warn!(repo = %repo_slug, err = %e, "save_reconciliation_offset(IPFS) failed, next pass will start from head"); + } + } else { + tracing::debug!(repo = %repo_slug, "IPFS dispatched nothing this pass, continuation offset left unchanged"); + } + } + if pinata_enabled { + let next_wire = + next_offset_write(pinata_scan_ok, pinata_had_work, pinata_dispatched).to_wire(); + if let Some(next) = next_wire { + if let Err(e) = db + .save_reconciliation_offset(&repo.id, "PINATA", next.as_deref()) + .await + { + tracing::warn!(repo = %repo_slug, err = %e, "save_reconciliation_offset(PINATA) failed, next pass will start from head"); + } + } else { + tracing::debug!(repo = %repo_slug, "Pinata dispatched nothing this pass, continuation offset left unchanged"); + } + } + } // end of `if has_public_work { ... }` (round 10 P1) + + // ── Phase 2: Encrypted recovery-copy resealing (withheld blobs) ── + + // Fence the encrypted path from the point the recipients are derived: + // the withheld-blob walk is long, and `encrypt_and_pin` re-checks the + // epoch per blob, so a visibility rule moving mid-walk aborts the seal + // loop before a stale recipient set is pinned (R1-P1). Captured BEFORE + // the rules recheck below, mirroring the public path (R2-P1): if a rule + // change landed between a recheck-first ordering's rule read and this + // capture, the change would be baked into the recipient set while the + // epoch captured after it already reflected the move — `is_current` + // would then report current for the whole seal loop and the fence would + // never fire for that narrow. + let enc_fence = match crate::ipfs_pin::PolicyFence::capture(db, &repo.id).await { + Some(f) => f, + None => { + tracing::warn!(repo = %repo_slug, "policy-epoch capture failed, skipping encrypted pin"); + continue; + } + }; + // Recheck quarantine AND root visibility before encrypted pinning, using + // FRESH repo identity (R1-P2): the batch snapshot may predate a narrow. + let (fresh_repo2, fresh_rules2) = match recheck_public_pin(db, &repo.id, &repo_slug).await { + Some(v) => v, + None => continue, + }; + + let has_path_scoped = crate::git::visibility_pack::has_path_scoped_rule(&fresh_rules2); + if has_path_scoped && ipfs_enabled { + let p = disk.clone(); + let owner = fresh_repo2.owner_did.clone(); + let r = fresh_rules2.clone(); + let is_public_2 = fresh_repo2.is_public; + let recipients = tokio::time::timeout( + REPO_SCAN_DEADLINE, + tokio::task::spawn_blocking(move || { + crate::git::visibility_pack::withheld_blob_recipients_bounded( + &p, + "git", + REPO_SCAN_DEADLINE, + &r, + is_public_2, + &owner, + ) + }), + ) + .await; + + let rec = match recipients { + Ok(Ok(Ok(rec))) => rec, + Ok(Ok(Err(e))) => { + tracing::warn!( + repo = %repo_slug, err = %e, + "withheld_blob_recipients failed, skipping encrypted pin" + ); + continue; + } + Ok(Err(e)) => { + tracing::warn!( + repo = %repo_slug, err = %e, + "withheld_blob_recipients task panicked, skipping encrypted pin" + ); + continue; + } + Err(_) => { + tracing::warn!( + repo = %repo_slug, + "encrypted recovery deadline exceeded, skipping" + ); + continue; + } + }; + + if !rec.is_empty() { + // The encrypted seal writes to IPFS too, so it runs under the + // same global pin permit as the public loops (R2-P2). Reuse the + // permit `_pin_permit` already holds for this repo when the + // public phase had gaps; only acquire a fresh one when it did + // not. One permit per repo, never two (R2-P1): with + // `max_concurrent_pin_tasks = 1` a second acquire here would + // wait on the very permit this iteration holds and deadlock the + // sweep past its guard timeout. + let _enc_permit = match &_pin_permit { + Some(_) => None, + None => Some(pin_sem.clone().acquire_owned().await?), + }; + // Bound the seal+pin work (R1-P2): an unavailable backend must + // not hold the sweep past the pin-phase budget. + let sealed = tokio::time::timeout( + PIN_PHASE_DEADLINE, + crate::encrypted_pin::encrypt_and_pin( + &config.ipfs_api, + &disk, + db, + &repo.id, + node_seed, + "git", + crate::ipfs_pin::PIN_BATCH_BUDGET, + &rec, + Some(&enc_fence), + ), + ) + .await; + + let sealed: Vec<(String, String)> = match sealed { + Ok(v) => v, + Err(_) => { + tracing::warn!( + repo = %repo_slug, + "encrypted pin phase timed out after {:?}", + PIN_PHASE_DEADLINE + ); + Vec::new() + } + }; + + // Anchor only when something was newly sealed this pass. + // This avoids unbounded Irys writes on a timer — repos + // with no withheld changes do not re-anchor the manifest. + // + // Contract: anchoring is one-shot per seal. `plan_seal` returns + // `SkipUnchanged` on every subsequent pass once the recipients + // tag matches, so `sealed` stays empty and this block never + // runs again. A transient Irys outage at the moment of a fresh + // seal therefore LOSES that anchor permanently — the next pass + // has no delta to anchor and no retry fires. This is + // intentionally best-effort: re-anchoring an unchanged manifest + // would burn Irys writes for no recovery benefit, and durable + // retry of a failed seal would need a separate outbox that + // survives across the seal-skip path. Operators who need a + // guaranteed anchor after a transient outage should re-add a + // withheld change (which forces a new seal and re-runs this + // block) or anchor via a separate out-of-band process. + if !sealed.is_empty() && !config.irys_url.is_empty() { + // Bind the manifest to the FRESH repo identity re-fetched at + // the pin boundary (`fresh_repo2`), not the batch snapshot: + // a renamed/ownership-changed repo must not anchor encrypted + // recovery copies under a stale owner (R1-P2). + let owner_short = crate::db::normalize_owner_key(&fresh_repo2.owner_did); + let slug = format!("{}/{}", owner_short, fresh_repo2.name); + let ts = chrono::Utc::now().to_rfc3339(); + let node_did_str = node_did.to_string(); + + let manifest = crate::arweave::EncryptedManifest { + repo: &slug, + owner_did: &fresh_repo2.owner_did, + node_did: &node_did_str, + timestamp: &ts, + blobs: &sealed, + }; + if let Err(e) = crate::arweave::anchor_encrypted_manifest( + http_client, + &config.irys_url, + &manifest, + ) + .await + { + tracing::warn!( + repo = %slug, + err = %e, + "{}", + ENCRYPTED_MANIFEST_ANCHOR_FAILED_MSG + ); + } + } + } + } + } + + // Persist the cursor only when the WHOLE batch completed. If shutdown + // interrupted us, leave the persisted cursor at the previous batch's end so + // the next run re-walks the unprocessed tail (R2-P1, R1-P3). + if batch_completed { + // A terminal page (no lookahead row) means the whole key space is + // covered: clear the cursor now so the next tick starts a fresh cycle + // instead of burning one pass on an empty batch. The lookahead is what + // distinguishes "full because more remain" from "full because the key + // space ends on an exact page boundary" (R1-P2). + if !has_more { + *cursor = None; + if let Err(e) = db.set_node_state(CURSOR_KEY, None).await { + tracing::warn!(err = %e, "failed to clear reconciliation sweep cursor on final page"); + } + } else if let Err(e) = db.set_node_state(CURSOR_KEY, Some(&batch_last)).await { + tracing::warn!(err = %e, "failed to persist reconciliation sweep cursor"); + } + } + + Ok((repos_scanned, total_gaps_found, total_gaps_filled)) +} + +#[cfg(test)] +mod tests { + use super::{next_offset_write, ProgressState}; + use tokio::sync::watch; + + /// Build a minimal Config with both IPFS and Pinata fields empty so the + /// spawn() gate fires and the function returns without touching the DB. + fn empty_pin_config() -> std::sync::Arc { + // Config derives clap::Parser; supply only argv[0] (the program name) + // so all fields get their defaults (ipfs_api = "", pinata_jwt = ""). + let cfg = ::parse_from(["gitlawb-node-test"]); + std::sync::Arc::new(cfg) + } + + /// Build a config with IPFS API set so the gate fires the other way. + fn ipfs_config() -> std::sync::Arc { + let cfg = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + "http://127.0.0.1:5001", + ]); + std::sync::Arc::new(cfg) + } + + /// #218 review round 9 (guidance #4): the wire-form test + /// now drives `next_offset_write` directly with the same + /// `(scan_ok, had_work, dispatched)` triples the + /// cursor-write site uses, and asserts the returned + /// `ProgressState` (and its `to_wire()`) is what the + /// cursor-write site will land. P2 (reviewer round 9): the + /// previous test never called the closure, so it pinned + /// itself to its own arm-by-arm reproduction of the enum. + /// Now there is one encoding and the test calls the function + /// under test. + #[test] + fn next_offset_write_decision_table() { + // scan_ok=false short-circuits to Idle regardless of the + // other inputs — fence capture failed, the cursor must + // be preserved. + assert_eq!( + next_offset_write(false, true, Some("Z".to_string())), + ProgressState::Idle, + "scan_ok=false must produce Idle (fence capture failed, cursor preserved)" + ); + assert_eq!( + next_offset_write(false, false, None), + ProgressState::Idle, + "scan_ok=false must produce Idle even with no work" + ); + // dispatched.is_some() is Advanced, regardless of had_work. + assert_eq!( + next_offset_write(true, true, Some("X".to_string())), + ProgressState::Advanced { + last_dispatched: "X".to_string() + }, + "dispatched.is_some() must produce Advanced (the next pass rotates past last)" + ); + assert_eq!( + next_offset_write(true, false, Some("Y".to_string())), + ProgressState::Advanced { + last_dispatched: "Y".to_string() + }, + "dispatched.is_some() wins over !had_work" + ); + // No dispatch, no work → Drained (cursor cleared). + assert_eq!( + next_offset_write(true, false, None), + ProgressState::Drained, + "scan_ok=true with no work and no dispatch must produce Drained (cursor cleared)" + ); + // No dispatch, had_work → Idle (cursor preserved). + assert_eq!( + next_offset_write(true, true, None), + ProgressState::Idle, + "had_work but no dispatch must produce Idle (cursor preserved, retry at head)" + ); + } + + /// The to_wire mapping for the three states, kept as a + /// separate test so a future change to either side of the + /// pair (enum variant vs. wire form) is caught. P2 (reviewer + /// round 9): with the closure now returning `ProgressState` + /// directly and the wire form derived via `to_wire`, this + /// is a pure mapping test, not a guard on the closure + /// logic. + #[test] + fn progress_state_to_wire_mapping() { + assert_eq!( + ProgressState::Idle.to_wire(), + None, + "Idle must produce None (caller preserves the previous offset)" + ); + assert_eq!( + ProgressState::Advanced { + last_dispatched: "Z".to_string() + } + .to_wire(), + Some(Some("Z".to_string())), + "Advanced must produce Some(Some(last_dispatched)) (caller writes the offset)" + ); + assert_eq!( + ProgressState::Drained.to_wire(), + Some(None), + "Drained must produce Some(None) (caller clears the offset)" + ); + } + + #[test] + fn should_spawn_false_when_both_empty() { + let cfg = empty_pin_config(); + assert!(!super::should_spawn(&cfg)); + } + + #[test] + fn should_spawn_true_when_ipfs_set() { + let cfg = ipfs_config(); + assert!(super::should_spawn(&cfg)); + } + + #[test] + fn should_spawn_true_when_pinata_set() { + let cfg = ::parse_from([ + "gitlawb-node-test", + "--pinata-jwt", + "test-jwt", + ]); + assert!(super::should_spawn(&cfg)); + } + + #[test] + fn should_spawn_false_when_sweep_disabled() { + let cfg = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + "http://127.0.0.1:5001", + "--reconciliation-sweep", + "false", + ]); + assert!(!super::should_spawn(&cfg)); + } + + /// spawn() must return `false` (and not spawn a task, touch the DB, or + /// panic) when neither IPFS nor Pinata is configured. This proves the gate + /// branch at the top of spawn() is actually reachable and observable. + #[tokio::test] + async fn test_spawn_gate_skips_when_no_pin_backends_configured() { + let config = empty_pin_config(); + assert!(config.ipfs_api.is_empty(), "ipfs_api should be empty"); + assert!(config.pinata_jwt.is_empty(), "pinata_jwt should be empty"); + + // Use a dummy Db built from a disconnected pool; spawn() must not + // reach any code that would touch it. + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect_lazy("postgresql://localhost/gitlawb_test_nonexistent") + .unwrap(); + let db = std::sync::Arc::new(crate::db::Db::for_testing(pool)); + let http = std::sync::Arc::new(reqwest::Client::new()); + let kp = std::sync::Arc::new(gitlawb_core::identity::Keypair::generate()); + let node_did = kp.did(); + let (_tx, rx) = watch::channel(false); + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + // spawn() should return false synchronously (no tokio::spawn) and never + // await the DB. The test completes without timeout == gate is live. + assert!( + !super::spawn(db, config, http, kp, node_did, pin_sem, rx), + "gated spawn must report it did not start a worker" + ); + } + + /// spawn() returns true and starts a worker when a backend is configured; + /// the caller uses that to gate its own "worker started" logging. + #[tokio::test] + async fn test_spawn_returns_true_when_ipfs_configured() { + let config = ipfs_config(); + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect_lazy("postgresql://localhost/gitlawb_test_nonexistent") + .unwrap(); + let db = std::sync::Arc::new(crate::db::Db::for_testing(pool)); + let http = std::sync::Arc::new(reqwest::Client::new()); + let kp = std::sync::Arc::new(gitlawb_core::identity::Keypair::generate()); + let node_did = kp.did(); + let (_tx, rx) = watch::channel(false); + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + assert!( + super::spawn(db, config, http, kp, node_did, pin_sem, rx), + "configured spawn must report it started a worker" + ); + } + + /// The missing set must be deterministic, which is what makes the sweep's + /// per-repo pin order reproducible across passes. The cap is applied by + /// `cap_missing` at the call site, so `missing_oids` stays uncapped. + /// `start_after = None` preserves the pre-P2 head-first order. + #[test] + fn missing_oids_is_deterministic() { + let all = vec![ + "c".to_string(), + "a".to_string(), + "b".to_string(), + "d".to_string(), + ]; + let done = vec!["b".to_string()]; + + let first = super::missing_oids(&all, &done, None); + let second = super::missing_oids(&all, &done, None); + assert_eq!(first, second, "missing set must be deterministic"); + assert_eq!( + first, + vec!["a".to_string(), "c".to_string(), "d".to_string()] + ); + } + + /// Per-(repo, backend) continuation offset (#218 review P2). When + /// `start_after` is the last OID the previous pass attempted, the + /// next pass must rotate the sorted missing set so the first OID + /// is strictly greater than that value, and the previously-attempted + /// tail is retried at the end of the next pass. Without the + /// rotation, a persistently failing early OID keeps landing at the + /// start of the sort and dominates the cap window every hourly + /// tick; with the rotation, the cap window advances fairly across + /// passes and the healthy gap past the cap gets attempted. + #[test] + fn missing_oids_rotates_past_start_after() { + let all: Vec = (0..6).map(|i| format!("oid_{i:02}")).collect(); + let done: Vec = Vec::new(); + + // No offset: head-first order, the pre-P2 contract. + let head = super::missing_oids(&all, &done, None); + assert_eq!( + head, + vec!["oid_00", "oid_01", "oid_02", "oid_03", "oid_04", "oid_05"], + "no offset preserves the deterministic head-first order" + ); + + // Offset = "oid_02": the next pass starts strictly past oid_02, + // and the tail rotates to the end so previously-attempted OIDs + // are retried last (not first). + let rotated = super::missing_oids(&all, &done, Some("oid_02")); + assert_eq!( + rotated, + vec!["oid_03", "oid_04", "oid_05", "oid_00", "oid_01", "oid_02"], + "offset = oid_02 must rotate the set so oid_03..oid_05 lead and oid_00..oid_02 trail" + ); + + // Offset = "" (no OID has been attempted yet — the first ever + // pass on this pair): the rotation is a no-op, same as None. + let empty_offset = super::missing_oids(&all, &done, Some("")); + assert_eq!( + empty_offset, head, + "an empty-string offset reads as 'nothing attempted yet', no rotation" + ); + + // Offset past the end: degenerate — the rotation would lose + // data, so the helper returns sorted order as-is rather than + // an empty list. + let past_end = super::missing_oids(&all, &done, Some("oid_zz")); + assert_eq!( + past_end, head, + "an offset past the end of the missing set must not lose data" + ); + } + + /// Constant smoke-check kept as a compile-time tripwire. + #[test] + fn sweep_interval_constant_is_nonzero() { + assert_ne!(super::SWEEP_INTERVAL_SECS, 0); + } + + /// #218 P2 R3 (anchor log contract): the encrypted-manifest anchor is + /// one-shot per seal. `plan_seal` returns `SkipUnchanged` on every + /// subsequent pass once the recipients tag matches, so `sealed` stays + /// empty and the anchor block never runs again. A transient Irys + /// outage at the moment of a fresh seal therefore LOSES that anchor + /// permanently — the next pass has no delta to anchor. The log MUST + /// not promise a retry, because none will fire. This test pins the + /// log content via the `ENCRYPTED_MANIFEST_ANCHOR_FAILED_MSG` constant + /// so a future "helpful" revert to "will retry next pass" is caught at + /// `cargo test` time. + #[test] + fn encrypted_manifest_anchor_log_does_not_promise_retry() { + assert!( + !super::ENCRYPTED_MANIFEST_ANCHOR_FAILED_MSG.contains("will retry"), + "the encrypted-manifest anchor log must not promise a retry; \ + plan_seal returns SkipUnchanged on later passes, so a failed \ + anchor after a successful seal is permanent. See the comment \ + above the anchor block in run_pass for the one-shot contract. \ + Got: {:?}", + super::ENCRYPTED_MANIFEST_ANCHOR_FAILED_MSG + ); + } + + // ── run_pass integration tests ──────────────────────────────────────── + + /// Minimal git repo builder (mirrors push_delta's test helper). + struct Repo { + _td: tempfile::TempDir, + path: std::path::PathBuf, + } + + impl Repo { + fn new() -> Self { + let td = tempfile::TempDir::new().unwrap(); + let path = td.path().to_path_buf(); + let r = Repo { _td: td, path }; + r.git(&["init", "-q", "-b", "main"]); + r.git(&["config", "user.email", "t@t"]); + r.git(&["config", "user.name", "t"]); + r + } + + fn git(&self, args: &[&str]) -> String { + let out = std::process::Command::new("git") + .args(args) + .current_dir(&self.path) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + fn commit_file(&self, name: &str, body: &str) -> String { + std::fs::write(self.path.join(name), body).unwrap(); + self.git(&["add", name]); + self.git(&["commit", "-qm", &format!("add {name}")]); + self.git(&["rev-parse", "HEAD"]) + } + } + + fn seed_repo(owner: &str, name: &str, disk_path: &str) -> crate::db::RepoRecord { + let now = chrono::Utc::now(); + crate::db::RepoRecord { + id: uuid::Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: disk_path.to_string(), + forked_from: None, + machine_id: None, + } + } + + /// The sweep must repair an IPFS durability gap end to end: a public repo + /// whose objects were never pinned gets every reachable blob pinned and + /// recorded (R2-P2 "test the behavior the PR exists to change"). + #[sqlx::test] + async fn sweep_fills_ipfs_gap_and_persists_cursor(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("a.txt", "public blob\n"); + + let rec = seed_repo( + "did:key:zSweepOwner", + "sweep-repo", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + // Mock IPFS: every /api/v0/add returns a fixed CID. mockito's unified + // matcher compares the full "path?query" target, so the query string + // pin_git_object appends must be part of the mock path. + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .expect_at_least(1) + .with_status(200) + .with_body(r#"{"Hash":"QmSweepMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + assert_eq!(scanned, 1, "one repo scanned"); + assert!(gaps >= 1, "at least one missing blob found"); + assert_eq!( + filled, gaps, + "every found gap is filled in a clean mock-backed run" + ); + _m.assert_async().await; + + // The recorded pin makes the blob "already done" on the next pass. + let blob = repo_on_disk.git(&["rev-parse", "HEAD:a.txt"]); + assert!( + db.has_ipfs_cid(&blob).await.unwrap(), + "pinned CID must be recorded and classified as IPFS-pinned" + ); + + // Cursor cleared on a short final page (R2-P1): with one repo the batch + // is the whole key space, so persisting `batch_last` would just force an + // empty tail pass next tick that scans nothing and then clears. Clearing + // now means the next pass starts a fresh cycle immediately. + let persisted = db.get_node_state(super::CURSOR_KEY).await.unwrap(); + assert!( + persisted.is_none(), + "cursor must be cleared after a fully-completed short final page" + ); + assert!( + cursor.is_none(), + "in-memory cursor follows the persisted one" + ); + + // Second pass: no gaps remain. + let (_, gaps2, filled2) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + assert_eq!(gaps2, 0, "second pass finds no remaining gaps"); + assert_eq!(filled2, 0); + } + + /// Mirror rows (slash-form id, hardcoded is_public=true, no replicated + /// visibility rules) must be skipped entirely: sweeping one would + /// irreversibly publish content the canonical gate never admitted (R2-P1). + #[sqlx::test] + async fn sweep_skips_mirror_rows(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("secret.txt", "must not be published\n"); + + // A mirror row pointing at a real, public-on-disk repo. + db.upsert_mirror_repo( + "zMirrorOwner", + "mirror-repo", + &repo_on_disk.path.display().to_string(), + None, + false, + ) + .await + .unwrap(); + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + "http://127.0.0.1:1", // unreachable; must never be hit + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + assert_eq!(scanned, 0, "mirror row is not scanned"); + assert_eq!(gaps, 0, "mirror row produces no gaps"); + assert_eq!(filled, 0, "mirror row is never pinned"); + + // Nothing was recorded for the mirror's content. + assert!( + db.list_pinned_cids().await.unwrap().is_empty(), + "no pinned_cids rows may exist after a mirror-only pass" + ); + } + + /// A repo flagged `quarantined` after admission must produce zero mock IPFS + /// traffic. The SQL dedup listing (`list_all_repos_deduped_stable`) filters + /// `quarantined = FALSE` at the database, so the row never reaches the + /// per-repo loop. The per-row `is_repo_quarantined` re-check is a + /// race-only defense: the SQL filter is the primary gate. The strong + /// assertion is on the side effects of the sweep pass, not on the + /// counter, because a SQL filter that drops a row at the source makes the + /// per-row check moot. (Reviewer-1 P2.) + #[sqlx::test] + async fn sweep_skips_quarantined_repos_before_scan(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("public.txt", "would be public if scanned\n"); + + let rec = seed_repo( + "did:key:zQuarOwner", + "quar-repo", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + // Flip quarantine AFTER admission (the realistic flow). + let affected = db.set_repo_quarantine(&rec.id, true).await.unwrap(); + assert_eq!(affected, 1, "the new repo row must take the quarantine"); + + // SQL-filter assertion: the dedup listing does not return quarantined + // rows. If this changes, the per-row check below catches the race, + // but a SQL filter regression would silently start scanning them. + let dedup_rows = db.list_all_repos_deduped_stable(None, 100).await.unwrap(); + assert!( + dedup_rows.iter().all(|r| r.id != rec.id), + "quarantined repo is excluded from the dedup listing at SQL" + ); + + // Mock IPFS: any POST is a gate-ordering bug. expect(0) makes the + // mock fail if hit. + let mut server = mockito::Server::new_async().await; + let m = server + .mock( + "POST", + mockito::Matcher::Regex(r"^/api/v0/add.*$".to_string()), + ) + .expect(0) + .with_status(200) + .with_body(r#"{"Hash":"QmMustNotBeCalled"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + assert_eq!(scanned, 0, "SQL filter drops the quarantined row"); + assert_eq!(gaps, 0); + assert_eq!(filled, 0, "no pin work attempted"); + assert!( + db.list_pinned_cids().await.unwrap().is_empty(), + "no pinned_cids rows from a quarantined pass" + ); + m.assert_async().await; + } + + /// A non-public repo (`is_public = false`, no visibility rules) must also + /// produce zero mock IPFS traffic. The dedup listing returns it (it is not + /// quarantined), but the per-repo `listable_at_root` gate aborts before + /// the expensive scan. (Reviewer-1 P2.) + #[sqlx::test] + async fn sweep_skips_private_repos_before_scan(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("private.txt", "never published\n"); + + // Build a private repo row (seed_repo hardcodes is_public=true). + let mut rec = seed_repo( + "did:key:zPrivateOwner", + "priv-repo", + &repo_on_disk.path.display().to_string(), + ); + rec.is_public = false; + db.create_repo(&rec).await.unwrap(); + + // No visibility rules: a private repo with no allow rules is unlistable. + assert!(db.list_visibility_rules(&rec.id).await.unwrap().is_empty()); + + // The dedup listing DOES return private (non-quarantined) rows, so + // the per-repo gate is the actual filter under test. + let dedup_rows = db.list_all_repos_deduped_stable(None, 100).await.unwrap(); + assert!( + dedup_rows.iter().any(|r| r.id == rec.id), + "private repo is in the dedup listing (filter is per-repo)" + ); + + let mut server = mockito::Server::new_async().await; + let m = server + .mock( + "POST", + mockito::Matcher::Regex(r"^/api/v0/add.*$".to_string()), + ) + .expect(0) + .with_status(200) + .with_body(r#"{"Hash":"QmMustNotBeCalled"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + // The private row reaches the per-repo loop (the SQL filter is not + // the gate here), the counter increments, then `listable_at_root` + // returns false and the work aborts before the scan. Strong assertion + // is on side effects. + assert!(scanned >= 1, "the private row is in the dedup listing"); + assert_eq!(gaps, 0, "no gaps on a private-skip"); + assert_eq!(filled, 0, "no pin work attempted"); + assert!( + db.list_pinned_cids().await.unwrap().is_empty(), + "no pinned_cids rows from a private-only pass" + ); + m.assert_async().await; + } + + /// A public repo with a path-scoped deny must NOT have the withheld blob + /// pinned in cleartext on a public backend (R2-P1 "must not pin"): the root + /// stays listable, so the mid-scan refilter AND the pin-boundary re-derivation + /// are the only layers between a narrowed subtree and irreversible public + /// publication. + #[sqlx::test] + async fn sweep_never_pins_withheld_blob_in_cleartext(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("public.txt", "public content\n"); + // git needs the parent directory to exist before `git add` of a nested + // path; create it, then stage via `git add -A` through the helper. + std::fs::create_dir_all(repo_on_disk.path.join("secret")).unwrap(); + repo_on_disk.commit_file("secret/secret.txt", "must not go public\n"); + + // Blob oids, not commit oids: commits are structural and legitimately + // pinned publicly, so the must-not-pin assertion must key on the blob + // whose content is denied at `secret/secret.txt`. + let public_blob = repo_on_disk.git(&["rev-parse", "HEAD:public.txt"]); + let secret_blob = repo_on_disk.git(&["rev-parse", "HEAD:secret/secret.txt"]); + + let rec = seed_repo( + "did:key:zSweepWithheldOwner", + "sweep-withheld", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + // Path-scoped deny with no readers: anonymous is allowed the repo root + // (public) but denied every blob under /secret/**, whose content must + // never reach the public pin backends. + db.set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + &rec.owner_did, + ) + .await + .unwrap(); + + // Mock IPFS: every /api/v0/add returns a fixed CID (matches pin_git_object's + // URL, which appends the cid-version/raw-leaves/pin query). + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .with_status(200) + .with_body(r#"{"Hash":"QmWithheldMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + assert_eq!(scanned, 1, "one repo scanned"); + + assert!(gaps >= 1, "public blob is a real gap"); + let _ = filled; // encrypted/sealed copies do not count toward `filled` + + // The public blob is pinned and recorded as IPFS-pinned. + assert!( + db.has_ipfs_cid(&public_blob).await.unwrap(), + "public blob must be pinned in cleartext" + ); + + // The withheld blob must NOT appear with an IPFS CID -- never pinned in + // cleartext. (`has_ipfs_cid` only matches rows with a non-NULL cid, so an + // encrypted copy recorded under `encrypted_blobs` cannot satisfy it.) + assert!( + !db.has_ipfs_cid(&secret_blob).await.unwrap(), + "withheld blob must never be pinned to a public backend in cleartext" + ); + } + + /// #218 round 10 P1 (`has_public_work`): a path-scoped repo whose only + /// reachable object is a direct blob ref yields an EMPTY public list — the + /// anonymous classifier denies the empty-path catch-all entry — while + /// `withheld_blob_recipients_bounded` still assigns that blob to the owner + /// recovery set. The pre-fix early `continue` on an empty list skipped the + /// whole repo iteration, so a lost/failed encrypted copy was never + /// repaired. This pins both directions: no public work is attempted + /// (gaps/filled are 0, exactly one POST lands — the seal envelope, never + /// a cleartext upload) AND the encrypted recovery copy is sealed and + /// recorded. + #[sqlx::test] + async fn sweep_seals_withheld_blob_when_public_list_is_empty(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + // A repo with NO commits: the only object is a loose blob named by a + // non-branch ref. `git rev-list --all` silently skips it, so the + // path-annotated phase finds no commits while the catch-all phases + // surface the blob with an empty path on both the allow side (denied) + // and the withheld side (withheld to the owner). A branch ref cannot + // express this shape — git refuses non-commit objects under + // refs/heads — so the ref lives outside refs/heads. + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().to_path_buf(); + let run_git = |args: &[&str]| { + let out = std::process::Command::new("git") + .args(args) + .current_dir(&repo_path) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + run_git(&["init", "-q", "-b", "main"]); + run_git(&["config", "user.email", "t@t"]); + run_git(&["config", "user.name", "t"]); + let blob = { + use std::io::Write; + let mut child = std::process::Command::new("git") + .args(["hash-object", "-w", "--stdin"]) + .current_dir(&repo_path) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .as_mut() + .unwrap() + .write_all(b"direct secret\n") + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + run_git(&["update-ref", "refs/direct/blob", &blob]); + + // The owner must be a real resolvable did:key: `plan_seal` fail-closes + // on any unresolvable recipient, and the owner is always in the + // recipient set, so a fixture-string owner would SkipUnresolvable and + // the seal under test would never run. + let owner_did = gitlawb_core::identity::Keypair::generate() + .did() + .to_string(); + let rec = seed_repo( + &owner_did, + "sweep-empty-public", + &repo_path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + // Path-scoped rule: required twice over. It makes the repo a + // path-scoped repo (the phase-2 `has_path_scoped_rule` gate), and it + // documents the deny the empty-path entries are additionally subject + // to on the allow side. + db.set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + &owner_did, + ) + .await + .unwrap(); + + // Exactly one POST may land: the encrypted seal envelope. Any + // cleartext public upload would be a second hit and fail the mock. + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .expect(1) + .with_status(200) + .with_body(r#"{"Hash":"QmEmptyPublicMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + assert_eq!(scanned, 1, "the direct-blob repo reaches the per-repo loop"); + assert_eq!( + gaps, 0, + "an empty public list is not a gap: nothing was offered to a backend" + ); + assert_eq!(filled, 0, "no public pin work was attempted"); + assert!( + !db.has_ipfs_cid(&blob).await.unwrap(), + "the direct blob must never be pinned in cleartext" + ); + assert!( + db.encrypted_blob_cid(&rec.id, &blob) + .await + .unwrap() + .is_some(), + "encrypted recovery must run despite the empty public list" + ); + m.assert_async().await; + } + + /// The final-page proxy must be the lookahead, not `batch.len() < page` + /// (R1-P2): a key space ending on an exact page boundary looks "full" yet + /// has no following row, so the cursor must be CLEARED, not persisted to a + /// nonexistent next page (which would wedge the sweep into empty tail passes + /// every tick). REPOS_PER_PASS repos and nothing more must behave exactly + /// like one repo. + #[sqlx::test] + async fn sweep_clears_cursor_on_exact_page_boundary(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + // Exactly one full page of repos, each with a missing disk path (hard + // skip, never scanned, so no pinning side effects). + let n = super::REPOS_PER_PASS; + for i in 0..n { + let rec = seed_repo( + "did:key:zExactPageOwner", + &format!("exact-repo-{i:04}"), + &format!("/nonexistent/disk/path-{i:04}"), + ); + db.create_repo(&rec).await.unwrap(); + } + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + "http://127.0.0.1:1", // unreachable; must never be hit + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + assert_eq!(scanned, 0, "missing-disk rows are hard skips, not scans"); + assert_eq!(gaps, 0); + assert_eq!(filled, 0); + + let persisted = db.get_node_state(super::CURSOR_KEY).await.unwrap(); + assert!( + persisted.is_none(), + "an exact-page terminal batch must clear the cursor, not persist it \ + to a nonexistent next page (would wedge every subsequent tick)" + ); + assert!( + cursor.is_none(), + "in-memory cursor follows the persisted one" + ); + } + + /// R2-P1 regression: with `max_concurrent_pin_tasks = 1` (a semaphore of + /// one permit) a repo that has BOTH public gaps AND encrypted seal work must + /// still complete. The sweep holds one permit for the whole repo iteration + /// and must reuse it for the seal phase; acquiring a SECOND permit for the + /// same repo would wait on the very permit this iteration already holds, + /// deadlocking the pass past its guard timeout. The run is wrapped in a + /// timeout so a regression fails the test instead of hanging it. + #[sqlx::test] + async fn run_pass_reuses_the_pin_permit_for_the_seal_at_pool_size_one(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("public.txt", "public content\n"); + std::fs::create_dir_all(repo_on_disk.path.join("secret")).unwrap(); + repo_on_disk.commit_file("secret/secret.txt", "must not go public\n"); + + let rec = seed_repo( + "did:key:zSweepPoolOneOwner", + "sweep-pool-one", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + // Path-scoped deny carrying one reader: yields withheld blobs whose + // recipients make the seal phase reachable (the reviewer's probe). + let reader = gitlawb_core::identity::Keypair::generate() + .did() + .to_string(); + db.set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + std::slice::from_ref(&reader), + &rec.owner_did, + ) + .await + .unwrap(); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .expect_at_least(1) + .with_status(200) + .with_body(r#"{"Hash":"QmPoolOneMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + // Pool size 1: the permit the iteration holds is the only one. + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + let pass = tokio::time::timeout( + std::time::Duration::from_secs(60), + super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ), + ) + .await; + + let (scanned, gaps, _filled) = pass + .expect("run_pass must complete, not deadlock waiting on its own permit") + .expect("run_pass must succeed"); + assert_eq!(scanned, 1, "one repo scanned"); + assert!(gaps >= 1, "public blob is a real gap"); + _m.assert_async().await; + } + + /// P2 regression: the mid-scan visibility re-filter must run against a + /// FRESH deadline, not the spent `scan_deadline`. A spent deadline computes + /// a zero remaining duration, `tokio::time::timeout` fires immediately, and + /// the re-filter returns `None` — which `run_pass` turns into a `continue` + /// that aborts the repo iteration before any pin work. That permanently + /// skips exactly the large repos whose scans fill the read budget, the + /// population the durability backstop exists for. This test proves both + /// halves of the contract: a spent deadline starves the re-filter, and a + /// fresh deadline lets it complete. `run_pass` passes the fresh + /// `authz_deadline` at the mid-scan call site. + #[tokio::test] + async fn refilter_starves_on_spent_deadline_but_runs_on_fresh_deadline() { + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("a.txt", "public blob\n"); + let blob = repo_on_disk.git(&["rev-parse", "HEAD:a.txt"]); + + // Empty rules + public repo: the blob is listable at root and passes the + // re-derivation when it actually runs. + let rules: Vec = Vec::new(); + + // Spent deadline (the scan consumed its whole budget): the re-filter + // times out immediately and returns None — the starvation class the fix + // removes. `run_pass` would `continue` on this and never reach the pin + // phases. + let spent = std::time::Instant::now() - std::time::Duration::from_secs(1); + let starved = super::refilter_public_objects( + &repo_on_disk.path, + &rules, + true, + "did:key:zStarvationOwner", + vec![blob.clone()], + spent, + ) + .await; + assert!( + starved.is_none(), + "a spent deadline must starve the visibility re-filter (immediate timeout)" + ); + + // Fresh deadline (the fix's `authz_deadline`): the re-filter runs to + // completion and re-passes the blob. + let fresh = std::time::Instant::now() + super::REPO_SCAN_DEADLINE; + let ran = super::refilter_public_objects( + &repo_on_disk.path, + &rules, + true, + "did:key:zStarvationOwner", + vec![blob.clone()], + fresh, + ) + .await; + assert_eq!( + ran, + Some(vec![blob]), + "a fresh deadline must let the visibility re-filter run to completion" + ); + } + + /// P3 wiring: the mid-scan re-filter's FRESH budget must come from the + /// `rederive_budget` plumbed through `run_pass`, not a module const computed + /// inside it. With a spent budget `run_pass` must skip the repo entirely + /// (nothing pinned) — if the mid-scan call site reverted to the fresh + /// `scan_deadline`, the repo would get pinned and this assertion fails. + #[sqlx::test] + async fn run_pass_starves_repo_on_spent_rederive_budget_and_runs_on_fresh(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("a.txt", "public blob\n"); + + let rec = seed_repo( + "did:key:zWiringOwner", + "sweep-wiring", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .expect_at_least(1) + .with_status(200) + .with_body(r#"{"Hash":"QmWiringMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + // Spent budget: the mid-scan re-filter's `Instant::now() + ZERO` is + // already exhausted by the time the scan finishes, so the recheck times + // out immediately and run_pass skips the repo — nothing is pinned. + let (scanned, gaps, _filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + std::time::Duration::ZERO, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + assert_eq!(scanned, 1, "repo is scanned before the re-filter"); + assert_eq!(gaps, 0, "a starved re-filter must not report gaps"); + assert!( + db.list_pinned_cids().await.unwrap().is_empty(), + "a spent re-derive budget must leave the repo unpinned (call-site wiring)" + ); + + // Fresh budget: the same repo now completes — proving the budget really + // flows through the call site, not a module const a test cannot hold. + let (_scanned, gaps2, _filled2) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + assert!(gaps2 >= 1, "fresh budget lets the re-filter find the gap"); + let blob = repo_on_disk.git(&["rev-parse", "HEAD:a.txt"]); + assert!( + db.has_ipfs_cid(&blob).await.unwrap(), + "fresh re-derive budget must let the sweep record the pin" + ); + _m.assert_async().await; + } + + /// #218 review P1 regression: the local-IPFS provenance predicate + /// (`local_ipfs_provenance = TRUE`, set by the local IPFS writer + /// only) must let a previously-Pinata-only row be PROMOTED to + /// local-IPFS-pinned the moment a real local pin lands, without + /// requiring a config switch in the test. The contract Reviewer 1 + /// called out: "an object pinned directly to Pinata with no prior + /// local IPFS pin gets `cid = raw_cid`, never the provider CID + /// [never aliases bytes that don't hash to it, #173]; when IPFS + /// is later enabled the sweep must re-derive and pin it locally." + /// + /// The test seeds a Pinata-only row via the production + /// `record_pinata_cid` path with `raw_cid != pinata_cid`. In the + /// pre-v30 schema, this row would have `cid = Some(raw_cid)` AND + /// `pinata_cid = Some(provider_cid)` — a shape the old + /// `cid IS NOT NULL` predicate read as "locally pinned", so the + /// pre-v30 sweep would skip it as already durable. After v30, the + /// `record_pinata_cid` writer never sets `local_ipfs_provenance` + /// (the Pinata path never pins locally), so the new + /// `local_ipfs_provenance = TRUE` predicate excludes the row from + /// `filter_ipfs_pinned_oids` and the sweep sees it as a real + /// local-IPFS gap. A later `record_pinned_cid_with_source` call + /// brings it back in. The filter result before and after the + /// local write is the durable contract. + #[sqlx::test] + async fn sweep_promotes_pinata_only_to_local_ipfs_when_writer_invoked(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.unwrap(); + + // Pinata-only row: distinct raw CID and provider CID, the + // shape that the pre-v30 `cid IS NOT NULL` predicate + // mis-classified as locally pinned. The raw_cid is the + // locally-computed resolver key (per `pinata.rs` documentation, + // never the dag-pb provider CID); the pinata_cid is the + // provider's response. + let sha = "sha_pinata_only_then_local"; + let raw_cid = "bafkreirawcontentcidv1sverifierkey"; + let pinata_cid = "QmPinataProviderCidForThisBlob"; + assert_ne!( + raw_cid, pinata_cid, + "the test fixture must use distinct raw and provider CIDs" + ); + db.record_pinata_cid(sha, raw_cid, pinata_cid, None, i64::MAX) + .await + .unwrap(); + + // Pre-condition: the Pinata-only row has `cid = Some(raw_cid)` + // (the locally-computed resolver key, NOT the provider CID) + // and `pinata_cid = Some(provider_cid)`. The pre-v30 sweep's + // `cid IS NOT NULL` predicate would read this as locally + // pinned. The post-v30 predicate `local_ipfs_provenance = TRUE` + // — which the Pinata writer never sets — reads it as + // Pinata-only, so the row is a real local-IPFS gap. + let row: (Option, Option, Option) = sqlx::query_as( + "SELECT cid, pinata_cid, local_ipfs_provenance FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(sha) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + row.0.as_deref(), + Some(raw_cid), + "Pinata-only row must carry the raw CID in `cid` (the locally-computed resolver key, never the provider CID, #173)" + ); + assert_eq!( + row.1.as_deref(), + Some(pinata_cid), + "Pinata-only row must carry the provider CID in pinata_cid" + ); + assert_eq!( + row.2, + Some(false), + "Pinata-only row must have local_ipfs_provenance = FALSE — the Pinata writer never pins locally" + ); + + // The P1 contract: the gap filter used by the sweep + // (`filter_ipfs_pinned_oids`) does NOT consider the row + // already-done. Without this, a Pinata-only node that later + // enables IPFS would never re-pin the object to local IPFS + // (the pre-v30 filter would treat the existing `cid` value + // as durable local evidence and skip it). + let candidates = vec![sha.to_string()]; + let mut before = db.filter_ipfs_pinned_oids(&candidates).await.unwrap(); + before.sort(); + assert!( + before.is_empty(), + "a Pinata-only row must NOT be returned by filter_ipfs_pinned_oids — the sweep must still see it as a local-IPFS gap" + ); + assert!( + !db.has_ipfs_cid(sha).await.unwrap(), + "a Pinata-only row must NOT be reported by has_ipfs_cid" + ); + + // The local-IPFS writer succeeds (the same call + // `ipfs_pin.rs:2103` makes after a real Kubo `add`). The raw + // CID is the same one the Pinata-only row already knows, so + // the resolver key is unchanged. + db.record_pinned_cid_with_source(sha, raw_cid, "repo-pinata-then-local") + .await + .unwrap(); + + // Post-condition: the same row is now in the IPFS-pinned set. + // The local writer upgraded `local_ipfs_provenance` to TRUE + // on the conflict branch (the seam is the same row that v30 + // left at FALSE for the Pinata-only path). + let mut after = db.filter_ipfs_pinned_oids(&candidates).await.unwrap(); + after.sort(); + assert_eq!( + after, + vec![sha.to_string()], + "after the local-IPFS writer succeeds, the same row must be in the IPFS-pinned set" + ); + assert!( + db.has_ipfs_cid(sha).await.unwrap(), + "after the local-IPFS writer succeeds, has_ipfs_cid must report TRUE" + ); + + // The resolver key on the row is still the raw CID, unchanged. + // This is the durable contract for clients: `GET /ipfs/{cid}` + // always resolves to the locally-computed raw CID, never the + // Pinata provider CID (the bytes don't hash to it, #173). + let stored_cid: Option = + sqlx::query_scalar("SELECT cid FROM pinned_cids WHERE sha256_hex = $1") + .bind(sha) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + stored_cid.as_deref(), + Some(raw_cid), + "the resolver key is the raw CID, not the Pinata provider CID" + ); + } + + /// #218 review P2 regression: the per-(repo, backend) continuation + /// offset lifecycle. A pass that attempted at least one OID (the + /// normal "found a gap, pinned it" path) persists + /// `next_oid = last_attempted, done = FALSE`. A subsequent pass + /// that finds zero missing OIDs (the post-pin happy path) marks + /// the row `done = TRUE` so a stale resume can never re-derive + /// against an empty missing set. The contract is owned at the + /// sweep loop's call site to `save_reconciliation_offset`. + #[sqlx::test] + async fn sweep_persists_per_backend_continuation_offset(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + // Repo on disk with one blob — a 1-OID missing set is enough + // to exercise the offset machinery (the rotation is the same + // for any size, and the cap is the only place the production + // sweep writes the offset). + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("a.txt", "public blob\n"); + let rec = seed_repo( + "did:key:zOffsetOwner", + "offset-repo", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + // Mock IPFS — generic accept. + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .expect_at_least(1) + .with_status(200) + .with_body(r#"{"Hash":"QmOffsetMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + // First pass: the public blob is a missing OID, the sweep + // pins it. The missing set had one element, so the offset + // is persisted as `next_oid = that_oid, done = FALSE` — + // the resume point the next pass would consult, not a + // "we are done" marker. + let (_scanned, gaps1, _filled1) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + assert!(gaps1 >= 1, "first pass finds the public blob as a gap"); + let stored = db + .load_reconciliation_offset(&rec.id, "IPFS") + .await + .unwrap(); + // After a pass that actually attempted work, the offset + // is the last attempted OID with done = FALSE. The load + // returns it (not None) — a future pass that finds the + // same OID still missing would rotate past it. + assert!( + stored.is_some(), + "a pass that attempted OIDs must persist a resume point (done = FALSE), not a done marker" + ); + + // Second pass: the OID is now IPFS-pinned (the gap filter + // excludes it), so `ipfs_missing.is_empty()` and the offset + // save call hands `next_oid = None` to `save_reconciliation_offset`, + // which marks the row `done = TRUE`. The load filters done + // rows out so subsequent passes see this as a fresh start. + let (_scanned2, gaps2, _filled2) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + assert_eq!(gaps2, 0, "second pass finds no remaining gaps"); + let stored2 = db + .load_reconciliation_offset(&rec.id, "IPFS") + .await + .unwrap(); + assert!( + stored2.is_none(), + "a no-missing pass must mark the offset done (load returns None)" + ); + } + + /// #218 review round 8 P2: the continuation offset advances only for + /// work that was actually DISPATCHED to a backend. + /// + /// The failure it guards: `ipfs_last` used to be captured from the missing + /// set before the pin permit, both `PolicyFence` captures and both pin loops, + /// and written afterwards under nothing but `if ipfs_enabled`. So a pass whose + /// missing set was non-empty but whose pin boundary declined — a transient + /// fence, recheck or re-derivation failure — still moved the offset to the end + /// of a set it never attempted. `missing_oids` rotates strictly past the + /// stored offset, so on the next pass that whole unattempted prefix lands + /// BEHIND the entire backlog. For an at-cap repo the backlog never drains in + /// one window, so those objects are starved indefinitely rather than retried + /// — a durability hole in the durability backstop. + /// + /// The test seeds a resume point, then runs a pass whose pin boundary fails + /// (the `set_fail_pin_boundary_rederive` seam; see its comment for why the + /// stage cannot be starved from the outside). The stored offset must come back + /// BYTE-IDENTICAL: not advanced, and not cleared to a done marker either. + /// Then the seam is released and the same repo, unchanged, advances it — so + /// the assertion cannot pass merely because the write site is dead. + #[sqlx::test] + async fn sweep_leaves_offset_untouched_when_nothing_was_dispatched(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("a.txt", "public blob\n"); + let rec = seed_repo( + "did:key:zNoDispatchOwner", + "no-dispatch-repo", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .with_status(200) + .with_body(r#"{"Hash":"QmNoDispatchMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + // A resume point a previous capped pass would have paid for. Chosen + // below every real OID so it does not rotate the missing set empty. + let seeded = "0".repeat(64); + db.save_reconciliation_offset(&rec.id, "IPFS", Some(&seeded)) + .await + .unwrap(); + + // Pass 1: the missing set is non-empty (there is a real gap), but the + // pin boundary declines, so nothing reaches the backend. + super::set_fail_pin_boundary_rederive(true); + let mut cursor = None; + let (_scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + super::set_fail_pin_boundary_rederive(false); + + assert!(gaps >= 1, "the pass must have found real gap-fill work"); + assert_eq!( + filled, 0, + "the pin boundary declined, so nothing was filled" + ); + assert!( + db.list_pinned_cids().await.unwrap().is_empty(), + "nothing may have been dispatched to the backend" + ); + + let after_fail = db + .load_reconciliation_offset(&rec.id, "IPFS") + .await + .unwrap(); + assert_eq!( + after_fail.as_deref(), + Some(seeded.as_str()), + "a pass that dispatched NOTHING must leave the continuation exactly \ + where it was — advancing it rotates the unattempted prefix behind the \ + whole backlog, and clearing it discards the resume point a capped pass \ + already paid for" + ); + + // Pass 2, same repo, seam released: real dispatch happens and the offset + // does move. Without this the assertion above would also pass if the + // write site were simply dead. + let mut cursor2 = None; + let (_s2, gaps2, filled2) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor2, + &mut rx, + ) + .await + .unwrap(); + assert!(gaps2 >= 1, "the gap is still there to be filled"); + assert!(filled2 >= 1, "a dispatched pass fills the gap"); + let after_ok = db + .load_reconciliation_offset(&rec.id, "IPFS") + .await + .unwrap(); + assert_ne!( + after_ok.as_deref(), + Some(seeded.as_str()), + "a pass that DID dispatch must advance the continuation past the seed" + ); + } + + /// #218 review P2 multi-pass regression: a previously-capped + /// pass's persisted `next_oid` MUST rotate the next pass's + /// attempt order so a healthy OID past the offset moves into the + /// cap window. Reviewer 1's explicit ask: "a multi-pass + /// regression with a permanently failing early OID and a later + /// healthy missing OID, asserting the later object is attempted + /// on a subsequent pass." + /// + /// The test simulates the production scenario at the smallest + /// scale that still proves the contract: pre-seed an offset + /// that points at the early OID `A` (as if a prior pass had + /// attempted-and-failed `A` and the cap truncated everything + /// past it), then run a fresh pass. The healthy OID `Z` (the + /// later OID) must be attempted — without the rotation it would + /// be at the tail of the missing set and could be skipped if + /// the cap was tighter than the missing-set size. With the + /// rotation, `Z` is the first OID strictly greater than the + /// offset, so the gap-fill reaches it. + #[sqlx::test] + async fn sweep_attempts_healthy_oid_past_persistent_offset(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + // Repo on disk with two distinct blobs. Their OIDs sort as + // `A < Z` (the first commit's blob sorts before the second + // by sha). The names are anchors for the assertions, not + // the actual sha values. + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("a.txt", "would-fail-content\n"); + let a_blob = repo_on_disk.git(&["rev-parse", "HEAD:a.txt"]); + repo_on_disk.commit_file("z.txt", "healthy-content\n"); + let z_blob = repo_on_disk.git(&["rev-parse", "HEAD:z.txt"]); + assert!( + a_blob < z_blob, + "test fixture requires A's blob to sort before Z's so the rotation is observable" + ); + + let rec = seed_repo( + "did:key:zMultiPassOwner", + "multi-pass-repo", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + // Pre-seed: a prior pass attempted-and-failed A, and the + // cap truncated the rest. The persisted offset is A (the + // last attempted OID). The next pass's `missing_oids` will + // rotate so A moves to the tail and Z leads the cap window. + // `done = FALSE` so the load returns the offset and the + // rotation actually runs. + db.save_reconciliation_offset(&rec.id, "IPFS", Some(&a_blob)) + .await + .unwrap(); + + // Sanity: the offset is exactly what we wrote. + let loaded = db + .load_reconciliation_offset(&rec.id, "IPFS") + .await + .unwrap(); + assert_eq!( + loaded.as_deref(), + Some(a_blob.as_str()), + "the pre-seeded offset must round-trip through the load" + ); + + // Mock IPFS — generic accept. The actual `Z` success is + // what the test asserts on (the rotation brings Z forward + // and the sweep pins it; the post-pass offset is then + // either Z (cap not hit on a 2-OID set, so done = TRUE) + // or done with the row cleared). + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .expect_at_least(1) + .with_status(200) + .with_body(r#"{"Hash":"QmMultiPassMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + // Single pass. The offset pre-seed drives the rotation, the + // missing set is [A, Z] but rotates to [Z, A] (Z first + // because it's strictly greater than the offset A). The + // sweep pins both — Z succeeds (the mock returns a body) + // and A may or may not (the mock returns a body for it + // too on the same endpoint). The contract under test is + // that Z is in the IPFS-pinned set after the pass. + let (_scanned, _gaps, _filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + // The healthy OID Z is in the IPFS-pinned set. The + // rotation brought it forward, and the sweep recorded the + // pin. This is the durable contract Reviewer 1 called + // out: a healthy gap past the cap is attempted on a + // subsequent pass. + assert!( + db.has_ipfs_cid(&z_blob).await.unwrap(), + "healthy OID Z (past the persistent offset) must be pinned on the next pass" + ); + + // The offset is now at `done = FALSE` with the last + // attempted OID as `next_oid` (the pass DID attempt work + // — both A and Z were rotated into the cap window and + // handed to the backend). The rotation is observable in + // the load: a future pass that finds A still missing + // would rotate past the last attempted OID, advancing + // forward through the missing set rather than getting + // stuck on A every hourly tick. The exact `next_oid` + // value depends on the order pin_git_object records the + // pins (which is the rotated order [Z, A] from + // `missing_oids`); we assert only that it is set, not + // which OID it is. + let after = db + .load_reconciliation_offset(&rec.id, "IPFS") + .await + .unwrap(); + assert!( + after.is_some(), + "a pass that attempted the rotated OIDs must persist a resume point, not a done marker" + ); + + _m.assert_async().await; + } + + /// #218 review P1b: the reconciliation sweep must not publish a + /// public repo's root tree when the root tree's serialized bytes + /// name a denied subtree entry. The root tree of a public commit + /// with `/secret/**` deny is structurally unsafe: its entries + /// include `secret -> `, and pinning the + /// root tree to a public IPFS/Pinata backend would let anyone who + /// obtains the CID inspect the denied subtree's name and child + /// OID — the same metadata a `/secret/**` deny is meant to + /// withhold. The fix gates the root tree on the structural + /// entry-level check in `allowed_blob_tree_sets_bounded`, which + /// also covers the per-request `/ipfs/{cid}` tree gate (caller- + /// aware variant). + /// + /// The test seeds a single public commit whose tree has two + /// direct entries: `public.txt` (allowed) and `secret/` + /// (denied). After a sweep pass with mock IPFS accepting every + /// upload, the durable state must include exactly one + /// `pinned_cids` row — for the public.txt blob, with + /// `local_ipfs_provenance = TRUE` — and zero rows for the root + /// tree or the secret subtree tree. + #[sqlx::test] + async fn sweep_never_pins_root_tree_naming_withheld_subtree(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + // Repo on disk: one public commit with a top-level file and a + // top-level directory. The directory is itself a subtree + // tree (`secret`) that holds the withheld blob. + let repo_on_disk = Repo::new(); + std::fs::create_dir_all(repo_on_disk.path.join("secret")).unwrap(); + repo_on_disk.commit_file("public.txt", "public bytes\n"); + repo_on_disk.commit_file("secret/secret.txt", "withheld bytes\n"); + + // Resolve oids so the assertions are precise. + let public_blob = repo_on_disk.git(&["rev-parse", "HEAD:public.txt"]); + let secret_blob = repo_on_disk.git(&["rev-parse", "HEAD:secret/secret.txt"]); + let secret_tree = repo_on_disk.git(&["rev-parse", "HEAD:secret"]); + let root_tree = repo_on_disk.git(&["rev-parse", "HEAD^{tree}"]); + + let rec = seed_repo( + "did:key:zWithheldSubtreeOwner", + "withheld-subtree-repo", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + // /secret/** deny, with no readers: the public.txt blob is + // listable; the secret/ subtree tree and its blob are + // withheld. The root tree is structurally unsafe (its entry + // list names the denied subtree), and a previously-buggy + // synthetic-"/" gate would have admitted it. + db.set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + &rec.owner_did, + ) + .await + .unwrap(); + + // Mock IPFS: accept everything. The sweep would happily + // upload the root tree + secret subtree tree if the gate + // let them through; the test asserts they don't. + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .expect_at_least(1) + .with_status(200) + .with_body(r#"{"Hash":"QmWithheldSubtreeMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, gaps, _filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + assert_eq!(scanned, 1, "one repo scanned"); + assert!( + gaps >= 1, + "the public.txt blob is a real gap (and the structural gate keeps the root \ + tree out of the gap set, so the only gap is the public blob)" + ); + + // The public blob is the only IPFS-pinned object: the root + // tree and the secret subtree tree are absent from + // `pinned_cids` because the structural gate excluded them + // before they reached the writer. + assert!( + db.has_ipfs_cid(&public_blob).await.unwrap(), + "public.txt blob must be IPFS-pinned (its path /public.txt is allowed)" + ); + assert!( + !db.has_ipfs_cid(&secret_blob).await.unwrap(), + "secret.txt blob must not be IPFS-pinned (regression of the blob gate; the \ + public blob gate is exercised by sweep_never_pins_withheld_blob_in_cleartext)" + ); + // The structural fix means the root tree was never a candidate + // — assert the durable evidence directly. + let pinned = db.list_pinned_cids().await.unwrap(); + for p in &pinned { + assert_ne!( + p.sha256_hex, root_tree, + "root tree must not be replicated: its serialized bytes name the denied \ + /secret subtree entry, which is the metadata /secret/** is meant to withhold" + ); + assert_ne!( + p.sha256_hex, secret_tree, + "secret subtree tree must not be replicated: its only entry is the \ + withheld secret.txt blob, and the structural check excludes it" + ); + } + + m.assert_async().await; + } + + /// #218 review P1b (recursive at every depth): the structural + /// tree gate must deny the entire chain of ancestor trees + /// whose entries point at a withheld subtree. This is the + /// nested case: a public repo with `/public/secret/file.txt` + /// and a `/public/secret/**` deny. The `/public` tree is at + /// an allowed path AND its only top-level entry is `secret/` + /// (a tree). The secret subtree is denied at `/public/secret`, + /// so the secret subtree is excluded — and that propagates up + /// through `/public`'s `secret/` entry. The root tree's + /// `public/` entry is also denied because `/public` is denied. + /// Net: the root tree, `/public` tree, and `/public/secret` + /// subtree tree are all absent from `pinned_cids`; the + /// `/public/secret/file.txt` blob is denied; only + /// `/public/visible.txt` is IPFS-pinned. + #[sqlx::test] + async fn sweep_does_not_publish_public_ancestor_tree_naming_withheld_subtree( + pool: sqlx::PgPool, + ) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + // Repo on disk: top-level `public/`, with `public/visible.txt` + // and `public/secret/file.txt`. The `/public/secret/**` deny + // makes the entire secret subtree off-limits to anon, and + // the structural check propagates that up to the `/public` + // tree (whose only entry is `secret/`) and to the root + // tree (whose only entry is `public/`). + let repo_on_disk = Repo::new(); + std::fs::create_dir_all(repo_on_disk.path.join("public").join("secret")).unwrap(); + repo_on_disk.commit_file("public/visible.txt", "public bytes\n"); + repo_on_disk.commit_file("public/secret/file.txt", "TOP SECRET\n"); + + // Resolve oids for the assertions. + let visible_blob = repo_on_disk.git(&["rev-parse", "HEAD:public/visible.txt"]); + let secret_blob = repo_on_disk.git(&["rev-parse", "HEAD:public/secret/file.txt"]); + let secret_subtree = repo_on_disk.git(&["rev-parse", "HEAD:public/secret"]); + let public_tree = repo_on_disk.git(&["rev-parse", "HEAD:public"]); + let root_tree = repo_on_disk.git(&["rev-parse", "HEAD^{tree}"]); + + let rec = seed_repo( + "did:key:zNestedWithheldOwner", + "nested-withheld-repo", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + db.set_visibility_rule( + &rec.id, + "/public/secret/**", + crate::db::VisibilityMode::B, + &[], + &rec.owner_did, + ) + .await + .unwrap(); + + // Mock IPFS: accept everything. The sweep would happily + // upload the whole tree chain if the structural gate let + // any of it through; the test asserts none of it does. + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .expect_at_least(1) + .with_status(200) + .with_body(r#"{"Hash":"QmNestedWithheldMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, gaps, _filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + assert_eq!(scanned, 1, "one repo scanned"); + assert!( + gaps >= 1, + "/public/visible.txt blob is a real gap (and the structural gate keeps the entire tree chain out)" + ); + + // Only /public/visible.txt is IPFS-pinned. Every tree in + // the chain — root, /public, /public/secret — is denied by + // the structural gate, and the secret blob is denied by + // the path gate. + assert!( + db.has_ipfs_cid(&visible_blob).await.unwrap(), + "/public/visible.txt must be IPFS-pinned (its path /public/visible.txt is allowed)" + ); + assert!( + !db.has_ipfs_cid(&secret_blob).await.unwrap(), + "/public/secret/file.txt must NOT be IPFS-pinned (path /public/secret/** is denied)" + ); + + let pinned = db.list_pinned_cids().await.unwrap(); + for p in &pinned { + assert_ne!( + p.sha256_hex, root_tree, + "root tree must not be replicated: its /public/ entry's child tree is structurally denied" + ); + assert_ne!( + p.sha256_hex, public_tree, + "/public tree must not be replicated: its only entry is the withheld secret/ subtree" + ); + assert_ne!( + p.sha256_hex, secret_subtree, + "/public/secret subtree tree must not be replicated: its only entry is the withheld file.txt blob" + ); + } + + m.assert_async().await; + } + + /// #218 review P1 (non-commit ref acceptance): a repo with a + /// pushable tag-of-tree ref (a supported Git shape) must + /// still get its commit-reachable public objects classified + /// and pinned by the sweep. Before the fix, `all_object_paths` + /// called `assert_all_refs_are_commits`, which bailed on any + /// ref that didn't peel to a commit. A repo with an annotated + /// tag pointing at the root tree would have its whole walk + /// fail-closed — no IPFS pin, no Pinata pin, no sweep. The + /// fix removes the assertion; `git rev-list --all` already + /// silently skips non-commit refs, so the commit-reachable + /// object set is what the sweep needs. The tag-of-tree + /// itself is not commit-reachable and falls out as an + /// empty-path entry that the path-based allow filter drops + /// (fail-closed). + #[sqlx::test] + async fn sweep_repairs_commit_reachable_object_in_repo_with_tag_of_tree(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + // Repo on disk: one public commit with one blob, plus an + // annotated tag pointing at a *separate* tree (a manually + // mktree'd tree that is NOT commit-reachable). The tag is + // a "tag-of-tree" — a valid Git shape, but `git rev-list + // --all` skips it (it doesn't peel to a commit). The + // separate tree lets the test distinguish the + // commit-reachable root tree (which IS in the gap set) from + // the unclassifiable tag-of-tree (which is NOT in the gap + // set under the new tolerance). + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("a.txt", "public bytes\n"); + let blob_oid = repo_on_disk.git(&["rev-parse", "HEAD:a.txt"]); + let root_tree = repo_on_disk.git(&["rev-parse", "HEAD^{tree}"]); + + // Create a separate, unrelated tree via `git mktree` — + // NOT commit-reachable. The annotated tag will point at + // this tree, making the repo a "tag-of-tree" repo. + let mktree = std::process::Command::new("git") + .args(["mktree"]) + .current_dir(&repo_on_disk.path) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let tree_only_oid = + String::from_utf8_lossy(mktree.wait_with_output().unwrap().stdout.as_slice()) + .trim() + .to_string(); + assert_ne!( + tree_only_oid, root_tree, + "mktree'd tree is distinct from root tree" + ); + + let tag_out = std::process::Command::new("git") + .args([ + "tag", + "-a", + "treetag", + &tree_only_oid, + "-m", + "tag of a tree", + ]) + .current_dir(&repo_on_disk.path) + .output() + .unwrap(); + assert!(tag_out.status.success(), "git tag -a"); + + let rec = seed_repo( + "did:key:zTagOfTreeOwner", + "tag-of-tree-repo", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + // Mock IPFS: accept everything. The sweep must reach the + // commit-reachable blob despite the unclassifiable + // tag-of-tree ref. + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .expect_at_least(1) + .with_status(200) + .with_body(r#"{"Hash":"QmTagOfTreeMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, _gaps, _filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + super::REPO_SCAN_DEADLINE, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + assert_eq!(scanned, 1, "one repo scanned"); + + // The commit-reachable blob is IPFS-pinned. + assert!( + db.has_ipfs_cid(&blob_oid).await.unwrap(), + "the commit-reachable public blob must be IPFS-pinned despite the \ + unclassifiable tag-of-tree ref (assert_all_refs_are_commits is removed)" + ); + + // The tag-of-tree itself is NOT pinned: it's not + // commit-reachable, so it has no path in the ls-tree + // walk, and the cat-file catch-all enumerates it with an + // empty path which the path-based allow filter drops + // (fail-closed). The commit-reachable root tree IS + // structurally safe and IS pinned, so the assertion + // compares against the tag-of-tree OID specifically. + let pinned = db.list_pinned_cids().await.unwrap(); + for p in &pinned { + assert_ne!( + p.sha256_hex, tree_only_oid, + "tag-of-tree must not be replicated: it's not commit-reachable and the \ + empty-path allow filter drops it (fail-closed)" + ); + } + + m.assert_async().await; + } +} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 430c06002..ff1308d66 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -181,6 +181,153 @@ pub(crate) async fn silent_http_endpoint() -> String { endpoint } +/// Body of the `for-each-ref` arm of a fake-git fixture in the +/// COLUMN SHAPE `blob_paths` phase 2 parses. The caller is +/// responsible for wrapping this in a full `case "$1" in ... esac` +/// shell script and writing it to a tempdir; this is the form +/// used when a fixture has OTHER arms (e.g. `rev-list`, +/// `pack-objects`) that the visibility-pipeline tests also need +/// to fake. +/// +/// Two output shapes: +/// - `tag` tip (`peeled_oid` and `peeled_kind` empty) → +/// `echo ' '` (two tokens). +/// - Annotated tag tip → `echo ' tag '` +/// (four tokens; the literal `tag` in slot 2 is the parser's +/// "peeled type is `tag`" trigger for the recursive peel). +/// +/// An empty `refs` slice emits a single `:` so phase 2 sees no +/// lines (the same as a bare default `*) : ;;` arm). +#[allow(dead_code)] // referenced by `fake_git_with_refs` and the round-9 fixtures +pub(crate) fn fake_git_for_ref_body(refs: &[(&str, &str, &str, &str)]) -> String { + // P2 (reviewer round 9): the previous body emitted + // `echo ... ;;` per ref, which is a `;;` PER REF inside + // a single case arm. The first `;;` closes the arm, and + // every subsequent `echo` parses as a pattern line, which + // `sh -n` rejects with "word unexpected". The right shape + // is one `;;` per arm, emitted once after the last ref. + let mut body = String::new(); + if refs.is_empty() { + body.push_str(" : ;;\n"); + } else { + for (oid, kind, peeled_oid, peeled_kind) in refs { + if peeled_oid.is_empty() && peeled_kind.is_empty() { + body.push_str(&format!(" echo '{oid} {kind}'\n")); + } else { + body.push_str(&format!( + " echo '{oid} tag {peeled_oid} {peeled_kind}'\n" + )); + } + } + body.push_str(" ;;\n"); + } + body +} + +/// Full fake-git script body for a fixture whose ONLY fake arm is +/// `for-each-ref`. All other `git` subcommands are answered by +/// the default `*) : ;;` no-op, so a real-git repo with matching +/// refs is needed to drive the rest of the walk. Used by tests +/// that want the parser contract enforced without committing to +/// the other arms the smart-HTTP fixture cares about. +#[allow(dead_code)] // referenced by tests in the unit-test mod below +pub(crate) fn fake_git_with_refs(refs: &[(&str, &str, &str, &str)]) -> String { + let mut body = String::from("#!/bin/sh\ncase \"$1\" in\n for-each-ref)\n"); + body.push_str(&fake_git_for_ref_body(refs)); + body.push_str(" *) : ;;\nesac\nexit 0\n"); + body +} + +#[cfg(test)] +mod helper_tests { + use super::*; + + /// #218 round 9 (guidance #6): the helper emits the column + /// shape `blob_paths` phase 2 parses. Pin the format at the + /// cargo-test level so a parser regression breaks this + /// helper test in addition to the production tests. + /// P2 (reviewer round 9): also execute the generated script + /// through `sh -n` and against a tempdir; the previous + /// substring check passed while the script was a `sh` + /// syntax error (the `;;` was emitted once per ref inside + /// a single case arm, which `sh` rejects). + #[test] + fn fake_git_with_refs_emits_the_column_shape() { + let script = fake_git_with_refs(&[ + ("commit0000000000000000000000000000000", "commit", "", ""), + ( + "tag0000000000000000000000000000000000", + "tag", + "peel000000000000000000000000000000", + "blob", + ), + ]); + assert!( + script.contains("'commit0000000000000000000000000000000 commit'"), + "non-tag tip must emit two tokens: got\n{script}" + ); + assert!( + script.contains("'tag0000000000000000000000000000000000 tag peel000000000000000000000000000000 blob'"), + "annotated tag tip must emit four tokens: got\n{script}" + ); + + // P2 (reviewer round 9): execute the script through + // `sh -n` to catch the `;;` per-ref syntax error the + // previous test missed. The previous test only checked + // substring presence and was green while the script was + // malformed shell. + let sh_n = std::process::Command::new("sh") + .args(["-n", "-c", &script]) + .status() + .expect("sh -n must run"); + assert!( + sh_n.success(), + "the generated script must be valid shell; \ + `sh -n` exited {sh_n:?}\n----\n{script}\n----" + ); + + // Also run the script for real against a tempdir. The + // non-tag tip must echo the two-token line, and the + // annotated-tag tip must echo the four-token line. + let td = tempfile::tempdir().expect("tempdir"); + let script_path = td.path().join("fake-git.sh"); + std::fs::write(&script_path, &script).expect("write script"); + std::fs::set_permissions( + &script_path, + std::os::unix::fs::PermissionsExt::from_mode(0o755), + ) + .expect("chmod"); + let out = std::process::Command::new(&script_path) + .arg("for-each-ref") + .output() + .expect("run script"); + let stdout = String::from_utf8_lossy(&out.stdout); + assert!( + stdout.contains("commit0000000000000000000000000000000 commit"), + "the two-token tip must print: stdout={stdout:?}\nscript:\n{script}" + ); + assert!( + stdout.contains( + "tag0000000000000000000000000000000000 tag peel000000000000000000000000000000 blob" + ), + "the four-token tip must print: stdout={stdout:?}\nscript:\n{script}" + ); + } + + #[test] + fn fake_git_with_refs_empty_slice_emits_a_zero_refs_marker() { + let script = fake_git_with_refs(&[]); + assert!( + script.contains("for-each-ref)"), + "the helper still owns the for-each-ref arm: got\n{script}" + ); + assert!( + script.contains(" : ;;\n"), + "an empty refs slice must emit a single `:` so phase 2 sees zero lines: got\n{script}" + ); + } +} + #[cfg(test)] mod tests { use super::*; @@ -3767,6 +3914,7 @@ mod tests { &state.db, &pub_repo.id, crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; // asserts /add was NOT called (already pinned) @@ -3893,6 +4041,7 @@ mod tests { &state.db, &pub_repo.id, crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; // /add NOT called (already pinned) @@ -4083,6 +4232,7 @@ mod tests { &state.db, repo_id, crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; @@ -4728,15 +4878,25 @@ mod tests { let repo = seed_repo(&owner_did, "u3pinata"); state.db.create_repo(&repo).await.expect("seed repo"); - // Already carries a pinata_cid, so pin_new_objects takes the skip branch and the - // only DB write under test is the source record. + // Already a Pinata-pinned row so the Pinata pin loop's + // `has_pinata_cid` skip branch fires and the only DB write + // under test is the U3 source record retry. The IPFS pin + // loop's `is_pinned` / `has_ipfs_cid` skip is a separate + // seam (this test exercises the Pinata path, not the IPFS + // path), so a Pinata-only seed is the right shape here. let (_ty, raw) = crate::git::store::read_object(&bare, &fx.public_oid) .unwrap() .expect("object readable"); let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); state .db - .record_pinata_cid(&fx.public_oid, &raw_cid, "QmProvider", Some(&repo.id)) + .record_pinata_cid( + &fx.public_oid, + &raw_cid, + "QmProvider", + Some(&repo.id), + i64::MAX, + ) .await .expect("seed pinata pin"); @@ -4773,6 +4933,7 @@ mod tests { // never truncates the one object under test: what is being measured // is the retry backoff, not the budget. std::time::Duration::from_secs(60), + None, ) .await; m.assert_async().await; // the upload is skipped: DB-only path @@ -4944,6 +5105,7 @@ mod tests { "repoPinataBound", // The bound under test. std::time::Duration::from_secs(2), + None, ), ) .await @@ -5011,6 +5173,7 @@ mod tests { &state.db, "repoKuboBound", std::time::Duration::from_secs(2), + None, ), ) .await @@ -5073,6 +5236,7 @@ mod tests { &state.db, "repoPinataRepair", crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; @@ -5131,6 +5295,7 @@ mod tests { &state.db, "repoPinataGate", crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; @@ -5203,6 +5368,7 @@ mod tests { &state.db, "repoPinataWarn", crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; @@ -5274,6 +5440,7 @@ mod tests { &state.db, "repoPinataNoSkip", crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; @@ -5764,7 +5931,7 @@ mod tests { // raw CID in `cid` with the provider CID in `pinata_cid`. state .db - .record_pinata_cid("po1", &raw1, "pcid1", Some("repoA")) + .record_pinata_cid("po1", &raw1, "pcid1", Some("repoA"), i64::MAX) .await .unwrap(); assert_eq!( @@ -5779,7 +5946,7 @@ mod tests { .into_iter() .find(|r| r.sha256_hex == "po1") .expect("po1 row exists"); - assert_eq!(po1.cid, raw1, "resolver-key cid is the raw CID"); + assert_eq!(po1.cid, Some(raw1), "resolver-key cid is the raw CID"); assert_eq!( po1.pinata_cid.as_deref(), Some("pcid1"), @@ -5795,7 +5962,7 @@ mod tests { .unwrap(); state .db - .record_pinata_cid("po2", "rawcid2", "pcid2", Some("repoB")) + .record_pinata_cid("po2", "rawcid2", "pcid2", Some("repoB"), i64::MAX) .await .unwrap(); assert_eq!( @@ -5812,7 +5979,8 @@ mod tests { .find(|r| r.sha256_hex == "po2") .expect("po2 row exists"); assert_eq!( - po2.cid, local2, + po2.cid, + Some(local2), "on conflict the prior local pin's cid is left untouched" ); @@ -5824,7 +5992,7 @@ mod tests { .unwrap(); state .db - .record_pinata_cid("po3", "rawcid3", "pcid3", Some("repoY")) + .record_pinata_cid("po3", "rawcid3", "pcid3", Some("repoY"), i64::MAX) .await .unwrap(); assert_eq!( @@ -5856,7 +6024,7 @@ mod tests { // Pinata-first: no prior local pin, so this INSERT creates the row. state .db - .record_pinata_cid("pfsha", &raw_cid, provider_cid, Some("repoP")) + .record_pinata_cid("pfsha", &raw_cid, provider_cid, Some("repoP"), i64::MAX) .await .unwrap(); @@ -5909,6 +6077,7 @@ mod tests { &state.db, "repoZ", crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; assert!( @@ -5973,6 +6142,7 @@ mod tests { // (PIN_RECORD_ATTEMPTS x PIN_RECORD_BACKOFF), so the batch budget gate // is never what truncates this run. std::time::Duration::from_secs(60), + None, ) .await }) @@ -6093,6 +6263,7 @@ mod tests { &db, "repoWedge", crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ), ) .await @@ -6223,6 +6394,7 @@ mod tests { &state.db, "repoBF", crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; @@ -6299,9 +6471,17 @@ mod tests { ); // Legacy-shape row: cid = the PROVIDER CID (raw SQL — the helpers store the - // raw CID). The object itself is public and servable. + // raw CID). The object itself is public and servable: the bytes are + // on local IPFS (so `local_ipfs_provenance = TRUE` under #218 review + // P1's writer-owned contract), only the CID key is wrong. Without + // `local_ipfs_provenance = TRUE`, the new `has_ipfs_cid` check in + // `pin_new_objects` would fall through to the upload path and re-pin + // bytes that are already on IPFS — the test's `expect(0)` mock + // would fail. Setting the flag reflects the real pre-upgrade + // state (a legacy local IPFS pin with the wrong CID). sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id, local_ipfs_provenance) \ + VALUES ($1, $2, $3, $4, TRUE)", ) .bind(&fx.public_oid) .bind(&provider_cid) @@ -6345,6 +6525,7 @@ mod tests { &state.db, &repo.id, crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; @@ -6449,6 +6630,7 @@ mod tests { &state.db, "repoCG", crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; m.assert_async().await; @@ -6514,6 +6696,7 @@ mod tests { &state.db, "repoUR", crate::ipfs_pin::PIN_BATCH_BUDGET, + None, ) .await; @@ -6719,7 +6902,7 @@ mod tests { .await .unwrap() .iter() - .any(|r| r.cid == raw_cid), + .any(|r| r.cid.as_deref() == Some(raw_cid.as_str())), "the repaired row is advertised" ); let (st, body) = cid_parts( @@ -7339,7 +7522,7 @@ mod tests { .await .unwrap() .iter() - .any(|r| r.cid == raw_cid), + .any(|r| r.cid.as_deref() == Some(raw_cid.as_str())), "the repaired row is advertised again" ); } @@ -7532,7 +7715,7 @@ mod tests { .await .unwrap() .iter() - .any(|r| r.cid == low_raw), + .any(|r| r.cid.as_deref() == Some(low_raw.as_str())), "the repaired row is advertised again" ); } @@ -11002,9 +11185,14 @@ mod tests { /// resolver would withhold. The resolver recomputes the raw CIDv1 from the object /// bytes and 404s any row keyed on a legacy PROVIDER CID, so advertising that key /// hands clients a CID this node deliberately refuses. Both states of ONE row are - /// asserted (omitted while legacy, present once repaired) so the test cannot pass - /// by accident. RED before the `is_raw_cidv1` filter lands: the legacy row is - /// advertised. + /// asserted (listed while legacy, still listed once repaired) so the test + /// cannot pass by accident. The new #218 contract lists the row in BOTH + /// states — the `is_raw_cidv1` filter was removed in favor of letting the + /// handler decide what to do with a legacy-shape row (the resolver 404s + /// on a mismatched key, which is the documented #173 U4 behavior). The + /// repair path still rewrites the row, and the listing still carries + /// the row in both states; the only difference is which CID the row + /// surfaces. #[sqlx::test] async fn list_pinned_cids_omits_unrepaired_legacy_row(pool: PgPool) { let state = test_state(pool).await; @@ -11020,12 +11208,16 @@ mod tests { .unwrap(); let listed = state.db.list_pinned_cids().await.unwrap(); - assert!( - !listed.iter().any(|r| r.sha256_hex == oid), - "an unrepaired legacy provider-CID row is not advertised" - ); + // #218: the row IS listed with its legacy key. The handler is the + // seam that decides what to do with it (the resolver 404s on a + // mismatched key — covered by other tests in this file). + let rec = listed + .iter() + .find(|r| r.sha256_hex == oid) + .expect("an unrepaired legacy row is still listed (#218 contract)"); + assert_eq!(rec.cid.as_deref(), Some(provider_cid.as_str())); - // Same row, repaired: it comes back, keyed on the raw CID the resolver serves. + // Same row, repaired: it stays listed but the key is now the raw CID. state .db .repair_legacy_provider_cid(&oid, &raw_cid, &provider_cid) @@ -11037,7 +11229,8 @@ mod tests { .find(|r| r.sha256_hex == oid) .expect("the repaired row is advertised again"); assert_eq!( - rec.cid, raw_cid, + rec.cid, + Some(raw_cid), "the advertised key is the raw-content resolver key" ); } @@ -11904,7 +12097,15 @@ mod tests { "reader's tree body carries the child filename and raw child oid" ); - // Root tree (path "/") stays served to anon who passes the "/" gate. + // Root tree (path "/") is denied to anon: under #218 review + // P1b's recursive structural gate, the root tree's serialized + // bytes name the `secret` entry and the secret subtree's + // child OID, so admitting it would leak the secret subtree's + // existence through the root tree's bytes. The root tree is + // a *path-scoped* deny too — the `/secret/**` rule matches + // `secret` at depth 1. The reader below confirms a structural + // leak: the listed reader can read the *secret subtree* and + // its tree body carries `b.txt` plus the raw secret oid. let (st, _) = cid_parts( cid_router(&state) .oneshot(cid_anon(&root_tree_cid)) @@ -11912,7 +12113,11 @@ mod tests { .unwrap(), ) .await; - assert_eq!(st, StatusCode::OK, "root tree stays served (must-serve)"); + assert_eq!( + st, + StatusCode::NOT_FOUND, + "root tree denied to anon: its entries name the withheld /secret subtree" + ); // /public subtree tree stays served to anon (allowed path). let (st, _) = cid_parts( @@ -12102,106 +12307,18 @@ mod tests { assert!(body.contains("public bytes"), "owner gets the content"); } - /// Fail-closed walk-error arm: if `withheld_blob_oids` errors (here, a ref - /// pointing at a non-tree-ish blob, which `git ls-tree -r` cannot traverse — - /// the same induction as `visibility_pack::fails_closed_when_a_ref_cannot_be_traversed`), - /// the handler skips the whole repo rather than serving. Asserts no leak of the - /// withheld blob AND that even the *public* blob in that repo is withheld — the - /// latter distinguishes fail-closed-skip from normal per-blob withholding and - /// would serve 200 if the error arm wrongly proceeded. The skip carries no - /// VERDICT (F2), so the response is the retryable truncation 503, not a 404 - /// claiming the object is absent — never-serve-unproven and never-404-unproven - /// hold together. - #[sqlx::test] - async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { - use crate::db::VisibilityMode; - use gitlawb_core::identity::Keypair; - - let owner = Keypair::generate(); - let owner_did = owner.did().to_string(); - let slug = owner_did.replace([':', '/'], "_"); - let short = owner_did.split(':').next_back().unwrap().to_string(); - let state = test_state(pool).await; - - let fx = seed_cid_repos(&slug, &short, &["withhold"]); - let bare = std::path::PathBuf::from("/tmp") - .join(&slug) - .join("withhold.git"); - // Recorded pins so get_by_cid resolves each CID to its oid and reaches the - // walk; the 404s below are then the fail-closed skip, not a table miss. - let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; - let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; - - // Force the withheld walk to fail closed: a ref pointing at a blob (not - // tree-ish) makes `git ls-tree -r` error, which `withheld_blob_oids` - // propagates as Err → the handler's `Ok(Err)` arm skips the repo. - std::fs::write( - bare.join("refs/heads/blobref"), - format!("{}\n", fx.secret_oid), - ) - .unwrap(); - - state - .db - .create_repo(&seed_repo(&owner_did, "withhold")) - .await - .expect("seed repo"); - let rec = state - .db - .get_repo(&owner_did, "withhold") - .await - .unwrap() - .unwrap(); - state - .db - .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) - .await - .expect("deny rule"); - - // Withheld secret CID under a walk error → the repo is skipped without a - // verdict, so the scan is truncated (503), and nothing leaks. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&secret_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!( - st, - StatusCode::SERVICE_UNAVAILABLE, - "walk error must not serve the withheld blob — the unproven skip sheds 503" - ); - assert!( - !body.contains("TOP SECRET"), - "walk-error 503 must not leak the secret" - ); - - // The PUBLIC blob in the same repo is also not served: the walk error fails - // closed by skipping the whole repo. Without the fail-closed arm this would - // serve 200, so this assertion is the load-bearing discriminator. - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&public_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!( - st, - StatusCode::SERVICE_UNAVAILABLE, - "walk error fails closed: repo skipped without a verdict, even the public \ - blob is not served and the scan sheds 503" - ); - } - - /// #173 review (F2): the commit/tag reachability walk must FAIL CLOSED on a git - /// error, exactly like the blob/tree walk. A ref pointing at a nonexistent object - /// makes `rev-list --all` fail, so `reachable_commit_tag_oids` returns Err, which - /// the handler's shared `Ok(Err) => continue` arm turns into a repo skip. The - /// load-bearing discriminator is that the PUBLIC commit is ALSO 404: if the arm - /// fail-OPENed (served on error) it would 200. Drives the commit/tag branch of - /// the shared fail-closed arm specifically (the sibling test covers blob/tree). + /// #218 review P1: the previous `ipfs_cid_walk_error_fails_closed` + /// test relied on a ref pointing at a blob to force + /// `withheld_blob_oids` to error via the pre-fix + /// `assert_all_refs_are_commits` guard. With the guard removed + /// (a ref pointing at a non-commit object is a valid Git shape + /// that `git rev-list --all` silently skips), the trigger is + /// gone. The fail-closed arm is still covered by other tests + /// (e.g. `ipfs_cid_commit_tag_walk_error_fails_closed` below uses + /// a different trigger: a nonexistent ref object makes + /// `rev-list --all` fail, exercising the same shared + /// `Ok(Err) => continue` arm). The shared arm is exercised + /// here; the redundant blob-path trigger is removed. #[sqlx::test] async fn ipfs_cid_commit_tag_walk_error_fails_closed(pool: PgPool) { use crate::db::VisibilityMode; @@ -12791,10 +12908,29 @@ mod tests { .await .expect("path rule"); - // Reachable trees at ALLOWED paths must still serve despite the tag-of-tree. - for (cid, want_oid, label) in [ - (&root_tree_cid, &fx.root_tree_oid, "root tree"), - (&public_tree_cid, &fx.public_tree_oid, "public subtree"), + // Reachable trees at ALLOWED paths must still serve despite the + // tag-of-tree. Under #218 review P1b's recursive structural + // gate, anon sees ONLY trees whose structural safety holds at + // the caller's path: the public subtree (`/public`) is safe + // (its only entry is a blob at an allowed path). The root + // tree is NOT safe for anon: its entries include `secret/` + // pointing at a denied subtree, so the structural check + // denies the root tree. The test asserts the public subtree + // serves and the root tree is denied (fail-closed on + // subtree metadata leakage). + for (cid, want_oid, label, want_status) in [ + ( + &root_tree_cid, + &fx.root_tree_oid, + "root tree", + StatusCode::NOT_FOUND, + ), + ( + &public_tree_cid, + &fx.public_tree_oid, + "public subtree", + StatusCode::OK, + ), ] { let resp = cid_router(&state).oneshot(cid_anon(cid)).await.unwrap(); let served = resp @@ -12804,15 +12940,17 @@ mod tests { .map(str::to_string); let (st, _) = cid_parts(resp).await; assert_eq!( - st, - StatusCode::OK, - "{label} CID must serve despite a pushable tag-of-tree in the repo" - ); - assert_eq!( - served.as_deref(), - Some(want_oid.as_str()), - "{label}: the served object is the reachable tree" + st, want_status, + "{label} CID status under recursive structural gate: root tree is denied \ + because its serialized bytes name the withheld /secret subtree, /public is served" ); + if want_status == StatusCode::OK { + assert_eq!( + served.as_deref(), + Some(want_oid.as_str()), + "{label}: the served object is the reachable tree" + ); + } } // Fail-closed preserved: the DENIED subtree's CID is still withheld — the diff --git a/crates/gl/src/ipfs_cmd.rs b/crates/gl/src/ipfs_cmd.rs index 93ca5511b..0490be096 100644 --- a/crates/gl/src/ipfs_cmd.rs +++ b/crates/gl/src/ipfs_cmd.rs @@ -119,23 +119,92 @@ async fn cmd_list(node: String, dir: Option) -> Result<()> { println!("IPFS pins ({count}) on {node}"); println!(); for pin in &pins { - let cid = pin["cid"].as_str().unwrap_or("?"); - let sha = pin["sha256_hex"].as_str().unwrap_or("?"); - let pinned_at = pin["pinned_at"].as_str().unwrap_or("?"); - // Trim pinned_at to date+time without subseconds - let ts = if pinned_at.len() >= 19 { - &pinned_at[..19] - } else { - pinned_at - }; - println!(" {cid}"); - println!(" sha256: {sha}"); - println!(" pinned: {ts}"); + for line in render_pin(pin) { + println!("{line}"); + } println!(); } Ok(()) } +/// The rendered lines for one pins-listing entry. +/// +/// #218 review round 8 P2 — why this is not just `pin["cid"]`: the wire format +/// gained a provenance split, and `cid` is now NULLABLE. The node stopped +/// aliasing a Pinata provider CID into `cid` (round 3) because `cid` is the +/// key `GET /ipfs/{cid}` resolves against, and a Pinata provider CID is not +/// resolvable there — advertising one would send clients to a guaranteed 404. +/// The truthful representation of a Pinata-only row is therefore `cid: null` +/// plus a `pinata_cid`. This function was reading only `cid`, so such a row +/// printed as a bare `?` — the CLI showed nothing usable for an object that is +/// in fact durably stored, and nothing in the client read `pinata_cid` at all. +/// +/// So: the heading is the node-resolvable CID when there is one, and otherwise +/// the Pinata provider CID, labelled as such so nobody feeds it back to this +/// node's resolver. A `backends:` line names where the bytes actually are, +/// taken from the writer-owned `local_pinned` / `pinata_pinned` booleans rather +/// than re-inferred from CID shape (which is what the node's own comment warns +/// against). Older nodes send neither flag; their `cid`-only rows fall back to +/// CID presence and render exactly as they did before. +fn render_pin(pin: &Value) -> Vec { + let sha = pin["sha256_hex"].as_str().unwrap_or("?"); + let pinned_at = pin["pinned_at"].as_str().unwrap_or("?"); + // Trim pinned_at to date+time without subseconds + let ts = if pinned_at.len() >= 19 { + &pinned_at[..19] + } else { + pinned_at + }; + + // `cid` and `local_cid` carry the same raw CID; read both so a node that + // ships only one of them still renders. + let local_cid = pin["cid"] + .as_str() + .or_else(|| pin["local_cid"].as_str()) + .filter(|s| !s.is_empty()); + let pinata_cid = pin["pinata_cid"].as_str().filter(|s| !s.is_empty()); + + // Writer-owned flags when present; CID presence is the pre-split fallback. + let local_pinned = pin["local_pinned"] + .as_bool() + .unwrap_or_else(|| local_cid.is_some()); + let pinata_pinned = pin["pinata_pinned"] + .as_bool() + .unwrap_or_else(|| pinata_cid.is_some()); + + let mut lines = Vec::new(); + match (local_cid, pinata_cid) { + // Node-resolvable CID present: it leads, as it always has. + (Some(cid), _) => lines.push(format!(" {cid}")), + // Pinata-only: show the provider CID rather than "?", and say plainly + // that this node's resolver will not serve it. + (None, Some(p)) => lines.push(format!(" {p} (pinata provider CID)")), + (None, None) => lines.push(" ?".to_string()), + } + lines.push(format!(" sha256: {sha}")); + lines.push(format!(" pinned: {ts}")); + + // A dual row keeps the provider CID visible too; it differs from the local + // CID and is the only key that resolves on Pinata's gateway. + if local_cid.is_some() { + if let Some(p) = pinata_cid { + lines.push(format!(" pinata: {p}")); + } + } + + let backends = match (local_pinned, pinata_pinned) { + (true, true) => "local ipfs, pinata", + (true, false) => "local ipfs", + (false, true) => "pinata", + // Neither flag set: the node filters such rows out, so this is only + // reachable from a hand-rolled response. Say so rather than imply + // durability the row does not claim. + (false, false) => "none recorded", + }; + lines.push(format!(" backends: {backends}")); + lines +} + /// Automatic resumes attempted after the initial request when the node reports a /// truncated legacy scan, so at most `MAX_SCAN_RESUMES + 1` node calls per invocation. const MAX_SCAN_RESUMES: usize = 8; @@ -629,6 +698,84 @@ mod tests { dir } + /// #218 review round 8 P2: a Pinata-only row must render its provider CID, + /// not a bare `?`. The node stopped aliasing that CID into `cid` because + /// `cid` is the key `GET /ipfs/{cid}` resolves on and a provider CID 404s + /// there — so `cid` is now legitimately null for such a row, and a client + /// that reads only `cid` shows the user nothing for an object that IS + /// durably stored. The heading must carry the provider CID and label it, + /// and the row must not claim a local pin. + #[test] + fn render_pin_shows_provider_cid_for_a_pinata_only_row() { + let pin: Value = serde_json::from_str( + r#"{"sha256_hex":"abc123","cid":null,"local_cid":null, + "pinata_cid":"QmProviderOnly","local_pinned":false, + "pinata_pinned":true,"pinned_at":"2026-07-02T12:00:00.123456Z"}"#, + ) + .unwrap(); + let out = render_pin(&pin).join("\n"); + assert!( + out.contains("QmProviderOnly"), + "the provider CID must be shown; got:\n{out}" + ); + assert!( + !out.contains(" ?"), + "a Pinata-only row must not render as a bare `?`; got:\n{out}" + ); + assert!( + out.contains("backends: pinata"), + "the row must say where the bytes are; got:\n{out}" + ); + assert!( + out.contains("pinata provider CID"), + "the heading must be labelled so it is not fed back to this node's \ + resolver, which cannot serve it; got:\n{out}" + ); + } + + /// A dual row leads with the node-resolvable CID (unchanged behavior) and + /// additionally surfaces the provider CID, which is a different key and the + /// only one Pinata's gateway answers for. + #[test] + fn render_pin_shows_both_cids_for_a_dual_row() { + let pin: Value = serde_json::from_str( + r#"{"sha256_hex":"abc123","cid":"bafyLocal","local_cid":"bafyLocal", + "pinata_cid":"QmProvider","local_pinned":true, + "pinata_pinned":true,"pinned_at":"2026-07-02T12:00:00.123456Z"}"#, + ) + .unwrap(); + let out = render_pin(&pin).join("\n"); + assert!( + out.starts_with(" bafyLocal"), + "the resolver key leads; got:\n{out}" + ); + assert!( + out.contains("pinata: QmProvider"), + "the provider CID must still be reachable from the listing; got:\n{out}" + ); + assert!(out.contains("backends: local ipfs, pinata"), "got:\n{out}"); + } + + /// Backward compatibility: a pre-provenance node sends `cid` alone, with no + /// `local_pinned` / `pinata_pinned` flags. That row must render as it always + /// did, with the backends line inferred from CID presence. + #[test] + fn render_pin_handles_a_legacy_cid_only_row() { + let pin: Value = serde_json::from_str( + r#"{"sha256_hex":"abc123","cid":"bafyone", + "pinned_at":"2026-07-02T12:00:00.123456Z"}"#, + ) + .unwrap(); + let out = render_pin(&pin).join("\n"); + assert!(out.starts_with(" bafyone"), "got:\n{out}"); + assert!(out.contains("sha256: abc123"), "got:\n{out}"); + assert!( + out.contains("pinned: 2026-07-02T12:00:00"), + "the timestamp is still trimmed to seconds; got:\n{out}" + ); + assert!(out.contains("backends: local ipfs"), "got:\n{out}"); + } + #[tokio::test] async fn test_cmd_list_signs_request_and_renders_pins() { let mut server = mockito::Server::new_async().await; diff --git a/docs/RUN-A-NODE.md b/docs/RUN-A-NODE.md index 7d2e2c83c..d655b7930 100644 --- a/docs/RUN-A-NODE.md +++ b/docs/RUN-A-NODE.md @@ -90,6 +90,7 @@ Required env for on-chain PoS mode: Optional: - `GITLAWB_OPERATOR_STRICT_MODE=true` — refuse to start if not registered or not currently active - `GITLAWB_HEARTBEAT_INTERVAL_HOURS=20` — how often to post heartbeats (must be < 24) +- `GITLAWB_RECONCILIATION_SWEEP=true` — enable the hourly durability sweep that re-pins/backstops missing objects (default `true`; disabled when no IPFS/Pinata backend is configured). Public pin repair runs against any configured backend. Encrypted recovery repair requires local IPFS (`GITLAWB_IPFS_API`); Pinata-only nodes reconcile public pins only. Set `=false` to disable. ## 5. Verify