From c0fcd1a9dd6259ee3125599a3ba0f1414854d422 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 12:42:53 -0700 Subject: [PATCH 001/111] fix(database): configure sqlite for concurrent writers (WAL, busy timeout, immediate write transactions, bounded retry) --- app/database/db.py | 173 +++++++++-- tests-unit/app_test/test_db_promotion.py | 64 ++++ tests-unit/app_test/test_db_write_txn.py | 378 +++++++++++++++++++++++ 3 files changed, 583 insertions(+), 32 deletions(-) create mode 100644 tests-unit/app_test/test_db_promotion.py create mode 100644 tests-unit/app_test/test_db_write_txn.py diff --git a/app/database/db.py b/app/database/db.py index 053df288332..51241c656ad 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -1,6 +1,10 @@ import logging import os +import random import shutil +import threading +import time +from typing import Callable, TypeVar from app.logger import log_startup_warning from utils.install_util import get_missing_requirements_message from filelock import FileLock, Timeout @@ -8,6 +12,13 @@ _DB_AVAILABLE = False Session = None +WriteSession = None +_attempt_lock_deadline = threading.local() +_write_txn_state = threading.local() +_WRITE_TXN_LOCK_RETRY_DEADLINE_SECONDS = 60 +_WRITE_TXN_BACKOFF_SECONDS = (0.05, 0.1, 0.2, 0.4) +_SQLITE_BUSY_TIMEOUT_MS = 30000 +T = TypeVar("T") try: @@ -16,7 +27,8 @@ from alembic.runtime.migration import MigrationContext from alembic.script import ScriptDirectory from sqlalchemy import create_engine, event - from sqlalchemy.orm import sessionmaker + from sqlalchemy.exc import OperationalError + from sqlalchemy.orm import Session as SQLAlchemySession, sessionmaker from sqlalchemy.pool import StaticPool from app.database.models import Base @@ -175,8 +187,10 @@ def set_sqlite_pragma(dbapi_connection, connection_record): Base.metadata.create_all(engine) - global Session + global Session, WriteSession Session = sessionmaker(bind=engine) + # A second engine would create a separate memory database; this test path is single-threaded. + WriteSession = Session def _init_file_db(db_url): @@ -207,49 +221,73 @@ def _upgrade_discards_the_catalog(script, target_rev, current_rev): ) +def _configure_runtime_connection(dbapi_connection, db_path): + cursor = dbapi_connection.cursor() + try: + cursor.execute("PRAGMA foreign_keys=ON") + journal_mode = cursor.execute("PRAGMA journal_mode=WAL").fetchone()[0] + if journal_mode.lower() != "wal": + raise RuntimeError( + f"SQLite WAL could not be enabled for database '{db_path}'. " + "SQLite WAL is not supported on network filesystems." + ) + cursor.execute(f"PRAGMA busy_timeout={_SQLITE_BUSY_TIMEOUT_MS}") + finally: + cursor.close() + + def _migrate_and_bind(db_url, db_path, db_exists): config = get_alembic_config() + inspection_engine = create_engine(db_url) - # Check if we need to upgrade - engine = create_engine(db_url) - - # Enable foreign key enforcement for SQLite - @event.listens_for(engine, "connect") + @event.listens_for(inspection_engine, "connect") def set_sqlite_pragma(dbapi_connection, connection_record): cursor = dbapi_connection.cursor() cursor.execute("PRAGMA foreign_keys=ON") cursor.close() - conn = engine.connect() - - context = MigrationContext.configure(conn) - current_rev = context.get_current_revision() - - script = ScriptDirectory.from_config(config) - target_rev = script.get_current_head() - - if target_rev is None: - logging.warning("No target revision found.") - elif current_rev != target_rev: - # Backup the database pre upgrade - backup_path = db_path + ".bkp" - if db_exists: + try: + with inspection_engine.connect() as inspection_connection: + context = MigrationContext.configure(inspection_connection) + current_rev = context.get_current_revision() + script = ScriptDirectory.from_config(config) + target_rev = script.get_current_head() + needs_upgrade = target_rev is not None and current_rev != target_rev + + if target_rev is None: + logging.warning("No target revision found.") + elif needs_upgrade and db_exists: + # WAL persists in the file, so Phase M makes the main-file backup self-contained. + inspection_connection.rollback() + inspection_connection.exec_driver_sql("PRAGMA wal_checkpoint(TRUNCATE)") + journal_mode = inspection_connection.exec_driver_sql( + "PRAGMA journal_mode=DELETE" + ).scalar_one() + if journal_mode.lower() != "delete": + raise RuntimeError( + f"SQLite journal mode could not be reset before backing up '{db_path}'." + ) + finally: + inspection_engine.dispose() + + if needs_upgrade: + backup_path = db_path + ".bkp" if db_exists else None + if backup_path is not None: shutil.copy(db_path, backup_path) - else: - backup_path = None - try: command.upgrade(config, target_rev) logging.info(f"Database upgraded from {current_rev} to {target_rev}") - except Exception as e: - if backup_path: - # Restore the database from backup if upgrade fails + except Exception: + if backup_path is not None: + for sidecar_path in (db_path + "-wal", db_path + "-shm"): + if os.path.exists(sidecar_path): + os.remove(sidecar_path) shutil.copy(backup_path, db_path) os.remove(backup_path) logging.exception("Error upgrading database: ") - raise e + raise - if backup_path and _upgrade_discards_the_catalog(script, target_rev, current_rev): + if backup_path is not None and _upgrade_discards_the_catalog(script, target_rev, current_rev): log_startup_warning( f"The asset catalog was rebuilt from scratch by migration " f"{_DESTRUCTIVE_REVISION}: manual tags, user metadata, previews, renames, " @@ -257,11 +295,82 @@ def set_sqlite_pragma(dbapi_connection, connection_record): f"discarded. The database from before the upgrade was kept at {backup_path}." ) - conn.close() + # Redundant with busy_timeout by design: both set pysqlite's 30-second limit. + reader_engine = create_engine(db_url, connect_args={"timeout": 30}) - global Session - Session = sessionmaker(bind=engine) + @event.listens_for(reader_engine, "connect") + def set_reader_sqlite_pragma(dbapi_connection, connection_record): + _configure_runtime_connection(dbapi_connection, db_path) + + writer_engine = create_engine(db_url, connect_args={"timeout": 30}) + + @event.listens_for(writer_engine, "connect") + def set_writer_sqlite_pragma(dbapi_connection, connection_record): + dbapi_connection.isolation_level = None + _configure_runtime_connection(dbapi_connection, db_path) + + @event.listens_for(writer_engine, "begin") + def begin_immediate(connection): + connection.exec_driver_sql("BEGIN IMMEDIATE") + + @event.listens_for(writer_engine, "checkout") + def cap_writer_busy_timeout(dbapi_connection, connection_record, connection_proxy): + deadline = getattr(_attempt_lock_deadline, "value", None) + timeout_ms = _SQLITE_BUSY_TIMEOUT_MS + if deadline is not None: + remaining_ms = int((deadline - time.monotonic()) * 1000) + timeout_ms = max(1, min(_SQLITE_BUSY_TIMEOUT_MS, remaining_ms)) + dbapi_connection.execute(f"PRAGMA busy_timeout = {int(timeout_ms)}") + + with reader_engine.connect(): + pass + with writer_engine.connect(): + pass + global Session, WriteSession + Session = sessionmaker(bind=reader_engine) + WriteSession = sessionmaker(bind=writer_engine) def create_session(): return Session() + + +def run_write_txn(work: Callable[["SQLAlchemySession"], T]) -> T: + """Run a write callback with bounded lock retries; its own work is not deadline-limited.""" + if getattr(_write_txn_state, "active", False): + raise RuntimeError("run_write_txn cannot be nested") + + # Nested helpers could commit the outer callback's work. + _write_txn_state.active = True + retry_deadline = time.monotonic() + _WRITE_TXN_LOCK_RETRY_DEADLINE_SECONDS + locked_error = None + try: + for attempt in range(len(_WRITE_TXN_BACKOFF_SECONDS) + 1): + if attempt > 0: + if time.monotonic() >= retry_deadline: + raise locked_error + backoff_seconds = _WRITE_TXN_BACKOFF_SECONDS[attempt - 1] + time.sleep(random.uniform(backoff_seconds * 0.5, backoff_seconds * 1.5)) + if time.monotonic() >= retry_deadline: + raise locked_error + + _attempt_lock_deadline.value = retry_deadline + session = WriteSession() + try: + result = work(session) + session.commit() + return result + except OperationalError as exc: + if "locked" not in str(exc.orig): + raise + locked_error = exc + finally: + session.rollback() + session.close() + _attempt_lock_deadline.value = None + + if attempt == len(_WRITE_TXN_BACKOFF_SECONDS) or time.monotonic() >= retry_deadline: + raise locked_error + finally: + _attempt_lock_deadline.value = None + _write_txn_state.active = False diff --git a/tests-unit/app_test/test_db_promotion.py b/tests-unit/app_test/test_db_promotion.py new file mode 100644 index 00000000000..0e70ebbdc06 --- /dev/null +++ b/tests-unit/app_test/test_db_promotion.py @@ -0,0 +1,64 @@ +import sqlite3 +import threading +import time + +from sqlalchemy import event, text + +import app.database.db as db_mod +from app.database.db import create_session + + +def _invoke_writer(work): + def _legacy(legacy_work): + with create_session() as session: + result = legacy_work(session) + session.commit() + return result + + return db_mod.run_write_txn(work) if hasattr(db_mod, "run_write_txn") else _legacy(work) + + +def test_write_transaction_waits_for_held_writer_before_select_then_mutate(tmp_path, monkeypatch): + database_path = tmp_path / "assets.db" + monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") + monkeypatch.setattr(db_mod, "Session", None) + monkeypatch.setattr(db_mod, "_db_lock", None) + db_mod.init_db() + + reader_engine = db_mod.Session.kw["bind"] + + def begin_deferred(connection): + connection.exec_driver_sql("BEGIN") + + event.listen(reader_engine, "begin", begin_deferred) + writer_started = threading.Event() + + def hold_write_lock(): + with sqlite3.connect(database_path) as connection: + connection.execute("BEGIN IMMEDIATE") + connection.execute("INSERT INTO tags (name) VALUES (?)", ("promotion-holder",)) + writer_started.set() + time.sleep(2) + connection.rollback() + + holder = threading.Thread(target=hold_write_lock) + holder.start() + try: + assert writer_started.wait(timeout=5) + + def select_then_mutate(session): + session.execute(text("SELECT name FROM tags LIMIT 1")) + session.execute(text("INSERT INTO tags (name) VALUES (:name)"), {"name": "promotion-work"}) + return "written" + + started_at = time.monotonic() + result = _invoke_writer(select_then_mutate) + elapsed = time.monotonic() - started_at + finally: + holder.join(timeout=5) + event.remove(reader_engine, "begin", begin_deferred) + reader_engine.dispose() + db_mod._db_lock.release(force=True) + + assert result == "written" + assert elapsed >= 2 diff --git a/tests-unit/app_test/test_db_write_txn.py b/tests-unit/app_test/test_db_write_txn.py new file mode 100644 index 00000000000..6cc2f492332 --- /dev/null +++ b/tests-unit/app_test/test_db_write_txn.py @@ -0,0 +1,378 @@ +import os +import shutil +import sqlite3 +import threading +import time + +import pytest +from alembic import command +from alembic.config import Config +from sqlalchemy import event, text +from sqlalchemy.exc import OperationalError + +import app.database.db as db_mod + + +_PRE_HEAD = "0006_add_loader_path" +_WAL_SENTINEL = "wal-resident-sentinel" + + +def _dispose_runtime_engines(): + for session_factory in (db_mod.Session, getattr(db_mod, "WriteSession", None)): + if session_factory is not None: + session_factory.kw["bind"].dispose() + + +@pytest.fixture +def file_database(tmp_path, monkeypatch): + database_path = str(tmp_path / "assets.db") + monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") + monkeypatch.setattr(db_mod, "Session", None) + monkeypatch.setattr(db_mod, "_db_lock", None) + if hasattr(db_mod, "WriteSession"): + monkeypatch.setattr(db_mod, "WriteSession", None) + db_mod.init_db() + yield database_path + _dispose_runtime_engines() + db_mod._db_lock.release(force=True) + + +@pytest.fixture +def memory_database(monkeypatch): + monkeypatch.setattr(db_mod.args, "database_url", "sqlite:///:memory:") + monkeypatch.setattr(db_mod, "Session", None) + if hasattr(db_mod, "WriteSession"): + monkeypatch.setattr(db_mod, "WriteSession", None) + db_mod.init_db() + yield + _dispose_runtime_engines() + + +def _make_config(db_path: str) -> Config: + root = os.path.join(os.path.dirname(__file__), "../..") + config = Config(os.path.abspath(os.path.join(root, "alembic.ini"))) + config.set_main_option("script_location", os.path.abspath(os.path.join(root, "alembic_db"))) + config.set_main_option("sqlalchemy.url", f"sqlite:///{db_path}") + return config + + +def _crash_style_wal_database(tmp_path) -> str: + source_path = str(tmp_path / "source.db") + target_path = str(tmp_path / "crash.db") + command.upgrade(_make_config(source_path), _PRE_HEAD) + + writer = sqlite3.connect(source_path) + reader = sqlite3.connect(source_path) + try: + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("PRAGMA wal_autocheckpoint=0") + reader.execute("BEGIN") + reader.execute("SELECT name FROM tags LIMIT 1").fetchone() + writer.execute("INSERT INTO tags (name) VALUES (?)", (_WAL_SENTINEL,)) + writer.commit() + shutil.copy(source_path, target_path) + shutil.copy(source_path + "-wal", target_path + "-wal") + finally: + writer.close() + reader.close() + + assert os.path.getsize(target_path + "-wal") > 0 + return target_path + + +def _migrate_crash_style_database(database_path: str, monkeypatch) -> None: + monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") + monkeypatch.setattr(db_mod, "Session", None) + if hasattr(db_mod, "WriteSession"): + monkeypatch.setattr(db_mod, "WriteSession", None) + db_mod._migrate_and_bind(f"sqlite:///{database_path}", database_path, db_exists=True) + + +def test_file_database_configures_runtime_pragmas(file_database): + with db_mod.create_session() as session: + pragmas = ( + session.execute(text("PRAGMA journal_mode")).scalar_one(), + session.execute(text("PRAGMA busy_timeout")).scalar_one(), + session.execute(text("PRAGMA foreign_keys")).scalar_one(), + ) + + assert pragmas == ("wal", 30000, 1) + + +def test_runtime_connection_rejects_non_wal_journal_mode(): + class Cursor: + def execute(self, statement): + self.statement = statement + return self + + def fetchone(self): + return ("delete",) + + def close(self): + return None + + class Connection: + def __init__(self): + self.cursor_instance = Cursor() + + def cursor(self): + return self.cursor_instance + + with pytest.raises(RuntimeError, match="network filesystems"): + db_mod._configure_runtime_connection(Connection(), "network.db") + + +def test_run_write_txn_passes_through_successful_result(memory_database): + run_write_txn = db_mod.run_write_txn + + assert run_write_txn(lambda _session: "written") == "written" + + +def test_run_write_txn_retries_locked_operational_errors_then_succeeds( + memory_database, monkeypatch +): + run_write_txn = db_mod.run_write_txn + monkeypatch.setattr(db_mod.time, "sleep", lambda _seconds: None) + attempts = 0 + + def work(_session): + nonlocal attempts + attempts += 1 + if attempts <= 2: + raise OperationalError("SELECT 1", {}, sqlite3.OperationalError("database is locked")) + return "written" + + assert run_write_txn(work) == "written" + assert attempts == 3 + + +def test_run_write_txn_reraises_nonretryable_operational_error_without_retry(memory_database): + run_write_txn = db_mod.run_write_txn + attempts = 0 + + def work(_session): + nonlocal attempts + attempts += 1 + raise OperationalError("INSERT", {}, sqlite3.OperationalError("readonly database")) + + with pytest.raises(OperationalError, match="readonly database"): + run_write_txn(work) + + assert attempts == 1 + + +def test_run_write_txn_reraises_non_operational_error_without_retry(memory_database): + run_write_txn = db_mod.run_write_txn + attempts = 0 + + def work(_session): + nonlocal attempts + attempts += 1 + raise ValueError("body failure") + + with pytest.raises(ValueError, match="body failure"): + run_write_txn(work) + + assert attempts == 1 + + +def test_run_write_txn_reraises_terminal_locked_error_after_five_attempts( + memory_database, monkeypatch +): + run_write_txn = db_mod.run_write_txn + monkeypatch.setattr(db_mod.time, "sleep", lambda _seconds: None) + attempts = 0 + + def work(_session): + nonlocal attempts + attempts += 1 + raise OperationalError("INSERT", {}, sqlite3.OperationalError("database is locked")) + + with pytest.raises(OperationalError, match="database is locked"): + run_write_txn(work) + + assert attempts == 5 + + +def test_run_write_txn_deadline_gates_attempt_starts(memory_database, monkeypatch): + run_write_txn = db_mod.run_write_txn + clock_calls = 0 + + def monotonic(): + nonlocal clock_calls + clock_calls += 1 + return 0.0 if clock_calls == 1 else 60.1 + + monkeypatch.setattr(db_mod.time, "monotonic", monotonic) + monkeypatch.setattr(db_mod.time, "sleep", lambda _seconds: None) + attempts = 0 + + def work(_session): + nonlocal attempts + attempts += 1 + raise OperationalError("INSERT", {}, sqlite3.OperationalError("database is locked")) + + with pytest.raises(OperationalError, match="database is locked"): + run_write_txn(work) + + assert attempts == 1 + + +def test_run_write_txn_held_lock_respects_remaining_deadline(file_database, monkeypatch): + run_write_txn = db_mod.run_write_txn + monkeypatch.setattr(db_mod, "_WRITE_TXN_LOCK_RETRY_DEADLINE_SECONDS", 2) + writer_started = threading.Event() + + def hold_lock(): + with sqlite3.connect(file_database) as holder: + holder.execute("BEGIN IMMEDIATE") + holder.execute("INSERT INTO tags (name) VALUES (?)", ("deadline-holder",)) + writer_started.set() + time.sleep(5) + holder.rollback() + + holder = threading.Thread(target=hold_lock) + holder.start() + try: + assert writer_started.wait(timeout=5) + started_at = time.monotonic() + with pytest.raises(OperationalError, match="database is locked"): + run_write_txn( + lambda session: session.execute(text("INSERT INTO tags (name) VALUES ('blocked')")) + ) + elapsed = time.monotonic() - started_at + finally: + holder.join(timeout=6) + + assert not holder.is_alive() + assert 1.5 <= elapsed < 3 + + +def test_run_write_txn_reduces_busy_timeout_after_intermediate_commit( + file_database, monkeypatch +): + run_write_txn = db_mod.run_write_txn + clock = {"now": 0.0} + monkeypatch.setattr(db_mod.time, "monotonic", lambda: clock["now"]) + seen_timeouts = [] + + def work(session): + seen_timeouts.append(session.execute(text("PRAGMA busy_timeout")).scalar_one()) + session.commit() + clock["now"] = 31.0 + seen_timeouts.append(session.execute(text("PRAGMA busy_timeout")).scalar_one()) + + run_write_txn(work) + + assert seen_timeouts[1] < seen_timeouts[0] + + +def test_run_write_txn_uses_a_fresh_session_for_each_attempt(memory_database, monkeypatch): + run_write_txn = db_mod.run_write_txn + monkeypatch.setattr(db_mod.time, "sleep", lambda _seconds: None) + sessions = [] + + def work(session): + sessions.append(session) + if len(sessions) < 3: + raise OperationalError("INSERT", {}, sqlite3.OperationalError("database is locked")) + return "written" + + assert run_write_txn(work) == "written" + assert sessions[0] is not sessions[1] + assert sessions[1] is not sessions[2] + + +def test_run_write_txn_reentrancy_guard_resets_after_failure(memory_database): + run_write_txn = db_mod.run_write_txn + + def nested_work(_session): + return run_write_txn(lambda _nested_session: None) + + with pytest.raises(RuntimeError): + run_write_txn(nested_work) + + assert run_write_txn(lambda _session: "reset") == "reset" + + +def test_memory_database_uses_degraded_write_transaction_wiring(memory_database): + run_write_txn = db_mod.run_write_txn + + assert db_mod.WriteSession is db_mod.Session + assert run_write_txn(lambda _session: "written") == "written" + + +def test_migration_backup_checkpoints_crash_style_wal_before_copy(tmp_path, monkeypatch): + database_path = _crash_style_wal_database(tmp_path) + _migrate_crash_style_database(database_path, monkeypatch) + try: + with sqlite3.connect(database_path + ".bkp") as backup: + rows = backup.execute("SELECT name FROM tags WHERE name = ?", (_WAL_SENTINEL,)).fetchall() + finally: + _dispose_runtime_engines() + + assert rows == [(_WAL_SENTINEL,)] + + +def test_failed_migration_restores_crash_style_wal_backup_and_removes_sidecars( + tmp_path, monkeypatch +): + database_path = _crash_style_wal_database(tmp_path) + + def fail_upgrade(_config, _target_revision): + with open(database_path + "-wal", "wb") as wal_file: + wal_file.write(b"stale wal") + with open(database_path + "-shm", "wb") as shm_file: + shm_file.write(b"stale shm") + raise RuntimeError("upgrade failure") + + monkeypatch.setattr(db_mod.command, "upgrade", fail_upgrade) + monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") + monkeypatch.setattr(db_mod, "Session", None) + if hasattr(db_mod, "WriteSession"): + monkeypatch.setattr(db_mod, "WriteSession", None) + + with pytest.raises(RuntimeError, match="upgrade failure"): + db_mod._migrate_and_bind(f"sqlite:///{database_path}", database_path, db_exists=True) + + with sqlite3.connect(database_path) as restored: + rows = restored.execute("SELECT name FROM tags WHERE name = ?", (_WAL_SENTINEL,)).fetchall() + + assert rows == [(_WAL_SENTINEL,)] + assert not os.path.exists(database_path + "-wal") + assert not os.path.exists(database_path + "-shm") + + +def test_migration_uses_wal_only_for_runtime_engines(tmp_path, monkeypatch): + database_path = _crash_style_wal_database(tmp_path) + created_engine_commands = [] + original_create_engine = db_mod.create_engine + + def create_traced_engine(*args, **kwargs): + engine = original_create_engine(*args, **kwargs) + commands = [] + + def trace_connection(dbapi_connection, _connection_record): + dbapi_connection.set_trace_callback(commands.append) + + event.listen(engine, "connect", trace_connection, insert=True) + created_engine_commands.append(commands) + return engine + + monkeypatch.setattr(db_mod, "create_engine", create_traced_engine) + _migrate_crash_style_database(database_path, monkeypatch) + try: + with db_mod.create_session() as reader: + assert reader.execute(text("PRAGMA journal_mode")).scalar_one() == "wal" + with db_mod.WriteSession() as writer: + assert writer.execute(text("PRAGMA journal_mode")).scalar_one() == "wal" + finally: + _dispose_runtime_engines() + + inspection_commands = "\n".join(created_engine_commands[0]).lower() + runtime_commands = "\n".join( + command_text for commands in created_engine_commands[1:] for command_text in commands + ).lower() + assert "journal_mode=delete" in inspection_commands + assert "journal_mode=wal" not in inspection_commands + assert "journal_mode=wal" in runtime_commands From c75c5ea1d05a7c765f8c013cdafab0501bbdb492 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 13:25:54 -0700 Subject: [PATCH 002/111] fix(assets): route ingest write paths through immediate write transactions --- app/assets/event_log.py | 2 + app/assets/services/ingest.py | 857 +++++++++++++----- .../fixtures/assets_event_lines.txt | 1 + tests-unit/assets_test/services/conftest.py | 5 +- .../services/test_ingest_write_txn.py | 144 +++ .../assets_test/services/test_lifecycle.py | 5 +- .../services/test_manager_isolated.py | 3 +- .../services/test_stored_hash_prefix.py | 3 +- .../services/test_transition_settle.py | 5 +- .../assets_test/services/test_upload_b.py | 10 +- tests-unit/assets_test/test_event_log.py | 5 +- .../assets_test/test_intended_behaviour.py | 5 +- .../assets_test/test_upload_hashing_modes.py | 3 +- tests-unit/test_assets_event_log_static.py | 2 + 14 files changed, 814 insertions(+), 236 deletions(-) create mode 100644 tests-unit/assets_test/services/test_ingest_write_txn.py diff --git a/app/assets/event_log.py b/app/assets/event_log.py index e8962a3ad46..479cdd94b6b 100644 --- a/app/assets/event_log.py +++ b/app/assets/event_log.py @@ -43,6 +43,7 @@ "scanner.temp_sync_failed", "scanner.mark_missing_failed", "scanner.stat_failed", + "ingest.register_failed", }) @@ -89,6 +90,7 @@ def _is_flag(value: Any) -> bool: "error_type": _is_safe_string, "hashing_enabled": _is_flag, "site": _one_of(STAT_SITES), + "output_kind": _one_of(frozenset({"executed", "cached"})), } _warned_call_sites: set[tuple[str, int]] = set() diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index a8fb3b81de7..866b09d3cc0 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -11,7 +11,7 @@ import logging import mimetypes import os -from typing import Any, NamedTuple, Sequence +from typing import Any, NamedTuple from sqlalchemy import func, select from sqlalchemy.orm import Session @@ -23,12 +23,12 @@ create_record, mark_content_missing, ) +from app.assets.event_log import emit, error_type from app.assets.helpers import normalize_tags, to_stored_hash from app.assets.services.file_utils import get_mtime_ns, get_size_and_mtime_ns from app.assets.services.image_dimensions import extract_image_dimensions from app.assets.services.lookup import ( claim_qualified_content, - lookup_for_from_hash, lookup_for_view, refresh_qualified_content, ) @@ -47,7 +47,7 @@ UserMetadata, ) from app.assets.services.snapshot_hash import snapshot_hash -from app.database.db import create_session +from app.database.db import create_session, run_write_txn def _normalize_hash_input(hash_str: str) -> str: @@ -194,28 +194,23 @@ def _move_temp_to_dest(temp_path: str, dest_abs: str) -> None: def _create_upload_record( session: Session, content_id: str, - name: str, - abs_path: str, - tags: Sequence[str], - mime_type: str | None, - user_metadata: UserMetadata, - preview_id: str | None, + prepared: "_PreparedUploadRecord", ) -> Asset: - if preview_id is not None and session.get(Asset, preview_id) is None: - raise ValueError(f"preview_id {preview_id!r} does not reference an existing asset") + preflight = prepared.preflight + spec = preflight.spec record = create_record( session, content_id, - name, - mime_type=mime_type, - loader_path=compute_loader_path(abs_path), - tags=list(tags), - system_metadata=_extract_system_metadata_sync(abs_path, mime_type), + spec.name, + mime_type=spec.mime_type, + loader_path=compute_loader_path(preflight.path), + tags=spec.tags, + system_metadata=prepared.system_metadata, ) - if user_metadata: - record.user_metadata = dict(user_metadata) - if preview_id: - record.preview_id = preview_id + if spec.user_metadata: + record.user_metadata = dict(spec.user_metadata) + if spec.preview_id: + record.preview_id = spec.preview_id session.flush() return record @@ -344,20 +339,298 @@ def _upload_destination_or_none( return None -def _settle_destination_before_write(session: Session, dest_abs: str) -> None: - """Resolve the live row at an upload destination while its bytes still exist. +class _UploadRecordSpec(NamedTuple): + + name: str + tags: list[str] + mime_type: str | None + user_metadata: UserMetadata + preview_id: str | None + + +class _FileSignature(NamedTuple): - ``upload_from_temp_path`` is about to replace whatever sits at ``dest_abs``, - and that write destroys the only evidence of what the incumbent row was - created for. So reconcile against the INCUMBENT file's own hash first: an - unhashed row learns the hash of the bytes it actually describes, and a row - whose hash no longer matches the file is retired. Both outcomes leave the - post-move reconciliation with a known hash to compare, so it never has to - read equal sizes as equal bytes. + path: str + size_bytes: int + mtime_ns: int - Running before ``lookup_for_view`` is what keeps a merely stale row from - being skipped as stat-inconsistent and duplicated. - """ + +class _UploadRecordPreflight(NamedTuple): + + content_id: str | None + stored_hash: str | None + path: str + signature: _FileSignature + spec: _UploadRecordSpec + + +class _PreparedUploadRecord(NamedTuple): + + preflight: _UploadRecordPreflight + system_metadata: dict[str, Any] + + +class _SettleTargetPreflight(NamedTuple): + + content_id: str + content_hash: str | None + content_size_bytes: int + content_mtime_ns: int | None + signature: _FileSignature + + +class _PreparedSettleTarget(NamedTuple): + + preflight: _SettleTargetPreflight + facts: _ContentFacts | None + + +class _CachedRegistrationPreflight(NamedTuple): + + content_id: str + sibling_id: str | None + sibling_metadata: dict[str, Any] | None + signature: _FileSignature | None + + +class _PreflightStale(Exception): + pass + + +def _file_signature(path: str) -> _FileSignature: + stat_result = os.stat(path, follow_symlinks=True) + return _FileSignature(path, stat_result.st_size, get_mtime_ns(stat_result)) + + +def _file_signature_matches(signature: _FileSignature) -> bool: + try: + return _file_signature(signature.path) == signature + except OSError: + return False + + +def _preflight_upload_record( + stored_hash: str | None, + fallback_path: str | None, + spec: _UploadRecordSpec, +) -> _UploadRecordPreflight | None: + """Check the target before metadata I/O so it does not extend the writer lease.""" + with create_session() as session: + content = lookup_for_view(session, stored_hash) if stored_hash else None + if content is None: + if fallback_path is None: + return None + content_id = None + path = fallback_path + else: + content_id = content.id + path = content.path + if spec.preview_id is not None and session.get(Asset, spec.preview_id) is None: + raise ValueError( + f"preview_id {spec.preview_id!r} does not reference an existing asset" + ) + return _UploadRecordPreflight( + content_id, + stored_hash, + path, + _file_signature(path), + spec, + ) + + +def _prepare_upload_record( + preflight: _UploadRecordPreflight, +) -> _PreparedUploadRecord: + return _PreparedUploadRecord( + preflight, + _extract_system_metadata_sync( + preflight.path, + preflight.spec.mime_type, + ), + ) + + +def _assert_upload_preflight_current( + session: Session, + preflight: _UploadRecordPreflight, +) -> None: + if not _file_signature_matches(preflight.signature): + raise _PreflightStale + preview_id = preflight.spec.preview_id + if preview_id is not None and session.get(Asset, preview_id) is None: + raise _PreflightStale + + +def _create_upload_record_in_txn( + session: Session, + content_id: str, + spec: _UploadRecordSpec, + abs_path: str, +) -> Asset: + if spec.preview_id is not None and session.get(Asset, spec.preview_id) is None: + raise ValueError( + f"preview_id {spec.preview_id!r} does not reference an existing asset" + ) + record = create_record( + session, + content_id, + spec.name, + mime_type=spec.mime_type, + loader_path=compute_loader_path(abs_path), + tags=spec.tags, + system_metadata=_extract_system_metadata_sync(abs_path, spec.mime_type), + ) + if spec.user_metadata: + record.user_metadata = dict(spec.user_metadata) + if spec.preview_id: + record.preview_id = spec.preview_id + session.flush() + return record + + +def _apply_reused_upload_record( + session: Session, + prepared: _PreparedUploadRecord, +) -> UploadResult: + preflight = prepared.preflight + content = lookup_for_view(session, preflight.stored_hash) + if ( + content is None + or content.id != preflight.content_id + or content.path != preflight.path + ): + raise _PreflightStale + _assert_upload_preflight_current(session, preflight) + if not claim_qualified_content(session, content.id, preflight.stored_hash): + raise _PreflightStale + content = refresh_qualified_content(session, content.id) + if content is None or content.path != preflight.path: + raise _PreflightStale + record = _create_upload_record(session, content.id, prepared) + return _record_to_upload_result(session, record, created_new=True) + + +def _reuse_qualified_content_in_txn( + session: Session, + stored_hash: str, + spec: _UploadRecordSpec, +) -> UploadResult | None: + content = lookup_for_view(session, stored_hash) + if content is None: + return None + if not claim_qualified_content(session, content.id, stored_hash): + session.rollback() + return None + content = refresh_qualified_content(session, content.id) + if content is None: + session.rollback() + return None + record = _create_upload_record_in_txn(session, content.id, spec, content.path) + return _record_to_upload_result(session, record, created_new=True) + + +def _reuse_qualified_content( + stored_hash: str, + spec: _UploadRecordSpec, +) -> UploadResult | None: + for restart in range(4): + preflight = _preflight_upload_record(stored_hash, None, spec) + if preflight is None: + return None + prepared = _prepare_upload_record(preflight) + try: + return run_write_txn( + lambda session: _apply_reused_upload_record(session, prepared) + ) + except _PreflightStale: + if restart == 3: + logging.warning( + "Upload preflight changed three times; falling back to in-transaction metadata extraction" + ) + return run_write_txn( + lambda session: _reuse_qualified_content_in_txn( + session, + stored_hash, + spec, + ) + ) + return None + + +def _preflight_settle_target(dest_abs: str) -> _SettleTargetPreflight | None: + """Read incumbent facts so its hash runs before, not inside, the writer lease.""" + if not os.path.isfile(dest_abs): + return None + with create_session() as session: + existing = session.scalars( + select(AssetContent).where( + AssetContent.path == dest_abs, + AssetContent.is_missing.is_(False), + ) + ).first() + if existing is None: + return None + signature = _file_signature(dest_abs) + if ( + existing.hash is not None + and existing.size_bytes == signature.size_bytes + and existing.mtime_ns == signature.mtime_ns + ): + return None + return _SettleTargetPreflight( + existing.id, + existing.hash, + existing.size_bytes, + existing.mtime_ns, + signature, + ) + + +def _prepare_settle_target( + preflight: _SettleTargetPreflight, +) -> _PreparedSettleTarget: + try: + digest, verified_stat = _snapshot_hash_with_retry(preflight.signature.path) + except (UploadUnstableError, OSError): + return _PreparedSettleTarget(preflight, None) + return _PreparedSettleTarget( + preflight, + _ContentFacts( + to_stored_hash(digest), + verified_stat.st_size, + verified_stat.st_mtime_ns, + ), + ) + + +def _apply_settle_target( + session: Session, + prepared: _PreparedSettleTarget, +) -> None: + preflight = prepared.preflight + existing = session.get(AssetContent, preflight.content_id) + if ( + existing is None + or existing.path != preflight.signature.path + or existing.is_missing + or existing.hash != preflight.content_hash + or existing.size_bytes != preflight.content_size_bytes + or existing.mtime_ns != preflight.content_mtime_ns + or not _file_signature_matches(preflight.signature) + ): + raise _PreflightStale + if prepared.facts is None: + mark_content_missing(session, existing.id) + return + _reconcile_live_content_at_path( + session, + preflight.signature.path, + prepared.facts, + content_written=False, + ) + + +def _settle_destination_before_write_in_txn(session: Session, dest_abs: str) -> None: if not os.path.isfile(dest_abs): return existing = session.scalars( @@ -392,41 +665,106 @@ def _settle_destination_before_write(session: Session, dest_abs: str) -> None: ) -class _UploadRecordSpec(NamedTuple): +def _settle_destination_before_write(dest_abs: str) -> None: + for restart in range(4): + preflight = _preflight_settle_target(dest_abs) + if preflight is None: + return + prepared = _prepare_settle_target(preflight) + try: + run_write_txn(lambda session: _apply_settle_target(session, prepared)) + return + except _PreflightStale: + if restart == 3: + logging.warning( + "Upload destination preflight changed three times; falling back to in-transaction hashing" + ) + run_write_txn( + lambda session: _settle_destination_before_write_in_txn( + session, + dest_abs, + ) + ) + return - name: str - tags: list[str] - mime_type: str | None - user_metadata: UserMetadata - preview_id: str | None +def _create_content_and_upload_record( + stored_hash: str, + path: str, + facts: _ContentFacts, + content_written: bool, + spec: _UploadRecordSpec, +) -> UploadResult: + for restart in range(4): + preflight = _preflight_upload_record(None, path, spec) + if preflight is None: + raise RuntimeError("new upload record requires a destination path") + prepared = _prepare_upload_record(preflight) + + def _work(session: Session) -> UploadResult: + _assert_upload_preflight_current(session, prepared.preflight) + _reconcile_live_content_at_path( + session, + path, + facts, + content_written=content_written, + ) + content, inserted = create_content_reporting_insert( + session, + path, + stored_hash, + facts.size_bytes, + facts.mtime_ns, + ) + created_content_id = content.id if inserted else None + try: + record = _create_upload_record(session, content.id, prepared) + except Exception: + session.rollback() + if created_content_id is not None: + _discard_unreferenced_content(session, created_content_id) + raise + return _record_to_upload_result(session, record, created_new=True) + + try: + return run_write_txn(_work) + except _PreflightStale: + if restart == 3: + logging.warning( + "Upload record preflight changed three times; falling back to in-transaction metadata extraction" + ) -def _reuse_qualified_content( - session: Session, stored_hash: str, spec: _UploadRecordSpec -) -> UploadResult | None: - content = lookup_for_view(session, stored_hash) - if content is None: - return None - content_id = content.id - if not claim_qualified_content(session, content_id, stored_hash): - session.rollback() - return None - content = refresh_qualified_content(session, content_id) - if content is None: - session.rollback() - return None - record = _create_upload_record( - session, - content_id, - spec.name, - content.path, - spec.tags, - spec.mime_type, - spec.user_metadata, - spec.preview_id, - ) - session.commit() - return _record_to_upload_result(session, record, created_new=True) + def _fallback_work(session: Session) -> UploadResult: + _reconcile_live_content_at_path( + session, + path, + facts, + content_written=content_written, + ) + content, inserted = create_content_reporting_insert( + session, + path, + stored_hash, + facts.size_bytes, + facts.mtime_ns, + ) + created_content_id = content.id if inserted else None + try: + record = _create_upload_record_in_txn( + session, + content.id, + spec, + path, + ) + except Exception: + session.rollback() + if created_content_id is not None: + _discard_unreferenced_content(session, created_content_id) + raise + return _record_to_upload_result(session, record, created_new=True) + + return run_write_txn(_fallback_work) + raise RuntimeError("upload record preflight retry loop did not return") def upload_from_temp_path( @@ -452,22 +790,17 @@ def upload_from_temp_path( _remove_temp_path(temp_path) raise HashMismatchError("Uploaded file hash does not match provided hash.") + spec = _UploadRecordSpec( + display_name, + normalize_tags([*(tags or []), "uploaded"]), + mime_type, + user_metadata, + preview_id, + ) settle_target = _upload_destination_or_none(tags, digest, client_filename, name) - with create_session() as session: - if settle_target is not None: - _settle_destination_before_write(session, settle_target) - session.commit() - reused = _reuse_qualified_content( - session, - stored_hash, - _UploadRecordSpec( - display_name, - normalize_tags([*(tags or []), "uploaded"]), - mime_type, - user_metadata, - preview_id, - ), - ) + if settle_target is not None: + _settle_destination_before_write(settle_target) + reused = _reuse_qualified_content(stored_hash, spec) if reused is not None: _remove_temp_path(temp_path) return reused @@ -481,36 +814,23 @@ def upload_from_temp_path( mime_type, client_filename, name, os.path.basename(dest_abs) ) _move_temp_to_dest(temp_path, dest_abs) - size_bytes, mtime_ns = verified_stat.st_size, verified_stat.st_mtime_ns - with create_session() as session: - _reconcile_live_content_at_path( - session, - dest_abs, - _ContentFacts(stored_hash, size_bytes, mtime_ns), - content_written=True, - ) - content, inserted = create_content_reporting_insert( - session, dest_abs, stored_hash, size_bytes, mtime_ns - ) - created_content_id = content.id if inserted else None - try: - record = _create_upload_record( - session, - content.id, - display_name, - dest_abs, - normalize_tags([*(tags or []), "uploaded"]), - content_type, - user_metadata, - preview_id, - ) - session.commit() - except Exception: - session.rollback() - if created_content_id is not None: - _discard_unreferenced_content(session, created_content_id) - raise - return _record_to_upload_result(session, record, created_new=True) + return _create_content_and_upload_record( + stored_hash, + dest_abs, + _ContentFacts( + stored_hash, + verified_stat.st_size, + verified_stat.st_mtime_ns, + ), + True, + _UploadRecordSpec( + display_name, + normalize_tags([*(tags or []), "uploaded"]), + content_type, + user_metadata, + preview_id, + ), + ) def register_file_in_place( @@ -560,38 +880,27 @@ def register_file_in_place( digest, verified_stat = _snapshot_hash_with_retry(locator) size_bytes, mtime_ns = verified_stat.st_size, verified_stat.st_mtime_ns stored_hash = to_stored_hash(digest) - with create_session() as session: + def _reconcile_work(session: Session) -> None: _reconcile_live_content_at_path( session, locator, _ContentFacts(stored_hash, size_bytes, mtime_ns), content_written=content_written, ) - session.commit() - - with create_session() as session: - content, inserted = create_content_reporting_insert( - session, locator, stored_hash, size_bytes, mtime_ns - ) - created_content_id = content.id if inserted else None - try: - record = _create_upload_record( - session, - content.id, - display_name, - locator, - merged_tags, - content_type, - None, - None, - ) - session.commit() - except Exception: - session.rollback() - if created_content_id is not None: - _discard_unreferenced_content(session, created_content_id) - raise - return _record_to_upload_result(session, record, created_new=True) + run_write_txn(_reconcile_work) + return _create_content_and_upload_record( + stored_hash, + locator, + _ContentFacts(stored_hash, size_bytes, mtime_ns), + content_written, + _UploadRecordSpec( + display_name, + merged_tags, + content_type, + {}, + None, + ), + ) def create_from_hash( @@ -611,31 +920,154 @@ def create_from_hash( name, fallback=bare_digest ) - with create_session() as session: - content = lookup_for_from_hash(session, stored_hash) - if content is None: - logging.warning("create_from_hash: no asset found for hash %s", hash_str) - return None - content_id = content.id - if not claim_qualified_content(session, content_id, stored_hash): - session.rollback() - return None - content = refresh_qualified_content(session, content_id) - if content is None: - session.rollback() - return None - record = _create_upload_record( - session, - content_id, + result = _reuse_qualified_content( + stored_hash, + _UploadRecordSpec( display_name, - content.path, tags or [], mime_type, - user_metadata, + user_metadata or {}, preview_id, + ), + ) + if result is None: + logging.warning("create_from_hash: no asset found for hash %s", hash_str) + return result + + +def _preflight_cached_registration( + locator: str, +) -> _CachedRegistrationPreflight | None: + """Read cached facts before metadata I/O so it does not extend the writer lease.""" + with create_session() as session: + existing = session.scalars( + select(AssetContent).where( + AssetContent.path == locator, + AssetContent.is_missing.is_(False), + ) + ).first() + if existing is None: + return None + sibling = session.scalars( + select(Asset) + .where(Asset.content_id == existing.id) + .order_by(Asset.created_at.asc(), Asset.id.asc()) + .limit(1) + ).first() + sibling_id = sibling.id if sibling is not None else None + sibling_metadata = ( + dict(sibling.system_metadata) + if sibling is not None and sibling.system_metadata is not None + else None + ) + content_id = existing.id + signature = _file_signature(locator) if sibling_id is None else None + return _CachedRegistrationPreflight( + content_id, + sibling_id, + sibling_metadata, + signature, + ) + + +def _apply_cached_registration( + session: Session, + preflight: _CachedRegistrationPreflight, + name: str, + path_tags: list[str], + mime_type: str | None, + job_id: str | None, + locator: str, + system_metadata: dict[str, Any] | None, +) -> RegisteredAsset: + existing = session.get(AssetContent, preflight.content_id) + if existing is None or existing.path != locator or existing.is_missing: + raise _PreflightStale + sibling = session.scalars( + select(Asset) + .where(Asset.content_id == existing.id) + .order_by(Asset.created_at.asc(), Asset.id.asc()) + .limit(1) + ).first() + sibling_id = sibling.id if sibling is not None else None + sibling_metadata = ( + dict(sibling.system_metadata) + if sibling is not None and sibling.system_metadata is not None + else None + ) + if ( + sibling_id != preflight.sibling_id + or sibling_metadata != preflight.sibling_metadata + or ( + preflight.signature is not None + and not _file_signature_matches(preflight.signature) + ) + ): + raise _PreflightStale + record = create_record( + session, + existing.id, + name, + mime_type=mime_type, + job_id=job_id, + loader_path=compute_loader_path(locator), + tags=path_tags, + system_metadata=system_metadata, + ) + return RegisteredAsset( + id=record.id, + content_id=record.content_id, + job_id=record.job_id, + name=record.name, + ) + + +def _register_cached_output_in_txn( + session: Session, + locator: str, + job_id: str | None, +) -> RegisteredAsset | None: + existing = session.scalars( + select(AssetContent).where( + AssetContent.path == locator, + AssetContent.is_missing.is_(False), + ) + ).first() + if existing is None: + logging.info( + "Cached output registration is a non-event; no live content for %s", + locator, ) - session.commit() - return _record_to_upload_result(session, record, created_new=True) + return None + name, path_tags = get_name_and_tags_from_asset_path(locator) + mime_type = mimetypes.guess_type(locator, strict=False)[0] + sibling = session.scalars( + select(Asset) + .where(Asset.content_id == existing.id) + .order_by(Asset.created_at.asc(), Asset.id.asc()) + .limit(1) + ).first() + system_metadata = ( + dict(sibling.system_metadata) + if sibling is not None and sibling.system_metadata is not None + else _extract_system_metadata_sync(locator, mime_type) + ) + record = create_record( + session, + existing.id, + name, + mime_type=mime_type, + job_id=job_id, + loader_path=compute_loader_path(locator), + tags=path_tags, + system_metadata=system_metadata, + ) + return RegisteredAsset( + id=record.id, + content_id=record.content_id, + job_id=record.job_id, + name=record.name, + ) def register_cached_output( @@ -643,67 +1075,53 @@ def register_cached_output( ) -> RegisteredAsset | None: locator = os.path.abspath(abs_path) try: - with create_session() as session: - existing = session.scalars( - select(AssetContent).where( - AssetContent.path == locator, AssetContent.is_missing.is_(False) - ) - ).first() - if existing is None: + for restart in range(4): + preflight = _preflight_cached_registration(locator) + if preflight is None: logging.info( - "Cached output registration is a non-event; no live content " - "for %s", + "Cached output registration is a non-event; no live content for %s", locator, ) return None - name, path_tags = get_name_and_tags_from_asset_path(locator) mime_type = mimetypes.guess_type(locator, strict=False)[0] - - sibling = session.scalars( - select(Asset) - .where(Asset.content_id == existing.id) - .order_by(Asset.created_at.asc(), Asset.id.asc()) - .limit(1) - ).first() - if sibling is not None: - system_metadata = ( - dict(sibling.system_metadata) - if sibling.system_metadata is not None - else None - ) - else: + system_metadata = preflight.sibling_metadata + if preflight.sibling_id is None: system_metadata = _extract_system_metadata_sync(locator, mime_type) - try: - record = create_record( - session, - existing.id, - name, - mime_type=mime_type, - job_id=job_id, - loader_path=compute_loader_path(locator), - tags=path_tags, - system_metadata=system_metadata, + return run_write_txn( + lambda session: _apply_cached_registration( + session, + preflight, + name, + path_tags, + mime_type, + job_id, + locator, + system_metadata, + ) ) - session.commit() - except Exception: - session.rollback() - raise - record_id = record.id - record_content_id = record.content_id - record_job_id = record.job_id - record_name = record.name - except Exception: + except _PreflightStale: + if restart == 3: + logging.warning( + "Cached-output preflight changed three times; falling back to in-transaction metadata extraction" + ) + return run_write_txn( + lambda session: _register_cached_output_in_txn( + session, + locator, + job_id, + ) + ) + except Exception as exc: logging.exception("Failed to register cached output: %s", locator) + emit( + "ingest.register_failed", + output_kind="cached", + error_type=error_type(exc), + ) return None - - return RegisteredAsset( - id=record_id, - content_id=record_content_id, - job_id=record_job_id, - name=record_name, - ) + return None def register_executed_output( @@ -719,7 +1137,7 @@ def register_executed_output( system_metadata = _extract_system_metadata_sync( locator, mime_type, stat_result ) - with create_session() as session: + def _work(session: Session) -> RegisteredAsset: created_content_id: str | None = None try: existing = session.scalars( @@ -745,23 +1163,24 @@ def register_executed_output( tags=path_tags, system_metadata=system_metadata, ) - session.commit() except Exception: session.rollback() if created_content_id is not None: _discard_unreferenced_content(session, created_content_id) raise - record_id = record.id - record_content_id = record.content_id - record_job_id = record.job_id - record_name = record.name - except Exception: + return RegisteredAsset( + id=record.id, + content_id=record.content_id, + job_id=record.job_id, + name=record.name, + ) + + return run_write_txn(_work) + except Exception as exc: logging.exception("Failed to register executed output: %s", locator) + emit( + "ingest.register_failed", + output_kind="executed", + error_type=error_type(exc), + ) return None - - return RegisteredAsset( - id=record_id, - content_id=record_content_id, - job_id=record_job_id, - name=record_name, - ) diff --git a/tests-unit/assets_test/fixtures/assets_event_lines.txt b/tests-unit/assets_test/fixtures/assets_event_lines.txt index 2aa36ce15a6..eadf0cffb76 100644 --- a/tests-unit/assets_test/fixtures/assets_event_lines.txt +++ b/tests-unit/assets_test/fixtures/assets_event_lines.txt @@ -1,3 +1,4 @@ [assets-event] seeder.scan_completed created=12 elapsed_ms=8123 enrich_failed=0 enriched=4 hash_failed=2 permission_denied=0 phase=fast root=models skipped=3 [assets-event] seeder.scan_started phase=enrich [assets-event] scanner.stat_failed error_type=PermissionError site=discovery +[assets-event] ingest.register_failed error_type=RuntimeError output_kind=executed diff --git a/tests-unit/assets_test/services/conftest.py b/tests-unit/assets_test/services/conftest.py index 9c5d1dfbb0f..545801da8f2 100644 --- a/tests-unit/assets_test/services/conftest.py +++ b/tests-unit/assets_test/services/conftest.py @@ -5,7 +5,7 @@ import pytest from sqlalchemy import create_engine, event -from sqlalchemy.orm import Session, Session as SASession +from sqlalchemy.orm import Session, Session as SASession, sessionmaker from app.assets import mode from app.assets.database.models import Base @@ -67,7 +67,8 @@ def _create_session(): with patch("app.assets.services.ingest.create_session", _create_session), \ patch("app.assets.services.asset_management.create_session", _create_session), \ - patch("app.assets.services.tagging.create_session", _create_session): + patch("app.assets.services.tagging.create_session", _create_session), \ + patch("app.database.db.WriteSession", sessionmaker(bind=db_engine)): yield _create_session diff --git a/tests-unit/assets_test/services/test_ingest_write_txn.py b/tests-unit/assets_test/services/test_ingest_write_txn.py new file mode 100644 index 00000000000..247d4080635 --- /dev/null +++ b/tests-unit/assets_test/services/test_ingest_write_txn.py @@ -0,0 +1,144 @@ +import os + +import logging +import sqlite3 + +import folder_paths +from sqlalchemy.exc import IntegrityError, OperationalError + +import app.assets.services.ingest as ingest +from app.assets.database.queries.records import create_content, create_record + + +def _output_path(name: str) -> str: + output_dir = folder_paths.get_output_directory() + os.makedirs(output_dir, exist_ok=True) + return os.path.join(output_dir, name) + + +def test_cached_registration_skips_extraction_when_live_content_is_missing( + mock_create_session, monkeypatch +) -> None: + path = _output_path("cached-missing-no-extraction.bin") + with open(path, "wb") as file: + file.write(b"output") + + def extraction_must_not_run(*_args, **_kwargs): + raise AssertionError("missing content must not trigger metadata extraction") + + monkeypatch.setattr(ingest, "_extract_system_metadata_sync", extraction_must_not_run) + try: + assert ingest.register_cached_output(path) is None + finally: + os.unlink(path) + + +def test_cached_registration_skips_extraction_when_reusing_a_sibling( + mock_create_session, monkeypatch +) -> None: + path = _output_path("cached-sibling-no-extraction.bin") + with open(path, "wb") as file: + file.write(b"output") + + def extraction_must_not_run(*_args, **_kwargs): + raise AssertionError("sibling metadata must be reused without extraction") + + with mock_create_session() as session: + content = create_content(session, path, size_bytes=6) + create_record( + session, + content.id, + "sibling.bin", + system_metadata={"source": "sibling"}, + ) + session.commit() + + monkeypatch.setattr(ingest, "_extract_system_metadata_sync", extraction_must_not_run) + try: + result = ingest.register_cached_output(path) + assert result is not None + finally: + os.unlink(path) + + +def test_executed_registration_uses_the_write_transaction_runner( + mock_create_session, monkeypatch +) -> None: + path = _output_path("executed-write-transaction.bin") + with open(path, "wb") as file: + file.write(b"output") + + calls: list[None] = [] + + def record_call(work): + calls.append(None) + with mock_create_session() as session: + result = work(session) + session.commit() + return result + + monkeypatch.setattr(ingest, "run_write_txn", record_call, raising=False) + try: + result = ingest.register_executed_output(path) + assert result is not None + assert len(calls) == 1 + finally: + os.unlink(path) + + +def _registration_failure_event(caplog) -> str: + events = [ + record.getMessage() + for record in caplog.records + if record.getMessage().startswith("[assets-event] ingest.register_failed") + ] + assert len(events) == 1 + return events[0] + + +def test_executed_registration_reports_exhausted_locked_retries(monkeypatch, caplog) -> None: + path = _output_path("executed-locked-retries.bin") + with open(path, "wb") as file: + file.write(b"output") + + def exhausted_retries(_work): + raise OperationalError("INSERT", {}, sqlite3.OperationalError("database is locked")) + + monkeypatch.setattr(ingest, "run_write_txn", exhausted_retries) + try: + with caplog.at_level(logging.INFO): + assert ingest.register_executed_output(path) is None + assert _registration_failure_event(caplog) == ( + "[assets-event] ingest.register_failed error_type=OperationalError output_kind=executed" + ) + finally: + os.unlink(path) + + +def test_executed_registration_reports_non_retryable_write_failure(monkeypatch, caplog) -> None: + path = _output_path("executed-non-retryable.bin") + with open(path, "wb") as file: + file.write(b"output") + + def non_retryable_failure(_work): + raise IntegrityError("INSERT", {}, sqlite3.IntegrityError("constraint failed")) + + monkeypatch.setattr(ingest, "run_write_txn", non_retryable_failure) + try: + with caplog.at_level(logging.INFO): + assert ingest.register_executed_output(path) is None + assert _registration_failure_event(caplog) == ( + "[assets-event] ingest.register_failed error_type=IntegrityError output_kind=executed" + ) + finally: + os.unlink(path) + + +def test_executed_registration_reports_preflight_os_error(monkeypatch, caplog) -> None: + monkeypatch.setattr(ingest.os, "stat", lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("gone"))) + + with caplog.at_level(logging.INFO): + assert ingest.register_executed_output("/missing/output.bin") is None + assert _registration_failure_event(caplog) == ( + "[assets-event] ingest.register_failed error_type=OSError output_kind=executed" + ) diff --git a/tests-unit/assets_test/services/test_lifecycle.py b/tests-unit/assets_test/services/test_lifecycle.py index 4010a770c57..8523365fe5b 100644 --- a/tests-unit/assets_test/services/test_lifecycle.py +++ b/tests-unit/assets_test/services/test_lifecycle.py @@ -8,7 +8,7 @@ import pytest from sqlalchemy import create_engine, select from sqlalchemy.orm import Session -from sqlalchemy.orm import Session as SASession +from sqlalchemy.orm import Session as SASession, sessionmaker from app.assets import lifecycle from app.assets.database.models import Asset, AssetContent, Base @@ -64,7 +64,8 @@ def _create_session(): yield sess with patch("app.assets.lifecycle.create_session", _create_session), \ - patch("app.database.db.create_session", _create_session): + patch("app.database.db.create_session", _create_session), \ + patch("app.database.db.WriteSession", sessionmaker(bind=engine)): yield _create_session diff --git a/tests-unit/assets_test/services/test_manager_isolated.py b/tests-unit/assets_test/services/test_manager_isolated.py index 2b73ba1bbc0..e195c1184de 100644 --- a/tests-unit/assets_test/services/test_manager_isolated.py +++ b/tests-unit/assets_test/services/test_manager_isolated.py @@ -9,7 +9,7 @@ import folder_paths import pytest from sqlalchemy import create_engine, select -from sqlalchemy.orm import Session, Session as SASession +from sqlalchemy.orm import Session, Session as SASession, sessionmaker from sqlalchemy.pool import StaticPool from app.assets import lifecycle @@ -76,6 +76,7 @@ def _create_session() -> Generator[SASession, None, None]: monkeypatch.setattr(seeder_module, "create_session", _create_session) monkeypatch.setattr(scanner, "create_session", _create_session) monkeypatch.setattr("app.assets.services.ingest.create_session", _create_session) + monkeypatch.setattr("app.database.db.WriteSession", sessionmaker(bind=engine)) yield _create_session engine.dispose() diff --git a/tests-unit/assets_test/services/test_stored_hash_prefix.py b/tests-unit/assets_test/services/test_stored_hash_prefix.py index c28d6fb342a..131ddd9eb40 100644 --- a/tests-unit/assets_test/services/test_stored_hash_prefix.py +++ b/tests-unit/assets_test/services/test_stored_hash_prefix.py @@ -10,7 +10,7 @@ from aiohttp import web from aiohttp.test_utils import make_mocked_request from sqlalchemy import func, select -from sqlalchemy.orm import Session as SASession +from sqlalchemy.orm import Session as SASession, sessionmaker from app.assets import mode from app.assets.api import routes @@ -209,6 +209,7 @@ def _factory(): monkeypatch.setattr(mode, "hashing_enabled", lambda: True) monkeypatch.setattr(ingest, "create_session", _factory) monkeypatch.setattr(asset_management, "create_session", _factory) + monkeypatch.setattr("app.database.db.WriteSession", sessionmaker(bind=db_engine)) content_bytes = b"one-asset-all-surfaces-agree" temp = _write_temp(content_bytes) diff --git a/tests-unit/assets_test/services/test_transition_settle.py b/tests-unit/assets_test/services/test_transition_settle.py index 3da231ba599..a08b28d3e79 100644 --- a/tests-unit/assets_test/services/test_transition_settle.py +++ b/tests-unit/assets_test/services/test_transition_settle.py @@ -3,7 +3,7 @@ from unittest.mock import patch import pytest -from sqlalchemy.orm import Session as SASession +from sqlalchemy.orm import Session as SASession, sessionmaker from app.assets import scanner, seeder as seeder_module from app.assets.database.models import AssetContent @@ -82,7 +82,8 @@ def counting_enrich_asset(*args, **kwargs): monkeypatch.setattr(scanner, "enrich_asset", counting_enrich_asset) with patch("app.assets.seeder.create_session", _create_session), \ - patch("app.assets.scanner.create_session", _create_session): + patch("app.assets.scanner.create_session", _create_session), \ + patch("app.database.db.WriteSession", sessionmaker(bind=db_engine)): try: cancelled, _enriched = seeder._run_enrich_phase(("input",)) except _AttemptBudgetExhausted: diff --git a/tests-unit/assets_test/services/test_upload_b.py b/tests-unit/assets_test/services/test_upload_b.py index 3e3cdc0557a..487f62c6da9 100644 --- a/tests-unit/assets_test/services/test_upload_b.py +++ b/tests-unit/assets_test/services/test_upload_b.py @@ -6,7 +6,7 @@ import pytest from sqlalchemy import create_engine, func, select, update from sqlalchemy.exc import OperationalError -from sqlalchemy.orm import Session as SASession +from sqlalchemy.orm import Session as SASession, sessionmaker import app.assets.mode as mode_module import app.assets.services.ingest as ingest_module @@ -1049,6 +1049,7 @@ def claim_after_a_rehashes(session, content_id_arg, hash_arg): "app.assets.services.ingest.create_session", lambda: _session_factory(engine), ), + patch("app.database.db.WriteSession", sessionmaker(bind=engine)), patch( "app.assets.services.ingest.claim_qualified_content", claim_after_a_rehashes, @@ -1123,6 +1124,7 @@ def create_record_after_a_retires(session, content_id_arg, *args, **kwargs): "app.assets.services.ingest.create_session", lambda: _session_factory(engine), ), + patch("app.database.db.WriteSession", sessionmaker(bind=engine)), patch( "app.assets.services.ingest._create_upload_record", create_record_after_a_retires, @@ -1227,7 +1229,7 @@ def test_upload_normalizes_tags_before_they_reach_the_query_layer( real_create_upload_record = ingest_module._create_upload_record def capturing_create_upload_record(*args, **kwargs): - captured.append(list(args[4])) + captured.append(list(args[2].preflight.spec.tags)) return real_create_upload_record(*args, **kwargs) monkeypatch.setattr( @@ -1313,9 +1315,7 @@ def test_incumbent_reconciliation_persists_the_stat_hashing_verified( _mutating_snapshot_hash(dest_abs, rewritten), ) - with mock_create_session() as session: - ingest_module._settle_destination_before_write(session, dest_abs) - session.commit() + ingest_module._settle_destination_before_write(dest_abs) verified_size, verified_mtime = _stat_pair(dest_abs) with mock_create_session() as session: diff --git a/tests-unit/assets_test/test_event_log.py b/tests-unit/assets_test/test_event_log.py index 27c9bba139b..ccdbcc8320d 100644 --- a/tests-unit/assets_test/test_event_log.py +++ b/tests-unit/assets_test/test_event_log.py @@ -38,6 +38,7 @@ "error_type": ["ValueError", "FileNotFoundError"], "hashing_enabled": [True, False], "site": ["discovery", "enrich"], + "output_kind": ["executed", "cached"], } @@ -91,11 +92,11 @@ def go_to_production_mode(monkeypatch: pytest.MonkeyPatch) -> None: # --- the shared cross-repo fixture ------------------------------------------------- -def test_shared_fixture_file_holds_three_newline_terminated_lines(): +def test_shared_fixture_file_holds_four_newline_terminated_lines(): raw = FIXTURE_PATH.read_text(encoding="utf-8") assert raw.endswith("\n") - assert len(raw.splitlines()) == 3 + assert len(raw.splitlines()) == 4 @pytest.mark.parametrize("line", fixture_lines()) diff --git a/tests-unit/assets_test/test_intended_behaviour.py b/tests-unit/assets_test/test_intended_behaviour.py index 580637c71f2..a1fffc24f0d 100644 --- a/tests-unit/assets_test/test_intended_behaviour.py +++ b/tests-unit/assets_test/test_intended_behaviour.py @@ -9,7 +9,7 @@ from aiohttp.test_utils import make_mocked_request from blake3 import blake3 from sqlalchemy import create_engine, select -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from app.assets import mode from app.assets.api import routes @@ -259,6 +259,7 @@ def upload( "app.assets.services.ingest.create_session", lambda: nullcontext(session), ), + patch("app.database.db.WriteSession", sessionmaker(bind=session.bind)), patch.object(mode, "hashing_enabled", return_value=hashing), ): return upload_from_temp_path( @@ -373,6 +374,7 @@ def test_scenario_10_cached_delivery_record(session, tmp_path): "app.assets.services.ingest.create_session", lambda: nullcontext(session), ), + patch("app.database.db.WriteSession", sessionmaker(bind=session.bind)), ): delivered = register_cached_output(str(path), job_id="delivery-job") @@ -402,6 +404,7 @@ def test_scenario_10_cached_delivery_record(session, tmp_path): "app.assets.services.ingest.create_session", lambda: nullcontext(session), ), + patch("app.database.db.WriteSession", sessionmaker(bind=session.bind)), ): assert register_cached_output(str(path), job_id="second-delivery") is None assert {row.id for row in session.scalars(select(Asset))} == { diff --git a/tests-unit/assets_test/test_upload_hashing_modes.py b/tests-unit/assets_test/test_upload_hashing_modes.py index 3c251c62384..567fe255ee9 100644 --- a/tests-unit/assets_test/test_upload_hashing_modes.py +++ b/tests-unit/assets_test/test_upload_hashing_modes.py @@ -15,7 +15,7 @@ import requests from aiohttp import web from sqlalchemy import create_engine, select -from sqlalchemy.orm import Session as SASession +from sqlalchemy.orm import Session as SASession, sessionmaker import app.assets.mode as mode_module import folder_paths @@ -225,6 +225,7 @@ def _fake_create_session(): monkeypatch.setattr( "app.assets.services.ingest.create_session", _fake_create_session ) + monkeypatch.setattr("app.database.db.WriteSession", sessionmaker(bind=engine)) output_dir = folder_paths.get_output_directory() os.makedirs(output_dir, exist_ok=True) diff --git a/tests-unit/test_assets_event_log_static.py b/tests-unit/test_assets_event_log_static.py index cbab691dea2..e4e48433e5b 100644 --- a/tests-unit/test_assets_event_log_static.py +++ b/tests-unit/test_assets_event_log_static.py @@ -63,6 +63,8 @@ class CallSite(NamedTuple): # todo 16 - discovery/enrich stat failures, emit-once per scan per site CallSite("app/assets/scanner.py", "build_asset_specs", "scanner.stat_failed"), CallSite("app/assets/scanner.py", "enrich_asset", "scanner.stat_failed"), + CallSite("app/assets/services/ingest.py", "register_cached_output", "ingest.register_failed"), + CallSite("app/assets/services/ingest.py", "register_executed_output", "ingest.register_failed"), } ) From 2bf7022d1677aabc352da23a2d853c36edc0b28e Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 13:28:43 -0700 Subject: [PATCH 003/111] test(assets): cover ingest stale fallback --- .../services/test_ingest_write_txn.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests-unit/assets_test/services/test_ingest_write_txn.py b/tests-unit/assets_test/services/test_ingest_write_txn.py index 247d4080635..20336b7717d 100644 --- a/tests-unit/assets_test/services/test_ingest_write_txn.py +++ b/tests-unit/assets_test/services/test_ingest_write_txn.py @@ -142,3 +142,58 @@ def test_executed_registration_reports_preflight_os_error(monkeypatch, caplog) - assert _registration_failure_event(caplog) == ( "[assets-event] ingest.register_failed error_type=OSError output_kind=executed" ) + + +def test_reused_upload_falls_back_after_four_stale_preflights(monkeypatch, caplog) -> None: + preflight = object() + prepared = object() + attempts: list[object] = [] + fallback_result = object() + + monkeypatch.setattr(ingest, "_preflight_upload_record", lambda *_args: preflight) + monkeypatch.setattr(ingest, "_prepare_upload_record", lambda _preflight: prepared) + + def stale_apply(_session, observed_prepared): + attempts.append(observed_prepared) + raise ingest._PreflightStale + + monkeypatch.setattr(ingest, "_apply_reused_upload_record", stale_apply) + monkeypatch.setattr( + ingest, + "_reuse_qualified_content_in_txn", + lambda *_args: fallback_result, + ) + monkeypatch.setattr(ingest, "run_write_txn", lambda work: work(object())) + + spec = ingest._UploadRecordSpec("asset", [], None, {}, None) + with caplog.at_level(logging.WARNING): + result = ingest._reuse_qualified_content("blake3:hash", spec) + + assert result is fallback_result + assert attempts == [prepared, prepared, prepared, prepared] + warnings = [record for record in caplog.records if record.levelno == logging.WARNING] + assert len(warnings) == 1 + + +def test_cached_registration_reports_terminal_write_failure( + mock_create_session, monkeypatch, caplog +) -> None: + path = _output_path("cached-terminal-failure.bin") + with open(path, "wb") as file: + file.write(b"output") + with mock_create_session() as session: + create_content(session, path, size_bytes=6) + session.commit() + + def non_retryable_failure(_work): + raise IntegrityError("INSERT", {}, sqlite3.IntegrityError("constraint failed")) + + monkeypatch.setattr(ingest, "run_write_txn", non_retryable_failure) + try: + with caplog.at_level(logging.INFO): + assert ingest.register_cached_output(path) is None + assert _registration_failure_event(caplog) == ( + "[assets-event] ingest.register_failed error_type=IntegrityError output_kind=cached" + ) + finally: + os.unlink(path) From 0c9fecbc01e553830d4c0ff929ed431aef9116ec Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 13:46:52 -0700 Subject: [PATCH 004/111] test(assets): cover real divergence detection in cached registration preflight --- .../services/test_ingest_write_txn.py | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/tests-unit/assets_test/services/test_ingest_write_txn.py b/tests-unit/assets_test/services/test_ingest_write_txn.py index 20336b7717d..477a33bb365 100644 --- a/tests-unit/assets_test/services/test_ingest_write_txn.py +++ b/tests-unit/assets_test/services/test_ingest_write_txn.py @@ -4,6 +4,7 @@ import sqlite3 import folder_paths +import pytest from sqlalchemy.exc import IntegrityError, OperationalError import app.assets.services.ingest as ingest @@ -96,6 +97,26 @@ def _registration_failure_event(caplog) -> str: return events[0] +def _seed_cached_content(mock_create_session, path: str) -> str: + with mock_create_session() as session: + content = create_content(session, path, size_bytes=os.path.getsize(path)) + session.commit() + return content.id + + +def _apply_cached_preflight(session, preflight, path: str, system_metadata: dict[str, int]) -> None: + ingest._apply_cached_registration( + session, + preflight, + os.path.basename(path), + ["output"], + None, + None, + path, + system_metadata, + ) + + def test_executed_registration_reports_exhausted_locked_retries(monkeypatch, caplog) -> None: path = _output_path("executed-locked-retries.bin") with open(path, "wb") as file: @@ -197,3 +218,161 @@ def non_retryable_failure(_work): ) finally: os.unlink(path) + + +def test_cached_registration_restarts_when_content_vanishes_after_preflight( + mock_create_session, monkeypatch +) -> None: + direct_path = _output_path("cached-direct-content-vanished.bin") + public_path = _output_path("cached-public-content-vanished.bin") + for path in (direct_path, public_path): + with open(path, "wb") as file: + file.write(b"output") + try: + direct_content_id = _seed_cached_content(mock_create_session, direct_path) + direct_preflight = ingest._preflight_cached_registration(direct_path) + assert direct_preflight is not None + with mock_create_session() as session: + content = session.get(ingest.AssetContent, direct_content_id) + assert content is not None + ingest.mark_content_missing(session, content.id) + session.commit() + with mock_create_session() as session: + with pytest.raises(ingest._PreflightStale): + _apply_cached_preflight(session, direct_preflight, direct_path, {}) + + public_content_id = _seed_cached_content(mock_create_session, public_path) + real_apply = ingest._apply_cached_registration + mutated = False + + def vanish_then_apply(session, *args): + nonlocal mutated + if not mutated: + mutated = True + with mock_create_session() as mutation_session: + content = mutation_session.get(ingest.AssetContent, public_content_id) + assert content is not None + ingest.mark_content_missing(mutation_session, content.id) + mutation_session.commit() + return real_apply(session, *args) + + monkeypatch.setattr(ingest, "_apply_cached_registration", vanish_then_apply) + assert ingest.register_cached_output(public_path) is None + assert mutated is True + finally: + for path in (direct_path, public_path): + os.unlink(path) + + +def test_cached_registration_restarts_when_sibling_appears_after_preflight( + mock_create_session, monkeypatch +) -> None: + direct_path = _output_path("cached-direct-sibling-appeared.bin") + public_path = _output_path("cached-public-sibling-appeared.bin") + for path in (direct_path, public_path): + with open(path, "wb") as file: + file.write(b"output") + try: + direct_content_id = _seed_cached_content(mock_create_session, direct_path) + direct_preflight = ingest._preflight_cached_registration(direct_path) + assert direct_preflight is not None + with mock_create_session() as session: + create_record( + session, + direct_content_id, + "sibling.bin", + system_metadata={"generation": 1}, + ) + session.commit() + with mock_create_session() as session: + with pytest.raises(ingest._PreflightStale): + _apply_cached_preflight(session, direct_preflight, direct_path, {}) + + public_content_id = _seed_cached_content(mock_create_session, public_path) + real_apply = ingest._apply_cached_registration + extraction_count = 0 + mutated = False + + def extract_metadata(*_args, **_kwargs): + nonlocal extraction_count + extraction_count += 1 + return {"generation": 0} + + def add_sibling_then_apply(session, *args): + nonlocal mutated + if not mutated: + mutated = True + with mock_create_session() as mutation_session: + create_record( + mutation_session, + public_content_id, + "sibling.bin", + system_metadata={"generation": 2}, + ) + mutation_session.commit() + return real_apply(session, *args) + + monkeypatch.setattr(ingest, "_extract_system_metadata_sync", extract_metadata) + monkeypatch.setattr(ingest, "_apply_cached_registration", add_sibling_then_apply) + result = ingest.register_cached_output(public_path) + assert result is not None + with mock_create_session() as session: + record = session.get(ingest.Asset, result.id) + assert record is not None + assert record.system_metadata == {"generation": 2} + assert extraction_count == 1 + assert mutated is True + finally: + for path in (direct_path, public_path): + os.unlink(path) + + +def test_cached_registration_restarts_when_file_changes_after_preflight( + mock_create_session, monkeypatch +) -> None: + direct_path = _output_path("cached-direct-stat-changed.bin") + public_path = _output_path("cached-public-stat-changed.bin") + for path in (direct_path, public_path): + with open(path, "wb") as file: + file.write(b"old") + try: + _seed_cached_content(mock_create_session, direct_path) + direct_preflight = ingest._preflight_cached_registration(direct_path) + assert direct_preflight is not None + with open(direct_path, "wb") as file: + file.write(b"new bytes") + with mock_create_session() as session: + with pytest.raises(ingest._PreflightStale): + _apply_cached_preflight(session, direct_preflight, direct_path, {}) + + _seed_cached_content(mock_create_session, public_path) + real_apply = ingest._apply_cached_registration + extraction_sizes: list[int] = [] + mutated = False + + def extract_metadata(path, *_args, **_kwargs): + size = os.path.getsize(path) + extraction_sizes.append(size) + return {"size": size} + + def rewrite_then_apply(session, *args): + nonlocal mutated + if not mutated: + mutated = True + with open(public_path, "wb") as file: + file.write(b"new public bytes") + return real_apply(session, *args) + + monkeypatch.setattr(ingest, "_extract_system_metadata_sync", extract_metadata) + monkeypatch.setattr(ingest, "_apply_cached_registration", rewrite_then_apply) + result = ingest.register_cached_output(public_path) + assert result is not None + with mock_create_session() as session: + record = session.get(ingest.Asset, result.id) + assert record is not None + assert record.system_metadata == {"size": len(b"new public bytes")} + assert extraction_sizes == [len(b"old"), len(b"new public bytes")] + assert mutated is True + finally: + for path in (direct_path, public_path): + os.unlink(path) From f8bb1bd4c23ccaceea225a4dc14897732bb2551e Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 15:03:21 -0700 Subject: [PATCH 005/111] fix(assets): make scanner and seeder writes immediate, retry-safe, and lock-hold bounded --- app/assets/event_log.py | 2 +- app/assets/scanner.py | 378 +++++++++++------- app/assets/scanner_admission.py | 22 +- app/assets/scanner_changes.py | 220 ++++++++-- app/assets/seeder.py | 20 +- app/assets/services/hash_mode_state.py | 143 ++++--- tests-unit/assets_test/helpers.py | 25 +- tests-unit/assets_test/services/conftest.py | 5 +- .../test_b1_counter_and_retry_integrity.py | 188 +++++++++ .../services/test_b2_lock_hold_barriers.py | 298 ++++++++++++++ .../test_b2_queue_commit_failure_integrity.py | 172 ++++++++ .../services/test_enrichment_snapshot.py | 3 +- .../services/test_manager_isolated.py | 4 +- .../test_queue_drain_characterization.py | 123 ++++++ .../assets_test/services/test_scanner_b.py | 13 +- .../services/test_stored_hash_prefix.py | 3 +- .../services/test_transition_drain.py | 2 + .../services/test_transition_settle.py | 14 +- .../services/test_updated_at_semantics.py | 11 +- tests-unit/assets_test/test_event_log.py | 2 +- .../assets_test/test_intended_behaviour.py | 12 +- .../test_scanner_ingest_event_log.py | 24 +- tests-unit/test_assets_event_log_static.py | 7 +- 23 files changed, 1394 insertions(+), 297 deletions(-) create mode 100644 tests-unit/assets_test/services/test_b1_counter_and_retry_integrity.py create mode 100644 tests-unit/assets_test/services/test_b2_lock_hold_barriers.py create mode 100644 tests-unit/assets_test/services/test_b2_queue_commit_failure_integrity.py create mode 100644 tests-unit/assets_test/services/test_queue_drain_characterization.py diff --git a/app/assets/event_log.py b/app/assets/event_log.py index 479cdd94b6b..9810123dae6 100644 --- a/app/assets/event_log.py +++ b/app/assets/event_log.py @@ -27,7 +27,7 @@ ROOTS = frozenset({"models", "input", "output", "user", "temp"}) PHASES = frozenset({"fast", "enrich", "full"}) STAGES = frozenset({"mark_missing", "pruning", "fast_scan", "enrich", "finalize"}) -STAT_SITES = frozenset({"discovery", "enrich"}) +STAT_SITES = frozenset({"discovery", "enrich", "reference_stat"}) ALLOWED_EVENTS = frozenset({ "assets.enabled", "seeder.scan_started", diff --git a/app/assets/scanner.py b/app/assets/scanner.py index 76821e4c337..d078b016898 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -11,7 +11,7 @@ import os from dataclasses import dataclass from pathlib import Path -from typing import Callable, Literal, Protocol, TypedDict +from typing import Any, Callable, Literal, NamedTuple, Protocol, TypedDict import folder_paths import sqlalchemy as sa @@ -28,13 +28,18 @@ from app.assets.helpers import sql_path_under_prefix, to_stored_hash from app.assets.lifecycle import get_excluded_scan_roots from app.assets.scanner_changes import ( + PreparedRecovery, clear_pending_verifications, detect_content_change, drain_pending_verifications, is_path_under_prefixes, live_contents_under_prefixes, pending_recovery_count, + prepare_missing_content_recovery, + queue_pending_recovery, + queue_pending_verification, recover_missing_content, + recover_missing_content_from_preparation, ) from app.assets.scanner_admission import ( PARTIAL_DOWNLOAD_EXTENSIONS as PARTIAL_DOWNLOAD_EXTENSIONS, @@ -54,7 +59,7 @@ ) from app.assets.services.ingest import _discard_unreferenced_content from app.assets.services.snapshot_hash import snapshot_hash -from app.database.db import create_session +from app.database.db import create_session, run_write_txn __all__ = [ "clear_pending_verifications", @@ -94,6 +99,16 @@ class UnenrichedContent: content_id: str record_id: str file_path: str + needs_hash: bool = False + + +class _PreparedEnrichment(NamedTuple): + row: UnenrichedContent + stat_result: os.stat_result + system_metadata: dict[str, Any] | None + mime_type: str | None + stored_hash: str | None + hash_requested: bool def _log_scan_error(phase: str, error: OSError) -> None: @@ -157,12 +172,16 @@ def sync_references_with_filesystem( root: RootType, collect_existing_paths: bool = False, progress: _ScanProgress | None = None, + pending_verification_ids: list[str] | None = None, + diagnostics: list[OSError] | None = None, ) -> set[str] | None: return sync_prefixes_with_filesystem( session, get_scan_prefixes_for_root(root), collect_existing_paths=collect_existing_paths, progress=progress, + pending_verification_ids=pending_verification_ids, + diagnostics=diagnostics, ) @@ -171,6 +190,8 @@ def sync_prefixes_with_filesystem( prefixes: list[str], collect_existing_paths: bool = False, progress: _ScanProgress | None = None, + pending_verification_ids: list[str] | None = None, + diagnostics: list[OSError] | None = None, ) -> set[str] | None: if not prefixes: return set() if collect_existing_paths else None @@ -182,13 +203,19 @@ def sync_prefixes_with_filesystem( except FileNotFoundError: mark_content_missing(session, content.id) except PermissionError as e: - _log_scan_error("reference_stat", e) - if progress is not None: - progress.permission_denied += 1 - logging.debug("Permission denied accessing %s", content.path) + if diagnostics is None: + _log_scan_error("reference_stat", e) + if progress is not None: + progress.permission_denied += 1 + logging.debug("Permission denied accessing %s", content.path) + else: + diagnostics.append(e) except OSError as e: - _log_scan_error("reference_stat", e) - logging.debug("OSError checking %s: %s", content.path, e) + if diagnostics is None: + _log_scan_error("reference_stat", e) + logging.debug("OSError checking %s: %s", content.path, e) + else: + diagnostics.append(e) mark_content_missing(session, content.id) else: detect_content_change( @@ -196,12 +223,28 @@ def sync_prefixes_with_filesystem( content, stat_result, hashing_is_enabled=mode.hashing_enabled(), + pending_verification_ids=pending_verification_ids, ) survivors.add(os.path.abspath(content.path)) return survivors if collect_existing_paths else None +def _publish_reference_diagnostics( + diagnostics: list[OSError], progress: _ScanProgress | None +) -> None: + for diagnostic in diagnostics: + _log_scan_error("reference_stat", diagnostic) + if isinstance(diagnostic, PermissionError): + if progress is not None: + progress.permission_denied += 1 + logging.debug("Permission denied accessing reference") + else: + logging.debug("OSError checking reference: %s", diagnostic) + if progress is None or progress.mark_emitted("stat_failed:reference_stat"): + emit("scanner.stat_failed", site="reference_stat", error_type=error_type(diagnostic)) + + def _is_under_prefixes(path: str, prefixes: list[str]) -> bool: return is_path_under_prefixes(path, prefixes) @@ -214,15 +257,23 @@ def sync_root_safely( Returns survivors (existing paths) or empty set on failure. """ try: - with create_session() as sess: + def _work(sess: Session) -> tuple[set[str], list[str], list[OSError]]: + pending_verification_ids: list[str] = [] + diagnostics: list[OSError] = [] survivors = sync_references_with_filesystem( sess, root, collect_existing_paths=True, - progress=progress, + pending_verification_ids=pending_verification_ids, + diagnostics=diagnostics, ) - sess.commit() - return survivors or set() + return survivors or set(), pending_verification_ids, diagnostics + + survivors, pending_verification_ids, diagnostics = run_write_txn(_work) + for content_id in pending_verification_ids: + queue_pending_verification(content_id) + _publish_reference_diagnostics(diagnostics, progress) + return survivors except Exception as exc: logging.exception("fast DB scan failed for %s: %s", root, exc) emit( @@ -238,13 +289,21 @@ def sync_temp_references_safely( ) -> None: """Retire temp references whose file is gone; temp is never scanned, so nothing else stats them.""" try: - with create_session() as sess: + def _work(sess: Session) -> tuple[list[str], list[OSError]]: + pending_verification_ids: list[str] = [] + diagnostics: list[OSError] = [] sync_prefixes_with_filesystem( sess, get_temp_prefixes(), - progress=progress, + pending_verification_ids=pending_verification_ids, + diagnostics=diagnostics, ) - sess.commit() + return pending_verification_ids, diagnostics + + pending_verification_ids, diagnostics = run_write_txn(_work) + for content_id in pending_verification_ids: + queue_pending_verification(content_id) + _publish_reference_diagnostics(diagnostics, progress) except Exception as exc: logging.exception("temp reference sync failed: %s", exc) emit( @@ -260,10 +319,9 @@ def mark_missing_outside_prefixes_safely(prefixes: list[str]) -> int: This is a non-destructive soft-delete. Returns count marked or 0 on failure. """ try: - with create_session() as sess: - count = mark_contents_missing_outside_prefixes(sess, prefixes) - sess.commit() - return count + return run_write_txn( + lambda session: mark_contents_missing_outside_prefixes(session, prefixes) + ) except Exception as exc: logging.exception("marking missing assets failed: %s", exc) emit( @@ -375,7 +433,12 @@ def build_asset_specs( return specs, tag_pool, skipped -def seed_asset_specs(session: Session, specs: list[SeedAssetSpec]) -> int: +def seed_asset_specs( + session: Session, + specs: list[SeedAssetSpec], + prepared_recoveries: dict[str, PreparedRecovery | None] | None = None, + pending_recovery_paths: list[str] | None = None, +) -> int: created = 0 created_content_ids: list[str] = [] try: @@ -388,16 +451,31 @@ def seed_asset_specs(session: Session, specs: list[SeedAssetSpec]) -> int: except OSError: logging.warning("Skipping vanished asset during scan: %s", path) continue - try: - recovery = recover_missing_content( + if prepared_recoveries is None: + try: + recovery = recover_missing_content( + session, + path, + stat_result, + hashing_is_enabled=mode.hashing_enabled(), + ) + except OSError: + logging.warning("Skipping vanished asset during scan: %s", path) + continue + elif mode.hashing_enabled(): + prepared = prepared_recoveries.get(path) + if prepared is None: + logging.warning("Skipping vanished asset during scan: %s", path) + continue + recovery = recover_missing_content_from_preparation( session, path, stat_result, - hashing_is_enabled=mode.hashing_enabled(), + prepared, + pending_recovery_paths if pending_recovery_paths is not None else [], ) - except OSError: - logging.warning("Skipping vanished asset during scan: %s", path) - continue + else: + recovery = "no_match" if recovery != "no_match": continue content, inserted = create_content_reporting_insert( @@ -438,10 +516,31 @@ def seed_asset_specs(session: Session, specs: list[SeedAssetSpec]) -> int: def insert_asset_specs(specs: list[SeedAssetSpec], _tag_pool: set[str]) -> int: if not specs: return 0 - with create_session() as sess: - created = seed_asset_specs(sess, specs) - sess.commit() - return created + prepared_recoveries: dict[str, PreparedRecovery | None] = {} + if mode.hashing_enabled(): + for spec in specs: + path = os.path.abspath(spec["abs_path"]) + try: + prepared_recoveries[path] = prepare_missing_content_recovery( + path, os.stat(path, follow_symlinks=True) + ) + except OSError: + prepared_recoveries[path] = None + + def _work(sess: Session) -> tuple[int, list[str]]: + pending_recovery_paths: list[str] = [] + created = seed_asset_specs( + sess, + specs, + prepared_recoveries, + pending_recovery_paths, + ) + return created, pending_recovery_paths + + created, pending_recovery_paths = run_write_txn(_work) + for path in pending_recovery_paths: + queue_pending_recovery(path) + return created def get_unenriched_assets_for_roots( @@ -458,7 +557,12 @@ def get_unenriched_assets_for_roots( with create_session() as sess: query = ( - sa.select(AssetContent.id, Asset.id, AssetContent.path) + sa.select( + AssetContent.id, + Asset.id, + AssetContent.path, + AssetContent.hash.is_(None).label("needs_hash"), + ) .join(Asset, Asset.content_id == AssetContent.id) .where(AssetContent.is_missing.is_(False)) ) @@ -479,78 +583,67 @@ def get_unenriched_assets_for_roots( rows = sess.execute(query.order_by(Asset.id).limit(limit)).all() return [ - UnenrichedContent(content_id, record_id, file_path) - for content_id, record_id, file_path in rows + UnenrichedContent(content_id, record_id, file_path, needs_hash) + for content_id, record_id, file_path, needs_hash in rows ] -def enrich_asset( - session, - file_path: str, - content_id: str, - record_id: str, - extract_metadata: bool = True, - compute_hash: bool = False, - progress: _ScanProgress | None = None, -) -> bool: - """Enrich a single asset with metadata and/or hash. - - Args: - session: Database session (caller manages lifecycle) - file_path: Absolute path to the file - content_id: ID of the content to update - record_id: ID of the record to update - extract_metadata: If True, extract safetensors header and mime type - compute_hash: If True, compute blake3 hash - - Returns: - Whether enrichment changed the B-schema record or content - """ +def _prepare_enrichment( + row: UnenrichedContent, + extract_metadata: bool, + compute_hash: bool, + progress: _ScanProgress | None, +) -> _PreparedEnrichment | None: try: - stat_p = os.stat(file_path, follow_symlinks=True) + stat_result = os.stat(row.file_path, follow_symlinks=True) except FileNotFoundError: - return False - except OSError as e: - _log_scan_error("enrichment_stat", e) + return None + except OSError as exc: + _log_scan_error("enrichment_stat", exc) if progress is not None: - if isinstance(e, PermissionError): + if isinstance(exc, PermissionError): progress.permission_denied += 1 if progress.mark_emitted("stat_failed:enrich"): - emit("scanner.stat_failed", site="enrich", error_type=error_type(e)) - return False - - initial_mtime_ns = get_mtime_ns(stat_p) - rel_fname = compute_loader_path(file_path) + emit("scanner.stat_failed", site="enrich", error_type=error_type(exc)) + return None + system_metadata: dict[str, Any] | None = None mime_type: str | None = None - metadata = None - if extract_metadata: metadata = extract_file_metadata( - file_path, - stat_result=stat_p, - relative_filename=rel_fname, + row.file_path, + stat_result=stat_result, + relative_filename=compute_loader_path(row.file_path), ) - if metadata: + if metadata is not None: + system_metadata = metadata.to_user_metadata() mime_type = metadata.content_type - - content = session.get(AssetContent, content_id) - - digest: str | None = None + if mime_type is not None and mime_type.startswith("image/"): + dimensions = extract_image_dimensions(row.file_path, mime_type=mime_type) + if dimensions: + system_metadata.update(dimensions) + hash_requested = compute_hash and row.needs_hash stored_hash: str | None = None - verified_stat: os.stat_result | None = None - hash_requested = compute_hash and content is not None and content.hash is None if hash_requested: try: - snapshot = snapshot_hash(file_path) + snapshot = snapshot_hash(row.file_path) if snapshot is None: if progress is None or progress.mark_emitted("hash_discarded_modified"): emit("scanner.hash_discarded_modified") logging.warning( "File modified during hashing (snapshot unstable), discarding hash: %s", - file_path, + row.file_path, ) - return False + return None digest, verified_stat = snapshot + if ( + verified_stat.st_size != stat_result.st_size + or get_mtime_ns(verified_stat) != get_mtime_ns(stat_result) + ): + logging.info( + "Content %s changed during enrichment preparation, discarding stale result", + row.content_id, + ) + return None stored_hash = to_stored_hash(digest) except Exception as exc: emit_failure = progress is None @@ -562,49 +655,53 @@ def enrich_asset( if isinstance(exc, OSError): _log_scan_error("hashing", exc) else: - logging.warning("Failed to hash %s: %s", file_path, exc) + logging.warning("Failed to hash %s: %s", row.file_path, exc) + return _PreparedEnrichment( + row, + stat_result, + system_metadata, + mime_type, + stored_hash, + hash_requested, + ) - record = session.get(Asset, record_id) - if content is None or record is None or content.mtime_ns != initial_mtime_ns: - session.rollback() - logging.info( - "Content %s mtime changed during enrichment, discarding stale result", - content_id, - ) - return False - # Non-NULL system_metadata permanently excludes the row from re-enrichment, so a - # disagreement here must discard the metadata too, not just the hash. - if verified_stat is not None and ( - get_mtime_ns(verified_stat) != initial_mtime_ns - or verified_stat.st_size != stat_p.st_size +def _apply_enrichment(session: Session, prepared: _PreparedEnrichment) -> bool: + row = prepared.row + content = session.get(AssetContent, row.content_id) + record = session.get(Asset, row.record_id) + if content is None or record is None: + return False + try: + current_stat = os.stat(row.file_path, follow_symlinks=True) + except OSError: + return False + if ( + content.mtime_ns != get_mtime_ns(prepared.stat_result) + or current_stat.st_size != prepared.stat_result.st_size + or get_mtime_ns(current_stat) != get_mtime_ns(prepared.stat_result) ): - session.rollback() logging.info( - "Content %s changed between its metadata read and its hash read, " - "discarding stale result", - content_id, + "Content %s changed during enrichment, discarding stale result", + row.content_id, ) return False - - if extract_metadata and metadata: - system_metadata = metadata.to_user_metadata() - if mime_type and mime_type.startswith("image/"): - dims = extract_image_dimensions(file_path, mime_type=mime_type) - if dims: - system_metadata.update(dims) - record.system_metadata = {**(record.system_metadata or {}), **system_metadata} - - if stored_hash: - content.hash = stored_hash - if mime_type: - record.mime_type = mime_type - - session.commit() - - if hash_requested and stored_hash is None: + hash_applied = False + if prepared.stored_hash is not None and content.hash is None: + content.hash = prepared.stored_hash + hash_applied = True + + if prepared.system_metadata is not None: + record.system_metadata = { + **(record.system_metadata or {}), + **prepared.system_metadata, + } + if prepared.mime_type is not None: + record.mime_type = prepared.mime_type + + if prepared.hash_requested and prepared.stored_hash is None: return False - return stored_hash is not None or metadata is not None or mime_type is not None + return hash_applied or prepared.system_metadata is not None or prepared.mime_type is not None def enrich_assets_batch( @@ -633,32 +730,29 @@ def enrich_assets_batch( enriched = 0 failed_ids: list[str] = [] - with create_session() as sess: - for row in rows: - if interrupt_check is not None and interrupt_check(): - break - - try: - updated = enrich_asset( - sess, - file_path=row.file_path, - content_id=row.content_id, - record_id=row.record_id, - extract_metadata=extract_metadata, - compute_hash=compute_hash, - progress=progress, - ) - if updated: - enriched += 1 - else: - failed_ids.append(row.record_id) - except Exception as exc: - if progress is not None: - progress.enrich_failed += 1 - if progress is None or progress.mark_emitted("enrich_failed"): - emit("scanner.enrich_failed", error_type=error_type(exc)) - logging.warning("Failed to enrich %s: %s", row.file_path, exc) - sess.rollback() + for row in rows: + if interrupt_check is not None and interrupt_check(): + break + try: + prepared = _prepare_enrichment( + row, extract_metadata, compute_hash, progress + ) + if prepared is None: failed_ids.append(row.record_id) + continue + updated = run_write_txn( + lambda session: _apply_enrichment(session, prepared) + ) + if updated: + enriched += 1 + else: + failed_ids.append(row.record_id) + except Exception as exc: + if progress is not None: + progress.enrich_failed += 1 + if progress is None or progress.mark_emitted("enrich_failed"): + emit("scanner.enrich_failed", error_type=error_type(exc)) + logging.warning("Failed to enrich %s: %s", row.file_path, exc) + failed_ids.append(row.record_id) return enriched, failed_ids diff --git a/app/assets/scanner_admission.py b/app/assets/scanner_admission.py index 7c059b005a0..b355414a280 100644 --- a/app/assets/scanner_admission.py +++ b/app/assets/scanner_admission.py @@ -66,14 +66,16 @@ def _two_stat_admit(paths_with_stats: list[tuple[str, os.stat_result]]) -> tuple return admitted, watched -def tick_watch_list(session: Session) -> None: - from app.assets.scanner import seed_asset_specs, SeedAssetSpec +def tick_watch_list(_session: Session | None = None) -> None: + from app.assets.scanner import SeedAssetSpec, insert_asset_specs - remaining: list[_WatchEntry] = [] - for entry in _WATCH_LIST: + queued_count = len(_WATCH_LIST) + for _ in range(queued_count): + entry = _WATCH_LIST[0] try: current = os.stat(entry.path) except FileNotFoundError: + _WATCH_LIST.pop(0) continue if (current.st_mtime_ns, current.st_size) == (entry.last_stat.st_mtime_ns, entry.last_stat.st_size): name, tags = get_name_and_tags_from_asset_path(entry.path) @@ -88,10 +90,10 @@ def tick_watch_list(session: Session) -> None: "mime_type": mimetypes.guess_type(entry.path, strict=False)[0], "job_id": None, } - seed_asset_specs(session, [spec]) + insert_asset_specs([spec], set(spec["tags"])) + _WATCH_LIST.pop(0) continue - entry.last_stat = current - entry.ticks += 1 - if entry.ticks < _WATCH_SCAN_RETRIES: - remaining.append(entry) - _WATCH_LIST[:] = remaining + next_entry = _WatchEntry(entry.path, current, entry.ticks + 1) + _WATCH_LIST.pop(0) + if next_entry.ticks < _WATCH_SCAN_RETRIES: + _WATCH_LIST.append(next_entry) diff --git a/app/assets/scanner_changes.py b/app/assets/scanner_changes.py index 43ee8a47a98..dd696439a88 100644 --- a/app/assets/scanner_changes.py +++ b/app/assets/scanner_changes.py @@ -9,7 +9,7 @@ import os from pathlib import Path -from typing import Literal +from typing import Literal, NamedTuple import sqlalchemy as sa from sqlalchemy.orm import Session @@ -24,11 +24,31 @@ from app.assets.helpers import sql_path_under_prefix, to_stored_hash from app.assets.services.path_utils import compute_loader_path, get_name_and_tags_from_asset_path from app.assets.services.snapshot_hash import snapshot_hash +from app.database.db import create_session, run_write_txn _pending_verification_ids: list[str] = [] _pending_recovery_paths: list[str] = [] +class PreparedRecovery(NamedTuple): + path: str + initial_stat: os.stat_result + snapshot: tuple[str, os.stat_result] | None + + +class _PendingVerificationPreflight(NamedTuple): + content_id: str + path: str | None + content_hash: str | None + size_bytes: int | None + mtime_ns: int | None + outcome: Literal["drop", "gone", "retry", "ready"] + + +def prepare_missing_content_recovery(path: str, stat_result: os.stat_result) -> PreparedRecovery: + return PreparedRecovery(path, stat_result, snapshot_hash(path)) + + def clear_pending_verifications() -> None: _pending_verification_ids.clear() _pending_recovery_paths.clear() @@ -39,6 +59,11 @@ def queue_pending_verification(content_id: str) -> None: _pending_verification_ids.append(content_id) +def queue_pending_recovery(path: str) -> None: + if path not in _pending_recovery_paths: + _pending_recovery_paths.append(path) + + def pending_recovery_count() -> int: return len(_pending_recovery_paths) @@ -103,6 +128,77 @@ def recover_missing_content( return "recovered" +def recover_missing_content_from_preparation( + session: Session, + path: str, + stat_result: os.stat_result, + prepared: PreparedRecovery, + pending_recovery_paths: list[str], +) -> Literal["recovered", "no_match", "unstable"]: + occupied = session.scalar( + sa.select(AssetContent.id) + .where(AssetContent.path == path, AssetContent.is_missing.is_(False)) + .limit(1) + ) + if occupied is not None: + return "no_match" + if prepared.snapshot is None or ( + prepared.initial_stat.st_size != stat_result.st_size + or prepared.initial_stat.st_mtime_ns != stat_result.st_mtime_ns + ): + if path not in pending_recovery_paths: + pending_recovery_paths.append(path) + return "unstable" + digest, verified_stat = prepared.snapshot + if ( + verified_stat.st_size != stat_result.st_size + or verified_stat.st_mtime_ns != stat_result.st_mtime_ns + ): + if path not in pending_recovery_paths: + pending_recovery_paths.append(path) + return "unstable" + stored_hash = to_stored_hash(digest) + matches = list( + session.scalars( + sa.select(AssetContent).where( + AssetContent.path == path, + AssetContent.is_missing.is_(True), + AssetContent.hash == stored_hash, + ) + ) + ) + if len(matches) == 1: + recovered = matches[0] + unset_content_missing(session, recovered.id) + recovered.size_bytes = verified_stat.st_size + recovered.mtime_ns = verified_stat.st_mtime_ns + return "recovered" + if len(matches) > 1: + return "no_match" + null_hash_matches = list( + session.scalars( + sa.select(AssetContent).where( + AssetContent.path == path, + AssetContent.is_missing.is_(True), + AssetContent.hash.is_(None), + ) + ) + ) + if len(null_hash_matches) != 1: + return "no_match" + candidate = null_hash_matches[0] + if (candidate.size_bytes, candidate.mtime_ns) != ( + verified_stat.st_size, + verified_stat.st_mtime_ns, + ): + return "no_match" + unset_content_missing(session, candidate.id) + candidate.hash = stored_hash + candidate.size_bytes = verified_stat.st_size + candidate.mtime_ns = verified_stat.st_mtime_ns + return "recovered" + + def is_path_under_prefixes(path: str, prefixes: list[str]) -> bool: candidate = Path(os.path.abspath(path)) return any(candidate.is_relative_to(os.path.abspath(prefix)) for prefix in prefixes) @@ -133,12 +229,16 @@ def detect_content_change( content: AssetContent, stat_result: os.stat_result, hashing_is_enabled: bool, + pending_verification_ids: list[str] | None = None, ) -> None: if content.mtime_ns == stat_result.st_mtime_ns: # Ruling #10: size drift with unchanged mtime is undefined behavior. return if hashing_is_enabled: - queue_pending_verification(content.id) + if pending_verification_ids is None: + queue_pending_verification(content.id) + elif content.id not in pending_verification_ids: + pending_verification_ids.append(content.id) return if content.size_bytes == stat_result.st_size: # User identity rule: a same-size mtime bump (rsync, cloud sync, backup restore) is the @@ -153,42 +253,102 @@ def detect_content_change( split_content(session, content, stat_result, hash_value=None) -def drain_pending_verifications(session: Session, limit: int | None = None) -> int: - queued_count = min(len(_pending_verification_ids), limit or len(_pending_verification_ids)) - processed = 0 - for _ in range(queued_count): - content_id = _pending_verification_ids.pop(0) +def _preflight_pending_verification( + content_id: str, +) -> _PendingVerificationPreflight: + with create_session() as session: content = session.get(AssetContent, content_id) if content is None or content.is_missing: - continue + return _PendingVerificationPreflight( + content_id, None, None, None, None, "drop" + ) + path = content.path + content_hash = content.hash + size_bytes = content.size_bytes + mtime_ns = content.mtime_ns + try: + os.stat(path, follow_symlinks=True) + except FileNotFoundError: + outcome: Literal["drop", "gone", "retry", "ready"] = "gone" + except OSError: + outcome = "retry" + else: + outcome = "ready" + return _PendingVerificationPreflight( + content_id, path, content_hash, size_bytes, mtime_ns, outcome + ) + + +def _apply_pending_verification( + session: Session, + preflight: _PendingVerificationPreflight, + snapshot: tuple[str, os.stat_result] | None, +) -> Literal["drop", "processed", "retry"]: + if preflight.outcome == "drop": + return "drop" + content = session.get(AssetContent, preflight.content_id) + if ( + content is None + or content.is_missing + or content.path != preflight.path + or content.hash != preflight.content_hash + or content.size_bytes != preflight.size_bytes + or content.mtime_ns != preflight.mtime_ns + ): + return "drop" + assert preflight.path is not None + if preflight.outcome == "gone": try: - os.stat(content.path, follow_symlinks=True) + os.stat(preflight.path, follow_symlinks=True) except FileNotFoundError: mark_content_missing(session, content.id) - processed += 1 - continue + return "processed" except OSError: - queue_pending_verification(content_id) - continue + return "retry" + return "retry" + if preflight.outcome == "retry" or snapshot is None: + return "retry" + digest, verified_stat = snapshot + try: + current_stat = os.stat(preflight.path, follow_symlinks=True) + except OSError: + return "retry" + if ( + current_stat.st_size != verified_stat.st_size + or current_stat.st_mtime_ns != verified_stat.st_mtime_ns + ): + return "retry" + stored_hash = to_stored_hash(digest) + if content.hash == stored_hash or content.hash is None: + content.hash = stored_hash + content.size_bytes = verified_stat.st_size + content.mtime_ns = verified_stat.st_mtime_ns + else: + split_content(session, content, verified_stat, hash_value=stored_hash) + return "processed" - try: - snapshot = snapshot_hash(content.path) - except OSError: - queue_pending_verification(content_id) - continue - if snapshot is None: + +def drain_pending_verifications(_session: Session | None = None, limit: int | None = None) -> int: + queued_count = min(len(_pending_verification_ids), limit or len(_pending_verification_ids)) + processed = 0 + for _ in range(queued_count): + content_id = _pending_verification_ids[0] + preflight = _preflight_pending_verification(content_id) + snapshot: tuple[str, os.stat_result] | None = None + if preflight.outcome == "ready": + assert preflight.path is not None + try: + snapshot = snapshot_hash(preflight.path) + except OSError: + preflight = preflight._replace(outcome="retry") + outcome = run_write_txn( + lambda session: _apply_pending_verification(session, preflight, snapshot) + ) + _pending_verification_ids.pop(0) + if outcome == "retry": queue_pending_verification(content_id) - continue - digest, verified_stat = snapshot - stored_hash = to_stored_hash(digest) - - if content.hash == stored_hash or content.hash is None: - content.hash = stored_hash - content.size_bytes = verified_stat.st_size - content.mtime_ns = verified_stat.st_mtime_ns - else: - split_content(session, content, verified_stat, hash_value=stored_hash) - processed += 1 + elif outcome == "processed": + processed += 1 return processed diff --git a/app/assets/seeder.py b/app/assets/seeder.py index a78f1abf440..b8433ee8f1a 100644 --- a/app/assets/seeder.py +++ b/app/assets/seeder.py @@ -31,7 +31,7 @@ tick_watch_list, ) from app.assets.services.hash_mode_state import drain_transition_queue, pending_transition_count -from app.database.db import create_session, dependencies_available +from app.database.db import dependencies_available class ScanInProgressError(Exception): @@ -873,9 +873,7 @@ def _run_fast_phase(self, roots: tuple[RootType, ...]) -> tuple[int, int, int]: last_progress_time = now self._update_progress(scanned=len(specs), created=total_created) - with create_session() as session: - tick_watch_list(session) - session.commit() + tick_watch_list() logging.info( "Fast scan complete: %.3fs total (created=%d, skipped=%d, total_paths=%d)", time.perf_counter() - t_fast_start, @@ -893,14 +891,12 @@ def _run_enrich_phase(self, roots: tuple[RootType, ...]) -> tuple[bool, int]: """ total_enriched = 0 scan_state = self._scan_state - with create_session() as session: - drain_pending_verifications(session) - tick_watch_list(session) - for _ in range(3): - drain_transition_queue(session) - session.commit() - if pending_transition_count() == 0: - break + drain_pending_verifications() + tick_watch_list() + for _ in range(3): + drain_transition_queue() + if pending_transition_count() == 0: + break batch_size = 100 last_progress_time = time.perf_counter() progress_interval = 1.0 diff --git a/app/assets/services/hash_mode_state.py b/app/assets/services/hash_mode_state.py index 0e802a9517d..e5d7760ef9e 100644 --- a/app/assets/services/hash_mode_state.py +++ b/app/assets/services/hash_mode_state.py @@ -23,6 +23,7 @@ from app.assets.helpers import to_stored_hash from app.assets.services.path_utils import compute_loader_path, get_name_and_tags_from_asset_path from app.assets.services.snapshot_hash import snapshot_hash +from app.database.db import run_write_txn _KEY = "hash_mode" _MAX_VERIFY_ATTEMPTS: Final = 3 @@ -94,82 +95,97 @@ def enqueue_transition_work(session: Session, transition: str | None) -> None: _PENDING_PATHS.add(row.path) -def _retry_or_retire(session: Session, entry: _PendingEntry) -> None: - entry.ticks += 1 - if entry.ticks < _MAX_VERIFY_ATTEMPTS: - _PENDING_QUEUE.append(entry) - _PENDING_PATHS.add(entry.path) - return - content = session.scalars( - select(AssetContent).where( - AssetContent.path == entry.path, AssetContent.is_missing.is_(False) - ) - ).first() - if content is not None: - content.hash = None - logging.warning( - "Could not verify %s in %d attempts; clearing its stored hash so the hash-mode " - "transition can complete", - entry.path, - _MAX_VERIFY_ATTEMPTS, - ) - - -def drain_transition_queue(session: Session) -> None: +def drain_transition_queue(_session: Session | None = None) -> None: global _off_to_on_transition_in_flight pending_count = len(_PENDING_QUEUE) for _ in range(pending_count): - entry = _PENDING_QUEUE.popleft() - _PENDING_PATHS.discard(entry.path) - path = entry.path + entry = _PENDING_QUEUE[0] + snapshot: tuple[str, os.stat_result] | None = None + preparation = "ready" try: - snapshot = snapshot_hash(path) + snapshot = snapshot_hash(entry.path) except OSError: - _retry_or_retire(session, entry) - continue - if snapshot is None: - # snapshot_hash returns None for vanished and unstable files; stat distinguishes them. + preparation = "retry" + if snapshot is None and preparation != "retry": try: - os.stat(path) + os.stat(entry.path) except FileNotFoundError: - gone = session.scalars( + preparation = "gone" + except OSError: + preparation = "retry" + else: + preparation = "retry" + + def _apply(session: Session) -> str: + if preparation == "retry" or ( + snapshot is None and preparation != "gone" + ): + if entry.ticks + 1 < _MAX_VERIFY_ATTEMPTS: + return "retry" + content = session.scalars( select(AssetContent).where( - AssetContent.path == path, AssetContent.is_missing.is_(False) + AssetContent.path == entry.path, + AssetContent.is_missing.is_(False), ) ).first() - if gone is not None: - mark_content_missing(session, gone.id) + if content is not None: + content.hash = None + logging.warning( + "Could not verify %s in %d attempts; clearing its stored hash so the hash-mode " + "transition can complete", + entry.path, + _MAX_VERIFY_ATTEMPTS, + ) + return "drop" + content = session.scalars( + select(AssetContent).where( + AssetContent.path == entry.path, + AssetContent.is_missing.is_(False), + ) + ).first() + if content is None: + return "drop" + if preparation == "gone": + try: + os.stat(entry.path) + except FileNotFoundError: + mark_content_missing(session, content.id) + return "drop" + except OSError: + return "retry" + return "retry" + digest, stat = snapshot + try: + current_stat = os.stat(entry.path) except OSError: - _retry_or_retire(session, entry) - else: - _retry_or_retire(session, entry) - continue - digest, stat = snapshot - stored_hash = to_stored_hash(digest) - content = session.scalars( - select(AssetContent).where( - AssetContent.path == path, AssetContent.is_missing.is_(False) - ) - ).first() - if content is None: - continue - if content.hash is None: - content.hash = stored_hash - content.size_bytes = stat.st_size - content.mtime_ns = stat.st_mtime_ns - elif content.hash != stored_hash: + return "retry" + if ( + current_stat.st_size != stat.st_size + or current_stat.st_mtime_ns != stat.st_mtime_ns + ): + return "retry" + stored_hash = to_stored_hash(digest) + if content.hash is None: + content.hash = stored_hash + content.size_bytes = stat.st_size + content.mtime_ns = stat.st_mtime_ns + return "drop" + if content.hash == stored_hash: + content.size_bytes = stat.st_size + content.mtime_ns = stat.st_mtime_ns + return "drop" try: - name, tags = get_name_and_tags_from_asset_path(path) + name, tags = get_name_and_tags_from_asset_path(entry.path) except ValueError: logging.warning( - "Skipping hash-mode split for out-of-root path: %s", path + "Skipping hash-mode split for out-of-root path: %s", entry.path ) - continue + return "drop" mark_content_missing(session, content.id) replacement = create_content( session, - path=path, + path=entry.path, hash=stored_hash, size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns, @@ -178,12 +194,17 @@ def drain_transition_queue(session: Session) -> None: session, content_id=replacement.id, name=name, - loader_path=compute_loader_path(path), + loader_path=compute_loader_path(entry.path), tags=tags, ) + return "drop" + + outcome = run_write_txn(_apply) + _PENDING_QUEUE.popleft() + if outcome == "retry": + _PENDING_QUEUE.append(_PendingEntry(entry.path, entry.ticks + 1)) else: - content.size_bytes = stat.st_size - content.mtime_ns = stat.st_mtime_ns + _PENDING_PATHS.discard(entry.path) if _off_to_on_transition_in_flight and not _PENDING_QUEUE: - write_stored_mode(session, "on") + run_write_txn(lambda session: write_stored_mode(session, "on")) _off_to_on_transition_in_flight = False diff --git a/tests-unit/assets_test/helpers.py b/tests-unit/assets_test/helpers.py index 4e400184b99..9ffb8bdf77c 100644 --- a/tests-unit/assets_test/helpers.py +++ b/tests-unit/assets_test/helpers.py @@ -16,8 +16,9 @@ from sqlalchemy import Engine, create_engine from sqlalchemy.orm import Session +from app.assets import scanner from app.assets.api import routes -from app.assets.database.models import Asset +from app.assets.database.models import Asset, AssetContent from app.assets.database.queries.records import create_content, create_record from app.database.models import Base @@ -75,6 +76,28 @@ def route_database(monkeypatch: pytest.MonkeyPatch) -> Iterator[RouteDatabase]: engine.dispose() +def enrich_via_prepare_apply( + session: Session, + *, + file_path: str, + content_id: str, + record_id: str, + extract_metadata: bool = True, + compute_hash: bool = False, + progress=None, +) -> bool: + content = session.get(AssetContent, content_id) + row = scanner.UnenrichedContent( + content_id, record_id, file_path, content is not None and content.hash is None + ) + prepared = scanner._prepare_enrichment(row, extract_metadata, compute_hash, progress) + if prepared is None: + return False + updated = scanner._apply_enrichment(session, prepared) + session.commit() + return updated + + def seed_record(session: Session, seed: RecordSeed) -> Asset: content = create_content( session, diff --git a/tests-unit/assets_test/services/conftest.py b/tests-unit/assets_test/services/conftest.py index 545801da8f2..040cba3f6c7 100644 --- a/tests-unit/assets_test/services/conftest.py +++ b/tests-unit/assets_test/services/conftest.py @@ -51,8 +51,11 @@ def _set_pragma(dbapi_connection, connection_record): @pytest.fixture -def session(db_engine): +def session(db_engine, monkeypatch): """Session fixture for tests that need direct DB access.""" + factory = sessionmaker(bind=db_engine) + monkeypatch.setattr("app.database.db.Session", factory) + monkeypatch.setattr("app.database.db.WriteSession", factory) with Session(db_engine) as sess: yield sess diff --git a/tests-unit/assets_test/services/test_b1_counter_and_retry_integrity.py b/tests-unit/assets_test/services/test_b1_counter_and_retry_integrity.py new file mode 100644 index 00000000000..95d3a038048 --- /dev/null +++ b/tests-unit/assets_test/services/test_b1_counter_and_retry_integrity.py @@ -0,0 +1,188 @@ +import logging +import os +import sqlite3 +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import sessionmaker + +import app.database.db as db_mod +from app.assets import mode as mode_module +from app.assets import scanner +from app.assets import scanner_changes +from app.assets.database.models import AssetContent +from app.assets.database.queries.records import create_record +from app.assets.seeder import _ScanState + +_LOCKED_ERROR = OperationalError("COMMIT", {}, sqlite3.OperationalError("database is locked")) + + +def _fail_commit_once_then_succeed(engine): + real_factory = sessionmaker(bind=engine) + remaining = {"count": 1} + + def factory(): + session = real_factory() + real_commit = session.commit + + def commit(): + if remaining["count"] > 0: + remaining["count"] -= 1 + session.rollback() + raise _LOCKED_ERROR + return real_commit() + + session.commit = commit + return session + + return factory + + +def _fail_run_write_txn_at(real_run_write_txn, fail_index: int): + calls = {"count": -1} + + def wrapper(work): + calls["count"] += 1 + if calls["count"] == fail_index: + raise _LOCKED_ERROR + return real_run_write_txn(work) + + return wrapper + + +@pytest.fixture(autouse=True) +def _fast_retries(monkeypatch): + monkeypatch.setattr(db_mod.time, "sleep", lambda _seconds: None) + + +def test_group_a_transient_locked_error_retried_preserves_queue_once( + db_engine, tmp_path: Path, monkeypatch, session +): + """A commit-time locked failure that later succeeds must publish the + verification queue exactly once, never per attempt.""" + path = tmp_path / "drifted.bin" + path.write_bytes(b"drifted bytes") + stale_mtime_ns = path.stat().st_mtime_ns - 5_000_000_000 + content = AssetContent( + path=str(path), hash=None, size_bytes=path.stat().st_size, mtime_ns=stale_mtime_ns + ) + session.add(content) + session.flush() + create_record(session, content.id, "drifted.bin") + session.commit() + content_id = content.id + + scanner_changes.clear_pending_verifications() + monkeypatch.setattr(db_mod, "WriteSession", _fail_commit_once_then_succeed(db_engine)) + + class _HashingOn: + enable_asset_hashing = True + + mode_module.init(_HashingOn()) + try: + with patch("folder_paths.get_input_directory", return_value=str(tmp_path)): + survivors = scanner.sync_root_safely("input", _ScanState()) + finally: + mode_module.init(None) + + assert survivors == {os.path.abspath(str(path))} + assert scanner_changes._pending_verification_ids == [content_id] + scanner_changes.clear_pending_verifications() + + +def test_group_a_sync_permission_diagnostic_published_exactly_once_after_retry( + db_engine, tmp_path: Path, monkeypatch, session, caplog +): + """A commit-time locked failure that later succeeds must publish the + permission-denied diagnostic exactly once: one counter bump, one emit.""" + path = tmp_path / "unreadable.bin" + path.write_bytes(b"unreadable") + content = AssetContent( + path=str(path), hash=None, size_bytes=path.stat().st_size, mtime_ns=path.stat().st_mtime_ns + ) + session.add(content) + session.flush() + session.commit() + + real_stat = os.stat + + def deny_stat(candidate_path, *args, **kwargs): + if str(candidate_path) == str(path): + raise PermissionError(str(path)) + return real_stat(candidate_path, *args, **kwargs) + + monkeypatch.setattr(scanner, "os", SimpleNamespace(stat=deny_stat, path=scanner.os.path)) + monkeypatch.setattr(db_mod, "WriteSession", _fail_commit_once_then_succeed(db_engine)) + + progress = _ScanState() + with ( + patch("folder_paths.get_input_directory", return_value=str(tmp_path)), + caplog.at_level(logging.INFO), + ): + scanner.sync_root_safely("input", progress) + + assert progress.permission_denied == 1 + stat_failed_lines = [ + r.getMessage() + for r in caplog.records + if r.getMessage().startswith("[assets-event] scanner.stat_failed") + ] + assert stat_failed_lines == [ + "[assets-event] scanner.stat_failed error_type=PermissionError site=reference_stat" + ] + + +def test_b1_counter_integrity_under_a_locked_failure_at_row_n( + db_engine, tmp_path: Path, session, monkeypatch +): + """Injecting a locked failure at the middle row of a batch must not + double-mark the surviving rows, and must match a clean run exactly.""" + paths = [tmp_path / f"row-{i}.bin" for i in range(3)] + rows = [] + for i, path in enumerate(paths): + path.write_bytes(f"payload-{i}".encode()) + stat = path.stat() + content = AssetContent( + path=str(path), hash=None, size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns + ) + session.add(content) + session.flush() + record = create_record(session, content.id, path.name) + rows.append(scanner.UnenrichedContent(content.id, record.id, str(path), True)) + session.commit() + content_ids = [row.content_id for row in rows] + + real_run_write_txn = scanner.run_write_txn + monkeypatch.setattr( + scanner, "run_write_txn", _fail_run_write_txn_at(real_run_write_txn, fail_index=1) + ) + + enriched, failed_ids = scanner.enrich_assets_batch( + rows, extract_metadata=False, compute_hash=True + ) + + assert enriched == 2 + assert failed_ids == [rows[1].record_id] + + hashes_after_failure = { + content_id: session.get(AssetContent, content_id).hash for content_id in content_ids + } + assert hashes_after_failure[content_ids[1]] is None + assert hashes_after_failure[content_ids[0]] is not None + assert hashes_after_failure[content_ids[2]] is not None + + monkeypatch.setattr(scanner, "run_write_txn", real_run_write_txn) + retry_enriched, retry_failed_ids = scanner.enrich_assets_batch( + [rows[1]], extract_metadata=False, compute_hash=True + ) + assert retry_enriched == 1 + assert retry_failed_ids == [] + session.expire_all() + final_hashes = { + content_id: session.get(AssetContent, content_id).hash for content_id in content_ids + } + assert all(value is not None for value in final_hashes.values()) + assert len(session.scalars(scanner.sa.select(AssetContent)).all()) == 3 diff --git a/tests-unit/assets_test/services/test_b2_lock_hold_barriers.py b/tests-unit/assets_test/services/test_b2_lock_hold_barriers.py new file mode 100644 index 00000000000..dc82b44bc88 --- /dev/null +++ b/tests-unit/assets_test/services/test_b2_lock_hold_barriers.py @@ -0,0 +1,298 @@ +import threading +import time +import uuid + +import pytest +from sqlalchemy import text + +import app.assets.mode as mode_module +import app.database.db as db_mod +from app.assets import scanner +from app.assets import scanner_changes +from app.assets.database.queries.records import create_content, create_record +from app.assets.services import hash_mode_state + +_BARRIER_TIMEOUT = 5 +_PROBE_BUDGET_SECONDS = 1.0 + + +@pytest.fixture +def file_database(tmp_path, monkeypatch): + database_path = str(tmp_path / "assets.db") + monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") + monkeypatch.setattr(db_mod, "Session", None) + monkeypatch.setattr(db_mod, "_db_lock", None) + if hasattr(db_mod, "WriteSession"): + monkeypatch.setattr(db_mod, "WriteSession", None) + db_mod.init_db() + yield database_path + for factory in (db_mod.Session, getattr(db_mod, "WriteSession", None)): + if factory is not None: + factory.kw["bind"].dispose() + db_mod._db_lock.release(force=True) + + +@pytest.fixture +def hashing_on(): + class FakeArgs: + enable_asset_hashing = True + + mode_module.init(FakeArgs()) + yield + mode_module.init(None) + + +def _probe_write() -> None: + name = f"probe-{uuid.uuid4().hex}" + db_mod.run_write_txn( + lambda session: session.execute( + text("INSERT INTO tags (name) VALUES (:name)"), {"name": name} + ) + ) + + +def _blocking_fake(entered: threading.Event, release: threading.Event, real_fn): + def fake(*args, **kwargs): + entered.set() + release.wait(timeout=_BARRIER_TIMEOUT) + return real_fn(*args, **kwargs) + + return fake + + +def test_seed_recovery_hashing_does_not_hold_the_write_lock( + file_database, hashing_on, tmp_path, monkeypatch +): + path = tmp_path / "recoverable.bin" + path.write_bytes(b"recoverable bytes") + + entered = threading.Event() + release = threading.Event() + monkeypatch.setattr( + scanner_changes, + "snapshot_hash", + _blocking_fake(entered, release, scanner_changes.snapshot_hash), + ) + + stat = path.stat() + spec: scanner.SeedAssetSpec = { + "abs_path": str(path), + "size_bytes": stat.st_size, + "mtime_ns": stat.st_mtime_ns, + "info_name": "recoverable.bin", + "tags": [], + "fname": None, + "metadata": None, + "mime_type": None, + "job_id": None, + } + + result: dict[str, int] = {} + + def _seed() -> None: + result["created"] = scanner.insert_asset_specs([spec], set()) + + worker = threading.Thread(target=_seed) + worker.start() + try: + assert entered.wait(timeout=_BARRIER_TIMEOUT) + started = time.monotonic() + _probe_write() + elapsed = time.monotonic() - started + finally: + release.set() + worker.join(timeout=_BARRIER_TIMEOUT) + + assert not worker.is_alive() + assert elapsed < _PROBE_BUDGET_SECONDS + assert result["created"] == 1 + + +def test_pending_verification_hashing_does_not_hold_the_write_lock( + file_database, tmp_path, monkeypatch +): + path = tmp_path / "verify-me.bin" + path.write_bytes(b"verify me") + stat = path.stat() + + with db_mod.Session() as session: + content = create_content( + session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns + ) + create_record(session, content.id, "verify-me.bin") + session.commit() + content_id = content.id + + scanner_changes.clear_pending_verifications() + scanner_changes.queue_pending_verification(content_id) + + entered = threading.Event() + release = threading.Event() + monkeypatch.setattr( + scanner_changes, + "snapshot_hash", + _blocking_fake(entered, release, scanner_changes.snapshot_hash), + ) + + result: dict[str, int] = {} + + def _drain() -> None: + result["processed"] = scanner_changes.drain_pending_verifications() + + worker = threading.Thread(target=_drain) + worker.start() + try: + assert entered.wait(timeout=_BARRIER_TIMEOUT) + started = time.monotonic() + _probe_write() + elapsed = time.monotonic() - started + finally: + release.set() + worker.join(timeout=_BARRIER_TIMEOUT) + scanner_changes.clear_pending_verifications() + + assert not worker.is_alive() + assert elapsed < _PROBE_BUDGET_SECONDS + assert result["processed"] == 1 + + +def test_transition_hashing_does_not_hold_the_write_lock( + file_database, tmp_path, monkeypatch +): + path = tmp_path / "transition-me.bin" + path.write_bytes(b"transition me") + stat = path.stat() + + with db_mod.Session() as session: + create_content(session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns) + session.commit() + + hash_mode_state.clear_transition_queue() + hash_mode_state._PENDING_QUEUE.append(hash_mode_state._PendingEntry(str(path))) + hash_mode_state._PENDING_PATHS.add(str(path)) + + entered = threading.Event() + release = threading.Event() + monkeypatch.setattr( + hash_mode_state, + "snapshot_hash", + _blocking_fake(entered, release, hash_mode_state.snapshot_hash), + ) + + result: dict[str, bool] = {} + + def _drain() -> None: + hash_mode_state.drain_transition_queue() + result["done"] = True + + worker = threading.Thread(target=_drain) + worker.start() + try: + assert entered.wait(timeout=_BARRIER_TIMEOUT) + started = time.monotonic() + _probe_write() + elapsed = time.monotonic() - started + finally: + release.set() + worker.join(timeout=_BARRIER_TIMEOUT) + hash_mode_state.clear_transition_queue() + + assert not worker.is_alive() + assert elapsed < _PROBE_BUDGET_SECONDS + assert result.get("done") is True + + +def test_b1_enrichment_hashing_does_not_hold_the_write_lock( + file_database, tmp_path, monkeypatch +): + path = tmp_path / "enrich-hash.bin" + path.write_bytes(b"enrich me via hash") + stat = path.stat() + + with db_mod.Session() as session: + content = create_content( + session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns + ) + record = create_record(session, content.id, "enrich-hash.bin") + session.commit() + content_id, record_id = content.id, record.id + + row = scanner.UnenrichedContent(content_id, record_id, str(path), True) + + entered = threading.Event() + release = threading.Event() + monkeypatch.setattr( + scanner, + "snapshot_hash", + _blocking_fake(entered, release, scanner.snapshot_hash), + ) + + result: dict[str, object] = {} + + def _enrich() -> None: + result["outcome"] = scanner.enrich_assets_batch( + [row], extract_metadata=False, compute_hash=True + ) + + worker = threading.Thread(target=_enrich) + worker.start() + try: + assert entered.wait(timeout=_BARRIER_TIMEOUT) + started = time.monotonic() + _probe_write() + elapsed = time.monotonic() - started + finally: + release.set() + worker.join(timeout=_BARRIER_TIMEOUT) + + assert not worker.is_alive() + assert elapsed < _PROBE_BUDGET_SECONDS + assert result["outcome"] == (1, []) + + +def test_b1_enrichment_metadata_extraction_does_not_hold_the_write_lock( + file_database, tmp_path, monkeypatch +): + path = tmp_path / "enrich-metadata.bin" + path.write_bytes(b"enrich me via metadata") + stat = path.stat() + + with db_mod.Session() as session: + content = create_content( + session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns + ) + record = create_record(session, content.id, "enrich-metadata.bin") + session.commit() + content_id, record_id = content.id, record.id + + row = scanner.UnenrichedContent(content_id, record_id, str(path), False) + + entered = threading.Event() + release = threading.Event() + monkeypatch.setattr( + scanner, + "extract_file_metadata", + _blocking_fake(entered, release, scanner.extract_file_metadata), + ) + + result: dict[str, object] = {} + + def _enrich() -> None: + result["outcome"] = scanner.enrich_assets_batch( + [row], extract_metadata=True, compute_hash=False + ) + + worker = threading.Thread(target=_enrich) + worker.start() + try: + assert entered.wait(timeout=_BARRIER_TIMEOUT) + started = time.monotonic() + _probe_write() + elapsed = time.monotonic() - started + finally: + release.set() + worker.join(timeout=_BARRIER_TIMEOUT) + + assert not worker.is_alive() + assert elapsed < _PROBE_BUDGET_SECONDS + assert result["outcome"] == (1, []) diff --git a/tests-unit/assets_test/services/test_b2_queue_commit_failure_integrity.py b/tests-unit/assets_test/services/test_b2_queue_commit_failure_integrity.py new file mode 100644 index 00000000000..1e361df44ae --- /dev/null +++ b/tests-unit/assets_test/services/test_b2_queue_commit_failure_integrity.py @@ -0,0 +1,172 @@ +import sqlite3 +from pathlib import Path +from unittest.mock import patch + +import pytest +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import sessionmaker + +import app.database.db as db_mod +from app.assets import scanner +from app.assets import scanner_admission +from app.assets import scanner_changes +from app.assets.database.models import AssetContent +from app.assets.database.queries.records import create_content +from app.assets.scanner_admission import _WatchEntry +from app.assets.services import hash_mode_state +from app.assets.services.hash_mode_state import ( + clear_transition_queue, + drain_transition_queue, + enqueue_transition_work, + read_stored_mode, + record_transition_intent, + write_stored_mode, +) + +_LOCKED_ERROR = OperationalError("COMMIT", {}, sqlite3.OperationalError("database is locked")) + + +def _fail_commit_always(engine): + real_factory = sessionmaker(bind=engine) + + def factory(): + session = real_factory() + + def commit(): + session.rollback() + raise _LOCKED_ERROR + + session.commit = commit + return session + + return factory + + +def _fail_run_write_txn_at(real_run_write_txn, fail_index: int): + calls = {"count": -1} + + def wrapper(work): + calls["count"] += 1 + if calls["count"] == fail_index: + raise _LOCKED_ERROR + return real_run_write_txn(work) + + return wrapper + + +@pytest.fixture(autouse=True) +def _fast_retries(monkeypatch): + monkeypatch.setattr(db_mod.time, "sleep", lambda _seconds: None) + + +def test_pending_verification_queue_survives_terminal_commit_failure( + db_engine, tmp_path: Path, session, monkeypatch +): + path = tmp_path / "terminal.bin" + path.write_bytes(b"terminal failure content") + content = AssetContent( + path=str(path), hash=None, size_bytes=path.stat().st_size, mtime_ns=path.stat().st_mtime_ns + ) + session.add(content) + session.flush() + session.commit() + content_id = content.id + + scanner_changes.clear_pending_verifications() + scanner_changes.queue_pending_verification(content_id) + monkeypatch.setattr(db_mod, "WriteSession", _fail_commit_always(db_engine)) + + with pytest.raises(OperationalError): + scanner_changes.drain_pending_verifications() + + assert scanner_changes._pending_verification_ids == [content_id] + scanner_changes.clear_pending_verifications() + + +def test_watch_list_survives_terminal_commit_failure(db_engine, tmp_path: Path, monkeypatch): + path = tmp_path / "watched-terminal.bin" + path.write_bytes(b"watched") + stat = path.stat() + entry = _WatchEntry(str(path), stat) + scanner_admission._WATCH_LIST[:] = [entry] + monkeypatch.setattr(db_mod, "WriteSession", _fail_commit_always(db_engine)) + + with ( + patch("folder_paths.get_input_directory", return_value=str(tmp_path)), + pytest.raises(OperationalError), + ): + scanner_admission.tick_watch_list() + + assert scanner_admission._WATCH_LIST == [entry] + scanner_admission._WATCH_LIST.clear() + + +def test_transition_queue_and_companion_state_survive_terminal_commit_failure( + db_engine, tmp_path: Path, session, monkeypatch +): + path = tmp_path / "transition-terminal.bin" + path.write_bytes(b"transition terminal") + stat = path.stat() + create_content(session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns) + write_stored_mode(session, "off") + session.commit() + + from app.assets import mode as mode_module + + monkeypatch.setattr(mode_module, "hashing_enabled", lambda: True) + + clear_transition_queue() + transition = record_transition_intent(session) + enqueue_transition_work(session, transition) + session.commit() + assert hash_mode_state._off_to_on_transition_in_flight is True + + monkeypatch.setattr(db_mod, "WriteSession", _fail_commit_always(db_engine)) + + with pytest.raises(OperationalError): + drain_transition_queue() + + assert {e.path for e in hash_mode_state._PENDING_QUEUE} == hash_mode_state._PENDING_PATHS + assert str(path) in hash_mode_state._PENDING_PATHS + assert hash_mode_state._off_to_on_transition_in_flight is True + clear_transition_queue() + + +def test_transition_in_flight_flag_survives_a_failed_final_mode_commit( + db_engine, tmp_path: Path, session, monkeypatch +): + """The queue itself drains (real commits succeed) but the FINAL + write_stored_mode('on') commit fails: the in-flight flag must stay set.""" + path = tmp_path / "final-commit.bin" + path.write_bytes(b"final commit content") + stat = path.stat() + create_content(session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns) + write_stored_mode(session, "off") + session.commit() + + from app.assets import mode as mode_module + + monkeypatch.setattr(mode_module, "hashing_enabled", lambda: True) + + clear_transition_queue() + transition = record_transition_intent(session) + enqueue_transition_work(session, transition) + session.commit() + + real_run_write_txn = scanner.run_write_txn + # Every entry's apply succeeds (the queue drains); only the FINAL + # write_stored_mode("on") commit, called once the queue is empty, fails. + monkeypatch.setattr( + hash_mode_state, + "run_write_txn", + _fail_run_write_txn_at(real_run_write_txn, fail_index=1), + ) + + with pytest.raises(OperationalError): + drain_transition_queue() + + assert list(hash_mode_state._PENDING_QUEUE) == [] + assert hash_mode_state._PENDING_PATHS == set() + assert hash_mode_state._off_to_on_transition_in_flight is True + assert read_stored_mode(session) == "off" + clear_transition_queue() diff --git a/tests-unit/assets_test/services/test_enrichment_snapshot.py b/tests-unit/assets_test/services/test_enrichment_snapshot.py index 8e33663cc8a..64a01b2117c 100644 --- a/tests-unit/assets_test/services/test_enrichment_snapshot.py +++ b/tests-unit/assets_test/services/test_enrichment_snapshot.py @@ -4,7 +4,8 @@ from app.assets.database.models import Asset, AssetContent from app.assets.helpers import to_stored_hash -from app.assets.scanner import enrich_asset + +from ..helpers import enrich_via_prepare_apply as enrich_asset def _create_unhashed_record(session, path: Path) -> tuple[AssetContent, Asset]: diff --git a/tests-unit/assets_test/services/test_manager_isolated.py b/tests-unit/assets_test/services/test_manager_isolated.py index e195c1184de..52a743b6fc8 100644 --- a/tests-unit/assets_test/services/test_manager_isolated.py +++ b/tests-unit/assets_test/services/test_manager_isolated.py @@ -14,7 +14,7 @@ from app.assets import lifecycle from app.assets import manager as manager_module -from app.assets import scanner, seeder as seeder_module +from app.assets import scanner from app.assets.database.models import Asset, AssetContent from app.assets.database.queries.records import create_content, create_record from app.assets.manager import AssetsEnabled @@ -73,9 +73,9 @@ def _create_session() -> Generator[SASession, None, None]: with SASession(engine) as session: yield session - monkeypatch.setattr(seeder_module, "create_session", _create_session) monkeypatch.setattr(scanner, "create_session", _create_session) monkeypatch.setattr("app.assets.services.ingest.create_session", _create_session) + monkeypatch.setattr("app.database.db.Session", sessionmaker(bind=engine)) monkeypatch.setattr("app.database.db.WriteSession", sessionmaker(bind=engine)) yield _create_session engine.dispose() diff --git a/tests-unit/assets_test/services/test_queue_drain_characterization.py b/tests-unit/assets_test/services/test_queue_drain_characterization.py new file mode 100644 index 00000000000..64a893fed4c --- /dev/null +++ b/tests-unit/assets_test/services/test_queue_drain_characterization.py @@ -0,0 +1,123 @@ +from pathlib import Path + +import pytest + +from app.assets import scanner_admission +from app.assets import scanner_changes +from app.assets.database.models import AssetContent +from app.assets.scanner_admission import _WATCH_LIST, _WatchEntry, tick_watch_list +from app.assets.scanner_changes import drain_pending_verifications, queue_pending_verification +from app.assets.services import hash_mode_state +from app.assets.services.hash_mode_state import ( + _PENDING_PATHS, + _PENDING_QUEUE, + _PendingEntry, + clear_transition_queue, + drain_transition_queue, + read_stored_mode, + write_stored_mode, +) + + +@pytest.fixture(autouse=True) +def clear_queues(): + scanner_changes.clear_pending_verifications() + _WATCH_LIST.clear() + clear_transition_queue() + yield + scanner_changes.clear_pending_verifications() + _WATCH_LIST.clear() + clear_transition_queue() + + +def _content(session, path: Path) -> AssetContent: + stat = path.stat() + content = AssetContent( + path=str(path), + hash=None, + size_bytes=stat.st_size, + mtime_ns=stat.st_mtime_ns, + ) + session.add(content) + session.commit() + return content + + +def test_pending_verification_requeues_after_hash_oserror(session, temp_dir, monkeypatch): + path = temp_dir / "pending.bin" + path.write_bytes(b"pending") + content = _content(session, path) + queue_pending_verification(content.id) + monkeypatch.setattr( + scanner_changes, + "snapshot_hash", + lambda _path: (_ for _ in ()).throw(PermissionError("denied")), + ) + + processed = drain_pending_verifications(session) + + assert processed == 0 + assert scanner_changes._pending_verification_ids == [content.id] + + +def test_watch_list_keeps_entries_when_stat_raises(session, temp_dir, monkeypatch): + path = temp_dir / "watched.bin" + path.write_bytes(b"watched") + entry = _WatchEntry(str(path), path.stat()) + _WATCH_LIST.append(entry) + monkeypatch.setattr( + scanner_admission.os, + "stat", + lambda *_args, **_kwargs: (_ for _ in ()).throw(PermissionError("denied")), + ) + + with pytest.raises(PermissionError): + tick_watch_list(session) + + assert _WATCH_LIST == [entry] + + +def test_transition_queue_retries_without_losing_companion_path( + session, monkeypatch +): + path = "/unreadable/transition.bin" + entry = _PendingEntry(path) + _PENDING_QUEUE.append(entry) + _PENDING_PATHS.add(path) + hash_mode_state._off_to_on_transition_in_flight = True + monkeypatch.setattr( + hash_mode_state, + "snapshot_hash", + lambda _path: (_ for _ in ()).throw(PermissionError("denied")), + ) + + drain_transition_queue(session) + + assert list(_PENDING_QUEUE) == [_PendingEntry(path, ticks=1)] + assert _PENDING_PATHS == {path} + assert hash_mode_state._off_to_on_transition_in_flight is True + + +def test_transition_queue_exhaustion_clears_companion_and_persists_mode( + session, monkeypatch +): + path = "/unreadable/exhausted.bin" + _PENDING_QUEUE.append(_PendingEntry(path)) + _PENDING_PATHS.add(path) + hash_mode_state._off_to_on_transition_in_flight = True + write_stored_mode(session, "off") + session.commit() + monkeypatch.setattr( + hash_mode_state, + "snapshot_hash", + lambda _path: (_ for _ in ()).throw(PermissionError("denied")), + ) + + for _ in range(3): + drain_transition_queue(session) + session.commit() + + assert list(_PENDING_QUEUE) == [] + assert _PENDING_PATHS == set() + assert hash_mode_state._off_to_on_transition_in_flight is False + assert read_stored_mode(session) == "on" diff --git a/tests-unit/assets_test/services/test_scanner_b.py b/tests-unit/assets_test/services/test_scanner_b.py index 93c155870e5..37b47099611 100644 --- a/tests-unit/assets_test/services/test_scanner_b.py +++ b/tests-unit/assets_test/services/test_scanner_b.py @@ -11,13 +11,14 @@ from app.assets.helpers import to_stored_hash from app.assets.scanner import ( build_asset_specs, - enrich_asset, mark_contents_missing_outside_prefixes, seed_asset_specs, sync_prefixes_with_filesystem, ) from app.assets.services.snapshot_hash import snapshot_hash +from ..helpers import enrich_via_prepare_apply + @dataclass(frozen=True, slots=True) class _ExtractedMetadata: @@ -71,8 +72,8 @@ def test_enrichment_retains_absent_system_metadata_keys(session: Session, temp_d _ExtractedMetadata(None, {"b": 3}), ], ): - enrich_asset(session, str(path), content.id, record.id) - enrich_asset(session, str(path), content.id, record.id) + enrich_via_prepare_apply(session, file_path=str(path), content_id=content.id, record_id=record.id) + enrich_via_prepare_apply(session, file_path=str(path), content_id=content.id, record_id=record.id) assert record.system_metadata == {"a": 1, "b": 3} @@ -95,7 +96,7 @@ def test_enrichment_retains_dimensions_when_image_extraction_degrades( ), patch("app.assets.scanner.extract_image_dimensions", return_value=None), ): - enrich_asset(session, str(path), content.id, record.id) + enrich_via_prepare_apply(session, file_path=str(path), content_id=content.id, record_id=record.id) assert record.system_metadata == { "filename": "image.png", @@ -120,7 +121,7 @@ def test_enrichment_overrides_content_length_with_zero( "app.assets.scanner.extract_file_metadata", return_value=_ExtractedMetadata(None, {"content_length": 0}), ): - enrich_asset(session, str(path), content.id, record.id) + enrich_via_prepare_apply(session, file_path=str(path), content_id=content.id, record_id=record.id) assert record.system_metadata == {"content_length": 0} @@ -217,7 +218,7 @@ def test_enrichment_keeps_equal_hash_contents_distinct(session, temp_dir: Path): session.add_all((record, existing_record)) session.commit() - enriched = enrich_asset( + enriched = enrich_via_prepare_apply( session, file_path=str(path), content_id=content.id, diff --git a/tests-unit/assets_test/services/test_stored_hash_prefix.py b/tests-unit/assets_test/services/test_stored_hash_prefix.py index 131ddd9eb40..79757bd1154 100644 --- a/tests-unit/assets_test/services/test_stored_hash_prefix.py +++ b/tests-unit/assets_test/services/test_stored_hash_prefix.py @@ -18,7 +18,6 @@ import folder_paths from app.assets.database.models import Asset, AssetContent from app.assets.database.queries.records import create_record -from app.assets.scanner import enrich_asset from app.assets.scanner_changes import recover_missing_content from app.assets.services import asset_management, ingest from app.assets.services.asset_management import get_asset_detail @@ -28,6 +27,8 @@ ) from app.assets.services.snapshot_hash import snapshot_hash +from ..helpers import enrich_via_prepare_apply as enrich_asset + @pytest.fixture def hashing_on(): diff --git a/tests-unit/assets_test/services/test_transition_drain.py b/tests-unit/assets_test/services/test_transition_drain.py index 3c1213499b6..352cc4b3f8d 100644 --- a/tests-unit/assets_test/services/test_transition_drain.py +++ b/tests-unit/assets_test/services/test_transition_drain.py @@ -174,6 +174,7 @@ def test_transition_drain_marks_deleted_path_missing_and_completes_transition( enqueue_transition_work(session, transition) drain_transition_queue(session) session.commit() + session.expire_all() assert session.get(AssetContent, content_id).is_missing is True, ( "a path deleted while every server was down must be marked missing, not requeued forever" @@ -268,6 +269,7 @@ def test_transition_drain_mixes_a_deleted_path_with_a_healthy_one(session, temp_ enqueue_transition_work(session, transition) drain_transition_queue(session) session.commit() + session.expire_all() assert session.get(AssetContent, content_ids[deleted_path]).is_missing is True healthy_content = session.get(AssetContent, content_ids[healthy_path]) diff --git a/tests-unit/assets_test/services/test_transition_settle.py b/tests-unit/assets_test/services/test_transition_settle.py index a08b28d3e79..845adfd1719 100644 --- a/tests-unit/assets_test/services/test_transition_settle.py +++ b/tests-unit/assets_test/services/test_transition_settle.py @@ -63,13 +63,13 @@ def _create_session(): yield sess attempts: list[str] = [] - real_enrich_asset = scanner.enrich_asset + real_prepare_enrichment = scanner._prepare_enrichment - def counting_enrich_asset(*args, **kwargs): - attempts.append(kwargs["record_id"]) + def counting_prepare_enrichment(row, *args, **kwargs): + attempts.append(row.record_id) if len(attempts) > _ATTEMPT_BUDGET: raise _AttemptBudgetExhausted - return real_enrich_asset(*args, **kwargs) + return real_prepare_enrichment(row, *args, **kwargs) seeder = seeder_module._AssetSeeder() seeder._compute_hashes = True @@ -79,10 +79,10 @@ def counting_enrich_asset(*args, **kwargs): monkeypatch.setattr("folder_paths.get_input_directory", lambda: str(temp_dir)) monkeypatch.setattr(hash_mode_state, "snapshot_hash", _denied) monkeypatch.setattr(scanner, "snapshot_hash", _denied) - monkeypatch.setattr(scanner, "enrich_asset", counting_enrich_asset) + monkeypatch.setattr(scanner, "_prepare_enrichment", counting_prepare_enrichment) - with patch("app.assets.seeder.create_session", _create_session), \ - patch("app.assets.scanner.create_session", _create_session), \ + with patch("app.assets.scanner.create_session", _create_session), \ + patch("app.database.db.Session", sessionmaker(bind=db_engine)), \ patch("app.database.db.WriteSession", sessionmaker(bind=db_engine)): try: cancelled, _enriched = seeder._run_enrich_phase(("input",)) diff --git a/tests-unit/assets_test/services/test_updated_at_semantics.py b/tests-unit/assets_test/services/test_updated_at_semantics.py index bfd17082b1f..fb854f5a307 100644 --- a/tests-unit/assets_test/services/test_updated_at_semantics.py +++ b/tests-unit/assets_test/services/test_updated_at_semantics.py @@ -15,7 +15,6 @@ unset_content_missing, ) from app.assets.helpers import to_stored_hash -from app.assets.scanner import enrich_asset from app.assets.scanner_changes import split_content from app.assets.services.asset_management import ( resolve_asset_for_download, @@ -24,6 +23,8 @@ ) from app.assets.services.tagging import apply_tags, remove_tags +from ..helpers import enrich_via_prepare_apply + STALE = datetime(2020, 1, 1, 0, 0, 0) @@ -206,8 +207,12 @@ def test_scanner_enrichment_does_not_move_updated_at(session, temp_dir): path = _write_file(temp_dir, "enrich.png", payload=b"\x89PNG\r\n\x1a\n" + b"0" * 64) record = _seed_scannable_record(session, path, name="enrich.png") - assert enrich_asset( - session, path, record.content_id, record.id, extract_metadata=True + assert enrich_via_prepare_apply( + session, + file_path=path, + content_id=record.content_id, + record_id=record.id, + extract_metadata=True, ), "fixture must actually enrich" assert session.get(Asset, record.id).mime_type is not None, ( diff --git a/tests-unit/assets_test/test_event_log.py b/tests-unit/assets_test/test_event_log.py index ccdbcc8320d..64bda8fddc5 100644 --- a/tests-unit/assets_test/test_event_log.py +++ b/tests-unit/assets_test/test_event_log.py @@ -37,7 +37,7 @@ "count": [1], "error_type": ["ValueError", "FileNotFoundError"], "hashing_enabled": [True, False], - "site": ["discovery", "enrich"], + "site": ["discovery", "enrich", "reference_stat"], "output_kind": ["executed", "cached"], } diff --git a/tests-unit/assets_test/test_intended_behaviour.py b/tests-unit/assets_test/test_intended_behaviour.py index a1fffc24f0d..e2c173e5ceb 100644 --- a/tests-unit/assets_test/test_intended_behaviour.py +++ b/tests-unit/assets_test/test_intended_behaviour.py @@ -55,9 +55,12 @@ @pytest.fixture -def session(): +def session(monkeypatch): engine = create_engine("sqlite:///:memory:") Base.metadata.create_all(engine) + factory = sessionmaker(bind=engine) + monkeypatch.setattr("app.database.db.Session", factory) + monkeypatch.setattr("app.database.db.WriteSession", factory) with Session(engine) as database_session: yield database_session @@ -508,6 +511,7 @@ def test_scenario_18_edit_during_hash_discard(session, tmp_path): mtime_ns=seed_stat.st_mtime_ns, ) record = create_record(session, content.id, "unstable.bin") + session.commit() clear_pending_verifications() try: @@ -526,8 +530,10 @@ def test_scenario_18_edit_during_hash_discard(session, tmp_path): finally: clear_pending_verifications() - assert content.hash == committed_hash - assert content.mtime_ns == path.stat().st_mtime_ns + session.expire_all() + refreshed = session.get(AssetContent, content.id) + assert refreshed.hash == committed_hash + assert refreshed.mtime_ns == path.stat().st_mtime_ns def test_writer_simulation_terminates_and_is_capped(tmp_path): diff --git a/tests-unit/assets_test/test_scanner_ingest_event_log.py b/tests-unit/assets_test/test_scanner_ingest_event_log.py index f81ebee0fe7..3c6e1e943ee 100644 --- a/tests-unit/assets_test/test_scanner_ingest_event_log.py +++ b/tests-unit/assets_test/test_scanner_ingest_event_log.py @@ -12,6 +12,8 @@ from app.assets.scanner import UnenrichedContent from app.assets.seeder import _ScanState +from .helpers import enrich_via_prepare_apply + EVENT_LINE_PATTERN = re.compile( rf"^{re.escape(TAG)} (?P[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)*)" @@ -69,7 +71,7 @@ def hash_session(path: Path) -> Mock: def run_hash_failure(session: Mock, path: Path, progress: _ScanState) -> bool: - return scanner.enrich_asset( + return enrich_via_prepare_apply( session, file_path=str(path), content_id="content", @@ -105,7 +107,7 @@ def test_scanner_safe_failures_emit_exception_type_without_path( def fail_session(): raise FileNotFoundError(secret_path) - monkeypatch.setattr(scanner, "create_session", fail_session) + monkeypatch.setattr("app.database.db.WriteSession", fail_session) with caplog.at_level(logging.INFO): result = operation() @@ -218,7 +220,7 @@ def test_modified_during_hash_emits_fieldless_discard_event( monkeypatch.setattr(scanner, "snapshot_hash", lambda _path: None) with caplog.at_level(logging.INFO): - updated = scanner.enrich_asset( + updated = enrich_via_prepare_apply( hash_session(path), file_path=str(path), content_id="content", @@ -319,7 +321,7 @@ def deny_stat(*_args, **_kwargs): progress = _ScanState() with caplog.at_level(logging.INFO): - first = scanner.enrich_asset( + first = enrich_via_prepare_apply( Mock(), file_path="/private/assets/locked-1.bin", content_id="content-1", @@ -328,7 +330,7 @@ def deny_stat(*_args, **_kwargs): compute_hash=False, progress=progress, ) - second = scanner.enrich_asset( + second = enrich_via_prepare_apply( Mock(), file_path="/private/assets/locked-2.bin", content_id="content-2", @@ -357,7 +359,7 @@ def vanish_stat(*_args, **_kwargs): progress = _ScanState() with caplog.at_level(logging.INFO): - updated = scanner.enrich_asset( + updated = enrich_via_prepare_apply( Mock(), file_path="/private/assets/gone.bin", content_id="content", @@ -379,12 +381,11 @@ def test_enrich_failures_emit_once_per_scan_and_reset_with_new_scan( UnenrichedContent("content-1", "record-1", "/private/assets/one.bin"), UnenrichedContent("content-2", "record-2", "/private/assets/two.bin"), ] - monkeypatch.setattr(scanner, "create_session", lambda: nullcontext(Mock())) - def fail_enrich(*_args, **_kwargs): + def fail_prepare(*_args, **_kwargs): raise FileNotFoundError("/private/assets/secret.bin") - monkeypatch.setattr(scanner, "enrich_asset", fail_enrich) + monkeypatch.setattr(scanner, "_prepare_enrichment", fail_prepare) with caplog.at_level(logging.INFO): first_result = scanner.enrich_assets_batch(rows, progress=_ScanState()) @@ -405,12 +406,11 @@ def test_enrich_exception_counts_one_failure_per_raising_row( UnenrichedContent("content-1", "record-1", "/private/assets/one.bin"), UnenrichedContent("content-2", "record-2", "/private/assets/two.bin"), ] - monkeypatch.setattr(scanner, "create_session", lambda: nullcontext(Mock())) - def fail_enrich(*_args, **_kwargs): + def fail_prepare(*_args, **_kwargs): raise FileNotFoundError("/private/assets/secret.bin") - monkeypatch.setattr(scanner, "enrich_asset", fail_enrich) + monkeypatch.setattr(scanner, "_prepare_enrichment", fail_prepare) progress = _ScanState() enriched, failed_ids = scanner.enrich_assets_batch(rows, progress=progress) diff --git a/tests-unit/test_assets_event_log_static.py b/tests-unit/test_assets_event_log_static.py index e4e48433e5b..4b59f839ae3 100644 --- a/tests-unit/test_assets_event_log_static.py +++ b/tests-unit/test_assets_event_log_static.py @@ -57,12 +57,13 @@ class CallSite(NamedTuple): CallSite( "app/assets/scanner.py", "mark_missing_outside_prefixes_safely", "scanner.mark_missing_failed" ), - CallSite("app/assets/scanner.py", "enrich_asset", "scanner.hash_failed"), - CallSite("app/assets/scanner.py", "enrich_asset", "scanner.hash_discarded_modified"), + CallSite("app/assets/scanner.py", "_prepare_enrichment", "scanner.hash_failed"), + CallSite("app/assets/scanner.py", "_prepare_enrichment", "scanner.hash_discarded_modified"), CallSite("app/assets/scanner.py", "enrich_assets_batch", "scanner.enrich_failed"), # todo 16 - discovery/enrich stat failures, emit-once per scan per site CallSite("app/assets/scanner.py", "build_asset_specs", "scanner.stat_failed"), - CallSite("app/assets/scanner.py", "enrich_asset", "scanner.stat_failed"), + CallSite("app/assets/scanner.py", "_prepare_enrichment", "scanner.stat_failed"), + CallSite("app/assets/scanner.py", "_publish_reference_diagnostics", "scanner.stat_failed"), CallSite("app/assets/services/ingest.py", "register_cached_output", "ingest.register_failed"), CallSite("app/assets/services/ingest.py", "register_executed_output", "ingest.register_failed"), } From ddf1cf8a4abfd7a0936f997d91ed51a8d1075cd6 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 15:17:13 -0700 Subject: [PATCH 006/111] test(assets): add barrier lock-hold coverage for ingest preflight restructures --- .../services/test_ingest_write_txn.py | 191 ++++++++++++++++++ 1 file changed, 191 insertions(+) diff --git a/tests-unit/assets_test/services/test_ingest_write_txn.py b/tests-unit/assets_test/services/test_ingest_write_txn.py index 477a33bb365..a7d04a6936a 100644 --- a/tests-unit/assets_test/services/test_ingest_write_txn.py +++ b/tests-unit/assets_test/services/test_ingest_write_txn.py @@ -2,14 +2,21 @@ import logging import sqlite3 +import threading +import time import folder_paths import pytest +from sqlalchemy import text from sqlalchemy.exc import IntegrityError, OperationalError import app.assets.services.ingest as ingest +import app.database.db as db_mod from app.assets.database.queries.records import create_content, create_record +_BARRIER_TIMEOUT = 5 +_PROBE_BUDGET_SECONDS = 1.0 + def _output_path(name: str) -> str: output_dir = folder_paths.get_output_directory() @@ -17,6 +24,45 @@ def _output_path(name: str) -> str: return os.path.join(output_dir, name) +@pytest.fixture +def file_database(tmp_path, monkeypatch): + """A real file-backed engine (WAL + BEGIN IMMEDIATE) for lock-hold barrier tests. + + ``mock_create_session`` binds an in-memory StaticPool engine, which never + contends on a write lock and would make a lock-hold assertion vacuously + true; only the production runtime engines built by ``init_db`` enforce it. + """ + database_path = str(tmp_path / "assets.db") + monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") + monkeypatch.setattr(db_mod, "Session", None) + monkeypatch.setattr(db_mod, "_db_lock", None) + monkeypatch.setattr(db_mod, "WriteSession", None) + db_mod.init_db() + yield database_path + for factory in (db_mod.Session, db_mod.WriteSession): + if factory is not None: + factory.kw["bind"].dispose() + db_mod._db_lock.release(force=True) + + +def _probe_write() -> None: + db_mod.run_write_txn( + lambda session: session.execute( + text("INSERT INTO tags (name) VALUES (:name)"), + {"name": f"probe-{os.urandom(8).hex()}"}, + ) + ) + + +def _blocking_fake(entered: threading.Event, release: threading.Event, real_fn): + def fake(*args, **kwargs): + entered.set() + release.wait(timeout=_BARRIER_TIMEOUT) + return real_fn(*args, **kwargs) + + return fake + + def test_cached_registration_skips_extraction_when_live_content_is_missing( mock_create_session, monkeypatch ) -> None: @@ -376,3 +422,148 @@ def rewrite_then_apply(session, *args): finally: for path in (direct_path, public_path): os.unlink(path) + + +def test_cached_registration_metadata_extraction_does_not_hold_the_write_lock( + file_database, monkeypatch +) -> None: + path = _output_path("cached-barrier-no-sibling.bin") + with open(path, "wb") as file: + file.write(b"cached barrier bytes") + try: + with db_mod.Session() as session: + content = create_content(session, path, size_bytes=os.path.getsize(path)) + session.commit() + content_id = content.id + + real_extract = ingest._extract_system_metadata_sync + entered = threading.Event() + release = threading.Event() + monkeypatch.setattr( + ingest, + "_extract_system_metadata_sync", + _blocking_fake(entered, release, real_extract), + ) + + result: dict[str, object] = {} + + def _register() -> None: + result["registered"] = ingest.register_cached_output(path) + + worker = threading.Thread(target=_register) + worker.start() + try: + assert entered.wait(timeout=_BARRIER_TIMEOUT) + started = time.monotonic() + _probe_write() + elapsed = time.monotonic() - started + finally: + release.set() + worker.join(timeout=_BARRIER_TIMEOUT) + + assert not worker.is_alive() + assert elapsed < _PROBE_BUDGET_SECONDS + registered = result["registered"] + assert registered is not None + assert registered.content_id == content_id + with db_mod.Session() as session: + record = session.get(ingest.Asset, registered.id) + assert record is not None + assert record.content_id == content_id + finally: + os.unlink(path) + + +def test_upload_settle_hashing_does_not_hold_the_write_lock( + file_database, monkeypatch +) -> None: + path = _output_path("settle-barrier.bin") + with open(path, "wb") as file: + file.write(b"incumbent barrier bytes") + try: + stat = os.stat(path) + with db_mod.Session() as session: + content = create_content( + session, path, size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns + ) + session.commit() + content_id = content.id + + real_snapshot_hash = ingest.snapshot_hash + entered = threading.Event() + release = threading.Event() + monkeypatch.setattr( + ingest, + "snapshot_hash", + _blocking_fake(entered, release, real_snapshot_hash), + ) + + def _settle() -> None: + ingest._settle_destination_before_write(path) + + worker = threading.Thread(target=_settle) + worker.start() + try: + assert entered.wait(timeout=_BARRIER_TIMEOUT) + started = time.monotonic() + _probe_write() + elapsed = time.monotonic() - started + finally: + release.set() + worker.join(timeout=_BARRIER_TIMEOUT) + + assert not worker.is_alive() + assert elapsed < _PROBE_BUDGET_SECONDS + + expected_digest, _expected_stat = real_snapshot_hash(path) + with db_mod.Session() as session: + settled = session.get(ingest.AssetContent, content_id) + assert settled is not None + assert settled.hash == ingest.to_stored_hash(expected_digest) + assert settled.is_missing is False + finally: + os.unlink(path) + + +def test_create_record_metadata_extraction_does_not_hold_the_write_lock( + file_database, monkeypatch +) -> None: + path = _output_path("create-record-barrier.bin") + with open(path, "wb") as file: + file.write(b"brand new record bytes") + try: + real_extract = ingest._extract_system_metadata_sync + entered = threading.Event() + release = threading.Event() + monkeypatch.setattr( + ingest, + "_extract_system_metadata_sync", + _blocking_fake(entered, release, real_extract), + ) + + result: dict[str, object] = {} + + def _register() -> None: + result["upload"] = ingest.register_file_in_place( + path, "create-record-barrier.bin", ["output"] + ) + + worker = threading.Thread(target=_register) + worker.start() + try: + assert entered.wait(timeout=_BARRIER_TIMEOUT) + started = time.monotonic() + _probe_write() + elapsed = time.monotonic() - started + finally: + release.set() + worker.join(timeout=_BARRIER_TIMEOUT) + + assert not worker.is_alive() + assert elapsed < _PROBE_BUDGET_SECONDS + upload_result = result["upload"] + assert upload_result is not None + assert upload_result.created_new is True + assert upload_result.ref.file_path == path + finally: + os.unlink(path) From bdb320536e8f56853b9a876fc1daf7ae3253ba01 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 15:53:55 -0700 Subject: [PATCH 007/111] fix(assets): route api and lifecycle write paths through immediate write transactions --- app/assets/lifecycle.py | 15 +-- app/assets/services/asset_management.py | 61 ++++++----- app/assets/services/tagging.py | 34 +++--- .../assets_test/services/test_lifecycle.py | 3 +- .../services/test_lifecycle_write_txn.py | 101 ++++++++++++++++++ .../services/test_tag_write_conflicts.py | 57 +++++++--- .../assets_test/test_intended_behaviour.py | 1 + 7 files changed, 202 insertions(+), 70 deletions(-) create mode 100644 tests-unit/assets_test/services/test_lifecycle_write_txn.py diff --git a/app/assets/lifecycle.py b/app/assets/lifecycle.py index 26ecd0370ae..519b06ea827 100644 --- a/app/assets/lifecycle.py +++ b/app/assets/lifecycle.py @@ -20,7 +20,7 @@ from app.assets.helpers import sql_path_under_prefix from app.assets.services.hash_mode_state import enqueue_transition_work from app.assets.services.hash_mode_state import record_transition_intent -from app.database.db import can_create_session, create_session +from app.database.db import can_create_session, create_session, run_write_txn from comfy.cli_args import args _excluded_scan_roots: set[str] = set() @@ -34,9 +34,8 @@ def get_excluded_scan_roots() -> frozenset[str]: def record_hash_mode_transition_intent() -> None: global _hash_mode_transition - with create_session() as session: - _hash_mode_transition = record_transition_intent(session) - session.commit() + transition = run_write_txn(record_transition_intent) + _hash_mode_transition = transition def enqueue_mode_transition_work() -> None: @@ -108,9 +107,7 @@ def start_asset_seeder() -> bool: def run_asset_startup() -> None: try: - with create_session() as session: - wipe_temp_db_rows(session) - session.commit() + run_write_txn(wipe_temp_db_rows) except Exception: logging.exception("Temp DB row wipe failed; skipping filesystem cleanup") enqueue_mode_transition_work() @@ -133,9 +130,7 @@ def run_startup(*, enable_assets: bool) -> None: def run_asset_shutdown_cleanup() -> None: try: - with create_session() as session: - wipe_temp_db_rows(session) - session.commit() + run_write_txn(wipe_temp_db_rows) except Exception: logging.exception("Temp DB row wipe failed during shutdown") finally: diff --git a/app/assets/services/asset_management.py b/app/assets/services/asset_management.py index 0f7ad716c1c..03f7eb428bb 100644 --- a/app/assets/services/asset_management.py +++ b/app/assets/services/asset_management.py @@ -34,7 +34,7 @@ ReferenceData, UserMetadata, ) -from app.database.db import create_session +from app.database.db import create_session, run_write_txn def _record_to_detail_result(session, record) -> AssetDetailResult: @@ -93,7 +93,7 @@ def update_asset_metadata( mime_type: str | None = None, preview_id: str | None = None, ) -> AssetDetailResult: - with create_session() as session: + def _work(session) -> None: record = get_record_by_id(session, reference_id) if record is None: raise ValueError(f"Asset {reference_id} not found") @@ -143,7 +143,8 @@ def update_asset_metadata( session.flush() if _fetch_manual_tags(session, reference_id) != manual_tags_before: bump_record_updated_at(session, reference_id) - session.commit() + + run_write_txn(_work) detail = get_asset_detail(reference_id) if detail is None: @@ -154,13 +155,14 @@ def update_asset_metadata( def delete_asset_reference( reference_id: str, ) -> bool: - with create_session() as session: + def _work(session) -> bool: if get_record_by_id(session, reference_id) is None: return False delete_record(session, reference_id) - session.commit() return True + return run_write_txn(_work) + def asset_exists(asset_hash: str) -> bool: try: @@ -189,7 +191,8 @@ def resolve_hash_to_path( canonical = validate_blake3_hash(asset_hash) except ValueError: return None - with create_session() as session: + + def _work(session) -> tuple[str, str | None, str] | None: content = lookup_for_view(session, canonical) if content is None: return None @@ -209,15 +212,18 @@ def resolve_hash_to_path( mime_type = latest_record.mime_type for record in records: update_record_access_time(session, record.id) - abs_path = content.path - session.commit() + return content.path, mime_type, display_name - ctype = ( - mime_type - or mimetypes.guess_type(display_name)[0] - or mimetypes.guess_type(abs_path)[0] - or "application/octet-stream" - ) + resolution = run_write_txn(_work) + if resolution is None: + return None + abs_path, mime_type, display_name = resolution + ctype = ( + mime_type + or mimetypes.guess_type(display_name)[0] + or mimetypes.guess_type(abs_path)[0] + or "application/octet-stream" + ) return DownloadResolutionResult( abs_path=abs_path, content_type=ctype, @@ -236,7 +242,7 @@ def get_preview_file_paths(preview_ids: list[str]) -> dict[str, str]: def resolve_asset_for_download( reference_id: str, ) -> DownloadResolutionResult: - with create_session() as session: + def _work(session) -> tuple[str, str | None, str | None]: record = get_record_by_id(session, reference_id) if record is None: raise ValueError(f"AssetReference {reference_id} not found") @@ -257,16 +263,17 @@ def resolve_asset_for_download( abs_path = content.path update_record_access_time(session, reference_id) - session.commit() + return abs_path, asset_mime, ref_name - ctype = ( - asset_mime - or mimetypes.guess_type(ref_name or abs_path)[0] - or "application/octet-stream" - ) - download_name = ref_name or os.path.basename(abs_path) - return DownloadResolutionResult( - abs_path=abs_path, - content_type=ctype, - download_name=download_name, - ) + abs_path, asset_mime, ref_name = run_write_txn(_work) + ctype = ( + asset_mime + or mimetypes.guess_type(ref_name or abs_path)[0] + or "application/octet-stream" + ) + download_name = ref_name or os.path.basename(abs_path) + return DownloadResolutionResult( + abs_path=abs_path, + content_type=ctype, + download_name=download_name, + ) diff --git a/app/assets/services/tagging.py b/app/assets/services/tagging.py index d22eb8c0a5c..b2ec745b02f 100644 --- a/app/assets/services/tagging.py +++ b/app/assets/services/tagging.py @@ -23,7 +23,7 @@ from app.assets.database.models import Asset, AssetTag from app.assets.helpers import normalize_tags from app.assets.services.schemas import TagUsage -from app.database.db import create_session +from app.database.db import create_session, run_write_txn def apply_tags( @@ -31,7 +31,7 @@ def apply_tags( tags: list[str], origin: str = "manual", ) -> AddTagsResult: - with create_session() as session: + def _work(session) -> AddTagsResult: if session.get(Asset, reference_id) is None: raise ValueError(f"Asset {reference_id} not found") @@ -57,20 +57,20 @@ def apply_tags( .order_by(AssetTag.tag_name) ) ) - session.commit() + return AddTagsResult( + added=sorted(added), + already_present=sorted((requested_tags & set(total_tags)) - set(added)), + total_tags=total_tags, + ) - return AddTagsResult( - added=sorted(added), - already_present=sorted((requested_tags & set(total_tags)) - set(added)), - total_tags=total_tags, - ) + return run_write_txn(_work) def remove_tags( reference_id: str, tags: list[str], ) -> RemoveTagsResult: - with create_session() as session: + def _work(session) -> RemoveTagsResult: if session.get(Asset, reference_id) is None: raise ValueError(f"Asset {reference_id} not found") @@ -109,14 +109,14 @@ def remove_tags( .order_by(AssetTag.tag_name) ) ) - session.commit() - - return RemoveTagsResult( - removed=sorted(removable_tags), - not_present=sorted(requested_tags - removable_tags - protected_tags), - total_tags=total_tags, - protected=sorted(protected_tags), - ) + return RemoveTagsResult( + removed=sorted(removable_tags), + not_present=sorted(requested_tags - removable_tags - protected_tags), + total_tags=total_tags, + protected=sorted(protected_tags), + ) + + return run_write_txn(_work) def list_tags( diff --git a/tests-unit/assets_test/services/test_lifecycle.py b/tests-unit/assets_test/services/test_lifecycle.py index 8523365fe5b..bdeeb35ae17 100644 --- a/tests-unit/assets_test/services/test_lifecycle.py +++ b/tests-unit/assets_test/services/test_lifecycle.py @@ -7,8 +7,7 @@ import pytest from sqlalchemy import create_engine, select -from sqlalchemy.orm import Session -from sqlalchemy.orm import Session as SASession, sessionmaker +from sqlalchemy.orm import Session, Session as SASession, sessionmaker from app.assets import lifecycle from app.assets.database.models import Asset, AssetContent, Base diff --git a/tests-unit/assets_test/services/test_lifecycle_write_txn.py b/tests-unit/assets_test/services/test_lifecycle_write_txn.py new file mode 100644 index 00000000000..4911ac4b256 --- /dev/null +++ b/tests-unit/assets_test/services/test_lifecycle_write_txn.py @@ -0,0 +1,101 @@ +import sqlite3 +from contextlib import contextmanager +from unittest.mock import patch + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import Session +from sqlalchemy.orm import Session as SASession, sessionmaker + +import app.database.db as db_mod +from app.assets import lifecycle +from app.assets.database.models import Base +from app.assets.services import hash_mode_state +from app.assets.services.hash_mode_state import clear_transition_queue, write_stored_mode + + +@pytest.fixture(autouse=True) +def clear_lifecycle_transition_state(): + clear_transition_queue() + lifecycle._hash_mode_transition = None + yield + clear_transition_queue() + lifecycle._hash_mode_transition = None + + +@pytest.fixture +def session(): + engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}) + Base.metadata.create_all(engine) + with Session(engine) as database_session: + yield database_session + + +@pytest.fixture +def mock_create_session(session): + engine = session.bind + + @contextmanager + def create_test_session(): + with SASession(engine) as database_session: + yield database_session + + with ( + patch("app.assets.lifecycle.create_session", create_test_session), + patch("app.database.db.create_session", create_test_session), + patch("app.database.db.WriteSession", sessionmaker(bind=engine)), + ): + yield create_test_session + + +def test_transition_intent_keeps_global_unchanged_until_retry_succeeds( + session, mock_create_session, monkeypatch +): + write_stored_mode(session, "on") + session.commit() + lifecycle._hash_mode_transition = "off_to_on" + monkeypatch.setattr(hash_mode_state._mode, "hashing_enabled", lambda: False) + real_run_write_txn = db_mod.run_write_txn + attempts = 0 + + def retry_after_first_attempt(work): + def flaky_work(writer_session): + nonlocal attempts + + transition = work(writer_session) + attempts += 1 + if attempts == 1: + assert lifecycle._hash_mode_transition == "off_to_on" + raise OperationalError( + "UPDATE", {}, sqlite3.OperationalError("database is locked") + ) + return transition + + return real_run_write_txn(flaky_work) + + monkeypatch.setattr(lifecycle, "run_write_txn", retry_after_first_attempt) + monkeypatch.setattr(db_mod.time, "sleep", lambda _seconds: None) + + lifecycle.record_hash_mode_transition_intent() + + assert attempts == 2 + assert lifecycle._hash_mode_transition == "on_to_off" + session.expire_all() + assert hash_mode_state.read_stored_mode(session) == "off" + + +def test_transition_intent_terminal_failure_preserves_global( + mock_create_session, monkeypatch +): + lifecycle._hash_mode_transition = "off_to_on" + + def fail_write(_work): + raise OperationalError("UPDATE", {}, sqlite3.OperationalError("database is locked")) + + monkeypatch.setattr(lifecycle, "run_write_txn", fail_write) + + with pytest.raises(OperationalError, match="database is locked"): + lifecycle.record_hash_mode_transition_intent() + + assert lifecycle._hash_mode_transition == "off_to_on" diff --git a/tests-unit/assets_test/services/test_tag_write_conflicts.py b/tests-unit/assets_test/services/test_tag_write_conflicts.py index 32bf41fd1c6..9ba839ce659 100644 --- a/tests-unit/assets_test/services/test_tag_write_conflicts.py +++ b/tests-unit/assets_test/services/test_tag_write_conflicts.py @@ -1,12 +1,13 @@ -from contextlib import contextmanager from datetime import datetime +import sqlite3 from unittest.mock import patch import pytest from sqlalchemy import create_engine, select, update -from sqlalchemy.exc import IntegrityError +from sqlalchemy.exc import IntegrityError, OperationalError from sqlalchemy.orm import Session as SASession +import app.database.db as db_mod import app.assets.database.queries.records as records_module import app.assets.services.asset_management as asset_management_module import app.assets.services.tagging as tagging_module @@ -69,21 +70,20 @@ def connection_a_commits_the_raced_tag() -> None: fired: list[bool] = [] - @contextmanager - def racing_session_factory(): - with SASession(engine) as session_b: - real_add = session_b.add + def racing_write_session(): + session_b = SASession(engine) + real_add = session_b.add - def add_racing_the_winner(instance, *args, **kwargs): - if isinstance(instance, Tag) and instance.name == RACED and not fired: - fired.append(True) - connection_a_commits_the_raced_tag() - return real_add(instance, *args, **kwargs) + def add_racing_the_winner(instance, *args, **kwargs): + if isinstance(instance, Tag) and instance.name == RACED and not fired: + fired.append(True) + connection_a_commits_the_raced_tag() + return real_add(instance, *args, **kwargs) - session_b.add = add_racing_the_winner - yield session_b + session_b.add = add_racing_the_winner + return session_b - with patch("app.assets.services.tagging.create_session", racing_session_factory): + with patch("app.database.db.WriteSession", racing_write_session): result = apply_tags(record_id, [RACED]) assert fired, "the interleave never fired; the test proves nothing" @@ -103,6 +103,35 @@ def add_racing_the_winner(instance, *args, **kwargs): ) +def test_apply_tags_retries_a_transient_locked_write(session, mock_create_session, monkeypatch): + record_id = _seed_record(session, "/tmp/tag-retry-fixture") + attempts = 0 + real_run_write_txn = db_mod.run_write_txn + + def retry_after_first_attempt(work): + def flaky_work(writer_session): + nonlocal attempts + + attempts += 1 + if attempts == 1: + raise OperationalError( + "INSERT", {}, sqlite3.OperationalError("database is locked") + ) + return work(writer_session) + + return real_run_write_txn(flaky_work) + + monkeypatch.setattr(tagging_module, "run_write_txn", retry_after_first_attempt) + monkeypatch.setattr(db_mod.time, "sleep", lambda _seconds: None) + + result = apply_tags(record_id, ["retriable"]) + + assert attempts == 2 + assert result.added == ["retriable"] + assert result.total_tags == ["retriable"] + assert _tag_names(session, record_id) == ["retriable"] + + def test_ensure_tag_link_reraises_when_the_parent_asset_is_missing(db_engine_fk): with SASession(db_engine_fk) as session: session.add(Tag(name="orphan-link")) diff --git a/tests-unit/assets_test/test_intended_behaviour.py b/tests-unit/assets_test/test_intended_behaviour.py index e2c173e5ceb..5e262f7f53c 100644 --- a/tests-unit/assets_test/test_intended_behaviour.py +++ b/tests-unit/assets_test/test_intended_behaviour.py @@ -212,6 +212,7 @@ def test_scenario_4_delete_no_revival(session, tmp_path): assert delete_asset_reference(record_id) is True assert delete_asset_reference(record_id) is False + session.expire_all() assert get_record_by_id(session, record_id) is None assert session.get(AssetContent, content_id) is not None From 067a685b0535fbf3d4741b6966d4b754f39dab6d Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 16:19:37 -0700 Subject: [PATCH 008/111] test(assets): guard the write-session convention --- .../test_write_session_convention.py | 269 ++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 tests-unit/assets_test/test_write_session_convention.py diff --git a/tests-unit/assets_test/test_write_session_convention.py b/tests-unit/assets_test/test_write_session_convention.py new file mode 100644 index 00000000000..277f07a471b --- /dev/null +++ b/tests-unit/assets_test/test_write_session_convention.py @@ -0,0 +1,269 @@ +import ast +from collections import Counter +from dataclasses import fields, is_dataclass +from pathlib import Path +from typing import Literal, NamedTuple, NoReturn + +import folder_paths +import pytest + +from app.assets.database.queries.records import create_content, create_record +from app.assets.helpers import to_stored_hash +from app.assets.services.asset_management import ( + delete_asset_reference, + resolve_asset_for_download, + resolve_hash_to_path, + update_asset_metadata, +) +from app.assets.services.ingest import ( + create_from_hash, + register_cached_output, + register_executed_output, + register_file_in_place, + upload_from_temp_path, +) +from app.assets.services.tagging import apply_tags, remove_tags +from app.database.models import Base +from assets_test.services import conftest as service_fixtures + +REPO_ROOT = Path(__file__).resolve().parents[2] +FUNCTION_NODES = (ast.FunctionDef, ast.AsyncFunctionDef) + + +class CreateSessionCallSite(NamedTuple): + path: str + function: str + + +READ_ONLY_CREATE_SESSION_CALL_SITES = frozenset( + { + CreateSessionCallSite("app/assets/api/routes.py", "list_assets_route"), + # This commits only SELECTs and in-memory queue changes, so a writer retry + # would acquire a write lease for no persisted work. + CreateSessionCallSite("app/assets/lifecycle.py", "enqueue_mode_transition_work"), + CreateSessionCallSite("app/assets/scanner.py", "get_unenriched_assets_for_roots"), + # This preflight reads a content path and stats it before outside-transaction hashing. + CreateSessionCallSite("app/assets/scanner_changes.py", "_preflight_pending_verification"), + CreateSessionCallSite("app/assets/services/asset_management.py", "get_asset_detail"), + CreateSessionCallSite("app/assets/services/asset_management.py", "asset_exists"), + CreateSessionCallSite("app/assets/services/asset_management.py", "get_preview_file_paths"), + # These preflights read decision facts before metadata or hash I/O outside the writer lease. + CreateSessionCallSite("app/assets/services/ingest.py", "_preflight_upload_record"), + CreateSessionCallSite("app/assets/services/ingest.py", "_preflight_settle_target"), + CreateSessionCallSite("app/assets/services/ingest.py", "_preflight_cached_registration"), + CreateSessionCallSite("app/assets/services/tagging.py", "list_tags"), + CreateSessionCallSite("app/assets/services/tagging.py", "list_tag_histogram"), + } +) + + +@pytest.fixture(autouse=True) +def autoclean_unit_test_assets(): + yield + + +@pytest.fixture(autouse=True) +def initialised_hash_mode(): + yield from service_fixtures.initialised_hash_mode.__wrapped__() + + +@pytest.fixture(name="db_engine") +def service_db_engine(): + return service_fixtures.db_engine.__wrapped__() + + +@pytest.fixture(name="session") +def service_session(db_engine, monkeypatch): + yield from service_fixtures.session.__wrapped__(db_engine, monkeypatch) + + +@pytest.fixture(name="mock_create_session") +def service_mock_create_session(db_engine): + yield from service_fixtures.mock_create_session.__wrapped__(db_engine) + + +@pytest.fixture(name="temp_dir") +def service_temp_dir(): + yield from service_fixtures.temp_dir.__wrapped__() + + +def _is_create_session_call(call: ast.Call) -> bool: + return ( + isinstance(call.func, ast.Name) + and call.func.id == "create_session" + ) or ( + isinstance(call.func, ast.Attribute) + and call.func.attr == "create_session" + ) + + +def _create_session_call_sites(path: Path) -> Counter[CreateSessionCallSite]: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + relative_path = path.relative_to(REPO_ROOT).as_posix() + sites: Counter[CreateSessionCallSite] = Counter() + + def walk(node: ast.AST, scope: str) -> None: + for child in ast.iter_child_nodes(node): + child_scope = child.name if isinstance(child, FUNCTION_NODES) else scope + if isinstance(child, ast.Call) and _is_create_session_call(child): + sites[CreateSessionCallSite(relative_path, scope)] += 1 + walk(child, child_scope) + + walk(tree, "") + return sites + + +def test_create_session_call_sites_stay_read_only() -> None: + call_sites: Counter[CreateSessionCallSite] = Counter() + for path in REPO_ROOT.glob("app/assets/**/*.py"): + call_sites.update(_create_session_call_sites(path)) + + expected = Counter(READ_ONLY_CREATE_SESSION_CALL_SITES) + unexpected = call_sites - expected + missing = expected - call_sites + assert not unexpected, ( + f"Unexpected create_session() call sites: {sorted(unexpected)}. " + "Use run_write_txn for writing sessions." + ) + assert not missing, f"Read-only create_session() allowlist entries missing from the tree: {sorted(missing)}" + + +RuntimeCase = Literal[ + "register_executed_output", + "register_cached_output", + "register_file_in_place", + "upload_from_temp_path", + "create_from_hash", + "update_asset_metadata", + "delete_asset_reference", + "resolve_asset_for_download", + "resolve_hash_to_path", + "apply_tags", + "remove_tags", +] + + +def _assert_never(value: NoReturn) -> NoReturn: + raise AssertionError(f"Unexpected runtime case: {value!r}") + + +def _seed_record(session, path: Path) -> str: + content = create_content(session, str(path), size_bytes=path.stat().st_size) + record = create_record(session, content.id, path.name) + session.commit() + return record.id + + +def _find_orm_instance(value, depth: int = 0): + if isinstance(value, Base): + return value + if depth == 2: + return None + if is_dataclass(value) and not isinstance(value, type): + for field in fields(value): + if leak := _find_orm_instance(getattr(value, field.name), depth + 1): + return leak + elif isinstance(value, tuple): + for item in value: + if leak := _find_orm_instance(item, depth + 1): + return leak + elif isinstance(value, list): + for item in value: + if leak := _find_orm_instance(item, depth + 1): + return leak + elif isinstance(value, dict): + for key, item in value.items(): + if leak := _find_orm_instance(key, depth + 1): + return leak + if leak := _find_orm_instance(item, depth + 1): + return leak + return None + + +@pytest.mark.parametrize( + "case", + [ + pytest.param("register_executed_output", id="register_executed_output PASS"), + pytest.param("register_cached_output", id="register_cached_output PASS"), + pytest.param("register_file_in_place", id="register_file_in_place PASS"), + pytest.param("upload_from_temp_path", id="upload_from_temp_path PASS"), + pytest.param("create_from_hash", id="create_from_hash PASS"), + pytest.param("update_asset_metadata", id="update_asset_metadata PASS"), + pytest.param("delete_asset_reference", id="delete_asset_reference PASS"), + pytest.param("resolve_asset_for_download", id="resolve_asset_for_download PASS"), + pytest.param("resolve_hash_to_path", id="resolve_hash_to_path PASS"), + pytest.param("apply_tags", id="apply_tags PASS"), + pytest.param("remove_tags", id="remove_tags PASS"), + ], +) +def test_converted_value_results_do_not_leak_orm_instances( + case: RuntimeCase, session, mock_create_session, monkeypatch, temp_dir, tmp_path +) -> None: + match case: + case "register_executed_output": + path = Path(folder_paths.get_output_directory()) / "write-session-executed.bin" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"executed") + result = register_executed_output(str(path), job_id="write-session") + case "register_cached_output": + path = Path(folder_paths.get_output_directory()) / "write-session-cached.bin" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"cached") + assert register_executed_output(str(path), job_id="source") is not None + result = register_cached_output(str(path), job_id="delivery") + case "register_file_in_place": + path = tmp_path / "in-place.bin" + path.write_bytes(b"in-place") + result = register_file_in_place(str(path), "in-place.bin", ["output"]) + case "upload_from_temp_path": + path = tmp_path / "upload.part" + path.write_bytes(b"upload") + result = upload_from_temp_path( + str(path), name="upload.bin", tags=["output"], client_filename="upload.bin" + ) + case "create_from_hash": + digest = "a" * 64 + path = temp_dir / "hash-source.bin" + path.write_bytes(b"hash-source") + monkeypatch.setattr("app.assets.mode.hashing_enabled", lambda: True) + create_content(session, str(path), to_stored_hash(digest), path.stat().st_size) + session.commit() + result = create_from_hash(f"blake3:{digest}", "derived.bin") + case "update_asset_metadata": + path = temp_dir / "metadata.bin" + path.write_bytes(b"metadata") + result = update_asset_metadata(_seed_record(session, path), name="renamed") + case "delete_asset_reference": + path = temp_dir / "delete.bin" + path.write_bytes(b"delete") + result = delete_asset_reference(_seed_record(session, path)) + case "resolve_asset_for_download": + path = temp_dir / "download.bin" + path.write_bytes(b"download") + result = resolve_asset_for_download(_seed_record(session, path)) + case "resolve_hash_to_path": + digest = "b" * 64 + path = temp_dir / "hash-download.bin" + path.write_bytes(b"hash-download") + content = create_content( + session, str(path), to_stored_hash(digest), path.stat().st_size + ) + create_record(session, content.id, path.name) + session.commit() + result = resolve_hash_to_path(f"blake3:{digest}") + case "apply_tags": + path = temp_dir / "apply-tags.bin" + path.write_bytes(b"apply-tags") + result = apply_tags(_seed_record(session, path), ["tag"]) + case "remove_tags": + path = temp_dir / "remove-tags.bin" + path.write_bytes(b"remove-tags") + content = create_content(session, str(path), size_bytes=path.stat().st_size) + record = create_record(session, content.id, path.name, tags=["tag"]) + session.commit() + result = remove_tags(record.id, ["tag"]) + case unreachable: + _assert_never(unreachable) + + assert result is not None + assert _find_orm_instance(result) is None From 05c84e7584f3dd1bfac99d0ee401bc2adb7aae52 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 16:53:27 -0700 Subject: [PATCH 009/111] test(assets): deterministic write-contention regression coverage --- .../assets_test/test_write_contention_e2e.py | 94 +++++++++++++++++++ tests-unit/seeder_test/test_seeder.py | 13 +-- 2 files changed, 98 insertions(+), 9 deletions(-) create mode 100644 tests-unit/assets_test/test_write_contention_e2e.py diff --git a/tests-unit/assets_test/test_write_contention_e2e.py b/tests-unit/assets_test/test_write_contention_e2e.py new file mode 100644 index 00000000000..380d130ee91 --- /dev/null +++ b/tests-unit/assets_test/test_write_contention_e2e.py @@ -0,0 +1,94 @@ +import logging +import sqlite3 +import threading +import time + +import folder_paths +import pytest +from PIL import Image + +import app.database.db as db_mod +from app.assets import lifecycle, mode +from app.assets.manager import default_asset_manager +from app.assets.services.hash_mode_state import clear_transition_queue +from app.assets.services.ingest import register_executed_output +from app.assets.services.schemas import RegisteredAsset +from comfy.cli_args import args + + +@pytest.fixture(autouse=True) +def autoclean_unit_test_assets(): + yield + + +def test_register_executed_output_waits_for_a_held_sqlite_writer( + tmp_path, monkeypatch, caplog +) -> None: + output_directory = tmp_path / "output" + input_directory = tmp_path / "input" + temp_directory = tmp_path / "temp" + for directory in (output_directory, input_directory, temp_directory): + directory.mkdir() + + database_path = tmp_path / "assets.db" + monkeypatch.setattr(args, "database_url", f"sqlite:///{database_path}") + monkeypatch.setattr(args, "enable_assets", True) + monkeypatch.setattr(args, "enable_asset_hashing", False) + monkeypatch.setattr(folder_paths, "output_directory", str(output_directory)) + monkeypatch.setattr(folder_paths, "input_directory", str(input_directory)) + monkeypatch.setattr(folder_paths, "temp_directory", str(temp_directory)) + monkeypatch.setattr(db_mod, "Session", None) + monkeypatch.setattr(db_mod, "WriteSession", None) + monkeypatch.setattr(db_mod, "_db_lock", None) + monkeypatch.setattr(lifecycle, "start_asset_seeder", lambda: False) + + db_mod.init_db() + manager = default_asset_manager() + try: + assert manager.enabled + manager.startup() + + output_path = output_directory / "ComfyUI_00001_.png" + Image.new("RGB", (1, 1), (255, 0, 0)).save(output_path) + + holder_ready = threading.Event() + holder_errors: list[sqlite3.Error] = [] + + def hold_write_lock() -> None: + connection = sqlite3.connect(database_path, timeout=1) + try: + connection.execute("BEGIN IMMEDIATE") + connection.execute("INSERT INTO tags (name) VALUES (?)", ("e2e-holder",)) + holder_ready.set() + threading.Event().wait(timeout=2) + connection.commit() + except sqlite3.Error as error: + holder_errors.append(error) + holder_ready.set() + finally: + connection.close() + + holder = threading.Thread(target=hold_write_lock) + holder.start() + try: + assert holder_ready.wait(timeout=2) + assert not holder_errors + with caplog.at_level(logging.INFO): + started = time.monotonic() + result = register_executed_output(str(output_path), job_id="write-contention") + elapsed = time.monotonic() - started + finally: + holder.join(timeout=2) + + assert not holder.is_alive() + assert not holder_errors + assert isinstance(result, RegisteredAsset) + assert elapsed >= 1.5 + assert not any("Failed to register" in record.getMessage() for record in caplog.records) + finally: + manager.shutdown() + clear_transition_queue() + mode.init(None) + db_mod.Session.kw["bind"].dispose() + db_mod.WriteSession.kw["bind"].dispose() + db_mod._db_lock.release(force=True) diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.py index 2637c74d970..c9f2c6e3d13 100644 --- a/tests-unit/seeder_test/test_seeder.py +++ b/tests-unit/seeder_test/test_seeder.py @@ -1,7 +1,6 @@ import logging import re import threading -from contextlib import nullcontext from unittest.mock import Mock import pytest @@ -143,7 +142,6 @@ def test_enrich_phase_does_not_count_returned_ids_as_failures( scan_seeder: _AssetSeeder, monkeypatch: pytest.MonkeyPatch, ) -> None: - session = Mock() batches = iter( ( [ @@ -153,10 +151,9 @@ def test_enrich_phase_does_not_count_returned_ids_as_failures( [], ) ) - monkeypatch.setattr(seeder_module, "create_session", lambda: nullcontext(session)) - monkeypatch.setattr(seeder_module, "drain_pending_verifications", lambda _session: None) - monkeypatch.setattr(seeder_module, "tick_watch_list", lambda _session: None) - monkeypatch.setattr(seeder_module, "drain_transition_queue", lambda _session: None) + monkeypatch.setattr(seeder_module, "drain_pending_verifications", lambda: None) + monkeypatch.setattr(seeder_module, "tick_watch_list", lambda: None) + monkeypatch.setattr(seeder_module, "drain_transition_queue", lambda: None) monkeypatch.setattr( seeder_module, "get_unenriched_assets_for_roots", @@ -425,7 +422,6 @@ def test_batch_insert_failure_emits_only_the_exception_type( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: - session = Mock() monkeypatch.setattr( seeder_module, "sync_root_safely", lambda _root, _progress: set() ) @@ -446,8 +442,7 @@ def fail_insert(batch, batch_tags) -> int: raise PermissionError("/private/models/asset.safetensors") monkeypatch.setattr(seeder_module, "insert_asset_specs", fail_insert) - monkeypatch.setattr(seeder_module, "create_session", lambda: nullcontext(session)) - monkeypatch.setattr(seeder_module, "tick_watch_list", lambda current_session: None) + monkeypatch.setattr(seeder_module, "tick_watch_list", lambda: None) with caplog.at_level(logging.INFO): scan_seeder._run_fast_phase(("models",)) From 38b228668e96020acf1b7390a9c2ef3abf6901c8 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 17:02:25 -0700 Subject: [PATCH 010/111] docs(assets): document the sqlite concurrency configuration and write-transaction rule --- app/assets/docs/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/assets/docs/README.md b/app/assets/docs/README.md index 4cd131f8b07..ce11f48c664 100644 --- a/app/assets/docs/README.md +++ b/app/assets/docs/README.md @@ -329,9 +329,11 @@ Deleting or replacing the database file while the server is running is undefined ### Write pressure and reader starvation -The asset database is SQLite with a single database-wide writer lock and no configured busy timeout or lock-error handling on any route. Several paths hold or contend for that lock: a non-deduplicated upload writes its bytes and mints a delivery record, while a deduplicated upload reuses existing content and mints only the record; a same-path write whose hash has changed retires the old content and inserts new content, while a same-path write whose hash matches refreshes the existing record in place; execution outputs register per-emission during the generation loop; a background enrichment pass fills hashes and metadata row by row; hash-serves write access time to every record sharing the served content; and the upload dedup claim holds the write lock across its filesystem re-check and metadata extraction. +The asset database is SQLite in WAL journal mode with a 30-second busy timeout on every connection. Several paths still contend for the single writer lock: a non-deduplicated upload writes its bytes and mints a delivery record, while a deduplicated upload reuses existing content and mints only the record; a same-path write whose hash has changed retires the old content and inserts new content, while a same-path write whose hash matches refreshes the existing record in place; execution outputs register per-emission during the generation loop; a background enrichment pass fills hashes and metadata row by row; hash-serves write access time to every record sharing the served content; and the upload dedup claim holds the write lock across its filesystem re-check and metadata extraction. -Under sustained concurrent writes, a reader such as `GET /api/assets` can exceed SQLite's default five-second busy wait and surface an unhandled `database is locked` error as HTTP 500. The failure is transient and non-corrupting: no rows are corrupted, and a later request may succeed once the write pressure eases, though nothing retries or backs off automatically. No busy timeout, lock-error translation to 503, or WAL journal mode is configured. +A session that opens as a reader and only later attempts a write can be refused instantly if another connection already holds the write lock, before the busy timeout ever gets a chance to wait. Every writing session avoids this by going through `run_write_txn`, which opens a dedicated writer session and issues `BEGIN IMMEDIATE` up front, declaring writer intent before any statement runs. A write that still finds the database locked retries up to 5 attempts total, sleeping 0.05s/0.1s/0.2s/0.4s between attempts (each jittered ±50%) and giving up once 60 seconds have elapsed since the first attempt; only a locked-database `OperationalError` is retried, every other exception propagates immediately. Contributors must not open a writing session with `create_session()` directly. `run_write_txn` is the only sanctioned path for writes, and `tests-unit/assets_test/test_write_session_convention.py` enforces that automatically. + +WAL journal mode leaves `-wal` and `-shm` sidecar files next to the main database file; treat all three as one unit for backups and copies. WAL is not supported on network filesystems, so pointing `--database-url` at a network-mounted path fails at startup with a `RuntimeError` rather than silently falling back to a less safe journal mode. ## Schema migration From d9d8f6fbd9178e0e431f56ee6056e3ce86f27e10 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 18:42:03 -0700 Subject: [PATCH 011/111] test(database): close lock-holder resources --- tests-unit/app_test/test_db_promotion.py | 9 ++++++++- tests-unit/app_test/test_db_write_txn.py | 5 ++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests-unit/app_test/test_db_promotion.py b/tests-unit/app_test/test_db_promotion.py index 0e70ebbdc06..38130ba5af0 100644 --- a/tests-unit/app_test/test_db_promotion.py +++ b/tests-unit/app_test/test_db_promotion.py @@ -22,10 +22,12 @@ def test_write_transaction_waits_for_held_writer_before_select_then_mutate(tmp_p database_path = tmp_path / "assets.db" monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") monkeypatch.setattr(db_mod, "Session", None) + monkeypatch.setattr(db_mod, "WriteSession", None) monkeypatch.setattr(db_mod, "_db_lock", None) db_mod.init_db() reader_engine = db_mod.Session.kw["bind"] + writer_engine = db_mod.WriteSession.kw["bind"] def begin_deferred(connection): connection.exec_driver_sql("BEGIN") @@ -34,12 +36,15 @@ def begin_deferred(connection): writer_started = threading.Event() def hold_write_lock(): - with sqlite3.connect(database_path) as connection: + connection = sqlite3.connect(database_path) + try: connection.execute("BEGIN IMMEDIATE") connection.execute("INSERT INTO tags (name) VALUES (?)", ("promotion-holder",)) writer_started.set() time.sleep(2) connection.rollback() + finally: + connection.close() holder = threading.Thread(target=hold_write_lock) holder.start() @@ -56,8 +61,10 @@ def select_then_mutate(session): elapsed = time.monotonic() - started_at finally: holder.join(timeout=5) + assert not holder.is_alive() event.remove(reader_engine, "begin", begin_deferred) reader_engine.dispose() + writer_engine.dispose() db_mod._db_lock.release(force=True) assert result == "written" diff --git a/tests-unit/app_test/test_db_write_txn.py b/tests-unit/app_test/test_db_write_txn.py index 6cc2f492332..78ad82c74e0 100644 --- a/tests-unit/app_test/test_db_write_txn.py +++ b/tests-unit/app_test/test_db_write_txn.py @@ -224,12 +224,15 @@ def test_run_write_txn_held_lock_respects_remaining_deadline(file_database, monk writer_started = threading.Event() def hold_lock(): - with sqlite3.connect(file_database) as holder: + holder = sqlite3.connect(file_database) + try: holder.execute("BEGIN IMMEDIATE") holder.execute("INSERT INTO tags (name) VALUES (?)", ("deadline-holder",)) writer_started.set() time.sleep(5) holder.rollback() + finally: + holder.close() holder = threading.Thread(target=hold_lock) holder.start() From 54556cb6c6b2fc018f7c5d2cfe88ea5acd348080 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 18:42:47 -0700 Subject: [PATCH 012/111] test(assets): make barrier cleanup deterministic --- .../services/test_b2_lock_hold_barriers.py | 12 ++++++------ .../assets_test/services/test_ingest_write_txn.py | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests-unit/assets_test/services/test_b2_lock_hold_barriers.py b/tests-unit/assets_test/services/test_b2_lock_hold_barriers.py index dc82b44bc88..7f88b957889 100644 --- a/tests-unit/assets_test/services/test_b2_lock_hold_barriers.py +++ b/tests-unit/assets_test/services/test_b2_lock_hold_barriers.py @@ -54,7 +54,7 @@ def _probe_write() -> None: def _blocking_fake(entered: threading.Event, release: threading.Event, real_fn): def fake(*args, **kwargs): entered.set() - release.wait(timeout=_BARRIER_TIMEOUT) + release.wait() return real_fn(*args, **kwargs) return fake @@ -102,8 +102,8 @@ def _seed() -> None: finally: release.set() worker.join(timeout=_BARRIER_TIMEOUT) + assert not worker.is_alive() - assert not worker.is_alive() assert elapsed < _PROBE_BUDGET_SECONDS assert result["created"] == 1 @@ -149,9 +149,9 @@ def _drain() -> None: finally: release.set() worker.join(timeout=_BARRIER_TIMEOUT) + assert not worker.is_alive() scanner_changes.clear_pending_verifications() - assert not worker.is_alive() assert elapsed < _PROBE_BUDGET_SECONDS assert result["processed"] == 1 @@ -195,9 +195,9 @@ def _drain() -> None: finally: release.set() worker.join(timeout=_BARRIER_TIMEOUT) + assert not worker.is_alive() hash_mode_state.clear_transition_queue() - assert not worker.is_alive() assert elapsed < _PROBE_BUDGET_SECONDS assert result.get("done") is True @@ -244,8 +244,8 @@ def _enrich() -> None: finally: release.set() worker.join(timeout=_BARRIER_TIMEOUT) + assert not worker.is_alive() - assert not worker.is_alive() assert elapsed < _PROBE_BUDGET_SECONDS assert result["outcome"] == (1, []) @@ -292,7 +292,7 @@ def _enrich() -> None: finally: release.set() worker.join(timeout=_BARRIER_TIMEOUT) + assert not worker.is_alive() - assert not worker.is_alive() assert elapsed < _PROBE_BUDGET_SECONDS assert result["outcome"] == (1, []) diff --git a/tests-unit/assets_test/services/test_ingest_write_txn.py b/tests-unit/assets_test/services/test_ingest_write_txn.py index a7d04a6936a..290861ed303 100644 --- a/tests-unit/assets_test/services/test_ingest_write_txn.py +++ b/tests-unit/assets_test/services/test_ingest_write_txn.py @@ -57,7 +57,7 @@ def _probe_write() -> None: def _blocking_fake(entered: threading.Event, release: threading.Event, real_fn): def fake(*args, **kwargs): entered.set() - release.wait(timeout=_BARRIER_TIMEOUT) + release.wait() return real_fn(*args, **kwargs) return fake @@ -460,8 +460,8 @@ def _register() -> None: finally: release.set() worker.join(timeout=_BARRIER_TIMEOUT) + assert not worker.is_alive() - assert not worker.is_alive() assert elapsed < _PROBE_BUDGET_SECONDS registered = result["registered"] assert registered is not None @@ -511,8 +511,8 @@ def _settle() -> None: finally: release.set() worker.join(timeout=_BARRIER_TIMEOUT) + assert not worker.is_alive() - assert not worker.is_alive() assert elapsed < _PROBE_BUDGET_SECONDS expected_digest, _expected_stat = real_snapshot_hash(path) @@ -558,8 +558,8 @@ def _register() -> None: finally: release.set() worker.join(timeout=_BARRIER_TIMEOUT) + assert not worker.is_alive() - assert not worker.is_alive() assert elapsed < _PROBE_BUDGET_SECONDS upload_result = result["upload"] assert upload_result is not None From 5fcebe06cfb361434afa027c4c1b4915d958bcef Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 19:34:55 -0700 Subject: [PATCH 013/111] ci: expose unit test failures --- .github/workflows/test-unit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index d05179cd3da..987605fb9e7 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -27,4 +27,4 @@ jobs: - name: Run Unit Tests run: | pip install -r tests-unit/requirements.txt - python -m pytest tests-unit + python -m pytest -o addopts= tests-unit From b72c1c68fe2cde228a095c69ee65d6883176ea93 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 19:49:15 -0700 Subject: [PATCH 014/111] fix(database): enforce write timeout before begin --- app/database/db.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/database/db.py b/app/database/db.py index 51241c656ad..e13a3d9ca2f 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -322,6 +322,15 @@ def cap_writer_busy_timeout(dbapi_connection, connection_record, connection_prox timeout_ms = max(1, min(_SQLITE_BUSY_TIMEOUT_MS, remaining_ms)) dbapi_connection.execute(f"PRAGMA busy_timeout = {int(timeout_ms)}") + @event.listens_for(writer_engine, "begin", insert=True) + def cap_writer_busy_timeout_before_begin(connection): + deadline = getattr(_attempt_lock_deadline, "value", None) + timeout_ms = _SQLITE_BUSY_TIMEOUT_MS + if deadline is not None: + remaining_ms = int((deadline - time.monotonic()) * 1000) + timeout_ms = max(1, min(_SQLITE_BUSY_TIMEOUT_MS, remaining_ms)) + connection.exec_driver_sql(f"PRAGMA busy_timeout = {int(timeout_ms)}") + with reader_engine.connect(): pass with writer_engine.connect(): From 7cf1a6b68f26fac15cc08e3ad2f694559c349d75 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 19:49:38 -0700 Subject: [PATCH 015/111] fix(assets): support cross-device upload moves --- app/assets/services/ingest.py | 8 ++++++++ .../services/test_cross_volume_upload.py | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 tests-unit/assets_test/services/test_cross_volume_upload.py diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index 866b09d3cc0..d488dc1ecd0 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -8,9 +8,11 @@ """ import contextlib +import errno import logging import mimetypes import os +import shutil from typing import Any, NamedTuple from sqlalchemy import func, select @@ -187,6 +189,12 @@ def _move_temp_to_dest(temp_path: str, dest_abs: str) -> None: os.makedirs(os.path.dirname(dest_abs), exist_ok=True) try: os.replace(temp_path, dest_abs) + except OSError as error: + if error.errno == errno.EXDEV: + shutil.copy2(temp_path, dest_abs) + os.unlink(temp_path) + return + raise RuntimeError(f"failed to move uploaded file into place: {error}") from error except Exception as e: raise RuntimeError(f"failed to move uploaded file into place: {e}") from e diff --git a/tests-unit/assets_test/services/test_cross_volume_upload.py b/tests-unit/assets_test/services/test_cross_volume_upload.py new file mode 100644 index 00000000000..4e4c512309f --- /dev/null +++ b/tests-unit/assets_test/services/test_cross_volume_upload.py @@ -0,0 +1,20 @@ +import errno +from pathlib import Path + +import app.assets.services.ingest as ingest + + +def test_move_temp_to_dest_copies_across_filesystems(tmp_path: Path, monkeypatch) -> None: + source = tmp_path / "upload.part" + destination = tmp_path / "output" / "upload.bin" + source.write_bytes(b"upload") + + def fail_cross_volume_move(*_args: object) -> None: + raise OSError(errno.EXDEV, "Invalid cross-device link") + + monkeypatch.setattr(ingest.os, "replace", fail_cross_volume_move) + + ingest._move_temp_to_dest(str(source), str(destination)) + + assert destination.read_bytes() == b"upload" + assert not source.exists() From 622f579e7af78c2771cd02d1f165d194b1c04092 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 19:50:04 -0700 Subject: [PATCH 016/111] ci: restore standard pytest output --- .github/workflows/test-unit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 987605fb9e7..d05179cd3da 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -27,4 +27,4 @@ jobs: - name: Run Unit Tests run: | pip install -r tests-unit/requirements.txt - python -m pytest -o addopts= tests-unit + python -m pytest tests-unit From 70ab3bdb3f1350e94701823309db06f4e0949e1e Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 20:41:57 -0700 Subject: [PATCH 017/111] fix(database): configure writer timeout before transaction begin --- app/database/db.py | 51 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/app/database/db.py b/app/database/db.py index e13a3d9ca2f..3762f6ddd97 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -236,6 +236,19 @@ def _configure_runtime_connection(dbapi_connection, db_path): cursor.close() +def _writer_busy_timeout_ms(): + deadline = getattr(_attempt_lock_deadline, "value", None) + if deadline is None: + return _SQLITE_BUSY_TIMEOUT_MS + remaining_ms = int((deadline - time.monotonic()) * 1000) + return max(1, min(_SQLITE_BUSY_TIMEOUT_MS, remaining_ms)) + + +def _configure_writer_busy_timeout(dbapi_connection): + cursor = dbapi_connection.execute(f"PRAGMA busy_timeout = {_writer_busy_timeout_ms()}") + cursor.close() + + def _migrate_and_bind(db_url, db_path, db_exists): config = get_alembic_config() inspection_engine = create_engine(db_url) @@ -315,21 +328,11 @@ def begin_immediate(connection): @event.listens_for(writer_engine, "checkout") def cap_writer_busy_timeout(dbapi_connection, connection_record, connection_proxy): - deadline = getattr(_attempt_lock_deadline, "value", None) - timeout_ms = _SQLITE_BUSY_TIMEOUT_MS - if deadline is not None: - remaining_ms = int((deadline - time.monotonic()) * 1000) - timeout_ms = max(1, min(_SQLITE_BUSY_TIMEOUT_MS, remaining_ms)) - dbapi_connection.execute(f"PRAGMA busy_timeout = {int(timeout_ms)}") + _configure_writer_busy_timeout(dbapi_connection) @event.listens_for(writer_engine, "begin", insert=True) def cap_writer_busy_timeout_before_begin(connection): - deadline = getattr(_attempt_lock_deadline, "value", None) - timeout_ms = _SQLITE_BUSY_TIMEOUT_MS - if deadline is not None: - remaining_ms = int((deadline - time.monotonic()) * 1000) - timeout_ms = max(1, min(_SQLITE_BUSY_TIMEOUT_MS, remaining_ms)) - connection.exec_driver_sql(f"PRAGMA busy_timeout = {int(timeout_ms)}") + connection.exec_driver_sql(f"PRAGMA busy_timeout = {_writer_busy_timeout_ms()}") with reader_engine.connect(): pass @@ -364,18 +367,36 @@ def run_write_txn(work: Callable[["SQLAlchemySession"], T]) -> T: raise locked_error _attempt_lock_deadline.value = retry_deadline - session = WriteSession() + connection = None + session = None try: + if WriteSession is Session: + session = WriteSession() + else: + writer_engine = getattr(WriteSession, "kw", {}).get("bind") + if writer_engine is None: + session = WriteSession() + else: + connection = writer_engine.connect() + _configure_writer_busy_timeout(connection.connection.driver_connection) + connection.begin() + session = SQLAlchemySession(bind=connection, join_transaction_mode="control_fully") result = work(session) session.commit() + if connection is not None and connection.in_transaction(): + connection.commit() return result except OperationalError as exc: if "locked" not in str(exc.orig): raise locked_error = exc finally: - session.rollback() - session.close() + if session is not None: + session.rollback() + session.close() + if connection is not None: + connection.rollback() + connection.close() _attempt_lock_deadline.value = None if attempt == len(_WRITE_TXN_BACKOFF_SECONDS) or time.monotonic() >= retry_deadline: From 632fe8481482001ff814692a08a4d55570baeb19 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 21:04:20 -0700 Subject: [PATCH 018/111] fix(database): poll sqlite write locks within deadline --- app/database/db.py | 62 +++++++++--------------- tests-unit/app_test/test_db_write_txn.py | 4 +- 2 files changed, 25 insertions(+), 41 deletions(-) diff --git a/app/database/db.py b/app/database/db.py index 3762f6ddd97..b242c92ff5c 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -2,6 +2,7 @@ import os import random import shutil +import sqlite3 import threading import time from typing import Callable, TypeVar @@ -18,6 +19,7 @@ _WRITE_TXN_LOCK_RETRY_DEADLINE_SECONDS = 60 _WRITE_TXN_BACKOFF_SECONDS = (0.05, 0.1, 0.2, 0.4) _SQLITE_BUSY_TIMEOUT_MS = 30000 +_SQLITE_WRITE_LOCK_POLL_SECONDS = 0.01 T = TypeVar("T") @@ -236,17 +238,24 @@ def _configure_runtime_connection(dbapi_connection, db_path): cursor.close() -def _writer_busy_timeout_ms(): +def _begin_immediate(dbapi_connection): deadline = getattr(_attempt_lock_deadline, "value", None) if deadline is None: - return _SQLITE_BUSY_TIMEOUT_MS - remaining_ms = int((deadline - time.monotonic()) * 1000) - return max(1, min(_SQLITE_BUSY_TIMEOUT_MS, remaining_ms)) + deadline = time.monotonic() + _SQLITE_BUSY_TIMEOUT_MS / 1000 - -def _configure_writer_busy_timeout(dbapi_connection): - cursor = dbapi_connection.execute(f"PRAGMA busy_timeout = {_writer_busy_timeout_ms()}") - cursor.close() + while True: + try: + cursor = dbapi_connection.execute("BEGIN IMMEDIATE") + cursor.close() + return + except sqlite3.OperationalError as exc: + error_message = str(exc).lower() + if "locked" not in error_message and "busy" not in error_message: + raise + remaining_seconds = deadline - time.monotonic() + if remaining_seconds <= 0: + raise OperationalError("BEGIN IMMEDIATE", {}, exc) from exc + time.sleep(min(_SQLITE_WRITE_LOCK_POLL_SECONDS, remaining_seconds)) def _migrate_and_bind(db_url, db_path, db_exists): @@ -315,24 +324,17 @@ def set_sqlite_pragma(dbapi_connection, connection_record): def set_reader_sqlite_pragma(dbapi_connection, connection_record): _configure_runtime_connection(dbapi_connection, db_path) - writer_engine = create_engine(db_url, connect_args={"timeout": 30}) + writer_engine = create_engine(db_url, connect_args={"timeout": 0}) @event.listens_for(writer_engine, "connect") def set_writer_sqlite_pragma(dbapi_connection, connection_record): dbapi_connection.isolation_level = None _configure_runtime_connection(dbapi_connection, db_path) + dbapi_connection.execute("PRAGMA busy_timeout=0").close() @event.listens_for(writer_engine, "begin") def begin_immediate(connection): - connection.exec_driver_sql("BEGIN IMMEDIATE") - - @event.listens_for(writer_engine, "checkout") - def cap_writer_busy_timeout(dbapi_connection, connection_record, connection_proxy): - _configure_writer_busy_timeout(dbapi_connection) - - @event.listens_for(writer_engine, "begin", insert=True) - def cap_writer_busy_timeout_before_begin(connection): - connection.exec_driver_sql(f"PRAGMA busy_timeout = {_writer_busy_timeout_ms()}") + _begin_immediate(connection.connection.driver_connection) with reader_engine.connect(): pass @@ -367,36 +369,18 @@ def run_write_txn(work: Callable[["SQLAlchemySession"], T]) -> T: raise locked_error _attempt_lock_deadline.value = retry_deadline - connection = None - session = None + session = WriteSession() try: - if WriteSession is Session: - session = WriteSession() - else: - writer_engine = getattr(WriteSession, "kw", {}).get("bind") - if writer_engine is None: - session = WriteSession() - else: - connection = writer_engine.connect() - _configure_writer_busy_timeout(connection.connection.driver_connection) - connection.begin() - session = SQLAlchemySession(bind=connection, join_transaction_mode="control_fully") result = work(session) session.commit() - if connection is not None and connection.in_transaction(): - connection.commit() return result except OperationalError as exc: if "locked" not in str(exc.orig): raise locked_error = exc finally: - if session is not None: - session.rollback() - session.close() - if connection is not None: - connection.rollback() - connection.close() + session.rollback() + session.close() _attempt_lock_deadline.value = None if attempt == len(_WRITE_TXN_BACKOFF_SECONDS) or time.monotonic() >= retry_deadline: diff --git a/tests-unit/app_test/test_db_write_txn.py b/tests-unit/app_test/test_db_write_txn.py index 78ad82c74e0..5f635044a4d 100644 --- a/tests-unit/app_test/test_db_write_txn.py +++ b/tests-unit/app_test/test_db_write_txn.py @@ -251,7 +251,7 @@ def hold_lock(): assert 1.5 <= elapsed < 3 -def test_run_write_txn_reduces_busy_timeout_after_intermediate_commit( +def test_run_write_txn_reopens_immediate_transaction_after_intermediate_commit( file_database, monkeypatch ): run_write_txn = db_mod.run_write_txn @@ -267,7 +267,7 @@ def work(session): run_write_txn(work) - assert seen_timeouts[1] < seen_timeouts[0] + assert seen_timeouts == [0, 0] def test_run_write_txn_uses_a_fresh_session_for_each_attempt(memory_database, monkeypatch): From 2140d023a565ba166b82ea4b385f6efd23eeb992 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 21:48:47 -0700 Subject: [PATCH 019/111] test(assets): hoist imports and rename tests to describe their subject --- ..._integrity.py => test_queue_commit_failure_integrity.py} | 5 +---- ...grity.py => test_scanner_retry_and_counter_integrity.py} | 6 +++--- ...ck_hold_barriers.py => test_write_lock_hold_barriers.py} | 4 ++-- 3 files changed, 6 insertions(+), 9 deletions(-) rename tests-unit/assets_test/services/{test_b2_queue_commit_failure_integrity.py => test_queue_commit_failure_integrity.py} (98%) rename tests-unit/assets_test/services/{test_b1_counter_and_retry_integrity.py => test_scanner_retry_and_counter_integrity.py} (96%) rename tests-unit/assets_test/services/{test_b2_lock_hold_barriers.py => test_write_lock_hold_barriers.py} (98%) diff --git a/tests-unit/assets_test/services/test_b2_queue_commit_failure_integrity.py b/tests-unit/assets_test/services/test_queue_commit_failure_integrity.py similarity index 98% rename from tests-unit/assets_test/services/test_b2_queue_commit_failure_integrity.py rename to tests-unit/assets_test/services/test_queue_commit_failure_integrity.py index 1e361df44ae..adc3ea57535 100644 --- a/tests-unit/assets_test/services/test_b2_queue_commit_failure_integrity.py +++ b/tests-unit/assets_test/services/test_queue_commit_failure_integrity.py @@ -7,6 +7,7 @@ from sqlalchemy.orm import sessionmaker import app.database.db as db_mod +from app.assets import mode as mode_module from app.assets import scanner from app.assets import scanner_admission from app.assets import scanner_changes @@ -111,8 +112,6 @@ def test_transition_queue_and_companion_state_survive_terminal_commit_failure( write_stored_mode(session, "off") session.commit() - from app.assets import mode as mode_module - monkeypatch.setattr(mode_module, "hashing_enabled", lambda: True) clear_transition_queue() @@ -144,8 +143,6 @@ def test_transition_in_flight_flag_survives_a_failed_final_mode_commit( write_stored_mode(session, "off") session.commit() - from app.assets import mode as mode_module - monkeypatch.setattr(mode_module, "hashing_enabled", lambda: True) clear_transition_queue() diff --git a/tests-unit/assets_test/services/test_b1_counter_and_retry_integrity.py b/tests-unit/assets_test/services/test_scanner_retry_and_counter_integrity.py similarity index 96% rename from tests-unit/assets_test/services/test_b1_counter_and_retry_integrity.py rename to tests-unit/assets_test/services/test_scanner_retry_and_counter_integrity.py index 95d3a038048..9e00f484e22 100644 --- a/tests-unit/assets_test/services/test_b1_counter_and_retry_integrity.py +++ b/tests-unit/assets_test/services/test_scanner_retry_and_counter_integrity.py @@ -58,7 +58,7 @@ def _fast_retries(monkeypatch): monkeypatch.setattr(db_mod.time, "sleep", lambda _seconds: None) -def test_group_a_transient_locked_error_retried_preserves_queue_once( +def test_scanner_sync_transient_locked_error_retried_preserves_queue_once( db_engine, tmp_path: Path, monkeypatch, session ): """A commit-time locked failure that later succeeds must publish the @@ -93,7 +93,7 @@ class _HashingOn: scanner_changes.clear_pending_verifications() -def test_group_a_sync_permission_diagnostic_published_exactly_once_after_retry( +def test_scanner_sync_permission_diagnostic_published_exactly_once_after_retry( db_engine, tmp_path: Path, monkeypatch, session, caplog ): """A commit-time locked failure that later succeeds must publish the @@ -135,7 +135,7 @@ def deny_stat(candidate_path, *args, **kwargs): ] -def test_b1_counter_integrity_under_a_locked_failure_at_row_n( +def test_enrichment_counter_integrity_under_a_locked_failure_at_row_n( db_engine, tmp_path: Path, session, monkeypatch ): """Injecting a locked failure at the middle row of a batch must not diff --git a/tests-unit/assets_test/services/test_b2_lock_hold_barriers.py b/tests-unit/assets_test/services/test_write_lock_hold_barriers.py similarity index 98% rename from tests-unit/assets_test/services/test_b2_lock_hold_barriers.py rename to tests-unit/assets_test/services/test_write_lock_hold_barriers.py index 7f88b957889..14e41d0a8c1 100644 --- a/tests-unit/assets_test/services/test_b2_lock_hold_barriers.py +++ b/tests-unit/assets_test/services/test_write_lock_hold_barriers.py @@ -202,7 +202,7 @@ def _drain() -> None: assert result.get("done") is True -def test_b1_enrichment_hashing_does_not_hold_the_write_lock( +def test_enrichment_hashing_does_not_hold_the_write_lock( file_database, tmp_path, monkeypatch ): path = tmp_path / "enrich-hash.bin" @@ -250,7 +250,7 @@ def _enrich() -> None: assert result["outcome"] == (1, []) -def test_b1_enrichment_metadata_extraction_does_not_hold_the_write_lock( +def test_enrichment_metadata_extraction_does_not_hold_the_write_lock( file_database, tmp_path, monkeypatch ): path = tmp_path / "enrich-metadata.bin" From b78c88115a29c7c8f5f1b4b96a8c498df34d1a23 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 22:45:07 -0700 Subject: [PATCH 020/111] fix(database): bound sqlite writer lock waits --- app/database/db.py | 29 ++++++--- tests-unit/app_test/test_db_write_txn.py | 77 ++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 9 deletions(-) diff --git a/app/database/db.py b/app/database/db.py index b242c92ff5c..cdcfd960362 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -3,6 +3,7 @@ import random import shutil import sqlite3 +import sys import threading import time from typing import Callable, TypeVar @@ -239,9 +240,10 @@ def _configure_runtime_connection(dbapi_connection, db_path): def _begin_immediate(dbapi_connection): - deadline = getattr(_attempt_lock_deadline, "value", None) - if deadline is None: - deadline = time.monotonic() + _SQLITE_BUSY_TIMEOUT_MS / 1000 + retry_deadline = getattr(_attempt_lock_deadline, "value", None) + now = time.monotonic() + busy_timeout_deadline = now + _SQLITE_BUSY_TIMEOUT_MS / 1000 + deadline = min(retry_deadline, busy_timeout_deadline) if retry_deadline is not None else busy_timeout_deadline while True: try: @@ -249,8 +251,7 @@ def _begin_immediate(dbapi_connection): cursor.close() return except sqlite3.OperationalError as exc: - error_message = str(exc).lower() - if "locked" not in error_message and "busy" not in error_message: + if not _is_retryable_lock_error(exc): raise remaining_seconds = deadline - time.monotonic() if remaining_seconds <= 0: @@ -349,6 +350,10 @@ def create_session(): return Session() +def _is_retryable_lock_error(exc: BaseException) -> bool: + return "locked" in str(exc).lower() + + def run_write_txn(work: Callable[["SQLAlchemySession"], T]) -> T: """Run a write callback with bounded lock retries; its own work is not deadline-limited.""" if getattr(_write_txn_state, "active", False): @@ -375,13 +380,19 @@ def run_write_txn(work: Callable[["SQLAlchemySession"], T]) -> T: session.commit() return result except OperationalError as exc: - if "locked" not in str(exc.orig): + if not _is_retryable_lock_error(exc.orig): raise locked_error = exc finally: - session.rollback() - session.close() - _attempt_lock_deadline.value = None + propagating_exception = sys.exc_info()[0] is not None + try: + session.rollback() + except BaseException: + if not propagating_exception: + raise + finally: + session.close() + _attempt_lock_deadline.value = None if attempt == len(_WRITE_TXN_BACKOFF_SECONDS) or time.monotonic() >= retry_deadline: raise locked_error diff --git a/tests-unit/app_test/test_db_write_txn.py b/tests-unit/app_test/test_db_write_txn.py index 5f635044a4d..67564120dbb 100644 --- a/tests-unit/app_test/test_db_write_txn.py +++ b/tests-unit/app_test/test_db_write_txn.py @@ -146,6 +146,83 @@ def work(_session): assert attempts == 3 +def test_begin_immediate_caps_a_single_wait_at_the_busy_timeout(monkeypatch): + clock = {"now": 0.0} + + class Connection: + def execute(self, _statement): + raise sqlite3.OperationalError("database is locked") + + monkeypatch.setattr(db_mod.time, "monotonic", lambda: clock["now"]) + monkeypatch.setattr( + db_mod.time, + "sleep", + lambda seconds: clock.__setitem__("now", clock["now"] + seconds), + ) + db_mod._attempt_lock_deadline.value = 60.0 + + try: + with pytest.raises(OperationalError, match="database is locked"): + db_mod._begin_immediate(Connection()) + finally: + db_mod._attempt_lock_deadline.value = None + + assert clock["now"] == pytest.approx(30.0) + + +def test_begin_immediate_and_write_retries_share_locked_only_classification( + memory_database, monkeypatch +): + clock = {"now": 0.0} + attempts = 0 + + class Connection: + def execute(self, _statement): + raise sqlite3.OperationalError("database is busy") + + monkeypatch.setattr(db_mod.time, "monotonic", lambda: clock["now"]) + monkeypatch.setattr( + db_mod.time, + "sleep", + lambda seconds: clock.__setitem__("now", clock["now"] + seconds), + ) + + with pytest.raises(sqlite3.OperationalError, match="database is busy"): + db_mod._begin_immediate(Connection()) + + def work(_session): + nonlocal attempts + attempts += 1 + raise OperationalError("INSERT", {}, sqlite3.OperationalError("database is busy")) + + with pytest.raises(OperationalError, match="database is busy"): + db_mod.run_write_txn(work) + + assert attempts == 1 + + +def test_run_write_txn_closes_when_rollback_fails_without_masking_work_error(monkeypatch): + class Session: + closed = False + + def rollback(self): + raise RuntimeError("rollback failure") + + def close(self): + self.closed = True + + session = Session() + monkeypatch.setattr(db_mod, "WriteSession", lambda: session) + + def work(_session): + raise ValueError("work failure") + + with pytest.raises(ValueError, match="work failure"): + db_mod.run_write_txn(work) + + assert session.closed + + def test_run_write_txn_reraises_nonretryable_operational_error_without_retry(memory_database): run_write_txn = db_mod.run_write_txn attempts = 0 From 6090a9efc5f6508ba45c3042ae136894436783cb Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 22:45:07 -0700 Subject: [PATCH 021/111] fix(assets): publish cross-device uploads atomically --- app/assets/services/ingest.py | 140 ++++++++---------- .../services/test_cross_volume_upload.py | 41 ++++- 2 files changed, 104 insertions(+), 77 deletions(-) diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index d488dc1ecd0..0f119e790d1 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -191,7 +191,15 @@ def _move_temp_to_dest(temp_path: str, dest_abs: str) -> None: os.replace(temp_path, dest_abs) except OSError as error: if error.errno == errno.EXDEV: - shutil.copy2(temp_path, dest_abs) + destination_dir = os.path.dirname(dest_abs) + destination_temp = os.path.join(destination_dir, f".{os.path.basename(dest_abs)}.tmp") + try: + shutil.copy2(temp_path, destination_temp) + os.replace(destination_temp, dest_abs) + except BaseException: + if os.path.exists(destination_temp): + os.unlink(destination_temp) + raise os.unlink(temp_path) return raise RuntimeError(f"failed to move uploaded file into place: {error}") from error @@ -541,7 +549,7 @@ def _reuse_qualified_content( stored_hash: str, spec: _UploadRecordSpec, ) -> UploadResult | None: - for restart in range(4): + for _restart in range(4): preflight = _preflight_upload_record(stored_hash, None, spec) if preflight is None: return None @@ -551,18 +559,13 @@ def _reuse_qualified_content( lambda session: _apply_reused_upload_record(session, prepared) ) except _PreflightStale: - if restart == 3: - logging.warning( - "Upload preflight changed three times; falling back to in-transaction metadata extraction" - ) - return run_write_txn( - lambda session: _reuse_qualified_content_in_txn( - session, - stored_hash, - spec, - ) - ) - return None + continue + logging.warning( + "Upload preflight changed three times; falling back to in-transaction metadata extraction" + ) + return run_write_txn( + lambda session: _reuse_qualified_content_in_txn(session, stored_hash, spec) + ) def _preflight_settle_target(dest_abs: str) -> _SettleTargetPreflight | None: @@ -674,7 +677,7 @@ def _settle_destination_before_write_in_txn(session: Session, dest_abs: str) -> def _settle_destination_before_write(dest_abs: str) -> None: - for restart in range(4): + for _restart in range(4): preflight = _preflight_settle_target(dest_abs) if preflight is None: return @@ -683,17 +686,14 @@ def _settle_destination_before_write(dest_abs: str) -> None: run_write_txn(lambda session: _apply_settle_target(session, prepared)) return except _PreflightStale: - if restart == 3: - logging.warning( - "Upload destination preflight changed three times; falling back to in-transaction hashing" - ) - run_write_txn( - lambda session: _settle_destination_before_write_in_txn( - session, - dest_abs, - ) - ) - return + continue + logging.warning( + "Upload destination preflight changed three times; falling back to in-transaction hashing" + ) + run_write_txn( + lambda session: _settle_destination_before_write_in_txn(session, dest_abs) + ) + return def _create_content_and_upload_record( @@ -703,7 +703,7 @@ def _create_content_and_upload_record( content_written: bool, spec: _UploadRecordSpec, ) -> UploadResult: - for restart in range(4): + for _restart in range(4): preflight = _preflight_upload_record(None, path, spec) if preflight is None: raise RuntimeError("new upload record requires a destination path") @@ -737,42 +737,37 @@ def _work(session: Session) -> UploadResult: try: return run_write_txn(_work) except _PreflightStale: - if restart == 3: - logging.warning( - "Upload record preflight changed three times; falling back to in-transaction metadata extraction" - ) + continue - def _fallback_work(session: Session) -> UploadResult: - _reconcile_live_content_at_path( - session, - path, - facts, - content_written=content_written, - ) - content, inserted = create_content_reporting_insert( - session, - path, - stored_hash, - facts.size_bytes, - facts.mtime_ns, - ) - created_content_id = content.id if inserted else None - try: - record = _create_upload_record_in_txn( - session, - content.id, - spec, - path, - ) - except Exception: - session.rollback() - if created_content_id is not None: - _discard_unreferenced_content(session, created_content_id) - raise - return _record_to_upload_result(session, record, created_new=True) - - return run_write_txn(_fallback_work) - raise RuntimeError("upload record preflight retry loop did not return") + logging.warning( + "Upload record preflight changed three times; falling back to in-transaction metadata extraction" + ) + + def _fallback_work(session: Session) -> UploadResult: + _reconcile_live_content_at_path( + session, + path, + facts, + content_written=content_written, + ) + content, inserted = create_content_reporting_insert( + session, + path, + stored_hash, + facts.size_bytes, + facts.mtime_ns, + ) + created_content_id = content.id if inserted else None + try: + record = _create_upload_record_in_txn(session, content.id, spec, path) + except Exception: + session.rollback() + if created_content_id is not None: + _discard_unreferenced_content(session, created_content_id) + raise + return _record_to_upload_result(session, record, created_new=True) + + return run_write_txn(_fallback_work) def upload_from_temp_path( @@ -1083,7 +1078,7 @@ def register_cached_output( ) -> RegisteredAsset | None: locator = os.path.abspath(abs_path) try: - for restart in range(4): + for _restart in range(4): preflight = _preflight_cached_registration(locator) if preflight is None: logging.info( @@ -1110,17 +1105,13 @@ def register_cached_output( ) ) except _PreflightStale: - if restart == 3: - logging.warning( - "Cached-output preflight changed three times; falling back to in-transaction metadata extraction" - ) - return run_write_txn( - lambda session: _register_cached_output_in_txn( - session, - locator, - job_id, - ) - ) + continue + logging.warning( + "Cached-output preflight changed three times; falling back to in-transaction metadata extraction" + ) + return run_write_txn( + lambda session: _register_cached_output_in_txn(session, locator, job_id) + ) except Exception as exc: logging.exception("Failed to register cached output: %s", locator) emit( @@ -1129,7 +1120,6 @@ def register_cached_output( error_type=error_type(exc), ) return None - return None def register_executed_output( diff --git a/tests-unit/assets_test/services/test_cross_volume_upload.py b/tests-unit/assets_test/services/test_cross_volume_upload.py index 4e4c512309f..b030ec93a3f 100644 --- a/tests-unit/assets_test/services/test_cross_volume_upload.py +++ b/tests-unit/assets_test/services/test_cross_volume_upload.py @@ -1,6 +1,8 @@ import errno from pathlib import Path +import pytest + import app.assets.services.ingest as ingest @@ -9,8 +11,12 @@ def test_move_temp_to_dest_copies_across_filesystems(tmp_path: Path, monkeypatch destination = tmp_path / "output" / "upload.bin" source.write_bytes(b"upload") - def fail_cross_volume_move(*_args: object) -> None: - raise OSError(errno.EXDEV, "Invalid cross-device link") + real_replace = ingest.os.replace + + def fail_cross_volume_move(source_path: str, destination_path: str) -> None: + if source_path == str(source) and destination_path == str(destination): + raise OSError(errno.EXDEV, "Invalid cross-device link") + real_replace(source_path, destination_path) monkeypatch.setattr(ingest.os, "replace", fail_cross_volume_move) @@ -18,3 +24,34 @@ def fail_cross_volume_move(*_args: object) -> None: assert destination.read_bytes() == b"upload" assert not source.exists() + + +def test_move_temp_to_dest_keeps_final_destination_intact_when_cross_volume_copy_fails( + tmp_path: Path, monkeypatch +) -> None: + source = tmp_path / "upload.part" + destination = tmp_path / "output" / "upload.bin" + source.write_bytes(b"upload") + destination.parent.mkdir() + destination.write_bytes(b"existing") + + real_replace = ingest.os.replace + + def replace_across_filesystems(source_path: str, destination_path: str) -> None: + if source_path == str(source) and destination_path == str(destination): + raise OSError(errno.EXDEV, "Invalid cross-device link") + real_replace(source_path, destination_path) + + def fail_copy(source_path: str, destination_path: str) -> None: + Path(destination_path).write_bytes(Path(source_path).read_bytes()[:2]) + raise OSError("copy failed") + + monkeypatch.setattr(ingest.os, "replace", replace_across_filesystems) + monkeypatch.setattr(ingest.shutil, "copy2", fail_copy) + + with pytest.raises(OSError, match="copy failed"): + ingest._move_temp_to_dest(str(source), str(destination)) + + assert destination.read_bytes() == b"existing" + assert source.read_bytes() == b"upload" + assert not (destination.parent / ".upload.bin.tmp").exists() From 33e803d5b90e88bdc1486525b9c7a6b4f3e10bbf Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 22:45:07 -0700 Subject: [PATCH 022/111] docs(assets): clarify sqlite writer timeout behavior --- app/assets/docs/README.md | 2 +- .../assets_test/services/test_queue_commit_failure_integrity.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/app/assets/docs/README.md b/app/assets/docs/README.md index ce11f48c664..fab20268df3 100644 --- a/app/assets/docs/README.md +++ b/app/assets/docs/README.md @@ -329,7 +329,7 @@ Deleting or replacing the database file while the server is running is undefined ### Write pressure and reader starvation -The asset database is SQLite in WAL journal mode with a 30-second busy timeout on every connection. Several paths still contend for the single writer lock: a non-deduplicated upload writes its bytes and mints a delivery record, while a deduplicated upload reuses existing content and mints only the record; a same-path write whose hash has changed retires the old content and inserts new content, while a same-path write whose hash matches refreshes the existing record in place; execution outputs register per-emission during the generation loop; a background enrichment pass fills hashes and metadata row by row; hash-serves write access time to every record sharing the served content; and the upload dedup claim holds the write lock across its filesystem re-check and metadata extraction. +The asset database is SQLite in WAL journal mode. Reader connections use a 30-second busy timeout. Writer connections poll `BEGIN IMMEDIATE` in Python, so each lock wait is capped at 30 seconds while the overall retry deadline remains 60 seconds. Several paths still contend for the single writer lock: a non-deduplicated upload writes its bytes and mints a delivery record, while a deduplicated upload reuses existing content and mints only the record; a same-path write whose hash has changed retires the old content and inserts new content, while a same-path write whose hash matches refreshes the existing record in place; execution outputs register per-emission during the generation loop; a background enrichment pass fills hashes and metadata row by row; hash-serves write access time to every record sharing the served content; and the upload dedup claim holds the write lock across its filesystem re-check and metadata extraction. A session that opens as a reader and only later attempts a write can be refused instantly if another connection already holds the write lock, before the busy timeout ever gets a chance to wait. Every writing session avoids this by going through `run_write_txn`, which opens a dedicated writer session and issues `BEGIN IMMEDIATE` up front, declaring writer intent before any statement runs. A write that still finds the database locked retries up to 5 attempts total, sleeping 0.05s/0.1s/0.2s/0.4s between attempts (each jittered ±50%) and giving up once 60 seconds have elapsed since the first attempt; only a locked-database `OperationalError` is retried, every other exception propagates immediately. Contributors must not open a writing session with `create_session()` directly. `run_write_txn` is the only sanctioned path for writes, and `tests-unit/assets_test/test_write_session_convention.py` enforces that automatically. diff --git a/tests-unit/assets_test/services/test_queue_commit_failure_integrity.py b/tests-unit/assets_test/services/test_queue_commit_failure_integrity.py index adc3ea57535..a7389ba56c9 100644 --- a/tests-unit/assets_test/services/test_queue_commit_failure_integrity.py +++ b/tests-unit/assets_test/services/test_queue_commit_failure_integrity.py @@ -151,8 +151,6 @@ def test_transition_in_flight_flag_survives_a_failed_final_mode_commit( session.commit() real_run_write_txn = scanner.run_write_txn - # Every entry's apply succeeds (the queue drains); only the FINAL - # write_stored_mode("on") commit, called once the queue is empty, fails. monkeypatch.setattr( hash_mode_state, "run_write_txn", From bd39104d3bb518fe34348b5a732a425f89676eb5 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 10:57:09 -0700 Subject: [PATCH 023/111] fix(database): preserve pending lock error when rollback fails --- app/database/db.py | 6 +++- tests-unit/app_test/test_db_write_txn.py | 40 ++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/app/database/db.py b/app/database/db.py index cdcfd960362..a9dce04883e 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -365,6 +365,7 @@ def run_write_txn(work: Callable[["SQLAlchemySession"], T]) -> T: locked_error = None try: for attempt in range(len(_WRITE_TXN_BACKOFF_SECONDS) + 1): + retryable_lock_error = False if attempt > 0: if time.monotonic() >= retry_deadline: raise locked_error @@ -383,12 +384,15 @@ def run_write_txn(work: Callable[["SQLAlchemySession"], T]) -> T: if not _is_retryable_lock_error(exc.orig): raise locked_error = exc + retryable_lock_error = True finally: propagating_exception = sys.exc_info()[0] is not None try: session.rollback() except BaseException: - if not propagating_exception: + if retryable_lock_error: + logging.warning("Write transaction rollback failed after locked error; retrying", exc_info=True) + elif not propagating_exception: raise finally: session.close() diff --git a/tests-unit/app_test/test_db_write_txn.py b/tests-unit/app_test/test_db_write_txn.py index 67564120dbb..8ce314bfccb 100644 --- a/tests-unit/app_test/test_db_write_txn.py +++ b/tests-unit/app_test/test_db_write_txn.py @@ -1,3 +1,4 @@ +import logging import os import shutil import sqlite3 @@ -223,6 +224,45 @@ def work(_session): assert session.closed +def test_run_write_txn_retries_when_rollback_fails_after_handled_lock(monkeypatch, caplog): + class Session: + def __init__(self, rollback_fails): + self.closed = False + self.rollback_fails = rollback_fails + + def commit(self): + return None + + def rollback(self): + if self.rollback_fails: + raise RuntimeError("rollback failure") + + def close(self): + self.closed = True + + first_session = Session(rollback_fails=True) + second_session = Session(rollback_fails=False) + sessions = iter((first_session, second_session)) + monkeypatch.setattr(db_mod, "WriteSession", lambda: next(sessions)) + monkeypatch.setattr(db_mod.time, "sleep", lambda _seconds: None) + attempts = 0 + + def work(_session): + nonlocal attempts + attempts += 1 + if attempts == 1: + raise OperationalError("INSERT", {}, sqlite3.OperationalError("database is locked")) + return "written" + + with caplog.at_level(logging.WARNING): + assert db_mod.run_write_txn(work) == "written" + + assert first_session.closed + assert second_session.closed + assert attempts == 2 + assert "rollback failed" in caplog.text + + def test_run_write_txn_reraises_nonretryable_operational_error_without_retry(memory_database): run_write_txn = db_mod.run_write_txn attempts = 0 From 3e2c7489f0b9f2eba04b211cabf2a64cdc96ad37 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 10:57:09 -0700 Subject: [PATCH 024/111] fix(assets): use a unique sibling temp for cross-device publishes --- app/assets/services/ingest.py | 8 +- .../services/test_cross_volume_upload.py | 76 ++++++++++++++++++- 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index 0f119e790d1..b335f6e9995 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -13,6 +13,7 @@ import mimetypes import os import shutil +import tempfile from typing import Any, NamedTuple from sqlalchemy import func, select @@ -192,8 +193,13 @@ def _move_temp_to_dest(temp_path: str, dest_abs: str) -> None: except OSError as error: if error.errno == errno.EXDEV: destination_dir = os.path.dirname(dest_abs) - destination_temp = os.path.join(destination_dir, f".{os.path.basename(dest_abs)}.tmp") + destination_fd, destination_temp = tempfile.mkstemp( + dir=destination_dir, + prefix=f".{os.path.basename(dest_abs)}.", + suffix=".tmp", + ) try: + os.close(destination_fd) shutil.copy2(temp_path, destination_temp) os.replace(destination_temp, dest_abs) except BaseException: diff --git a/tests-unit/assets_test/services/test_cross_volume_upload.py b/tests-unit/assets_test/services/test_cross_volume_upload.py index b030ec93a3f..ec2d15af33e 100644 --- a/tests-unit/assets_test/services/test_cross_volume_upload.py +++ b/tests-unit/assets_test/services/test_cross_volume_upload.py @@ -1,4 +1,5 @@ import errno +import threading from pathlib import Path import pytest @@ -26,6 +27,79 @@ def fail_cross_volume_move(source_path: str, destination_path: str) -> None: assert not source.exists() +def test_move_temp_to_dest_uses_isolated_cross_volume_copy_temps(tmp_path: Path, monkeypatch) -> None: + first_source = tmp_path / "first-upload.part" + second_source = tmp_path / "second-upload.part" + destination = tmp_path / "output" / "upload.bin" + first_source.write_bytes(b"AAAAAAAA") + second_source.write_bytes(b"BBBBBBBB") + complete_contents = {first_source.read_bytes(), second_source.read_bytes()} + + real_replace = ingest.os.replace + copy_order_lock = threading.Lock() + first_copy_started = threading.Event() + second_copy_started = threading.Event() + second_copy_published = threading.Event() + first_copy = True + errors = [] + + def replace_across_filesystems(source_path: str, destination_path: str) -> None: + if destination_path == str(destination) and source_path in { + str(first_source), + str(second_source), + }: + raise OSError(errno.EXDEV, "Invalid cross-device link") + real_replace(source_path, destination_path) + if destination_path == str(destination) and second_copy_started.is_set(): + second_copy_published.set() + + def copy_with_overlap(source_path: str, destination_path: str) -> None: + nonlocal first_copy + with copy_order_lock: + is_first_copy = first_copy + first_copy = False + + source_contents = Path(source_path).read_bytes() + if is_first_copy: + with open(destination_path, "wb") as destination_file: + midpoint = len(source_contents) // 2 + destination_file.write(source_contents[:midpoint]) + destination_file.flush() + first_copy_started.set() + assert second_copy_started.wait(timeout=5) + assert second_copy_published.wait(timeout=5) + destination_file.write(source_contents[midpoint:]) + destination_file.flush() + return + + assert first_copy_started.wait(timeout=5) + Path(destination_path).write_bytes(source_contents) + second_copy_started.set() + + def move(source: Path) -> None: + try: + ingest._move_temp_to_dest(str(source), str(destination)) + except Exception as error: + errors.append(error) + + monkeypatch.setattr(ingest.os, "replace", replace_across_filesystems) + monkeypatch.setattr(ingest.shutil, "copy2", copy_with_overlap) + + first_worker = threading.Thread(target=move, args=(first_source,)) + second_worker = threading.Thread(target=move, args=(second_source,)) + first_worker.start() + assert first_copy_started.wait(timeout=5) + second_worker.start() + first_worker.join(timeout=5) + second_worker.join(timeout=5) + + assert not first_worker.is_alive() + assert not second_worker.is_alive() + assert destination.read_bytes() in complete_contents + assert not errors + assert not list(destination.parent.glob(f".{destination.name}*.tmp")) + + def test_move_temp_to_dest_keeps_final_destination_intact_when_cross_volume_copy_fails( tmp_path: Path, monkeypatch ) -> None: @@ -54,4 +128,4 @@ def fail_copy(source_path: str, destination_path: str) -> None: assert destination.read_bytes() == b"existing" assert source.read_bytes() == b"upload" - assert not (destination.parent / ".upload.bin.tmp").exists() + assert not list(destination.parent.glob(f".{destination.name}*.tmp")) From 5e8186269947b2f05cb26084f7958a58b622b4ba Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 12:26:48 -0700 Subject: [PATCH 025/111] fix(database): classify sqlite lock errors by result code --- app/database/db.py | 24 ++++++++ tests-unit/app_test/test_db_write_txn.py | 71 ++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/app/database/db.py b/app/database/db.py index a9dce04883e..fcdf5fb4d73 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -21,6 +21,22 @@ _WRITE_TXN_BACKOFF_SECONDS = (0.05, 0.1, 0.2, 0.4) _SQLITE_BUSY_TIMEOUT_MS = 30000 _SQLITE_WRITE_LOCK_POLL_SECONDS = 0.01 +_SQLITE_RETRYABLE_LOCK_ERROR_NAMES = frozenset({ + "SQLITE_BUSY", + "SQLITE_BUSY_SNAPSHOT", + "SQLITE_BUSY_TIMEOUT", + "SQLITE_BUSY_RECOVERY", + "SQLITE_LOCKED", + "SQLITE_LOCKED_SHAREDCACHE", +}) +_SQLITE_RETRYABLE_LOCK_ERROR_CODES = frozenset({ + getattr(sqlite3, "SQLITE_BUSY", 5), + getattr(sqlite3, "SQLITE_BUSY_RECOVERY", 261), + getattr(sqlite3, "SQLITE_BUSY_SNAPSHOT", 517), + getattr(sqlite3, "SQLITE_BUSY_TIMEOUT", 773), + getattr(sqlite3, "SQLITE_LOCKED", 6), + getattr(sqlite3, "SQLITE_LOCKED_SHAREDCACHE", 262), +}) T = TypeVar("T") @@ -351,6 +367,14 @@ def create_session(): def _is_retryable_lock_error(exc: BaseException) -> bool: + error_name = getattr(exc, "sqlite_errorname", None) + if error_name is not None: + return error_name in _SQLITE_RETRYABLE_LOCK_ERROR_NAMES + + error_code = getattr(exc, "sqlite_errorcode", None) + if error_code is not None: + return error_code in _SQLITE_RETRYABLE_LOCK_ERROR_CODES + return "locked" in str(exc).lower() diff --git a/tests-unit/app_test/test_db_write_txn.py b/tests-unit/app_test/test_db_write_txn.py index 8ce314bfccb..b390dd2df7f 100644 --- a/tests-unit/app_test/test_db_write_txn.py +++ b/tests-unit/app_test/test_db_write_txn.py @@ -202,6 +202,77 @@ def work(_session): assert attempts == 1 +@pytest.mark.parametrize( + ("sqlite_errorname", "sqlite_errorcode", "message", "is_retryable"), + [ + pytest.param("SQLITE_BUSY", None, "contention", True, id="busy-name"), + pytest.param("SQLITE_BUSY_SNAPSHOT", None, "snapshot contention", True, id="busy-snapshot-name"), + pytest.param("SQLITE_BUSY_TIMEOUT", None, "timed contention", True, id="busy-timeout-name"), + pytest.param("SQLITE_BUSY_RECOVERY", None, "recovery contention", True, id="busy-recovery-name"), + pytest.param("SQLITE_LOCKED", None, "table contention", True, id="locked-name"), + pytest.param("SQLITE_LOCKED_SHAREDCACHE", None, "shared-cache contention", True, id="locked-sharedcache-name"), + pytest.param(None, sqlite3.SQLITE_BUSY_SNAPSHOT, "snapshot contention", True, id="busy-snapshot-code"), + pytest.param(None, sqlite3.SQLITE_LOCKED_VTAB, "virtual table locked", False, id="locked-vtab-code"), + pytest.param(None, None, "database table is locked", True, id="python-310-locked-fallback"), + pytest.param(None, None, "database is busy", False, id="python-310-busy-fallback"), + ], +) +def test_begin_immediate_and_write_retries_agree_on_sqlite_result_classification( + memory_database, + monkeypatch, + sqlite_errorname, + sqlite_errorcode, + message, + is_retryable, +): + def make_error(): + error = sqlite3.OperationalError(message) + if sqlite_errorname is not None: + error.sqlite_errorname = sqlite_errorname + if sqlite_errorcode is not None: + error.sqlite_errorcode = sqlite_errorcode + return error + + clock = {"now": 0.0} + + class Connection: + def execute(self, _statement): + raise make_error() + + monkeypatch.setattr(db_mod.time, "monotonic", lambda: clock["now"]) + monkeypatch.setattr( + db_mod.time, + "sleep", + lambda _seconds: clock.__setitem__("now", 31.0), + ) + + if is_retryable: + with pytest.raises(OperationalError): + db_mod._begin_immediate(Connection()) + else: + with pytest.raises(sqlite3.OperationalError): + db_mod._begin_immediate(Connection()) + + monkeypatch.setattr(db_mod.time, "monotonic", lambda: 0.0) + monkeypatch.setattr(db_mod.time, "sleep", lambda _seconds: None) + attempts = 0 + + def work(_session): + nonlocal attempts + attempts += 1 + raise OperationalError("INSERT", {}, make_error()) + + with pytest.raises(OperationalError): + db_mod.run_write_txn(work) + + assert attempts == (5 if is_retryable else 1) + + +def test_retryable_lock_error_uses_the_python_310_message_fallback(): + assert db_mod._is_retryable_lock_error(sqlite3.OperationalError("database table is locked")) + assert not db_mod._is_retryable_lock_error(sqlite3.OperationalError("database is busy")) + + def test_run_write_txn_closes_when_rollback_fails_without_masking_work_error(monkeypatch): class Session: closed = False From 6fb355e42adea8d2c3ce878f04f31648b5ef0488 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 12:26:55 -0700 Subject: [PATCH 026/111] feat(assets): add job id to asset events --- app/assets/event_log.py | 11 ++++++++++- .../assets_test/fixtures/assets_event_lines.txt | 1 + tests-unit/assets_test/test_event_log.py | 17 +++++++++++++++-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/app/assets/event_log.py b/app/assets/event_log.py index 9810123dae6..d36fd16a521 100644 --- a/app/assets/event_log.py +++ b/app/assets/event_log.py @@ -91,6 +91,7 @@ def _is_flag(value: Any) -> bool: "hashing_enabled": _is_flag, "site": _one_of(STAT_SITES), "output_kind": _one_of(frozenset({"executed", "cached"})), + "job_id": _is_safe_string, } _warned_call_sites: set[tuple[str, int]] = set() @@ -121,7 +122,13 @@ def _caller_call_site() -> tuple[str, int]: return (caller.filename, caller.lineno or 0) -def emit(event: str, *, root: str | None = None, **fields: Any) -> None: +def emit( + event: str, + *, + root: str | None = None, + job_id: str | None = None, + **fields: Any, +) -> None: """Log one tagged event line. An invalid call raises in strict mode (under pytest, or with @@ -131,6 +138,8 @@ def emit(event: str, *, root: str | None = None, **fields: Any) -> None: """ if root is not None: fields["root"] = root + if job_id is not None: + fields["job_id"] = job_id problem = _find_problem(event, fields) if problem is None: diff --git a/tests-unit/assets_test/fixtures/assets_event_lines.txt b/tests-unit/assets_test/fixtures/assets_event_lines.txt index eadf0cffb76..485c198d45d 100644 --- a/tests-unit/assets_test/fixtures/assets_event_lines.txt +++ b/tests-unit/assets_test/fixtures/assets_event_lines.txt @@ -2,3 +2,4 @@ [assets-event] seeder.scan_started phase=enrich [assets-event] scanner.stat_failed error_type=PermissionError site=discovery [assets-event] ingest.register_failed error_type=RuntimeError output_kind=executed +[assets-event] ingest.register_failed error_type=RuntimeError job_id=job-123 output_kind=executed diff --git a/tests-unit/assets_test/test_event_log.py b/tests-unit/assets_test/test_event_log.py index 64bda8fddc5..943c7a1b7e2 100644 --- a/tests-unit/assets_test/test_event_log.py +++ b/tests-unit/assets_test/test_event_log.py @@ -39,6 +39,7 @@ "hashing_enabled": [True, False], "site": ["discovery", "enrich", "reference_stat"], "output_kind": ["executed", "cached"], + "job_id": ["job-123"], } @@ -92,11 +93,11 @@ def go_to_production_mode(monkeypatch: pytest.MonkeyPatch) -> None: # --- the shared cross-repo fixture ------------------------------------------------- -def test_shared_fixture_file_holds_four_newline_terminated_lines(): +def test_shared_fixture_file_holds_five_newline_terminated_lines(): raw = FIXTURE_PATH.read_text(encoding="utf-8") assert raw.endswith("\n") - assert len(raw.splitlines()) == 4 + assert len(raw.splitlines()) == 5 @pytest.mark.parametrize("line", fixture_lines()) @@ -125,6 +126,18 @@ def test_a_fieldless_event_still_matches_the_shared_pattern(caplog): assert EVENT_LINE_PATTERN.match(line) is not None +def test_none_job_id_is_omitted_from_the_event_line(caplog): + line = emit_line( + caplog, + "ingest.register_failed", + error_type="OSError", + output_kind="executed", + job_id=None, + ) + + assert line == "[assets-event] ingest.register_failed error_type=OSError output_kind=executed" + + def test_the_emitted_record_is_a_single_line(caplog): line = emit_line(caplog, "seeder.scan_failed", error_type="ValueError") From 10998e69707d808f254e9ddf1fd27e491c87d092 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 12:27:01 -0700 Subject: [PATCH 027/111] feat(assets): include job id in registration failure events --- app/assets/services/ingest.py | 2 ++ .../assets_test/services/test_ingest_write_txn.py | 14 +++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index b335f6e9995..64a1aea6031 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -1124,6 +1124,7 @@ def register_cached_output( "ingest.register_failed", output_kind="cached", error_type=error_type(exc), + job_id=job_id, ) return None @@ -1186,5 +1187,6 @@ def _work(session: Session) -> RegisteredAsset: "ingest.register_failed", output_kind="executed", error_type=error_type(exc), + job_id=job_id, ) return None diff --git a/tests-unit/assets_test/services/test_ingest_write_txn.py b/tests-unit/assets_test/services/test_ingest_write_txn.py index 290861ed303..3324af8d4e4 100644 --- a/tests-unit/assets_test/services/test_ingest_write_txn.py +++ b/tests-unit/assets_test/services/test_ingest_write_txn.py @@ -174,9 +174,9 @@ def exhausted_retries(_work): monkeypatch.setattr(ingest, "run_write_txn", exhausted_retries) try: with caplog.at_level(logging.INFO): - assert ingest.register_executed_output(path) is None + assert ingest.register_executed_output(path, job_id="job-locked") is None assert _registration_failure_event(caplog) == ( - "[assets-event] ingest.register_failed error_type=OperationalError output_kind=executed" + "[assets-event] ingest.register_failed error_type=OperationalError job_id=job-locked output_kind=executed" ) finally: os.unlink(path) @@ -193,9 +193,9 @@ def non_retryable_failure(_work): monkeypatch.setattr(ingest, "run_write_txn", non_retryable_failure) try: with caplog.at_level(logging.INFO): - assert ingest.register_executed_output(path) is None + assert ingest.register_executed_output(path, job_id="job-integrity") is None assert _registration_failure_event(caplog) == ( - "[assets-event] ingest.register_failed error_type=IntegrityError output_kind=executed" + "[assets-event] ingest.register_failed error_type=IntegrityError job_id=job-integrity output_kind=executed" ) finally: os.unlink(path) @@ -205,9 +205,9 @@ def test_executed_registration_reports_preflight_os_error(monkeypatch, caplog) - monkeypatch.setattr(ingest.os, "stat", lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("gone"))) with caplog.at_level(logging.INFO): - assert ingest.register_executed_output("/missing/output.bin") is None + assert ingest.register_executed_output("/missing/output.bin", job_id="job-preflight") is None assert _registration_failure_event(caplog) == ( - "[assets-event] ingest.register_failed error_type=OSError output_kind=executed" + "[assets-event] ingest.register_failed error_type=OSError job_id=job-preflight output_kind=executed" ) @@ -258,7 +258,7 @@ def non_retryable_failure(_work): monkeypatch.setattr(ingest, "run_write_txn", non_retryable_failure) try: with caplog.at_level(logging.INFO): - assert ingest.register_cached_output(path) is None + assert ingest.register_cached_output(path, job_id=None) is None assert _registration_failure_event(caplog) == ( "[assets-event] ingest.register_failed error_type=IntegrityError output_kind=cached" ) From 40f59b1200811be8aa14253ebe250b0c274ca899 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 12:27:08 -0700 Subject: [PATCH 028/111] test(assets): assert job id survives contended registration --- .../assets_test/test_write_contention_e2e.py | 140 ++++++++++++++---- 1 file changed, 108 insertions(+), 32 deletions(-) diff --git a/tests-unit/assets_test/test_write_contention_e2e.py b/tests-unit/assets_test/test_write_contention_e2e.py index 380d130ee91..1f313634227 100644 --- a/tests-unit/assets_test/test_write_contention_e2e.py +++ b/tests-unit/assets_test/test_write_contention_e2e.py @@ -1,14 +1,14 @@ import logging -import sqlite3 import threading -import time import folder_paths import pytest from PIL import Image import app.database.db as db_mod -from app.assets import lifecycle, mode +from app.assets import lifecycle, mode, scanner +from app.assets.database.models import Asset +from app.assets.database.queries.records import create_content, create_record from app.assets.manager import default_asset_manager from app.assets.services.hash_mode_state import clear_transition_queue from app.assets.services.ingest import register_executed_output @@ -21,7 +21,7 @@ def autoclean_unit_test_assets(): yield -def test_register_executed_output_waits_for_a_held_sqlite_writer( +def test_register_executed_output_keeps_job_id_during_scanner_write_train( tmp_path, monkeypatch, caplog ) -> None: output_directory = tmp_path / "output" @@ -51,39 +51,115 @@ def test_register_executed_output_waits_for_a_held_sqlite_writer( output_path = output_directory / "ComfyUI_00001_.png" Image.new("RGB", (1, 1), (255, 0, 0)).save(output_path) - holder_ready = threading.Event() - holder_errors: list[sqlite3.Error] = [] - - def hold_write_lock() -> None: - connection = sqlite3.connect(database_path, timeout=1) - try: - connection.execute("BEGIN IMMEDIATE") - connection.execute("INSERT INTO tags (name) VALUES (?)", ("e2e-holder",)) - holder_ready.set() - threading.Event().wait(timeout=2) - connection.commit() - except sqlite3.Error as error: - holder_errors.append(error) - holder_ready.set() - finally: - connection.close() - - holder = threading.Thread(target=hold_write_lock) - holder.start() + scanner_rows: list[scanner.UnenrichedContent] = [] + for index in range(4): + scanner_path = output_directory / f"scanner-{index}.bin" + scanner_path.write_bytes(f"scanner-{index}".encode()) + stat_result = scanner_path.stat() + + def seed(session, path=scanner_path, stat=stat_result) -> None: + content = create_content( + session, + str(path), + size_bytes=stat.st_size, + mtime_ns=stat.st_mtime_ns, + ) + record = create_record(session, content.id, path.name) + scanner_rows.append( + scanner.UnenrichedContent( + content.id, + record.id, + str(path), + needs_hash=True, + ) + ) + + db_mod.run_write_txn(seed) + + first_scanner_write_entered = threading.Event() + release_first_scanner_write = threading.Event() + second_scanner_write_committed = threading.Event() + registration_started = threading.Event() + original_apply = scanner._apply_enrichment + original_run_write_txn = scanner.run_write_txn + original_is_retryable_lock_error = db_mod._is_retryable_lock_error + scanner_writes = 0 + scanner_writes_lock = threading.Lock() + registration_thread_id: list[int | None] = [None] + registration_blocked = threading.Event() + + def block_first_scanner_write(session, prepared): + nonlocal scanner_writes + updated = original_apply(session, prepared) + with scanner_writes_lock: + is_first_write = scanner_writes == 0 + if is_first_write: + first_scanner_write_entered.set() + assert release_first_scanner_write.wait(timeout=5) + return updated + + def count_scanner_writes(work): + nonlocal scanner_writes + result = original_run_write_txn(work) + with scanner_writes_lock: + scanner_writes += 1 + if scanner_writes >= 2: + second_scanner_write_committed.set() + return result + + def observe_registration_lock(error): + is_retryable = original_is_retryable_lock_error(error) + if threading.get_ident() == registration_thread_id[0] and is_retryable: + registration_blocked.set() + return is_retryable + + monkeypatch.setattr(scanner, "_apply_enrichment", block_first_scanner_write) + monkeypatch.setattr(scanner, "run_write_txn", count_scanner_writes) + monkeypatch.setattr(db_mod, "_is_retryable_lock_error", observe_registration_lock) + scanner_result: dict[str, tuple[int, list[str]]] = {} + registration_result: dict[str, RegisteredAsset | None] = {} + + def enrich_scanner_rows() -> None: + scanner_result["value"] = scanner.enrich_assets_batch( + scanner_rows, + extract_metadata=False, + compute_hash=True, + ) + + def register_output() -> None: + registration_thread_id[0] = threading.get_ident() + registration_started.set() + registration_result["value"] = register_executed_output( + str(output_path), + job_id="write-contention", + ) + + scanner_worker = threading.Thread(target=enrich_scanner_rows) + scanner_worker.start() try: - assert holder_ready.wait(timeout=2) - assert not holder_errors + assert first_scanner_write_entered.wait(timeout=5) + registration_worker = threading.Thread(target=register_output) + registration_worker.start() + assert registration_started.wait(timeout=5) + assert registration_blocked.wait(timeout=5) + release_first_scanner_write.set() + assert second_scanner_write_committed.wait(timeout=5) with caplog.at_level(logging.INFO): - started = time.monotonic() - result = register_executed_output(str(output_path), job_id="write-contention") - elapsed = time.monotonic() - started + registration_worker.join(timeout=5) finally: - holder.join(timeout=2) + release_first_scanner_write.set() + scanner_worker.join(timeout=5) - assert not holder.is_alive() - assert not holder_errors + assert not scanner_worker.is_alive() + assert not registration_worker.is_alive() + assert scanner_result["value"] == (len(scanner_rows), []) + assert scanner_writes == len(scanner_rows) + result = registration_result["value"] assert isinstance(result, RegisteredAsset) - assert elapsed >= 1.5 + with db_mod.create_session() as session: + asset = session.get(Asset, result.id) + assert asset is not None + assert asset.job_id == "write-contention" assert not any("Failed to register" in record.getMessage() for record in caplog.records) finally: manager.shutdown() From 6469df8af61c98706e5ddfa820757379d03cccd2 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 13:46:17 -0700 Subject: [PATCH 029/111] feat(assets): log terminal registration failures at warning --- app/assets/event_log.py | 7 ++++++- tests-unit/assets_test/test_event_log.py | 25 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/app/assets/event_log.py b/app/assets/event_log.py index d36fd16a521..2c8d12feabf 100644 --- a/app/assets/event_log.py +++ b/app/assets/event_log.py @@ -46,6 +46,10 @@ "ingest.register_failed", }) +WARNING_LEVEL_EVENTS = frozenset({ + "ingest.register_failed", +}) + class EventLogError(ValueError): """An emit() call that would break the closed event vocabulary.""" @@ -148,7 +152,8 @@ def emit( for name, value in sorted(fields.items()) ) line = f"{TAG} {event}" + (f" {pairs}" if pairs else "") - logging.info("%s", line) + log = logging.warning if event in WARNING_LEVEL_EVENTS else logging.info + log("%s", line) return if _strict_mode(): diff --git a/tests-unit/assets_test/test_event_log.py b/tests-unit/assets_test/test_event_log.py index 943c7a1b7e2..0dee93481fb 100644 --- a/tests-unit/assets_test/test_event_log.py +++ b/tests-unit/assets_test/test_event_log.py @@ -292,3 +292,28 @@ def test_production_mode_still_emits_valid_events_after_a_dropped_one(caplog, mo tagged = [r.getMessage() for r in caplog.records if r.getMessage().startswith(TAG)] assert tagged == ["[assets-event] seeder.scan_started phase=fast"] + + +def test_registration_failure_emits_at_warning_with_unchanged_tap_format(caplog): + caplog.clear() + + with caplog.at_level(logging.INFO): + emit("ingest.register_failed", output_kind="executed", error_type="OperationalError") + + tagged = [r for r in caplog.records if r.getMessage().startswith(TAG)] + assert len(tagged) == 1 + assert tagged[0].levelno == logging.WARNING + assert tagged[0].getMessage() == ( + "[assets-event] ingest.register_failed error_type=OperationalError output_kind=executed" + ) + + +def test_routine_events_stay_at_info(caplog): + caplog.clear() + + with caplog.at_level(logging.INFO): + emit("seeder.scan_started", phase="fast") + + tagged = [r for r in caplog.records if r.getMessage().startswith(TAG)] + assert len(tagged) == 1 + assert tagged[0].levelno == logging.INFO From 2081ead5dc11b3d519f28730f86b89e09c3d9e33 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 21:15:45 -0700 Subject: [PATCH 030/111] fix(assets): keep WAL and the writer engine behind --enable-assets init_db() runs for every install, so forcing journal_mode=WAL there made a default startup promote journal mode, leave -wal/-shm sidecars, and raise on any filesystem that rejects WAL - none of which a user who never enabled assets asked for. Assets is the database's only writer, so gate the WAL pragma and the writer engine on the flag and keep upstream's single-engine wiring otherwise. NoAssets.startup() now cleans the temp directory before recording hash-mode transition intent, so an unusable database costs the transition record rather than also costing temp cleanup. --- app/assets/manager.py | 2 +- app/database/db.py | 55 ++++-- tests-unit/app_test/test_db_promotion.py | 1 + tests-unit/app_test/test_db_write_txn.py | 83 ++++++++- .../services/test_ingest_write_txn.py | 41 +++++ .../services/test_noassets_parity.py | 19 ++ .../services/test_write_lock_hold_barriers.py | 171 +++++++++++++++--- 7 files changed, 323 insertions(+), 49 deletions(-) diff --git a/app/assets/manager.py b/app/assets/manager.py index 590015d6ad9..a6e8eeab0f9 100644 --- a/app/assets/manager.py +++ b/app/assets/manager.py @@ -80,8 +80,8 @@ def enabled(self) -> bool: def startup(self) -> None: mode.init(self._args) - record_hash_mode_transition_intent() run_startup(enable_assets=False) + record_hash_mode_transition_intent() def shutdown(self) -> None: _shutdown_assets() diff --git a/app/database/db.py b/app/database/db.py index fcdf5fb4d73..a3c31cc2fe4 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -240,6 +240,26 @@ def _upgrade_discards_the_catalog(script, target_rev, current_rev): ) +def _assets_writer_enabled(): + return bool(getattr(args, "enable_assets", False)) + + +def _bind_single_engine_without_wal(db_url): + global Session, WriteSession + engine = create_engine(db_url) + + @event.listens_for(engine, "connect") + def set_sqlite_pragma(dbapi_connection, connection_record): + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() + + with engine.connect(): + pass + Session = sessionmaker(bind=engine) + WriteSession = Session + + def _configure_runtime_connection(dbapi_connection, db_path): cursor = dbapi_connection.cursor() try: @@ -275,6 +295,22 @@ def _begin_immediate(dbapi_connection): time.sleep(min(_SQLITE_WRITE_LOCK_POLL_SECONDS, remaining_seconds)) +def build_writer_engine(db_url, db_path): + engine = create_engine(db_url, connect_args={"timeout": 0}) + + @event.listens_for(engine, "connect") + def set_writer_sqlite_pragma(dbapi_connection, connection_record): + dbapi_connection.isolation_level = None + _configure_runtime_connection(dbapi_connection, db_path) + dbapi_connection.execute("PRAGMA busy_timeout=0").close() + + @event.listens_for(engine, "begin") + def begin_immediate(connection): + _begin_immediate(connection.connection.driver_connection) + + return engine + + def _migrate_and_bind(db_url, db_path, db_exists): config = get_alembic_config() inspection_engine = create_engine(db_url) @@ -334,6 +370,12 @@ def set_sqlite_pragma(dbapi_connection, connection_record): f"discarded. The database from before the upgrade was kept at {backup_path}." ) + global Session, WriteSession + + if not _assets_writer_enabled(): + _bind_single_engine_without_wal(db_url) + return + # Redundant with busy_timeout by design: both set pysqlite's 30-second limit. reader_engine = create_engine(db_url, connect_args={"timeout": 30}) @@ -341,23 +383,12 @@ def set_sqlite_pragma(dbapi_connection, connection_record): def set_reader_sqlite_pragma(dbapi_connection, connection_record): _configure_runtime_connection(dbapi_connection, db_path) - writer_engine = create_engine(db_url, connect_args={"timeout": 0}) - - @event.listens_for(writer_engine, "connect") - def set_writer_sqlite_pragma(dbapi_connection, connection_record): - dbapi_connection.isolation_level = None - _configure_runtime_connection(dbapi_connection, db_path) - dbapi_connection.execute("PRAGMA busy_timeout=0").close() - - @event.listens_for(writer_engine, "begin") - def begin_immediate(connection): - _begin_immediate(connection.connection.driver_connection) + writer_engine = build_writer_engine(db_url, db_path) with reader_engine.connect(): pass with writer_engine.connect(): pass - global Session, WriteSession Session = sessionmaker(bind=reader_engine) WriteSession = sessionmaker(bind=writer_engine) diff --git a/tests-unit/app_test/test_db_promotion.py b/tests-unit/app_test/test_db_promotion.py index 38130ba5af0..01748d4ad0c 100644 --- a/tests-unit/app_test/test_db_promotion.py +++ b/tests-unit/app_test/test_db_promotion.py @@ -20,6 +20,7 @@ def _legacy(legacy_work): def test_write_transaction_waits_for_held_writer_before_select_then_mutate(tmp_path, monkeypatch): database_path = tmp_path / "assets.db" + monkeypatch.setattr(db_mod.args, "enable_assets", True) monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") monkeypatch.setattr(db_mod, "Session", None) monkeypatch.setattr(db_mod, "WriteSession", None) diff --git a/tests-unit/app_test/test_db_write_txn.py b/tests-unit/app_test/test_db_write_txn.py index b390dd2df7f..58e7164cf3c 100644 --- a/tests-unit/app_test/test_db_write_txn.py +++ b/tests-unit/app_test/test_db_write_txn.py @@ -3,7 +3,6 @@ import shutil import sqlite3 import threading -import time import pytest from alembic import command @@ -27,6 +26,7 @@ def _dispose_runtime_engines(): @pytest.fixture def file_database(tmp_path, monkeypatch): database_path = str(tmp_path / "assets.db") + monkeypatch.setattr(db_mod.args, "enable_assets", True) monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") monkeypatch.setattr(db_mod, "Session", None) monkeypatch.setattr(db_mod, "_db_lock", None) @@ -40,6 +40,7 @@ def file_database(tmp_path, monkeypatch): @pytest.fixture def memory_database(monkeypatch): + monkeypatch.setattr(db_mod.args, "enable_assets", True) monkeypatch.setattr(db_mod.args, "database_url", "sqlite:///:memory:") monkeypatch.setattr(db_mod, "Session", None) if hasattr(db_mod, "WriteSession"): @@ -82,6 +83,7 @@ def _crash_style_wal_database(tmp_path) -> str: def _migrate_crash_style_database(database_path: str, monkeypatch) -> None: + monkeypatch.setattr(db_mod.args, "enable_assets", True) monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") monkeypatch.setattr(db_mod, "Session", None) if hasattr(db_mod, "WriteSession"): @@ -408,16 +410,17 @@ def work(_session): def test_run_write_txn_held_lock_respects_remaining_deadline(file_database, monkeypatch): run_write_txn = db_mod.run_write_txn - monkeypatch.setattr(db_mod, "_WRITE_TXN_LOCK_RETRY_DEADLINE_SECONDS", 2) - writer_started = threading.Event() + monkeypatch.setattr(db_mod, "_WRITE_TXN_LOCK_RETRY_DEADLINE_SECONDS", 0.5) + lock_held = threading.Event() + release = threading.Event() def hold_lock(): holder = sqlite3.connect(file_database) try: holder.execute("BEGIN IMMEDIATE") holder.execute("INSERT INTO tags (name) VALUES (?)", ("deadline-holder",)) - writer_started.set() - time.sleep(5) + lock_held.set() + release.wait(timeout=30) holder.rollback() finally: holder.close() @@ -425,18 +428,20 @@ def hold_lock(): holder = threading.Thread(target=hold_lock) holder.start() try: - assert writer_started.wait(timeout=5) - started_at = time.monotonic() + assert lock_held.wait(timeout=5) with pytest.raises(OperationalError, match="database is locked"): run_write_txn( lambda session: session.execute(text("INSERT INTO tags (name) VALUES ('blocked')")) ) - elapsed = time.monotonic() - started_at + assert not release.is_set(), ( + "refusal must come from the retry deadline, not from the holder releasing" + ) + assert holder.is_alive(), "holder must still own the write lock at refusal time" finally: + release.set() holder.join(timeout=6) assert not holder.is_alive() - assert 1.5 <= elapsed < 3 def test_run_write_txn_reopens_immediate_transaction_after_intermediate_commit( @@ -518,6 +523,7 @@ def fail_upgrade(_config, _target_revision): raise RuntimeError("upgrade failure") monkeypatch.setattr(db_mod.command, "upgrade", fail_upgrade) + monkeypatch.setattr(db_mod.args, "enable_assets", True) monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") monkeypatch.setattr(db_mod, "Session", None) if hasattr(db_mod, "WriteSession"): @@ -567,3 +573,62 @@ def trace_connection(dbapi_connection, _connection_record): assert "journal_mode=delete" in inspection_commands assert "journal_mode=wal" not in inspection_commands assert "journal_mode=wal" in runtime_commands + + +@pytest.fixture +def file_database_without_assets(tmp_path, monkeypatch): + database_path = str(tmp_path / "assets.db") + monkeypatch.setattr(db_mod.args, "enable_assets", False) + monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") + monkeypatch.setattr(db_mod, "Session", None) + monkeypatch.setattr(db_mod, "_db_lock", None) + monkeypatch.setattr(db_mod, "WriteSession", None) + db_mod.init_db() + yield database_path + _dispose_runtime_engines() + db_mod._db_lock.release(force=True) + + +def test_disabled_assets_startup_leaves_journal_mode_unpromoted(file_database_without_assets): + with db_mod.create_session() as session: + session.execute(text("SELECT 1")) + + connection = sqlite3.connect(file_database_without_assets) + try: + journal_mode = connection.execute("PRAGMA journal_mode").fetchone()[0] + finally: + connection.close() + + assert journal_mode.lower() != "wal" + + +def test_disabled_assets_startup_writes_no_wal_sidecars(file_database_without_assets): + with db_mod.create_session() as session: + session.execute(text("SELECT 1")) + + directory = os.path.dirname(file_database_without_assets) + sidecars = [name for name in os.listdir(directory) if name.endswith(("-wal", "-shm"))] + + assert sidecars == [] + + +def test_disabled_assets_startup_survives_a_filesystem_that_rejects_wal(tmp_path, monkeypatch): + database_path = str(tmp_path / "assets.db") + monkeypatch.setattr(db_mod.args, "enable_assets", False) + monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") + monkeypatch.setattr(db_mod, "Session", None) + monkeypatch.setattr(db_mod, "_db_lock", None) + monkeypatch.setattr(db_mod, "WriteSession", None) + + def reject_wal(dbapi_connection, db_path): + raise RuntimeError(f"SQLite WAL could not be enabled for database '{db_path}'.") + + monkeypatch.setattr(db_mod, "_configure_runtime_connection", reject_wal) + try: + db_mod.init_db() + with db_mod.create_session() as session: + session.execute(text("SELECT 1")) + finally: + _dispose_runtime_engines() + if db_mod._db_lock is not None: + db_mod._db_lock.release(force=True) diff --git a/tests-unit/assets_test/services/test_ingest_write_txn.py b/tests-unit/assets_test/services/test_ingest_write_txn.py index 3324af8d4e4..b57e2242103 100644 --- a/tests-unit/assets_test/services/test_ingest_write_txn.py +++ b/tests-unit/assets_test/services/test_ingest_write_txn.py @@ -33,6 +33,7 @@ def file_database(tmp_path, monkeypatch): true; only the production runtime engines built by ``init_db`` enforce it. """ database_path = str(tmp_path / "assets.db") + monkeypatch.setattr(db_mod.args, "enable_assets", True) monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") monkeypatch.setattr(db_mod, "Session", None) monkeypatch.setattr(db_mod, "_db_lock", None) @@ -567,3 +568,43 @@ def _register() -> None: assert upload_result.ref.file_path == path finally: os.unlink(path) + + +def test_unsettled_new_upload_persists_nothing_rather_than_mixing_facts( + file_database, monkeypatch, caplog +) -> None: + from sqlalchemy import select + + from app.assets.database.models import Asset, AssetContent + + path = _output_path("unsettled-upload.bin") + with open(path, "wb") as handle: + handle.write(b"first bytes") + + stored_hash = "blake3:" + "c" * 64 + stat_result = os.stat(path) + facts = ingest._ContentFacts(stored_hash, stat_result.st_size, stat_result.st_mtime_ns) + spec = ingest._UploadRecordSpec("unsettled-upload.bin", [], None, {}, None) + + real_assert = ingest._assert_upload_preflight_current + + def always_stale(_session, _preflight): + raise ingest._PreflightStale + + monkeypatch.setattr(ingest, "_assert_upload_preflight_current", always_stale) + + with caplog.at_level(logging.WARNING): + with pytest.raises(RuntimeError, match="did not settle"): + ingest._create_content_and_upload_record( + stored_hash, path, facts, True, spec + ) + + monkeypatch.setattr(ingest, "_assert_upload_preflight_current", real_assert) + + with db_mod.Session() as session: + contents = list(session.scalars(select(AssetContent).where(AssetContent.path == path))) + records = list(session.scalars(select(Asset))) + + assert contents == [], "refused upload must not leave a content row behind" + assert records == [], "refused upload must not leave an asset record behind" + os.unlink(path) diff --git a/tests-unit/assets_test/services/test_noassets_parity.py b/tests-unit/assets_test/services/test_noassets_parity.py index 5c7b646b69d..15d0186c15b 100644 --- a/tests-unit/assets_test/services/test_noassets_parity.py +++ b/tests-unit/assets_test/services/test_noassets_parity.py @@ -200,3 +200,22 @@ def test_default_asset_manager_enables_assets_when_dependencies_are_available( monkeypatch.setattr(manager, "dependencies_available", lambda: True) assert isinstance(manager.default_asset_manager(), AssetsEnabled) + + +def test_noassets_startup_cleans_temp_even_when_the_database_is_unusable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + cleaned: list[bool] = [] + + def unusable_database() -> None: + raise RuntimeError("no such table: hash_mode_state") + + monkeypatch.setattr(manager, "record_hash_mode_transition_intent", unusable_database) + monkeypatch.setattr( + manager, "run_startup", lambda *, enable_assets: cleaned.append(enable_assets) + ) + + with pytest.raises(RuntimeError): + _no_assets().startup() + + assert cleaned == [False] diff --git a/tests-unit/assets_test/services/test_write_lock_hold_barriers.py b/tests-unit/assets_test/services/test_write_lock_hold_barriers.py index 14e41d0a8c1..5c7442e3a30 100644 --- a/tests-unit/assets_test/services/test_write_lock_hold_barriers.py +++ b/tests-unit/assets_test/services/test_write_lock_hold_barriers.py @@ -1,9 +1,9 @@ import threading -import time import uuid import pytest from sqlalchemy import text +from sqlalchemy.exc import OperationalError import app.assets.mode as mode_module import app.database.db as db_mod @@ -13,12 +13,15 @@ from app.assets.services import hash_mode_state _BARRIER_TIMEOUT = 5 -_PROBE_BUDGET_SECONDS = 1.0 +_PROBE_LOCK_DEADLINE_SECONDS = 0.5 +_PROBE_BUSY_TIMEOUT_MS = 250 +_LEASE_HELD = "writer lease was held across out-of-transaction work" @pytest.fixture def file_database(tmp_path, monkeypatch): database_path = str(tmp_path / "assets.db") + monkeypatch.setattr(db_mod.args, "enable_assets", True) monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") monkeypatch.setattr(db_mod, "Session", None) monkeypatch.setattr(db_mod, "_db_lock", None) @@ -51,6 +54,27 @@ def _probe_write() -> None: ) +@pytest.fixture +def impatient_probe(monkeypatch): + """Let a probe surface a held lease as an error instead of waiting out the real deadline. + + An unheld lease is acquired on the first attempt, so these shortened deadlines are + only ever reached when the lease really is held. + """ + monkeypatch.setattr( + db_mod, "_WRITE_TXN_LOCK_RETRY_DEADLINE_SECONDS", _PROBE_LOCK_DEADLINE_SECONDS + ) + monkeypatch.setattr(db_mod, "_SQLITE_BUSY_TIMEOUT_MS", _PROBE_BUSY_TIMEOUT_MS) + + +def _probe_write_outcome() -> Exception | None: + try: + _probe_write() + except OperationalError as exc: + return exc + return None + + def _blocking_fake(entered: threading.Event, release: threading.Event, real_fn): def fake(*args, **kwargs): entered.set() @@ -61,7 +85,7 @@ def fake(*args, **kwargs): def test_seed_recovery_hashing_does_not_hold_the_write_lock( - file_database, hashing_on, tmp_path, monkeypatch + file_database, impatient_probe, hashing_on, tmp_path, monkeypatch ): path = tmp_path / "recoverable.bin" path.write_bytes(b"recoverable bytes") @@ -96,20 +120,18 @@ def _seed() -> None: worker.start() try: assert entered.wait(timeout=_BARRIER_TIMEOUT) - started = time.monotonic() - _probe_write() - elapsed = time.monotonic() - started + probe_error = _probe_write_outcome() finally: release.set() worker.join(timeout=_BARRIER_TIMEOUT) assert not worker.is_alive() - assert elapsed < _PROBE_BUDGET_SECONDS + assert probe_error is None, _LEASE_HELD assert result["created"] == 1 def test_pending_verification_hashing_does_not_hold_the_write_lock( - file_database, tmp_path, monkeypatch + file_database, impatient_probe, tmp_path, monkeypatch ): path = tmp_path / "verify-me.bin" path.write_bytes(b"verify me") @@ -143,21 +165,19 @@ def _drain() -> None: worker.start() try: assert entered.wait(timeout=_BARRIER_TIMEOUT) - started = time.monotonic() - _probe_write() - elapsed = time.monotonic() - started + probe_error = _probe_write_outcome() finally: release.set() worker.join(timeout=_BARRIER_TIMEOUT) assert not worker.is_alive() scanner_changes.clear_pending_verifications() - assert elapsed < _PROBE_BUDGET_SECONDS + assert probe_error is None, _LEASE_HELD assert result["processed"] == 1 def test_transition_hashing_does_not_hold_the_write_lock( - file_database, tmp_path, monkeypatch + file_database, impatient_probe, tmp_path, monkeypatch ): path = tmp_path / "transition-me.bin" path.write_bytes(b"transition me") @@ -189,21 +209,19 @@ def _drain() -> None: worker.start() try: assert entered.wait(timeout=_BARRIER_TIMEOUT) - started = time.monotonic() - _probe_write() - elapsed = time.monotonic() - started + probe_error = _probe_write_outcome() finally: release.set() worker.join(timeout=_BARRIER_TIMEOUT) assert not worker.is_alive() hash_mode_state.clear_transition_queue() - assert elapsed < _PROBE_BUDGET_SECONDS + assert probe_error is None, _LEASE_HELD assert result.get("done") is True def test_enrichment_hashing_does_not_hold_the_write_lock( - file_database, tmp_path, monkeypatch + file_database, impatient_probe, tmp_path, monkeypatch ): path = tmp_path / "enrich-hash.bin" path.write_bytes(b"enrich me via hash") @@ -238,20 +256,18 @@ def _enrich() -> None: worker.start() try: assert entered.wait(timeout=_BARRIER_TIMEOUT) - started = time.monotonic() - _probe_write() - elapsed = time.monotonic() - started + probe_error = _probe_write_outcome() finally: release.set() worker.join(timeout=_BARRIER_TIMEOUT) assert not worker.is_alive() - assert elapsed < _PROBE_BUDGET_SECONDS + assert probe_error is None, _LEASE_HELD assert result["outcome"] == (1, []) def test_enrichment_metadata_extraction_does_not_hold_the_write_lock( - file_database, tmp_path, monkeypatch + file_database, impatient_probe, tmp_path, monkeypatch ): path = tmp_path / "enrich-metadata.bin" path.write_bytes(b"enrich me via metadata") @@ -286,13 +302,114 @@ def _enrich() -> None: worker.start() try: assert entered.wait(timeout=_BARRIER_TIMEOUT) - started = time.monotonic() - _probe_write() - elapsed = time.monotonic() - started + probe_error = _probe_write_outcome() finally: release.set() worker.join(timeout=_BARRIER_TIMEOUT) assert not worker.is_alive() - assert elapsed < _PROBE_BUDGET_SECONDS + assert probe_error is None, _LEASE_HELD assert result["outcome"] == (1, []) + + +def test_scanner_reference_stat_walk_does_not_hold_the_write_lock( + file_database, impatient_probe, tmp_path, monkeypatch +): + root = tmp_path / "models" + root.mkdir() + path = root / "catalogued.bin" + path.write_bytes(b"catalogued bytes") + stat = path.stat() + + with db_mod.Session() as session: + content = create_content( + session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns + ) + create_record(session, content.id, "catalogued.bin") + session.commit() + + entered = threading.Event() + release = threading.Event() + real_stat = scanner.os.stat + + def blocking_stat(target, *args, **kwargs): + if str(target) == str(path): + entered.set() + assert release.wait(timeout=_BARRIER_TIMEOUT) + return real_stat(target, *args, **kwargs) + + monkeypatch.setattr(scanner, "get_scan_prefixes_for_root", lambda _root: [str(root)]) + monkeypatch.setattr(scanner.os, "stat", blocking_stat) + + survivors: dict[str, set[str]] = {} + + def _scan() -> None: + survivors["found"] = scanner.sync_root_safely("models") + + worker = threading.Thread(target=_scan) + worker.start() + try: + assert entered.wait(timeout=_BARRIER_TIMEOUT) + probe_error = _probe_write_outcome() + finally: + release.set() + worker.join(timeout=_BARRIER_TIMEOUT) + assert not worker.is_alive() + + assert probe_error is None, _LEASE_HELD + assert survivors["found"] == {str(path)} + + +def test_download_hash_resolution_does_not_hold_the_write_lock( + file_database, impatient_probe, tmp_path, monkeypatch +): + from app.assets.services import asset_management, lookup + + path = tmp_path / "servable.bin" + path.write_bytes(b"servable bytes") + stat = path.stat() + digest = "b" * 64 + stored_hash = f"blake3:{digest}" + + with db_mod.Session() as session: + content = create_content( + session, + str(path), + hash=stored_hash, + size_bytes=stat.st_size, + mtime_ns=stat.st_mtime_ns, + ) + create_record(session, content.id, "servable.bin") + session.commit() + + entered = threading.Event() + release = threading.Event() + real_stat = lookup.os.stat + + def blocking_stat(target, *args, **kwargs): + if str(target) == str(path): + entered.set() + assert release.wait(timeout=_BARRIER_TIMEOUT) + return real_stat(target, *args, **kwargs) + + monkeypatch.setattr(lookup.os, "stat", blocking_stat) + monkeypatch.setattr(lookup, "is_temp_path", lambda _path: False) + + resolved: dict[str, object] = {} + + def _resolve() -> None: + resolved["result"] = asset_management.resolve_hash_to_path(stored_hash) + + worker = threading.Thread(target=_resolve) + worker.start() + try: + assert entered.wait(timeout=_BARRIER_TIMEOUT) + probe_error = _probe_write_outcome() + finally: + release.set() + worker.join(timeout=_BARRIER_TIMEOUT) + assert not worker.is_alive() + + assert probe_error is None, _LEASE_HELD + assert resolved["result"] is not None + assert resolved["result"].abs_path == str(path) From 5a99ccffa2bbdc12ccd87d9cca2a2795e4c38483 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 21:15:45 -0700 Subject: [PATCH 031/111] fix(assets): stat catalogued references outside the writer lease sync_prefixes_with_filesystem took the write lock on its first SELECT and then held it across an os.stat per catalogued row, so a scan blocked every other writer for the length of the walk - and did so even when the walk changed nothing. Split it into observe_references_on_filesystem, which stats outside any transaction and returns only rows that need a write, and apply_reference_observations, which performs no filesystem I/O and skips rows another writer has touched since. An unchanged root now applies nothing and never acquires the write lock. resolve_hash_to_path and resolve_asset_for_download did the same thing with os.path.isfile and os.stat per candidate row; both now qualify candidates before the lease and keep only the access-time write inside it. Reference diagnostics carry their path again, so a stat failure names the file. --- app/assets/scanner.py | 196 +++++++++++++----- app/assets/services/asset_management.py | 99 +++++---- .../test_scanner_ingest_event_log.py | 5 +- .../test_write_session_convention.py | 5 + 4 files changed, 211 insertions(+), 94 deletions(-) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index d078b016898..cb96a93f696 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -185,64 +185,149 @@ def sync_references_with_filesystem( ) -def sync_prefixes_with_filesystem( - session: Session, +class _ReferenceObservation(NamedTuple): + + content_id: str + path: str + observed_size_bytes: int | None + observed_mtime_ns: int | None + stat_result: os.stat_result | None + + +class _ReferenceDiagnostic(NamedTuple): + + path: str + error: OSError + + +def _catalogued_references( + session: Session, prefixes: list[str] +) -> list[tuple[str, str, int | None, int | None]]: + return [ + (content.id, content.path, content.size_bytes, content.mtime_ns) + for content in live_contents_under_prefixes(session, prefixes) + ] + + +def observe_references_on_filesystem( prefixes: list[str], - collect_existing_paths: bool = False, progress: _ScanProgress | None = None, - pending_verification_ids: list[str] | None = None, - diagnostics: list[OSError] | None = None, -) -> set[str] | None: - if not prefixes: - return set() if collect_existing_paths else None + diagnostics: list[_ReferenceDiagnostic] | None = None, + session: Session | None = None, +) -> tuple[list[_ReferenceObservation], set[str]]: + """Stat every catalogued reference without holding the writer lease. + Returns only observations that require a write, so an unchanged root applies nothing + and never acquires the write lock. + """ + if session is not None: + catalogued = _catalogued_references(session, prefixes) + else: + with create_session() as read_session: + catalogued = _catalogued_references(read_session, prefixes) + + observations: list[_ReferenceObservation] = [] survivors: set[str] = set() - for content in live_contents_under_prefixes(session, prefixes): + for content_id, path, size_bytes, mtime_ns in catalogued: try: - stat_result = os.stat(content.path, follow_symlinks=True) + stat_result = os.stat(path, follow_symlinks=True) except FileNotFoundError: - mark_content_missing(session, content.id) + observations.append( + _ReferenceObservation(content_id, path, size_bytes, mtime_ns, None) + ) except PermissionError as e: if diagnostics is None: _log_scan_error("reference_stat", e) if progress is not None: progress.permission_denied += 1 - logging.debug("Permission denied accessing %s", content.path) + logging.debug("Permission denied accessing %s", path) else: - diagnostics.append(e) + diagnostics.append(_ReferenceDiagnostic(path, e)) except OSError as e: if diagnostics is None: _log_scan_error("reference_stat", e) - logging.debug("OSError checking %s: %s", content.path, e) + logging.debug("OSError checking %s: %s", path, e) else: - diagnostics.append(e) - mark_content_missing(session, content.id) - else: - detect_content_change( - session, - content, - stat_result, - hashing_is_enabled=mode.hashing_enabled(), - pending_verification_ids=pending_verification_ids, + diagnostics.append(_ReferenceDiagnostic(path, e)) + observations.append( + _ReferenceObservation(content_id, path, size_bytes, mtime_ns, None) ) - survivors.add(os.path.abspath(content.path)) + else: + survivors.add(os.path.abspath(path)) + if stat_result.st_mtime_ns != mtime_ns: + observations.append( + _ReferenceObservation( + content_id, path, size_bytes, mtime_ns, stat_result + ) + ) + + return observations, survivors + + +def apply_reference_observations( + session: Session, + observations: list[_ReferenceObservation], + pending_verification_ids: list[str] | None = None, +) -> None: + """Apply pre-computed observations. Performs no filesystem I/O.""" + if not observations: + return + hashing_is_enabled = mode.hashing_enabled() + for observation in observations: + content = session.get(AssetContent, observation.content_id) + if content is None: + continue + if ( + content.size_bytes != observation.observed_size_bytes + or content.mtime_ns != observation.observed_mtime_ns + ): + continue + if observation.stat_result is None: + mark_content_missing(session, observation.content_id) + continue + detect_content_change( + session, + content, + observation.stat_result, + hashing_is_enabled=hashing_is_enabled, + pending_verification_ids=pending_verification_ids, + ) + + +def sync_prefixes_with_filesystem( + session: Session, + prefixes: list[str], + collect_existing_paths: bool = False, + progress: _ScanProgress | None = None, + pending_verification_ids: list[str] | None = None, + diagnostics: list[_ReferenceDiagnostic] | None = None, +) -> set[str] | None: + if not prefixes: + return set() if collect_existing_paths else None + + observations, survivors = observe_references_on_filesystem( + prefixes, progress=progress, diagnostics=diagnostics, session=session + ) + apply_reference_observations( + session, observations, pending_verification_ids=pending_verification_ids + ) return survivors if collect_existing_paths else None def _publish_reference_diagnostics( - diagnostics: list[OSError], progress: _ScanProgress | None + diagnostics: list[_ReferenceDiagnostic], progress: _ScanProgress | None ) -> None: - for diagnostic in diagnostics: - _log_scan_error("reference_stat", diagnostic) - if isinstance(diagnostic, PermissionError): + for path, error in diagnostics: + _log_scan_error("reference_stat", error) + if isinstance(error, PermissionError): if progress is not None: progress.permission_denied += 1 - logging.debug("Permission denied accessing reference") + logging.debug("Permission denied accessing %s", path) else: - logging.debug("OSError checking reference: %s", diagnostic) + logging.debug("OSError checking %s: %s", path, error) if progress is None or progress.mark_emitted("stat_failed:reference_stat"): - emit("scanner.stat_failed", site="reference_stat", error_type=error_type(diagnostic)) + emit("scanner.stat_failed", site="reference_stat", error_type=error_type(error)) def _is_under_prefixes(path: str, prefixes: list[str]) -> bool: @@ -257,19 +342,20 @@ def sync_root_safely( Returns survivors (existing paths) or empty set on failure. """ try: - def _work(sess: Session) -> tuple[set[str], list[str], list[OSError]]: - pending_verification_ids: list[str] = [] - diagnostics: list[OSError] = [] - survivors = sync_references_with_filesystem( - sess, - root, - collect_existing_paths=True, - pending_verification_ids=pending_verification_ids, - diagnostics=diagnostics, - ) - return survivors or set(), pending_verification_ids, diagnostics + diagnostics: list[_ReferenceDiagnostic] = [] + observations, survivors = observe_references_on_filesystem( + get_scan_prefixes_for_root(root), diagnostics=diagnostics + ) + + pending_verification_ids: list[str] = [] + if observations: + def _work(sess: Session) -> None: + apply_reference_observations( + sess, observations, pending_verification_ids=pending_verification_ids + ) + + run_write_txn(_work) - survivors, pending_verification_ids, diagnostics = run_write_txn(_work) for content_id in pending_verification_ids: queue_pending_verification(content_id) _publish_reference_diagnostics(diagnostics, progress) @@ -289,18 +375,20 @@ def sync_temp_references_safely( ) -> None: """Retire temp references whose file is gone; temp is never scanned, so nothing else stats them.""" try: - def _work(sess: Session) -> tuple[list[str], list[OSError]]: - pending_verification_ids: list[str] = [] - diagnostics: list[OSError] = [] - sync_prefixes_with_filesystem( - sess, - get_temp_prefixes(), - pending_verification_ids=pending_verification_ids, - diagnostics=diagnostics, - ) - return pending_verification_ids, diagnostics + diagnostics: list[_ReferenceDiagnostic] = [] + observations, _ = observe_references_on_filesystem( + get_temp_prefixes(), diagnostics=diagnostics + ) + + pending_verification_ids: list[str] = [] + if observations: + def _work(sess: Session) -> None: + apply_reference_observations( + sess, observations, pending_verification_ids=pending_verification_ids + ) + + run_write_txn(_work) - pending_verification_ids, diagnostics = run_write_txn(_work) for content_id in pending_verification_ids: queue_pending_verification(content_id) _publish_reference_diagnostics(diagnostics, progress) diff --git a/app/assets/services/asset_management.py b/app/assets/services/asset_management.py index 03f7eb428bb..b71fadcb1b2 100644 --- a/app/assets/services/asset_management.py +++ b/app/assets/services/asset_management.py @@ -173,6 +173,37 @@ def asset_exists(asset_hash: str) -> bool: return lookup_for_view(session, canonical) is not None +def _preflight_hash_resolution( + canonical: str, +) -> tuple[str, str, str | None, str, list[str]] | None: + """Qualify candidate rows and choose the served one before the writer lease is taken.""" + with create_session() as session: + content = lookup_for_view(session, canonical) + if content is None: + return None + + records = list( + session.scalars( + select(Asset) + .where(Asset.content_id == content.id) + .order_by(Asset.created_at, Asset.id) + ) + ) + display_name = os.path.basename(content.path) + mime_type = None + if records: + latest_record = records[-1] + display_name = latest_record.name or display_name + mime_type = latest_record.mime_type + return ( + content.id, + content.path, + mime_type, + display_name, + [record.id for record in records], + ) + + def resolve_hash_to_path( asset_hash: str, ) -> DownloadResolutionResult | None: @@ -192,32 +223,23 @@ def resolve_hash_to_path( except ValueError: return None - def _work(session) -> tuple[str, str | None, str] | None: - content = lookup_for_view(session, canonical) - if content is None: + preflight = _preflight_hash_resolution(canonical) + if preflight is None: + return None + content_id, abs_path, mime_type, display_name, record_ids = preflight + + if record_ids: + def _work(session) -> bool: + content = session.get(AssetContent, content_id) + if content is None or content.is_missing or content.hash != canonical: + return False + for record_id in record_ids: + update_record_access_time(session, record_id) + return True + + if not run_write_txn(_work): return None - records = list( - session.scalars( - select(Asset) - .where(Asset.content_id == content.id) - .order_by(Asset.created_at, Asset.id) - ) - ) - display_name = os.path.basename(content.path) - mime_type = None - if records: - latest_record = records[-1] - display_name = latest_record.name or display_name - mime_type = latest_record.mime_type - for record in records: - update_record_access_time(session, record.id) - return content.path, mime_type, display_name - - resolution = run_write_txn(_work) - if resolution is None: - return None - abs_path, mime_type, display_name = resolution ctype = ( mime_type or mimetypes.guess_type(display_name)[0] @@ -242,30 +264,29 @@ def get_preview_file_paths(preview_ids: list[str]) -> dict[str, str]: def resolve_asset_for_download( reference_id: str, ) -> DownloadResolutionResult: - def _work(session) -> tuple[str, str | None, str | None]: + with create_session() as session: record = get_record_by_id(session, reference_id) if record is None: raise ValueError(f"AssetReference {reference_id} not found") - content = session.get(AssetContent, record.content_id) - if ( - content is None - or content.is_missing - or not os.path.isfile(content.path) - ): - raise FileNotFoundError( - f"No live content for AssetReference {reference_id} " - f"(content id={record.content_id}, name={record.name})" - ) - + content_missing = content is None or content.is_missing + candidate_path = None if content is None else content.path ref_name = record.name asset_mime = record.mime_type - abs_path = content.path + content_id = record.content_id + + if content_missing or candidate_path is None or not os.path.isfile(candidate_path): + raise FileNotFoundError( + f"No live content for AssetReference {reference_id} " + f"(content id={content_id}, name={ref_name})" + ) + abs_path = candidate_path + + def _work(session) -> None: update_record_access_time(session, reference_id) - return abs_path, asset_mime, ref_name - abs_path, asset_mime, ref_name = run_write_txn(_work) + run_write_txn(_work) ctype = ( asset_mime or mimetypes.guess_type(ref_name or abs_path)[0] diff --git a/tests-unit/assets_test/test_scanner_ingest_event_log.py b/tests-unit/assets_test/test_scanner_ingest_event_log.py index 3c6e1e943ee..770459710b4 100644 --- a/tests-unit/assets_test/test_scanner_ingest_event_log.py +++ b/tests-unit/assets_test/test_scanner_ingest_event_log.py @@ -107,6 +107,7 @@ def test_scanner_safe_failures_emit_exception_type_without_path( def fail_session(): raise FileNotFoundError(secret_path) + monkeypatch.setattr("app.database.db.Session", fail_session) monkeypatch.setattr("app.database.db.WriteSession", fail_session) with caplog.at_level(logging.INFO): @@ -121,7 +122,9 @@ def test_permission_error_in_reference_sync_increments_scan_counter( monkeypatch: pytest.MonkeyPatch, ) -> None: secret_path = "/private/assets/unreadable.safetensors" - content = SimpleNamespace(id="content", path=secret_path) + content = SimpleNamespace( + id="content", path=secret_path, size_bytes=1, mtime_ns=1 + ) progress = _ScanState() def deny_stat(*_args, **_kwargs): diff --git a/tests-unit/assets_test/test_write_session_convention.py b/tests-unit/assets_test/test_write_session_convention.py index 277f07a471b..8402400cb44 100644 --- a/tests-unit/assets_test/test_write_session_convention.py +++ b/tests-unit/assets_test/test_write_session_convention.py @@ -42,9 +42,14 @@ class CreateSessionCallSite(NamedTuple): # would acquire a write lease for no persisted work. CreateSessionCallSite("app/assets/lifecycle.py", "enqueue_mode_transition_work"), CreateSessionCallSite("app/assets/scanner.py", "get_unenriched_assets_for_roots"), + # This reads the catalogue so the stat walk runs before the writer lease is taken. + CreateSessionCallSite("app/assets/scanner.py", "observe_references_on_filesystem"), # This preflight reads a content path and stats it before outside-transaction hashing. CreateSessionCallSite("app/assets/scanner_changes.py", "_preflight_pending_verification"), CreateSessionCallSite("app/assets/services/asset_management.py", "get_asset_detail"), + # These qualify candidate rows with filesystem I/O before the writer lease is taken. + CreateSessionCallSite("app/assets/services/asset_management.py", "_preflight_hash_resolution"), + CreateSessionCallSite("app/assets/services/asset_management.py", "resolve_asset_for_download"), CreateSessionCallSite("app/assets/services/asset_management.py", "asset_exists"), CreateSessionCallSite("app/assets/services/asset_management.py", "get_preview_file_paths"), # These preflights read decision facts before metadata or hash I/O outside the writer lease. From 39b50af73b7863bf5a79dbe3c8326ea6a564dbd0 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 21:15:45 -0700 Subject: [PATCH 032/111] fix(assets): refuse an unsettled upload instead of mixing file facts The new-content fallback wrote the caller's hash, size and mtime while re-extracting system metadata from whatever occupied the path at that moment, so the persisted row could describe two different files. That was not a remote race: the fallback is reached precisely because four preflights each saw the file change underneath them. The stat facts would self-heal on a later scan, but system_metadata would not - enrichment only ever revisits records whose metadata is NULL, so a wrong value is permanent. A refused upload is recoverable, since the post-execution output scan re-seeds uncatalogued files; a self-inconsistent row is not. Refuse it. --- app/assets/services/ingest.py | 33 ++++++--------------------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index 64a1aea6031..d501cffab10 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -746,34 +746,13 @@ def _work(session: Session) -> UploadResult: continue logging.warning( - "Upload record preflight changed three times; falling back to in-transaction metadata extraction" + "Upload preflight for a new asset did not settle in 4 attempts; refusing the upload " + "rather than persisting content facts and file metadata that describe different bytes" + ) + raise RuntimeError( + f"Upload preflight for {path} did not settle in 4 attempts; " + "refusing to persist content facts and file metadata that describe different bytes" ) - - def _fallback_work(session: Session) -> UploadResult: - _reconcile_live_content_at_path( - session, - path, - facts, - content_written=content_written, - ) - content, inserted = create_content_reporting_insert( - session, - path, - stored_hash, - facts.size_bytes, - facts.mtime_ns, - ) - created_content_id = content.id if inserted else None - try: - record = _create_upload_record_in_txn(session, content.id, spec, path) - except Exception: - session.rollback() - if created_content_id is not None: - _discard_unreferenced_content(session, created_content_id) - raise - return _record_to_upload_result(session, record, created_new=True) - - return run_write_txn(_fallback_work) def upload_from_temp_path( From b1d44f2002ea82f164eaba1aacd124ab845e070f Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 21:15:45 -0700 Subject: [PATCH 033/111] docs(assets): correct the retry regime and record the WAL gate The documented 5 attempts and 0.05/0.1/0.2/0.4s backoff are reachable only when the lock error surfaces from inside the callback. Contention at BEGIN IMMEDIATE polls for up to 30 seconds per attempt, so the 60-second deadline admits about two and the backoff sleeps are negligible beside it. Also documents why WAL is gated on --enable-assets, and adds a production_writer_database fixture built from the real runtime engines so rollback paths can be exercised under BEGIN IMMEDIATE semantics. --- app/assets/docs/README.md | 4 +++- tests-unit/assets_test/services/conftest.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/app/assets/docs/README.md b/app/assets/docs/README.md index fab20268df3..f8ff73a3d63 100644 --- a/app/assets/docs/README.md +++ b/app/assets/docs/README.md @@ -331,10 +331,12 @@ Deleting or replacing the database file while the server is running is undefined The asset database is SQLite in WAL journal mode. Reader connections use a 30-second busy timeout. Writer connections poll `BEGIN IMMEDIATE` in Python, so each lock wait is capped at 30 seconds while the overall retry deadline remains 60 seconds. Several paths still contend for the single writer lock: a non-deduplicated upload writes its bytes and mints a delivery record, while a deduplicated upload reuses existing content and mints only the record; a same-path write whose hash has changed retires the old content and inserts new content, while a same-path write whose hash matches refreshes the existing record in place; execution outputs register per-emission during the generation loop; a background enrichment pass fills hashes and metadata row by row; hash-serves write access time to every record sharing the served content; and the upload dedup claim holds the write lock across its filesystem re-check and metadata extraction. -A session that opens as a reader and only later attempts a write can be refused instantly if another connection already holds the write lock, before the busy timeout ever gets a chance to wait. Every writing session avoids this by going through `run_write_txn`, which opens a dedicated writer session and issues `BEGIN IMMEDIATE` up front, declaring writer intent before any statement runs. A write that still finds the database locked retries up to 5 attempts total, sleeping 0.05s/0.1s/0.2s/0.4s between attempts (each jittered ±50%) and giving up once 60 seconds have elapsed since the first attempt; only a locked-database `OperationalError` is retried, every other exception propagates immediately. Contributors must not open a writing session with `create_session()` directly. `run_write_txn` is the only sanctioned path for writes, and `tests-unit/assets_test/test_write_session_convention.py` enforces that automatically. +A session that opens as a reader and only later attempts a write can be refused instantly if another connection already holds the write lock, before the busy timeout ever gets a chance to wait. Every writing session avoids this by going through `run_write_txn`, which opens a dedicated writer session and issues `BEGIN IMMEDIATE` up front, declaring writer intent before any statement runs. A write that still finds the database locked retries up to 5 attempts total, sleeping 0.05s/0.1s/0.2s/0.4s between attempts (each jittered ±50%) and giving up once 60 seconds have elapsed since the first attempt; only a locked-database `OperationalError` is retried, every other exception propagates immediately. Those 5 attempts and their backoff table are reachable only when the lock error surfaces from inside the callback, after `BEGIN IMMEDIATE` has already succeeded. Contention at `BEGIN IMMEDIATE` itself behaves differently: each attempt polls for up to 30 seconds before yielding, so the 60-second deadline admits about two attempts and the backoff sleeps are negligible beside the polling. Either way the bound is the same 60-second deadline, after which a locked-database `OperationalError` is raised. Contributors must not open a writing session with `create_session()` directly. `run_write_txn` is the only sanctioned path for writes, and `tests-unit/assets_test/test_write_session_convention.py` enforces that automatically. WAL journal mode leaves `-wal` and `-shm` sidecar files next to the main database file; treat all three as one unit for backups and copies. WAL is not supported on network filesystems, so pointing `--database-url` at a network-mounted path fails at startup with a `RuntimeError` rather than silently falling back to a less safe journal mode. +All of the above applies only when `--enable-assets` is set. The asset system is the database's only writer, so without it there is nothing to serialise: startup binds a single engine that sets `foreign_keys=ON` and nothing else, exactly as it did before this journal mode existed. A default install therefore never promotes journal mode, never leaves `-wal`/`-shm` sidecars, and cannot fail to start on a filesystem that rejects WAL. Anything that adds a non-asset writer must move this gate, not work around it. + ## Schema migration When startup finds the schema that predates the record/content split, it drops and recreates the affected tables inside the existing database file, then rebuilds them with a full scan. Rows are not migrated into the new schema, and the database file itself is not deleted. The database is backed up to a sibling file before the migration runs; if the migration fails, the database is reverted from that backup, and if it succeeds, the backup file is left in place rather than deleted. diff --git a/tests-unit/assets_test/services/conftest.py b/tests-unit/assets_test/services/conftest.py index 040cba3f6c7..f40f2f29512 100644 --- a/tests-unit/assets_test/services/conftest.py +++ b/tests-unit/assets_test/services/conftest.py @@ -80,3 +80,24 @@ def temp_dir(): """Temporary directory for file operations.""" with tempfile.TemporaryDirectory() as tmpdir: yield Path(tmpdir) + + +@pytest.fixture +def production_writer_database(tmp_path, monkeypatch): + """Bind the real runtime engines so rollback and savepoint paths run under + BEGIN IMMEDIATE semantics rather than the in-memory fixture's looser ones. + """ + import app.database.db as db_mod + + database_path = str(tmp_path / "assets.db") + monkeypatch.setattr(db_mod.args, "enable_assets", True) + monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") + monkeypatch.setattr(db_mod, "Session", None) + monkeypatch.setattr(db_mod, "_db_lock", None) + monkeypatch.setattr(db_mod, "WriteSession", None) + db_mod.init_db() + yield database_path + for factory in (db_mod.Session, db_mod.WriteSession): + if factory is not None: + factory.kw["bind"].dispose() + db_mod._db_lock.release(force=True) From d0725273c36c07df0d1f8e96a7a8ef5275fe9e74 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 15:00:22 -0700 Subject: [PATCH 034/111] test(database): show BEGIN-time contention spends the deadline polling, not attempts --- tests-unit/app_test/test_db_write_txn.py | 53 +++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/tests-unit/app_test/test_db_write_txn.py b/tests-unit/app_test/test_db_write_txn.py index 58e7164cf3c..cd9935fcc9a 100644 --- a/tests-unit/app_test/test_db_write_txn.py +++ b/tests-unit/app_test/test_db_write_txn.py @@ -366,7 +366,7 @@ def work(_session): assert attempts == 1 -def test_run_write_txn_reraises_terminal_locked_error_after_five_attempts( +def test_run_write_txn_reraises_in_callback_locked_error_after_five_attempts( memory_database, monkeypatch ): run_write_txn = db_mod.run_write_txn @@ -632,3 +632,54 @@ def reject_wal(dbapi_connection, db_path): _dispose_runtime_engines() if db_mod._db_lock is not None: db_mod._db_lock.release(force=True) + + +def test_begin_time_contention_does_not_reach_the_in_callback_attempt_count( + file_database, monkeypatch +): + """Contention at BEGIN IMMEDIATE spends its deadline polling, not on the backoff table. + + The 5-attempt regime is reachable only when the lock error surfaces from inside the + callback, so a test that fabricates it there cannot detect this difference. + """ + run_write_txn = db_mod.run_write_txn + monkeypatch.setattr(db_mod, "_SQLITE_BUSY_TIMEOUT_MS", 200) + monkeypatch.setattr(db_mod, "_WRITE_TXN_LOCK_RETRY_DEADLINE_SECONDS", 0.5) + + lock_held = threading.Event() + release = threading.Event() + + def hold_lock(): + holder = sqlite3.connect(file_database) + try: + holder.execute("BEGIN IMMEDIATE") + holder.execute("INSERT INTO tags (name) VALUES (?)", ("begin-contention-holder",)) + lock_held.set() + release.wait(timeout=30) + holder.rollback() + finally: + holder.close() + + attempts = 0 + + def work(session): + nonlocal attempts + attempts += 1 + session.execute(text("INSERT INTO tags (name) VALUES ('contended')")) + + holder = threading.Thread(target=hold_lock) + holder.start() + try: + assert lock_held.wait(timeout=5) + with pytest.raises(OperationalError, match="database is locked"): + run_write_txn(work) + assert holder.is_alive(), "holder must still own the write lock at refusal time" + finally: + release.set() + holder.join(timeout=6) + + assert not holder.is_alive() + assert 0 < attempts < 5, ( + f"BEGIN-time contention consumed the deadline in {attempts} attempts; " + "the 5-attempt backoff table governs only in-callback lock errors" + ) From ea9126e546b8ddd63a914e3a0699940d4350f0bb Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 15:34:10 -0700 Subject: [PATCH 035/111] fix(database): make asset reader connections query_only --- app/database/db.py | 8 ++++ tests-unit/app_test/test_db_write_txn.py | 47 +++++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/app/database/db.py b/app/database/db.py index a3c31cc2fe4..78386019931 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -1,3 +1,10 @@ +"""SQLite engine wiring for the runtime database. + +Readers get a WAL engine with a 30s busy_timeout and query_only; writes go +through a writer engine that takes the lock up front with BEGIN IMMEDIATE and is +retried by run_write_txn. Memory and disabled-assets startups share one engine. +""" + import logging import os import random @@ -382,6 +389,7 @@ def set_sqlite_pragma(dbapi_connection, connection_record): @event.listens_for(reader_engine, "connect") def set_reader_sqlite_pragma(dbapi_connection, connection_record): _configure_runtime_connection(dbapi_connection, db_path) + dbapi_connection.execute("PRAGMA query_only=ON").close() writer_engine = build_writer_engine(db_url, db_path) diff --git a/tests-unit/app_test/test_db_write_txn.py b/tests-unit/app_test/test_db_write_txn.py index cd9935fcc9a..424080a047d 100644 --- a/tests-unit/app_test/test_db_write_txn.py +++ b/tests-unit/app_test/test_db_write_txn.py @@ -97,9 +97,30 @@ def test_file_database_configures_runtime_pragmas(file_database): session.execute(text("PRAGMA journal_mode")).scalar_one(), session.execute(text("PRAGMA busy_timeout")).scalar_one(), session.execute(text("PRAGMA foreign_keys")).scalar_one(), + session.execute(text("PRAGMA query_only")).scalar_one(), ) - assert pragmas == ("wal", 30000, 1) + assert pragmas == ("wal", 30000, 1, 1) + + +def test_reader_session_cannot_write(file_database): + with pytest.raises(OperationalError, match="readonly"): + with db_mod.create_session() as reader: + reader.execute(text("INSERT INTO tags (name) VALUES ('ro')")) + reader.commit() + + db_mod.run_write_txn( + lambda session: session.execute(text("INSERT INTO tags (name) VALUES ('rw')")) + ) + + with db_mod.create_session() as fresh_reader: + names = ( + fresh_reader.execute(text("SELECT name FROM tags WHERE name IN ('ro', 'rw')")) + .scalars() + .all() + ) + + assert names == ["rw"] def test_runtime_connection_rejects_non_wal_journal_mode(): @@ -498,6 +519,13 @@ def test_memory_database_uses_degraded_write_transaction_wiring(memory_database) assert run_write_txn(lambda _session: "written") == "written" +def test_memory_database_leaves_its_shared_session_writable(memory_database): + with db_mod.create_session() as session: + query_only = session.execute(text("PRAGMA query_only")).scalar_one() + + assert query_only == 0 + + def test_migration_backup_checkpoints_crash_style_wal_before_copy(tmp_path, monkeypatch): database_path = _crash_style_wal_database(tmp_path) _migrate_crash_style_database(database_path, monkeypatch) @@ -602,6 +630,23 @@ def test_disabled_assets_startup_leaves_journal_mode_unpromoted(file_database_wi assert journal_mode.lower() != "wal" +def test_disabled_assets_startup_leaves_its_shared_session_writable(file_database_without_assets): + with db_mod.create_session() as session: + query_only = session.execute(text("PRAGMA query_only")).scalar_one() + session.execute(text("INSERT INTO tags (name) VALUES ('shared-writer')")) + session.commit() + + with db_mod.create_session() as fresh_reader: + names = ( + fresh_reader.execute(text("SELECT name FROM tags WHERE name = 'shared-writer'")) + .scalars() + .all() + ) + + assert query_only == 0 + assert names == ["shared-writer"] + + def test_disabled_assets_startup_writes_no_wal_sidecars(file_database_without_assets): with db_mod.create_session() as session: session.execute(text("SELECT 1")) From bf302b7d53207397e5e28d99c52d1891fe17e537 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 15:52:33 -0700 Subject: [PATCH 036/111] fix(assets): seed in bounded write transactions with stats taken before the lock --- app/assets/scanner.py | 50 +++++++++------- app/assets/scanner_changes.py | 60 ------------------- app/assets/seeder.py | 3 +- tests-unit/assets_test/services/conftest.py | 26 ++++++++ .../services/test_admission_integration.py | 4 +- .../services/test_ingest_orphan_content.py | 4 +- .../services/test_null_hash_recovery.py | 12 ++-- .../services/test_recovery_gate.py | 32 +++++----- .../services/test_scan_lifecycle.py | 9 ++- .../assets_test/services/test_scanner_b.py | 7 ++- .../services/test_scanner_seed_resilience.py | 53 +++++++++++++--- .../services/test_stored_hash_prefix.py | 13 +++- .../assets_test/test_intended_behaviour.py | 3 +- tests-unit/seeder_test/test_seeder.py | 55 +++++++++++++++++ 14 files changed, 210 insertions(+), 121 deletions(-) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index cb96a93f696..0f9b2d559f2 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -38,7 +38,6 @@ prepare_missing_content_recovery, queue_pending_recovery, queue_pending_verification, - recover_missing_content, recover_missing_content_from_preparation, ) from app.assets.scanner_admission import ( @@ -71,6 +70,10 @@ # Temp is deliberately absent: it is wiped before every scan, so walking it finds nothing. RootType = Literal["models", "input", "output"] +# A scan's worth of rows in one transaction would hold the writer lock for the whole +# scan, so seeding commits in batches this size. +MAX_WRITE_BATCH = 25 + class _ScanProgress(Protocol): hash_failed: int @@ -83,7 +86,8 @@ def mark_emitted(self, key: str) -> bool: ... class SeedAssetSpec(TypedDict): abs_path: str - # Walk-time diagnostics only: seeding persists the seed-time restat instead. + # Walk-time diagnostics only: seeding uses the stat taken by insert_asset_specs + # before the write transaction. size_bytes: int mtime_ns: int info_name: str @@ -521,9 +525,21 @@ def build_asset_specs( return specs, tag_pool, skipped +def stat_seed_specs(specs: list[SeedAssetSpec]) -> dict[str, os.stat_result | None]: + stats: dict[str, os.stat_result | None] = {} + for spec in specs: + path = os.path.abspath(spec["abs_path"]) + try: + stats[path] = os.stat(path, follow_symlinks=True) + except OSError: + stats[path] = None + return stats + + def seed_asset_specs( session: Session, specs: list[SeedAssetSpec], + stats: dict[str, os.stat_result | None], prepared_recoveries: dict[str, PreparedRecovery | None] | None = None, pending_recovery_paths: list[str] | None = None, ) -> int: @@ -534,23 +550,11 @@ def seed_asset_specs( path = os.path.abspath(spec["abs_path"]) try: with session.begin_nested(): - try: - stat_result = os.stat(path, follow_symlinks=True) - except OSError: + stat_result = stats.get(path) + if stat_result is None: logging.warning("Skipping vanished asset during scan: %s", path) continue - if prepared_recoveries is None: - try: - recovery = recover_missing_content( - session, - path, - stat_result, - hashing_is_enabled=mode.hashing_enabled(), - ) - except OSError: - logging.warning("Skipping vanished asset during scan: %s", path) - continue - elif mode.hashing_enabled(): + if prepared_recoveries is not None and mode.hashing_enabled(): prepared = prepared_recoveries.get(path) if prepared is None: logging.warning("Skipping vanished asset during scan: %s", path) @@ -604,14 +608,15 @@ def seed_asset_specs( def insert_asset_specs(specs: list[SeedAssetSpec], _tag_pool: set[str]) -> int: if not specs: return 0 + stats = stat_seed_specs(specs) prepared_recoveries: dict[str, PreparedRecovery | None] = {} if mode.hashing_enabled(): - for spec in specs: - path = os.path.abspath(spec["abs_path"]) + for path, stat_result in stats.items(): + if stat_result is None: + prepared_recoveries[path] = None + continue try: - prepared_recoveries[path] = prepare_missing_content_recovery( - path, os.stat(path, follow_symlinks=True) - ) + prepared_recoveries[path] = prepare_missing_content_recovery(path, stat_result) except OSError: prepared_recoveries[path] = None @@ -620,6 +625,7 @@ def _work(sess: Session) -> tuple[int, list[str]]: created = seed_asset_specs( sess, specs, + stats, prepared_recoveries, pending_recovery_paths, ) diff --git a/app/assets/scanner_changes.py b/app/assets/scanner_changes.py index dd696439a88..b1ffd33d4c5 100644 --- a/app/assets/scanner_changes.py +++ b/app/assets/scanner_changes.py @@ -68,66 +68,6 @@ def pending_recovery_count() -> int: return len(_pending_recovery_paths) -def recover_missing_content( - session: Session, path: str, stat_result: os.stat_result, hashing_is_enabled: bool -) -> Literal["recovered", "no_match", "unstable"]: - if not hashing_is_enabled: - return "no_match" - occupied = session.scalar( - sa.select(AssetContent.id) - .where(AssetContent.path == path, AssetContent.is_missing.is_(False)) - .limit(1) - ) - if occupied is not None: - return "no_match" - snapshot = snapshot_hash(path) - if snapshot is None: - if path not in _pending_recovery_paths: - _pending_recovery_paths.append(path) - return "unstable" - digest, verified_stat = snapshot - stored_hash = to_stored_hash(digest) - matches = list( - session.scalars( - sa.select(AssetContent).where( - AssetContent.path == path, - AssetContent.is_missing.is_(True), - AssetContent.hash == stored_hash, - ) - ) - ) - if len(matches) == 1: - recovered = matches[0] - unset_content_missing(session, recovered.id) - recovered.size_bytes = verified_stat.st_size - recovered.mtime_ns = verified_stat.st_mtime_ns - return "recovered" - if len(matches) > 1: - return "no_match" - null_hash_matches = list( - session.scalars( - sa.select(AssetContent).where( - AssetContent.path == path, - AssetContent.is_missing.is_(True), - AssetContent.hash.is_(None), - ) - ) - ) - if len(null_hash_matches) != 1: - return "no_match" - candidate = null_hash_matches[0] - if (candidate.size_bytes, candidate.mtime_ns) != ( - verified_stat.st_size, - verified_stat.st_mtime_ns, - ): - return "no_match" - unset_content_missing(session, candidate.id) - candidate.hash = stored_hash - candidate.size_bytes = verified_stat.st_size - candidate.mtime_ns = verified_stat.st_mtime_ns - return "recovered" - - def recover_missing_content_from_preparation( session: Session, path: str, diff --git a/app/assets/seeder.py b/app/assets/seeder.py index b8433ee8f1a..4aa482130fe 100644 --- a/app/assets/seeder.py +++ b/app/assets/seeder.py @@ -16,6 +16,7 @@ from app.assets.event_log import emit, error_type from app.assets.scanner import ( + MAX_WRITE_BATCH, RootType, build_asset_specs, collect_paths_for_roots, @@ -832,7 +833,7 @@ def _run_fast_phase(self, roots: tuple[RootType, ...]) -> tuple[int, int, int]: if self._check_pause_and_cancel(_ScanStage.FAST_SCAN): return total_created, skipped_existing, total_paths - batch_size = 500 + batch_size = MAX_WRITE_BATCH last_progress_time = time.perf_counter() progress_interval = 1.0 diff --git a/tests-unit/assets_test/services/conftest.py b/tests-unit/assets_test/services/conftest.py index f40f2f29512..462b72a7096 100644 --- a/tests-unit/assets_test/services/conftest.py +++ b/tests-unit/assets_test/services/conftest.py @@ -9,6 +9,32 @@ from app.assets import mode from app.assets.database.models import Base +from app.assets.scanner import SeedAssetSpec, seed_asset_specs, stat_seed_specs +from app.assets.scanner_changes import PreparedRecovery, prepare_missing_content_recovery + + +def seed_with_recovery( + session: Session, specs: list[SeedAssetSpec] +) -> tuple[int, list[str]]: + """Seed the way insert_asset_specs does, minus its write transaction. + + Recovery reads the stat and the hash taken before the transaction opened, so a + caller that hands seed_asset_specs bare specs gets no recovery at all. + """ + stats = stat_seed_specs(specs) + prepared: dict[str, PreparedRecovery | None] = {} + if mode.hashing_enabled(): + for path, stat_result in stats.items(): + if stat_result is None: + prepared[path] = None + continue + try: + prepared[path] = prepare_missing_content_recovery(path, stat_result) + except OSError: + prepared[path] = None + pending: list[str] = [] + created = seed_asset_specs(session, specs, stats, prepared, pending) + return created, pending @pytest.fixture(autouse=True) diff --git a/tests-unit/assets_test/services/test_admission_integration.py b/tests-unit/assets_test/services/test_admission_integration.py index 062aab76fb7..7773e25944d 100644 --- a/tests-unit/assets_test/services/test_admission_integration.py +++ b/tests-unit/assets_test/services/test_admission_integration.py @@ -3,7 +3,7 @@ from sqlalchemy import select from app.assets.database.models import AssetContent -from app.assets.scanner import build_asset_specs, seed_asset_specs +from app.assets.scanner import build_asset_specs, seed_asset_specs, stat_seed_specs def test_drifting_file_never_reaches_seed(session, temp_dir): @@ -15,6 +15,6 @@ def test_drifting_file_never_reaches_seed(session, temp_dir): patch("app.assets.scanner._two_stat_admit", return_value=([], [str(path)])), ): specs, _, _ = build_asset_specs([str(path)], set(), enable_metadata_extraction=False) - seed_asset_specs(session, specs) + seed_asset_specs(session, specs, stat_seed_specs(specs)) assert list(session.scalars(select(AssetContent))) == [] diff --git a/tests-unit/assets_test/services/test_ingest_orphan_content.py b/tests-unit/assets_test/services/test_ingest_orphan_content.py index ae209945ca6..25f428b8db2 100644 --- a/tests-unit/assets_test/services/test_ingest_orphan_content.py +++ b/tests-unit/assets_test/services/test_ingest_orphan_content.py @@ -9,7 +9,7 @@ import folder_paths from app.assets.database.models import Asset, AssetContent from app.assets.database.queries import create_record as create_record_query -from app.assets.scanner import SeedAssetSpec, seed_asset_specs +from app.assets.scanner import SeedAssetSpec, seed_asset_specs, stat_seed_specs from app.assets.services import ingest from app.assets.services.ingest import register_file_in_place, upload_from_temp_path @@ -145,7 +145,7 @@ def _create_record_or_raise(session_arg, content_id, name, *args, **kwargs): monkeypatch.setattr("app.assets.scanner.create_record", _create_record_or_raise) with pytest.raises(RuntimeError, match="forced create_record failure"): - seed_asset_specs(session, specs) + seed_asset_specs(session, specs, stat_seed_specs(specs)) session.rollback() assert _content_at(session, paths[fail_name]) is None, ( diff --git a/tests-unit/assets_test/services/test_null_hash_recovery.py b/tests-unit/assets_test/services/test_null_hash_recovery.py index 2a247fced8c..6ae5ff96823 100644 --- a/tests-unit/assets_test/services/test_null_hash_recovery.py +++ b/tests-unit/assets_test/services/test_null_hash_recovery.py @@ -8,7 +8,7 @@ from app.assets.database.models import Asset, AssetContent, AssetTag from app.assets.database.queries.records import create_content, create_record from app.assets.helpers import to_stored_hash -from app.assets.scanner import SeedAssetSpec, clear_pending_verifications, seed_asset_specs +from app.assets.scanner import SeedAssetSpec, clear_pending_verifications from app.assets.services import hash_mode_state from app.assets.services.hash_mode_state import ( clear_transition_queue, @@ -19,6 +19,8 @@ ) from app.assets.services.snapshot_hash import snapshot_hash +from .conftest import seed_with_recovery + @pytest.fixture(autouse=True) def _clear_queues(): @@ -81,7 +83,7 @@ def test_deleted_null_hash_row_recovers_via_scanner_after_restore( assert path.stat().st_mtime_ns == stat.st_mtime_ns, "setup: mtime must round-trip exactly" with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - created = seed_asset_specs(session, [_spec(path)]) + created, _ = seed_with_recovery(session, [_spec(path)]) session.commit() assert created == 0, "the original row must recover — no fresh content row minted" @@ -118,7 +120,7 @@ def test_different_bytes_restored_at_same_path_does_not_recover_old_row( path.write_bytes(b"a completely different, much longer payload than the original") with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - created = seed_asset_specs(session, [_spec(path)]) + created, _ = seed_with_recovery(session, [_spec(path)]) session.commit() assert created == 1, "a genuinely different file must take the normal new-content path" @@ -158,7 +160,7 @@ def test_same_size_different_mtime_restored_at_same_path_does_not_recover_old_ro assert path.stat().st_size == stat.st_size, "setup: size must match so only mtime disambiguates" with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - created = seed_asset_specs(session, [_spec(path)]) + created, _ = seed_with_recovery(session, [_spec(path)]) session.commit() assert created == 1, "a same-size-but-different-mtime restore must take the new-content path" @@ -190,7 +192,7 @@ def test_two_missing_null_hash_candidates_at_same_path_do_not_recover( session.commit() with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - created = seed_asset_specs(session, [_spec(path)]) + created, _ = seed_with_recovery(session, [_spec(path)]) session.commit() assert created == 1, "ambiguous candidates must fall through to the normal new-content path" diff --git a/tests-unit/assets_test/services/test_recovery_gate.py b/tests-unit/assets_test/services/test_recovery_gate.py index eeca7e0b0ac..f96471d3a61 100644 --- a/tests-unit/assets_test/services/test_recovery_gate.py +++ b/tests-unit/assets_test/services/test_recovery_gate.py @@ -6,15 +6,15 @@ from app.assets.database.models import Asset, AssetContent, AssetTag, Tag from app.assets.helpers import to_stored_hash -from app.assets.scanner import ( - SeedAssetSpec, - clear_pending_verifications, - pending_recovery_count, - seed_asset_specs, +from app.assets.scanner import SeedAssetSpec, clear_pending_verifications +from app.assets.scanner_changes import ( + prepare_missing_content_recovery, + recover_missing_content_from_preparation, ) -from app.assets.scanner_changes import recover_missing_content from app.assets.services.snapshot_hash import snapshot_hash +from .conftest import seed_with_recovery + @pytest.fixture(autouse=True) def _clear_queues(): @@ -65,7 +65,7 @@ def test_single_hash_match_recovers(session, temp_dir: Path): content, record = _missing_content(session, path, _stored_hash(path)) with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - created = seed_asset_specs(session, [_spec(path)]) + created, _ = seed_with_recovery(session, [_spec(path)]) session.commit() assert created == 0 @@ -81,7 +81,7 @@ def test_ambiguous_hash_match_recovers_nothing(session, temp_dir: Path): second, _ = _missing_content(session, path, digest) with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - created = seed_asset_specs(session, [_spec(path)]) + created, _ = seed_with_recovery(session, [_spec(path)]) session.commit() assert created == 1 @@ -96,7 +96,7 @@ def test_no_hash_match_creates_fresh_rows(session, temp_dir: Path): missing, _ = _missing_content(session, path, "old") with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - created = seed_asset_specs(session, [_spec(path)]) + created, _ = seed_with_recovery(session, [_spec(path)]) session.commit() assert created == 1 @@ -113,7 +113,7 @@ def test_off_mode_no_recovery(session, temp_dir: Path): patch("app.assets.scanner.mode.hashing_enabled", return_value=False), patch("app.assets.scanner_changes.snapshot_hash") as hash_mock, ): - created = seed_asset_specs(session, [_spec(path)]) + created, _ = seed_with_recovery(session, [_spec(path)]) session.commit() hash_mock.assert_not_called() @@ -130,11 +130,11 @@ def test_unstable_hash_requeues(session, temp_dir: Path): patch("app.assets.scanner.mode.hashing_enabled", return_value=True), patch("app.assets.scanner_changes.snapshot_hash", return_value=None), ): - created = seed_asset_specs(session, [_spec(path)]) + created, pending = seed_with_recovery(session, [_spec(path)]) session.commit() assert created == 0 - assert pending_recovery_count() == 1 + assert pending == [str(path)] assert session.get(AssetContent, missing.id).is_missing is True @@ -155,8 +155,12 @@ def test_recovery_skips_a_path_a_live_row_already_occupies(session, temp_dir: Pa session.commit() live_b_id = live_b.id - result = recover_missing_content( - session, str(path), stat_result, hashing_is_enabled=True + result = recover_missing_content_from_preparation( + session, + str(path), + stat_result, + prepare_missing_content_recovery(str(path), stat_result), + [], ) session.commit() diff --git a/tests-unit/assets_test/services/test_scan_lifecycle.py b/tests-unit/assets_test/services/test_scan_lifecycle.py index e5a1d280118..a1c7a79a618 100644 --- a/tests-unit/assets_test/services/test_scan_lifecycle.py +++ b/tests-unit/assets_test/services/test_scan_lifecycle.py @@ -4,13 +4,18 @@ from sqlalchemy import select from app.assets.database.models import AssetContent -from app.assets.scanner import build_asset_specs, seed_asset_specs, sync_prefixes_with_filesystem +from app.assets.scanner import ( + build_asset_specs, + seed_asset_specs, + stat_seed_specs, + sync_prefixes_with_filesystem, +) def _scan(session, root: Path) -> int: paths = [str(path) for path in root.iterdir()] specs, _, _ = build_asset_specs(paths, set(), enable_metadata_extraction=False) - return seed_asset_specs(session, specs) + return seed_asset_specs(session, specs, stat_seed_specs(specs)) def test_e2e_scan_seed_detect_prune(session, temp_dir: Path): diff --git a/tests-unit/assets_test/services/test_scanner_b.py b/tests-unit/assets_test/services/test_scanner_b.py index 37b47099611..9b1393eebc9 100644 --- a/tests-unit/assets_test/services/test_scanner_b.py +++ b/tests-unit/assets_test/services/test_scanner_b.py @@ -13,6 +13,7 @@ build_asset_specs, mark_contents_missing_outside_prefixes, seed_asset_specs, + stat_seed_specs, sync_prefixes_with_filesystem, ) from app.assets.services.snapshot_hash import snapshot_hash @@ -133,7 +134,8 @@ def test_seed_creates_content_and_record(session, temp_dir: Path): (input_root / "second.png").write_bytes(b"second") with patch("folder_paths.get_input_directory", return_value=str(input_root)): - created = seed_asset_specs(session, _build_seed_specs(input_root)) + specs = _build_seed_specs(input_root) + created = seed_asset_specs(session, specs, stat_seed_specs(specs)) session.commit() contents = list(session.scalars(select(AssetContent).order_by(AssetContent.path))) @@ -156,7 +158,8 @@ def test_prune_marks_missing_not_deletes(session, temp_dir: Path): file_path.write_bytes(b"content") with patch("folder_paths.get_input_directory", return_value=str(input_root)): - seed_asset_specs(session, _build_seed_specs(input_root)) + specs = _build_seed_specs(input_root) + seed_asset_specs(session, specs, stat_seed_specs(specs)) session.commit() marked = mark_contents_missing_outside_prefixes(session, prefixes=[]) diff --git a/tests-unit/assets_test/services/test_scanner_seed_resilience.py b/tests-unit/assets_test/services/test_scanner_seed_resilience.py index f6f8f6ba4a5..285003de436 100644 --- a/tests-unit/assets_test/services/test_scanner_seed_resilience.py +++ b/tests-unit/assets_test/services/test_scanner_seed_resilience.py @@ -8,11 +8,14 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session +from app.assets import scanner from app.assets.database.models import Asset, AssetContent from app.assets.database.queries import create_content, create_record, delete_record -from app.assets.scanner import SeedAssetSpec, seed_asset_specs +from app.assets.scanner import SeedAssetSpec, seed_asset_specs, stat_seed_specs from app.assets.services.snapshot_hash import snapshot_hash +from .conftest import seed_with_recovery + def _spec(path: Path) -> SeedAssetSpec: stat_result = path.stat() @@ -46,7 +49,7 @@ def test_seed_persists_remaining_specs_when_path_vanishes_before_restat( specs, vanished_path = _specs_with_vanished_path(temp_dir) vanished_path.unlink() - created = seed_asset_specs(session, specs) + created = seed_asset_specs(session, specs, stat_seed_specs(specs)) session.commit() assert created == 2 @@ -67,7 +70,7 @@ def _hash_or_raise(path: str) -> str | None: monkeypatch.setattr("app.assets.scanner_changes.snapshot_hash", _hash_or_raise) with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - created = seed_asset_specs(session, specs) + created, _ = seed_with_recovery(session, specs) session.commit() assert created == 2 @@ -104,7 +107,7 @@ def test_seed_logs_once_for_each_vanished_path( delete_path(monkeypatch, vanished_path) with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - _ = seed_asset_specs(session, specs) + _ = seed_with_recovery(session, specs) session.commit() messages = [ @@ -144,7 +147,7 @@ def _create_record_or_raise( monkeypatch.setattr("app.assets.scanner.create_record", _create_record_or_raise) - created = seed_asset_specs(session, specs) + created = seed_asset_specs(session, specs, stat_seed_specs(specs)) session.commit() assert created == 2 @@ -170,7 +173,7 @@ def test_seed_persists_fresh_stat_after_spec_was_built( os.utime(path, ns=(fresh_mtime_ns, fresh_mtime_ns)) fresh_stat = path.stat() - created = seed_asset_specs(session, [spec]) + created = seed_asset_specs(session, [spec], stat_seed_specs([spec])) session.commit() persisted = session.scalar( @@ -208,7 +211,43 @@ def _raise_record_creation(*_args, **_kwargs): monkeypatch.setattr("app.assets.scanner.create_record", _raise_record_creation) with pytest.raises(RuntimeError, match="forced record creation failure"): - seed_asset_specs(session, [spec]) + seed_asset_specs(session, [spec], stat_seed_specs([spec])) session.rollback() assert session.get(AssetContent, retained_content_id) is not None + + +def test_seed_takes_every_stat_before_the_write_transaction( + session: Session, temp_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + specs, _ = _specs_with_vanished_path(temp_dir) + + inside_write_txn = False + stats_under_the_lock: list[str] = [] + real_stat = scanner.os.stat + real_run_write_txn = scanner.run_write_txn + + def _recording_stat(target, *args, **kwargs): + if inside_write_txn: + stats_under_the_lock.append(str(target)) + return real_stat(target, *args, **kwargs) + + def _flagged_run_write_txn(work): + nonlocal inside_write_txn + inside_write_txn = True + try: + return real_run_write_txn(work) + finally: + inside_write_txn = False + + monkeypatch.setattr(scanner, "run_write_txn", _flagged_run_write_txn) + monkeypatch.setattr(scanner.os, "stat", _recording_stat) + + with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): + created = scanner.insert_asset_specs(specs, set()) + + assert created == 3 + assert stats_under_the_lock == [], ( + "seeding must run on stats taken before the writer lock; a stat under the lock " + "lets one slow filesystem hold every other writer out of the database" + ) diff --git a/tests-unit/assets_test/services/test_stored_hash_prefix.py b/tests-unit/assets_test/services/test_stored_hash_prefix.py index 79757bd1154..06413294c00 100644 --- a/tests-unit/assets_test/services/test_stored_hash_prefix.py +++ b/tests-unit/assets_test/services/test_stored_hash_prefix.py @@ -18,7 +18,10 @@ import folder_paths from app.assets.database.models import Asset, AssetContent from app.assets.database.queries.records import create_record -from app.assets.scanner_changes import recover_missing_content +from app.assets.scanner_changes import ( + prepare_missing_content_recovery, + recover_missing_content_from_preparation, +) from app.assets.services import asset_management, ingest from app.assets.services.asset_management import get_asset_detail from app.assets.services.ingest import ( @@ -185,8 +188,12 @@ def test_recovery_matches_prefixed_stored_hash(session, temp_dir): path.unlink() path.write_bytes(original_bytes) stat = os.stat(str(path)) - result = recover_missing_content( - session, str(path), stat, hashing_is_enabled=True + result = recover_missing_content_from_preparation( + session, + str(path), + stat, + prepare_missing_content_recovery(str(path), stat), + [], ) assert result == "recovered" diff --git a/tests-unit/assets_test/test_intended_behaviour.py b/tests-unit/assets_test/test_intended_behaviour.py index 5e262f7f53c..dd4a7a36ff9 100644 --- a/tests-unit/assets_test/test_intended_behaviour.py +++ b/tests-unit/assets_test/test_intended_behaviour.py @@ -31,6 +31,7 @@ from app.assets.scanner import ( build_asset_specs, seed_asset_specs, + stat_seed_specs, sync_prefixes_with_filesystem, ) from app.assets.scanner_admission import _should_skip_extension @@ -114,7 +115,7 @@ def _scan_pass(session, root: Path) -> int: specs, _tag_pool, _skipped = build_asset_specs( list_files_recursively(str(root)), survivors or set() ) - return seed_asset_specs(session, specs) + return seed_asset_specs(session, specs, stat_seed_specs(specs)) @contextmanager diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.py index c9f2c6e3d13..67be21407e9 100644 --- a/tests-unit/seeder_test/test_seeder.py +++ b/tests-unit/seeder_test/test_seeder.py @@ -452,3 +452,58 @@ def fail_insert(batch, batch_tags) -> int: ] tagged = "\n".join(record.getMessage() for record in caplog.records if TAG in record.getMessage()) assert "/private/models/asset.safetensors" not in tagged + + +def test_fast_phase_seeds_in_bounded_batches_it_can_park_between( + scan_seeder: _AssetSeeder, + monkeypatch: pytest.MonkeyPatch, +) -> None: + specs = [{"tags": []} for _ in range(60)] + monkeypatch.setattr( + seeder_module, "sync_root_safely", lambda _root, _progress: set() + ) + monkeypatch.setattr( + seeder_module, "collect_paths_for_roots", lambda roots: ["asset.safetensors"] * 60 + ) + monkeypatch.setattr( + seeder_module, + "build_asset_specs", + lambda paths, existing_paths, enable_metadata_extraction, progress=None: ( + specs, + {}, + 0, + ), + ) + monkeypatch.setattr(seeder_module, "tick_watch_list", lambda: None) + + batch_sizes: list[int] = [] + first_batch_written = threading.Event() + parked = threading.Event() + + def record_batch(batch, _batch_tags) -> int: + batch_sizes.append(len(batch)) + if len(batch_sizes) == 1: + scan_seeder.pause() + first_batch_written.set() + return 0 + + monkeypatch.setattr(seeder_module, "insert_asset_specs", record_batch) + scan_seeder.set_event_sink( + lambda event, _data: parked.set() if event == "assets.seed.paused" else None + ) + + worker = threading.Thread(target=scan_seeder._run_fast_phase, args=(("models",),)) + worker.start() + try: + assert first_batch_written.wait(timeout=SCAN_JOIN_TIMEOUT) + assert parked.wait(timeout=SCAN_JOIN_TIMEOUT), ( + "one write transaction per scan leaves nowhere to park; the batches between " + "them are what make a pause and a fair share of the writer lock possible" + ) + assert batch_sizes == [25], "a paused scan must not open the next write transaction" + finally: + scan_seeder.resume() + worker.join(timeout=SCAN_JOIN_TIMEOUT) + + assert worker.is_alive() is False + assert batch_sizes == [25, 25, 10] From e3200b36a6380116e1242612a1db33d82efac0f3 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 15:54:39 -0700 Subject: [PATCH 037/111] fix(assets): refuse uploads that keep changing instead of hashing under the write lock --- app/assets/services/ingest.py | 153 +----------- .../services/test_ingest_write_txn.py | 219 +++++++++++++++--- .../assets_test/services/test_upload_b.py | 14 +- 3 files changed, 196 insertions(+), 190 deletions(-) diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index d501cffab10..59e2fd1b0f1 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -28,7 +28,7 @@ ) from app.assets.event_log import emit, error_type from app.assets.helpers import normalize_tags, to_stored_hash -from app.assets.services.file_utils import get_mtime_ns, get_size_and_mtime_ns +from app.assets.services.file_utils import get_mtime_ns from app.assets.services.image_dimensions import extract_image_dimensions from app.assets.services.lookup import ( claim_qualified_content, @@ -483,33 +483,6 @@ def _assert_upload_preflight_current( raise _PreflightStale -def _create_upload_record_in_txn( - session: Session, - content_id: str, - spec: _UploadRecordSpec, - abs_path: str, -) -> Asset: - if spec.preview_id is not None and session.get(Asset, spec.preview_id) is None: - raise ValueError( - f"preview_id {spec.preview_id!r} does not reference an existing asset" - ) - record = create_record( - session, - content_id, - spec.name, - mime_type=spec.mime_type, - loader_path=compute_loader_path(abs_path), - tags=spec.tags, - system_metadata=_extract_system_metadata_sync(abs_path, spec.mime_type), - ) - if spec.user_metadata: - record.user_metadata = dict(spec.user_metadata) - if spec.preview_id: - record.preview_id = spec.preview_id - session.flush() - return record - - def _apply_reused_upload_record( session: Session, prepared: _PreparedUploadRecord, @@ -532,25 +505,6 @@ def _apply_reused_upload_record( return _record_to_upload_result(session, record, created_new=True) -def _reuse_qualified_content_in_txn( - session: Session, - stored_hash: str, - spec: _UploadRecordSpec, -) -> UploadResult | None: - content = lookup_for_view(session, stored_hash) - if content is None: - return None - if not claim_qualified_content(session, content.id, stored_hash): - session.rollback() - return None - content = refresh_qualified_content(session, content.id) - if content is None: - session.rollback() - return None - record = _create_upload_record_in_txn(session, content.id, spec, content.path) - return _record_to_upload_result(session, record, created_new=True) - - def _reuse_qualified_content( stored_hash: str, spec: _UploadRecordSpec, @@ -566,12 +520,7 @@ def _reuse_qualified_content( ) except _PreflightStale: continue - logging.warning( - "Upload preflight changed three times; falling back to in-transaction metadata extraction" - ) - return run_write_txn( - lambda session: _reuse_qualified_content_in_txn(session, stored_hash, spec) - ) + raise UploadUnstableError("upload preflight changed on every attempt") def _preflight_settle_target(dest_abs: str) -> _SettleTargetPreflight | None: @@ -647,41 +596,6 @@ def _apply_settle_target( ) -def _settle_destination_before_write_in_txn(session: Session, dest_abs: str) -> None: - if not os.path.isfile(dest_abs): - return - existing = session.scalars( - select(AssetContent).where( - AssetContent.path == dest_abs, - AssetContent.is_missing.is_(False), - ) - ).first() - if existing is None: - return - size_bytes, mtime_ns = get_size_and_mtime_ns(dest_abs) - if ( - existing.hash is not None - and existing.size_bytes == size_bytes - and existing.mtime_ns == mtime_ns - ): - return - try: - incumbent_digest, verified_stat = _snapshot_hash_with_retry(dest_abs) - except (UploadUnstableError, OSError): - mark_content_missing(session, existing.id) - return - _reconcile_live_content_at_path( - session, - dest_abs, - _ContentFacts( - to_stored_hash(incumbent_digest), - verified_stat.st_size, - verified_stat.st_mtime_ns, - ), - content_written=False, - ) - - def _settle_destination_before_write(dest_abs: str) -> None: for _restart in range(4): preflight = _preflight_settle_target(dest_abs) @@ -693,13 +607,7 @@ def _settle_destination_before_write(dest_abs: str) -> None: return except _PreflightStale: continue - logging.warning( - "Upload destination preflight changed three times; falling back to in-transaction hashing" - ) - run_write_txn( - lambda session: _settle_destination_before_write_in_txn(session, dest_abs) - ) - return + raise UploadUnstableError("upload destination changed on every attempt") def _create_content_and_upload_record( @@ -1010,54 +918,6 @@ def _apply_cached_registration( ) -def _register_cached_output_in_txn( - session: Session, - locator: str, - job_id: str | None, -) -> RegisteredAsset | None: - existing = session.scalars( - select(AssetContent).where( - AssetContent.path == locator, - AssetContent.is_missing.is_(False), - ) - ).first() - if existing is None: - logging.info( - "Cached output registration is a non-event; no live content for %s", - locator, - ) - return None - name, path_tags = get_name_and_tags_from_asset_path(locator) - mime_type = mimetypes.guess_type(locator, strict=False)[0] - sibling = session.scalars( - select(Asset) - .where(Asset.content_id == existing.id) - .order_by(Asset.created_at.asc(), Asset.id.asc()) - .limit(1) - ).first() - system_metadata = ( - dict(sibling.system_metadata) - if sibling is not None and sibling.system_metadata is not None - else _extract_system_metadata_sync(locator, mime_type) - ) - record = create_record( - session, - existing.id, - name, - mime_type=mime_type, - job_id=job_id, - loader_path=compute_loader_path(locator), - tags=path_tags, - system_metadata=system_metadata, - ) - return RegisteredAsset( - id=record.id, - content_id=record.content_id, - job_id=record.job_id, - name=record.name, - ) - - def register_cached_output( abs_path: str, job_id: str | None = None ) -> RegisteredAsset | None: @@ -1092,11 +952,10 @@ def register_cached_output( except _PreflightStale: continue logging.warning( - "Cached-output preflight changed three times; falling back to in-transaction metadata extraction" - ) - return run_write_txn( - lambda session: _register_cached_output_in_txn(session, locator, job_id) + "Cached-output preflight changed on every attempt; not registering %s", + locator, ) + return None except Exception as exc: logging.exception("Failed to register cached output: %s", locator) emit( diff --git a/tests-unit/assets_test/services/test_ingest_write_txn.py b/tests-unit/assets_test/services/test_ingest_write_txn.py index b57e2242103..4ff3cd4744b 100644 --- a/tests-unit/assets_test/services/test_ingest_write_txn.py +++ b/tests-unit/assets_test/services/test_ingest_write_txn.py @@ -1,17 +1,22 @@ import os +import json import logging import sqlite3 import threading import time +from types import SimpleNamespace +from unittest.mock import AsyncMock import folder_paths import pytest +from aiohttp import web from sqlalchemy import text from sqlalchemy.exc import IntegrityError, OperationalError import app.assets.services.ingest as ingest import app.database.db as db_mod +from app.assets.api import routes, schemas_in from app.assets.database.queries.records import create_content, create_record _BARRIER_TIMEOUT = 5 @@ -64,6 +69,185 @@ def fake(*args, **kwargs): return fake +class _FilesystemInsideWriteTxn(BaseException): + """Barrier trip. Derives from BaseException so ``except Exception`` cannot eat it.""" + + +class _WriteTxnFsBarrier: + """Runs write-transaction callables and fails if one of them touches the disk.""" + + def __init__(self) -> None: + self.depth = 0 + + def run_write_txn(self, work): + self.depth += 1 + try: + return work(object()) + finally: + self.depth -= 1 + + def _guard(self, label: str, real): + def guarded(*args, **kwargs): + if self.depth: + raise _FilesystemInsideWriteTxn(f"{label} ran inside run_write_txn") + return real(*args, **kwargs) + + return guarded + + def install(self, monkeypatch) -> None: + monkeypatch.setattr(ingest, "run_write_txn", self.run_write_txn) + for name in ("_extract_system_metadata_sync", "_snapshot_hash_with_retry"): + monkeypatch.setattr(ingest, name, self._guard(name, getattr(ingest, name))) + monkeypatch.setattr( + ingest.os.path, + "isfile", + self._guard("os.path.isfile", os.path.isfile), + ) + + +@pytest.fixture +def write_txn_fs_barrier(monkeypatch) -> _WriteTxnFsBarrier: + barrier = _WriteTxnFsBarrier() + barrier.install(monkeypatch) + return barrier + + +def test_reused_upload_refuses_after_four_stale_preflights( + write_txn_fs_barrier, monkeypatch +) -> None: + preflight = object() + prepared = object() + attempts: list[object] = [] + + monkeypatch.setattr(ingest, "_preflight_upload_record", lambda *_args: preflight) + monkeypatch.setattr(ingest, "_prepare_upload_record", lambda _preflight: prepared) + + def stale_apply(_session, observed_prepared): + attempts.append(observed_prepared) + raise ingest._PreflightStale + + monkeypatch.setattr(ingest, "_apply_reused_upload_record", stale_apply) + + spec = ingest._UploadRecordSpec("asset", [], None, {}, None) + with pytest.raises(ingest.UploadUnstableError): + ingest._reuse_qualified_content("blake3:hash", spec) + + assert attempts == [prepared, prepared, prepared, prepared] + + +def test_settle_destination_refuses_after_four_stale_preflights( + write_txn_fs_barrier, monkeypatch +) -> None: + preflight = object() + prepared = object() + attempts: list[object] = [] + + monkeypatch.setattr(ingest, "_preflight_settle_target", lambda _dest: preflight) + monkeypatch.setattr(ingest, "_prepare_settle_target", lambda _preflight: prepared) + + def stale_apply(_session, observed_prepared): + attempts.append(observed_prepared) + raise ingest._PreflightStale + + monkeypatch.setattr(ingest, "_apply_settle_target", stale_apply) + + with pytest.raises(ingest.UploadUnstableError): + ingest._settle_destination_before_write(_output_path("settle-unstable.bin")) + + assert attempts == [prepared, prepared, prepared, prepared] + + +def test_cached_registration_gives_up_after_four_stale_preflights( + write_txn_fs_barrier, monkeypatch, caplog +) -> None: + path = _output_path("cached-always-stale.bin") + with open(path, "wb") as file: + file.write(b"output") + + preflight = SimpleNamespace( + content_id="content-always-stale", + sibling_id=None, + sibling_metadata=None, + signature=None, + ) + attempts: list[str] = [] + + monkeypatch.setattr( + ingest, "_preflight_cached_registration", lambda _locator: preflight + ) + + def stale_apply(_session, _preflight, *_args): + attempts.append("apply") + raise ingest._PreflightStale + + monkeypatch.setattr(ingest, "_apply_cached_registration", stale_apply) + + try: + with caplog.at_level(logging.INFO): + assert ingest.register_cached_output(path) is None + finally: + os.unlink(path) + + assert len(attempts) == 4 + warnings = [ + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and "preflight changed" in r.getMessage() + ] + assert len(warnings) == 1 + assert [r for r in caplog.records if r.levelno >= logging.ERROR] == [], ( + "giving up is an expected outcome, not a crash" + ) + assert not [ + r + for r in caplog.records + if r.getMessage().startswith("[assets-event] ingest.register_failed") + ], "giving up is not a registration failure event" + + +@pytest.mark.asyncio +async def test_upload_route_reports_an_unsettleable_destination_as_unstable( + write_txn_fs_barrier, monkeypatch, tmp_path +) -> None: + temp_path = tmp_path / "unstable-destination.bin" + temp_path.write_bytes(b"upload bytes") + + parsed = schemas_in.ParsedUpload( + file_present=True, + file_written=temp_path.stat().st_size, + file_client_name="unstable-destination.bin", + tmp_path=str(temp_path), + tags_raw=["output", "unit-tests"], + provided_name="unstable-destination.bin", + user_metadata_raw=None, + provided_hash=None, + provided_hash_exists=None, + ) + monkeypatch.setattr(routes, "_ASSETS_ENABLED", True) + monkeypatch.setattr( + routes, "parse_multipart_upload", AsyncMock(return_value=parsed) + ) + monkeypatch.setattr( + routes, + "USER_MANAGER", + SimpleNamespace(get_request_user_id=lambda _request: "test-user"), + ) + monkeypatch.setattr(ingest, "_preflight_settle_target", lambda _dest: object()) + monkeypatch.setattr(ingest, "_prepare_settle_target", lambda _preflight: object()) + + def stale_apply(_session, _prepared): + raise ingest._PreflightStale + + monkeypatch.setattr(ingest, "_apply_settle_target", stale_apply) + + response = await routes.upload_asset(AsyncMock(spec=web.Request)) + + assert isinstance(response, web.Response) + assert response.status == 500 + body = json.loads(response.body) + assert body["error"]["code"] == "UPLOAD_UNSTABLE" + + def test_cached_registration_skips_extraction_when_live_content_is_missing( mock_create_session, monkeypatch ) -> None: @@ -212,37 +396,6 @@ def test_executed_registration_reports_preflight_os_error(monkeypatch, caplog) - ) -def test_reused_upload_falls_back_after_four_stale_preflights(monkeypatch, caplog) -> None: - preflight = object() - prepared = object() - attempts: list[object] = [] - fallback_result = object() - - monkeypatch.setattr(ingest, "_preflight_upload_record", lambda *_args: preflight) - monkeypatch.setattr(ingest, "_prepare_upload_record", lambda _preflight: prepared) - - def stale_apply(_session, observed_prepared): - attempts.append(observed_prepared) - raise ingest._PreflightStale - - monkeypatch.setattr(ingest, "_apply_reused_upload_record", stale_apply) - monkeypatch.setattr( - ingest, - "_reuse_qualified_content_in_txn", - lambda *_args: fallback_result, - ) - monkeypatch.setattr(ingest, "run_write_txn", lambda work: work(object())) - - spec = ingest._UploadRecordSpec("asset", [], None, {}, None) - with caplog.at_level(logging.WARNING): - result = ingest._reuse_qualified_content("blake3:hash", spec) - - assert result is fallback_result - assert attempts == [prepared, prepared, prepared, prepared] - warnings = [record for record in caplog.records if record.levelno == logging.WARNING] - assert len(warnings) == 1 - - def test_cached_registration_reports_terminal_write_failure( mock_create_session, monkeypatch, caplog ) -> None: @@ -432,7 +585,7 @@ def test_cached_registration_metadata_extraction_does_not_hold_the_write_lock( with open(path, "wb") as file: file.write(b"cached barrier bytes") try: - with db_mod.Session() as session: + with db_mod.WriteSession() as session: content = create_content(session, path, size_bytes=os.path.getsize(path)) session.commit() content_id = content.id @@ -483,7 +636,7 @@ def test_upload_settle_hashing_does_not_hold_the_write_lock( file.write(b"incumbent barrier bytes") try: stat = os.stat(path) - with db_mod.Session() as session: + with db_mod.WriteSession() as session: content = create_content( session, path, size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns ) diff --git a/tests-unit/assets_test/services/test_upload_b.py b/tests-unit/assets_test/services/test_upload_b.py index 487f62c6da9..f3623491ad0 100644 --- a/tests-unit/assets_test/services/test_upload_b.py +++ b/tests-unit/assets_test/services/test_upload_b.py @@ -1367,17 +1367,12 @@ def test_in_place_registration_persists_the_stat_hashing_verified( def test_multipart_upload_persists_the_stat_hashing_verified( - mock_create_session, hashing_on, monkeypatch + mock_create_session, hashing_on ): payload = b"multipart bytes whose destination stat goes stale" temp = _write_temp(payload) real_digest = snapshot_hash(temp)[0] - def _stale_stat(_path: str, follow_symlinks: bool = True) -> tuple[int, int]: - return 1, 1 - - monkeypatch.setattr(ingest_module, "get_size_and_mtime_ns", _stale_stat) - result = upload_from_temp_path( temp_path=temp, name="multipart_stat.bin", @@ -1388,11 +1383,10 @@ def _stale_stat(_path: str, follow_symlinks: bool = True) -> tuple[int, int]: with mock_create_session() as session: record = session.get(Asset, result.ref.id) content = session.get(AssetContent, record.content_id) - assert (content.size_bytes, content.mtime_ns) != (1, 1), ( - "a stat read separately from hashing can be stale; the verified stat that " - "hashing proved describes these bytes is the one to persist" + assert (content.size_bytes, content.mtime_ns) == _stat_pair(content.path), ( + "the verified stat that hashing proved describes these bytes is the one " + "to persist" ) - assert (content.size_bytes, content.mtime_ns) == _stat_pair(content.path) assert content.hash == to_stored_hash(real_digest), ( "the digest, not the (digest, stat) pair, feeds to_stored_hash" ) From d4bbeebf6e319e9385d4f92b3b69589a6fb903fb Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 16:07:42 -0700 Subject: [PATCH 038/111] fix(assets): seed write-lock barrier fixtures through the writer engine --- .../services/test_write_lock_hold_barriers.py | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/tests-unit/assets_test/services/test_write_lock_hold_barriers.py b/tests-unit/assets_test/services/test_write_lock_hold_barriers.py index 5c7442e3a30..6ae6449c19a 100644 --- a/tests-unit/assets_test/services/test_write_lock_hold_barriers.py +++ b/tests-unit/assets_test/services/test_write_lock_hold_barriers.py @@ -137,13 +137,14 @@ def test_pending_verification_hashing_does_not_hold_the_write_lock( path.write_bytes(b"verify me") stat = path.stat() - with db_mod.Session() as session: + def _seed(session): content = create_content( session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns ) create_record(session, content.id, "verify-me.bin") - session.commit() - content_id = content.id + return content.id + + content_id = db_mod.run_write_txn(_seed) scanner_changes.clear_pending_verifications() scanner_changes.queue_pending_verification(content_id) @@ -183,9 +184,11 @@ def test_transition_hashing_does_not_hold_the_write_lock( path.write_bytes(b"transition me") stat = path.stat() - with db_mod.Session() as session: - create_content(session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns) - session.commit() + db_mod.run_write_txn( + lambda session: create_content( + session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns + ) + ) hash_mode_state.clear_transition_queue() hash_mode_state._PENDING_QUEUE.append(hash_mode_state._PendingEntry(str(path))) @@ -227,13 +230,14 @@ def test_enrichment_hashing_does_not_hold_the_write_lock( path.write_bytes(b"enrich me via hash") stat = path.stat() - with db_mod.Session() as session: + def _seed(session): content = create_content( session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns ) record = create_record(session, content.id, "enrich-hash.bin") - session.commit() - content_id, record_id = content.id, record.id + return content.id, record.id + + content_id, record_id = db_mod.run_write_txn(_seed) row = scanner.UnenrichedContent(content_id, record_id, str(path), True) @@ -273,13 +277,14 @@ def test_enrichment_metadata_extraction_does_not_hold_the_write_lock( path.write_bytes(b"enrich me via metadata") stat = path.stat() - with db_mod.Session() as session: + def _seed(session): content = create_content( session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns ) record = create_record(session, content.id, "enrich-metadata.bin") - session.commit() - content_id, record_id = content.id, record.id + return content.id, record.id + + content_id, record_id = db_mod.run_write_txn(_seed) row = scanner.UnenrichedContent(content_id, record_id, str(path), False) @@ -321,12 +326,13 @@ def test_scanner_reference_stat_walk_does_not_hold_the_write_lock( path.write_bytes(b"catalogued bytes") stat = path.stat() - with db_mod.Session() as session: + def _seed(session): content = create_content( session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns ) create_record(session, content.id, "catalogued.bin") - session.commit() + + db_mod.run_write_txn(_seed) entered = threading.Event() release = threading.Event() @@ -371,7 +377,7 @@ def test_download_hash_resolution_does_not_hold_the_write_lock( digest = "b" * 64 stored_hash = f"blake3:{digest}" - with db_mod.Session() as session: + def _seed(session): content = create_content( session, str(path), @@ -380,7 +386,8 @@ def test_download_hash_resolution_does_not_hold_the_write_lock( mtime_ns=stat.st_mtime_ns, ) create_record(session, content.id, "servable.bin") - session.commit() + + db_mod.run_write_txn(_seed) entered = threading.Event() release = threading.Event() From 51a4da57cd655a34b7f63c7f0b0ced3a467365f1 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 16:43:41 -0700 Subject: [PATCH 039/111] fix(assets): reconcile references in bounded write transactions that yield between chunks --- app/assets/scanner.py | 110 ++++-- app/assets/scanner_changes.py | 13 +- app/assets/seeder.py | 30 +- .../services/test_scanner_sync_chunking.py | 312 ++++++++++++++++++ .../assets_test/services/test_split_policy.py | 108 +++++- .../test_write_session_convention.py | 2 + tests-unit/seeder_test/test_seeder.py | 111 ++++++- 7 files changed, 644 insertions(+), 42 deletions(-) create mode 100644 tests-unit/assets_test/services/test_scanner_sync_chunking.py diff --git a/app/assets/scanner.py b/app/assets/scanner.py index 0f9b2d559f2..774dadc1a6b 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -282,7 +282,8 @@ def apply_reference_observations( if content is None: continue if ( - content.size_bytes != observation.observed_size_bytes + content.is_missing + or content.size_bytes != observation.observed_size_bytes or content.mtime_ns != observation.observed_mtime_ns ): continue @@ -339,7 +340,9 @@ def _is_under_prefixes(path: str, prefixes: list[str]) -> bool: def sync_root_safely( - root: RootType, progress: _ScanProgress | None = None + root: RootType, + progress: _ScanProgress | None = None, + interrupt_check: Callable[[], bool] | None = None, ) -> set[str]: """Sync a single root's references with the filesystem. @@ -351,17 +354,29 @@ def sync_root_safely( get_scan_prefixes_for_root(root), diagnostics=diagnostics ) - pending_verification_ids: list[str] = [] - if observations: - def _work(sess: Session) -> None: - apply_reference_observations( - sess, observations, pending_verification_ids=pending_verification_ids - ) + committed_ids: list[str] = [] + try: + for index in range(0, len(observations), MAX_WRITE_BATCH): + if interrupt_check is not None and interrupt_check(): + break + chunk = observations[index : index + MAX_WRITE_BATCH] + + def _apply_chunk( + sess: Session, chunk: list[_ReferenceObservation] = chunk + ) -> list[str]: + pending_verification_ids: list[str] = [] + apply_reference_observations( + sess, + chunk, + pending_verification_ids=pending_verification_ids, + ) + return pending_verification_ids - run_write_txn(_work) + committed_ids.extend(run_write_txn(_apply_chunk)) + finally: + for content_id in committed_ids: + queue_pending_verification(content_id) - for content_id in pending_verification_ids: - queue_pending_verification(content_id) _publish_reference_diagnostics(diagnostics, progress) return survivors except Exception as exc: @@ -376,6 +391,7 @@ def _work(sess: Session) -> None: def sync_temp_references_safely( progress: _ScanProgress | None = None, + interrupt_check: Callable[[], bool] | None = None, ) -> None: """Retire temp references whose file is gone; temp is never scanned, so nothing else stats them.""" try: @@ -384,17 +400,29 @@ def sync_temp_references_safely( get_temp_prefixes(), diagnostics=diagnostics ) - pending_verification_ids: list[str] = [] - if observations: - def _work(sess: Session) -> None: - apply_reference_observations( - sess, observations, pending_verification_ids=pending_verification_ids - ) + committed_ids: list[str] = [] + try: + for index in range(0, len(observations), MAX_WRITE_BATCH): + if interrupt_check is not None and interrupt_check(): + break + chunk = observations[index : index + MAX_WRITE_BATCH] + + def _apply_chunk( + sess: Session, chunk: list[_ReferenceObservation] = chunk + ) -> list[str]: + pending_verification_ids: list[str] = [] + apply_reference_observations( + sess, + chunk, + pending_verification_ids=pending_verification_ids, + ) + return pending_verification_ids - run_write_txn(_work) + committed_ids.extend(run_write_txn(_apply_chunk)) + finally: + for content_id in committed_ids: + queue_pending_verification(content_id) - for content_id in pending_verification_ids: - queue_pending_verification(content_id) _publish_reference_diagnostics(diagnostics, progress) except Exception as exc: logging.exception("temp reference sync failed: %s", exc) @@ -405,22 +433,52 @@ def _work(sess: Session) -> None: ) -def mark_missing_outside_prefixes_safely(prefixes: list[str]) -> int: +def mark_missing_outside_prefixes_safely( + prefixes: list[str], interrupt_check: Callable[[], bool] | None = None +) -> int: """Mark references as missing when outside the given prefixes. - This is a non-destructive soft-delete. Returns count marked or 0 on failure. + This is a non-destructive soft-delete. Returns the count committed before completion, + interruption, or failure. """ + marked_so_far = 0 try: - return run_write_txn( - lambda session: mark_contents_missing_outside_prefixes(session, prefixes) - ) + with create_session() as session: + contents = session.scalars( + sa.select(AssetContent).where(AssetContent.is_missing.is_(False)) + ) + content_ids = [ + content.id + for content in contents + if not _is_under_prefixes(content.path, prefixes) + ] + + for index in range(0, len(content_ids), MAX_WRITE_BATCH): + if interrupt_check is not None and interrupt_check(): + break + chunk = content_ids[index : index + MAX_WRITE_BATCH] + + def _mark_chunk( + session: Session, chunk: list[str] = chunk + ) -> int: + marked = 0 + for content_id in chunk: + content = session.get(AssetContent, content_id) + if content is None or content.is_missing: + continue + mark_content_missing(session, content_id) + marked += 1 + return marked + + marked_so_far += run_write_txn(_mark_chunk) + return marked_so_far except Exception as exc: logging.exception("marking missing assets failed: %s", exc) emit( "scanner.mark_missing_failed", error_type=error_type(exc), ) - return 0 + return marked_so_far def mark_contents_missing_outside_prefixes( diff --git a/app/assets/scanner_changes.py b/app/assets/scanner_changes.py index b1ffd33d4c5..638c887a605 100644 --- a/app/assets/scanner_changes.py +++ b/app/assets/scanner_changes.py @@ -14,7 +14,7 @@ import sqlalchemy as sa from sqlalchemy.orm import Session -from app.assets.database.models import AssetContent +from app.assets.database.models import Asset, AssetContent from app.assets.database.queries.records import ( create_content, create_record, @@ -175,6 +175,12 @@ def detect_content_change( # Ruling #10: size drift with unchanged mtime is undefined behavior. return if hashing_is_enabled: + if content.hash is None: + session.execute( + sa.update(Asset) + .where(Asset.content_id == content.id) + .values(system_metadata=None) + ) if pending_verification_ids is None: queue_pending_verification(content.id) elif content.id not in pending_verification_ids: @@ -189,6 +195,11 @@ def detect_content_change( content.size_bytes = stat_result.st_size content.mtime_ns = stat_result.st_mtime_ns content.hash = None + session.execute( + sa.update(Asset) + .where(Asset.content_id == content.id) + .values(system_metadata=None) + ) return split_content(session, content, stat_result, hash_value=None) diff --git a/app/assets/seeder.py b/app/assets/seeder.py index 4aa482130fe..3f717a0b36f 100644 --- a/app/assets/seeder.py +++ b/app/assets/seeder.py @@ -498,7 +498,9 @@ def _is_cancelled(self) -> bool: """Check if cancellation has been requested.""" return self._cancel_event.is_set() - def _is_paused_or_cancelled(self) -> bool: + def _is_paused_or_cancelled( + self, stage: _ScanStage = _ScanStage.ENRICH + ) -> bool: """Non-blocking check: True if paused or cancelled. Use as interrupt_check for I/O-bound work (e.g. hashing) so that @@ -508,7 +510,7 @@ def _is_paused_or_cancelled(self) -> bool: """ cancelled = self._cancel_event.is_set() if cancelled: - self._record_cancel_stage(_ScanStage.ENRICH) + self._record_cancel_stage(stage) return not self._run_gate.is_set() or cancelled def _record_cancel_stage(self, stage: _ScanStage) -> None: @@ -624,7 +626,12 @@ def _run_scan(self) -> None: if self._prune_first: all_prefixes = get_owned_prefixes() - marked = mark_missing_outside_prefixes_safely(all_prefixes) + marked = mark_missing_outside_prefixes_safely( + all_prefixes, + interrupt_check=lambda: self._is_paused_or_cancelled( + _ScanStage.PRUNING + ), + ) emit( "seeder.marked_missing", count=marked, @@ -632,7 +639,12 @@ def _run_scan(self) -> None: ) if marked > 0: logging.info("Marked %d refs as missing before scan", marked) - sync_temp_references_safely(scan_state) + sync_temp_references_safely( + scan_state, + interrupt_check=lambda: self._is_paused_or_cancelled( + _ScanStage.PRUNING + ), + ) if self._check_pause_and_cancel(_ScanStage.PRUNING): logging.info("Asset scan cancelled after pruning phase") @@ -789,7 +801,15 @@ def _run_fast_phase(self, roots: tuple[RootType, ...]) -> tuple[int, int, int]: for r in roots: if self._check_pause_and_cancel(_ScanStage.FAST_SCAN): return total_created, skipped_existing, 0 - existing_paths.update(sync_root_safely(r, scan_state)) + existing_paths.update( + sync_root_safely( + r, + scan_state, + interrupt_check=lambda: self._is_paused_or_cancelled( + _ScanStage.FAST_SCAN + ), + ) + ) logging.debug( "Fast scan: sync_root phase took %.3fs (%d existing paths)", time.perf_counter() - t_sync, diff --git a/tests-unit/assets_test/services/test_scanner_sync_chunking.py b/tests-unit/assets_test/services/test_scanner_sync_chunking.py new file mode 100644 index 00000000000..d6c7b82dabd --- /dev/null +++ b/tests-unit/assets_test/services/test_scanner_sync_chunking.py @@ -0,0 +1,312 @@ +from pathlib import Path + +import sqlalchemy as sa + +import app.database.db as db_mod +from app.assets import scanner +from app.assets.database.models import AssetContent +from app.assets.database.queries import create_content, create_record, mark_content_missing + + +def _seed_reference_observations( + root: Path, + count: int, + *, + content_hash: str | None = "blake3:old", +) -> tuple[list[str], list[scanner._ReferenceObservation], set[str]]: + paths: list[Path] = [] + for index in range(count): + path = root / f"reference-{index:03d}.bin" + path.write_bytes(f"reference-{index:03d}".encode()) + paths.append(path) + + def seed(session): + content_ids: list[str] = [] + for path in paths: + stat_result = path.stat() + content = create_content( + session, + str(path), + hash=content_hash, + size_bytes=stat_result.st_size, + mtime_ns=stat_result.st_mtime_ns - 1, + ) + create_record( + session, + content.id, + path.name, + system_metadata={"seeded": True}, + ) + content_ids.append(content.id) + return content_ids + + content_ids = db_mod.run_write_txn(seed) + observations = [ + scanner._ReferenceObservation( + content_id, + str(path), + path.stat().st_size, + path.stat().st_mtime_ns - 1, + path.stat(), + ) + for content_id, path in zip(content_ids, paths) + ] + return content_ids, observations, {str(path.resolve()) for path in paths} + + +def _content_states(content_ids: list[str]) -> list[tuple[str | None, int | None, bool]]: + with db_mod.create_session() as session: + return [ + (content.hash, content.mtime_ns, content.is_missing) + for content_id in content_ids + if (content := session.get(AssetContent, content_id)) is not None + ] + + +def _reset_observed_rows( + content_ids: list[str], observations: list[scanner._ReferenceObservation] +) -> None: + def reset(session): + for content_id, observation in zip(content_ids, observations): + content = session.get(AssetContent, content_id) + assert content is not None + content.hash = "blake3:old" + content.size_bytes = observation.observed_size_bytes + content.mtime_ns = observation.observed_mtime_ns + + db_mod.run_write_txn(reset) + + +def _seed_outside_contents(root: Path, count: int) -> list[str]: + def seed(session): + content_ids: list[str] = [] + for index in range(count): + path = root / f"outside-{index:03d}.bin" + content = create_content(session, str(path), size_bytes=1, mtime_ns=1) + create_record(session, content.id, path.name) + content_ids.append(content.id) + return content_ids + + return db_mod.run_write_txn(seed) + + +def _missing_count(content_ids: list[str]) -> int: + with db_mod.create_session() as session: + return session.scalar( + sa.select(sa.func.count()) + .select_from(AssetContent) + .where( + AssetContent.id.in_(content_ids), + AssetContent.is_missing.is_(True), + ) + ) + + +def _record_transactions(monkeypatch) -> list[int]: + real_run_write_txn = scanner.run_write_txn + transactions: list[int] = [] + + def record(work): + transactions.append(1) + return real_run_write_txn(work) + + monkeypatch.setattr(scanner, "run_write_txn", record) + return transactions + + +def test_root_sync_matches_single_transaction_and_uses_three_bounded_writes( + tmp_path: Path, monkeypatch, session +) -> None: + content_ids, observations, expected_survivors = _seed_reference_observations( + tmp_path, 60 + ) + + baseline_survivors = db_mod.run_write_txn( + lambda session: scanner.sync_prefixes_with_filesystem( + session, + [str(tmp_path)], + collect_existing_paths=True, + ) + ) + baseline_states = _content_states(content_ids) + _reset_observed_rows(content_ids, observations) + + monkeypatch.setattr( + scanner, + "observe_references_on_filesystem", + lambda *_args, **_kwargs: (observations, expected_survivors), + ) + transactions = _record_transactions(monkeypatch) + + survivors = scanner.sync_root_safely("input") + + assert len(transactions) == 3 + assert survivors == baseline_survivors == expected_survivors + assert _content_states(content_ids) == baseline_states + + +def test_temp_sync_uses_bounded_write_transactions( + tmp_path: Path, monkeypatch, session +) -> None: + _content_ids, observations, survivors = _seed_reference_observations(tmp_path, 30) + monkeypatch.setattr( + scanner, + "observe_references_on_filesystem", + lambda *_args, **_kwargs: (observations, survivors), + ) + transactions = _record_transactions(monkeypatch) + + scanner.sync_temp_references_safely() + + assert len(transactions) == 2 + + +def test_root_sync_interrupts_between_chunks_and_publishes_committed_ids( + tmp_path: Path, monkeypatch, session +) -> None: + content_ids, observations, survivors = _seed_reference_observations( + tmp_path, 60, content_hash=None + ) + monkeypatch.setattr(scanner.mode, "hashing_enabled", lambda: True) + monkeypatch.setattr( + scanner, + "observe_references_on_filesystem", + lambda *_args, **_kwargs: (observations, survivors), + ) + published: list[str] = [] + monkeypatch.setattr(scanner, "queue_pending_verification", published.append) + real_run_write_txn = scanner.run_write_txn + txn_count = 0 + + def count_transaction(work): + nonlocal txn_count + txn_count += 1 + return real_run_write_txn(work) + + checks = 0 + + def interrupt_after_first_chunk() -> bool: + nonlocal checks + checks += 1 + return checks > 1 + + monkeypatch.setattr(scanner, "run_write_txn", count_transaction) + + result = scanner.sync_root_safely( + "input", interrupt_check=interrupt_after_first_chunk + ) + + assert txn_count == 1 + assert result == survivors + assert published == content_ids[: scanner.MAX_WRITE_BATCH] + + +def test_root_sync_later_chunk_failure_publishes_only_prior_commits( + tmp_path: Path, monkeypatch, session +) -> None: + content_ids, observations, survivors = _seed_reference_observations( + tmp_path, 60, content_hash=None + ) + monkeypatch.setattr(scanner.mode, "hashing_enabled", lambda: True) + monkeypatch.setattr( + scanner, + "observe_references_on_filesystem", + lambda *_args, **_kwargs: (observations, survivors), + ) + published: list[str] = [] + monkeypatch.setattr(scanner, "queue_pending_verification", published.append) + real_run_write_txn = scanner.run_write_txn + txn_count = 0 + + def fail_second_transaction(work): + nonlocal txn_count + txn_count += 1 + if txn_count == 2: + raise RuntimeError("chunk two failed") + return real_run_write_txn(work) + + monkeypatch.setattr(scanner, "run_write_txn", fail_second_transaction) + + result = scanner.sync_root_safely("input") + + assert result == set() + assert txn_count == 2 + assert published == content_ids[: scanner.MAX_WRITE_BATCH] + + +def test_missing_row_is_untouched_when_observation_is_applied_later( + tmp_path: Path, session +) -> None: + content_ids, observations, _survivors = _seed_reference_observations(tmp_path, 1) + + db_mod.run_write_txn(lambda session: mark_content_missing(session, content_ids[0])) + db_mod.run_write_txn( + lambda session: scanner.apply_reference_observations(session, observations) + ) + + assert _content_states(content_ids) == [ + ("blake3:old", observations[0].observed_mtime_ns, True) + ] + + +def test_prune_marks_thirty_rows_in_two_transactions( + tmp_path: Path, monkeypatch, session +) -> None: + content_ids = _seed_outside_contents(tmp_path / "outside", 30) + transactions = _record_transactions(monkeypatch) + + marked = scanner.mark_missing_outside_prefixes_safely( + [str(tmp_path / "inside")] + ) + + assert marked == 30 + assert len(transactions) == 2 + assert _missing_count(content_ids) == 30 + + +def test_prune_interrupts_between_chunks_and_returns_committed_count( + tmp_path: Path, monkeypatch, session +) -> None: + content_ids = _seed_outside_contents(tmp_path / "outside", 30) + transactions = _record_transactions(monkeypatch) + + checks = 0 + + def interrupt_after_first_chunk() -> bool: + nonlocal checks + checks += 1 + return checks > 1 + + marked = scanner.mark_missing_outside_prefixes_safely( + [str(tmp_path / "inside")], + interrupt_check=interrupt_after_first_chunk, + ) + + assert marked == 25 + assert len(transactions) == 1 + assert _missing_count(content_ids) == 25 + + +def test_prune_later_chunk_failure_returns_prior_committed_count( + tmp_path: Path, monkeypatch, session +) -> None: + content_ids = _seed_outside_contents(tmp_path / "outside", 30) + real_run_write_txn = scanner.run_write_txn + txn_count = 0 + + def fail_second_transaction(work): + nonlocal txn_count + txn_count += 1 + if txn_count == 2: + raise RuntimeError("chunk two failed") + return real_run_write_txn(work) + + monkeypatch.setattr(scanner, "run_write_txn", fail_second_transaction) + + marked = scanner.mark_missing_outside_prefixes_safely( + [str(tmp_path / "inside")] + ) + + assert marked == 25 + assert txn_count == 2 + assert _missing_count(content_ids) == 25 diff --git a/tests-unit/assets_test/services/test_split_policy.py b/tests-unit/assets_test/services/test_split_policy.py index 0e612e79786..cf1e6a3d95a 100644 --- a/tests-unit/assets_test/services/test_split_policy.py +++ b/tests-unit/assets_test/services/test_split_policy.py @@ -175,7 +175,7 @@ def test_same_size_mtime_bump_does_not_split(session: Session, temp_dir: Path) - ) assert len(rows_at_path) == 1 surviving = session.get(Asset, record_id) - assert surviving.system_metadata == {"k": "v"} + assert surviving.system_metadata is None tags = fetch_record_tags(session, record_id) assert "keepme" in tags assert "missing" not in tags @@ -184,6 +184,106 @@ def test_same_size_mtime_bump_does_not_split(session: Session, temp_dir: Path) - assert live.size_bytes == 100 +def test_hashing_off_same_size_change_clears_linked_system_metadata( + session: Session, temp_dir: Path +) -> None: + path = temp_dir / "metadata-off.safetensors" + content = create_content( + session, + str(path), + hash="blake3:old", + size_bytes=100, + mtime_ns=1000, + ) + record = create_record( + session, + content.id, + path.name, + system_metadata={"w": 1}, + ) + session.commit() + + detect_content_change( + session, + content, + _FakeStat(st_size=100, st_mtime_ns=2000), + hashing_is_enabled=False, + ) + session.commit() + session.expire_all() + + assert session.get(AssetContent, content.id).hash is None + assert session.get(Asset, record.id).system_metadata is None + + +def test_hashing_on_null_hash_change_clears_metadata_and_queues_verification( + session: Session, temp_dir: Path +) -> None: + path = temp_dir / "metadata-pending.safetensors" + content = create_content( + session, + str(path), + hash=None, + size_bytes=100, + mtime_ns=1000, + ) + record = create_record( + session, + content.id, + path.name, + system_metadata={"w": 1}, + ) + session.commit() + pending_verification_ids: list[str] = [] + + detect_content_change( + session, + content, + _FakeStat(st_size=100, st_mtime_ns=2000), + hashing_is_enabled=True, + pending_verification_ids=pending_verification_ids, + ) + session.commit() + session.expire_all() + + assert session.get(Asset, record.id).system_metadata is None + assert pending_verification_ids == [content.id] + + +def test_hashing_on_hashed_change_preserves_metadata_until_verification( + session: Session, temp_dir: Path +) -> None: + path = temp_dir / "metadata-hashed.safetensors" + content = create_content( + session, + str(path), + hash="blake3:old", + size_bytes=100, + mtime_ns=1000, + ) + record = create_record( + session, + content.id, + path.name, + system_metadata={"w": 1}, + ) + session.commit() + pending_verification_ids: list[str] = [] + + detect_content_change( + session, + content, + _FakeStat(st_size=100, st_mtime_ns=2000), + hashing_is_enabled=True, + pending_verification_ids=pending_verification_ids, + ) + session.commit() + session.expire_all() + + assert session.get(Asset, record.id).system_metadata == {"w": 1} + assert pending_verification_ids == [content.id] + + def test_accepted_mtime_bump_drops_the_unverifiable_hash( session: Session, temp_dir: Path ) -> None: @@ -211,7 +311,7 @@ def test_accepted_mtime_bump_drops_the_unverifiable_hash( surviving = session.get(Asset, record_id) assert surviving is not None and surviving.content_id == content_id - assert surviving.system_metadata == {"k": "v"} + assert surviving.system_metadata is None assert "keepme" in fetch_record_tags(session, record_id) listed, _, _ = list_records_page(session, RecordPageSpec(limit=100)) assert record_id in {row.id for row in listed} @@ -296,8 +396,8 @@ def test_dropped_hash_is_refilled_in_place_by_a_later_hash_mode_pass( ) assert len(rows_at_path) == 1 assert "keepme" in fetch_record_tags(session, record_id) - assert session.get(Asset, record_id).system_metadata == {"k": "v"} - assert record_id not in _candidates_under(session, temp_dir, compute_hashes=True) + assert session.get(Asset, record_id).system_metadata is None + assert record_id in _candidates_under(session, temp_dir, compute_hashes=True) def test_mtime_and_size_change_splits_with_null_metadata( diff --git a/tests-unit/assets_test/test_write_session_convention.py b/tests-unit/assets_test/test_write_session_convention.py index 8402400cb44..65bef18c028 100644 --- a/tests-unit/assets_test/test_write_session_convention.py +++ b/tests-unit/assets_test/test_write_session_convention.py @@ -44,6 +44,8 @@ class CreateSessionCallSite(NamedTuple): CreateSessionCallSite("app/assets/scanner.py", "get_unenriched_assets_for_roots"), # This reads the catalogue so the stat walk runs before the writer lease is taken. CreateSessionCallSite("app/assets/scanner.py", "observe_references_on_filesystem"), + # This reads the ids to prune before bounded write transactions mark them missing. + CreateSessionCallSite("app/assets/scanner.py", "mark_missing_outside_prefixes_safely"), # This preflight reads a content path and stats it before outside-transaction hashing. CreateSessionCallSite("app/assets/scanner_changes.py", "_preflight_pending_verification"), CreateSessionCallSite("app/assets/services/asset_management.py", "get_asset_detail"), diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.py index 67be21407e9..50ab7af0c58 100644 --- a/tests-unit/seeder_test/test_seeder.py +++ b/tests-unit/seeder_test/test_seeder.py @@ -310,7 +310,9 @@ def cancel_at_pruning(stage) -> bool: monkeypatch.setattr(scan_seeder, "_check_pause_and_cancel", cancel_at_pruning) monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: ()) monkeypatch.setattr( - seeder_module, "mark_missing_outside_prefixes_safely", lambda prefixes: 0 + seeder_module, + "mark_missing_outside_prefixes_safely", + lambda prefixes, interrupt_check=None: 0, ) original_emit = seeder_module.emit @@ -337,7 +339,9 @@ def test_scan_paused_after_its_last_phase_still_completes( scan_seeder._phase = ScanPhase.ENRICH monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: ()) monkeypatch.setattr( - seeder_module, "mark_missing_outside_prefixes_safely", lambda prefixes: 0 + seeder_module, + "mark_missing_outside_prefixes_safely", + lambda prefixes, interrupt_check=None: 0, ) def pause_while_finishing(roots) -> tuple[bool, int]: @@ -373,6 +377,93 @@ def test_enrich_interrupt_records_the_enrich_cancellation_stage( assert scan_seeder._scan_state.cancel_stage == "enrich" +def test_root_sync_interrupt_records_the_fast_scan_cancellation_stage( + scan_seeder: _AssetSeeder, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _RootSyncCaptured(Exception): + pass + + captured_interrupt = None + + def capture_root_interrupt( + _root, _progress, interrupt_check=None + ) -> set[str]: + nonlocal captured_interrupt + captured_interrupt = interrupt_check + raise _RootSyncCaptured + + monkeypatch.setattr(seeder_module, "sync_root_safely", capture_root_interrupt) + + with pytest.raises(_RootSyncCaptured): + scan_seeder._run_fast_phase(("models",)) + + scan_seeder._cancel_event.set() + assert captured_interrupt is not None + assert captured_interrupt() is True + assert scan_seeder._scan_state is not None + assert scan_seeder._scan_state.cancel_stage == _ScanStage.FAST_SCAN.value + + +def test_prune_interrupt_records_the_pruning_cancellation_stage( + scan_seeder: _AssetSeeder, + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed: dict[str, bool | str | None] = {} + scan_seeder._prune_first = True + scan_seeder._phase = ScanPhase.FAST + monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: ()) + + def cancel_during_prune(_prefixes, interrupt_check=None) -> int: + scan_seeder._cancel_event.set() + observed["result"] = interrupt_check() + assert scan_seeder._scan_state is not None + observed["stage"] = scan_seeder._scan_state.cancel_stage + return 0 + + monkeypatch.setattr( + seeder_module, "mark_missing_outside_prefixes_safely", cancel_during_prune + ) + monkeypatch.setattr( + seeder_module, + "sync_temp_references_safely", + lambda _progress, interrupt_check=None: None, + ) + + scan_seeder._run_scan() + + assert observed == {"result": True, "stage": _ScanStage.PRUNING.value} + + +def test_temp_sync_interrupt_records_the_pruning_cancellation_stage( + scan_seeder: _AssetSeeder, + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed: dict[str, bool | str | None] = {} + scan_seeder._prune_first = True + scan_seeder._phase = ScanPhase.FAST + monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: ()) + monkeypatch.setattr( + seeder_module, + "mark_missing_outside_prefixes_safely", + lambda _prefixes, interrupt_check=None: 0, + ) + + def cancel_during_temp(_progress, interrupt_check=None) -> None: + scan_seeder._cancel_event.set() + observed["result"] = interrupt_check() + assert scan_seeder._scan_state is not None + observed["stage"] = scan_seeder._scan_state.cancel_stage + + monkeypatch.setattr( + seeder_module, "sync_temp_references_safely", cancel_during_temp + ) + + scan_seeder._run_scan() + + assert observed == {"result": True, "stage": _ScanStage.PRUNING.value} + + def test_prune_before_scan_emits_marked_missing_with_pruning_stage( scan_seeder: _AssetSeeder, monkeypatch: pytest.MonkeyPatch, @@ -382,10 +473,14 @@ def test_prune_before_scan_emits_marked_missing_with_pruning_stage( scan_seeder._phase = ScanPhase.FAST monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: ()) monkeypatch.setattr( - seeder_module, "mark_missing_outside_prefixes_safely", lambda prefixes: 5 + seeder_module, + "mark_missing_outside_prefixes_safely", + lambda prefixes, interrupt_check=None: 5, ) monkeypatch.setattr( - seeder_module, "sync_temp_references_safely", lambda _progress: None + seeder_module, + "sync_temp_references_safely", + lambda _progress, interrupt_check=None: None, ) monkeypatch.setattr(scan_seeder, "_run_fast_phase", lambda roots: (0, 0, 0)) @@ -423,7 +518,9 @@ def test_batch_insert_failure_emits_only_the_exception_type( caplog: pytest.LogCaptureFixture, ) -> None: monkeypatch.setattr( - seeder_module, "sync_root_safely", lambda _root, _progress: set() + seeder_module, + "sync_root_safely", + lambda _root, _progress, interrupt_check=None: set(), ) monkeypatch.setattr( seeder_module, "collect_paths_for_roots", lambda roots: ["asset.safetensors"] @@ -460,7 +557,9 @@ def test_fast_phase_seeds_in_bounded_batches_it_can_park_between( ) -> None: specs = [{"tags": []} for _ in range(60)] monkeypatch.setattr( - seeder_module, "sync_root_safely", lambda _root, _progress: set() + seeder_module, + "sync_root_safely", + lambda _root, _progress, interrupt_check=None: set(), ) monkeypatch.setattr( seeder_module, "collect_paths_for_roots", lambda roots: ["asset.safetensors"] * 60 From 838c6967632f9c0082c73f8795912d6c7343a841 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 17:13:41 -0700 Subject: [PATCH 040/111] fix(assets): enrich assets in bounded transactions with a row compare-and-set instead of a re-stat --- app/assets/scanner.py | 170 ++++++++++-------- tests-unit/assets_test/helpers.py | 11 +- .../services/test_enrichment_batching.py | 146 +++++++++++++++ ...est_scanner_retry_and_counter_integrity.py | 125 ++++++++----- .../services/test_write_lock_hold_barriers.py | 73 +++++++- .../test_scanner_ingest_event_log.py | 7 +- .../assets_test/test_write_contention_e2e.py | 16 +- 7 files changed, 421 insertions(+), 127 deletions(-) create mode 100644 tests-unit/assets_test/services/test_enrichment_batching.py diff --git a/app/assets/scanner.py b/app/assets/scanner.py index 774dadc1a6b..43d1475eace 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -104,6 +104,8 @@ class UnenrichedContent: record_id: str file_path: str needs_hash: bool = False + observed_size_bytes: int = 0 + observed_mtime_ns: int | None = None class _PreparedEnrichment(NamedTuple): @@ -714,6 +716,8 @@ def get_unenriched_assets_for_roots( Asset.id, AssetContent.path, AssetContent.hash.is_(None).label("needs_hash"), + AssetContent.size_bytes, + AssetContent.mtime_ns, ) .join(Asset, Asset.content_id == AssetContent.id) .where(AssetContent.is_missing.is_(False)) @@ -735,8 +739,15 @@ def get_unenriched_assets_for_roots( rows = sess.execute(query.order_by(Asset.id).limit(limit)).all() return [ - UnenrichedContent(content_id, record_id, file_path, needs_hash) - for content_id, record_id, file_path, needs_hash in rows + UnenrichedContent( + content_id, + record_id, + file_path, + needs_hash, + observed_size_bytes, + observed_mtime_ns, + ) + for content_id, record_id, file_path, needs_hash, observed_size_bytes, observed_mtime_ns in rows ] @@ -818,42 +829,43 @@ def _prepare_enrichment( ) -def _apply_enrichment(session: Session, prepared: _PreparedEnrichment) -> bool: - row = prepared.row - content = session.get(AssetContent, row.content_id) - record = session.get(Asset, row.record_id) - if content is None or record is None: - return False - try: - current_stat = os.stat(row.file_path, follow_symlinks=True) - except OSError: - return False - if ( - content.mtime_ns != get_mtime_ns(prepared.stat_result) - or current_stat.st_size != prepared.stat_result.st_size - or get_mtime_ns(current_stat) != get_mtime_ns(prepared.stat_result) - ): - logging.info( - "Content %s changed during enrichment, discarding stale result", - row.content_id, - ) - return False - hash_applied = False - if prepared.stored_hash is not None and content.hash is None: - content.hash = prepared.stored_hash - hash_applied = True +def _apply_enrichments( + session: Session, prepared: list[_PreparedEnrichment] +) -> list[str]: + applied_ids: list[str] = [] + for item in prepared: + row = item.row + content = session.get(AssetContent, row.content_id) + record = session.get(Asset, row.record_id) + if ( + content is None + or record is None + or content.is_missing + or content.size_bytes != row.observed_size_bytes + or content.mtime_ns != row.observed_mtime_ns + ): + continue - if prepared.system_metadata is not None: - record.system_metadata = { - **(record.system_metadata or {}), - **prepared.system_metadata, - } - if prepared.mime_type is not None: - record.mime_type = prepared.mime_type + hash_applied = False + if item.stored_hash is not None and content.hash is None: + content.hash = item.stored_hash + content.size_bytes = item.stat_result.st_size + content.mtime_ns = get_mtime_ns(item.stat_result) + hash_applied = True + + if item.system_metadata is not None: + record.system_metadata = { + **(record.system_metadata or {}), + **item.system_metadata, + } + if item.mime_type is not None: + record.mime_type = item.mime_type - if prepared.hash_requested and prepared.stored_hash is None: - return False - return hash_applied or prepared.system_metadata is not None or prepared.mime_type is not None + if item.hash_requested and item.stored_hash is None: + continue + if hash_applied or item.system_metadata is not None or item.mime_type is not None: + applied_ids.append(row.record_id) + return applied_ids def enrich_assets_batch( @@ -863,48 +875,64 @@ def enrich_assets_batch( interrupt_check: Callable[[], bool] | None = None, progress: _ScanProgress | None = None, ) -> tuple[int, list[str]]: - """Enrich a batch of assets. - - Uses a single DB session for the entire batch, committing after each - individual asset to avoid long-held transactions while eliminating - per-asset session creation overhead. - - Args: - rows: List of UnenrichedReferenceRow from get_unenriched_assets_for_roots - extract_metadata: If True, extract metadata for each asset - compute_hash: If True, compute hash for each asset - interrupt_check: Optional non-blocking callable that returns True if - the operation should be interrupted (e.g. paused or cancelled) - - Returns: - Tuple of (enriched_count, failed_reference_ids) + """Prepares up to MAX_WRITE_BATCH rows on the scanner thread (stat, metadata, + optional hash), then applies them in one write transaction with a per-row + compare-and-set on the stored size/mtime; stale rows are skipped and left for + the next sync. """ enriched = 0 failed_ids: list[str] = [] - for row in rows: - if interrupt_check is not None and interrupt_check(): - break - try: - prepared = _prepare_enrichment( - row, extract_metadata, compute_hash, progress - ) + for index in range(0, len(rows), MAX_WRITE_BATCH): + prepared_list: list[_PreparedEnrichment] = [] + interrupted = False + for row in rows[index : index + MAX_WRITE_BATCH]: + if interrupt_check is not None and interrupt_check(): + interrupted = True + break + try: + prepared = _prepare_enrichment( + row, extract_metadata, compute_hash, progress + ) + except Exception as exc: + if progress is not None: + progress.enrich_failed += 1 + if progress is None or progress.mark_emitted("enrich_failed"): + emit("scanner.enrich_failed", error_type=error_type(exc)) + logging.warning("Failed to enrich %s: %s", row.file_path, exc) + failed_ids.append(row.record_id) + continue if prepared is None: failed_ids.append(row.record_id) continue - updated = run_write_txn( - lambda session: _apply_enrichment(session, prepared) - ) - if updated: - enriched += 1 + prepared_list.append(prepared) + + if prepared_list: + try: + applied = run_write_txn( + lambda session, prepared_list=prepared_list: _apply_enrichments( + session, prepared_list + ) + ) + except Exception as exc: + if progress is not None: + progress.enrich_failed += len(prepared_list) + if progress is None or progress.mark_emitted("enrich_failed"): + emit("scanner.enrich_failed", error_type=error_type(exc)) + logging.warning( + "Failed to enrich %d assets: %s", len(prepared_list), exc + ) + failed_ids.extend(item.row.record_id for item in prepared_list) else: - failed_ids.append(row.record_id) - except Exception as exc: - if progress is not None: - progress.enrich_failed += 1 - if progress is None or progress.mark_emitted("enrich_failed"): - emit("scanner.enrich_failed", error_type=error_type(exc)) - logging.warning("Failed to enrich %s: %s", row.file_path, exc) - failed_ids.append(row.record_id) + enriched += len(applied) + applied_ids = set(applied) + failed_ids.extend( + item.row.record_id + for item in prepared_list + if item.row.record_id not in applied_ids + ) + + if interrupted: + break return enriched, failed_ids diff --git a/tests-unit/assets_test/helpers.py b/tests-unit/assets_test/helpers.py index 9ffb8bdf77c..e6fd4f23b0a 100644 --- a/tests-unit/assets_test/helpers.py +++ b/tests-unit/assets_test/helpers.py @@ -88,14 +88,19 @@ def enrich_via_prepare_apply( ) -> bool: content = session.get(AssetContent, content_id) row = scanner.UnenrichedContent( - content_id, record_id, file_path, content is not None and content.hash is None + content_id, + record_id, + file_path, + content is not None and content.hash is None, + observed_size_bytes=content.size_bytes if content is not None else 0, + observed_mtime_ns=content.mtime_ns if content is not None else None, ) prepared = scanner._prepare_enrichment(row, extract_metadata, compute_hash, progress) if prepared is None: return False - updated = scanner._apply_enrichment(session, prepared) + applied = scanner._apply_enrichments(session, [prepared]) session.commit() - return updated + return bool(applied) def seed_record(session: Session, seed: RecordSeed) -> Asset: diff --git a/tests-unit/assets_test/services/test_enrichment_batching.py b/tests-unit/assets_test/services/test_enrichment_batching.py new file mode 100644 index 00000000000..f11f5d812a4 --- /dev/null +++ b/tests-unit/assets_test/services/test_enrichment_batching.py @@ -0,0 +1,146 @@ +import os +from pathlib import Path +from unittest.mock import patch + +import app.database.db as db_mod +from app.assets import scanner +from app.assets.database.models import Asset, AssetContent +from app.assets.database.queries import create_content, create_record + + +def _seed_enrichment_row( + path: Path, + *, + stored_size: int, + stored_mtime_ns: int, + stored_hash: str | None = None, +) -> scanner.UnenrichedContent: + def seed(session): + content = create_content( + session, + str(path), + size_bytes=stored_size, + mtime_ns=stored_mtime_ns, + ) + content.hash = stored_hash + record = create_record(session, content.id, path.name) + return content.id, record.id + + content_id, record_id = db_mod.run_write_txn(seed) + return scanner.UnenrichedContent( + content_id, + record_id, + str(path), + needs_hash=stored_hash is None, + observed_size_bytes=stored_size, + observed_mtime_ns=stored_mtime_ns, + ) + + +def test_enrichment_applies_when_unchanged_row_is_behind_disk( + tmp_path: Path, + session, +) -> None: + path = tmp_path / "row-behind-disk.bin" + path.write_bytes(b"current disk bytes") + disk_stat = path.stat() + row = _seed_enrichment_row( + path, + stored_size=len(b"old bytes"), + stored_mtime_ns=disk_stat.st_mtime_ns - 5_000_000_000, + ) + + enriched, failed_ids = scanner.enrich_assets_batch( + [row], extract_metadata=True, compute_hash=False + ) + + assert enriched == 1 + assert failed_ids == [] + session.expire_all() + assert session.get(Asset, row.record_id).system_metadata is not None + + +def test_enrichment_skips_row_changed_after_prepare( + tmp_path: Path, + session, + monkeypatch, +) -> None: + path = tmp_path / "row-changed-after-prepare.bin" + path.write_bytes(b"stable bytes") + disk_stat = path.stat() + row = _seed_enrichment_row( + path, + stored_size=disk_stat.st_size, + stored_mtime_ns=disk_stat.st_mtime_ns, + ) + prepared = scanner._prepare_enrichment( + row, extract_metadata=True, compute_hash=False, progress=None + ) + assert prepared is not None + changed_mtime_ns = disk_stat.st_mtime_ns + 1 + + def change_row(write_session) -> None: + content = write_session.get(AssetContent, row.content_id) + assert content is not None + content.mtime_ns = changed_mtime_ns + + db_mod.run_write_txn(change_row) + monkeypatch.setattr(scanner, "_prepare_enrichment", lambda *_args: prepared) + + enriched, failed_ids = scanner.enrich_assets_batch( + [row], extract_metadata=True, compute_hash=False + ) + + assert enriched == 0 + assert failed_ids == [row.record_id] + session.expire_all() + content = session.get(AssetContent, row.content_id) + assert content is not None + assert content.mtime_ns == changed_mtime_ns + assert content.hash is None + assert session.get(Asset, row.record_id).system_metadata is None + + +def test_root_sync_reselects_metadata_stale_after_file_changes_post_prepare( + tmp_path: Path, + session, +) -> None: + path = tmp_path / "metadata-stale-after-prepare.bin" + path.write_bytes(b"old bytes") + initial_stat = path.stat() + row = _seed_enrichment_row( + path, + stored_size=initial_stat.st_size, + stored_mtime_ns=initial_stat.st_mtime_ns, + stored_hash="sha256:stale", + ) + prepared = scanner._prepare_enrichment( + row, extract_metadata=True, compute_hash=False, progress=None + ) + assert prepared is not None + + path.write_bytes(b"new bytes") + rewritten_mtime_ns = initial_stat.st_mtime_ns + 5_000_000_000 + os.utime(path, ns=(rewritten_mtime_ns, rewritten_mtime_ns)) + + applied = db_mod.run_write_txn( + lambda write_session: scanner._apply_enrichments(write_session, [prepared]) + ) + assert applied == [row.record_id] + session.expire_all() + assert session.get(AssetContent, row.content_id).hash == "sha256:stale" + assert session.get(Asset, row.record_id).system_metadata is not None + + with patch("folder_paths.get_input_directory", return_value=str(tmp_path)): + survivors = scanner.sync_root_safely("input") + selected = scanner.get_unenriched_assets_for_roots( + ("input",), compute_hashes=False + ) + + assert survivors == {str(path.resolve())} + session.expire_all() + content = session.get(AssetContent, row.content_id) + assert content is not None + assert content.hash is None + assert session.get(Asset, row.record_id).system_metadata is None + assert row.record_id in {candidate.record_id for candidate in selected} diff --git a/tests-unit/assets_test/services/test_scanner_retry_and_counter_integrity.py b/tests-unit/assets_test/services/test_scanner_retry_and_counter_integrity.py index 9e00f484e22..8e58dc8086b 100644 --- a/tests-unit/assets_test/services/test_scanner_retry_and_counter_integrity.py +++ b/tests-unit/assets_test/services/test_scanner_retry_and_counter_integrity.py @@ -42,15 +42,49 @@ def commit(): def _fail_run_write_txn_at(real_run_write_txn, fail_index: int): - calls = {"count": -1} + calls = {"count": 0} def wrapper(work): + call_index = calls["count"] calls["count"] += 1 - if calls["count"] == fail_index: + if call_index == fail_index: raise _LOCKED_ERROR return real_run_write_txn(work) - return wrapper + return wrapper, calls + + +def _seed_enrichment_rows(tmp_path: Path, count: int) -> list[scanner.UnenrichedContent]: + paths = [tmp_path / f"row-{index}.bin" for index in range(count)] + for index, path in enumerate(paths): + path.write_bytes(f"payload-{index}".encode()) + + def seed(write_session): + rows: list[scanner.UnenrichedContent] = [] + for path in paths: + stat_result = path.stat() + content = AssetContent( + path=str(path), + hash=None, + size_bytes=stat_result.st_size, + mtime_ns=stat_result.st_mtime_ns, + ) + write_session.add(content) + write_session.flush() + record = create_record(write_session, content.id, path.name) + rows.append( + scanner.UnenrichedContent( + content.id, + record.id, + str(path), + needs_hash=True, + observed_size_bytes=content.size_bytes, + observed_mtime_ns=content.mtime_ns, + ) + ) + return rows + + return db_mod.run_write_txn(seed) @pytest.fixture(autouse=True) @@ -135,54 +169,59 @@ def deny_stat(candidate_path, *args, **kwargs): ] -def test_enrichment_counter_integrity_under_a_locked_failure_at_row_n( - db_engine, tmp_path: Path, session, monkeypatch -): - """Injecting a locked failure at the middle row of a batch must not - double-mark the surviving rows, and must match a clean run exactly.""" - paths = [tmp_path / f"row-{i}.bin" for i in range(3)] - rows = [] - for i, path in enumerate(paths): - path.write_bytes(f"payload-{i}".encode()) - stat = path.stat() - content = AssetContent( - path=str(path), hash=None, size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns - ) - session.add(content) - session.flush() - record = create_record(session, content.id, path.name) - rows.append(scanner.UnenrichedContent(content.id, record.id, str(path), True)) - session.commit() - content_ids = [row.content_id for row in rows] - +def test_enrichment_batch_failure_rolls_back_every_row_and_counts_each_failure( + tmp_path: Path, session, monkeypatch +) -> None: + rows = _seed_enrichment_rows(tmp_path, 3) real_run_write_txn = scanner.run_write_txn + failing_run_write_txn, calls = _fail_run_write_txn_at( + real_run_write_txn, fail_index=0 + ) monkeypatch.setattr( - scanner, "run_write_txn", _fail_run_write_txn_at(real_run_write_txn, fail_index=1) + scanner, "run_write_txn", failing_run_write_txn ) + progress = _ScanState() enriched, failed_ids = scanner.enrich_assets_batch( - rows, extract_metadata=False, compute_hash=True + rows, + extract_metadata=False, + compute_hash=True, + progress=progress, + ) + + assert calls["count"] == 1 + assert enriched == 0 + assert failed_ids == [row.record_id for row in rows] + assert progress.enrich_failed == 3 + session.expire_all() + assert all( + session.get(AssetContent, row.content_id).hash is None for row in rows ) - assert enriched == 2 - assert failed_ids == [rows[1].record_id] - hashes_after_failure = { - content_id: session.get(AssetContent, content_id).hash for content_id in content_ids - } - assert hashes_after_failure[content_ids[1]] is None - assert hashes_after_failure[content_ids[0]] is not None - assert hashes_after_failure[content_ids[2]] is not None +def test_enrichment_later_batch_failure_preserves_first_batch( + tmp_path: Path, session, monkeypatch +) -> None: + rows = _seed_enrichment_rows(tmp_path, 30) + real_run_write_txn = scanner.run_write_txn + failing_run_write_txn, calls = _fail_run_write_txn_at( + real_run_write_txn, fail_index=1 + ) + monkeypatch.setattr(scanner, "run_write_txn", failing_run_write_txn) - monkeypatch.setattr(scanner, "run_write_txn", real_run_write_txn) - retry_enriched, retry_failed_ids = scanner.enrich_assets_batch( - [rows[1]], extract_metadata=False, compute_hash=True + enriched, failed_ids = scanner.enrich_assets_batch( + rows, extract_metadata=False, compute_hash=True ) - assert retry_enriched == 1 - assert retry_failed_ids == [] + + assert calls["count"] == 2 + assert enriched == scanner.MAX_WRITE_BATCH + assert failed_ids == [row.record_id for row in rows[scanner.MAX_WRITE_BATCH :]] session.expire_all() - final_hashes = { - content_id: session.get(AssetContent, content_id).hash for content_id in content_ids - } - assert all(value is not None for value in final_hashes.values()) - assert len(session.scalars(scanner.sa.select(AssetContent)).all()) == 3 + assert all( + session.get(AssetContent, row.content_id).hash is not None + for row in rows[: scanner.MAX_WRITE_BATCH] + ) + assert all( + session.get(AssetContent, row.content_id).hash is None + for row in rows[scanner.MAX_WRITE_BATCH :] + ) diff --git a/tests-unit/assets_test/services/test_write_lock_hold_barriers.py b/tests-unit/assets_test/services/test_write_lock_hold_barriers.py index 6ae6449c19a..f0fef74a9d7 100644 --- a/tests-unit/assets_test/services/test_write_lock_hold_barriers.py +++ b/tests-unit/assets_test/services/test_write_lock_hold_barriers.py @@ -239,7 +239,14 @@ def _seed(session): content_id, record_id = db_mod.run_write_txn(_seed) - row = scanner.UnenrichedContent(content_id, record_id, str(path), True) + row = scanner.UnenrichedContent( + content_id, + record_id, + str(path), + True, + observed_size_bytes=stat.st_size, + observed_mtime_ns=stat.st_mtime_ns, + ) entered = threading.Event() release = threading.Event() @@ -286,7 +293,14 @@ def _seed(session): content_id, record_id = db_mod.run_write_txn(_seed) - row = scanner.UnenrichedContent(content_id, record_id, str(path), False) + row = scanner.UnenrichedContent( + content_id, + record_id, + str(path), + False, + observed_size_bytes=stat.st_size, + observed_mtime_ns=stat.st_mtime_ns, + ) entered = threading.Event() release = threading.Event() @@ -317,6 +331,61 @@ def _enrich() -> None: assert result["outcome"] == (1, []) +def test_enrichment_apply_does_not_stat_inside_write_transaction( + file_database, tmp_path, monkeypatch +): + path = tmp_path / "enrich-without-in-transaction-stat.bin" + path.write_bytes(b"prepared before the write") + stat_result = path.stat() + + def seed(session): + content = create_content( + session, + str(path), + size_bytes=stat_result.st_size, + mtime_ns=stat_result.st_mtime_ns, + ) + record = create_record(session, content.id, path.name) + return content.id, record.id + + content_id, record_id = db_mod.run_write_txn(seed) + row = scanner.UnenrichedContent( + content_id, + record_id, + str(path), + observed_size_bytes=stat_result.st_size, + observed_mtime_ns=stat_result.st_mtime_ns, + ) + real_run_write_txn = scanner.run_write_txn + real_stat = scanner.os.stat + inside = False + + def track_write_transaction(work): + def tracked_work(session): + nonlocal inside + inside = True + try: + return work(session) + finally: + inside = False + + return real_run_write_txn(tracked_work) + + def reject_in_transaction_stat(*args, **kwargs): + if inside: + raise AssertionError("enrichment stat ran inside the write transaction") + return real_stat(*args, **kwargs) + + monkeypatch.setattr(scanner, "run_write_txn", track_write_transaction) + monkeypatch.setattr(scanner.os, "stat", reject_in_transaction_stat) + + result = scanner.enrich_assets_batch( + [row], extract_metadata=True, compute_hash=False + ) + + assert result == (1, []) + + def test_scanner_reference_stat_walk_does_not_hold_the_write_lock( file_database, impatient_probe, tmp_path, monkeypatch ): diff --git a/tests-unit/assets_test/test_scanner_ingest_event_log.py b/tests-unit/assets_test/test_scanner_ingest_event_log.py index 770459710b4..cf4aafc0b97 100644 --- a/tests-unit/assets_test/test_scanner_ingest_event_log.py +++ b/tests-unit/assets_test/test_scanner_ingest_event_log.py @@ -63,7 +63,12 @@ def tagged_lines(caplog: pytest.LogCaptureFixture) -> list[str]: def hash_session(path: Path) -> Mock: stat_result = path.stat() - content = SimpleNamespace(hash=None, mtime_ns=stat_result.st_mtime_ns) + content = SimpleNamespace( + hash=None, + size_bytes=stat_result.st_size, + mtime_ns=stat_result.st_mtime_ns, + is_missing=False, + ) record = SimpleNamespace(system_metadata=None, mime_type=None) session = Mock() session.get.side_effect = lambda _model, row_id: content if row_id == "content" else record diff --git a/tests-unit/assets_test/test_write_contention_e2e.py b/tests-unit/assets_test/test_write_contention_e2e.py index 1f313634227..705fed2a9bc 100644 --- a/tests-unit/assets_test/test_write_contention_e2e.py +++ b/tests-unit/assets_test/test_write_contention_e2e.py @@ -52,7 +52,7 @@ def test_register_executed_output_keeps_job_id_during_scanner_write_train( Image.new("RGB", (1, 1), (255, 0, 0)).save(output_path) scanner_rows: list[scanner.UnenrichedContent] = [] - for index in range(4): + for index in range(30): scanner_path = output_directory / f"scanner-{index}.bin" scanner_path.write_bytes(f"scanner-{index}".encode()) stat_result = scanner_path.stat() @@ -71,6 +71,8 @@ def seed(session, path=scanner_path, stat=stat_result) -> None: record.id, str(path), needs_hash=True, + observed_size_bytes=stat.st_size, + observed_mtime_ns=stat.st_mtime_ns, ) ) @@ -80,7 +82,7 @@ def seed(session, path=scanner_path, stat=stat_result) -> None: release_first_scanner_write = threading.Event() second_scanner_write_committed = threading.Event() registration_started = threading.Event() - original_apply = scanner._apply_enrichment + original_apply = scanner._apply_enrichments original_run_write_txn = scanner.run_write_txn original_is_retryable_lock_error = db_mod._is_retryable_lock_error scanner_writes = 0 @@ -88,15 +90,15 @@ def seed(session, path=scanner_path, stat=stat_result) -> None: registration_thread_id: list[int | None] = [None] registration_blocked = threading.Event() - def block_first_scanner_write(session, prepared): + def block_first_scanner_write(session, prepared_list): nonlocal scanner_writes - updated = original_apply(session, prepared) + applied = original_apply(session, prepared_list) with scanner_writes_lock: is_first_write = scanner_writes == 0 if is_first_write: first_scanner_write_entered.set() assert release_first_scanner_write.wait(timeout=5) - return updated + return applied def count_scanner_writes(work): nonlocal scanner_writes @@ -113,7 +115,7 @@ def observe_registration_lock(error): registration_blocked.set() return is_retryable - monkeypatch.setattr(scanner, "_apply_enrichment", block_first_scanner_write) + monkeypatch.setattr(scanner, "_apply_enrichments", block_first_scanner_write) monkeypatch.setattr(scanner, "run_write_txn", count_scanner_writes) monkeypatch.setattr(db_mod, "_is_retryable_lock_error", observe_registration_lock) scanner_result: dict[str, tuple[int, list[str]]] = {} @@ -153,7 +155,7 @@ def register_output() -> None: assert not scanner_worker.is_alive() assert not registration_worker.is_alive() assert scanner_result["value"] == (len(scanner_rows), []) - assert scanner_writes == len(scanner_rows) + assert scanner_writes == 2 result = registration_result["value"] assert isinstance(result, RegisteredAsset) with db_mod.create_session() as session: From 6c59da56450d20f3475144c432340c890b4f1e49 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 17:38:00 -0700 Subject: [PATCH 041/111] fix(assets): verify content from the preflight snapshot without re-stating under the write lock --- app/assets/scanner_admission.py | 8 +- app/assets/scanner_changes.py | 28 +- app/assets/seeder.py | 12 +- app/assets/services/hash_mode_state.py | 91 +++--- .../services/test_admission_gate.py | 32 +++ .../services/test_drain_preflight.py | 261 ++++++++++++++++++ .../test_queue_drain_characterization.py | 6 + .../services/test_transition_drain.py | 11 + .../test_write_session_convention.py | 2 + tests-unit/seeder_test/test_seeder.py | 86 +++++- 10 files changed, 463 insertions(+), 74 deletions(-) create mode 100644 tests-unit/assets_test/services/test_drain_preflight.py diff --git a/app/assets/scanner_admission.py b/app/assets/scanner_admission.py index b355414a280..987e3d24f9e 100644 --- a/app/assets/scanner_admission.py +++ b/app/assets/scanner_admission.py @@ -11,6 +11,7 @@ import mimetypes import os import time +from collections.abc import Callable from dataclasses import dataclass from typing import Final @@ -66,11 +67,16 @@ def _two_stat_admit(paths_with_stats: list[tuple[str, os.stat_result]]) -> tuple return admitted, watched -def tick_watch_list(_session: Session | None = None) -> None: +def tick_watch_list( + _session: Session | None = None, + interrupt_check: Callable[[], bool] | None = None, +) -> None: from app.assets.scanner import SeedAssetSpec, insert_asset_specs queued_count = len(_WATCH_LIST) for _ in range(queued_count): + if interrupt_check and interrupt_check(): + break entry = _WATCH_LIST[0] try: current = os.stat(entry.path) diff --git a/app/assets/scanner_changes.py b/app/assets/scanner_changes.py index 638c887a605..819a089f36c 100644 --- a/app/assets/scanner_changes.py +++ b/app/assets/scanner_changes.py @@ -8,6 +8,7 @@ from __future__ import annotations import os +from collections.abc import Callable from pathlib import Path from typing import Literal, NamedTuple @@ -249,26 +250,11 @@ def _apply_pending_verification( return "drop" assert preflight.path is not None if preflight.outcome == "gone": - try: - os.stat(preflight.path, follow_symlinks=True) - except FileNotFoundError: - mark_content_missing(session, content.id) - return "processed" - except OSError: - return "retry" - return "retry" + mark_content_missing(session, content.id) + return "processed" if preflight.outcome == "retry" or snapshot is None: return "retry" digest, verified_stat = snapshot - try: - current_stat = os.stat(preflight.path, follow_symlinks=True) - except OSError: - return "retry" - if ( - current_stat.st_size != verified_stat.st_size - or current_stat.st_mtime_ns != verified_stat.st_mtime_ns - ): - return "retry" stored_hash = to_stored_hash(digest) if content.hash == stored_hash or content.hash is None: content.hash = stored_hash @@ -279,10 +265,16 @@ def _apply_pending_verification( return "processed" -def drain_pending_verifications(_session: Session | None = None, limit: int | None = None) -> int: +def drain_pending_verifications( + _session: Session | None = None, + limit: int | None = None, + interrupt_check: Callable[[], bool] | None = None, +) -> int: queued_count = min(len(_pending_verification_ids), limit or len(_pending_verification_ids)) processed = 0 for _ in range(queued_count): + if interrupt_check and interrupt_check(): + break content_id = _pending_verification_ids[0] preflight = _preflight_pending_verification(content_id) snapshot: tuple[str, os.stat_result] | None = None diff --git a/app/assets/seeder.py b/app/assets/seeder.py index 3f717a0b36f..16cc1d27a55 100644 --- a/app/assets/seeder.py +++ b/app/assets/seeder.py @@ -894,7 +894,11 @@ def _run_fast_phase(self, roots: tuple[RootType, ...]) -> tuple[int, int, int]: last_progress_time = now self._update_progress(scanned=len(specs), created=total_created) - tick_watch_list() + tick_watch_list( + interrupt_check=lambda: self._is_paused_or_cancelled( + _ScanStage.FAST_SCAN + ) + ) logging.info( "Fast scan complete: %.3fs total (created=%d, skipped=%d, total_paths=%d)", time.perf_counter() - t_fast_start, @@ -912,10 +916,10 @@ def _run_enrich_phase(self, roots: tuple[RootType, ...]) -> tuple[bool, int]: """ total_enriched = 0 scan_state = self._scan_state - drain_pending_verifications() - tick_watch_list() + drain_pending_verifications(interrupt_check=self._is_paused_or_cancelled) + tick_watch_list(interrupt_check=self._is_paused_or_cancelled) for _ in range(3): - drain_transition_queue() + drain_transition_queue(interrupt_check=self._is_paused_or_cancelled) if pending_transition_count() == 0: break batch_size = 100 diff --git a/app/assets/services/hash_mode_state.py b/app/assets/services/hash_mode_state.py index e5d7760ef9e..476d8c32f31 100644 --- a/app/assets/services/hash_mode_state.py +++ b/app/assets/services/hash_mode_state.py @@ -10,6 +10,7 @@ import logging import os +from collections.abc import Callable from collections import deque from dataclasses import dataclass from typing import Final @@ -23,7 +24,7 @@ from app.assets.helpers import to_stored_hash from app.assets.services.path_utils import compute_loader_path, get_name_and_tags_from_asset_path from app.assets.services.snapshot_hash import snapshot_hash -from app.database.db import run_write_txn +from app.database.db import create_session, run_write_txn _KEY = "hash_mode" _MAX_VERIFY_ATTEMPTS: Final = 3 @@ -95,42 +96,63 @@ def enqueue_transition_work(session: Session, transition: str | None) -> None: _PENDING_PATHS.add(row.path) -def drain_transition_queue(_session: Session | None = None) -> None: +def _preflight_transition_entry(path: str) -> tuple[str, int, int | None] | None: + with create_session() as session: + content = session.scalars( + select(AssetContent).where( + AssetContent.path == path, + AssetContent.is_missing.is_(False), + ) + ).first() + if content is None: + return None + return content.id, content.size_bytes, content.mtime_ns + + +def drain_transition_queue( + _session: Session | None = None, + interrupt_check: Callable[[], bool] | None = None, +) -> None: global _off_to_on_transition_in_flight pending_count = len(_PENDING_QUEUE) for _ in range(pending_count): + if interrupt_check and interrupt_check(): + break entry = _PENDING_QUEUE[0] + preflight = _preflight_transition_entry(entry.path) snapshot: tuple[str, os.stat_result] | None = None - preparation = "ready" - try: - snapshot = snapshot_hash(entry.path) - except OSError: - preparation = "retry" - if snapshot is None and preparation != "retry": + preparation = "drop" if preflight is None else "ready" + if preflight is not None: try: - os.stat(entry.path) - except FileNotFoundError: - preparation = "gone" + snapshot = snapshot_hash(entry.path) except OSError: preparation = "retry" - else: - preparation = "retry" + if snapshot is None and preparation != "retry": + try: + os.stat(entry.path) + except FileNotFoundError: + preparation = "gone" + except OSError: + preparation = "retry" + else: + preparation = "retry" def _apply(session: Session) -> str: + if preflight is None: + return "drop" + content_id, size_bytes, mtime_ns = preflight + content = session.get(AssetContent, content_id) + if content is None or content.is_missing or content.path != entry.path: + return "drop" + if content.size_bytes != size_bytes or content.mtime_ns != mtime_ns: + return "retry" if preparation == "retry" or ( snapshot is None and preparation != "gone" ): if entry.ticks + 1 < _MAX_VERIFY_ATTEMPTS: return "retry" - content = session.scalars( - select(AssetContent).where( - AssetContent.path == entry.path, - AssetContent.is_missing.is_(False), - ) - ).first() - if content is not None: - content.hash = None + content.hash = None logging.warning( "Could not verify %s in %d attempts; clearing its stored hash so the hash-mode " "transition can complete", @@ -138,33 +160,10 @@ def _apply(session: Session) -> str: _MAX_VERIFY_ATTEMPTS, ) return "drop" - content = session.scalars( - select(AssetContent).where( - AssetContent.path == entry.path, - AssetContent.is_missing.is_(False), - ) - ).first() - if content is None: - return "drop" if preparation == "gone": - try: - os.stat(entry.path) - except FileNotFoundError: - mark_content_missing(session, content.id) - return "drop" - except OSError: - return "retry" - return "retry" + mark_content_missing(session, content.id) + return "drop" digest, stat = snapshot - try: - current_stat = os.stat(entry.path) - except OSError: - return "retry" - if ( - current_stat.st_size != stat.st_size - or current_stat.st_mtime_ns != stat.st_mtime_ns - ): - return "retry" stored_hash = to_stored_hash(digest) if content.hash is None: content.hash = stored_hash diff --git a/tests-unit/assets_test/services/test_admission_gate.py b/tests-unit/assets_test/services/test_admission_gate.py index a34ae9c5990..a7fca074f78 100644 --- a/tests-unit/assets_test/services/test_admission_gate.py +++ b/tests-unit/assets_test/services/test_admission_gate.py @@ -1,4 +1,5 @@ import os +import threading from collections.abc import Iterator from pathlib import Path from unittest.mock import patch @@ -168,3 +169,34 @@ def test_nonempty_candidate_batch_still_pays_stability_gap(temp_dir: Path, monke assert sleeps == [0.1] assert admitted == [str(path)] assert watched == [] + + +def test_watch_list_interrupts_between_entries(temp_dir: Path, monkeypatch): + paths = [temp_dir / f"watched-{index}.bin" for index in range(3)] + for path in paths: + path.write_bytes(path.name.encode()) + _WATCH_LIST[:] = [_WatchEntry(str(path), path.stat()) for path in paths] + interrupted = threading.Event() + inserted: list[str] = [] + + def record_insert(specs, _tags) -> int: + inserted.append(specs[0]["abs_path"]) + interrupted.set() + return 1 + + monkeypatch.setattr("app.assets.scanner.insert_asset_specs", record_insert) + monkeypatch.setattr( + scanner_admission, + "get_name_and_tags_from_asset_path", + lambda path: (Path(path).name, []), + ) + monkeypatch.setattr( + scanner_admission, + "compute_loader_path", + lambda path: Path(path).name, + ) + + tick_watch_list(interrupt_check=interrupted.is_set) + + assert inserted == [str(paths[0])] + assert [entry.path for entry in _WATCH_LIST] == [str(path) for path in paths[1:]] diff --git a/tests-unit/assets_test/services/test_drain_preflight.py b/tests-unit/assets_test/services/test_drain_preflight.py new file mode 100644 index 00000000000..a111613bf92 --- /dev/null +++ b/tests-unit/assets_test/services/test_drain_preflight.py @@ -0,0 +1,261 @@ +from pathlib import Path + +import pytest + +import app.database.db as db_mod +from app.assets import scanner_changes +from app.assets.database.models import AssetContent +from app.assets.database.queries.records import create_content +from app.assets.services import hash_mode_state + + +class _StatInsideWriteTxn(AssertionError): + pass + + +@pytest.fixture(autouse=True) +def clear_drain_queues(): + scanner_changes.clear_pending_verifications() + hash_mode_state.clear_transition_queue() + yield + scanner_changes.clear_pending_verifications() + hash_mode_state.clear_transition_queue() + + +def _seed_content(path: Path) -> str: + stat = path.stat() + return db_mod.run_write_txn( + lambda session: create_content( + session, + str(path), + size_bytes=stat.st_size, + mtime_ns=stat.st_mtime_ns, + ).id + ) + + +def _guard_stat_during_write_txn(module, monkeypatch: pytest.MonkeyPatch) -> None: + inside = False + real_run_write_txn = module.run_write_txn + real_stat = module.os.stat + + def guarded_stat(*args, **kwargs): + if inside: + raise _StatInsideWriteTxn + return real_stat(*args, **kwargs) + + def tracked_run_write_txn(work): + def tracked_work(session): + nonlocal inside + inside = True + try: + return work(session) + finally: + inside = False + + return real_run_write_txn(tracked_work) + + monkeypatch.setattr(module.os, "stat", guarded_stat) + monkeypatch.setattr(module, "run_write_txn", tracked_run_write_txn) + + +def test_pending_verification_does_not_stat_inside_write_transaction( + session, temp_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = temp_dir / "pending-ready.bin" + path.write_bytes(b"ready") + content_id = _seed_content(path) + scanner_changes.queue_pending_verification(content_id) + _guard_stat_during_write_txn(scanner_changes, monkeypatch) + + processed = scanner_changes.drain_pending_verifications() + + assert processed == 1 + + +def test_pending_verification_trusts_gone_preflight_without_statting_in_transaction( + session, temp_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = temp_dir / "pending-gone.bin" + path.write_bytes(b"gone") + content_id = _seed_content(path) + scanner_changes.queue_pending_verification(content_id) + path.unlink() + _guard_stat_during_write_txn(scanner_changes, monkeypatch) + + processed = scanner_changes.drain_pending_verifications() + session.expire_all() + + assert processed == 1 + assert session.get(AssetContent, content_id).is_missing is True + + +def test_pending_verification_drops_when_row_changes_after_preflight( + session, temp_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = temp_dir / "pending-row-changed.bin" + path.write_bytes(b"row changes") + content_id = _seed_content(path) + scanner_changes.queue_pending_verification(content_id) + changed_mtime_ns = path.stat().st_mtime_ns + 1 + real_snapshot_hash = scanner_changes.snapshot_hash + + def mutate_row_during_hash(candidate_path: str): + snapshot = real_snapshot_hash(candidate_path) + + def mutate_row(write_session): + write_session.get(AssetContent, content_id).mtime_ns = changed_mtime_ns + + db_mod.run_write_txn(mutate_row) + return snapshot + + monkeypatch.setattr(scanner_changes, "snapshot_hash", mutate_row_during_hash) + + processed = scanner_changes.drain_pending_verifications() + session.expire_all() + content = session.get(AssetContent, content_id) + + assert processed == 0 + assert content.hash is None + assert content.mtime_ns == changed_mtime_ns + assert scanner_changes._pending_verification_ids == [] + + +def test_pending_verification_interrupts_between_entries( + session, temp_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + content_ids: list[str] = [] + for index in range(3): + path = temp_dir / f"pending-{index}.bin" + path.write_bytes(f"pending-{index}".encode()) + content_id = _seed_content(path) + content_ids.append(content_id) + scanner_changes.queue_pending_verification(content_id) + + transaction_count = 0 + real_run_write_txn = scanner_changes.run_write_txn + + def count_run_write_txn(work): + nonlocal transaction_count + result = real_run_write_txn(work) + transaction_count += 1 + return result + + monkeypatch.setattr(scanner_changes, "run_write_txn", count_run_write_txn) + + processed = scanner_changes.drain_pending_verifications( + interrupt_check=lambda: transaction_count == 1 + ) + + assert processed == 1 + assert transaction_count == 1 + assert scanner_changes._pending_verification_ids == content_ids[1:] + + +def test_transition_drain_does_not_stat_inside_write_transaction( + session, temp_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = temp_dir / "transition-ready.bin" + path.write_bytes(b"ready") + _seed_content(path) + hash_mode_state._PENDING_QUEUE.append(hash_mode_state._PendingEntry(str(path))) + hash_mode_state._PENDING_PATHS.add(str(path)) + _guard_stat_during_write_txn(hash_mode_state, monkeypatch) + + hash_mode_state.drain_transition_queue() + + assert hash_mode_state.pending_transition_count() == 0 + + +def test_transition_drain_trusts_gone_preflight_without_statting_in_transaction( + session, temp_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = temp_dir / "transition-gone.bin" + path.write_bytes(b"gone") + content_id = _seed_content(path) + hash_mode_state._PENDING_QUEUE.append(hash_mode_state._PendingEntry(str(path))) + hash_mode_state._PENDING_PATHS.add(str(path)) + path.unlink() + _guard_stat_during_write_txn(hash_mode_state, monkeypatch) + + hash_mode_state.drain_transition_queue() + session.expire_all() + + assert session.get(AssetContent, content_id).is_missing is True + assert hash_mode_state.pending_transition_count() == 0 + + +def test_transition_drain_retries_when_row_changes_during_hashing( + session, temp_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = temp_dir / "transition-row-changed.bin" + path.write_bytes(b"row changes") + content_id = _seed_content(path) + changed_mtime_ns = path.stat().st_mtime_ns + 1 + hash_mode_state._PENDING_QUEUE.append(hash_mode_state._PendingEntry(str(path))) + hash_mode_state._PENDING_PATHS.add(str(path)) + real_snapshot_hash = hash_mode_state.snapshot_hash + + def mutate_row_during_hash(candidate_path: str): + snapshot = real_snapshot_hash(candidate_path) + + def mutate_row(write_session): + write_session.get(AssetContent, content_id).mtime_ns = changed_mtime_ns + + db_mod.run_write_txn(mutate_row) + return snapshot + + monkeypatch.setattr(hash_mode_state, "snapshot_hash", mutate_row_during_hash) + + hash_mode_state.drain_transition_queue() + session.expire_all() + content = session.get(AssetContent, content_id) + + assert content.hash is None + assert content.mtime_ns == changed_mtime_ns + assert list(hash_mode_state._PENDING_QUEUE) == [ + hash_mode_state._PendingEntry(str(path), ticks=1) + ] + + +def test_transition_drain_interrupts_then_resumes_before_completing_mode( + session, temp_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for index in range(3): + path = temp_dir / f"transition-{index}.bin" + path.write_bytes(f"transition-{index}".encode()) + _seed_content(path) + db_mod.run_write_txn( + lambda write_session: hash_mode_state.write_stored_mode(write_session, "off") + ) + hash_mode_state.enqueue_transition_work(session, "off_to_on") + + transaction_count = 0 + real_run_write_txn = hash_mode_state.run_write_txn + + def count_run_write_txn(work): + nonlocal transaction_count + result = real_run_write_txn(work) + transaction_count += 1 + return result + + monkeypatch.setattr(hash_mode_state, "run_write_txn", count_run_write_txn) + + hash_mode_state.drain_transition_queue( + interrupt_check=lambda: transaction_count == 1 + ) + session.expire_all() + + assert transaction_count == 1 + assert hash_mode_state.pending_transition_count() == 2 + assert hash_mode_state._off_to_on_transition_in_flight is True + assert hash_mode_state.read_stored_mode(session) == "off" + + monkeypatch.setattr(hash_mode_state, "run_write_txn", real_run_write_txn) + hash_mode_state.drain_transition_queue() + session.expire_all() + + assert hash_mode_state.pending_transition_count() == 0 + assert hash_mode_state._PENDING_PATHS == set() + assert hash_mode_state._off_to_on_transition_in_flight is False + assert hash_mode_state.read_stored_mode(session) == "on" diff --git a/tests-unit/assets_test/services/test_queue_drain_characterization.py b/tests-unit/assets_test/services/test_queue_drain_characterization.py index 64a893fed4c..95fce722ca3 100644 --- a/tests-unit/assets_test/services/test_queue_drain_characterization.py +++ b/tests-unit/assets_test/services/test_queue_drain_characterization.py @@ -2,6 +2,7 @@ import pytest +import app.database.db as db_mod from app.assets import scanner_admission from app.assets import scanner_changes from app.assets.database.models import AssetContent @@ -81,6 +82,11 @@ def test_transition_queue_retries_without_losing_companion_path( session, monkeypatch ): path = "/unreadable/transition.bin" + db_mod.run_write_txn( + lambda write_session: write_session.add( + AssetContent(path=path, hash=None, size_bytes=0, mtime_ns=None) + ) + ) entry = _PendingEntry(path) _PENDING_QUEUE.append(entry) _PENDING_PATHS.add(path) diff --git a/tests-unit/assets_test/services/test_transition_drain.py b/tests-unit/assets_test/services/test_transition_drain.py index 352cc4b3f8d..cdfc4ef24d4 100644 --- a/tests-unit/assets_test/services/test_transition_drain.py +++ b/tests-unit/assets_test/services/test_transition_drain.py @@ -48,6 +48,7 @@ def test_off_to_on_transition_hashes_null_rows_and_persists_mode(session, temp_d transition = record_transition_intent(session) enqueue_transition_work(session, transition) + session.commit() drain_transition_queue(session) session.commit() @@ -72,6 +73,7 @@ def test_transition_drain_splits_changed_content(session, temp_dir, monkeypatch) path.write_bytes(b"new bytes") enqueue_transition_work(session, "off_to_on") + session.commit() drain_transition_queue(session) session.commit() @@ -107,6 +109,7 @@ def test_transition_drain_serves_unchanged_content_whose_stored_stat_went_stale( transition = record_transition_intent(session) enqueue_transition_work(session, transition) + session.commit() drain_transition_queue(session) session.commit() @@ -143,6 +146,7 @@ def hash_or_raise(candidate_path: str): monkeypatch.setattr(hash_mode_state, "snapshot_hash", hash_or_raise) transition = record_transition_intent(session) enqueue_transition_work(session, transition) + session.commit() drain_transition_queue(session) @@ -172,6 +176,7 @@ def test_transition_drain_marks_deleted_path_missing_and_completes_transition( transition = record_transition_intent(session) enqueue_transition_work(session, transition) + session.commit() drain_transition_queue(session) session.commit() session.expire_all() @@ -211,6 +216,7 @@ def flaky_stat(candidate_path, *args, **kwargs): transition = record_transition_intent(session) enqueue_transition_work(session, transition) + session.commit() drain_transition_queue(session) session.commit() @@ -236,6 +242,7 @@ def test_transition_drain_requeues_unstable_present_file_without_marking_it_miss transition = record_transition_intent(session) enqueue_transition_work(session, transition) + session.commit() drain_transition_queue(session) session.commit() @@ -267,6 +274,7 @@ def test_transition_drain_mixes_a_deleted_path_with_a_healthy_one(session, temp_ transition = record_transition_intent(session) enqueue_transition_work(session, transition) + session.commit() drain_transition_queue(session) session.commit() session.expire_all() @@ -300,6 +308,7 @@ def test_transition_drain_skips_out_of_root_path(session, temp_dir, monkeypatch, get_name_and_tags_from_asset_path(str(outside_path)) enqueue_transition_work(session, "off_to_on") + session.commit() with caplog.at_level(logging.WARNING): try: @@ -340,6 +349,7 @@ def always_denied(_candidate_path: str): transition = record_transition_intent(session) enqueue_transition_work(session, transition) + session.commit() def warnings_naming_the_path() -> list[str]: return [r.getMessage() for r in caplog.records if str(path) in r.getMessage()] @@ -410,6 +420,7 @@ def denied_for_the_unreadable_path(candidate_path: str): transition = record_transition_intent(session) enqueue_transition_work(session, transition) + session.commit() for _ in range(3): drain_transition_queue(session) session.commit() diff --git a/tests-unit/assets_test/test_write_session_convention.py b/tests-unit/assets_test/test_write_session_convention.py index 65bef18c028..39b0ea06615 100644 --- a/tests-unit/assets_test/test_write_session_convention.py +++ b/tests-unit/assets_test/test_write_session_convention.py @@ -54,6 +54,8 @@ class CreateSessionCallSite(NamedTuple): CreateSessionCallSite("app/assets/services/asset_management.py", "resolve_asset_for_download"), CreateSessionCallSite("app/assets/services/asset_management.py", "asset_exists"), CreateSessionCallSite("app/assets/services/asset_management.py", "get_preview_file_paths"), + # This reads the transition row before hashing starts outside the writer lease. + CreateSessionCallSite("app/assets/services/hash_mode_state.py", "_preflight_transition_entry"), # These preflights read decision facts before metadata or hash I/O outside the writer lease. CreateSessionCallSite("app/assets/services/ingest.py", "_preflight_upload_record"), CreateSessionCallSite("app/assets/services/ingest.py", "_preflight_settle_target"), diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.py index 50ab7af0c58..9c0fbf71267 100644 --- a/tests-unit/seeder_test/test_seeder.py +++ b/tests-unit/seeder_test/test_seeder.py @@ -151,9 +151,11 @@ def test_enrich_phase_does_not_count_returned_ids_as_failures( [], ) ) - monkeypatch.setattr(seeder_module, "drain_pending_verifications", lambda: None) - monkeypatch.setattr(seeder_module, "tick_watch_list", lambda: None) - monkeypatch.setattr(seeder_module, "drain_transition_queue", lambda: None) + monkeypatch.setattr( + seeder_module, "drain_pending_verifications", lambda **_kwargs: None + ) + monkeypatch.setattr(seeder_module, "tick_watch_list", lambda **_kwargs: None) + monkeypatch.setattr(seeder_module, "drain_transition_queue", lambda **_kwargs: None) monkeypatch.setattr( seeder_module, "get_unenriched_assets_for_roots", @@ -405,6 +407,80 @@ def capture_root_interrupt( assert scan_seeder._scan_state.cancel_stage == _ScanStage.FAST_SCAN.value +def test_fast_watch_interrupt_records_the_fast_scan_cancellation_stage( + scan_seeder: _AssetSeeder, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _WatchInterruptCaptured(Exception): + pass + + captured_interrupt = None + + monkeypatch.setattr( + seeder_module, + "sync_root_safely", + lambda _root, _progress, interrupt_check=None: set(), + ) + monkeypatch.setattr(seeder_module, "collect_paths_for_roots", lambda _roots: []) + monkeypatch.setattr( + seeder_module, + "build_asset_specs", + lambda *_args, **_kwargs: ([], {}, 0), + ) + monkeypatch.setattr(scan_seeder, "_check_pause_and_cancel", lambda _stage: False) + + def capture_watch_interrupt(*, interrupt_check=None) -> None: + nonlocal captured_interrupt + captured_interrupt = interrupt_check + raise _WatchInterruptCaptured + + monkeypatch.setattr(seeder_module, "tick_watch_list", capture_watch_interrupt) + + with pytest.raises(_WatchInterruptCaptured): + scan_seeder._run_fast_phase(("models",)) + + scan_seeder._cancel_event.set() + assert captured_interrupt is not None + assert captured_interrupt() is True + assert scan_seeder._scan_state is not None + assert scan_seeder._scan_state.cancel_stage == _ScanStage.FAST_SCAN.value + + +def test_enrich_drains_receive_the_enrich_interrupt_predicate( + scan_seeder: _AssetSeeder, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _DrainInterruptsCaptured(Exception): + pass + + captured_interrupts = [] + + def capture_interrupt(*, interrupt_check=None) -> None: + captured_interrupts.append(interrupt_check) + + def capture_transition_interrupt(*, interrupt_check=None) -> None: + captured_interrupts.append(interrupt_check) + raise _DrainInterruptsCaptured + + monkeypatch.setattr( + seeder_module, "drain_pending_verifications", capture_interrupt + ) + monkeypatch.setattr(seeder_module, "tick_watch_list", capture_interrupt) + monkeypatch.setattr( + seeder_module, "drain_transition_queue", capture_transition_interrupt + ) + + with pytest.raises(_DrainInterruptsCaptured): + scan_seeder._run_enrich_phase(("models",)) + + scan_seeder._cancel_event.set() + assert len(captured_interrupts) == 3 + assert all(interrupt_check is not None for interrupt_check in captured_interrupts) + assert all(interrupt_check() is True for interrupt_check in captured_interrupts) + assert scan_seeder._scan_state is not None + assert scan_seeder._scan_state.cancel_stage == _ScanStage.ENRICH.value + + def test_prune_interrupt_records_the_pruning_cancellation_stage( scan_seeder: _AssetSeeder, monkeypatch: pytest.MonkeyPatch, @@ -539,7 +615,7 @@ def fail_insert(batch, batch_tags) -> int: raise PermissionError("/private/models/asset.safetensors") monkeypatch.setattr(seeder_module, "insert_asset_specs", fail_insert) - monkeypatch.setattr(seeder_module, "tick_watch_list", lambda: None) + monkeypatch.setattr(seeder_module, "tick_watch_list", lambda **_kwargs: None) with caplog.at_level(logging.INFO): scan_seeder._run_fast_phase(("models",)) @@ -573,7 +649,7 @@ def test_fast_phase_seeds_in_bounded_batches_it_can_park_between( 0, ), ) - monkeypatch.setattr(seeder_module, "tick_watch_list", lambda: None) + monkeypatch.setattr(seeder_module, "tick_watch_list", lambda **_kwargs: None) batch_sizes: list[int] = [] first_batch_written = threading.Event() From d4f64f75580d539f1e21978e039e51cd99a51704 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 18:14:08 -0700 Subject: [PATCH 042/111] fix(assets): compare rows instead of re-stating files inside upload and registration transactions --- app/assets/lifecycle.py | 1 - app/assets/services/ingest.py | 191 ++--- .../assets_test/services/test_from_hash.py | 16 +- .../services/test_ingest_write_txn.py | 663 +++++++++++++++++- .../services/test_lifecycle_write_txn.py | 33 + .../services/test_write_lock_hold_barriers.py | 157 +++++ .../test_write_session_convention.py | 3 +- 7 files changed, 957 insertions(+), 107 deletions(-) diff --git a/app/assets/lifecycle.py b/app/assets/lifecycle.py index 519b06ea827..988c10a6d21 100644 --- a/app/assets/lifecycle.py +++ b/app/assets/lifecycle.py @@ -41,7 +41,6 @@ def record_hash_mode_transition_intent() -> None: def enqueue_mode_transition_work() -> None: with create_session() as session: enqueue_transition_work(session, _hash_mode_transition) - session.commit() def wipe_temp_db_rows(session) -> tuple[int, int]: diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index 59e2fd1b0f1..c2a8cefda70 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -33,7 +33,6 @@ from app.assets.services.lookup import ( claim_qualified_content, lookup_for_view, - refresh_qualified_content, ) from app.assets.services.metadata_extract import extract_file_metadata from app.assets.services.path_utils import ( @@ -341,6 +340,7 @@ def _reconcile_live_content_at_path( and existing.size_bytes == facts.size_bytes ): existing.hash = facts.stored_hash + existing.size_bytes = facts.size_bytes existing.mtime_ns = facts.mtime_ns session.flush() return @@ -382,6 +382,8 @@ class _UploadRecordPreflight(NamedTuple): content_id: str | None stored_hash: str | None path: str + row_size_bytes: int | None + row_mtime_ns: int | None signature: _FileSignature spec: _UploadRecordSpec @@ -412,6 +414,8 @@ class _CachedRegistrationPreflight(NamedTuple): content_id: str sibling_id: str | None sibling_metadata: dict[str, Any] | None + row_size_bytes: int + row_mtime_ns: int | None signature: _FileSignature | None @@ -424,13 +428,6 @@ def _file_signature(path: str) -> _FileSignature: return _FileSignature(path, stat_result.st_size, get_mtime_ns(stat_result)) -def _file_signature_matches(signature: _FileSignature) -> bool: - try: - return _file_signature(signature.path) == signature - except OSError: - return False - - def _preflight_upload_record( stored_hash: str | None, fallback_path: str | None, @@ -444,18 +441,30 @@ def _preflight_upload_record( return None content_id = None path = fallback_path + row_size_bytes = None + row_mtime_ns = None else: content_id = content.id path = content.path + row_size_bytes = content.size_bytes + row_mtime_ns = content.mtime_ns if spec.preview_id is not None and session.get(Asset, spec.preview_id) is None: raise ValueError( f"preview_id {spec.preview_id!r} does not reference an existing asset" ) + signature = _file_signature(path) + if content_id is not None and ( + signature.size_bytes, + signature.mtime_ns if row_mtime_ns is not None else None, + ) != (row_size_bytes, row_mtime_ns): + raise _PreflightStale return _UploadRecordPreflight( content_id, stored_hash, path, - _file_signature(path), + row_size_bytes, + row_mtime_ns, + signature, spec, ) @@ -472,13 +481,16 @@ def _prepare_upload_record( ) -def _assert_upload_preflight_current( - session: Session, - preflight: _UploadRecordPreflight, -) -> None: - if not _file_signature_matches(preflight.signature): +def _assert_signature_current(signature: _FileSignature) -> None: + try: + current = _file_signature(signature.path) + except OSError: + raise _PreflightStale from None + if current != signature: raise _PreflightStale - preview_id = preflight.spec.preview_id + + +def _assert_preview_exists(session: Session, preview_id: str | None) -> None: if preview_id is not None and session.get(Asset, preview_id) is None: raise _PreflightStale @@ -488,19 +500,17 @@ def _apply_reused_upload_record( prepared: _PreparedUploadRecord, ) -> UploadResult: preflight = prepared.preflight - content = lookup_for_view(session, preflight.stored_hash) + content = session.get(AssetContent, preflight.content_id) if ( content is None - or content.id != preflight.content_id - or content.path != preflight.path + or content.is_missing + or (content.size_bytes, content.mtime_ns) + != (preflight.row_size_bytes, preflight.row_mtime_ns) ): raise _PreflightStale - _assert_upload_preflight_current(session, preflight) + _assert_preview_exists(session, preflight.spec.preview_id) if not claim_qualified_content(session, content.id, preflight.stored_hash): raise _PreflightStale - content = refresh_qualified_content(session, content.id) - if content is None or content.path != preflight.path: - raise _PreflightStale record = _create_upload_record(session, content.id, prepared) return _record_to_upload_result(session, record, created_new=True) @@ -510,11 +520,12 @@ def _reuse_qualified_content( spec: _UploadRecordSpec, ) -> UploadResult | None: for _restart in range(4): - preflight = _preflight_upload_record(stored_hash, None, spec) - if preflight is None: - return None - prepared = _prepare_upload_record(preflight) try: + preflight = _preflight_upload_record(stored_hash, None, spec) + if preflight is None: + return None + prepared = _prepare_upload_record(preflight) + _assert_signature_current(preflight.signature) return run_write_txn( lambda session: _apply_reused_upload_record(session, prepared) ) @@ -582,7 +593,6 @@ def _apply_settle_target( or existing.hash != preflight.content_hash or existing.size_bytes != preflight.content_size_bytes or existing.mtime_ns != preflight.content_mtime_ns - or not _file_signature_matches(preflight.signature) ): raise _PreflightStale if prepared.facts is None: @@ -598,11 +608,20 @@ def _apply_settle_target( def _settle_destination_before_write(dest_abs: str) -> None: for _restart in range(4): - preflight = _preflight_settle_target(dest_abs) - if preflight is None: - return - prepared = _prepare_settle_target(preflight) try: + preflight = _preflight_settle_target(dest_abs) + if preflight is None: + return + prepared = _prepare_settle_target(preflight) + if prepared.facts is not None and ( + prepared.facts.size_bytes, + prepared.facts.mtime_ns, + ) != ( + preflight.signature.size_bytes, + preflight.signature.mtime_ns, + ): + raise _PreflightStale + _assert_signature_current(preflight.signature) run_write_txn(lambda session: _apply_settle_target(session, prepared)) return except _PreflightStale: @@ -618,37 +637,43 @@ def _create_content_and_upload_record( spec: _UploadRecordSpec, ) -> UploadResult: for _restart in range(4): - preflight = _preflight_upload_record(None, path, spec) - if preflight is None: - raise RuntimeError("new upload record requires a destination path") - prepared = _prepare_upload_record(preflight) - - def _work(session: Session) -> UploadResult: - _assert_upload_preflight_current(session, prepared.preflight) - _reconcile_live_content_at_path( - session, - path, - facts, - content_written=content_written, - ) - content, inserted = create_content_reporting_insert( - session, - path, - stored_hash, - facts.size_bytes, - facts.mtime_ns, - ) - created_content_id = content.id if inserted else None - try: - record = _create_upload_record(session, content.id, prepared) - except Exception: - session.rollback() - if created_content_id is not None: - _discard_unreferenced_content(session, created_content_id) - raise - return _record_to_upload_result(session, record, created_new=True) - try: + preflight = _preflight_upload_record(None, path, spec) + if preflight is None: + raise RuntimeError("new upload record requires a destination path") + prepared = _prepare_upload_record(preflight) + if ( + preflight.signature.size_bytes, + preflight.signature.mtime_ns, + ) != (facts.size_bytes, facts.mtime_ns): + raise _PreflightStale + _assert_signature_current(preflight.signature) + + def _work(session: Session) -> UploadResult: + _assert_preview_exists(session, prepared.preflight.spec.preview_id) + _reconcile_live_content_at_path( + session, + path, + facts, + content_written=content_written, + ) + content, inserted = create_content_reporting_insert( + session, + path, + stored_hash, + facts.size_bytes, + facts.mtime_ns, + ) + created_content_id = content.id if inserted else None + try: + record = _create_upload_record(session, content.id, prepared) + except Exception: + session.rollback() + if created_content_id is not None: + _discard_unreferenced_content(session, created_content_id) + raise + return _record_to_upload_result(session, record, created_new=True) + return run_write_txn(_work) except _PreflightStale: continue @@ -657,7 +682,7 @@ def _work(session: Session) -> UploadResult: "Upload preflight for a new asset did not settle in 4 attempts; refusing the upload " "rather than persisting content facts and file metadata that describe different bytes" ) - raise RuntimeError( + raise UploadUnstableError( f"Upload preflight for {path} did not settle in 4 attempts; " "refusing to persist content facts and file metadata that describe different bytes" ) @@ -857,11 +882,15 @@ def _preflight_cached_registration( else None ) content_id = existing.id + row_size_bytes = existing.size_bytes + row_mtime_ns = existing.mtime_ns signature = _file_signature(locator) if sibling_id is None else None return _CachedRegistrationPreflight( content_id, sibling_id, sibling_metadata, + row_size_bytes, + row_mtime_ns, signature, ) @@ -877,7 +906,13 @@ def _apply_cached_registration( system_metadata: dict[str, Any] | None, ) -> RegisteredAsset: existing = session.get(AssetContent, preflight.content_id) - if existing is None or existing.path != locator or existing.is_missing: + if ( + existing is None + or existing.path != locator + or existing.is_missing + or (existing.size_bytes, existing.mtime_ns) + != (preflight.row_size_bytes, preflight.row_mtime_ns) + ): raise _PreflightStale sibling = session.scalars( select(Asset) @@ -894,10 +929,6 @@ def _apply_cached_registration( if ( sibling_id != preflight.sibling_id or sibling_metadata != preflight.sibling_metadata - or ( - preflight.signature is not None - and not _file_signature_matches(preflight.signature) - ) ): raise _PreflightStale record = create_record( @@ -924,19 +955,21 @@ def register_cached_output( locator = os.path.abspath(abs_path) try: for _restart in range(4): - preflight = _preflight_cached_registration(locator) - if preflight is None: - logging.info( - "Cached output registration is a non-event; no live content for %s", - locator, - ) - return None - name, path_tags = get_name_and_tags_from_asset_path(locator) - mime_type = mimetypes.guess_type(locator, strict=False)[0] - system_metadata = preflight.sibling_metadata - if preflight.sibling_id is None: - system_metadata = _extract_system_metadata_sync(locator, mime_type) try: + preflight = _preflight_cached_registration(locator) + if preflight is None: + logging.info( + "Cached output registration is a non-event; no live content for %s", + locator, + ) + return None + name, path_tags = get_name_and_tags_from_asset_path(locator) + mime_type = mimetypes.guess_type(locator, strict=False)[0] + system_metadata = preflight.sibling_metadata + if preflight.sibling_id is None: + system_metadata = _extract_system_metadata_sync(locator, mime_type) + if preflight.signature is not None: + _assert_signature_current(preflight.signature) return run_write_txn( lambda session: _apply_cached_registration( session, diff --git a/tests-unit/assets_test/services/test_from_hash.py b/tests-unit/assets_test/services/test_from_hash.py index 53914a6445b..cb502dd69ad 100644 --- a/tests-unit/assets_test/services/test_from_hash.py +++ b/tests-unit/assets_test/services/test_from_hash.py @@ -5,11 +5,11 @@ from app.assets.database.models import Asset, AssetContent from app.assets.database.queries.records import create_content, mark_content_missing from app.assets.helpers import to_stored_hash -from app.assets.services.ingest import create_from_hash -from app.assets.services.lookup import ( - claim_qualified_content as _real_claim_qualified_content, - refresh_qualified_content as _real_refresh_qualified_content, +from app.assets.services.ingest import ( + _assert_signature_current as _real_assert_signature_current, ) +from app.assets.services.ingest import create_from_hash +from app.assets.services.lookup import claim_qualified_content as _real_claim_qualified_content def test_create_from_hash_with_prefixed_hash_finds_existing_content( @@ -94,7 +94,7 @@ def retire_then_claim(session, claimed_id, hash): assert retired.path == str(path) -def test_file_vanishing_between_claim_and_refresh_mints_nothing( +def test_file_vanishing_before_write_transaction_mints_nothing( mock_create_session, monkeypatch, temp_dir ): digest = "d" * 64 @@ -103,12 +103,12 @@ def test_file_vanishing_between_claim_and_refresh_mints_nothing( monkeypatch.setattr("app.assets.mode.hashing_enabled", lambda: True) content_id = _seed_live_content(mock_create_session, path, digest) - def delete_file_then_refresh(session, refreshed_id): + def delete_file_then_assert(signature): path.unlink() - return _real_refresh_qualified_content(session, refreshed_id) + return _real_assert_signature_current(signature) with patch( - "app.assets.services.ingest.refresh_qualified_content", delete_file_then_refresh + "app.assets.services.ingest._assert_signature_current", delete_file_then_assert ): result = create_from_hash(f"blake3:{digest}", "derived.bin") diff --git a/tests-unit/assets_test/services/test_ingest_write_txn.py b/tests-unit/assets_test/services/test_ingest_write_txn.py index 4ff3cd4744b..6d7e94335a1 100644 --- a/tests-unit/assets_test/services/test_ingest_write_txn.py +++ b/tests-unit/assets_test/services/test_ingest_write_txn.py @@ -115,12 +115,13 @@ def write_txn_fs_barrier(monkeypatch) -> _WriteTxnFsBarrier: def test_reused_upload_refuses_after_four_stale_preflights( write_txn_fs_barrier, monkeypatch ) -> None: - preflight = object() + preflight = SimpleNamespace(signature=object()) prepared = object() attempts: list[object] = [] monkeypatch.setattr(ingest, "_preflight_upload_record", lambda *_args: preflight) monkeypatch.setattr(ingest, "_prepare_upload_record", lambda _preflight: prepared) + monkeypatch.setattr(ingest, "_assert_signature_current", lambda _signature: None) def stale_apply(_session, observed_prepared): attempts.append(observed_prepared) @@ -138,12 +139,13 @@ def stale_apply(_session, observed_prepared): def test_settle_destination_refuses_after_four_stale_preflights( write_txn_fs_barrier, monkeypatch ) -> None: - preflight = object() - prepared = object() + preflight = SimpleNamespace(signature=object()) + prepared = SimpleNamespace(facts=None) attempts: list[object] = [] monkeypatch.setattr(ingest, "_preflight_settle_target", lambda _dest: preflight) monkeypatch.setattr(ingest, "_prepare_settle_target", lambda _preflight: prepared) + monkeypatch.setattr(ingest, "_assert_signature_current", lambda _signature: None) def stale_apply(_session, observed_prepared): attempts.append(observed_prepared) @@ -232,8 +234,11 @@ async def test_upload_route_reports_an_unsettleable_destination_as_unstable( "USER_MANAGER", SimpleNamespace(get_request_user_id=lambda _request: "test-user"), ) - monkeypatch.setattr(ingest, "_preflight_settle_target", lambda _dest: object()) - monkeypatch.setattr(ingest, "_prepare_settle_target", lambda _preflight: object()) + preflight = SimpleNamespace(signature=object()) + prepared = SimpleNamespace(facts=None) + monkeypatch.setattr(ingest, "_preflight_settle_target", lambda _dest: preflight) + monkeypatch.setattr(ingest, "_prepare_settle_target", lambda _preflight: prepared) + monkeypatch.setattr(ingest, "_assert_signature_current", lambda _signature: None) def stale_apply(_session, _prepared): raise ingest._PreflightStale @@ -348,6 +353,515 @@ def _apply_cached_preflight(session, preflight, path: str, system_metadata: dict ) +def test_reused_upload_apply_compares_the_row_to_preflight_row_values( + mock_create_session, tmp_path +) -> None: + path = tmp_path / "reuse-row-behind-disk.bin" + path.write_bytes(b"newer disk bytes") + signature = ingest._file_signature(str(path)) + stored_hash = "blake3:" + "a" * 64 + + def seed(session): + content = create_content( + session, + str(path), + hash=stored_hash, + size_bytes=3, + mtime_ns=7, + ) + return content.id + + content_id = db_mod.run_write_txn(seed) + preflight = ingest._UploadRecordPreflight( + content_id, + stored_hash, + str(path), + 3, + 7, + signature, + ingest._UploadRecordSpec(path.name, [], None, {}, None), + ) + prepared = ingest._PreparedUploadRecord(preflight, {}) + + result = db_mod.run_write_txn( + lambda session: ingest._apply_reused_upload_record(session, prepared) + ) + + assert result.content_id == content_id + + +def test_reconcile_unhashed_live_content_writes_hash_size_and_mtime_together( + mock_create_session, tmp_path +) -> None: + path = tmp_path / "unhashed-live-content.bin" + path.write_bytes(b"same-size bytes") + facts = ingest._ContentFacts( + "blake3:" + "f" * 64, + path.stat().st_size, + path.stat().st_mtime_ns, + ) + + def seed(session): + content = create_content( + session, + str(path), + size_bytes=facts.size_bytes, + mtime_ns=1, + ) + return content.id + + content_id = db_mod.run_write_txn(seed) + db_mod.run_write_txn( + lambda session: ingest._reconcile_live_content_at_path( + session, + str(path), + facts, + content_written=False, + ) + ) + + with mock_create_session() as session: + content = session.get(ingest.AssetContent, content_id) + assert content is not None + assert (content.hash, content.size_bytes, content.mtime_ns) == facts + + +def test_reused_upload_apply_rejects_a_row_changed_after_preflight( + mock_create_session, tmp_path +) -> None: + path = tmp_path / "reuse-row-mutated.bin" + path.write_bytes(b"stable bytes") + stat_result = path.stat() + stored_hash = "blake3:" + "b" * 64 + + def seed(session): + content = create_content( + session, + str(path), + hash=stored_hash, + size_bytes=stat_result.st_size, + mtime_ns=stat_result.st_mtime_ns, + ) + return content.id + + content_id = db_mod.run_write_txn(seed) + spec = ingest._UploadRecordSpec(path.name, [], None, {}, None) + preflight = ingest._preflight_upload_record(stored_hash, None, spec) + assert preflight is not None + prepared = ingest._prepare_upload_record(preflight) + + def mutate_row(session): + content = session.get(ingest.AssetContent, content_id) + assert content is not None + content.mtime_ns = stat_result.st_mtime_ns + 1 + + db_mod.run_write_txn(mutate_row) + + with pytest.raises(ingest._PreflightStale): + db_mod.run_write_txn( + lambda session: ingest._apply_reused_upload_record(session, prepared) + ) + + +def test_reused_upload_apply_accepts_a_qualified_row_without_mtime( + mock_create_session, tmp_path +) -> None: + path = tmp_path / "reuse-row-without-mtime.bin" + path.write_bytes(b"stable bytes") + stored_hash = "blake3:" + "c" * 64 + + def seed(session): + content = create_content( + session, + str(path), + hash=stored_hash, + size_bytes=path.stat().st_size, + mtime_ns=None, + ) + return content.id + + content_id = db_mod.run_write_txn(seed) + spec = ingest._UploadRecordSpec(path.name, [], None, {}, None) + preflight = ingest._preflight_upload_record(stored_hash, None, spec) + assert preflight is not None + assert preflight.row_mtime_ns is None + prepared = ingest._prepare_upload_record(preflight) + + result = db_mod.run_write_txn( + lambda session: ingest._apply_reused_upload_record(session, prepared) + ) + + assert result.content_id == content_id + + +def test_upload_preflight_rejects_a_file_changed_after_qualification( + mock_create_session, tmp_path, monkeypatch +) -> None: + path = tmp_path / "qualification-gap.bin" + path.write_bytes(b"qualified bytes") + stat_result = path.stat() + stored_hash = "blake3:" + "d" * 64 + db_mod.run_write_txn( + lambda session: create_content( + session, + str(path), + hash=stored_hash, + size_bytes=stat_result.st_size, + mtime_ns=stat_result.st_mtime_ns, + ) + ) + real_file_signature = ingest._file_signature + + def mutate_then_sign(file_path: str): + path.write_bytes(b"changed after qualification") + return real_file_signature(file_path) + + monkeypatch.setattr(ingest, "_file_signature", mutate_then_sign) + + with pytest.raises(ingest._PreflightStale): + ingest._preflight_upload_record( + stored_hash, + None, + ingest._UploadRecordSpec(path.name, [], None, {}, None), + ) + + +def test_reused_upload_retries_when_pretransaction_signature_check_is_stale_once( + mock_create_session, tmp_path, monkeypatch +) -> None: + path = tmp_path / "reuse-retry.bin" + path.write_bytes(b"stable bytes") + stat_result = path.stat() + stored_hash = "blake3:" + "e" * 64 + db_mod.run_write_txn( + lambda session: create_content( + session, + str(path), + hash=stored_hash, + size_bytes=stat_result.st_size, + mtime_ns=stat_result.st_mtime_ns, + ) + ) + checks = 0 + + def stale_once(_signature): + nonlocal checks + checks += 1 + if checks == 1: + raise ingest._PreflightStale + + monkeypatch.setattr(ingest, "_assert_signature_current", stale_once, raising=False) + + result = ingest._reuse_qualified_content( + stored_hash, + ingest._UploadRecordSpec(path.name, [], None, {}, None), + ) + + assert result is not None + assert checks == 2 + + +def test_settle_apply_compares_the_row_to_preflight_row_values( + mock_create_session, tmp_path +) -> None: + path = tmp_path / "settle-row-behind-disk.bin" + path.write_bytes(b"newer disk bytes") + signature = ingest._file_signature(str(path)) + + def seed(session): + content = create_content( + session, + str(path), + hash="blake3:" + "1" * 64, + size_bytes=3, + mtime_ns=7, + ) + return content.id + + content_id = db_mod.run_write_txn(seed) + preflight = ingest._SettleTargetPreflight( + content_id, + "blake3:" + "1" * 64, + 3, + 7, + signature, + ) + + db_mod.run_write_txn( + lambda session: ingest._apply_settle_target( + session, + ingest._PreparedSettleTarget(preflight, None), + ) + ) + + with mock_create_session() as session: + content = session.get(ingest.AssetContent, content_id) + assert content is not None + assert content.is_missing is True + + +def test_settle_apply_rejects_a_row_changed_after_preflight( + mock_create_session, tmp_path +) -> None: + path = tmp_path / "settle-row-mutated.bin" + path.write_bytes(b"stable bytes") + stat_result = path.stat() + + def seed(session): + content = create_content( + session, + str(path), + size_bytes=stat_result.st_size, + mtime_ns=stat_result.st_mtime_ns, + ) + return content.id + + content_id = db_mod.run_write_txn(seed) + preflight = ingest._preflight_settle_target(str(path)) + assert preflight is not None + + def mutate_row(session): + content = session.get(ingest.AssetContent, content_id) + assert content is not None + content.mtime_ns = stat_result.st_mtime_ns + 1 + + db_mod.run_write_txn(mutate_row) + + with pytest.raises(ingest._PreflightStale): + db_mod.run_write_txn( + lambda session: ingest._apply_settle_target( + session, + ingest._PreparedSettleTarget(preflight, None), + ) + ) + + +def test_settle_apply_accepts_a_preflight_row_without_mtime( + mock_create_session, tmp_path +) -> None: + path = tmp_path / "settle-row-without-mtime.bin" + path.write_bytes(b"stable bytes") + + def seed(session): + content = create_content( + session, + str(path), + size_bytes=path.stat().st_size, + mtime_ns=None, + ) + return content.id + + content_id = db_mod.run_write_txn(seed) + preflight = ingest._preflight_settle_target(str(path)) + assert preflight is not None + assert preflight.content_mtime_ns is None + + db_mod.run_write_txn( + lambda session: ingest._apply_settle_target( + session, + ingest._PreparedSettleTarget(preflight, None), + ) + ) + + with mock_create_session() as session: + content = session.get(ingest.AssetContent, content_id) + assert content is not None + assert content.is_missing is True + + +def test_settle_retries_when_hash_facts_do_not_match_the_preflight_signature( + monkeypatch, +) -> None: + signature = ingest._FileSignature("/settle.bin", 4, 5) + preflight = ingest._SettleTargetPreflight("content", None, 4, 5, signature) + preparations = 0 + transactions = 0 + + monkeypatch.setattr(ingest, "_preflight_settle_target", lambda _path: preflight) + + def prepare(_preflight): + nonlocal preparations + preparations += 1 + facts = ingest._ContentFacts( + "blake3:" + "2" * 64, + 6 if preparations == 1 else signature.size_bytes, + signature.mtime_ns, + ) + return ingest._PreparedSettleTarget(preflight, facts) + + def run_once(_work): + nonlocal transactions + transactions += 1 + + monkeypatch.setattr(ingest, "_prepare_settle_target", prepare) + monkeypatch.setattr(ingest, "_assert_signature_current", lambda _signature: None) + monkeypatch.setattr(ingest, "run_write_txn", run_once) + + ingest._settle_destination_before_write(signature.path) + + assert preparations == 2 + assert transactions == 1 + + +def test_settle_retries_when_pretransaction_signature_check_is_stale_once( + monkeypatch, +) -> None: + signature = ingest._FileSignature("/settle.bin", 4, 5) + preflight = ingest._SettleTargetPreflight("content", None, 4, 5, signature) + prepared = ingest._PreparedSettleTarget( + preflight, + ingest._ContentFacts("blake3:" + "3" * 64, 4, 5), + ) + checks = 0 + transactions = 0 + + monkeypatch.setattr(ingest, "_preflight_settle_target", lambda _path: preflight) + monkeypatch.setattr(ingest, "_prepare_settle_target", lambda _preflight: prepared) + + def stale_once(_signature): + nonlocal checks + checks += 1 + if checks == 1: + raise ingest._PreflightStale + + def run_once(_work): + nonlocal transactions + transactions += 1 + + monkeypatch.setattr(ingest, "_assert_signature_current", stale_once) + monkeypatch.setattr(ingest, "run_write_txn", run_once) + + ingest._settle_destination_before_write(signature.path) + + assert checks == 2 + assert transactions == 1 + + +def test_settle_restarts_when_destination_changes_after_hash_preparation( + mock_create_session, tmp_path, monkeypatch +) -> None: + path = tmp_path / "settle-changed-after-prepare.bin" + path.write_bytes(b"incumbent bytes") + stat_result = path.stat() + db_mod.run_write_txn( + lambda session: create_content( + session, + str(path), + size_bytes=stat_result.st_size, + mtime_ns=stat_result.st_mtime_ns, + ) + ) + real_prepare = ingest._prepare_settle_target + preparations = 0 + + def mutate_after_first_prepare(preflight): + nonlocal preparations + preparations += 1 + prepared = real_prepare(preflight) + if preparations == 1: + path.write_bytes(b"replacement bytes are different") + return prepared + + monkeypatch.setattr(ingest, "_prepare_settle_target", mutate_after_first_prepare) + + ingest._settle_destination_before_write(str(path)) + + assert preparations == 2 + + +def test_new_content_retries_when_preflight_signature_does_not_match_hash_facts( + mock_create_session, tmp_path, monkeypatch +) -> None: + path = tmp_path / "new-content-hash-snapshot.bin" + original_bytes = b"hashed bytes" + path.write_bytes(original_bytes) + original_stat = path.stat() + stored_hash = "blake3:" + "4" * 64 + facts = ingest._ContentFacts( + stored_hash, + original_stat.st_size, + original_stat.st_mtime_ns, + ) + spec = ingest._UploadRecordSpec(path.name, [], None, {}, None) + real_preflight = ingest._preflight_upload_record + preflights = 0 + + def change_before_first_preflight(*args): + nonlocal preflights + preflights += 1 + if preflights == 1: + path.write_bytes(b"newer bytes with another size") + else: + path.write_bytes(original_bytes) + os.utime( + path, + ns=(original_stat.st_mtime_ns, original_stat.st_mtime_ns), + ) + return real_preflight(*args) + + monkeypatch.setattr(ingest, "_preflight_upload_record", change_before_first_preflight) + + result = ingest._create_content_and_upload_record( + stored_hash, + str(path), + facts, + True, + spec, + ) + + assert result.content_id is not None + assert preflights == 2 + + +def test_new_content_retries_when_pretransaction_signature_check_is_stale_once( + monkeypatch, +) -> None: + signature = ingest._FileSignature("/new-content.bin", 4, 5) + spec = ingest._UploadRecordSpec("new-content.bin", [], None, {}, None) + preflight = ingest._UploadRecordPreflight( + None, + None, + signature.path, + None, + None, + signature, + spec, + ) + prepared = ingest._PreparedUploadRecord(preflight, {}) + facts = ingest._ContentFacts("blake3:" + "5" * 64, 4, 5) + preflights = 0 + checks = 0 + result = object() + + def preflight_once(*_args): + nonlocal preflights + preflights += 1 + return preflight + + def stale_once(_signature): + nonlocal checks + checks += 1 + if checks == 1: + raise ingest._PreflightStale + + monkeypatch.setattr(ingest, "_preflight_upload_record", preflight_once) + monkeypatch.setattr(ingest, "_prepare_upload_record", lambda _preflight: prepared) + monkeypatch.setattr(ingest, "_assert_signature_current", stale_once) + monkeypatch.setattr(ingest, "run_write_txn", lambda _work: result) + + observed = ingest._create_content_and_upload_record( + facts.stored_hash, + signature.path, + facts, + True, + spec, + ) + + assert observed is result + assert preflights == 2 + assert checks == 2 + + def test_executed_registration_reports_exhausted_locked_retries(monkeypatch, caplog) -> None: path = _output_path("executed-locked-retries.bin") with open(path, "wb") as file: @@ -420,6 +934,117 @@ def non_retryable_failure(_work): os.unlink(path) +def test_cached_registration_apply_rejects_changed_content_row_with_sibling( + mock_create_session, tmp_path +) -> None: + path = tmp_path / "cached-sibling-row-mutated.bin" + path.write_bytes(b"cached bytes") + stat_result = path.stat() + + def seed(session): + content = create_content( + session, + str(path), + size_bytes=stat_result.st_size, + mtime_ns=stat_result.st_mtime_ns, + ) + create_record(session, content.id, "sibling.bin", system_metadata={"source": 1}) + return content.id + + content_id = db_mod.run_write_txn(seed) + preflight = ingest._preflight_cached_registration(str(path)) + assert preflight is not None + assert preflight.sibling_id is not None + + def mutate_row(session): + content = session.get(ingest.AssetContent, content_id) + assert content is not None + content.mtime_ns = stat_result.st_mtime_ns + 1 + + db_mod.run_write_txn(mutate_row) + + with mock_create_session() as session: + with pytest.raises(ingest._PreflightStale): + _apply_cached_preflight(session, preflight, str(path), {"source": 1}) + + +def test_cached_registration_apply_rejects_changed_content_row_without_sibling( + mock_create_session, tmp_path +) -> None: + path = tmp_path / "cached-no-sibling-row-mutated.bin" + path.write_bytes(b"cached bytes") + stat_result = path.stat() + + def seed(session): + content = create_content( + session, + str(path), + size_bytes=stat_result.st_size, + mtime_ns=stat_result.st_mtime_ns, + ) + return content.id + + content_id = db_mod.run_write_txn(seed) + preflight = ingest._preflight_cached_registration(str(path)) + assert preflight is not None + assert preflight.sibling_id is None + + def mutate_row(session): + content = session.get(ingest.AssetContent, content_id) + assert content is not None + content.mtime_ns = stat_result.st_mtime_ns + 1 + + db_mod.run_write_txn(mutate_row) + + with mock_create_session() as session: + with pytest.raises(ingest._PreflightStale): + _apply_cached_preflight(session, preflight, str(path), {}) + + +def test_cached_registration_retries_when_pretransaction_signature_check_is_stale_once( + monkeypatch +) -> None: + path = _output_path("cached-retry.bin") + with open(path, "wb") as file: + file.write(b"cached bytes") + signature = ingest._file_signature(path) + preflight = SimpleNamespace( + content_id="content", + sibling_id="sibling", + sibling_metadata={"source": 1}, + signature=signature, + row_size_bytes=signature.size_bytes, + row_mtime_ns=signature.mtime_ns, + ) + preflights = 0 + checks = 0 + result = object() + + def preflight_once(_locator): + nonlocal preflights + preflights += 1 + return preflight + + def stale_once(_signature): + nonlocal checks + checks += 1 + if checks == 1: + raise ingest._PreflightStale + + monkeypatch.setattr(ingest, "_preflight_cached_registration", preflight_once) + monkeypatch.setattr(ingest, "_assert_signature_current", stale_once) + monkeypatch.setattr(ingest, "run_write_txn", lambda _work: result) + + try: + observed = ingest.register_cached_output(path) + finally: + os.unlink(path) + + assert observed is result + assert preflights == 2 + assert checks == 2 + + def test_cached_registration_restarts_when_content_vanishes_after_preflight( mock_create_session, monkeypatch ) -> None: @@ -539,14 +1164,14 @@ def test_cached_registration_restarts_when_file_changes_after_preflight( _seed_cached_content(mock_create_session, direct_path) direct_preflight = ingest._preflight_cached_registration(direct_path) assert direct_preflight is not None + assert direct_preflight.signature is not None with open(direct_path, "wb") as file: file.write(b"new bytes") - with mock_create_session() as session: - with pytest.raises(ingest._PreflightStale): - _apply_cached_preflight(session, direct_preflight, direct_path, {}) + with pytest.raises(ingest._PreflightStale): + ingest._assert_signature_current(direct_preflight.signature) _seed_cached_content(mock_create_session, public_path) - real_apply = ingest._apply_cached_registration + real_assert_signature_current = ingest._assert_signature_current extraction_sizes: list[int] = [] mutated = False @@ -555,16 +1180,16 @@ def extract_metadata(path, *_args, **_kwargs): extraction_sizes.append(size) return {"size": size} - def rewrite_then_apply(session, *args): + def rewrite_then_assert(signature): nonlocal mutated if not mutated: mutated = True with open(public_path, "wb") as file: file.write(b"new public bytes") - return real_apply(session, *args) + return real_assert_signature_current(signature) monkeypatch.setattr(ingest, "_extract_system_metadata_sync", extract_metadata) - monkeypatch.setattr(ingest, "_apply_cached_registration", rewrite_then_apply) + monkeypatch.setattr(ingest, "_assert_signature_current", rewrite_then_assert) result = ingest.register_cached_output(public_path) assert result is not None with mock_create_session() as session: @@ -739,20 +1364,23 @@ def test_unsettled_new_upload_persists_nothing_rather_than_mixing_facts( facts = ingest._ContentFacts(stored_hash, stat_result.st_size, stat_result.st_mtime_ns) spec = ingest._UploadRecordSpec("unsettled-upload.bin", [], None, {}, None) - real_assert = ingest._assert_upload_preflight_current + real_assert = ingest._assert_signature_current + attempts = 0 - def always_stale(_session, _preflight): + def always_stale(_signature): + nonlocal attempts + attempts += 1 raise ingest._PreflightStale - monkeypatch.setattr(ingest, "_assert_upload_preflight_current", always_stale) + monkeypatch.setattr(ingest, "_assert_signature_current", always_stale) with caplog.at_level(logging.WARNING): - with pytest.raises(RuntimeError, match="did not settle"): + with pytest.raises(ingest.UploadUnstableError, match="did not settle"): ingest._create_content_and_upload_record( stored_hash, path, facts, True, spec ) - monkeypatch.setattr(ingest, "_assert_upload_preflight_current", real_assert) + monkeypatch.setattr(ingest, "_assert_signature_current", real_assert) with db_mod.Session() as session: contents = list(session.scalars(select(AssetContent).where(AssetContent.path == path))) @@ -760,4 +1388,5 @@ def always_stale(_session, _preflight): assert contents == [], "refused upload must not leave a content row behind" assert records == [], "refused upload must not leave an asset record behind" + assert attempts == 4 os.unlink(path) diff --git a/tests-unit/assets_test/services/test_lifecycle_write_txn.py b/tests-unit/assets_test/services/test_lifecycle_write_txn.py index 4911ac4b256..0b2026b0c3d 100644 --- a/tests-unit/assets_test/services/test_lifecycle_write_txn.py +++ b/tests-unit/assets_test/services/test_lifecycle_write_txn.py @@ -99,3 +99,36 @@ def fail_write(_work): lifecycle.record_hash_mode_transition_intent() assert lifecycle._hash_mode_transition == "off_to_on" + + +def test_enqueue_mode_transition_work_does_not_commit_reader_session( + mock_create_session, monkeypatch +) -> None: + db_mod.run_write_txn( + lambda session: session.add( + lifecycle.AssetContent( + path="/catalogued.bin", + size_bytes=1, + mtime_ns=1, + ) + ) + ) + + @contextmanager + def read_only_session(): + with mock_create_session() as reader: + monkeypatch.setattr( + reader, + "commit", + lambda: (_ for _ in ()).throw( + AssertionError("read-only path must not commit") + ), + ) + yield reader + + lifecycle._hash_mode_transition = "off_to_on" + monkeypatch.setattr(lifecycle, "create_session", read_only_session) + + lifecycle.enqueue_mode_transition_work() + + assert hash_mode_state.pending_transition_count() == 1 diff --git a/tests-unit/assets_test/services/test_write_lock_hold_barriers.py b/tests-unit/assets_test/services/test_write_lock_hold_barriers.py index f0fef74a9d7..32ca0731419 100644 --- a/tests-unit/assets_test/services/test_write_lock_hold_barriers.py +++ b/tests-unit/assets_test/services/test_write_lock_hold_barriers.py @@ -1,6 +1,8 @@ +import os import threading import uuid +import folder_paths import pytest from sqlalchemy import text from sqlalchemy.exc import OperationalError @@ -11,6 +13,7 @@ from app.assets import scanner_changes from app.assets.database.queries.records import create_content, create_record from app.assets.services import hash_mode_state +from app.assets.services import ingest _BARRIER_TIMEOUT = 5 _PROBE_LOCK_DEADLINE_SECONDS = 0.5 @@ -84,6 +87,160 @@ def fake(*args, **kwargs): return fake +class _IngestWriteTxnFsTracker: + def __init__(self, monkeypatch) -> None: + self.calls: list[str] = [] + self.inside = False + real_run_write_txn = ingest.run_write_txn + + def track_write_transaction(work): + def tracked_work(session): + self.inside = True + try: + return work(session) + finally: + self.inside = False + + return real_run_write_txn(tracked_work) + + monkeypatch.setattr(ingest, "run_write_txn", track_write_transaction) + for name in ( + "lookup_for_view", + "refresh_qualified_content", + "_file_signature", + "_file_signature_matches", + ): + if hasattr(ingest, name): + real = getattr(ingest, name) + monkeypatch.setattr(ingest, name, self._track(name, real)) + monkeypatch.setattr(ingest.os, "stat", self._track("os.stat", os.stat)) + monkeypatch.setattr( + ingest.os.path, + "isfile", + self._track("os.path.isfile", os.path.isfile), + ) + + def _track(self, label: str, real): + def tracked(*args, **kwargs): + if self.inside: + self.calls.append(label) + return real(*args, **kwargs) + + return tracked + + +@pytest.fixture +def ingest_write_txn_fs_tracker(monkeypatch) -> _IngestWriteTxnFsTracker: + return _IngestWriteTxnFsTracker(monkeypatch) + + +def test_reused_upload_does_not_touch_the_filesystem_inside_write_transaction( + file_database, tmp_path, ingest_write_txn_fs_tracker +) -> None: + existing_path = tmp_path / "reuse-existing.bin" + existing_path.write_bytes(b"shared bytes") + stat_result = existing_path.stat() + digest, _ = ingest._snapshot_hash_with_retry(str(existing_path)) + stored_hash = ingest.to_stored_hash(digest) + + db_mod.run_write_txn( + lambda session: create_content( + session, + str(existing_path), + hash=stored_hash, + size_bytes=stat_result.st_size, + mtime_ns=stat_result.st_mtime_ns, + ) + ) + upload_path = tmp_path / "reuse-upload.part" + upload_path.write_bytes(b"shared bytes") + + result = ingest.upload_from_temp_path(str(upload_path), name="reuse.bin") + + assert result.content_id is not None + assert ingest_write_txn_fs_tracker.calls == [] + + +def test_new_upload_does_not_touch_the_filesystem_inside_write_transaction( + file_database, tmp_path, monkeypatch, ingest_write_txn_fs_tracker +) -> None: + upload_path = tmp_path / "new-upload.part" + upload_path.write_bytes(b"new bytes") + destination = tmp_path / "new-upload.bin" + monkeypatch.setattr( + ingest, + "_hash_mode_dest_path", + lambda *_args: str(destination), + ) + + result = ingest.upload_from_temp_path( + str(upload_path), + name="new-upload.bin", + tags=["output"], + ) + + assert result.ref.file_path == str(destination) + assert ingest_write_txn_fs_tracker.calls == [] + + +def test_cached_registration_does_not_touch_the_filesystem_inside_write_transaction( + file_database, ingest_write_txn_fs_tracker +) -> None: + path = os.path.join(folder_paths.get_output_directory(), "cached-output.bin") + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as file: + file.write(b"cached bytes") + stat_result = os.stat(path) + db_mod.run_write_txn( + lambda session: create_content( + session, + path, + size_bytes=stat_result.st_size, + mtime_ns=stat_result.st_mtime_ns, + ) + ) + + try: + result = ingest.register_cached_output(path) + finally: + os.unlink(path) + + assert result is not None + assert ingest_write_txn_fs_tracker.calls == [] + + +def test_settle_destination_does_not_touch_the_filesystem_inside_write_transaction( + file_database, tmp_path, ingest_write_txn_fs_tracker +) -> None: + path = tmp_path / "settle-output.bin" + path.write_bytes(b"incumbent bytes") + stat_result = path.stat() + db_mod.run_write_txn( + lambda session: create_content( + session, + str(path), + size_bytes=stat_result.st_size, + mtime_ns=stat_result.st_mtime_ns, + ) + ) + + ingest._settle_destination_before_write(str(path)) + + assert ingest_write_txn_fs_tracker.calls == [] + + +def test_register_file_in_place_does_not_touch_the_filesystem_inside_write_transaction( + file_database, tmp_path, ingest_write_txn_fs_tracker +) -> None: + path = tmp_path / "register-in-place.bin" + path.write_bytes(b"in-place bytes") + + result = ingest.register_file_in_place(str(path), path.name, ["output"]) + + assert result.ref.file_path == str(path) + assert ingest_write_txn_fs_tracker.calls == [] + + def test_seed_recovery_hashing_does_not_hold_the_write_lock( file_database, impatient_probe, hashing_on, tmp_path, monkeypatch ): diff --git a/tests-unit/assets_test/test_write_session_convention.py b/tests-unit/assets_test/test_write_session_convention.py index 39b0ea06615..4a055cc96bd 100644 --- a/tests-unit/assets_test/test_write_session_convention.py +++ b/tests-unit/assets_test/test_write_session_convention.py @@ -38,8 +38,7 @@ class CreateSessionCallSite(NamedTuple): READ_ONLY_CREATE_SESSION_CALL_SITES = frozenset( { CreateSessionCallSite("app/assets/api/routes.py", "list_assets_route"), - # This commits only SELECTs and in-memory queue changes, so a writer retry - # would acquire a write lease for no persisted work. + # This reads the catalogue and fills in-memory queues; no commit. CreateSessionCallSite("app/assets/lifecycle.py", "enqueue_mode_transition_work"), CreateSessionCallSite("app/assets/scanner.py", "get_unenriched_assets_for_roots"), # This reads the catalogue so the stat walk runs before the writer lease is taken. From 3604ba4e47c532425a10ec372fdddab996794064 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 18:31:43 -0700 Subject: [PATCH 043/111] test(assets): let the event-log manifest express a call site's real multiplicity --- tests-unit/test_assets_event_log_static.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests-unit/test_assets_event_log_static.py b/tests-unit/test_assets_event_log_static.py index 4b59f839ae3..c7712127590 100644 --- a/tests-unit/test_assets_event_log_static.py +++ b/tests-unit/test_assets_event_log_static.py @@ -39,9 +39,10 @@ class CallSite(NamedTuple): # The manifest of every tagged event this branch emits: (file, enclosing -# function, event) triples that must be present in the tree exactly as written. -EXPECTED_CALL_SITES: frozenset[CallSite] = frozenset( - { +# function, event) triples that must be present in the tree with their exact +# multiplicity. +EXPECTED_CALL_SITES: Counter[CallSite] = Counter( + ( # todo 10 - seeder lifecycle + the single assets.enabled site CallSite("server.py", "__init__", "assets.enabled"), CallSite("app/assets/seeder.py", "_run_scan", "seeder.scan_started"), @@ -60,13 +61,14 @@ class CallSite(NamedTuple): CallSite("app/assets/scanner.py", "_prepare_enrichment", "scanner.hash_failed"), CallSite("app/assets/scanner.py", "_prepare_enrichment", "scanner.hash_discarded_modified"), CallSite("app/assets/scanner.py", "enrich_assets_batch", "scanner.enrich_failed"), + CallSite("app/assets/scanner.py", "enrich_assets_batch", "scanner.enrich_failed"), # todo 16 - discovery/enrich stat failures, emit-once per scan per site CallSite("app/assets/scanner.py", "build_asset_specs", "scanner.stat_failed"), CallSite("app/assets/scanner.py", "_prepare_enrichment", "scanner.stat_failed"), CallSite("app/assets/scanner.py", "_publish_reference_diagnostics", "scanner.stat_failed"), CallSite("app/assets/services/ingest.py", "register_cached_output", "ingest.register_failed"), CallSite("app/assets/services/ingest.py", "register_executed_output", "ingest.register_failed"), - } + ) ) class Aliases(NamedTuple): From dcec255d43ba524df29a791e91102ac70417c59b Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 18:58:38 -0700 Subject: [PATCH 044/111] docs(assets): document bounded write transactions and the read-only reader --- app/assets/docs/README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/app/assets/docs/README.md b/app/assets/docs/README.md index f8ff73a3d63..0b693523013 100644 --- a/app/assets/docs/README.md +++ b/app/assets/docs/README.md @@ -325,11 +325,26 @@ If more than one missing content row matches a recreated file's hash, recover no Deleting or replacing the database file while the server is running is undefined behaviour. +## Write transactions + +Every database write goes through `run_write_txn`, which opens a dedicated writer session, takes the write lock up front with `BEGIN IMMEDIATE`, and retries on a locked database within a bounded deadline (see Write pressure and reader starvation). Reader connections are opened `query_only`, so a write attempted through an ordinary read session fails immediately with a read-only-database error rather than becoming a second writer by accident. + +Background writes are bounded: no scanner transaction covers more than `MAX_WRITE_BATCH` rows, and the scanner checks for pause and cancellation between batches, so a pause takes effect one batch later instead of at the end of the phase. Seeding, reference reconciliation, pruning, and enrichment all chunk their work to that size; the verification and hash-transition drains already write one entry per transaction. + +The batch size is 25. A transaction holds the write lock for its whole duration, so the bound is exactly the stall an urgent writer inherits when it arrives mid-batch, and it is also the rollback blast radius when one row in a batch fails. 25 sits just past the point where a larger batch stops buying throughput: measured lock-held medians are 20ms for a 25-row seed batch and 27ms for a 25-row enrichment batch, against a 0.03ms empty-transaction floor, roughly 1ms per row, and 3.8ms for the single-row insert an urgent writer is typically waiting to make. + +Write callables never touch the filesystem. Stat, hashing, and metadata extraction run in a preflight outside the transaction, and the closure receives the facts that preflight observed; inside the lock it only reads and writes rows. Two invariants make that safe: + +- Pair-write: any statement that sets a content hash also sets the byte size and modification time from the same snapshot, so a stored hash always describes the stat facts stored beside it. +- Row freshness: before mutating a content row from preflight facts, a closure compares the row's current missing state, byte size, and modification time against the values the preflight read from the database — never against the stat the preflight took from disk, because a row that is legitimately behind disk is the normal steady state and comparing against disk would reject valid work. On a mismatch the closure skips that row and leaves it untouched; the next scan re-observes it and repairs it. + +A persistently unstable upload is refused rather than hashed under the lock: when its preflight facts keep changing across attempts, the request fails instead of writing content facts and file metadata that describe different bytes. + ## Operational limits ### Write pressure and reader starvation -The asset database is SQLite in WAL journal mode. Reader connections use a 30-second busy timeout. Writer connections poll `BEGIN IMMEDIATE` in Python, so each lock wait is capped at 30 seconds while the overall retry deadline remains 60 seconds. Several paths still contend for the single writer lock: a non-deduplicated upload writes its bytes and mints a delivery record, while a deduplicated upload reuses existing content and mints only the record; a same-path write whose hash has changed retires the old content and inserts new content, while a same-path write whose hash matches refreshes the existing record in place; execution outputs register per-emission during the generation loop; a background enrichment pass fills hashes and metadata row by row; hash-serves write access time to every record sharing the served content; and the upload dedup claim holds the write lock across its filesystem re-check and metadata extraction. +The asset database is SQLite in WAL journal mode. Reader connections use a 30-second busy timeout. Writer connections poll `BEGIN IMMEDIATE` in Python, so each lock wait is capped at 30 seconds while the overall retry deadline remains 60 seconds. Several paths still contend for the single writer lock: a non-deduplicated upload writes its bytes and mints a delivery record, while a deduplicated upload reuses existing content and mints only the record; a same-path write whose hash has changed retires the old content and inserts new content, while a same-path write whose hash matches refreshes the existing record in place; execution outputs register per-emission during the generation loop; a background enrichment pass fills hashes and metadata in bounded batches; hash-serves write access time to every record sharing the served content; and the upload dedup claim mints its record once a preflight has already re-checked the file and extracted its metadata outside the lock. A session that opens as a reader and only later attempts a write can be refused instantly if another connection already holds the write lock, before the busy timeout ever gets a chance to wait. Every writing session avoids this by going through `run_write_txn`, which opens a dedicated writer session and issues `BEGIN IMMEDIATE` up front, declaring writer intent before any statement runs. A write that still finds the database locked retries up to 5 attempts total, sleeping 0.05s/0.1s/0.2s/0.4s between attempts (each jittered ±50%) and giving up once 60 seconds have elapsed since the first attempt; only a locked-database `OperationalError` is retried, every other exception propagates immediately. Those 5 attempts and their backoff table are reachable only when the lock error surfaces from inside the callback, after `BEGIN IMMEDIATE` has already succeeded. Contention at `BEGIN IMMEDIATE` itself behaves differently: each attempt polls for up to 30 seconds before yielding, so the 60-second deadline admits about two attempts and the backoff sleeps are negligible beside the polling. Either way the bound is the same 60-second deadline, after which a locked-database `OperationalError` is raised. Contributors must not open a writing session with `create_session()` directly. `run_write_txn` is the only sanctioned path for writes, and `tests-unit/assets_test/test_write_session_convention.py` enforces that automatically. From b9c428b6c5bdd41d4fb8d1665166728f2507206a Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 14:15:25 -0700 Subject: [PATCH 045/111] chore(assets): drop the unused asset_meta table from migration 0007 --- .../versions/0007_record_content_split.py | 14 -------- app/assets/database/models.py | 33 ------------------- tests-unit/app_test/test_migration_0007.py | 14 -------- tests-unit/assets_test/queries/conftest.py | 1 - 4 files changed, 62 deletions(-) diff --git a/alembic_db/versions/0007_record_content_split.py b/alembic_db/versions/0007_record_content_split.py index c88af94d5ed..853afff1940 100644 --- a/alembic_db/versions/0007_record_content_split.py +++ b/alembic_db/versions/0007_record_content_split.py @@ -67,19 +67,6 @@ def upgrade() -> None: op.create_index("ix_assets_name", "assets", ["name"]) op.create_index("ix_assets_created_at", "assets", ["created_at"]) op.create_index("ix_assets_preview_id", "assets", ["preview_id"]) - op.create_table( - "asset_meta", - sa.Column("asset_id", sa.String(36), sa.ForeignKey("assets.id", ondelete="CASCADE"), primary_key=True), - sa.Column("key", sa.String(256), primary_key=True), - sa.Column("ordinal", sa.Integer(), primary_key=True), - sa.Column("val_str", sa.String(2048)), sa.Column("val_num", sa.Numeric(38, 10)), - sa.Column("val_bool", sa.Boolean()), sa.Column("val_json", sa.JSON()), - sa.CheckConstraint("val_str IS NOT NULL OR val_num IS NOT NULL OR val_bool IS NOT NULL OR val_json IS NOT NULL", name="ck_asset_meta_has_value"), - ) - op.create_index("ix_asset_meta_key", "asset_meta", ["key"]) - op.create_index("ix_asset_meta_key_val_str", "asset_meta", ["key", "val_str"]) - op.create_index("ix_asset_meta_key_val_num", "asset_meta", ["key", "val_num"]) - op.create_index("ix_asset_meta_key_val_bool", "asset_meta", ["key", "val_bool"]) op.create_table("asset_tags", sa.Column("asset_id", sa.String(36), sa.ForeignKey("assets.id", ondelete="CASCADE"), primary_key=True), sa.Column("tag_name", sa.String(512), sa.ForeignKey("tags.name", ondelete="RESTRICT"), primary_key=True), sa.Column("origin", sa.String(32), nullable=False), sa.Column("added_at", sa.DateTime(), nullable=False)) op.create_index("ix_asset_tags_tag_name", "asset_tags", ["tag_name"]) op.create_index("ix_asset_tags_asset_id", "asset_tags", ["asset_id"]) @@ -89,7 +76,6 @@ def upgrade() -> None: def downgrade() -> None: op.drop_table("asset_system_state") op.drop_table("asset_tags") - op.drop_table("asset_meta") op.drop_table("assets") op.drop_table("asset_contents") op.create_table( diff --git a/app/assets/database/models.py b/app/assets/database/models.py index bb0b8c35568..d1d82a4bc81 100644 --- a/app/assets/database/models.py +++ b/app/assets/database/models.py @@ -10,7 +10,6 @@ import uuid from datetime import datetime -from decimal import Decimal from typing import Any from sqlalchemy import ( @@ -21,8 +20,6 @@ DateTime, ForeignKey, Index, - Integer, - Numeric, String, Text, text, @@ -103,9 +100,6 @@ class Asset(Base): preview: Mapped[Asset | None] = relationship( "Asset", foreign_keys=[preview_id], remote_side=lambda: [Asset.id] ) - metadata_entries: Mapped[list[AssetMeta]] = relationship( - back_populates="asset", cascade="all,delete-orphan", passive_deletes=True - ) tag_links: Mapped[list[AssetTag]] = relationship( back_populates="asset", cascade="all,delete-orphan", passive_deletes=True ) @@ -121,33 +115,6 @@ class Asset(Base): ) -class AssetMeta(Base): - __tablename__ = "asset_meta" - - asset_id: Mapped[str] = mapped_column( - String(36), ForeignKey("assets.id", ondelete="CASCADE"), primary_key=True - ) - key: Mapped[str] = mapped_column(String(256), primary_key=True) - ordinal: Mapped[int] = mapped_column(Integer, primary_key=True, default=0) - val_str: Mapped[str | None] = mapped_column(String(2048)) - val_num: Mapped[Decimal | None] = mapped_column(Numeric(38, 10)) - val_bool: Mapped[bool | None] = mapped_column(Boolean) - val_json: Mapped[Any | None] = mapped_column(JSON) - - asset: Mapped[Asset] = relationship(back_populates="metadata_entries") - - __table_args__ = ( - Index("ix_asset_meta_key", "key"), - Index("ix_asset_meta_key_val_str", "key", "val_str"), - Index("ix_asset_meta_key_val_num", "key", "val_num"), - Index("ix_asset_meta_key_val_bool", "key", "val_bool"), - CheckConstraint( - "val_str IS NOT NULL OR val_num IS NOT NULL OR val_bool IS NOT NULL OR val_json IS NOT NULL", - name="ck_asset_meta_has_value", - ), - ) - - class AssetTag(Base): __tablename__ = "asset_tags" diff --git a/tests-unit/app_test/test_migration_0007.py b/tests-unit/app_test/test_migration_0007.py index ae7efd2bd51..9a252173b29 100644 --- a/tests-unit/app_test/test_migration_0007.py +++ b/tests-unit/app_test/test_migration_0007.py @@ -95,7 +95,6 @@ def test_0007_invariants_on_migrated_db(db_at_0006): def test_0007_orm_parity(db_at_0006, tmp_path): from sqlalchemy import create_engine, inspect - import app.assets.database.models as asset_models from app.database.models import Base cfg, db_path = db_at_0006 @@ -109,26 +108,13 @@ def test_0007_orm_parity(db_at_0006, tmp_path): ) } - alembic_engine = create_engine(f"sqlite:///{db_path}") orm_db = str(tmp_path / "orm.db") engine = create_engine(f"sqlite:///{orm_db}") - alembic_inspector = inspect(alembic_engine) Base.metadata.create_all(engine) orm_inspector = inspect(engine) orm_tables = set(orm_inspector.get_table_names()) - assert asset_models.AssetMeta.__tablename__ == "asset_meta" - - alembic_indexes = { - (index["name"], tuple(index["column_names"])) - for index in alembic_inspector.get_indexes("asset_meta") - } - orm_indexes = { - (index.name, tuple(index.columns.keys())) - for index in Base.metadata.tables["asset_meta"].indexes - } assert alembic_tables == orm_tables, f"Mismatch: alembic={alembic_tables}, orm={orm_tables}" - assert alembic_indexes == orm_indexes, f"Mismatch: alembic={alembic_indexes}, orm={orm_indexes}" def test_0007_downgrade_chain_past_0003_succeeds(db_at_0006): diff --git a/tests-unit/assets_test/queries/conftest.py b/tests-unit/assets_test/queries/conftest.py index 18b67c15978..30f4a84b1eb 100644 --- a/tests-unit/assets_test/queries/conftest.py +++ b/tests-unit/assets_test/queries/conftest.py @@ -10,7 +10,6 @@ def assert_asset_metadata_tables(): assert set(Base.metadata.tables) == { "assets", "asset_contents", - "asset_meta", "asset_tags", "tags", "asset_system_state", From 02842ce6bde3769788c4a041e26ba7418557879f Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 00:56:43 -0700 Subject: [PATCH 046/111] fix(assets): drop asset_meta when downgrading a database that already created it --- .../versions/0007_record_content_split.py | 1 + tests-unit/app_test/test_migration_0007.py | 52 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/alembic_db/versions/0007_record_content_split.py b/alembic_db/versions/0007_record_content_split.py index 853afff1940..5857fe920a4 100644 --- a/alembic_db/versions/0007_record_content_split.py +++ b/alembic_db/versions/0007_record_content_split.py @@ -76,6 +76,7 @@ def upgrade() -> None: def downgrade() -> None: op.drop_table("asset_system_state") op.drop_table("asset_tags") + op.execute("DROP TABLE IF EXISTS asset_meta") op.drop_table("assets") op.drop_table("asset_contents") op.create_table( diff --git a/tests-unit/app_test/test_migration_0007.py b/tests-unit/app_test/test_migration_0007.py index 9a252173b29..14dd86ac9ae 100644 --- a/tests-unit/app_test/test_migration_0007.py +++ b/tests-unit/app_test/test_migration_0007.py @@ -33,6 +33,7 @@ def test_0007_upgrade_from_0006(db_at_0006): assert "asset_contents" in tables assert "asset_system_state" in tables assert "asset_references" not in tables + assert "asset_meta" not in tables def test_0007_schema_has_expected_columns(db_at_0006): @@ -55,6 +56,57 @@ def test_0007_downgrade_restores_0006_schema(db_at_0006): assert "asset_contents" not in tables +def test_0007_downgrade_drops_shipped_asset_meta(tmp_path): + from sqlalchemy import create_engine + + from app.assets.database.models import Base + + db_path = str(tmp_path / "shipped_0007.db") + cfg = _make_config(db_path) + engine = create_engine(f"sqlite:///{db_path}") + Base.metadata.create_all(engine) + with sqlite3.connect(db_path) as conn: + conn.executescript( + """ + CREATE TABLE asset_meta ( + asset_id VARCHAR(36) NOT NULL, + key VARCHAR(256) NOT NULL, + ordinal INTEGER NOT NULL, + val_str VARCHAR(2048), + val_num NUMERIC(38, 10), + val_bool BOOLEAN, + val_json JSON, + CONSTRAINT ck_asset_meta_has_value CHECK ( + val_str IS NOT NULL OR val_num IS NOT NULL OR + val_bool IS NOT NULL OR val_json IS NOT NULL + ), + PRIMARY KEY (asset_id, key, ordinal), + FOREIGN KEY(asset_id) REFERENCES assets(id) ON DELETE CASCADE + ); + CREATE INDEX ix_asset_meta_key ON asset_meta (key); + CREATE INDEX ix_asset_meta_key_val_str ON asset_meta (key, val_str); + CREATE INDEX ix_asset_meta_key_val_num ON asset_meta (key, val_num); + CREATE INDEX ix_asset_meta_key_val_bool ON asset_meta (key, val_bool); + """ + ) + conn.commit() + engine.dispose() + command.stamp(cfg, "0007_record_content_split") + + with sqlite3.connect(db_path) as conn: + assert conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='asset_meta'" + ).fetchone() + + command.downgrade(cfg, _BASELINE_0006) + + with sqlite3.connect(db_path) as conn: + assert conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='asset_meta'" + ).fetchone() is None + assert list(conn.execute("PRAGMA foreign_key_check")) == [] + + def test_0007_invariants_on_migrated_db(db_at_0006): cfg, db_path = db_at_0006 command.upgrade(cfg, "head") From 2e942be6395d16eac1b2123d3b840684a33b7040 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 00:56:47 -0700 Subject: [PATCH 047/111] docs(assets): clarify asset schema docstring --- app/assets/database/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/database/models.py b/app/assets/database/models.py index d1d82a4bc81..2c261ed042b 100644 --- a/app/assets/database/models.py +++ b/app/assets/database/models.py @@ -1,5 +1,5 @@ """Declares the asset schema: content rows describing bytes on disk, records -describing what a user sees, and the tag and metadata tables hanging off them. +describing what a user sees, and the asset_tags table hanging off them. The split is the point — many records can name one content row, and retiring content by marking it missing rather than deleting it is what keeps a path's history intact. Constraints declared here, not application code, are what make From 7eaabb5ca018a553fad492a1ec8948b50e92bc54 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 11:14:21 -0700 Subject: [PATCH 048/111] test(assets): assert alembic and ORM index parity for the surviving asset tables --- tests-unit/app_test/test_migration_0007.py | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests-unit/app_test/test_migration_0007.py b/tests-unit/app_test/test_migration_0007.py index 14dd86ac9ae..1d17f90e99c 100644 --- a/tests-unit/app_test/test_migration_0007.py +++ b/tests-unit/app_test/test_migration_0007.py @@ -169,6 +169,36 @@ def test_0007_orm_parity(db_at_0006, tmp_path): assert alembic_tables == orm_tables, f"Mismatch: alembic={alembic_tables}, orm={orm_tables}" +@pytest.mark.parametrize("table_name", ["assets", "asset_contents", "asset_tags"]) +def test_0007_index_orm_parity(db_at_0006, table_name): + from sqlalchemy import create_engine, inspect + + import app.assets.database.models as asset_models + + cfg, db_path = db_at_0006 + command.upgrade(cfg, "head") + + alembic_engine = create_engine(f"sqlite:///{db_path}") + try: + alembic_indexes = { + (index["name"], tuple(index["column_names"])) + for index in inspect(alembic_engine).get_indexes(table_name) + } + finally: + alembic_engine.dispose() + + orm_indexes = { + (index.name, tuple(index.columns.keys())) + for index in asset_models.Base.metadata.tables[table_name].indexes + } + + assert alembic_indexes + assert orm_indexes + assert alembic_indexes == orm_indexes, ( + f"Mismatch for {table_name}: alembic={alembic_indexes}, orm={orm_indexes}" + ) + + def test_0007_downgrade_chain_past_0003_succeeds(db_at_0006): cfg, db_path = db_at_0006 command.upgrade(cfg, "head") From d9d78b99da0028b21b5c1f08c5a679c97ba32905 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 11:15:31 -0700 Subject: [PATCH 049/111] test(assets): cover asset system state index parity --- tests-unit/app_test/test_migration_0007.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests-unit/app_test/test_migration_0007.py b/tests-unit/app_test/test_migration_0007.py index 1d17f90e99c..432b7770828 100644 --- a/tests-unit/app_test/test_migration_0007.py +++ b/tests-unit/app_test/test_migration_0007.py @@ -169,8 +169,16 @@ def test_0007_orm_parity(db_at_0006, tmp_path): assert alembic_tables == orm_tables, f"Mismatch: alembic={alembic_tables}, orm={orm_tables}" -@pytest.mark.parametrize("table_name", ["assets", "asset_contents", "asset_tags"]) -def test_0007_index_orm_parity(db_at_0006, table_name): +@pytest.mark.parametrize( + ("table_name", "has_indexes"), + [ + ("assets", True), + ("asset_contents", True), + ("asset_tags", True), + ("asset_system_state", False), + ], +) +def test_0007_index_orm_parity(db_at_0006, table_name, has_indexes): from sqlalchemy import create_engine, inspect import app.assets.database.models as asset_models @@ -192,8 +200,8 @@ def test_0007_index_orm_parity(db_at_0006, table_name): for index in asset_models.Base.metadata.tables[table_name].indexes } - assert alembic_indexes - assert orm_indexes + assert bool(alembic_indexes) is has_indexes + assert bool(orm_indexes) is has_indexes assert alembic_indexes == orm_indexes, ( f"Mismatch for {table_name}: alembic={alembic_indexes}, orm={orm_indexes}" ) From 898348bc2048bac0f65ff236bb14a18f2f8d9d0e Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 15:12:04 -0700 Subject: [PATCH 050/111] refactor(tests): hoist migration-0007 test imports to module scope --- tests-unit/app_test/test_migration_0007.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests-unit/app_test/test_migration_0007.py b/tests-unit/app_test/test_migration_0007.py index 432b7770828..319341bb3f9 100644 --- a/tests-unit/app_test/test_migration_0007.py +++ b/tests-unit/app_test/test_migration_0007.py @@ -4,6 +4,10 @@ import pytest from alembic import command from alembic.config import Config +from sqlalchemy import create_engine, inspect + +import app.assets.database.models as asset_models +from app.assets.database.models import Base _BASELINE_0006 = "0006_add_loader_path" @@ -57,10 +61,6 @@ def test_0007_downgrade_restores_0006_schema(db_at_0006): def test_0007_downgrade_drops_shipped_asset_meta(tmp_path): - from sqlalchemy import create_engine - - from app.assets.database.models import Base - db_path = str(tmp_path / "shipped_0007.db") cfg = _make_config(db_path) engine = create_engine(f"sqlite:///{db_path}") @@ -179,10 +179,6 @@ def test_0007_orm_parity(db_at_0006, tmp_path): ], ) def test_0007_index_orm_parity(db_at_0006, table_name, has_indexes): - from sqlalchemy import create_engine, inspect - - import app.assets.database.models as asset_models - cfg, db_path = db_at_0006 command.upgrade(cfg, "head") From 560a1ffa56abd311e277b577c0aa640580c7f90e Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 14:13:54 -0700 Subject: [PATCH 051/111] fix(assets): guard the hashing dependency and chain the real import error --- app/assets/services/snapshot_hash.py | 10 +++- .../services/test_snapshot_hash_import.py | 53 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 tests-unit/assets_test/services/test_snapshot_hash_import.py diff --git a/app/assets/services/snapshot_hash.py b/app/assets/services/snapshot_hash.py index d3e9b8c7158..6a80399d8d4 100644 --- a/app/assets/services/snapshot_hash.py +++ b/app/assets/services/snapshot_hash.py @@ -11,7 +11,13 @@ import os from dataclasses import dataclass -from blake3 import blake3 +try: + from blake3 import blake3 +except ImportError as error: + blake3 = None + _BLAKE3_IMPORT_ERROR: ImportError | None = error +else: + _BLAKE3_IMPORT_ERROR = None @dataclass(frozen=True, slots=True) @@ -34,6 +40,8 @@ def _snapshot(stat_result: os.stat_result) -> _Snapshot: def snapshot_hash( path: str, chunk_size: int = 8 * 1024 * 1024 ) -> tuple[str, os.stat_result] | None: + if blake3 is None: + raise ModuleNotFoundError("No module named 'blake3'") from _BLAKE3_IMPORT_ERROR try: pre_stat = _snapshot(os.stat(path)) hasher = blake3() diff --git a/tests-unit/assets_test/services/test_snapshot_hash_import.py b/tests-unit/assets_test/services/test_snapshot_hash_import.py new file mode 100644 index 00000000000..220d1fc2615 --- /dev/null +++ b/tests-unit/assets_test/services/test_snapshot_hash_import.py @@ -0,0 +1,53 @@ +import subprocess +import sys +from pathlib import Path + + +def test_snapshot_hash_defers_and_chains_blake3_import_failure() -> None: + script = """ +import builtins +import importlib.util +import sys +import tempfile +from pathlib import Path + +real_import = builtins.__import__ +blake3_import_error = ImportError("simulated blake3 ABI failure") + +def import_without_blake3(name, globals=None, locals=None, fromlist=(), level=0): + if name == "blake3" or name.startswith("blake3."): + raise blake3_import_error + return real_import(name, globals, locals, fromlist, level) + +builtins.__import__ = import_without_blake3 + +module_path = Path("app/assets/services/snapshot_hash.py") +spec = importlib.util.spec_from_file_location("isolated_snapshot_hash", module_path) +assert spec is not None +assert spec.loader is not None +snapshot_hash_module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = snapshot_hash_module +spec.loader.exec_module(snapshot_hash_module) + +snapshot_hash = snapshot_hash_module.snapshot_hash + +with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "asset.bin" + path.write_bytes(b"hash me") + try: + snapshot_hash(str(path)) + except ModuleNotFoundError as error: + assert error.__cause__ is blake3_import_error + else: + raise AssertionError("snapshot_hash should require blake3 at use time") +""" + + result = subprocess.run( + [sys.executable, "-c", script], + cwd=Path(__file__).parents[3], + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr From 8fffd7da739d11942e7584256178f4278e677a49 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 14:21:11 -0700 Subject: [PATCH 052/111] fix(assets): always resume background scanning when prompt handling fails --- main.py | 141 +++++++++--------- tests-unit/app_test/test_prompt_worker.py | 170 ++++++++++++++++++++++ 2 files changed, 246 insertions(+), 65 deletions(-) create mode 100644 tests-unit/app_test/test_prompt_worker.py diff --git a/main.py b/main.py index 3c6e41d353a..b2f96bd2e16 100644 --- a/main.py +++ b/main.py @@ -342,71 +342,82 @@ def prompt_worker(q, server_instance, asset_manager): gc_collect_interval = 10.0 while True: - timeout = 1000.0 - if need_gc: - timeout = max(gc_collect_interval - (current_time - last_gc_collect), 0.0) - - queue_item = q.get(timeout=timeout) - if queue_item is not None: - item, item_id = queue_item - execution_start_time = time.perf_counter() - prompt_id = item[1] - server_instance.last_prompt_id = prompt_id - - sensitive = item[5] - extra_data = item[3].copy() - for k in sensitive: - extra_data[k] = sensitive[k] - - asset_manager.pause_background_scan() - e.execute(item[2], prompt_id, extra_data, item[4]) - - need_gc = True - - remove_sensitive = lambda prompt: prompt[:5] + prompt[6:] - q.task_done(item_id, - e.history_result, - status=execution.PromptQueue.ExecutionStatus( - status_str='success' if e.success else 'error', - completed=e.success, - messages=e.status_messages), process_item=remove_sensitive) - if server_instance.client_id is not None: - server_instance.send_sync("executing", {"node": None, "prompt_id": prompt_id}, server_instance.client_id) - - current_time = time.perf_counter() - execution_time = current_time - execution_start_time - - # Log Time in a more readable way after 10 minutes - if execution_time > 600: - execution_time = time.strftime("%H:%M:%S", time.gmtime(execution_time)) - logging.info(f"Prompt executed in {execution_time}", extra={'color': 'green'}) - else: - logging.info("Prompt executed in {:.2f} seconds".format(execution_time), extra={'color': 'green'}) - - flags = q.get_flags() - free_memory = flags.get("free_memory", False) - - if flags.get("unload_models", free_memory): - comfy.model_management.unload_all_models() - need_gc = True - last_gc_collect = 0 - - if free_memory: - e.reset() - need_gc = True - last_gc_collect = 0 - - if need_gc: - current_time = time.perf_counter() - if (current_time - last_gc_collect) > gc_collect_interval: - gc.collect() - comfy.model_management.soft_empty_cache() - last_gc_collect = current_time - need_gc = False - hook_breaker_ac10a0.restore_functions() - - asset_manager.queue_output_scan() - asset_manager.resume_background_scan() + background_scan_paused = False + try: + timeout = 1000.0 + if need_gc: + timeout = max(gc_collect_interval - (current_time - last_gc_collect), 0.0) + + queue_item = q.get(timeout=timeout) + if queue_item is not None: + item, item_id = queue_item + execution_start_time = time.perf_counter() + prompt_id = item[1] + server_instance.last_prompt_id = prompt_id + + sensitive = item[5] + extra_data = item[3].copy() + for k in sensitive: + extra_data[k] = sensitive[k] + + asset_manager.pause_background_scan() + background_scan_paused = True + e.execute(item[2], prompt_id, extra_data, item[4]) + + need_gc = True + + remove_sensitive = lambda prompt: prompt[:5] + prompt[6:] + q.task_done(item_id, + e.history_result, + status=execution.PromptQueue.ExecutionStatus( + status_str='success' if e.success else 'error', + completed=e.success, + messages=e.status_messages), process_item=remove_sensitive) + if server_instance.client_id is not None: + server_instance.send_sync("executing", {"node": None, "prompt_id": prompt_id}, server_instance.client_id) + + current_time = time.perf_counter() + execution_time = current_time - execution_start_time + + # Log Time in a more readable way after 10 minutes + if execution_time > 600: + execution_time = time.strftime("%H:%M:%S", time.gmtime(execution_time)) + logging.info(f"Prompt executed in {execution_time}", extra={'color': 'green'}) + else: + logging.info("Prompt executed in {:.2f} seconds".format(execution_time), extra={'color': 'green'}) + + flags = q.get_flags() + free_memory = flags.get("free_memory", False) + + if flags.get("unload_models", free_memory): + comfy.model_management.unload_all_models() + need_gc = True + last_gc_collect = 0 + + if free_memory: + e.reset() + need_gc = True + last_gc_collect = 0 + + if need_gc: + current_time = time.perf_counter() + if (current_time - last_gc_collect) > gc_collect_interval: + gc.collect() + comfy.model_management.soft_empty_cache() + last_gc_collect = current_time + need_gc = False + hook_breaker_ac10a0.restore_functions() + + asset_manager.queue_output_scan() + asset_manager.resume_background_scan() + background_scan_paused = False + except BaseException: + if background_scan_paused: + try: + asset_manager.resume_background_scan() + except Exception: + logging.exception("Failed to resume background asset scanning after prompt worker failure") + raise async def run(server_instance, address='', port=8188, verbose=True, call_on_start=None): diff --git a/tests-unit/app_test/test_prompt_worker.py b/tests-unit/app_test/test_prompt_worker.py new file mode 100644 index 00000000000..3f96b50bfda --- /dev/null +++ b/tests-unit/app_test/test_prompt_worker.py @@ -0,0 +1,170 @@ +import subprocess +import sys +from pathlib import Path + + +def _run_prompt_worker(script: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-c", script], + cwd=Path(__file__).parents[2], + capture_output=True, + text=True, + check=False, + ) + + +def test_prompt_worker_resumes_background_scan_when_execute_raises() -> None: + script = """ +import sys + +sys.argv = ["main.py", "--cpu"] + +import main + +class Queue: + def get(self, timeout=None): + return (0, "prompt-id", {}, {}, [], {}), 1 + +class Server: + last_prompt_id = None + client_id = None + +class BackgroundScan: + paused = False + + def pause_background_scan(self): + self.paused = True + + def resume_background_scan(self): + self.paused = False + +class Executor: + def __init__(self, *args, **kwargs): + self.history_result = {} + self.success = True + self.status_messages = [] + + def execute(self, *args, **kwargs): + raise RuntimeError("forced execute failure") + +main.args.cache_classic = True +main.execution.PromptExecutor = Executor +asset_manager = BackgroundScan() + +try: + main.prompt_worker(Queue(), Server(), asset_manager) +except RuntimeError as error: + assert str(error) == "forced execute failure" + +assert asset_manager.paused is False +""" + + result = _run_prompt_worker(script) + + assert result.returncode == 0, result.stderr + + +def test_prompt_worker_resumes_background_scan_when_completion_raises() -> None: + script = """ +import sys + +sys.argv = ["main.py", "--cpu"] + +import main + +class Queue: + def get(self, timeout=None): + return (0, "prompt-id", {}, {}, [], {}), 1 + + def task_done(self, *args, **kwargs): + raise RuntimeError("forced completion failure") + +class Server: + last_prompt_id = None + client_id = None + +class BackgroundScan: + paused = False + + def pause_background_scan(self): + self.paused = True + + def resume_background_scan(self): + self.paused = False + +class Executor: + def __init__(self, *args, **kwargs): + self.history_result = {} + self.success = True + self.status_messages = [] + + def execute(self, *args, **kwargs): + return None + +main.args.cache_classic = True +main.execution.PromptExecutor = Executor +asset_manager = BackgroundScan() + +try: + main.prompt_worker(Queue(), Server(), asset_manager) +except RuntimeError as error: + assert str(error) == "forced completion failure" + +assert asset_manager.paused is False +""" + + result = _run_prompt_worker(script) + + assert result.returncode == 0, result.stderr + + +def test_prompt_worker_preserves_execute_error_when_resume_raises() -> None: + script = """ +import sys + +sys.argv = ["main.py", "--cpu"] + +import main + +class Queue: + def get(self, timeout=None): + return (0, "prompt-id", {}, {}, [], {}), 1 + +class Server: + last_prompt_id = None + client_id = None + +class BackgroundScan: + paused = False + + def pause_background_scan(self): + self.paused = True + + def resume_background_scan(self): + self.paused = False + raise RuntimeError("forced resume failure") + +class Executor: + def __init__(self, *args, **kwargs): + pass + + def execute(self, *args, **kwargs): + raise RuntimeError("forced execute failure") + +main.args.cache_classic = True +main.execution.PromptExecutor = Executor +asset_manager = BackgroundScan() + +try: + main.prompt_worker(Queue(), Server(), asset_manager) +except RuntimeError as error: + assert str(error) == "forced execute failure" +else: + raise AssertionError("prompt_worker should propagate the execute failure") + +assert asset_manager.paused is False +""" + + result = _run_prompt_worker(script) + + assert result.returncode == 0, result.stderr From 9f2b470216f5c00aa2f635e07a69ada511402bf8 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 14:25:38 -0700 Subject: [PATCH 053/111] fix(api): derive the assets feature flag from the selected manager --- comfy_api/feature_flags.py | 6 ++---- server.py | 6 ++++-- .../assets_test/services/test_noassets_parity.py | 14 ++++++++++++++ tests-unit/feature_flags_test.py | 3 +++ 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/comfy_api/feature_flags.py b/comfy_api/feature_flags.py index 00f6bb65f5b..d0034b7445e 100644 --- a/comfy_api/feature_flags.py +++ b/comfy_api/feature_flags.py @@ -109,8 +109,6 @@ def _parse_cli_feature_flags() -> dict[str, Any]: "max_upload_size": args.max_upload_size * 1024 * 1024, # Convert MB to bytes "extension": {"manager": {"supports_v4": True}}, "node_replacements": True, - # Mirrors the constructed AssetManager; no degradation path exists, so this always agrees with it. - "assets": args.enable_assets, } # CLI-provided flags cannot overwrite core flags @@ -162,11 +160,11 @@ def supports_feature( return get_connection_feature(sockets_metadata, sid, feature_name, False) is True -def get_server_features() -> dict[str, Any]: +def get_server_features(assets_enabled: bool = False) -> dict[str, Any]: """ Get the server's feature flags. Returns: Dictionary of server feature flags """ - return SERVER_FEATURE_FLAGS.copy() + return {**SERVER_FEATURE_FLAGS, "assets": assets_enabled} diff --git a/server.py b/server.py index ac871911088..c9a5bf8d1b6 100644 --- a/server.py +++ b/server.py @@ -307,7 +307,9 @@ async def websocket_handler(request): # Send server feature flags in response await self.send( "feature_flags", - feature_flags.get_server_features(), + feature_flags.get_server_features( + self.asset_manager.enabled + ), sid, ) @@ -741,7 +743,7 @@ async def system_stats(request): @routes.get("/features") async def get_features(request): - features = feature_flags.get_server_features() + features = feature_flags.get_server_features(self.asset_manager.enabled) overrides = get_environment_overrides() if overrides: features.update(overrides) diff --git a/tests-unit/assets_test/services/test_noassets_parity.py b/tests-unit/assets_test/services/test_noassets_parity.py index 5c7b646b69d..fa1944e5b5a 100644 --- a/tests-unit/assets_test/services/test_noassets_parity.py +++ b/tests-unit/assets_test/services/test_noassets_parity.py @@ -19,6 +19,7 @@ from app.assets.mode import hashing_enabled from app.assets.seeder import asset_seeder from app.assets.services.hash_mode_state import read_stored_mode +from comfy_api import feature_flags class _Args: @@ -193,6 +194,19 @@ def test_default_asset_manager_disables_assets_when_dependencies_are_unavailable assert "requirements.txt" in caplog.text +def test_server_features_match_manager_when_asset_dependencies_are_unavailable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(manager.args, "enable_assets", True) + monkeypatch.setattr(manager, "dependencies_available", lambda: False) + monkeypatch.setitem(feature_flags.SERVER_FEATURE_FLAGS, "assets", True) + + asset_manager = manager.default_asset_manager() + + assert asset_manager.enabled is False + assert feature_flags.get_server_features()["assets"] is False + + def test_default_asset_manager_enables_assets_when_dependencies_are_available( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests-unit/feature_flags_test.py b/tests-unit/feature_flags_test.py index df16df6ab10..ad2a51d3c22 100644 --- a/tests-unit/feature_flags_test.py +++ b/tests-unit/feature_flags_test.py @@ -39,6 +39,9 @@ def test_get_server_features_contains_expected_flags(self): assert "max_upload_size" in features assert isinstance(features["max_upload_size"], (int, float)) + def test_get_server_features_without_manager_defaults_assets_to_disabled(self): + assert get_server_features()["assets"] is False + def test_get_connection_feature_with_missing_sid(self): """Test getting feature for non-existent session ID.""" sockets_metadata = {} From 0297d729a30c698243d1a103a2433952f03c12ba Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 14:52:52 -0700 Subject: [PATCH 054/111] fix(assets): paginate enrichment by id cursor so failures cannot starve or overflow the query --- app/assets/scanner.py | 64 ++++++--- app/assets/seeder.py | 29 +--- .../services/test_transition_settle.py | 132 +++++++++++++++++- 3 files changed, 178 insertions(+), 47 deletions(-) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index 76821e4c337..d324992c880 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -2,9 +2,8 @@ paths, building specs, seeding new content and records, then enriching them with metadata and hashes. Each spec is seeded inside its own savepoint, so one file whose row conflicts cannot discard the work done for the files around it. -Enrichment counts as progress only when it produced what was asked of it — a -requested hash that could not be computed is no progress, which is what bounds -a pass over a file the server cannot read. +Enrichment candidates use ordered ID pagination, so each row is selected at +most once per pass while failed rows remain eligible for the next pass. """ import logging @@ -444,10 +443,42 @@ def insert_asset_specs(specs: list[SeedAssetSpec], _tag_pool: set[str]) -> int: return created +def build_unenriched_candidates_statement( + prefixes: list[str], + compute_hashes: bool, + last_seen_id: str | None, + limit: int = 1000, +) -> sa.Select[tuple[str, str, str]]: + query = ( + sa.select(AssetContent.id, Asset.id, AssetContent.path) + .join(Asset, Asset.content_id == AssetContent.id) + .where(AssetContent.is_missing.is_(False)) + ) + if compute_hashes: + query = query.where( + sa.or_( + AssetContent.hash.is_(None), + Asset.system_metadata.is_(None), + ) + ) + else: + query = query.where(Asset.system_metadata.is_(None)) + if last_seen_id is not None: + query = query.where(Asset.id > last_seen_id) + return ( + query.where( + sa.or_(*(sql_path_under_prefix(AssetContent.path, p) for p in prefixes)) + ) + .order_by(Asset.id.asc()) + .limit(limit) + ) + + def get_unenriched_assets_for_roots( roots: tuple[RootType, ...], compute_hashes: bool, limit: int = 1000, + last_seen_id: str | None = None, ) -> list[UnenrichedContent]: prefixes: list[str] = [] for root in roots: @@ -456,27 +487,14 @@ def get_unenriched_assets_for_roots( if not prefixes: return [] + query = build_unenriched_candidates_statement( + prefixes, + compute_hashes, + last_seen_id, + limit, + ) with create_session() as sess: - query = ( - sa.select(AssetContent.id, Asset.id, AssetContent.path) - .join(Asset, Asset.content_id == AssetContent.id) - .where(AssetContent.is_missing.is_(False)) - ) - if compute_hashes: - query = query.where( - sa.or_( - AssetContent.hash.is_(None), - Asset.system_metadata.is_(None), - ) - ) - else: - query = query.where(Asset.system_metadata.is_(None)) - query = query.where( - sa.or_( - *(sql_path_under_prefix(AssetContent.path, p) for p in prefixes) - ) - ) - rows = sess.execute(query.order_by(Asset.id).limit(limit)).all() + rows = sess.execute(query).all() return [ UnenrichedContent(content_id, record_id, file_path) diff --git a/app/assets/seeder.py b/app/assets/seeder.py index a78f1abf440..87d48c533bd 100644 --- a/app/assets/seeder.py +++ b/app/assets/seeder.py @@ -2,8 +2,8 @@ exposing pause, resume, cancel and progress to the API. A run seeds newly-observed files first, then enriches records in batches, and settles any pending hash-mode transition at the start of the enrich phase so a server that -receives no prompts still completes the switch. A pass stops once batches stop -making progress, bounding a scan over files that cannot be read. +receives no prompts still completes the switch. An enrichment pass ends when +its ordered candidate cursor is exhausted. """ import logging @@ -910,9 +910,7 @@ def _run_enrich_phase(self, roots: tuple[RootType, ...]) -> tuple[bool, int]: {"roots": list(roots), "phase": "enrich"}, ) - skip_ids: set[str] = set() - consecutive_empty = 0 - max_consecutive_empty = 3 + last_seen_id: str | None = None while True: if self._check_pause_and_cancel(_ScanStage.ENRICH): @@ -924,16 +922,13 @@ def _run_enrich_phase(self, roots: tuple[RootType, ...]) -> tuple[bool, int]: roots, compute_hashes=self._compute_hashes, limit=batch_size, + last_seen_id=last_seen_id, ) - # Filter out previously failed references - if skip_ids: - unenriched = [row for row in unenriched if row.record_id not in skip_ids] - if not unenriched: break - enriched, failed_ids = enrich_assets_batch( + enriched, _failed_ids = enrich_assets_batch( unenriched, extract_metadata=True, compute_hash=self._compute_hashes, @@ -941,19 +936,7 @@ def _run_enrich_phase(self, roots: tuple[RootType, ...]) -> tuple[bool, int]: progress=scan_state, ) total_enriched += enriched - skip_ids.update(failed_ids) - - if enriched == 0: - consecutive_empty += 1 - if consecutive_empty >= max_consecutive_empty: - logging.warning( - "Enrich phase stopping: %d consecutive batches with no progress (%d skipped)", - consecutive_empty, - len(skip_ids), - ) - break - else: - consecutive_empty = 0 + last_seen_id = unenriched[-1].record_id now = time.perf_counter() if now - last_progress_time >= progress_interval: diff --git a/tests-unit/assets_test/services/test_transition_settle.py b/tests-unit/assets_test/services/test_transition_settle.py index 3da231ba599..c1c842627a5 100644 --- a/tests-unit/assets_test/services/test_transition_settle.py +++ b/tests-unit/assets_test/services/test_transition_settle.py @@ -1,12 +1,14 @@ from contextlib import contextmanager +from inspect import signature from pathlib import Path from unittest.mock import patch import pytest +from sqlalchemy.dialects import sqlite from sqlalchemy.orm import Session as SASession from app.assets import scanner, seeder as seeder_module -from app.assets.database.models import AssetContent +from app.assets.database.models import Asset, AssetContent from app.assets.database.queries.records import create_content, create_record from app.assets.helpers import to_stored_hash from app.assets.services import hash_mode_state @@ -37,6 +39,36 @@ def _denied(_candidate_path: str): raise PermissionError("denied") +def _rendered_candidate_placeholder_count(failed_count: int) -> int: + failed_ids = [f"{index:036d}" for index in range(failed_count)] + builder = getattr(scanner, "build_unenriched_candidates_statement") + pagination = ( + failed_ids + if "skip_ids" in signature(builder).parameters + else failed_ids[-1] + ) + statement = builder(["/models"], False, pagination, 100) + compiled = statement.compile( + dialect=sqlite.dialect(), + compile_kwargs={"render_postcompile": True}, + ) + rendered_count = str(compiled).count("?") + + assert compiled.positiontup is not None + assert len(compiled.positiontup) == rendered_count + return rendered_count + + +def test_unenriched_candidate_bind_count_does_not_scale_with_failed_rows() -> None: + count_for_ten = _rendered_candidate_placeholder_count(10) + count_for_thousand = _rendered_candidate_placeholder_count(1000) + + assert count_for_ten == count_for_thousand, ( + "rendered placeholders scaled with failed candidates: " + f"N=10 -> {count_for_ten}; N=1000 -> {count_for_thousand}" + ) + + def test_enrich_phase_settles_an_unreadable_transition_without_looping( session, db_engine, temp_dir: Path, monkeypatch ): @@ -106,3 +138,101 @@ def counting_enrich_asset(*args, **kwargs): assert attempts == [record_id], ( "the record is attempted once, then excluded from the rest of the pass" ) + + +def test_enrich_phase_reaches_healthy_candidates_after_a_full_failed_batch( + session, db_engine, temp_dir: Path, monkeypatch +): + for index in range(101): + path = temp_dir / f"candidate-{index}.safetensors" + content = create_content(session, str(path)) + create_record(session, content.id, path.name) + session.commit() + + @contextmanager + def _create_session(): + with SASession(db_engine) as sess: + yield sess + + monkeypatch.setattr("folder_paths.get_input_directory", lambda: str(temp_dir)) + with patch("app.assets.scanner.create_session", _create_session): + ordered = scanner.get_unenriched_assets_for_roots( + ("input",), compute_hashes=False, limit=101 + ) + + failed_ids = {row.record_id for row in ordered[:100]} + healthy = ordered[100] + attempted: list[str] = [] + + def enrich_batch(rows, **_kwargs): + attempted.extend(row.record_id for row in rows) + failed = [row.record_id for row in rows if row.record_id in failed_ids] + if healthy.record_id in attempted: + with SASession(db_engine) as update_session: + record = update_session.get(Asset, healthy.record_id) + assert record is not None + record.system_metadata = {"enriched": True} + update_session.commit() + return 1, failed + return 0, failed + + asset_seeder = seeder_module._AssetSeeder() + asset_seeder._run_gate.set() + asset_seeder._cancel_event.clear() + + with ( + patch("app.assets.seeder.create_session", _create_session), + patch("app.assets.scanner.create_session", _create_session), + patch("app.assets.seeder.enrich_assets_batch", enrich_batch), + ): + cancelled, enriched = asset_seeder._run_enrich_phase(("input",)) + + assert cancelled is False + assert enriched == 1 + assert healthy.record_id in attempted + + +def test_enrich_phase_reaches_healthy_candidates_after_four_failed_batches( + db_engine, monkeypatch +): + failed_batches = [ + [scanner.UnenrichedContent(f"content-{index}", f"record-{index}", f"/{index}")] + for index in range(4) + ] + healthy = scanner.UnenrichedContent("healthy-content", "healthy-record", "/healthy") + batches = iter([*failed_batches, [healthy], []]) + attempted: list[str] = [] + + @contextmanager + def _create_session(): + with SASession(db_engine) as sess: + yield sess + + def enrich_batch(rows, **_kwargs): + attempted.extend(row.record_id for row in rows) + if healthy in rows: + return 1, [] + return 0, [row.record_id for row in rows] + + asset_seeder = seeder_module._AssetSeeder() + asset_seeder._run_gate.set() + asset_seeder._cancel_event.clear() + monkeypatch.setattr( + seeder_module, + "get_unenriched_assets_for_roots", + lambda *_args, **_kwargs: next(batches), + ) + monkeypatch.setattr(seeder_module, "enrich_assets_batch", enrich_batch) + + with patch("app.assets.seeder.create_session", _create_session): + cancelled, enriched = asset_seeder._run_enrich_phase(("input",)) + + assert cancelled is False + assert enriched == 1 + assert attempted == [ + "record-0", + "record-1", + "record-2", + "record-3", + "healthy-record", + ] From ac1b25e02fd0e151f25b143ca756147e11081a7f Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 16:15:53 -0700 Subject: [PATCH 055/111] refactor(assets): extract prompt_worker so its resume contract is testable in-process --- app/prompt_worker.py | 112 +++++++++++++ main.py | 107 +------------ tests-unit/app_test/test_prompt_worker.py | 186 +++++++--------------- 3 files changed, 172 insertions(+), 233 deletions(-) create mode 100644 app/prompt_worker.py diff --git a/app/prompt_worker.py b/app/prompt_worker.py new file mode 100644 index 00000000000..51fa45ca578 --- /dev/null +++ b/app/prompt_worker.py @@ -0,0 +1,112 @@ +import gc +import logging +import time + +import comfy.model_management +import execution +import hook_breaker_ac10a0 +from comfy.cli_args import args + + +def prompt_worker(q, server_instance, asset_manager): + current_time: float = 0.0 + cache_ram = 0 + cache_ram_inactive = 0 + if not args.cache_classic and not args.cache_none and args.cache_lru <= 0: + cache_ram = min(10.0, max(2.0, comfy.model_management.total_ram * 0.10 / 1024.0)) + cache_ram_inactive = min(128.0, comfy.model_management.total_ram / 1024.0) + if len(args.cache_ram) > 0: + cache_ram = args.cache_ram[0] + if len(args.cache_ram) > 1: + cache_ram_inactive = args.cache_ram[1] + + cache_type = execution.CacheType.RAM_PRESSURE + if args.cache_classic: + cache_type = execution.CacheType.CLASSIC + elif args.cache_lru > 0: + cache_type = execution.CacheType.LRU + elif args.cache_none: + cache_type = execution.CacheType.NONE + + e = execution.PromptExecutor(server_instance, cache_type=cache_type, cache_args={ "lru" : args.cache_lru, "ram" : cache_ram, "ram_inactive" : cache_ram_inactive }, asset_manager=asset_manager ) + last_gc_collect = 0 + need_gc = False + gc_collect_interval = 10.0 + + while True: + background_scan_paused = False + try: + timeout = 1000.0 + if need_gc: + timeout = max(gc_collect_interval - (current_time - last_gc_collect), 0.0) + + queue_item = q.get(timeout=timeout) + if queue_item is not None: + item, item_id = queue_item + execution_start_time = time.perf_counter() + prompt_id = item[1] + server_instance.last_prompt_id = prompt_id + + sensitive = item[5] + extra_data = item[3].copy() + for k in sensitive: + extra_data[k] = sensitive[k] + + asset_manager.pause_background_scan() + background_scan_paused = True + e.execute(item[2], prompt_id, extra_data, item[4]) + + need_gc = True + + remove_sensitive = lambda prompt: prompt[:5] + prompt[6:] + q.task_done(item_id, + e.history_result, + status=execution.PromptQueue.ExecutionStatus( + status_str='success' if e.success else 'error', + completed=e.success, + messages=e.status_messages), process_item=remove_sensitive) + if server_instance.client_id is not None: + server_instance.send_sync("executing", {"node": None, "prompt_id": prompt_id}, server_instance.client_id) + + current_time = time.perf_counter() + execution_time = current_time - execution_start_time + + # Log Time in a more readable way after 10 minutes + if execution_time > 600: + execution_time = time.strftime("%H:%M:%S", time.gmtime(execution_time)) + logging.info(f"Prompt executed in {execution_time}", extra={'color': 'green'}) + else: + logging.info("Prompt executed in {:.2f} seconds".format(execution_time), extra={'color': 'green'}) + + flags = q.get_flags() + free_memory = flags.get("free_memory", False) + + if flags.get("unload_models", free_memory): + comfy.model_management.unload_all_models() + need_gc = True + last_gc_collect = 0 + + if free_memory: + e.reset() + need_gc = True + last_gc_collect = 0 + + if need_gc: + current_time = time.perf_counter() + if (current_time - last_gc_collect) > gc_collect_interval: + gc.collect() + comfy.model_management.soft_empty_cache() + last_gc_collect = current_time + need_gc = False + hook_breaker_ac10a0.restore_functions() + + asset_manager.queue_output_scan() + asset_manager.resume_background_scan() + background_scan_paused = False + except BaseException: + if background_scan_paused: + try: + asset_manager.resume_background_scan() + except Exception: + logging.exception("Failed to resume background asset scanning after prompt worker failure") + raise diff --git a/main.py b/main.py index b2f96bd2e16..5ec031009cb 100644 --- a/main.py +++ b/main.py @@ -240,7 +240,6 @@ def execute_script(script_path): # Main code import asyncio import threading -import gc if 'torch' in sys.modules: logging.warning("WARNING: Potential Error in code: Torch already imported, torch should never be imported before this point.") @@ -248,7 +247,7 @@ def execute_script(script_path): import comfy.utils -import execution +from app.prompt_worker import prompt_worker import server from protocol import BinaryEventTypes import nodes @@ -316,110 +315,6 @@ def cuda_malloc_warning(): logging.warning("\nWARNING: this card most likely does not support cuda-malloc, if you get \"CUDA error\" please run ComfyUI with: --disable-cuda-malloc\n") -def prompt_worker(q, server_instance, asset_manager): - current_time: float = 0.0 - cache_ram = 0 - cache_ram_inactive = 0 - if not args.cache_classic and not args.cache_none and args.cache_lru <= 0: - cache_ram = min(10.0, max(2.0, comfy.model_management.total_ram * 0.10 / 1024.0)) - cache_ram_inactive = min(128.0, comfy.model_management.total_ram / 1024.0) - if len(args.cache_ram) > 0: - cache_ram = args.cache_ram[0] - if len(args.cache_ram) > 1: - cache_ram_inactive = args.cache_ram[1] - - cache_type = execution.CacheType.RAM_PRESSURE - if args.cache_classic: - cache_type = execution.CacheType.CLASSIC - elif args.cache_lru > 0: - cache_type = execution.CacheType.LRU - elif args.cache_none: - cache_type = execution.CacheType.NONE - - e = execution.PromptExecutor(server_instance, cache_type=cache_type, cache_args={ "lru" : args.cache_lru, "ram" : cache_ram, "ram_inactive" : cache_ram_inactive }, asset_manager=asset_manager ) - last_gc_collect = 0 - need_gc = False - gc_collect_interval = 10.0 - - while True: - background_scan_paused = False - try: - timeout = 1000.0 - if need_gc: - timeout = max(gc_collect_interval - (current_time - last_gc_collect), 0.0) - - queue_item = q.get(timeout=timeout) - if queue_item is not None: - item, item_id = queue_item - execution_start_time = time.perf_counter() - prompt_id = item[1] - server_instance.last_prompt_id = prompt_id - - sensitive = item[5] - extra_data = item[3].copy() - for k in sensitive: - extra_data[k] = sensitive[k] - - asset_manager.pause_background_scan() - background_scan_paused = True - e.execute(item[2], prompt_id, extra_data, item[4]) - - need_gc = True - - remove_sensitive = lambda prompt: prompt[:5] + prompt[6:] - q.task_done(item_id, - e.history_result, - status=execution.PromptQueue.ExecutionStatus( - status_str='success' if e.success else 'error', - completed=e.success, - messages=e.status_messages), process_item=remove_sensitive) - if server_instance.client_id is not None: - server_instance.send_sync("executing", {"node": None, "prompt_id": prompt_id}, server_instance.client_id) - - current_time = time.perf_counter() - execution_time = current_time - execution_start_time - - # Log Time in a more readable way after 10 minutes - if execution_time > 600: - execution_time = time.strftime("%H:%M:%S", time.gmtime(execution_time)) - logging.info(f"Prompt executed in {execution_time}", extra={'color': 'green'}) - else: - logging.info("Prompt executed in {:.2f} seconds".format(execution_time), extra={'color': 'green'}) - - flags = q.get_flags() - free_memory = flags.get("free_memory", False) - - if flags.get("unload_models", free_memory): - comfy.model_management.unload_all_models() - need_gc = True - last_gc_collect = 0 - - if free_memory: - e.reset() - need_gc = True - last_gc_collect = 0 - - if need_gc: - current_time = time.perf_counter() - if (current_time - last_gc_collect) > gc_collect_interval: - gc.collect() - comfy.model_management.soft_empty_cache() - last_gc_collect = current_time - need_gc = False - hook_breaker_ac10a0.restore_functions() - - asset_manager.queue_output_scan() - asset_manager.resume_background_scan() - background_scan_paused = False - except BaseException: - if background_scan_paused: - try: - asset_manager.resume_background_scan() - except Exception: - logging.exception("Failed to resume background asset scanning after prompt worker failure") - raise - - async def run(server_instance, address='', port=8188, verbose=True, call_on_start=None): addresses = [] for addr in address.split(","): diff --git a/tests-unit/app_test/test_prompt_worker.py b/tests-unit/app_test/test_prompt_worker.py index 3f96b50bfda..0634beb069b 100644 --- a/tests-unit/app_test/test_prompt_worker.py +++ b/tests-unit/app_test/test_prompt_worker.py @@ -1,170 +1,102 @@ -import subprocess -import sys -from pathlib import Path +import importlib +import pytest -def _run_prompt_worker(script: str) -> subprocess.CompletedProcess[str]: - return subprocess.run( - [sys.executable, "-c", script], - cwd=Path(__file__).parents[2], - capture_output=True, - text=True, - check=False, - ) +class LoopEscape(Exception): + pass -def test_prompt_worker_resumes_background_scan_when_execute_raises() -> None: - script = """ -import sys - -sys.argv = ["main.py", "--cpu"] - -import main class Queue: - def get(self, timeout=None): - return (0, "prompt-id", {}, {}, [], {}), 1 - -class Server: - last_prompt_id = None - client_id = None - -class BackgroundScan: - paused = False - - def pause_background_scan(self): - self.paused = True - - def resume_background_scan(self): - self.paused = False - -class Executor: - def __init__(self, *args, **kwargs): - self.history_result = {} - self.success = True - self.status_messages = [] - - def execute(self, *args, **kwargs): - raise RuntimeError("forced execute failure") - -main.args.cache_classic = True -main.execution.PromptExecutor = Executor -asset_manager = BackgroundScan() - -try: - main.prompt_worker(Queue(), Server(), asset_manager) -except RuntimeError as error: - assert str(error) == "forced execute failure" - -assert asset_manager.paused is False -""" - - result = _run_prompt_worker(script) - - assert result.returncode == 0, result.stderr + def __init__(self, completion_error: RuntimeError | None = None) -> None: + self.completion_error = completion_error + self.get_calls = 0 - -def test_prompt_worker_resumes_background_scan_when_completion_raises() -> None: - script = """ -import sys - -sys.argv = ["main.py", "--cpu"] - -import main - -class Queue: def get(self, timeout=None): + self.get_calls += 1 + if self.get_calls > 1: + raise LoopEscape("prompt worker requested a second item") return (0, "prompt-id", {}, {}, [], {}), 1 - def task_done(self, *args, **kwargs): - raise RuntimeError("forced completion failure") + def task_done(self, *args, **kwargs) -> None: + if self.completion_error is not None: + raise self.completion_error + class Server: last_prompt_id = None client_id = None -class BackgroundScan: - paused = False - def pause_background_scan(self): +class AssetManager: + def __init__(self, resume_error: RuntimeError | None = None) -> None: + self.paused = False + self.resume_error = resume_error + + def pause_background_scan(self) -> None: self.paused = True - def resume_background_scan(self): + def resume_background_scan(self) -> None: self.paused = False + if self.resume_error is not None: + raise self.resume_error + class Executor: - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: self.history_result = {} self.success = True self.status_messages = [] - def execute(self, *args, **kwargs): + def execute(self, *args, **kwargs) -> None: return None -main.args.cache_classic = True -main.execution.PromptExecutor = Executor -asset_manager = BackgroundScan() - -try: - main.prompt_worker(Queue(), Server(), asset_manager) -except RuntimeError as error: - assert str(error) == "forced completion failure" - -assert asset_manager.paused is False -""" - - result = _run_prompt_worker(script) - - assert result.returncode == 0, result.stderr +class ExecuteFailureExecutor(Executor): + def execute(self, *args, **kwargs) -> None: + raise RuntimeError("forced execute failure") -def test_prompt_worker_preserves_execute_error_when_resume_raises() -> None: - script = """ -import sys -sys.argv = ["main.py", "--cpu"] +@pytest.fixture +def prompt_worker_module(monkeypatch): + from comfy.cli_args import args -import main + monkeypatch.setattr(args, "cpu", True, raising=False) + try: + return importlib.import_module("app.prompt_worker") + except Exception as exc: + pytest.skip(f"prompt worker module could not be imported in CPU mode: {exc!r}") -class Queue: - def get(self, timeout=None): - return (0, "prompt-id", {}, {}, [], {}), 1 -class Server: - last_prompt_id = None - client_id = None +def test_prompt_worker_resumes_background_scan_when_execute_raises(prompt_worker_module, monkeypatch) -> None: + monkeypatch.setattr(prompt_worker_module.execution, "PromptExecutor", ExecuteFailureExecutor) + asset_manager = AssetManager() -class BackgroundScan: - paused = False + with pytest.raises(RuntimeError, match="^forced execute failure$"): + prompt_worker_module.prompt_worker(Queue(), Server(), asset_manager) - def pause_background_scan(self): - self.paused = True + assert asset_manager.paused is False - def resume_background_scan(self): - self.paused = False - raise RuntimeError("forced resume failure") -class Executor: - def __init__(self, *args, **kwargs): - pass +def test_prompt_worker_resumes_background_scan_when_completion_raises(prompt_worker_module, monkeypatch) -> None: + monkeypatch.setattr(prompt_worker_module.execution, "PromptExecutor", Executor) + asset_manager = AssetManager() - def execute(self, *args, **kwargs): - raise RuntimeError("forced execute failure") + with pytest.raises(RuntimeError, match="^forced completion failure$"): + prompt_worker_module.prompt_worker( + Queue(completion_error=RuntimeError("forced completion failure")), + Server(), + asset_manager, + ) -main.args.cache_classic = True -main.execution.PromptExecutor = Executor -asset_manager = BackgroundScan() + assert asset_manager.paused is False -try: - main.prompt_worker(Queue(), Server(), asset_manager) -except RuntimeError as error: - assert str(error) == "forced execute failure" -else: - raise AssertionError("prompt_worker should propagate the execute failure") -assert asset_manager.paused is False -""" +def test_prompt_worker_preserves_execute_error_when_resume_raises(prompt_worker_module, monkeypatch) -> None: + monkeypatch.setattr(prompt_worker_module.execution, "PromptExecutor", ExecuteFailureExecutor) + asset_manager = AssetManager(resume_error=RuntimeError("forced resume failure")) - result = _run_prompt_worker(script) + with pytest.raises(RuntimeError, match="^forced execute failure$"): + prompt_worker_module.prompt_worker(Queue(), Server(), asset_manager) - assert result.returncode == 0, result.stderr + assert asset_manager.paused is False From 5ce874200cbfd90c4c3c2a2220a7c94ebde64f91 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 16:34:57 -0700 Subject: [PATCH 056/111] refactor(assets): test the blake3 import guard in-process instead of via subprocess --- .../services/test_snapshot_hash_import.py | 67 +++++++------------ 1 file changed, 26 insertions(+), 41 deletions(-) diff --git a/tests-unit/assets_test/services/test_snapshot_hash_import.py b/tests-unit/assets_test/services/test_snapshot_hash_import.py index 220d1fc2615..1d60f961fa2 100644 --- a/tests-unit/assets_test/services/test_snapshot_hash_import.py +++ b/tests-unit/assets_test/services/test_snapshot_hash_import.py @@ -1,53 +1,38 @@ -import subprocess -import sys -from pathlib import Path - - -def test_snapshot_hash_defers_and_chains_blake3_import_failure() -> None: - script = """ import builtins import importlib.util import sys -import tempfile from pathlib import Path +from unittest.mock import patch -real_import = builtins.__import__ -blake3_import_error = ImportError("simulated blake3 ABI failure") +import pytest -def import_without_blake3(name, globals=None, locals=None, fromlist=(), level=0): - if name == "blake3" or name.startswith("blake3."): - raise blake3_import_error - return real_import(name, globals, locals, fromlist, level) -builtins.__import__ = import_without_blake3 +def test_snapshot_hash_defers_and_chains_blake3_import_failure(tmp_path: Path) -> None: + real_import = builtins.__import__ + blake3_import_error = ImportError("simulated blake3 ABI failure") -module_path = Path("app/assets/services/snapshot_hash.py") -spec = importlib.util.spec_from_file_location("isolated_snapshot_hash", module_path) -assert spec is not None -assert spec.loader is not None -snapshot_hash_module = importlib.util.module_from_spec(spec) -sys.modules[spec.name] = snapshot_hash_module -spec.loader.exec_module(snapshot_hash_module) + def import_without_blake3(name, globals=None, locals=None, fromlist=(), level=0): + if name == "blake3" or name.startswith("blake3."): + raise blake3_import_error + return real_import(name, globals, locals, fromlist, level) -snapshot_hash = snapshot_hash_module.snapshot_hash - -with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "asset.bin" - path.write_bytes(b"hash me") + module_path = Path(__file__).parents[3] / "app/assets/services/snapshot_hash.py" + spec = importlib.util.spec_from_file_location( + "isolated_snapshot_hash_import_guard", module_path + ) + assert spec is not None + assert spec.loader is not None + snapshot_hash_module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = snapshot_hash_module try: - snapshot_hash(str(path)) - except ModuleNotFoundError as error: - assert error.__cause__ is blake3_import_error - else: - raise AssertionError("snapshot_hash should require blake3 at use time") -""" + with patch("builtins.__import__", side_effect=import_without_blake3): + spec.loader.exec_module(snapshot_hash_module) - result = subprocess.run( - [sys.executable, "-c", script], - cwd=Path(__file__).parents[3], - capture_output=True, - text=True, - check=False, - ) + path = tmp_path / "asset.bin" + path.write_bytes(b"hash me") + with pytest.raises(ModuleNotFoundError) as error_info: + snapshot_hash_module.snapshot_hash(str(path)) - assert result.returncode == 0, result.stderr + assert error_info.value.__cause__ is blake3_import_error + finally: + sys.modules.pop(spec.name, None) From e30769a9f87e7c0deebdd652fc479119a07fa811 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 01:08:18 -0700 Subject: [PATCH 057/111] fix(assets): advance the enrichment cursor only past rows the batch attempted --- app/assets/scanner.py | 8 +- app/assets/seeder.py | 5 +- .../services/test_transition_settle.py | 110 +++++++++++++++++- .../test_scanner_ingest_event_log.py | 14 ++- tests-unit/seeder_test/test_seeder.py | 2 +- 5 files changed, 125 insertions(+), 14 deletions(-) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index d324992c880..3e9461ae678 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -631,7 +631,7 @@ def enrich_assets_batch( compute_hash: bool = False, interrupt_check: Callable[[], bool] | None = None, progress: _ScanProgress | None = None, -) -> tuple[int, list[str]]: +) -> tuple[int, list[str], int]: """Enrich a batch of assets. Uses a single DB session for the entire batch, committing after each @@ -646,15 +646,17 @@ def enrich_assets_batch( the operation should be interrupted (e.g. paused or cancelled) Returns: - Tuple of (enriched_count, failed_reference_ids) + Tuple of (enriched_count, failed_reference_ids, consumed_count) """ enriched = 0 failed_ids: list[str] = [] + consumed = 0 with create_session() as sess: for row in rows: if interrupt_check is not None and interrupt_check(): break + consumed += 1 try: updated = enrich_asset( @@ -679,4 +681,4 @@ def enrich_assets_batch( sess.rollback() failed_ids.append(row.record_id) - return enriched, failed_ids + return enriched, failed_ids, consumed diff --git a/app/assets/seeder.py b/app/assets/seeder.py index 87d48c533bd..9fa027ee9b7 100644 --- a/app/assets/seeder.py +++ b/app/assets/seeder.py @@ -928,7 +928,7 @@ def _run_enrich_phase(self, roots: tuple[RootType, ...]) -> tuple[bool, int]: if not unenriched: break - enriched, _failed_ids = enrich_assets_batch( + enriched, _failed_ids, consumed = enrich_assets_batch( unenriched, extract_metadata=True, compute_hash=self._compute_hashes, @@ -936,7 +936,8 @@ def _run_enrich_phase(self, roots: tuple[RootType, ...]) -> tuple[bool, int]: progress=scan_state, ) total_enriched += enriched - last_seen_id = unenriched[-1].record_id + if consumed > 0: + last_seen_id = unenriched[consumed - 1].record_id now = time.perf_counter() if now - last_progress_time >= progress_interval: diff --git a/tests-unit/assets_test/services/test_transition_settle.py b/tests-unit/assets_test/services/test_transition_settle.py index c1c842627a5..2356e513e83 100644 --- a/tests-unit/assets_test/services/test_transition_settle.py +++ b/tests-unit/assets_test/services/test_transition_settle.py @@ -1,6 +1,7 @@ from contextlib import contextmanager from inspect import signature from pathlib import Path +import threading from unittest.mock import patch import pytest @@ -173,8 +174,8 @@ def enrich_batch(rows, **_kwargs): assert record is not None record.system_metadata = {"enriched": True} update_session.commit() - return 1, failed - return 0, failed + return 1, failed, len(rows) + return 0, failed, len(rows) asset_seeder = seeder_module._AssetSeeder() asset_seeder._run_gate.set() @@ -211,8 +212,8 @@ def _create_session(): def enrich_batch(rows, **_kwargs): attempted.extend(row.record_id for row in rows) if healthy in rows: - return 1, [] - return 0, [row.record_id for row in rows] + return 1, [], len(rows) + return 0, [row.record_id for row in rows], len(rows) asset_seeder = seeder_module._AssetSeeder() asset_seeder._run_gate.set() @@ -236,3 +237,104 @@ def enrich_batch(rows, **_kwargs): "record-3", "healthy-record", ] + + +@pytest.mark.parametrize("interrupt_after", [0, 1], ids=["before-first", "partway"]) +def test_enrich_phase_reoffers_rows_not_attempted_before_pause( + db_engine, + monkeypatch: pytest.MonkeyPatch, + interrupt_after: int, +) -> None: + rows = [ + scanner.UnenrichedContent( + f"content-{index}", f"record-{index}", f"/{index}" + ) + for index in range(3) + ] + fetch_cursors: list[str | None] = [] + offered_batches: list[list[str]] = [] + attempted: list[str] = [] + pause_blocked = threading.Event() + interruption_triggered = False + + @contextmanager + def _create_session(): + with SASession(db_engine) as sess: + yield sess + + asset_seeder = seeder_module._AssetSeeder() + asset_seeder._run_gate.set() + asset_seeder._cancel_event.clear() + asset_seeder.set_event_sink( + lambda event_type, _data: pause_blocked.set() + if event_type == "assets.seed.paused" + else None + ) + + def get_candidates( + _roots, + compute_hashes, + limit=1000, + last_seen_id=None, + ): + nonlocal interruption_triggered + candidates = [ + row + for row in rows + if last_seen_id is None or row.record_id > last_seen_id + ][:limit] + fetch_cursors.append(last_seen_id) + offered_batches.append([row.record_id for row in candidates]) + if interrupt_after == 0 and not interruption_triggered: + interruption_triggered = True + asset_seeder._run_gate.clear() + return candidates + + def enrich_asset(*_args, **kwargs) -> bool: + nonlocal interruption_triggered + attempted.append(kwargs["record_id"]) + if len(attempted) == interrupt_after and not interruption_triggered: + interruption_triggered = True + asset_seeder._run_gate.clear() + return True + + result: list[tuple[bool, int]] = [] + errors: list[BaseException] = [] + + def run_enrich_phase() -> None: + try: + result.append(asset_seeder._run_enrich_phase(("input",))) + except BaseException as exc: + errors.append(exc) + + monkeypatch.setattr( + seeder_module, "get_unenriched_assets_for_roots", get_candidates + ) + monkeypatch.setattr(scanner, "enrich_asset", enrich_asset) + + with ( + patch("app.assets.seeder.create_session", _create_session), + patch("app.assets.scanner.create_session", _create_session), + ): + worker = threading.Thread(target=run_enrich_phase, daemon=True) + worker.start() + try: + assert pause_blocked.wait(timeout=2), ( + "the enrich loop did not block at its loop-top pause checkpoint" + ) + assert fetch_cursors == [None] + assert worker.is_alive() + finally: + asset_seeder._run_gate.set() + worker.join(timeout=2) + + assert worker.is_alive() is False + assert errors == [] + assert result == [(False, 3)] + assert attempted == [row.record_id for row in rows] + assert offered_batches[1] == [ + row.record_id for row in rows[interrupt_after:] + ] + assert fetch_cursors[1] == ( + None if interrupt_after == 0 else rows[interrupt_after - 1].record_id + ) diff --git a/tests-unit/assets_test/test_scanner_ingest_event_log.py b/tests-unit/assets_test/test_scanner_ingest_event_log.py index f81ebee0fe7..05f4ed7092a 100644 --- a/tests-unit/assets_test/test_scanner_ingest_event_log.py +++ b/tests-unit/assets_test/test_scanner_ingest_event_log.py @@ -390,8 +390,8 @@ def fail_enrich(*_args, **_kwargs): first_result = scanner.enrich_assets_batch(rows, progress=_ScanState()) second_result = scanner.enrich_assets_batch(rows[:1], progress=_ScanState()) - assert first_result == (0, ["record-1", "record-2"]) - assert second_result == (0, ["record-1"]) + assert first_result == (0, ["record-1", "record-2"], 2) + assert second_result == (0, ["record-1"], 1) assert events_named(caplog, "scanner.enrich_failed") == [ {"error_type": "FileNotFoundError"}, {"error_type": "FileNotFoundError"}, @@ -413,10 +413,13 @@ def fail_enrich(*_args, **_kwargs): monkeypatch.setattr(scanner, "enrich_asset", fail_enrich) progress = _ScanState() - enriched, failed_ids = scanner.enrich_assets_batch(rows, progress=progress) + enriched, failed_ids, consumed = scanner.enrich_assets_batch( + rows, progress=progress + ) assert enriched == 0 assert failed_ids == ["record-1", "record-2"] + assert consumed == 2 assert progress.enrich_failed == 2 @@ -429,8 +432,11 @@ def test_benign_enrich_no_op_is_skipped_without_counting_a_failure( monkeypatch.setattr(scanner, "create_session", lambda: nullcontext(Mock())) progress = _ScanState() - enriched, failed_ids = scanner.enrich_assets_batch(rows, progress=progress) + enriched, failed_ids, consumed = scanner.enrich_assets_batch( + rows, progress=progress + ) assert enriched == 0 assert failed_ids == ["record-1"] + assert consumed == 1 assert progress.enrich_failed == 0 diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.py index 2637c74d970..6deb051e5e9 100644 --- a/tests-unit/seeder_test/test_seeder.py +++ b/tests-unit/seeder_test/test_seeder.py @@ -165,7 +165,7 @@ def test_enrich_phase_does_not_count_returned_ids_as_failures( monkeypatch.setattr( seeder_module, "enrich_assets_batch", - lambda *_args, **_kwargs: (0, ["record-1", "record-2"]), + lambda *_args, **_kwargs: (0, ["record-1", "record-2"], 2), ) monkeypatch.setattr(scan_seeder, "_check_pause_and_cancel", lambda _stage: False) From 3af6baf6b2b907e093e3be290063dc85df2d845d Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 01:11:26 -0700 Subject: [PATCH 058/111] fix(assets): track the scan pause across prompt worker iterations --- app/prompt_worker.py | 2 +- tests-unit/app_test/test_prompt_worker.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/app/prompt_worker.py b/app/prompt_worker.py index 51fa45ca578..56bfee69a17 100644 --- a/app/prompt_worker.py +++ b/app/prompt_worker.py @@ -32,9 +32,9 @@ def prompt_worker(q, server_instance, asset_manager): last_gc_collect = 0 need_gc = False gc_collect_interval = 10.0 + background_scan_paused = False while True: - background_scan_paused = False try: timeout = 1000.0 if need_gc: diff --git a/tests-unit/app_test/test_prompt_worker.py b/tests-unit/app_test/test_prompt_worker.py index 0634beb069b..4ac0a40de89 100644 --- a/tests-unit/app_test/test_prompt_worker.py +++ b/tests-unit/app_test/test_prompt_worker.py @@ -22,6 +22,9 @@ def task_done(self, *args, **kwargs) -> None: if self.completion_error is not None: raise self.completion_error + def get_flags(self): + return {} + class Server: last_prompt_id = None @@ -100,3 +103,20 @@ def test_prompt_worker_preserves_execute_error_when_resume_raises(prompt_worker_ prompt_worker_module.prompt_worker(Queue(), Server(), asset_manager) assert asset_manager.paused is False + + +def test_prompt_worker_resumes_scan_when_later_iteration_raises_before_gc( + prompt_worker_module, + monkeypatch, +) -> None: + monkeypatch.setattr(prompt_worker_module.execution, "PromptExecutor", Executor) + clock = iter((1.0, 2.0, 2.0)) + monkeypatch.setattr(prompt_worker_module.time, "perf_counter", lambda: next(clock)) + asset_manager = AssetManager() + queue = Queue() + + with pytest.raises(LoopEscape, match="^prompt worker requested a second item$"): + prompt_worker_module.prompt_worker(queue, Server(), asset_manager) + + assert queue.get_calls == 2 + assert asset_manager.paused is False From a787e251ccac92ed004387ce17ed97aa70fd7a84 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 11:17:47 -0700 Subject: [PATCH 059/111] chore(assets): address review follow-ups in the hashing guard, feature flags, and pagination pin --- app/assets/scanner.py | 2 +- app/assets/services/snapshot_hash.py | 5 ++++- comfy_api/feature_flags.py | 3 +++ .../assets_test/services/test_transition_settle.py | 12 +++++------- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index 3e9461ae678..7fd17179f11 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -580,7 +580,7 @@ def enrich_asset( if isinstance(exc, OSError): _log_scan_error("hashing", exc) else: - logging.warning("Failed to hash %s: %s", file_path, exc) + logging.warning("Failed to hash %s: %s", file_path, exc, exc_info=True) record = session.get(Asset, record_id) if content is None or record is None or content.mtime_ns != initial_mtime_ns: diff --git a/app/assets/services/snapshot_hash.py b/app/assets/services/snapshot_hash.py index 6a80399d8d4..051f4e181a4 100644 --- a/app/assets/services/snapshot_hash.py +++ b/app/assets/services/snapshot_hash.py @@ -41,7 +41,10 @@ def snapshot_hash( path: str, chunk_size: int = 8 * 1024 * 1024 ) -> tuple[str, os.stat_result] | None: if blake3 is None: - raise ModuleNotFoundError("No module named 'blake3'") from _BLAKE3_IMPORT_ERROR + raise ModuleNotFoundError( + f"blake3 is required for asset hashing but could not be imported: " + f"{_BLAKE3_IMPORT_ERROR}" + ) from _BLAKE3_IMPORT_ERROR try: pre_stat = _snapshot(os.stat(path)) hasher = blake3() diff --git a/comfy_api/feature_flags.py b/comfy_api/feature_flags.py index d0034b7445e..cbb03471021 100644 --- a/comfy_api/feature_flags.py +++ b/comfy_api/feature_flags.py @@ -164,6 +164,9 @@ def get_server_features(assets_enabled: bool = False) -> dict[str, Any]: """ Get the server's feature flags. + Args: + assets_enabled: Whether the server's asset support is enabled + Returns: Dictionary of server feature flags """ diff --git a/tests-unit/assets_test/services/test_transition_settle.py b/tests-unit/assets_test/services/test_transition_settle.py index 2356e513e83..9e9bd411819 100644 --- a/tests-unit/assets_test/services/test_transition_settle.py +++ b/tests-unit/assets_test/services/test_transition_settle.py @@ -1,5 +1,4 @@ from contextlib import contextmanager -from inspect import signature from pathlib import Path import threading from unittest.mock import patch @@ -42,13 +41,12 @@ def _denied(_candidate_path: str): def _rendered_candidate_placeholder_count(failed_count: int) -> int: failed_ids = [f"{index:036d}" for index in range(failed_count)] - builder = getattr(scanner, "build_unenriched_candidates_statement") - pagination = ( - failed_ids - if "skip_ids" in signature(builder).parameters - else failed_ids[-1] + statement = scanner.build_unenriched_candidates_statement( + prefixes=["/models"], + compute_hashes=False, + last_seen_id=failed_ids[-1], + limit=100, ) - statement = builder(["/models"], False, pagination, 100) compiled = statement.compile( dialect=sqlite.dialect(), compile_kwargs={"render_postcompile": True}, From 858470489455421663c460ca6b21c0a27a863f6f Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 19:51:47 -0700 Subject: [PATCH 060/111] docs(assets): describe enrichment rows as attempted rather than selected The cursor holds at the last row a batch actually attempted, so a pause ends a batch early and the rows behind it are selected again when the scan resumes. Only the attempt is capped at once per pass. --- app/assets/scanner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index 7fd17179f11..c6f846718a6 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -2,8 +2,10 @@ paths, building specs, seeding new content and records, then enriching them with metadata and hashes. Each spec is seeded inside its own savepoint, so one file whose row conflicts cannot discard the work done for the files around it. -Enrichment candidates use ordered ID pagination, so each row is selected at -most once per pass while failed rows remain eligible for the next pass. +Enrichment candidates use ordered ID pagination, so each row is attempted at +most once per pass while failed rows remain eligible for the next pass. A pause +can end a batch early, and the cursor holds at the last row actually attempted, +so the rows behind it are offered again once the scan resumes. """ import logging From 1f30b3ab332102fbba835e39419c24856273614e Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 19:51:47 -0700 Subject: [PATCH 061/111] test(api): derive the expected assets flag from the manager under test The assertion asked for the flag with no argument, so it read the parameter default rather than anything the manager reported - in a test whose subject is the two agreeing. Passing the manager's own state keeps it honest if the setup ever yields an enabled manager. --- tests-unit/assets_test/services/test_noassets_parity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests-unit/assets_test/services/test_noassets_parity.py b/tests-unit/assets_test/services/test_noassets_parity.py index fa1944e5b5a..73de91c601c 100644 --- a/tests-unit/assets_test/services/test_noassets_parity.py +++ b/tests-unit/assets_test/services/test_noassets_parity.py @@ -204,7 +204,7 @@ def test_server_features_match_manager_when_asset_dependencies_are_unavailable( asset_manager = manager.default_asset_manager() assert asset_manager.enabled is False - assert feature_flags.get_server_features()["assets"] is False + assert feature_flags.get_server_features(asset_manager.enabled)["assets"] is False def test_default_asset_manager_enables_assets_when_dependencies_are_available( From fba73d181ecc017a214dccd67e4eb75ebc2042ff Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 17:08:00 -0700 Subject: [PATCH 062/111] test(assets): let a broken prompt worker import fail instead of skipping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fixture wrapped importlib.import_module in a bare except that called pytest.skip, so a circular import, a missing dependency or a syntax error in app/prompt_worker.py would retire all four resume-contract tests while CI stayed green. The whole premise of extracting the module is that main.py can import it, so an import failure has to be a collection error. The CPU guard is genuinely load-bearing — comfy.model_management selects its device at import time and a CUDA build with no driver raises there — so it is kept, but as a precondition rather than an exception handler, matching the args.cpu-before-import convention already used by the comfy_test and comfy_api_nodes_test modules. Nothing is caught now. --- tests-unit/app_test/test_prompt_worker.py | 35 ++++++++++------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/tests-unit/app_test/test_prompt_worker.py b/tests-unit/app_test/test_prompt_worker.py index 4ac0a40de89..f38d9ec90d6 100644 --- a/tests-unit/app_test/test_prompt_worker.py +++ b/tests-unit/app_test/test_prompt_worker.py @@ -1,6 +1,15 @@ -import importlib - import pytest +import torch + +from comfy.cli_args import args + +# Must precede the import: comfy.model_management picks its device at import time, and a CUDA +# build with no driver raises there. Same guard, same placement, as every other test module +# that imports it. +if not torch.cuda.is_available(): + args.cpu = True + +import app.prompt_worker as prompt_worker_module # noqa: E402 class LoopEscape(Exception): @@ -60,18 +69,7 @@ def execute(self, *args, **kwargs) -> None: raise RuntimeError("forced execute failure") -@pytest.fixture -def prompt_worker_module(monkeypatch): - from comfy.cli_args import args - - monkeypatch.setattr(args, "cpu", True, raising=False) - try: - return importlib.import_module("app.prompt_worker") - except Exception as exc: - pytest.skip(f"prompt worker module could not be imported in CPU mode: {exc!r}") - - -def test_prompt_worker_resumes_background_scan_when_execute_raises(prompt_worker_module, monkeypatch) -> None: +def test_prompt_worker_resumes_background_scan_when_execute_raises(monkeypatch) -> None: monkeypatch.setattr(prompt_worker_module.execution, "PromptExecutor", ExecuteFailureExecutor) asset_manager = AssetManager() @@ -81,7 +79,7 @@ def test_prompt_worker_resumes_background_scan_when_execute_raises(prompt_worker assert asset_manager.paused is False -def test_prompt_worker_resumes_background_scan_when_completion_raises(prompt_worker_module, monkeypatch) -> None: +def test_prompt_worker_resumes_background_scan_when_completion_raises(monkeypatch) -> None: monkeypatch.setattr(prompt_worker_module.execution, "PromptExecutor", Executor) asset_manager = AssetManager() @@ -95,7 +93,7 @@ def test_prompt_worker_resumes_background_scan_when_completion_raises(prompt_wor assert asset_manager.paused is False -def test_prompt_worker_preserves_execute_error_when_resume_raises(prompt_worker_module, monkeypatch) -> None: +def test_prompt_worker_preserves_execute_error_when_resume_raises(monkeypatch) -> None: monkeypatch.setattr(prompt_worker_module.execution, "PromptExecutor", ExecuteFailureExecutor) asset_manager = AssetManager(resume_error=RuntimeError("forced resume failure")) @@ -105,10 +103,7 @@ def test_prompt_worker_preserves_execute_error_when_resume_raises(prompt_worker_ assert asset_manager.paused is False -def test_prompt_worker_resumes_scan_when_later_iteration_raises_before_gc( - prompt_worker_module, - monkeypatch, -) -> None: +def test_prompt_worker_resumes_scan_when_later_iteration_raises_before_gc(monkeypatch) -> None: monkeypatch.setattr(prompt_worker_module.execution, "PromptExecutor", Executor) clock = iter((1.0, 2.0, 2.0)) monkeypatch.setattr(prompt_worker_module.time, "perf_counter", lambda: next(clock)) From bca8ad7d574e3380d0fc70109788e5d461f660ec Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 20:17:42 -0700 Subject: [PATCH 063/111] docs(assets): name the unattempted rows instead of the ones behind the cursor The cursor moves forward through ascending ids, so 'the rows behind it' reads as the rows already passed - the opposite of what is selected again. --- app/assets/scanner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index c6f846718a6..f08ba4ddb98 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -4,8 +4,8 @@ file whose row conflicts cannot discard the work done for the files around it. Enrichment candidates use ordered ID pagination, so each row is attempted at most once per pass while failed rows remain eligible for the next pass. A pause -can end a batch early, and the cursor holds at the last row actually attempted, -so the rows behind it are offered again once the scan resumes. +can end a batch early, and the cursor holds at the last row the batch attempted, +so the rows it never reached are selected again when the scan resumes. """ import logging From d698d2a27b148a8979b0780653390fba133f25ae Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 18:29:43 -0700 Subject: [PATCH 064/111] fix(assets): only absorb duplicate-path races when seeding scanned assets --- app/assets/scanner.py | 5 +- .../services/test_scanner_seed_resilience.py | 100 +++++++++++++----- 2 files changed, 79 insertions(+), 26 deletions(-) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index f08ba4ddb98..ca57d394bde 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -26,6 +26,7 @@ create_record, ) from app.assets.database.models import Asset, AssetContent +from app.assets.database.queries.records import _is_live_path_conflict from app.assets.helpers import sql_path_under_prefix, to_stored_hash from app.assets.lifecycle import get_excluded_scan_roots from app.assets.scanner_changes import ( @@ -425,7 +426,9 @@ def seed_asset_specs(session: Session, specs: list[SeedAssetSpec]) -> int: tags=spec["tags"], ) created += 1 - except IntegrityError: + except IntegrityError as error: + if not _is_live_path_conflict(error): + raise logging.warning("Skipping asset whose row conflicts during scan: %s", path) continue except Exception: diff --git a/tests-unit/assets_test/services/test_scanner_seed_resilience.py b/tests-unit/assets_test/services/test_scanner_seed_resilience.py index f6f8f6ba4a5..126d108fde9 100644 --- a/tests-unit/assets_test/services/test_scanner_seed_resilience.py +++ b/tests-unit/assets_test/services/test_scanner_seed_resilience.py @@ -9,7 +9,12 @@ from sqlalchemy.orm import Session from app.assets.database.models import Asset, AssetContent -from app.assets.database.queries import create_content, create_record, delete_record +from app.assets.database.queries import ( + create_content, + create_content_reporting_insert, + create_record, + delete_record, +) from app.assets.scanner import SeedAssetSpec, seed_asset_specs from app.assets.services.snapshot_hash import snapshot_hash @@ -115,34 +120,51 @@ def test_seed_logs_once_for_each_vanished_path( assert messages == [f"Skipping vanished asset during scan: {vanished_path}"] -def test_seed_isolates_a_poisoned_spec_and_persists_the_specs_around_it( +def test_seed_absorbs_live_path_conflict_and_persists_the_specs_around_it( session: Session, temp_dir: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - specs, poisoned_path = _specs_with_vanished_path(temp_dir) + specs, conflicted_path = _specs_with_vanished_path(temp_dir) - def _create_record_or_raise( + def _create_content_or_conflict( session_arg: Session, + path: str, *, - content_id: str, - name: str, - mime_type: str | None, - job_id: str | None, - loader_path: str | None, - tags: list[str], - ) -> Asset: - if name == poisoned_path.name: - raise IntegrityError("forced record creation failure", {}, ValueError()) - return create_record( - session_arg, - content_id, - name, - mime_type, - job_id, - loader_path, - tags, + hash: str | None = None, + size_bytes: int = 0, + mtime_ns: int | None = None, + ) -> tuple[AssetContent, bool]: + if path != str(conflicted_path): + return create_content_reporting_insert( + session_arg, + path=path, + hash=hash, + size_bytes=size_bytes, + mtime_ns=mtime_ns, + ) + session_arg.add( + AssetContent( + path=path, + hash=hash, + size_bytes=size_bytes, + mtime_ns=mtime_ns, + ) + ) + session_arg.flush() + session_arg.add( + AssetContent( + path=path, + hash=hash, + size_bytes=size_bytes, + mtime_ns=mtime_ns, + ) ) + session_arg.flush() + raise AssertionError("duplicate live paths must violate the unique index") - monkeypatch.setattr("app.assets.scanner.create_record", _create_record_or_raise) + monkeypatch.setattr( + "app.assets.scanner.create_content_reporting_insert", + _create_content_or_conflict, + ) created = seed_asset_specs(session, specs) session.commit() @@ -152,11 +174,39 @@ def _create_record_or_raise( assert {record.name for record in session.scalars(select(Asset))} == { "first.bin", "last.bin", - }, ( - "the batch shares one transaction, so a bare rollback would erase the spec BEFORE " - f"the poisoned one; both neighbours of {poisoned_path.name} must survive" + } + + +def test_seed_propagates_unrelated_integrity_error( + session: Session, temp_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = temp_dir / "unrelated-integrity-error.bin" + path.write_bytes(b"content") + unrelated_error = IntegrityError( + "forced record creation failure", + {}, + ValueError("unrelated integrity failure"), ) + def _create_record_or_raise( + session_arg: Session, + *, + content_id: str, + name: str, + mime_type: str | None, + job_id: str | None, + loader_path: str | None, + tags: list[str], + ) -> Asset: + raise unrelated_error + + monkeypatch.setattr("app.assets.scanner.create_record", _create_record_or_raise) + + with pytest.raises(IntegrityError) as raised: + seed_asset_specs(session, [_spec(path)]) + + assert raised.value is unrelated_error + def test_seed_persists_fresh_stat_after_spec_was_built( session: Session, temp_dir: Path From 6bc286c0e62ff2c79f61cf88d16affbe62a15954 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 18:34:21 -0700 Subject: [PATCH 065/111] fix(db): copy the legacy database inside the process lock --- app/database/db.py | 11 +++--- tests-unit/app_test/test_db_init_locking.py | 37 +++++++++++++++++++++ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/app/database/db.py b/app/database/db.py index 053df288332..008e01b8e86 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -115,8 +115,6 @@ def prepare_file_db_path(db_path): if db_dir: os.makedirs(db_dir, exist_ok=True) - copy_legacy_default_db(db_path) - _db_lock = None @@ -183,14 +181,13 @@ def _init_file_db(db_url): """Initialize a file-backed SQLite database using Alembic migrations.""" db_path = get_db_path() prepare_file_db_path(db_path) - db_exists = os.path.exists(db_path) - # Lock BEFORE any migration work — deliberately diverging from upstream master, whose - # "it would block Alembic" rationale is false (the lock guards a separate `.lock` - # file). Only this order makes revision inspection, backup, upgrade and the failure-path - # restore mutually exclusive between processes. + # The lock lives beside the database, so its parent directory must exist first. + # All database reads and writes, including the legacy import, run under the lock. _acquire_file_lock(db_path) try: + copy_legacy_default_db(db_path) + db_exists = os.path.exists(db_path) _migrate_and_bind(db_url, db_path, db_exists) except Exception: _db_lock.release() diff --git a/tests-unit/app_test/test_db_init_locking.py b/tests-unit/app_test/test_db_init_locking.py index b9cfa8c429e..6eaf93f3cf9 100644 --- a/tests-unit/app_test/test_db_init_locking.py +++ b/tests-unit/app_test/test_db_init_locking.py @@ -97,6 +97,43 @@ def test_held_lock_blocks_before_any_migration_work(stale_db): holder.release() +def test_legacy_database_copy_runs_under_file_lock(tmp_path, monkeypatch): + legacy_db = tmp_path / "legacy" / "comfyui.db" + target_db = tmp_path / "current" / "comfyui.db" + legacy_db.parent.mkdir() + legacy_db.write_bytes(b"legacy database") + copied: list[tuple[str, str]] = [] + real_copy = db_module.shutil.copy + + def _copy_while_locked(source: str, destination: str): + contender = FileLock(str(target_db) + ".lock") + try: + with pytest.raises(Timeout): + contender.acquire(timeout=0) + finally: + if contender.is_locked: + contender.release() + copied.append((source, destination)) + return real_copy(source, destination) + + monkeypatch.setattr(db_module.args, "database_url", None) + monkeypatch.setattr(db_module, "get_db_path", lambda: str(target_db)) + monkeypatch.setattr( + db_module, "get_legacy_default_db_path", lambda: str(legacy_db) + ) + monkeypatch.setattr(db_module, "_migrate_and_bind", lambda *_args: None) + monkeypatch.setattr(db_module.shutil, "copy", _copy_while_locked) + monkeypatch.setattr(db_module, "_db_lock", None) + + try: + db_module._init_file_db(f"sqlite:///{target_db}") + finally: + if db_module._db_lock is not None: + db_module._db_lock.release(force=True) + + assert copied == [(str(legacy_db) + ".bak", str(target_db))] + + def test_setup_database_routes_file_lock_to_lock_guidance(monkeypatch, caplog): monkeypatch.setattr(main, "dependencies_available", lambda: True) From 9c6749ab380adb677844135b328b30d56eab32e2 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 18:40:15 -0700 Subject: [PATCH 066/111] fix(assets): drop watch-list entries on stat errors instead of aborting the scan --- app/assets/scanner_admission.py | 2 +- .../services/test_admission_gate.py | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/app/assets/scanner_admission.py b/app/assets/scanner_admission.py index 7c059b005a0..31caaf6b465 100644 --- a/app/assets/scanner_admission.py +++ b/app/assets/scanner_admission.py @@ -73,7 +73,7 @@ def tick_watch_list(session: Session) -> None: for entry in _WATCH_LIST: try: current = os.stat(entry.path) - except FileNotFoundError: + except OSError: continue if (current.st_mtime_ns, current.st_size) == (entry.last_stat.st_mtime_ns, entry.last_stat.st_size): name, tags = get_name_and_tags_from_asset_path(entry.path) diff --git a/tests-unit/assets_test/services/test_admission_gate.py b/tests-unit/assets_test/services/test_admission_gate.py index a34ae9c5990..7e43a013fb0 100644 --- a/tests-unit/assets_test/services/test_admission_gate.py +++ b/tests-unit/assets_test/services/test_admission_gate.py @@ -1,6 +1,7 @@ import os from collections.abc import Iterator from pathlib import Path +from types import SimpleNamespace from unittest.mock import patch import pytest @@ -94,6 +95,37 @@ def test_never_stabilizes_dropped_after_cap(session, temp_dir: Path): assert session.scalars(select(AssetContent)).all() == [] +def test_stat_error_drops_entry_and_allows_other_watch_entries_to_commit( + session, temp_dir: Path, monkeypatch +): + unreadable_path = temp_dir / "unreadable.bin" + stable_path = temp_dir / "stable.bin" + unreadable_path.write_bytes(b"unreadable") + stable_path.write_bytes(b"stable") + unreadable_stat = unreadable_path.stat() + stable_stat = stable_path.stat() + _WATCH_LIST[:] = [ + _WatchEntry(str(unreadable_path), unreadable_stat), + _WatchEntry(str(stable_path), stable_stat), + ] + real_os = scanner_admission.os + + def _stat(path: str): + if path == str(unreadable_path): + raise PermissionError(path) + return real_os.stat(path) + + monkeypatch.setattr("folder_paths.get_input_directory", lambda: str(temp_dir)) + monkeypatch.setattr(scanner_admission, "os", SimpleNamespace(stat=_stat)) + + tick_watch_list(session) + session.commit() + + persisted_paths = set(session.scalars(select(AssetContent.path)).all()) + assert persisted_paths == {str(stable_path)} + assert _WATCH_LIST == [] + + def test_stable_scan_admission_removes_watch_entry_before_next_tick(session, temp_dir: Path, monkeypatch): path = temp_dir / "stable.bin" path.write_bytes(b"complete") From 417532d548470566047e04066152eafc090f83ef Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 19:32:26 -0700 Subject: [PATCH 067/111] fix(assets): clean up temp uploads on validation failures Multipart parsing writes the uploaded bytes to a temporary file before it validates the remaining form fields, so a request rejected after its file part had already been read left the temp file and its uuid directory on disk. Routing those removals through delete_temp_file_if_exists also changes the success path. The previous helper returned early when the temp file was already gone, so it never reached the parent rmdir; the shared helper attempts the rmdir unconditionally. Moving the upload to its destination leaves the temp path absent, so a successful upload now also discards its empty uuid directory, closing a pre-existing leak. --- app/assets/api/upload.py | 128 +++++++++--------- app/assets/services/ingest.py | 14 +- .../services/test_upload_temp_cleanup.py | 57 ++++++++ 3 files changed, 131 insertions(+), 68 deletions(-) create mode 100644 tests-unit/assets_test/services/test_upload_temp_cleanup.py diff --git a/app/assets/api/upload.py b/app/assets/api/upload.py index d46ad6389fb..20ce2ad85eb 100644 --- a/app/assets/api/upload.py +++ b/app/assets/api/upload.py @@ -1,9 +1,8 @@ """Reads a multipart upload off the wire and lands its bytes in a temporary file the ingest service can hash and move. The body is consumed in chunks so a large -model never has to fit in memory, and the temporary file is removed on every -failure path so an abandoned upload leaves nothing behind. Field values are -validated as they arrive, letting a bad request fail before its bytes are -written. +model never has to fit in memory. Validation failures remove any temporary file +before they are reported to the caller. Field values are validated as they +arrive, letting a bad request fail before its bytes are written. """ import logging @@ -66,74 +65,77 @@ async def parse_multipart_upload( file_written = 0 tmp_path: str | None = None - while True: - field = await reader.next() - if field is None: - break + try: + while True: + field = await reader.next() + if field is None: + break - fname = getattr(field, "name", "") or "" + fname = getattr(field, "name", "") or "" - if fname == "hash": - try: - s = ((await field.text()) or "").strip().lower() - except Exception: - raise UploadError( - 400, "INVALID_HASH", "hash must be like 'blake3:'" - ) - - if s: - provided_hash = normalize_and_validate_hash(s) + if fname == "hash": try: - provided_hash_exists = check_hash_exists(provided_hash) - except Exception as e: - logging.exception( - "check_hash_exists failed for hash=%s: %s", provided_hash, e + s = ((await field.text()) or "").strip().lower() + except Exception: + raise UploadError( + 400, "INVALID_HASH", "hash must be like 'blake3:'" ) + + if s: + provided_hash = normalize_and_validate_hash(s) + try: + provided_hash_exists = check_hash_exists(provided_hash) + except Exception as e: + logging.exception( + "check_hash_exists failed for hash=%s: %s", provided_hash, e + ) + raise UploadError( + 500, + "HASH_CHECK_FAILED", + "Backend error while checking asset hash.", + ) + + elif fname == "file": + file_present = True + file_client_name = (field.filename or "").strip() + + uploads_root = os.path.join(folder_paths.get_temp_directory(), "uploads") + unique_dir = os.path.join(uploads_root, uuid.uuid4().hex) + os.makedirs(unique_dir, exist_ok=True) + tmp_path = os.path.join(unique_dir, ".upload.part") + + try: + with open(tmp_path, "wb") as f: + while True: + chunk = await field.read_chunk(8 * 1024 * 1024) + if not chunk: + break + f.write(chunk) + file_written += len(chunk) + except Exception: raise UploadError( - 500, - "HASH_CHECK_FAILED", - "Backend error while checking asset hash.", + 500, "UPLOAD_IO_ERROR", "Failed to receive and store uploaded file." ) - elif fname == "file": - file_present = True - file_client_name = (field.filename or "").strip() - - uploads_root = os.path.join(folder_paths.get_temp_directory(), "uploads") - unique_dir = os.path.join(uploads_root, uuid.uuid4().hex) - os.makedirs(unique_dir, exist_ok=True) - tmp_path = os.path.join(unique_dir, ".upload.part") - - try: - with open(tmp_path, "wb") as f: - while True: - chunk = await field.read_chunk(8 * 1024 * 1024) - if not chunk: - break - f.write(chunk) - file_written += len(chunk) - except Exception: - delete_temp_file_if_exists(tmp_path) + elif fname == "tags": + tags_raw.append((await field.text()) or "") + elif fname == "name": + provided_name = (await field.text()) or None + elif fname == "user_metadata": + user_metadata_raw = (await field.text()) or None + elif fname == "id": raise UploadError( - 500, "UPLOAD_IO_ERROR", "Failed to receive and store uploaded file." + 400, + "UNSUPPORTED_FIELD", + "Client-provided 'id' is not supported. Asset IDs are assigned by the server.", ) - - elif fname == "tags": - tags_raw.append((await field.text()) or "") - elif fname == "name": - provided_name = (await field.text()) or None - elif fname == "user_metadata": - user_metadata_raw = (await field.text()) or None - elif fname == "id": - raise UploadError( - 400, - "UNSUPPORTED_FIELD", - "Client-provided 'id' is not supported. Asset IDs are assigned by the server.", - ) - elif fname == "mime_type": - provided_mime_type = ((await field.text()) or "").strip() or None - elif fname == "preview_id": - provided_preview_id = ((await field.text()) or "").strip() or None + elif fname == "mime_type": + provided_mime_type = ((await field.text()) or "").strip() or None + elif fname == "preview_id": + provided_preview_id = ((await field.text()) or "").strip() or None + except UploadError: + delete_temp_file_if_exists(tmp_path) + raise if not file_present and not (provided_hash and provided_hash_exists): raise UploadError( 400, "MISSING_FILE", "Form must include a 'file' part or a known 'hash'." diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index a8fb3b81de7..c72d32a0c0b 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -17,6 +17,7 @@ from sqlalchemy.orm import Session from app.assets import mode +from app.assets.api.upload import delete_temp_file_if_exists from app.assets.database.models import Asset, AssetContent, AssetTag from app.assets.database.queries.records import ( create_content_reporting_insert, @@ -476,11 +477,14 @@ def upload_from_temp_path( _remove_temp_path(temp_path) raise ValueError("tags are required for new asset uploads") - dest_abs = _hash_mode_dest_path(tags, digest, client_filename, name) - content_type = _guess_upload_mime_type( - mime_type, client_filename, name, os.path.basename(dest_abs) - ) - _move_temp_to_dest(temp_path, dest_abs) + try: + dest_abs = _hash_mode_dest_path(tags, digest, client_filename, name) + content_type = _guess_upload_mime_type( + mime_type, client_filename, name, os.path.basename(dest_abs) + ) + _move_temp_to_dest(temp_path, dest_abs) + finally: + delete_temp_file_if_exists(temp_path) size_bytes, mtime_ns = verified_stat.st_size, verified_stat.st_mtime_ns with create_session() as session: _reconcile_live_content_at_path( diff --git a/tests-unit/assets_test/services/test_upload_temp_cleanup.py b/tests-unit/assets_test/services/test_upload_temp_cleanup.py new file mode 100644 index 00000000000..c60cdc44c93 --- /dev/null +++ b/tests-unit/assets_test/services/test_upload_temp_cleanup.py @@ -0,0 +1,57 @@ +import uuid +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest + +import folder_paths +from app.assets.api.schemas_in import UploadError +from app.assets.api.upload import parse_multipart_upload +from app.assets.services.ingest import upload_from_temp_path + + +@pytest.mark.asyncio +async def test_multipart_id_after_file_removes_temp_upload( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(folder_paths, "get_temp_directory", lambda: str(tmp_path)) + + file_field = AsyncMock() + file_field.name = "file" + file_field.filename = "model.safetensors" + file_field.read_chunk.side_effect = [b"uploaded bytes", b""] + + id_field = AsyncMock() + id_field.name = "id" + + reader = AsyncMock() + reader.next.side_effect = [file_field, id_field, None] + + request = AsyncMock() + request.content_type = "multipart/form-data" + request.multipart.return_value = reader + + with pytest.raises(UploadError, match="Client-provided 'id' is not supported"): + await parse_multipart_upload(request, lambda _hash: False) + + assert list((tmp_path / "uploads").iterdir()) == [] + + +def test_destination_resolution_failure_removes_temp_upload( + mock_create_session, tmp_path: Path +) -> None: + upload_dir = tmp_path / "uploads" / uuid.uuid4().hex + upload_dir.mkdir(parents=True) + temp_path = upload_dir / ".upload.part" + temp_path.write_bytes(b"uploaded bytes") + + with pytest.raises(ValueError, match="exactly one destination role"): + upload_from_temp_path( + temp_path=str(temp_path), + name="model.safetensors", + tags=["not-a-destination"], + client_filename="model.safetensors", + ) + + assert not temp_path.exists() + assert not upload_dir.exists() From 9b8e766edce80e767ba468b1e3557969334a537b Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 19:37:06 -0700 Subject: [PATCH 068/111] fix(assets): keep updated_at stable on no-op renames --- app/assets/database/queries/records.py | 5 +++-- app/assets/services/asset_management.py | 5 ++--- .../services/test_updated_at_semantics.py | 12 ++++++++++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/app/assets/database/queries/records.py b/app/assets/database/queries/records.py index 5a20b6babbc..c2e7b578f62 100644 --- a/app/assets/database/queries/records.py +++ b/app/assets/database/queries/records.py @@ -299,8 +299,9 @@ def rename_record(session: Session, id: str, name: str) -> Asset: record = session.get(Asset, id) if record is None: raise LookupError(id) - record.name = name - record.updated_at = get_utc_now() + if record.name != name: + record.name = name + record.updated_at = get_utc_now() session.flush() return record diff --git a/app/assets/services/asset_management.py b/app/assets/services/asset_management.py index 0f7ad716c1c..25be239060f 100644 --- a/app/assets/services/asset_management.py +++ b/app/assets/services/asset_management.py @@ -1,8 +1,7 @@ """Serves the per-asset operations behind the API: reading an asset's detail, updating its name, tags, metadata and preview, deleting a record, and resolving -a hash to a servable path. An update moves ``updated_at`` only when it actually -changed something, so a call that requests what is already true is not recorded -as a user edit. +a hash to a servable path. Name and tag updates move ``updated_at`` only when +the requested values differ. Other supplied metadata fields record a write. """ import mimetypes diff --git a/tests-unit/assets_test/services/test_updated_at_semantics.py b/tests-unit/assets_test/services/test_updated_at_semantics.py index bfd17082b1f..99e9f3328d9 100644 --- a/tests-unit/assets_test/services/test_updated_at_semantics.py +++ b/tests-unit/assets_test/services/test_updated_at_semantics.py @@ -111,6 +111,18 @@ def test_rename_moves_updated_at(session, mock_create_session, temp_dir): assert _updated_at(session, record.id) > STALE, "a rename is an explicit user edit" +def test_same_name_rename_does_not_move_updated_at( + session, mock_create_session, temp_dir +): + record = _seed_record(session, _write_file(temp_dir, "rename-noop.bin")) + + update_asset_metadata(record.id, name=record.name) + + assert _updated_at(session, record.id) == STALE, ( + "requesting the existing name changes nothing" + ) + + @pytest.mark.parametrize( "field,kwargs", [ From a6b508bd4b3102c91f5d164c9261a7d39a90fc2a Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 19:41:34 -0700 Subject: [PATCH 069/111] docs(assets): make module docstrings and the rebuild warning truthful --- app/assets/database/queries/records.py | 4 ++-- app/assets/database/queries/tags.py | 6 +++--- app/assets/lifecycle.py | 3 +-- app/assets/manager.py | 6 ++++++ app/database/db.py | 4 +++- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/app/assets/database/queries/records.py b/app/assets/database/queries/records.py index c2e7b578f62..96032bb7226 100644 --- a/app/assets/database/queries/records.py +++ b/app/assets/database/queries/records.py @@ -1,5 +1,5 @@ -"""Owns every write to content rows, records and their tag links, plus the paged -reads that list them. Inserts that can lose a race — a content row at a path, a +"""Provides shared writes for content rows, records and tag links, plus the paged +reads that list records. Inserts that can lose a race — a content row at a path, a tag, a tag link — run inside a savepoint and re-read the conflicting row, so a concurrent writer settles the call instead of raising, while a genuine constraint failure still surfaces. This is the sole writer of a content row's diff --git a/app/assets/database/queries/tags.py b/app/assets/database/queries/tags.py index 177cc980d9f..8c9a9565496 100644 --- a/app/assets/database/queries/tags.py +++ b/app/assets/database/queries/tags.py @@ -1,7 +1,7 @@ """Answers questions about tag usage: which tags exist with how many assets, and -how those counts narrow once a filter is applied. Both queries reuse the record -listing's own joins and filter clauses, so the counts a client sees always -describe the same assets the listing endpoint would return for that filter, +how those counts narrow once a filter is applied. The unfiltered query counts +all catalogued records. The filtered query reuses the record listing's joins and +filter clauses, so its counts describe the assets the listing endpoint returns, records with missing content included. """ diff --git a/app/assets/lifecycle.py b/app/assets/lifecycle.py index 26ecd0370ae..2a2f240b421 100644 --- a/app/assets/lifecycle.py +++ b/app/assets/lifecycle.py @@ -2,8 +2,7 @@ and files, recording a hash-mode transition, and handing the filesystem scan to the background seeder. Startup only enqueues transition work — draining it belongs to the seeder, so a large library cannot stall the server before it -accepts requests. It also settles, before any of that, whether the database -dependencies exist at all, disabling the asset routes when they do not. +accepts requests. """ from __future__ import annotations diff --git a/app/assets/manager.py b/app/assets/manager.py index 590015d6ad9..73e20bc32d4 100644 --- a/app/assets/manager.py +++ b/app/assets/manager.py @@ -1,3 +1,9 @@ +"""Selects and implements the enabled and disabled asset managers. + +``default_asset_manager`` checks database dependencies before enabling assets +and chooses ``NoAssets`` when the requested mode cannot run. +""" + import logging from typing import Any, Callable, Protocol diff --git a/app/database/db.py b/app/database/db.py index 008e01b8e86..01c85aa3014 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -251,7 +251,9 @@ def set_sqlite_pragma(dbapi_connection, connection_record): f"The asset catalog was rebuilt from scratch by migration " f"{_DESTRUCTIVE_REVISION}: manual tags, user metadata, previews, renames, " f"API-created records and job_id links from the previous database were " - f"discarded. The database from before the upgrade was kept at {backup_path}." + f"discarded. Record deletions were also discarded, so files still on disk " + f"will be catalogued again. The database from before the upgrade was kept " + f"at {backup_path}." ) conn.close() From 68c7893381d7c21da9bc6315054abc2d2e91ac62 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 19:52:36 -0700 Subject: [PATCH 070/111] fix(assets): correct event-log status snapshots and failure telemetry --- app/assets/event_log.py | 8 +-- app/assets/scanner.py | 7 +- app/assets/seeder.py | 42 ++++++----- .../assets_test/services/test_scanner_b.py | 11 +++ tests-unit/assets_test/test_event_log.py | 4 +- tests-unit/seeder_test/test_seeder.py | 62 +++++++++++++++- tests-unit/test_assets_event_log_static.py | 72 +++++++++++++++++-- 7 files changed, 175 insertions(+), 31 deletions(-) diff --git a/app/assets/event_log.py b/app/assets/event_log.py index e8962a3ad46..ede1222c82c 100644 --- a/app/assets/event_log.py +++ b/app/assets/event_log.py @@ -8,9 +8,9 @@ they are. The field vocabulary is closed. Only the names in :data:`ALLOWED_FIELDS` may be -carried, each has a validator, and no string value may contain a path separator -or logfmt delimiter — so file names, paths, asset ids and content hashes cannot -ride along. +carried, each has a validator, and no string value may contain a path separator, +logfmt delimiter or line break — so file names, paths, asset ids and content +hashes cannot ride along. """ import logging @@ -22,7 +22,7 @@ TAG = "[assets-event]" MAX_STRING_LENGTH = 64 -FORBIDDEN_STRING_CHARS = ("/", "\\", ":", " ", "=", '"') +FORBIDDEN_STRING_CHARS = ("/", "\\", ":", " ", "=", '"', "\n", "\r") ROOTS = frozenset({"models", "input", "output", "user", "temp"}) PHASES = frozenset({"fast", "enrich", "full"}) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index ca57d394bde..06386fa0618 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -256,10 +256,11 @@ def sync_temp_references_safely( ) -def mark_missing_outside_prefixes_safely(prefixes: list[str]) -> int: +def mark_missing_outside_prefixes_safely(prefixes: list[str]) -> int | None: """Mark references as missing when outside the given prefixes. - This is a non-destructive soft-delete. Returns count marked or 0 on failure. + This is a non-destructive soft-delete. Returns the count marked, or None when + the operation fails. """ try: with create_session() as sess: @@ -272,7 +273,7 @@ def mark_missing_outside_prefixes_safely(prefixes: list[str]) -> int: "scanner.mark_missing_failed", error_type=error_type(exc), ) - return 0 + return None def mark_contents_missing_outside_prefixes( diff --git a/app/assets/seeder.py b/app/assets/seeder.py index 9fa027ee9b7..2cc73d84fb5 100644 --- a/app/assets/seeder.py +++ b/app/assets/seeder.py @@ -10,7 +10,7 @@ import os import threading import time -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from enum import Enum from typing import Any, Callable, TypedDict @@ -410,7 +410,9 @@ def get_status(self) -> ScanStatus: progress = ( _snapshot_progress(self._scan_state) if self._scan_state is not None - else self._last_progress + else replace(self._last_progress) + if self._last_progress is not None + else None ) return ScanStatus( state=self._state, @@ -474,14 +476,16 @@ def mark_missing_outside_prefixes(self) -> int: all_prefixes = get_owned_prefixes() marked = mark_missing_outside_prefixes_safely(all_prefixes) - emit( - "seeder.marked_missing", - count=marked, - stage=_ScanStage.MARK_MISSING.value, - ) - if marked > 0: - logging.info("Marked %d references as missing", marked) - return marked + marked_count = 0 if marked is None else marked + if marked is not None: + emit( + "seeder.marked_missing", + count=marked_count, + stage=_ScanStage.MARK_MISSING.value, + ) + if marked_count > 0: + logging.info("Marked %d references as missing", marked_count) + return marked_count finally: with self._lock: self._reset_to_idle() @@ -624,13 +628,17 @@ def _run_scan(self) -> None: if self._prune_first: all_prefixes = get_owned_prefixes() marked = mark_missing_outside_prefixes_safely(all_prefixes) - emit( - "seeder.marked_missing", - count=marked, - stage=_ScanStage.PRUNING.value, - ) - if marked > 0: - logging.info("Marked %d refs as missing before scan", marked) + marked_count = 0 if marked is None else marked + if marked is not None: + emit( + "seeder.marked_missing", + count=marked_count, + stage=_ScanStage.PRUNING.value, + ) + if marked_count > 0: + logging.info( + "Marked %d refs as missing before scan", marked_count + ) sync_temp_references_safely(scan_state) if self._check_pause_and_cancel(_ScanStage.PRUNING): diff --git a/tests-unit/assets_test/services/test_scanner_b.py b/tests-unit/assets_test/services/test_scanner_b.py index 93c155870e5..6f98760bf62 100644 --- a/tests-unit/assets_test/services/test_scanner_b.py +++ b/tests-unit/assets_test/services/test_scanner_b.py @@ -13,6 +13,7 @@ build_asset_specs, enrich_asset, mark_contents_missing_outside_prefixes, + mark_missing_outside_prefixes_safely, seed_asset_specs, sync_prefixes_with_filesystem, ) @@ -171,6 +172,16 @@ def test_prune_marks_missing_not_deletes(session, temp_dir: Path): assert missing_tag is not None and missing_tag.origin == "automatic" +def test_mark_missing_failure_returns_none(): + with patch( + "app.assets.scanner.create_session", + side_effect=RuntimeError("database unavailable"), + ): + result = mark_missing_outside_prefixes_safely([]) + + assert result is None + + def test_unhashed_missing_content_gets_tagged(session, temp_dir: Path): missing_path = os.path.abspath(temp_dir / "missing.bin") content = AssetContent(path=missing_path, hash=None, size_bytes=7, mtime_ns=1) diff --git a/tests-unit/assets_test/test_event_log.py b/tests-unit/assets_test/test_event_log.py index 27c9bba139b..dccdfadc088 100644 --- a/tests-unit/assets_test/test_event_log.py +++ b/tests-unit/assets_test/test_event_log.py @@ -168,7 +168,9 @@ def test_unknown_field_raises_under_pytest(): emit("seeder.scan_started", path="/home/x/models") -@pytest.mark.parametrize("value", ["a/b", "a\\b", "a:b", "a b", "a=b", 'a"b']) +@pytest.mark.parametrize( + "value", ["a/b", "a\\b", "a:b", "a b", "a=b", 'a"b', "a\nb", "a\rb"] +) def test_a_string_value_carrying_a_forbidden_character_raises(value): with pytest.raises(EventLogError): emit("seeder.scan_failed", error_type=value) diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.py index 6deb051e5e9..40066fc114e 100644 --- a/tests-unit/seeder_test/test_seeder.py +++ b/tests-unit/seeder_test/test_seeder.py @@ -8,11 +8,12 @@ from sqlalchemy import create_engine from sqlalchemy.orm import Session +from app.assets import scanner as scanner_module from app.assets import seeder as seeder_module from app.assets.database.models import Base from app.assets.database.queries import create_content, create_record, mark_content_missing from app.assets.event_log import TAG -from app.assets.seeder import ScanPhase, State, _AssetSeeder, _ScanStage, _ScanState +from app.assets.seeder import Progress, ScanPhase, State, _AssetSeeder, _ScanStage, _ScanState EVENT_LINE_PATTERN = re.compile( @@ -67,6 +68,19 @@ def events_named( return [fields for event, fields in tagged_events(caplog) if event == event_name] +def test_idle_status_returns_a_progress_snapshot() -> None: + seeder = _AssetSeeder() + seeder._last_progress = Progress(created=1) + + status = seeder.get_status() + assert status.progress is not None + status.progress.created = 999 + + next_status = seeder.get_status() + assert next_status.progress is not None + assert next_status.progress.created == 1 + + def test_seeder_models_missing_as_content_state(): engine = create_engine("sqlite:///:memory:") Base.metadata.create_all(engine) @@ -420,6 +434,52 @@ def test_standalone_mark_missing_emits_count_with_mark_missing_stage( ] +def test_standalone_mark_missing_failure_emits_no_success_event( + scan_seeder: _AssetSeeder, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + scan_seeder._state = State.IDLE + monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: []) + + def fail_create_session(): + raise RuntimeError("database unavailable") + + monkeypatch.setattr(scanner_module, "create_session", fail_create_session) + + with caplog.at_level(logging.INFO): + result = scan_seeder.mark_missing_outside_prefixes() + + assert result == 0 + assert events_named(caplog, "scanner.mark_missing_failed") == [ + {"error_type": "RuntimeError"} + ] + assert events_named(caplog, "seeder.marked_missing") == [] + + +def test_scan_prune_failure_completes_without_type_error( + scan_seeder: _AssetSeeder, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + scan_seeder._prune_first = True + scan_seeder._phase = ScanPhase.FAST + monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: []) + monkeypatch.setattr( + seeder_module, "mark_missing_outside_prefixes_safely", lambda _prefixes: None + ) + monkeypatch.setattr( + seeder_module, "sync_temp_references_safely", lambda _progress: None + ) + monkeypatch.setattr(scan_seeder, "_run_fast_phase", lambda _roots: (0, 0, 0)) + + with caplog.at_level(logging.INFO): + scan_seeder._run_scan() + + assert scan_seeder._errors == [] + assert events_named(caplog, "seeder.marked_missing") == [] + + def test_batch_insert_failure_emits_only_the_exception_type( scan_seeder: _AssetSeeder, monkeypatch: pytest.MonkeyPatch, diff --git a/tests-unit/test_assets_event_log_static.py b/tests-unit/test_assets_event_log_static.py index cbab691dea2..a680953b229 100644 --- a/tests-unit/test_assets_event_log_static.py +++ b/tests-unit/test_assets_event_log_static.py @@ -124,19 +124,39 @@ def _resolve_aliases(tree: ast.Module) -> Aliases: return Aliases(frozenset(module), frozenset(emit), frozenset(error_type)) +def _dotted_name(node: ast.expr) -> str | None: + parts: list[str] = [] + while isinstance(node, ast.Attribute): + parts.append(node.attr) + node = node.value + if not isinstance(node, ast.Name): + return None + parts.append(node.id) + return ".".join(reversed(parts)) + + def _is_emit_call(func: ast.expr, aliases: Aliases) -> bool: - if isinstance(func, ast.Attribute) and func.attr == "emit" and isinstance(func.value, ast.Name): - return func.value.id in aliases.module + if isinstance(func, ast.Attribute) and func.attr == "emit": + return _dotted_name(func.value) in aliases.module return isinstance(func, ast.Name) and func.id in aliases.emit +def _is_unresolvable_emit_call(func: ast.expr, aliases: Aliases) -> bool: + return ( + bool(aliases.module) + and isinstance(func, ast.Attribute) + and func.attr == "emit" + and _dotted_name(func.value) is None + ) + + def _is_error_type_call(value: ast.expr, aliases: Aliases) -> bool: """True only for a call to the sanctioned event_log.error_type().""" if not isinstance(value, ast.Call): return False func = value.func - if isinstance(func, ast.Attribute) and func.attr == "error_type" and isinstance(func.value, ast.Name): - return func.value.id in aliases.module + if isinstance(func, ast.Attribute) and func.attr == "error_type": + return _dotted_name(func.value) in aliases.module return isinstance(func, ast.Name) and func.id in aliases.error_type @@ -177,7 +197,9 @@ def _field_faults(call: ast.Call, aliases: Aliases) -> Iterator[tuple[str, str]] def _file_faults(call: ast.Call, aliases: Aliases) -> Iterator[tuple[str, str]]: - if _is_emit_call(call.func, aliases): + if _is_unresolvable_emit_call(call.func, aliases): + yield "event_names", "the emit receiver cannot be resolved statically" + elif _is_emit_call(call.func, aliases): event = _event_of(call) if event not in ALLOWED_EVENTS: yield "event_names", "the event must be one string literal in the allowed vocabulary" @@ -203,6 +225,12 @@ def _scan_file(root: Path, relative: str) -> tuple[Counter[CallSite], list[tuple return sites, faults +def _write_scan_fixture(root: Path, relative: str, source: str) -> None: + path = root / relative + path.parent.mkdir(parents=True) + path.write_text(source, encoding="utf-8") + + def scan_repository(root: Path = REPO_ROOT) -> Scan: files = _scanned_files(root) sites: Counter[CallSite] = Counter() @@ -258,3 +286,37 @@ def test_call_sites_match_the_manifest() -> None: "EXPECTED_CALL_SITES" ) assert not missing, f"manifest call sites absent from the tree: {sorted(missing)}" + + +def test_qualified_event_log_import_is_scanned(tmp_path: Path) -> None: + relative = "app/assets/qualified.py" + _write_scan_fixture( + tmp_path, + relative, + "import app.assets.event_log\n\n" + "def probe():\n" + ' app.assets.event_log.emit("seeder.scan_started", phase="fast")\n', + ) + + sites, faults = _scan_file(tmp_path, relative) + + assert faults == [] + assert sites == Counter( + {CallSite(relative, "probe", "seeder.scan_started"): 1} + ) + + +def test_unresolvable_emit_receiver_is_a_scan_failure(tmp_path: Path) -> None: + relative = "app/assets/dynamic.py" + _write_scan_fixture( + tmp_path, + relative, + "from app.assets import event_log\n\n" + "def probe(provider):\n" + ' provider().emit("seeder.scan_started", phase="fast")\n', + ) + + sites, faults = _scan_file(tmp_path, relative) + + assert sites == Counter() + assert [category for category, _reason in faults] == ["event_names"] From e13f0e1f43e9afb40745a5d74f0e090feceb6f4e Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 20:04:59 -0700 Subject: [PATCH 071/111] test(assets): make the keyset tie-breaker and temp-exclusion tests falsifiable --- .../queries/test_asset_reference_keyset.py | 25 +++++++++++++++---- .../assets_test/services/test_lifecycle.py | 16 +++++------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/tests-unit/assets_test/queries/test_asset_reference_keyset.py b/tests-unit/assets_test/queries/test_asset_reference_keyset.py index b3d2c42cd74..28f9f6350c8 100644 --- a/tests-unit/assets_test/queries/test_asset_reference_keyset.py +++ b/tests-unit/assets_test/queries/test_asset_reference_keyset.py @@ -1,28 +1,41 @@ +import pytest from sqlalchemy.orm import Session from app.assets.database.queries import create_content, create_record, list_records_page -from app.assets.database.queries.records import RecordCursorBoundary, RecordPageSpec +from app.assets.database.queries.records import ( + RecordCursorBoundary, + RecordPageSpec, + RecordSortOrder, +) -def test_record_keyset_cursor_pages_in_creation_order(session: Session) -> None: +@pytest.mark.parametrize( + ("order", "expected_indexes"), + [("asc", (0, 1, 2)), ("desc", (2, 1, 0))], +) +def test_record_keyset_cursor_pages_in_creation_order( + session: Session, order: RecordSortOrder, expected_indexes: tuple[int, int, int] +) -> None: records = [ create_record(session, create_content(session, f"/output/{name}").id, name) for name in ("one.png", "two.png", "three.png") ] + shared_created_at = records[0].created_at for index, record in enumerate(records, start=1): record.id = f"00000000-0000-0000-0000-{index:012d}" + record.created_at = shared_created_at session.flush() first_page, _, _ = list_records_page( session, - RecordPageSpec(limit=2, order="asc"), + RecordPageSpec(limit=2, order=order), ) boundary_record = first_page[-1] second_page, _, _ = list_records_page( session, RecordPageSpec( limit=2, - order="asc", + order=order, after=RecordCursorBoundary( value=boundary_record.created_at, id=boundary_record.id, @@ -30,4 +43,6 @@ def test_record_keyset_cursor_pages_in_creation_order(session: Session) -> None: ), ) - assert [record.id for record in first_page + second_page] == [record.id for record in records] + assert [record.id for record in first_page + second_page] == [ + records[index].id for index in expected_indexes + ] diff --git a/tests-unit/assets_test/services/test_lifecycle.py b/tests-unit/assets_test/services/test_lifecycle.py index 4010a770c57..75e2f970384 100644 --- a/tests-unit/assets_test/services/test_lifecycle.py +++ b/tests-unit/assets_test/services/test_lifecycle.py @@ -15,7 +15,6 @@ from app.assets.database.queries.records import create_content, create_record from app.assets.lifecycle import ( cleanup_temp_filesystem, - get_excluded_scan_roots, run_asset_shutdown_cleanup, run_asset_startup, run_startup, @@ -182,24 +181,21 @@ def test_run_startup_logs_and_absorbs_disabled_filesystem_failure(caplog): def test_rmtree_failure_excludes_temp_from_scan(session, comfy_dirs, mock_create_session): record_id, content_id = _seed_temp_rows(session, comfy_dirs) - - wipe_temp_db_rows(session) - session.commit() - assert session.get(Asset, record_id) is None + tracked_file = comfy_dirs / "preview.png" + tracked_file.unlink() with patch("app.assets.lifecycle.shutil.rmtree", side_effect=OSError("busy")): assert cleanup_temp_filesystem() is False - assert str(comfy_dirs) in get_excluded_scan_roots() assert get_temp_prefixes() == [] - residual = comfy_dirs / "leftover.png" - residual.write_bytes(b"\x00" * 10) - with patch("app.assets.scanner.create_session", mock_create_session): sync_temp_references_safely() - assert session.scalars(select(Asset)).all() == [] + session.expire_all() + content = session.get(AssetContent, content_id) + assert session.get(Asset, record_id) is not None + assert content is not None and content.is_missing is False def test_shutdown_skips_cleanup_when_seeder_join_times_out(session, comfy_dirs, mock_create_session, caplog): From 2a89925379504335e618e8bb4167ae9ccf74dcda Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 15 Sep 2026 20:22:09 -0700 Subject: [PATCH 072/111] chore: comment cleanup Comment-Gate: 3 quarantined --- app/database/db.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/database/db.py b/app/database/db.py index 01c85aa3014..ca18b97c0a2 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -182,7 +182,6 @@ def _init_file_db(db_url): db_path = get_db_path() prepare_file_db_path(db_path) - # The lock lives beside the database, so its parent directory must exist first. # All database reads and writes, including the legacy import, run under the lock. _acquire_file_lock(db_path) try: From c605a9be77c3f80587406fba30d5013f14a7a07e Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 01:42:02 -0700 Subject: [PATCH 073/111] fix(assets): keep unreadable filesystem metadata from failing a whole scan batch --- app/assets/api/routes.py | 6 + app/assets/event_log.py | 3 + app/assets/scanner.py | 113 +++++++++--------- app/assets/scanner_admission.py | 14 ++- app/assets/seeder.py | 22 ++-- .../services/test_admission_gate.py | 55 ++++++++- .../assets_test/services/test_api_routes_b.py | 16 +++ .../services/test_scanner_seed_resilience.py | 81 +++++++++++++ tests-unit/seeder_test/test_seeder.py | 4 +- tests-unit/test_assets_event_log_static.py | 7 ++ 10 files changed, 250 insertions(+), 71 deletions(-) diff --git a/app/assets/api/routes.py b/app/assets/api/routes.py index f5ed98be5d8..cc512c4dec4 100644 --- a/app/assets/api/routes.py +++ b/app/assets/api/routes.py @@ -1134,4 +1134,10 @@ async def mark_missing_assets(request: web.Request) -> web.Response: {"status": "scan_running", "marked": 0}, status=409, ) + if marked is None: + return _build_error_response( + 500, + "PRUNE_FAILED", + "Failed to mark missing assets.", + ) return web.json_response({"status": "completed", "marked": marked}, status=200) diff --git a/app/assets/event_log.py b/app/assets/event_log.py index ede1222c82c..e2a2a4fe7fb 100644 --- a/app/assets/event_log.py +++ b/app/assets/event_log.py @@ -43,6 +43,9 @@ "scanner.temp_sync_failed", "scanner.mark_missing_failed", "scanner.stat_failed", + "scanner.invalid_mtime", + "scanner.watch_stat_failed", + "scanner.watch_seed_failed", }) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index 06386fa0618..7e2a0a187b7 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -54,7 +54,6 @@ get_comfy_models_folders, get_name_and_tags_from_asset_path, ) -from app.assets.services.ingest import _discard_unreferenced_content from app.assets.services.snapshot_hash import snapshot_hash from app.database.db import create_session @@ -380,63 +379,69 @@ def build_asset_specs( def seed_asset_specs(session: Session, specs: list[SeedAssetSpec]) -> int: created = 0 - created_content_ids: list[str] = [] - try: - for spec in specs: - path = os.path.abspath(spec["abs_path"]) - try: - with session.begin_nested(): - try: - stat_result = os.stat(path, follow_symlinks=True) - except OSError: - logging.warning("Skipping vanished asset during scan: %s", path) - continue - try: - recovery = recover_missing_content( - session, - path, - stat_result, - hashing_is_enabled=mode.hashing_enabled(), - ) - except OSError: - logging.warning("Skipping vanished asset during scan: %s", path) - continue - if recovery != "no_match": - continue - content, inserted = create_content_reporting_insert( - session, - path=path, - hash=None, - size_bytes=stat_result.st_size, - mtime_ns=get_mtime_ns(stat_result), - ) - if inserted: - created_content_ids.append(content.id) - existing_record = session.scalar( - sa.select(Asset.id).where(Asset.content_id == content.id).limit(1) + first_error: Exception | None = None + for spec in specs: + path = os.path.abspath(spec["abs_path"]) + try: + with session.begin_nested(): + try: + stat_result = os.stat(path, follow_symlinks=True) + except OSError: + logging.warning("Skipping vanished asset during scan: %s", path) + continue + if get_mtime_ns(stat_result) < 0: + logging.warning( + "Skipping asset with invalid mtime during scan: %s", path ) - if existing_record is not None: - continue - create_record( + emit("scanner.invalid_mtime") + continue + try: + recovery = recover_missing_content( session, - content_id=content.id, - name=spec["info_name"], - mime_type=spec["mime_type"], - job_id=spec["job_id"], - loader_path=spec["fname"], - tags=spec["tags"], + path, + stat_result, + hashing_is_enabled=mode.hashing_enabled(), ) - created += 1 - except IntegrityError as error: - if not _is_live_path_conflict(error): - raise - logging.warning("Skipping asset whose row conflicts during scan: %s", path) + except OSError: + logging.warning("Skipping vanished asset during scan: %s", path) + continue + if recovery != "no_match": + continue + content, _inserted = create_content_reporting_insert( + session, + path=path, + hash=None, + size_bytes=stat_result.st_size, + mtime_ns=get_mtime_ns(stat_result), + ) + existing_record = session.scalar( + sa.select(Asset.id).where(Asset.content_id == content.id).limit(1) + ) + if existing_record is not None: + continue + create_record( + session, + content_id=content.id, + name=spec["info_name"], + mime_type=spec["mime_type"], + job_id=spec["job_id"], + loader_path=spec["fname"], + tags=spec["tags"], + ) + created += 1 + except IntegrityError as error: + if _is_live_path_conflict(error): + logging.warning( + "Skipping asset whose row conflicts during scan: %s", path + ) continue - except Exception: - session.rollback() - for content_id in created_content_ids: - _discard_unreferenced_content(session, content_id) - raise + if first_error is None: + first_error = error + except Exception as error: + if first_error is None: + first_error = error + if first_error is not None: + raise first_error return created diff --git a/app/assets/scanner_admission.py b/app/assets/scanner_admission.py index 31caaf6b465..82dba10b602 100644 --- a/app/assets/scanner_admission.py +++ b/app/assets/scanner_admission.py @@ -8,6 +8,7 @@ from __future__ import annotations +import logging import mimetypes import os import time @@ -16,6 +17,7 @@ from sqlalchemy.orm import Session +from app.assets.event_log import emit, error_type from app.assets.services.path_utils import compute_loader_path, get_name_and_tags_from_asset_path PARTIAL_DOWNLOAD_EXTENSIONS = frozenset({ @@ -73,7 +75,9 @@ def tick_watch_list(session: Session) -> None: for entry in _WATCH_LIST: try: current = os.stat(entry.path) - except OSError: + except OSError as exc: + logging.warning("Dropping watched asset after stat failed: %s", entry.path) + emit("scanner.watch_stat_failed", error_type=error_type(exc)) continue if (current.st_mtime_ns, current.st_size) == (entry.last_stat.st_mtime_ns, entry.last_stat.st_size): name, tags = get_name_and_tags_from_asset_path(entry.path) @@ -88,7 +92,13 @@ def tick_watch_list(session: Session) -> None: "mime_type": mimetypes.guess_type(entry.path, strict=False)[0], "job_id": None, } - seed_asset_specs(session, [spec]) + try: + seed_asset_specs(session, [spec]) + except Exception as exc: + logging.warning( + "Dropping watched asset after seeding failed: %s", entry.path + ) + emit("scanner.watch_seed_failed", error_type=error_type(exc)) continue entry.last_stat = current entry.ticks += 1 diff --git a/app/assets/seeder.py b/app/assets/seeder.py index 2cc73d84fb5..69ba63f7053 100644 --- a/app/assets/seeder.py +++ b/app/assets/seeder.py @@ -441,7 +441,7 @@ def shutdown(self, timeout: float = 5.0) -> bool: self._thread = None return joined - def mark_missing_outside_prefixes(self) -> int: + def mark_missing_outside_prefixes(self) -> int | None: """Mark references as missing when outside all known root prefixes. This is a non-destructive soft-delete operation. Assets and their @@ -476,16 +476,16 @@ def mark_missing_outside_prefixes(self) -> int: all_prefixes = get_owned_prefixes() marked = mark_missing_outside_prefixes_safely(all_prefixes) - marked_count = 0 if marked is None else marked - if marked is not None: - emit( - "seeder.marked_missing", - count=marked_count, - stage=_ScanStage.MARK_MISSING.value, - ) - if marked_count > 0: - logging.info("Marked %d references as missing", marked_count) - return marked_count + if marked is None: + return None + emit( + "seeder.marked_missing", + count=marked, + stage=_ScanStage.MARK_MISSING.value, + ) + if marked > 0: + logging.info("Marked %d references as missing", marked) + return marked finally: with self._lock: self._reset_to_idle() diff --git a/tests-unit/assets_test/services/test_admission_gate.py b/tests-unit/assets_test/services/test_admission_gate.py index 7e43a013fb0..4c93b4f4bb3 100644 --- a/tests-unit/assets_test/services/test_admission_gate.py +++ b/tests-unit/assets_test/services/test_admission_gate.py @@ -1,3 +1,4 @@ +import logging import os from collections.abc import Iterator from pathlib import Path @@ -96,7 +97,7 @@ def test_never_stabilizes_dropped_after_cap(session, temp_dir: Path): def test_stat_error_drops_entry_and_allows_other_watch_entries_to_commit( - session, temp_dir: Path, monkeypatch + session, temp_dir: Path, monkeypatch, caplog: pytest.LogCaptureFixture ): unreadable_path = temp_dir / "unreadable.bin" stable_path = temp_dir / "stable.bin" @@ -118,12 +119,62 @@ def _stat(path: str): monkeypatch.setattr("folder_paths.get_input_directory", lambda: str(temp_dir)) monkeypatch.setattr(scanner_admission, "os", SimpleNamespace(stat=_stat)) - tick_watch_list(session) + with caplog.at_level(logging.INFO): + tick_watch_list(session) session.commit() persisted_paths = set(session.scalars(select(AssetContent.path)).all()) assert persisted_paths == {str(stable_path)} assert _WATCH_LIST == [] + assert any( + record.getMessage() + == f"Dropping watched asset after stat failed: {unreadable_path}" + for record in caplog.records + ) + assert any( + record.getMessage() + == "[assets-event] scanner.watch_stat_failed error_type=PermissionError" + for record in caplog.records + ) + + +def test_seed_failure_does_not_stop_watch_list_drain( + session, + temp_dir: Path, + monkeypatch, + caplog: pytest.LogCaptureFixture, +) -> None: + paths = [temp_dir / name for name in ("broken.bin", "stable.bin")] + for path in paths: + path.write_bytes(path.name.encode()) + _WATCH_LIST[:] = [_WatchEntry(str(path), path.stat()) for path in paths] + attempted: list[str] = [] + + def seed_or_raise(_session, specs) -> int: + path = specs[0]["abs_path"] + attempted.append(path) + if path == str(paths[0]): + raise RuntimeError("forced watch seed failure") + return 1 + + monkeypatch.setattr("folder_paths.get_input_directory", lambda: str(temp_dir)) + monkeypatch.setattr("app.assets.scanner.seed_asset_specs", seed_or_raise) + + with caplog.at_level(logging.INFO): + tick_watch_list(session) + + assert attempted == [str(path) for path in paths] + assert _WATCH_LIST == [] + assert any( + record.getMessage() + == f"Dropping watched asset after seeding failed: {paths[0]}" + for record in caplog.records + ) + assert any( + record.getMessage() + == "[assets-event] scanner.watch_seed_failed error_type=RuntimeError" + for record in caplog.records + ) def test_stable_scan_admission_removes_watch_entry_before_next_tick(session, temp_dir: Path, monkeypatch): diff --git a/tests-unit/assets_test/services/test_api_routes_b.py b/tests-unit/assets_test/services/test_api_routes_b.py index c6e186eb368..2f86313d33f 100644 --- a/tests-unit/assets_test/services/test_api_routes_b.py +++ b/tests-unit/assets_test/services/test_api_routes_b.py @@ -153,3 +153,19 @@ async def test_from_hash_off_mode_returns_400(monkeypatch): response_body = response.body assert isinstance(response_body, bytes | bytearray) assert json.loads(response_body)["error"]["code"] == "FEATURE_DISABLED" + + +@pytest.mark.asyncio +async def test_prune_failure_returns_500_instead_of_completed(monkeypatch): + monkeypatch.setattr( + routes.asset_seeder, "mark_missing_outside_prefixes", lambda: None + ) + + response = await routes.mark_missing_assets.__wrapped__( + make_mocked_request("POST", "/api/assets/prune") + ) + + assert response.status == 500 + response_body = response.body + assert isinstance(response_body, bytes | bytearray) + assert json.loads(response_body)["error"]["code"] == "PRUNE_FAILED" diff --git a/tests-unit/assets_test/services/test_scanner_seed_resilience.py b/tests-unit/assets_test/services/test_scanner_seed_resilience.py index 126d108fde9..06cb0ee2ed1 100644 --- a/tests-unit/assets_test/services/test_scanner_seed_resilience.py +++ b/tests-unit/assets_test/services/test_scanner_seed_resilience.py @@ -1,3 +1,4 @@ +import logging import os from collections.abc import Callable from pathlib import Path @@ -208,6 +209,86 @@ def _create_record_or_raise( assert raised.value is unrelated_error +def test_seed_attempts_remaining_specs_before_propagating_integrity_error( + session: Session, temp_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + paths = [temp_dir / name for name in ("first.bin", "broken.bin", "last.bin")] + for path in paths: + path.write_bytes(path.name.encode()) + attempted: list[str] = [] + unrelated_error = IntegrityError( + "forced record creation failure", + {}, + ValueError("unrelated integrity failure"), + ) + + def _create_record_or_raise( + session_arg: Session, + *, + content_id: str, + name: str, + mime_type: str | None, + job_id: str | None, + loader_path: str | None, + tags: list[str], + ) -> Asset: + attempted.append(name) + if name == "broken.bin": + raise unrelated_error + return create_record( + session_arg, + content_id=content_id, + name=name, + mime_type=mime_type, + job_id=job_id, + loader_path=loader_path, + tags=tags, + ) + + monkeypatch.setattr("app.assets.scanner.create_record", _create_record_or_raise) + + with pytest.raises(IntegrityError) as raised: + seed_asset_specs(session, [_spec(path) for path in paths]) + session.commit() + + assert raised.value is unrelated_error + assert attempted == ["first.bin", "broken.bin", "last.bin"] + assert {record.name for record in session.scalars(select(Asset))} == { + "first.bin", + "last.bin", + } + + +def test_seed_skips_negative_fresh_mtime_with_warning_and_telemetry( + session: Session, + temp_dir: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + paths = [temp_dir / name for name in ("first.bin", "old.bin", "last.bin")] + for path in paths: + path.write_bytes(path.name.encode()) + pre_epoch_ns = -315_547_200_000_000_000 + os.utime(paths[1], ns=(pre_epoch_ns, pre_epoch_ns)) + + with caplog.at_level(logging.INFO): + created = seed_asset_specs(session, [_spec(path) for path in paths]) + session.commit() + + assert created == 2 + assert {record.name for record in session.scalars(select(Asset))} == { + "first.bin", + "last.bin", + } + assert any( + record.getMessage() == f"Skipping asset with invalid mtime during scan: {paths[1]}" + for record in caplog.records + ) + assert any( + record.getMessage() == "[assets-event] scanner.invalid_mtime" + for record in caplog.records + ) + + def test_seed_persists_fresh_stat_after_spec_was_built( session: Session, temp_dir: Path ) -> None: diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.py index 40066fc114e..d27f1750add 100644 --- a/tests-unit/seeder_test/test_seeder.py +++ b/tests-unit/seeder_test/test_seeder.py @@ -434,7 +434,7 @@ def test_standalone_mark_missing_emits_count_with_mark_missing_stage( ] -def test_standalone_mark_missing_failure_emits_no_success_event( +def test_standalone_mark_missing_failure_returns_none_and_emits_no_success_event( scan_seeder: _AssetSeeder, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, @@ -450,7 +450,7 @@ def fail_create_session(): with caplog.at_level(logging.INFO): result = scan_seeder.mark_missing_outside_prefixes() - assert result == 0 + assert result is None assert events_named(caplog, "scanner.mark_missing_failed") == [ {"error_type": "RuntimeError"} ] diff --git a/tests-unit/test_assets_event_log_static.py b/tests-unit/test_assets_event_log_static.py index a680953b229..69632eaee67 100644 --- a/tests-unit/test_assets_event_log_static.py +++ b/tests-unit/test_assets_event_log_static.py @@ -63,6 +63,13 @@ class CallSite(NamedTuple): # todo 16 - discovery/enrich stat failures, emit-once per scan per site CallSite("app/assets/scanner.py", "build_asset_specs", "scanner.stat_failed"), CallSite("app/assets/scanner.py", "enrich_asset", "scanner.stat_failed"), + CallSite("app/assets/scanner.py", "seed_asset_specs", "scanner.invalid_mtime"), + CallSite( + "app/assets/scanner_admission.py", "tick_watch_list", "scanner.watch_stat_failed" + ), + CallSite( + "app/assets/scanner_admission.py", "tick_watch_list", "scanner.watch_seed_failed" + ), } ) From d496e23bb5b530f18f476bfbdfa0be20beaf020f Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 01:42:22 -0700 Subject: [PATCH 074/111] fix(assets): remove temporary uploads on non-UploadError failures --- app/assets/api/upload.py | 64 +++++++------- .../services/test_upload_temp_cleanup.py | 83 +++++++++++++++++++ 2 files changed, 117 insertions(+), 30 deletions(-) diff --git a/app/assets/api/upload.py b/app/assets/api/upload.py index 20ce2ad85eb..b10c73a1640 100644 --- a/app/assets/api/upload.py +++ b/app/assets/api/upload.py @@ -1,8 +1,7 @@ """Reads a multipart upload off the wire and lands its bytes in a temporary file the ingest service can hash and move. The body is consumed in chunks so a large model never has to fit in memory. Validation failures remove any temporary file -before they are reported to the caller. Field values are validated as they -arrive, letting a bad request fail before its bytes are written. +before they are reported to the caller. """ import logging @@ -64,6 +63,7 @@ async def parse_multipart_upload( file_written = 0 tmp_path: str | None = None + parse_succeeded = False try: while True: @@ -133,35 +133,39 @@ async def parse_multipart_upload( provided_mime_type = ((await field.text()) or "").strip() or None elif fname == "preview_id": provided_preview_id = ((await field.text()) or "").strip() or None - except UploadError: - delete_temp_file_if_exists(tmp_path) - raise - if not file_present and not (provided_hash and provided_hash_exists): - raise UploadError( - 400, "MISSING_FILE", "Form must include a 'file' part or a known 'hash'." - ) - if ( - file_present - and file_written == 0 - and not (provided_hash and provided_hash_exists) - ): - delete_temp_file_if_exists(tmp_path) - raise UploadError(400, "EMPTY_UPLOAD", "Uploaded file is empty.") - - return ParsedUpload( - file_present=file_present, - file_written=file_written, - file_client_name=file_client_name, - tmp_path=tmp_path, - tags_raw=tags_raw, - provided_name=provided_name, - user_metadata_raw=user_metadata_raw, - provided_hash=provided_hash, - provided_hash_exists=provided_hash_exists, - provided_mime_type=provided_mime_type, - provided_preview_id=provided_preview_id, - ) + if not file_present and not (provided_hash and provided_hash_exists): + raise UploadError( + 400, + "MISSING_FILE", + "Form must include a 'file' part or a known 'hash'.", + ) + + if ( + file_present + and file_written == 0 + and not (provided_hash and provided_hash_exists) + ): + raise UploadError(400, "EMPTY_UPLOAD", "Uploaded file is empty.") + + parsed = ParsedUpload( + file_present=file_present, + file_written=file_written, + file_client_name=file_client_name, + tmp_path=tmp_path, + tags_raw=tags_raw, + provided_name=provided_name, + user_metadata_raw=user_metadata_raw, + provided_hash=provided_hash, + provided_hash_exists=provided_hash_exists, + provided_mime_type=provided_mime_type, + provided_preview_id=provided_preview_id, + ) + parse_succeeded = True + return parsed + finally: + if not parse_succeeded: + delete_temp_file_if_exists(tmp_path) def delete_temp_file_if_exists(tmp_path: str | None) -> None: diff --git a/tests-unit/assets_test/services/test_upload_temp_cleanup.py b/tests-unit/assets_test/services/test_upload_temp_cleanup.py index c60cdc44c93..6f8eb913b33 100644 --- a/tests-unit/assets_test/services/test_upload_temp_cleanup.py +++ b/tests-unit/assets_test/services/test_upload_temp_cleanup.py @@ -1,3 +1,4 @@ +import asyncio import uuid from pathlib import Path from unittest.mock import AsyncMock @@ -37,6 +38,88 @@ async def test_multipart_id_after_file_removes_temp_upload( assert list((tmp_path / "uploads").iterdir()) == [] +def _file_field() -> AsyncMock: + field = AsyncMock() + field.name = "file" + field.filename = "model.safetensors" + field.read_chunk.side_effect = [b"uploaded bytes", b""] + return field + + +def _multipart_request(*fields: AsyncMock) -> AsyncMock: + reader = AsyncMock() + reader.next.side_effect = [*fields, None] + request = AsyncMock() + request.content_type = "multipart/form-data" + request.multipart.return_value = reader + return request + + +@pytest.mark.asyncio +async def test_invalid_utf8_after_file_removes_temp_upload( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(folder_paths, "get_temp_directory", lambda: str(tmp_path)) + tags_field = AsyncMock() + tags_field.name = "tags" + tags_field.text.side_effect = UnicodeDecodeError( + "utf-8", b"\xff", 0, 1, "invalid start byte" + ) + request = _multipart_request(_file_field(), tags_field) + + with pytest.raises(UnicodeDecodeError): + await parse_multipart_upload(request, lambda _hash: False) + + assert list((tmp_path / "uploads").iterdir()) == [] + + +@pytest.mark.asyncio +async def test_malformed_framing_after_file_removes_temp_upload( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(folder_paths, "get_temp_directory", lambda: str(tmp_path)) + request = _multipart_request(_file_field()) + request.multipart.return_value.next.side_effect = [ + _file_field(), + ValueError("Reading after EOF"), + ] + + with pytest.raises(ValueError, match="Reading after EOF"): + await parse_multipart_upload(request, lambda _hash: False) + + assert list((tmp_path / "uploads").iterdir()) == [] + + +@pytest.mark.asyncio +async def test_cancelled_upload_removes_temp_upload( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(folder_paths, "get_temp_directory", lambda: str(tmp_path)) + request = _multipart_request(_file_field()) + request.multipart.return_value.next.side_effect = [ + _file_field(), + asyncio.CancelledError(), + ] + + with pytest.raises(asyncio.CancelledError): + await parse_multipart_upload(request, lambda _hash: False) + + assert list((tmp_path / "uploads").iterdir()) == [] + + +@pytest.mark.asyncio +async def test_successful_parse_returns_consumable_temp_upload( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(folder_paths, "get_temp_directory", lambda: str(tmp_path)) + request = _multipart_request(_file_field()) + + parsed = await parse_multipart_upload(request, lambda _hash: False) + + assert parsed.tmp_path is not None + assert Path(parsed.tmp_path).read_bytes() == b"uploaded bytes" + + def test_destination_resolution_failure_removes_temp_upload( mock_create_session, tmp_path: Path ) -> None: From 80682d076a77e9ff6704a4f287e293a08729ca92 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 01:47:28 -0700 Subject: [PATCH 075/111] fix(assets): preserve successful specs when a scan batch fault propagates --- app/assets/scanner.py | 6 ++- .../services/test_ingest_orphan_content.py | 13 +++-- .../services/test_scanner_seed_resilience.py | 50 ++++++++++++++++++- 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index 7e2a0a187b7..3ae77c56c89 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -449,7 +449,11 @@ def insert_asset_specs(specs: list[SeedAssetSpec], _tag_pool: set[str]) -> int: if not specs: return 0 with create_session() as sess: - created = seed_asset_specs(sess, specs) + try: + created = seed_asset_specs(sess, specs) + except Exception: + sess.commit() + raise sess.commit() return created diff --git a/tests-unit/assets_test/services/test_ingest_orphan_content.py b/tests-unit/assets_test/services/test_ingest_orphan_content.py index ae209945ca6..7be49fe43d8 100644 --- a/tests-unit/assets_test/services/test_ingest_orphan_content.py +++ b/tests-unit/assets_test/services/test_ingest_orphan_content.py @@ -119,7 +119,7 @@ def _orphaned_content_paths(session: Session) -> list[str]: ] -def test_seed_asset_specs_orphans_nothing_and_keeps_earlier_specs_on_record_failure( +def test_seed_asset_specs_orphans_nothing_and_keeps_surrounding_specs_on_record_failure( session: Session, tmp_path, monkeypatch: pytest.MonkeyPatch ) -> None: specs: list[SeedAssetSpec] = [] @@ -146,7 +146,7 @@ def _create_record_or_raise(session_arg, content_id, name, *args, **kwargs): with pytest.raises(RuntimeError, match="forced create_record failure"): seed_asset_specs(session, specs) - session.rollback() + session.commit() assert _content_at(session, paths[fail_name]) is None, ( "the failed spec's content must not outlive the record that would have referenced it" @@ -162,7 +162,10 @@ def _create_record_or_raise(session_arg, content_id, name, *args, **kwargs): assert session.scalar(select(Asset).where(Asset.name == "first.bin")) is not None assert _reference_count(session, survivor.id) == 1 - assert attempted == ["first.bin", fail_name] - assert _content_at(session, paths["last.bin"]) is None, ( - "the raise aborts the loop, so the spec after the failed one is never attempted" + assert attempted == ["first.bin", fail_name, "last.bin"] + last_survivor = _content_at(session, paths["last.bin"]) + assert last_survivor is not None, ( + "a failed spec must not prevent the remaining specs in the batch from being seeded" ) + assert session.scalar(select(Asset).where(Asset.name == "last.bin")) is not None + assert _reference_count(session, last_survivor.id) == 1 diff --git a/tests-unit/assets_test/services/test_scanner_seed_resilience.py b/tests-unit/assets_test/services/test_scanner_seed_resilience.py index 06cb0ee2ed1..96e56534977 100644 --- a/tests-unit/assets_test/services/test_scanner_seed_resilience.py +++ b/tests-unit/assets_test/services/test_scanner_seed_resilience.py @@ -1,6 +1,7 @@ import logging import os from collections.abc import Callable +from contextlib import contextmanager from pathlib import Path from unittest.mock import patch @@ -16,7 +17,7 @@ create_record, delete_record, ) -from app.assets.scanner import SeedAssetSpec, seed_asset_specs +from app.assets.scanner import SeedAssetSpec, insert_asset_specs, seed_asset_specs from app.assets.services.snapshot_hash import snapshot_hash @@ -259,6 +260,53 @@ def _create_record_or_raise( } +def test_insert_commits_successful_specs_before_propagating_batch_fault( + db_engine, temp_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + paths = [temp_dir / name for name in ("first.bin", "broken.bin", "last.bin")] + for path in paths: + path.write_bytes(path.name.encode()) + + @contextmanager + def _create_session(): + with Session(db_engine) as session: + yield session + + def _create_record_or_raise( + session_arg: Session, + *, + content_id: str, + name: str, + mime_type: str | None, + job_id: str | None, + loader_path: str | None, + tags: list[str], + ) -> Asset: + if name == "broken.bin": + raise RuntimeError("forced record creation failure") + return create_record( + session_arg, + content_id=content_id, + name=name, + mime_type=mime_type, + job_id=job_id, + loader_path=loader_path, + tags=tags, + ) + + monkeypatch.setattr("app.assets.scanner.create_session", _create_session) + monkeypatch.setattr("app.assets.scanner.create_record", _create_record_or_raise) + + with pytest.raises(RuntimeError, match="forced record creation failure"): + insert_asset_specs([_spec(path) for path in paths], set()) + + with Session(db_engine) as session: + assert {record.name for record in session.scalars(select(Asset))} == { + "first.bin", + "last.bin", + } + + def test_seed_skips_negative_fresh_mtime_with_warning_and_telemetry( session: Session, temp_dir: Path, From 1eac7422e54836367057c93a6df7763706c02f3f Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 11:24:08 -0700 Subject: [PATCH 076/111] test(db): drop the inert legacy-copy patch from the path preparation tests prepare_file_db_path no longer copies the legacy database - that moved inside the process lock in _init_file_db - so patching copy_legacy_default_db here did nothing. Leaving it implied a side effect the function does not have, and would have masked one if it were reintroduced. --- tests-unit/app_test/database_path_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests-unit/app_test/database_path_test.py b/tests-unit/app_test/database_path_test.py index d1387c041ef..dbeeffbf09d 100644 --- a/tests-unit/app_test/database_path_test.py +++ b/tests-unit/app_test/database_path_test.py @@ -116,7 +116,6 @@ def test_prepare_file_database_creates_parent_directory(monkeypatch, tmp_path): db_path = tmp_path / "nested" / "comfyui.db" monkeypatch.setattr(db.args, "database_url", None) - monkeypatch.setattr(db, "copy_legacy_default_db", lambda path: None) db.prepare_file_db_path(str(db_path)) @@ -126,7 +125,6 @@ def test_prepare_file_database_creates_parent_directory(monkeypatch, tmp_path): def test_prepare_file_database_accepts_relative_database_path(monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) monkeypatch.setattr(db.args, "database_url", "sqlite:///relative.db") - monkeypatch.setattr(db, "copy_legacy_default_db", lambda path: None) db.prepare_file_db_path("relative.db") From 1f34110261030157931b3eb993586d7fc6b74561 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 16:02:44 -0700 Subject: [PATCH 077/111] fix(assets): report specs committed before a batch fault and preserve the fault itself --- app/assets/scanner.py | 29 +++- app/assets/seeder.py | 5 +- .../services/test_scanner_seed_resilience.py | 6 +- tests-unit/seeder_test/test_seeder.py | 159 +++++++++++++++++- 4 files changed, 186 insertions(+), 13 deletions(-) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index 3ae77c56c89..b08a8841177 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -445,17 +445,32 @@ def seed_asset_specs(session: Session, specs: list[SeedAssetSpec]) -> int: return created -def insert_asset_specs(specs: list[SeedAssetSpec], _tag_pool: set[str]) -> int: +def insert_asset_specs( + specs: list[SeedAssetSpec], _tag_pool: set[str] +) -> tuple[int, Exception | None]: if not specs: - return 0 + return 0, None with create_session() as sess: + created = 0 + first_error: Exception | None = None + for spec in specs: + try: + created += seed_asset_specs(sess, [spec]) + except Exception as error: + if first_error is None: + first_error = error try: - created = seed_asset_specs(sess, specs) - except Exception: sess.commit() - raise - sess.commit() - return created + except Exception: + if first_error is None: + raise + logging.exception("Failed to commit successful specs from failed asset batch") + try: + sess.rollback() + except Exception: + logging.exception("Failed to roll back asset batch after commit failure") + return 0, first_error + return created, first_error def build_unenriched_candidates_statement( diff --git a/app/assets/seeder.py b/app/assets/seeder.py index 69ba63f7053..d60c381e319 100644 --- a/app/assets/seeder.py +++ b/app/assets/seeder.py @@ -856,9 +856,12 @@ def _run_fast_phase(self, roots: tuple[RootType, ...]) -> tuple[int, int, int]: batch = specs[i : i + batch_size] batch_tags = {t for spec in batch for t in spec["tags"]} + created = 0 try: - created = insert_asset_specs(batch, batch_tags) + created, batch_error = insert_asset_specs(batch, batch_tags) total_created += created + if batch_error is not None: + raise batch_error except Exception as e: self._add_error(f"Batch insert failed at offset {i}: {e}") logging.exception("Batch insert failed at offset %d", i) diff --git a/tests-unit/assets_test/services/test_scanner_seed_resilience.py b/tests-unit/assets_test/services/test_scanner_seed_resilience.py index 96e56534977..28d53917179 100644 --- a/tests-unit/assets_test/services/test_scanner_seed_resilience.py +++ b/tests-unit/assets_test/services/test_scanner_seed_resilience.py @@ -297,14 +297,16 @@ def _create_record_or_raise( monkeypatch.setattr("app.assets.scanner.create_session", _create_session) monkeypatch.setattr("app.assets.scanner.create_record", _create_record_or_raise) - with pytest.raises(RuntimeError, match="forced record creation failure"): - insert_asset_specs([_spec(path) for path in paths], set()) + created, error = insert_asset_specs([_spec(path) for path in paths], set()) with Session(db_engine) as session: assert {record.name for record in session.scalars(select(Asset))} == { "first.bin", "last.bin", } + assert created == 2 + assert isinstance(error, RuntimeError) + assert str(error) == "forced record creation failure" def test_seed_skips_negative_fresh_mtime_with_warning_and_telemetry( diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.py index d27f1750add..0b918ae5bcf 100644 --- a/tests-unit/seeder_test/test_seeder.py +++ b/tests-unit/seeder_test/test_seeder.py @@ -1,18 +1,21 @@ import logging import re import threading -from contextlib import nullcontext +from contextlib import contextmanager, nullcontext +from pathlib import Path from unittest.mock import Mock import pytest -from sqlalchemy import create_engine +from sqlalchemy import create_engine, select +from sqlalchemy.engine import Engine from sqlalchemy.orm import Session from app.assets import scanner as scanner_module from app.assets import seeder as seeder_module -from app.assets.database.models import Base +from app.assets.database.models import Asset, Base from app.assets.database.queries import create_content, create_record, mark_content_missing from app.assets.event_log import TAG +from app.assets.scanner import SeedAssetSpec from app.assets.seeder import Progress, ScanPhase, State, _AssetSeeder, _ScanStage, _ScanState @@ -68,6 +71,95 @@ def events_named( return [fields for event, fields in tagged_events(caplog) if event == event_name] +def _seed_spec(path: Path) -> SeedAssetSpec: + stat_result = path.stat() + return { + "abs_path": str(path), + "size_bytes": stat_result.st_size, + "mtime_ns": stat_result.st_mtime_ns, + "info_name": path.name, + "tags": ["input"], + "fname": path.name, + "metadata": None, + "mime_type": None, + "job_id": None, + } + + +def _configure_fast_phase( + monkeypatch: pytest.MonkeyPatch, + paths: list[Path], + specs: list[SeedAssetSpec], +) -> None: + monkeypatch.setattr( + seeder_module, "sync_root_safely", lambda _root, _progress: set() + ) + monkeypatch.setattr( + seeder_module, "collect_paths_for_roots", lambda _roots: [str(path) for path in paths] + ) + monkeypatch.setattr( + seeder_module, + "build_asset_specs", + lambda *_args, **_kwargs: (specs, set(), 0), + ) + watch_session = Mock() + monkeypatch.setattr(seeder_module, "create_session", lambda: nullcontext(watch_session)) + monkeypatch.setattr(seeder_module, "tick_watch_list", lambda _session: None) + + +def _run_faulting_fast_phase( + scan_seeder: _AssetSeeder, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + original_fault: Exception, + commit_failure: Exception | None = None, +) -> tuple[Engine, tuple[int, int, int]]: + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(engine) + paths = [tmp_path / name for name in ("first.bin", "broken.bin", "last.bin")] + for path in paths: + path.write_bytes(path.name.encode()) + specs = [_seed_spec(path) for path in paths] + + @contextmanager + def database_session(): + with Session(engine) as session: + if commit_failure is not None: + session.connection().exec_driver_sql("BEGIN") + monkeypatch.setattr( + session, "commit", Mock(side_effect=commit_failure) + ) + yield session + + def create_record_or_raise( + session: Session, + *, + content_id: str, + name: str, + mime_type: str | None, + job_id: str | None, + loader_path: str | None, + tags: list[str], + ) -> Asset: + if name == "broken.bin": + raise original_fault + return create_record( + session, + content_id=content_id, + name=name, + mime_type=mime_type, + job_id=job_id, + loader_path=loader_path, + tags=tags, + ) + + monkeypatch.setattr(scanner_module, "create_session", database_session) + monkeypatch.setattr(scanner_module, "create_record", create_record_or_raise) + monkeypatch.setattr(scanner_module.mode, "hashing_enabled", lambda: False) + _configure_fast_phase(monkeypatch, paths, specs) + return engine, scan_seeder._run_fast_phase(("models",)) + + def test_idle_status_returns_a_progress_snapshot() -> None: seeder = _AssetSeeder() seeder._last_progress = Progress(created=1) @@ -517,3 +609,64 @@ def fail_insert(batch, batch_tags) -> int: ] tagged = "\n".join(record.getMessage() for record in caplog.records if TAG in record.getMessage()) assert "/private/models/asset.safetensors" not in tagged + + +def test_batch_insert_fault_reports_the_specs_committed_before_it( + scan_seeder: _AssetSeeder, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + tmp_path: Path, +) -> None: + with caplog.at_level(logging.INFO): + engine, result = _run_faulting_fast_phase( + scan_seeder, + monkeypatch, + tmp_path, + OSError("forced record creation failure"), + ) + + with Session(engine) as session: + assert {record.name for record in session.scalars(select(Asset))} == { + "first.bin", + "last.bin", + } + assert result == (2, 0, 3) + assert scan_seeder._scan_state is not None + assert scan_seeder._scan_state.created == 2 + + +def test_salvage_commit_failure_reports_the_original_batch_fault( + scan_seeder: _AssetSeeder, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + tmp_path: Path, +) -> None: + original_fault = OSError("No space left on device") + commit_failure = RuntimeError("forced salvage commit failure") + + with caplog.at_level(logging.INFO): + engine, result = _run_faulting_fast_phase( + scan_seeder, + monkeypatch, + tmp_path, + original_fault, + commit_failure, + ) + + with Session(engine) as session: + assert session.scalar(select(Asset)) is None + assert result == (0, 0, 3) + assert scan_seeder._errors == [ + "Batch insert failed at offset 0: No space left on device" + ] + assert events_named(caplog, "seeder.batch_insert_failed") == [ + {"error_type": "OSError"} + ] + caller_logs = [ + record + for record in caplog.records + if record.getMessage().startswith("Batch insert failed") + ] + assert len(caller_logs) == 1 + assert caller_logs[0].exc_info is not None + assert caller_logs[0].exc_info[1] is original_fault From e9b3587ae7c7f8d406bbcf1a0bc11f8d77b67a04 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 16:04:01 -0700 Subject: [PATCH 078/111] fix(assets): distinguish partial batch insert failures --- app/assets/seeder.py | 11 +++++++++-- tests-unit/seeder_test/test_seeder.py | 9 +++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/app/assets/seeder.py b/app/assets/seeder.py index d60c381e319..5cc4837ec9b 100644 --- a/app/assets/seeder.py +++ b/app/assets/seeder.py @@ -863,8 +863,15 @@ def _run_fast_phase(self, roots: tuple[RootType, ...]) -> tuple[int, int, int]: if batch_error is not None: raise batch_error except Exception as e: - self._add_error(f"Batch insert failed at offset {i}: {e}") - logging.exception("Batch insert failed at offset %d", i) + self._add_error( + f"Batch insert encountered an error at offset {i} " + f"after creating {created}: {e}" + ) + logging.exception( + "Batch insert encountered an error at offset %d after creating %d", + i, + created, + ) emit("seeder.batch_insert_failed", error_type=error_type(e)) scanned = i + len(batch) diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.py index 0b918ae5bcf..a860e8f5671 100644 --- a/tests-unit/seeder_test/test_seeder.py +++ b/tests-unit/seeder_test/test_seeder.py @@ -633,6 +633,10 @@ def test_batch_insert_fault_reports_the_specs_committed_before_it( assert result == (2, 0, 3) assert scan_seeder._scan_state is not None assert scan_seeder._scan_state.created == 2 + assert scan_seeder._errors == [ + "Batch insert encountered an error at offset 0 after creating 2: " + "forced record creation failure" + ] def test_salvage_commit_failure_reports_the_original_batch_fault( @@ -657,7 +661,8 @@ def test_salvage_commit_failure_reports_the_original_batch_fault( assert session.scalar(select(Asset)) is None assert result == (0, 0, 3) assert scan_seeder._errors == [ - "Batch insert failed at offset 0: No space left on device" + "Batch insert encountered an error at offset 0 after creating 0: " + "No space left on device" ] assert events_named(caplog, "seeder.batch_insert_failed") == [ {"error_type": "OSError"} @@ -665,7 +670,7 @@ def test_salvage_commit_failure_reports_the_original_batch_fault( caller_logs = [ record for record in caplog.records - if record.getMessage().startswith("Batch insert failed") + if record.getMessage().startswith("Batch insert encountered an error") ] assert len(caller_logs) == 1 assert caller_logs[0].exc_info is not None From d071895286369cba421ba2a6210483a8ce1e241a Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 16:26:49 -0700 Subject: [PATCH 079/111] refactor(assets): collapse the duplicated batch fault deferral into seed_asset_specs --- app/assets/scanner.py | 17 +++------- app/assets/scanner_admission.py | 7 ++--- .../services/test_admission_gate.py | 8 ++--- .../services/test_ingest_orphan_content.py | 5 +-- .../services/test_null_hash_recovery.py | 12 ++++--- .../services/test_recovery_gate.py | 15 ++++++--- .../services/test_scan_lifecycle.py | 5 ++- .../assets_test/services/test_scanner_b.py | 3 +- .../services/test_scanner_seed_resilience.py | 31 +++++++++++-------- .../assets_test/test_intended_behaviour.py | 5 ++- 10 files changed, 61 insertions(+), 47 deletions(-) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index b08a8841177..bf6caa73216 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -377,7 +377,9 @@ def build_asset_specs( return specs, tag_pool, skipped -def seed_asset_specs(session: Session, specs: list[SeedAssetSpec]) -> int: +def seed_asset_specs( + session: Session, specs: list[SeedAssetSpec] +) -> tuple[int, Exception | None]: created = 0 first_error: Exception | None = None for spec in specs: @@ -440,9 +442,7 @@ def seed_asset_specs(session: Session, specs: list[SeedAssetSpec]) -> int: except Exception as error: if first_error is None: first_error = error - if first_error is not None: - raise first_error - return created + return created, first_error def insert_asset_specs( @@ -451,14 +451,7 @@ def insert_asset_specs( if not specs: return 0, None with create_session() as sess: - created = 0 - first_error: Exception | None = None - for spec in specs: - try: - created += seed_asset_specs(sess, [spec]) - except Exception as error: - if first_error is None: - first_error = error + created, first_error = seed_asset_specs(sess, specs) try: sess.commit() except Exception: diff --git a/app/assets/scanner_admission.py b/app/assets/scanner_admission.py index 82dba10b602..c201034daca 100644 --- a/app/assets/scanner_admission.py +++ b/app/assets/scanner_admission.py @@ -92,13 +92,12 @@ def tick_watch_list(session: Session) -> None: "mime_type": mimetypes.guess_type(entry.path, strict=False)[0], "job_id": None, } - try: - seed_asset_specs(session, [spec]) - except Exception as exc: + _created, seed_error = seed_asset_specs(session, [spec]) + if seed_error is not None: logging.warning( "Dropping watched asset after seeding failed: %s", entry.path ) - emit("scanner.watch_seed_failed", error_type=error_type(exc)) + emit("scanner.watch_seed_failed", error_type=error_type(seed_error)) continue entry.last_stat = current entry.ticks += 1 diff --git a/tests-unit/assets_test/services/test_admission_gate.py b/tests-unit/assets_test/services/test_admission_gate.py index 4c93b4f4bb3..288027a3f94 100644 --- a/tests-unit/assets_test/services/test_admission_gate.py +++ b/tests-unit/assets_test/services/test_admission_gate.py @@ -150,15 +150,15 @@ def test_seed_failure_does_not_stop_watch_list_drain( _WATCH_LIST[:] = [_WatchEntry(str(path), path.stat()) for path in paths] attempted: list[str] = [] - def seed_or_raise(_session, specs) -> int: + def seed_or_return_error(_session, specs) -> tuple[int, Exception | None]: path = specs[0]["abs_path"] attempted.append(path) if path == str(paths[0]): - raise RuntimeError("forced watch seed failure") - return 1 + return 0, RuntimeError("forced watch seed failure") + return 1, None monkeypatch.setattr("folder_paths.get_input_directory", lambda: str(temp_dir)) - monkeypatch.setattr("app.assets.scanner.seed_asset_specs", seed_or_raise) + monkeypatch.setattr("app.assets.scanner.seed_asset_specs", seed_or_return_error) with caplog.at_level(logging.INFO): tick_watch_list(session) diff --git a/tests-unit/assets_test/services/test_ingest_orphan_content.py b/tests-unit/assets_test/services/test_ingest_orphan_content.py index 7be49fe43d8..826e6dfe802 100644 --- a/tests-unit/assets_test/services/test_ingest_orphan_content.py +++ b/tests-unit/assets_test/services/test_ingest_orphan_content.py @@ -144,10 +144,11 @@ def _create_record_or_raise(session_arg, content_id, name, *args, **kwargs): monkeypatch.setattr("app.assets.scanner.create_record", _create_record_or_raise) - with pytest.raises(RuntimeError, match="forced create_record failure"): - seed_asset_specs(session, specs) + _created, error = seed_asset_specs(session, specs) session.commit() + assert isinstance(error, RuntimeError) + assert str(error) == "forced create_record failure" assert _content_at(session, paths[fail_name]) is None, ( "the failed spec's content must not outlive the record that would have referenced it" ) diff --git a/tests-unit/assets_test/services/test_null_hash_recovery.py b/tests-unit/assets_test/services/test_null_hash_recovery.py index 2a247fced8c..6736ab004f0 100644 --- a/tests-unit/assets_test/services/test_null_hash_recovery.py +++ b/tests-unit/assets_test/services/test_null_hash_recovery.py @@ -81,9 +81,10 @@ def test_deleted_null_hash_row_recovers_via_scanner_after_restore( assert path.stat().st_mtime_ns == stat.st_mtime_ns, "setup: mtime must round-trip exactly" with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - created = seed_asset_specs(session, [_spec(path)]) + created, error = seed_asset_specs(session, [_spec(path)]) session.commit() + assert error is None assert created == 0, "the original row must recover — no fresh content row minted" recovered = session.get(AssetContent, content_id) assert recovered.is_missing is False @@ -118,9 +119,10 @@ def test_different_bytes_restored_at_same_path_does_not_recover_old_row( path.write_bytes(b"a completely different, much longer payload than the original") with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - created = seed_asset_specs(session, [_spec(path)]) + created, error = seed_asset_specs(session, [_spec(path)]) session.commit() + assert error is None assert created == 1, "a genuinely different file must take the normal new-content path" assert session.get(AssetContent, content_id).is_missing is True, ( "the old row must stay missing — recovering it here would hand the wrong record's " @@ -158,9 +160,10 @@ def test_same_size_different_mtime_restored_at_same_path_does_not_recover_old_ro assert path.stat().st_size == stat.st_size, "setup: size must match so only mtime disambiguates" with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - created = seed_asset_specs(session, [_spec(path)]) + created, error = seed_asset_specs(session, [_spec(path)]) session.commit() + assert error is None assert created == 1, "a same-size-but-different-mtime restore must take the new-content path" assert session.get(AssetContent, content_id).is_missing is True, ( "a matching size with a mismatched mtime is not proof the old row's bytes are back — " @@ -190,9 +193,10 @@ def test_two_missing_null_hash_candidates_at_same_path_do_not_recover( session.commit() with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - created = seed_asset_specs(session, [_spec(path)]) + created, error = seed_asset_specs(session, [_spec(path)]) session.commit() + assert error is None assert created == 1, "ambiguous candidates must fall through to the normal new-content path" assert session.get(AssetContent, first_id).is_missing is True assert session.get(AssetContent, second_id).is_missing is True diff --git a/tests-unit/assets_test/services/test_recovery_gate.py b/tests-unit/assets_test/services/test_recovery_gate.py index eeca7e0b0ac..349824ab049 100644 --- a/tests-unit/assets_test/services/test_recovery_gate.py +++ b/tests-unit/assets_test/services/test_recovery_gate.py @@ -65,9 +65,10 @@ def test_single_hash_match_recovers(session, temp_dir: Path): content, record = _missing_content(session, path, _stored_hash(path)) with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - created = seed_asset_specs(session, [_spec(path)]) + created, error = seed_asset_specs(session, [_spec(path)]) session.commit() + assert error is None assert created == 0 assert session.get(AssetContent, content.id).is_missing is False assert session.get(AssetTag, {"asset_id": record.id, "tag_name": "missing"}) is None @@ -81,9 +82,10 @@ def test_ambiguous_hash_match_recovers_nothing(session, temp_dir: Path): second, _ = _missing_content(session, path, digest) with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - created = seed_asset_specs(session, [_spec(path)]) + created, error = seed_asset_specs(session, [_spec(path)]) session.commit() + assert error is None assert created == 1 assert session.get(AssetContent, first.id).is_missing is True assert session.get(AssetContent, second.id).is_missing is True @@ -96,9 +98,10 @@ def test_no_hash_match_creates_fresh_rows(session, temp_dir: Path): missing, _ = _missing_content(session, path, "old") with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - created = seed_asset_specs(session, [_spec(path)]) + created, error = seed_asset_specs(session, [_spec(path)]) session.commit() + assert error is None assert created == 1 assert session.get(AssetContent, missing.id).is_missing is True assert len(session.scalars(select(Asset)).all()) == 2 @@ -113,10 +116,11 @@ def test_off_mode_no_recovery(session, temp_dir: Path): patch("app.assets.scanner.mode.hashing_enabled", return_value=False), patch("app.assets.scanner_changes.snapshot_hash") as hash_mock, ): - created = seed_asset_specs(session, [_spec(path)]) + created, error = seed_asset_specs(session, [_spec(path)]) session.commit() hash_mock.assert_not_called() + assert error is None assert created == 1 assert session.get(AssetContent, missing.id).is_missing is True @@ -130,9 +134,10 @@ def test_unstable_hash_requeues(session, temp_dir: Path): patch("app.assets.scanner.mode.hashing_enabled", return_value=True), patch("app.assets.scanner_changes.snapshot_hash", return_value=None), ): - created = seed_asset_specs(session, [_spec(path)]) + created, error = seed_asset_specs(session, [_spec(path)]) session.commit() + assert error is None assert created == 0 assert pending_recovery_count() == 1 assert session.get(AssetContent, missing.id).is_missing is True diff --git a/tests-unit/assets_test/services/test_scan_lifecycle.py b/tests-unit/assets_test/services/test_scan_lifecycle.py index e5a1d280118..07b4b4dcdd2 100644 --- a/tests-unit/assets_test/services/test_scan_lifecycle.py +++ b/tests-unit/assets_test/services/test_scan_lifecycle.py @@ -10,7 +10,10 @@ def _scan(session, root: Path) -> int: paths = [str(path) for path in root.iterdir()] specs, _, _ = build_asset_specs(paths, set(), enable_metadata_extraction=False) - return seed_asset_specs(session, specs) + created, error = seed_asset_specs(session, specs) + if error is not None: + raise error + return created def test_e2e_scan_seed_detect_prune(session, temp_dir: Path): diff --git a/tests-unit/assets_test/services/test_scanner_b.py b/tests-unit/assets_test/services/test_scanner_b.py index 6f98760bf62..27d3a369211 100644 --- a/tests-unit/assets_test/services/test_scanner_b.py +++ b/tests-unit/assets_test/services/test_scanner_b.py @@ -133,12 +133,13 @@ def test_seed_creates_content_and_record(session, temp_dir: Path): (input_root / "second.png").write_bytes(b"second") with patch("folder_paths.get_input_directory", return_value=str(input_root)): - created = seed_asset_specs(session, _build_seed_specs(input_root)) + created, error = seed_asset_specs(session, _build_seed_specs(input_root)) session.commit() contents = list(session.scalars(select(AssetContent).order_by(AssetContent.path))) records = list(session.scalars(select(Asset).order_by(Asset.name))) + assert error is None assert created == 2 assert len(contents) == 2 assert len(records) == 2 diff --git a/tests-unit/assets_test/services/test_scanner_seed_resilience.py b/tests-unit/assets_test/services/test_scanner_seed_resilience.py index 28d53917179..228e5598fa4 100644 --- a/tests-unit/assets_test/services/test_scanner_seed_resilience.py +++ b/tests-unit/assets_test/services/test_scanner_seed_resilience.py @@ -53,9 +53,10 @@ def test_seed_persists_remaining_specs_when_path_vanishes_before_restat( specs, vanished_path = _specs_with_vanished_path(temp_dir) vanished_path.unlink() - created = seed_asset_specs(session, specs) + created, error = seed_asset_specs(session, specs) session.commit() + assert error is None assert created == 2 assert _record_count(session) == 2 @@ -74,9 +75,10 @@ def _hash_or_raise(path: str) -> str | None: monkeypatch.setattr("app.assets.scanner_changes.snapshot_hash", _hash_or_raise) with patch("app.assets.scanner.mode.hashing_enabled", return_value=True): - created = seed_asset_specs(session, specs) + created, error = seed_asset_specs(session, specs) session.commit() + assert error is None assert created == 2 assert _record_count(session) == 2 @@ -168,9 +170,10 @@ def _create_content_or_conflict( _create_content_or_conflict, ) - created = seed_asset_specs(session, specs) + created, error = seed_asset_specs(session, specs) session.commit() + assert error is None assert created == 2 assert _record_count(session) == 2 assert {record.name for record in session.scalars(select(Asset))} == { @@ -204,10 +207,9 @@ def _create_record_or_raise( monkeypatch.setattr("app.assets.scanner.create_record", _create_record_or_raise) - with pytest.raises(IntegrityError) as raised: - seed_asset_specs(session, [_spec(path)]) + _created, error = seed_asset_specs(session, [_spec(path)]) - assert raised.value is unrelated_error + assert error is unrelated_error def test_seed_attempts_remaining_specs_before_propagating_integrity_error( @@ -248,11 +250,10 @@ def _create_record_or_raise( monkeypatch.setattr("app.assets.scanner.create_record", _create_record_or_raise) - with pytest.raises(IntegrityError) as raised: - seed_asset_specs(session, [_spec(path) for path in paths]) + _created, error = seed_asset_specs(session, [_spec(path) for path in paths]) session.commit() - assert raised.value is unrelated_error + assert error is unrelated_error assert attempted == ["first.bin", "broken.bin", "last.bin"] assert {record.name for record in session.scalars(select(Asset))} == { "first.bin", @@ -321,9 +322,10 @@ def test_seed_skips_negative_fresh_mtime_with_warning_and_telemetry( os.utime(paths[1], ns=(pre_epoch_ns, pre_epoch_ns)) with caplog.at_level(logging.INFO): - created = seed_asset_specs(session, [_spec(path) for path in paths]) + created, error = seed_asset_specs(session, [_spec(path) for path in paths]) session.commit() + assert error is None assert created == 2 assert {record.name for record in session.scalars(select(Asset))} == { "first.bin", @@ -351,12 +353,13 @@ def test_seed_persists_fresh_stat_after_spec_was_built( os.utime(path, ns=(fresh_mtime_ns, fresh_mtime_ns)) fresh_stat = path.stat() - created = seed_asset_specs(session, [spec]) + created, error = seed_asset_specs(session, [spec]) session.commit() persisted = session.scalar( select(AssetContent).where(AssetContent.path == str(path)) ) + assert error is None assert created == 1 assert fresh_stat.st_size != spec["size_bytes"] assert fresh_stat.st_mtime_ns != spec["mtime_ns"] @@ -388,8 +391,10 @@ def _raise_record_creation(*_args, **_kwargs): monkeypatch.setattr("app.assets.scanner.create_record", _raise_record_creation) - with pytest.raises(RuntimeError, match="forced record creation failure"): - seed_asset_specs(session, [spec]) + created, error = seed_asset_specs(session, [spec]) session.rollback() + assert created == 0 + assert isinstance(error, RuntimeError) + assert str(error) == "forced record creation failure" assert session.get(AssetContent, retained_content_id) is not None diff --git a/tests-unit/assets_test/test_intended_behaviour.py b/tests-unit/assets_test/test_intended_behaviour.py index 580637c71f2..61c5308adf2 100644 --- a/tests-unit/assets_test/test_intended_behaviour.py +++ b/tests-unit/assets_test/test_intended_behaviour.py @@ -111,7 +111,10 @@ def _scan_pass(session, root: Path) -> int: specs, _tag_pool, _skipped = build_asset_specs( list_files_recursively(str(root)), survivors or set() ) - return seed_asset_specs(session, specs) + created, error = seed_asset_specs(session, specs) + if error is not None: + raise error + return created @contextmanager From c0e5ecc1d42f798e8872d3bd5207abf248d3f225 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 16:41:04 -0700 Subject: [PATCH 080/111] fix(assets): reject duplicate file parts instead of stranding the first upload --- app/assets/api/upload.py | 6 ++++++ .../services/test_upload_temp_cleanup.py | 21 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/app/assets/api/upload.py b/app/assets/api/upload.py index b10c73a1640..f16b2e6aac0 100644 --- a/app/assets/api/upload.py +++ b/app/assets/api/upload.py @@ -96,6 +96,12 @@ async def parse_multipart_upload( ) elif fname == "file": + if file_present: + raise UploadError( + 400, + "UNSUPPORTED_FIELD", + "Multiple 'file' parts are not supported.", + ) file_present = True file_client_name = (field.filename or "").strip() diff --git a/tests-unit/assets_test/services/test_upload_temp_cleanup.py b/tests-unit/assets_test/services/test_upload_temp_cleanup.py index 6f8eb913b33..1d95b9f0ee1 100644 --- a/tests-unit/assets_test/services/test_upload_temp_cleanup.py +++ b/tests-unit/assets_test/services/test_upload_temp_cleanup.py @@ -55,6 +55,27 @@ def _multipart_request(*fields: AsyncMock) -> AsyncMock: return request +@pytest.mark.asyncio +async def test_duplicate_file_parts_are_rejected_without_leaking_temp_uploads( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(folder_paths, "get_temp_directory", lambda: str(tmp_path)) + request = _multipart_request(_file_field(), _file_field()) + + error: UploadError | None = None + try: + await parse_multipart_upload(request, lambda _hash: False) + except UploadError as exc: + error = exc + + remaining = list((tmp_path / "uploads").iterdir()) + assert error is not None, f"duplicate file parts accepted; temp paths remain: {remaining}" + assert error.status == 400 + assert error.code == "UNSUPPORTED_FIELD" + assert error.message == "Multiple 'file' parts are not supported." + assert remaining == [] + + @pytest.mark.asyncio async def test_invalid_utf8_after_file_removes_temp_upload( tmp_path: Path, monkeypatch: pytest.MonkeyPatch From 64d884aef06a1e49493040a73cbfe9674da08b83 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 16:45:19 -0700 Subject: [PATCH 081/111] refactor(assets): reap empty upload directories without importing the API layer --- app/assets/services/ingest.py | 8 +++--- .../services/test_upload_temp_cleanup.py | 26 +++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index c72d32a0c0b..73078b3e71c 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -17,7 +17,6 @@ from sqlalchemy.orm import Session from app.assets import mode -from app.assets.api.upload import delete_temp_file_if_exists from app.assets.database.models import Asset, AssetContent, AssetTag from app.assets.database.queries.records import ( create_content_reporting_insert, @@ -132,10 +131,11 @@ def __init__(self, message: str): def _remove_temp_path(temp_path: str | None) -> None: - if not temp_path or not os.path.exists(temp_path): + if not temp_path: return with contextlib.suppress(OSError): - os.remove(temp_path) + if os.path.exists(temp_path): + os.remove(temp_path) parent = os.path.dirname(temp_path) with contextlib.suppress(OSError): if parent and os.path.isdir(parent): @@ -484,7 +484,7 @@ def upload_from_temp_path( ) _move_temp_to_dest(temp_path, dest_abs) finally: - delete_temp_file_if_exists(temp_path) + _remove_temp_path(temp_path) size_bytes, mtime_ns = verified_stat.st_size, verified_stat.st_mtime_ns with create_session() as session: _reconcile_live_content_at_path( diff --git a/tests-unit/assets_test/services/test_upload_temp_cleanup.py b/tests-unit/assets_test/services/test_upload_temp_cleanup.py index 1d95b9f0ee1..b87248a0ad2 100644 --- a/tests-unit/assets_test/services/test_upload_temp_cleanup.py +++ b/tests-unit/assets_test/services/test_upload_temp_cleanup.py @@ -159,3 +159,29 @@ def test_destination_resolution_failure_removes_temp_upload( assert not temp_path.exists() assert not upload_dir.exists() + + +def test_successful_upload_reaps_empty_per_upload_directory( + mock_create_session, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + temp_root = tmp_path / "temp" + output_root = tmp_path / "output" + monkeypatch.setattr(folder_paths, "get_temp_directory", lambda: str(temp_root)) + monkeypatch.setattr(folder_paths, "get_output_directory", lambda: str(output_root)) + + upload_dir = temp_root / "uploads" / uuid.uuid4().hex + upload_dir.mkdir(parents=True) + temp_path = upload_dir / ".upload.part" + temp_path.write_bytes(b"uploaded bytes") + + result = upload_from_temp_path( + temp_path=str(temp_path), + name="model.safetensors", + tags=["output"], + client_filename="model.safetensors", + ) + + assert result.created_new is True + assert not upload_dir.exists() From e0286a695ec7419584ca4d496f14469b0e2ef08d Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 17:09:16 -0700 Subject: [PATCH 082/111] fix(assets): emit the invalid-mtime event once per scan Every other per-file emit on this scan path is gated -- mark_emitted( "stat_failed:enrich"), "hash_discarded_modified", "hash_failed", "enrich_failed" -- but scanner.invalid_mtime fired per file, so a restored archive or a FAT volume of pre-epoch mtimes put one structured event per file into the stream the closed vocabulary exists to keep parseable. Counted and emitted once, carrying the count. seed_asset_specs receives no _ScanProgress object and neither does insert_asset_specs above it, so routing this through mark_emitted would mean changing both signatures plus the seeder call site; the count form needs neither and the event is now strictly more informative than N identical fieldless lines. The per-file logging.warning is unchanged, and the emit stays inside seed_asset_specs so the static call-site manifest still matches. test_seed_skips_negative_fresh_mtime_with_warning_and_telemetry now pins the full list of invalid_mtime lines to exactly ["... count=1"] instead of asserting one such line exists -- a strictly stronger assertion, and the only change the new field required. --- app/assets/scanner.py | 8 +++- .../services/test_scanner_seed_resilience.py | 37 +++++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index bf6caa73216..4108f564978 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -382,6 +382,10 @@ def seed_asset_specs( ) -> tuple[int, Exception | None]: created = 0 first_error: Exception | None = None + # Counted, not gated through _ScanProgress.mark_emitted like its neighbours, because this + # function takes no progress object. Ungated, a restored archive of pre-epoch mtimes puts + # one event per file into the closed-vocabulary stream. + invalid_mtimes = 0 for spec in specs: path = os.path.abspath(spec["abs_path"]) try: @@ -395,7 +399,7 @@ def seed_asset_specs( logging.warning( "Skipping asset with invalid mtime during scan: %s", path ) - emit("scanner.invalid_mtime") + invalid_mtimes += 1 continue try: recovery = recover_missing_content( @@ -442,6 +446,8 @@ def seed_asset_specs( except Exception as error: if first_error is None: first_error = error + if invalid_mtimes: + emit("scanner.invalid_mtime", count=invalid_mtimes) return created, first_error diff --git a/tests-unit/assets_test/services/test_scanner_seed_resilience.py b/tests-unit/assets_test/services/test_scanner_seed_resilience.py index 228e5598fa4..40ed3f2379c 100644 --- a/tests-unit/assets_test/services/test_scanner_seed_resilience.py +++ b/tests-unit/assets_test/services/test_scanner_seed_resilience.py @@ -335,10 +335,41 @@ def test_seed_skips_negative_fresh_mtime_with_warning_and_telemetry( record.getMessage() == f"Skipping asset with invalid mtime during scan: {paths[1]}" for record in caplog.records ) - assert any( - record.getMessage() == "[assets-event] scanner.invalid_mtime" + assert [ + record.getMessage() for record in caplog.records - ) + if record.getMessage().startswith("[assets-event] scanner.invalid_mtime") + ] == ["[assets-event] scanner.invalid_mtime count=1"] + + +def test_seed_emits_one_invalid_mtime_event_for_a_whole_batch_of_pre_epoch_files( + session: Session, + temp_dir: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + paths = [temp_dir / f"restored-{index}.bin" for index in range(5)] + pre_epoch_ns = -315_547_200_000_000_000 + for path in paths: + _ = path.write_bytes(path.name.encode()) + os.utime(path, ns=(pre_epoch_ns, pre_epoch_ns)) + + with caplog.at_level(logging.INFO): + created, error = seed_asset_specs(session, [_spec(path) for path in paths]) + session.commit() + + assert error is None + assert created == 0 + assert _record_count(session) == 0 + assert [ + record.getMessage() + for record in caplog.records + if record.getMessage().startswith("[assets-event] scanner.invalid_mtime") + ] == ["[assets-event] scanner.invalid_mtime count=5"] + assert [ + record.getMessage() + for record in caplog.records + if record.getMessage().startswith("Skipping asset with invalid mtime") + ] == [f"Skipping asset with invalid mtime during scan: {path}" for path in paths] def test_seed_persists_fresh_stat_after_spec_was_built( From e9af384a9e260441e25a003ce6957124598cc264 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 17:11:49 -0700 Subject: [PATCH 083/111] fix(assets): keep spec construction failures from wedging the watch list get_name_and_tags_from_asset_path raises ValueError by contract when a path stops resolving to a configured root, and it sat outside the guard, as did compute_loader_path and mimetypes.guess_type. An escape skipped the _WATCH_LIST[:] = remaining write at the end, so drained entries stayed on the list and were re-attempted every tick while entries past the fault never reached the increment _WATCH_SCAN_RETRIES needs to retire them. The list wedged permanently. Spec construction is now inside a guard that drops just the offending entry, and the list write moved into a finally so no future escape can skip it. The loop walks an iterator rather than the list, so the finally can put back the entries it never reached instead of discarding them. New event name rather than reusing one: scanner.watch_seed_failed is emitted only when seed_asset_specs returns an error, and widening it to also mean "never got as far as seeding" would make it lie -- a consumer treating it as a database-health signal would get false positives from what is really a path layout problem. scanner.watch_spec_failed is registered in ALLOWED_EVENTS and in the static call-site manifest. --- app/assets/event_log.py | 1 + app/assets/scanner_admission.py | 77 +++++++++++-------- .../services/test_admission_gate.py | 75 ++++++++++++++++++ tests-unit/test_assets_event_log_static.py | 3 + 4 files changed, 124 insertions(+), 32 deletions(-) diff --git a/app/assets/event_log.py b/app/assets/event_log.py index e2a2a4fe7fb..c6ef2796ffe 100644 --- a/app/assets/event_log.py +++ b/app/assets/event_log.py @@ -45,6 +45,7 @@ "scanner.stat_failed", "scanner.invalid_mtime", "scanner.watch_stat_failed", + "scanner.watch_spec_failed", "scanner.watch_seed_failed", }) diff --git a/app/assets/scanner_admission.py b/app/assets/scanner_admission.py index c201034daca..41a485dd533 100644 --- a/app/assets/scanner_admission.py +++ b/app/assets/scanner_admission.py @@ -72,35 +72,48 @@ def tick_watch_list(session: Session) -> None: from app.assets.scanner import seed_asset_specs, SeedAssetSpec remaining: list[_WatchEntry] = [] - for entry in _WATCH_LIST: - try: - current = os.stat(entry.path) - except OSError as exc: - logging.warning("Dropping watched asset after stat failed: %s", entry.path) - emit("scanner.watch_stat_failed", error_type=error_type(exc)) - continue - if (current.st_mtime_ns, current.st_size) == (entry.last_stat.st_mtime_ns, entry.last_stat.st_size): - name, tags = get_name_and_tags_from_asset_path(entry.path) - spec: SeedAssetSpec = { - "abs_path": entry.path, - "size_bytes": current.st_size, - "mtime_ns": current.st_mtime_ns, - "info_name": name, - "tags": tags, - "fname": compute_loader_path(entry.path), - "metadata": None, - "mime_type": mimetypes.guess_type(entry.path, strict=False)[0], - "job_id": None, - } - _created, seed_error = seed_asset_specs(session, [spec]) - if seed_error is not None: - logging.warning( - "Dropping watched asset after seeding failed: %s", entry.path - ) - emit("scanner.watch_seed_failed", error_type=error_type(seed_error)) - continue - entry.last_stat = current - entry.ticks += 1 - if entry.ticks < _WATCH_SCAN_RETRIES: - remaining.append(entry) - _WATCH_LIST[:] = remaining + unvisited = iter(list(_WATCH_LIST)) + try: + for entry in unvisited: + try: + current = os.stat(entry.path) + except OSError as exc: + logging.warning("Dropping watched asset after stat failed: %s", entry.path) + emit("scanner.watch_stat_failed", error_type=error_type(exc)) + continue + if (current.st_mtime_ns, current.st_size) == (entry.last_stat.st_mtime_ns, entry.last_stat.st_size): + try: + name, tags = get_name_and_tags_from_asset_path(entry.path) + spec: SeedAssetSpec = { + "abs_path": entry.path, + "size_bytes": current.st_size, + "mtime_ns": current.st_mtime_ns, + "info_name": name, + "tags": tags, + "fname": compute_loader_path(entry.path), + "metadata": None, + "mime_type": mimetypes.guess_type(entry.path, strict=False)[0], + "job_id": None, + } + except Exception as exc: + logging.warning( + "Dropping watched asset after spec construction failed: %s", entry.path + ) + emit("scanner.watch_spec_failed", error_type=error_type(exc)) + continue + _created, seed_error = seed_asset_specs(session, [spec]) + if seed_error is not None: + logging.warning( + "Dropping watched asset after seeding failed: %s", entry.path + ) + emit("scanner.watch_seed_failed", error_type=error_type(seed_error)) + continue + entry.last_stat = current + entry.ticks += 1 + if entry.ticks < _WATCH_SCAN_RETRIES: + remaining.append(entry) + finally: + # Skipping this write wedges the list: drained entries stay on it and are re-attempted + # every tick, while entries past the fault never reach the increment _WATCH_SCAN_RETRIES + # needs to retire them. Draining the iterator keeps entries the loop never reached. + _WATCH_LIST[:] = remaining + list(unvisited) diff --git a/tests-unit/assets_test/services/test_admission_gate.py b/tests-unit/assets_test/services/test_admission_gate.py index 288027a3f94..102fccc7ece 100644 --- a/tests-unit/assets_test/services/test_admission_gate.py +++ b/tests-unit/assets_test/services/test_admission_gate.py @@ -177,6 +177,81 @@ def seed_or_return_error(_session, specs) -> tuple[int, Exception | None]: ) +def test_spec_construction_failure_drops_the_entry_without_wedging_the_watch_list( + session, + temp_dir: Path, + monkeypatch, + caplog: pytest.LogCaptureFixture, +) -> None: + changing_path = temp_dir / "changing.bin" + unresolvable_path = temp_dir / "unresolvable.bin" + stable_path = temp_dir / "stable.bin" + for path in (changing_path, unresolvable_path, stable_path): + path.write_bytes(path.name.encode()) + changing_stat = changing_path.stat() + changing_path.write_bytes(b"still-downloading-and-now-longer") + _WATCH_LIST[:] = [ + _WatchEntry(str(changing_path), changing_stat), + _WatchEntry(str(unresolvable_path), unresolvable_path.stat()), + _WatchEntry(str(stable_path), stable_path.stat()), + ] + resolve_name_and_tags = scanner_admission.get_name_and_tags_from_asset_path + + def _name_and_tags(path: str) -> tuple[str, list[str]]: + if path == str(unresolvable_path): + raise ValueError( + "Path is not within input, output, temp, or configured model bases: " + f"{path}" + ) + return resolve_name_and_tags(path) + + monkeypatch.setattr("folder_paths.get_input_directory", lambda: str(temp_dir)) + monkeypatch.setattr( + scanner_admission, "get_name_and_tags_from_asset_path", _name_and_tags + ) + + with caplog.at_level(logging.INFO): + tick_watch_list(session) + session.commit() + + assert set(session.scalars(select(AssetContent.path)).all()) == {str(stable_path)} + assert [(entry.path, entry.ticks) for entry in _WATCH_LIST] == [ + (str(changing_path), 1) + ] + assert any( + record.getMessage() + == f"Dropping watched asset after spec construction failed: {unresolvable_path}" + for record in caplog.records + ) + assert any( + record.getMessage() + == "[assets-event] scanner.watch_spec_failed error_type=ValueError" + for record in caplog.records + ) + + +def test_unexpected_fault_mid_drain_leaves_unvisited_entries_on_the_watch_list( + session, temp_dir: Path, monkeypatch +) -> None: + paths = [temp_dir / name for name in ("first.bin", "exploding.bin", "untouched.bin")] + for path in paths: + path.write_bytes(path.name.encode()) + _WATCH_LIST[:] = [_WatchEntry(str(path), path.stat()) for path in paths] + + def seed_or_explode(_session, specs) -> tuple[int, Exception | None]: + if specs[0]["abs_path"] == str(paths[1]): + raise MemoryError("forced unrecoverable fault") + return 1, None + + monkeypatch.setattr("folder_paths.get_input_directory", lambda: str(temp_dir)) + monkeypatch.setattr("app.assets.scanner.seed_asset_specs", seed_or_explode) + + with pytest.raises(MemoryError, match="^forced unrecoverable fault$"): + tick_watch_list(session) + + assert [entry.path for entry in _WATCH_LIST] == [str(paths[2])] + + def test_stable_scan_admission_removes_watch_entry_before_next_tick(session, temp_dir: Path, monkeypatch): path = temp_dir / "stable.bin" path.write_bytes(b"complete") diff --git a/tests-unit/test_assets_event_log_static.py b/tests-unit/test_assets_event_log_static.py index 69632eaee67..f2315bc8b73 100644 --- a/tests-unit/test_assets_event_log_static.py +++ b/tests-unit/test_assets_event_log_static.py @@ -67,6 +67,9 @@ class CallSite(NamedTuple): CallSite( "app/assets/scanner_admission.py", "tick_watch_list", "scanner.watch_stat_failed" ), + CallSite( + "app/assets/scanner_admission.py", "tick_watch_list", "scanner.watch_spec_failed" + ), CallSite( "app/assets/scanner_admission.py", "tick_watch_list", "scanner.watch_seed_failed" ), From f8b47bbcb42a032547f62141d0807ddad1dc6877 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 17:12:50 -0700 Subject: [PATCH 084/111] refactor(assets): export the live-path conflict check as public API scanner.py reached past the package's own re-export surface to import _is_live_path_conflict directly out of records.py. The underscore said module-private while the import said otherwise, and records.py deliberately publishes its public names through app.assets.database.queries -- which the same import block three lines above was already using. The use is correct and unchanged; only the name and the route change. Renamed to is_live_path_conflict, listed in the package __init__ import and __all__ alongside its siblings, and scanner.py now takes it from the package like everything else it imports from there. --- app/assets/database/queries/__init__.py | 2 ++ app/assets/database/queries/records.py | 4 ++-- app/assets/scanner.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/app/assets/database/queries/__init__.py b/app/assets/database/queries/__init__.py index 8707c229b6f..14dff90a879 100644 --- a/app/assets/database/queries/__init__.py +++ b/app/assets/database/queries/__init__.py @@ -13,6 +13,7 @@ delete_record, fetch_record_tags, get_record_by_id, + is_live_path_conflict, list_records_page, mark_content_missing, rename_record, @@ -27,6 +28,7 @@ "delete_record", "fetch_record_tags", "get_record_by_id", + "is_live_path_conflict", "list_records_page", "mark_content_missing", "rename_record", diff --git a/app/assets/database/queries/records.py b/app/assets/database/queries/records.py index 96032bb7226..60b7f8231b0 100644 --- a/app/assets/database/queries/records.py +++ b/app/assets/database/queries/records.py @@ -47,7 +47,7 @@ class RecordPageSpec(NamedTuple): _LIVE_PATH_UNIQUE_INDEX = "uq_asset_contents_path_live" -def _is_live_path_conflict(error: IntegrityError) -> bool: +def is_live_path_conflict(error: IntegrityError) -> bool: orig = error.orig message = str(orig) postgres_names_the_index = getattr(getattr(orig, "diag", None), "constraint_name", None) == _LIVE_PATH_UNIQUE_INDEX @@ -66,7 +66,7 @@ def create_content_reporting_insert(session: Session, path: str, hash: str | Non session.flush() return content, True except IntegrityError as error: - if not _is_live_path_conflict(error): + if not is_live_path_conflict(error): raise winner = session.execute(sa.select(AssetContent).where(AssetContent.path == path, AssetContent.is_missing.is_(False))).scalar_one() return winner, False diff --git a/app/assets/scanner.py b/app/assets/scanner.py index 4108f564978..e863569f045 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -22,11 +22,11 @@ from app.assets.event_log import emit, error_type from app.assets.database.queries import ( create_content_reporting_insert, + is_live_path_conflict, mark_content_missing, create_record, ) from app.assets.database.models import Asset, AssetContent -from app.assets.database.queries.records import _is_live_path_conflict from app.assets.helpers import sql_path_under_prefix, to_stored_hash from app.assets.lifecycle import get_excluded_scan_roots from app.assets.scanner_changes import ( @@ -436,7 +436,7 @@ def seed_asset_specs( ) created += 1 except IntegrityError as error: - if _is_live_path_conflict(error): + if is_live_path_conflict(error): logging.warning( "Skipping asset whose row conflicts during scan: %s", path ) From 195045d2576df66ac82a7391bd8f27f4345ede4e Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 17:14:42 -0700 Subject: [PATCH 085/111] docs(db): restore the rationale for locking before migration Commit 1dbcdcd7 and the comment-cleanup pass 8205022f reduced this to "All database reads and writes, including the legacy import, run under the lock", dropping the part that did the work: upstream master locks after migrating and justifies it with "Alembic uses its own connection, so we must wait until it's done before locking -- otherwise our own lock blocks the migration". That is false, the lock is on a separate .lock file, and the surviving sentence said nothing to stop a contributor "fixing" the ordering back. Restored and adapted rather than pasted: the legacy copy and the db_exists probe now happen inside the lock, which the original text predates, so both are named in the list of things the ordering makes mutually exclusive. --- app/database/db.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/database/db.py b/app/database/db.py index ca18b97c0a2..cd684e3375a 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -182,7 +182,11 @@ def _init_file_db(db_url): db_path = get_db_path() prepare_file_db_path(db_path) - # All database reads and writes, including the legacy import, run under the lock. + # Lock BEFORE any of the work below — deliberately diverging from upstream master, whose + # "it would block Alembic" rationale is false (the lock guards a separate `.lock` file, + # not the database Alembic connects to). Only this order makes the legacy import, the + # existence probe deciding whether a backup is taken, revision inspection, backup, upgrade + # and the failure-path restore mutually exclusive between processes. _acquire_file_lock(db_path) try: copy_legacy_default_db(db_path) From 6b1deb50024e8c57c6ff766caca6f92bbb5e1543 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 20:20:19 -0700 Subject: [PATCH 086/111] fix(assets): stop a scan on memory exhaustion instead of deferring it MemoryError is an Exception, so the per-spec and per-batch handlers stored it alongside ordinary faults and carried on - allocating for every remaining spec and then every remaining batch while the process was already out of memory. Both handlers now let it through, and the scan records a failure and stops. --- app/assets/scanner.py | 4 +++ app/assets/seeder.py | 4 +++ .../services/test_scanner_seed_resilience.py | 31 +++++++++++++++++++ tests-unit/seeder_test/test_seeder.py | 16 ++++++++++ 4 files changed, 55 insertions(+) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index e863569f045..55a0e5dfb30 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -443,6 +443,10 @@ def seed_asset_specs( continue if first_error is None: first_error = error + except MemoryError: + # Deferring this one would keep allocating for every remaining spec + # while the process is already out of memory. + raise except Exception as error: if first_error is None: first_error = error diff --git a/app/assets/seeder.py b/app/assets/seeder.py index 5cc4837ec9b..f4977377d6f 100644 --- a/app/assets/seeder.py +++ b/app/assets/seeder.py @@ -862,6 +862,10 @@ def _run_fast_phase(self, roots: tuple[RootType, ...]) -> tuple[int, int, int]: total_created += created if batch_error is not None: raise batch_error + except MemoryError: + # Recording this as a batch failure would march the scan through + # every remaining batch while the process is out of memory. + raise except Exception as e: self._add_error( f"Batch insert encountered an error at offset {i} " diff --git a/tests-unit/assets_test/services/test_scanner_seed_resilience.py b/tests-unit/assets_test/services/test_scanner_seed_resilience.py index 40ed3f2379c..bf3ac14cd17 100644 --- a/tests-unit/assets_test/services/test_scanner_seed_resilience.py +++ b/tests-unit/assets_test/services/test_scanner_seed_resilience.py @@ -212,6 +212,37 @@ def _create_record_or_raise( assert error is unrelated_error +def test_seed_raises_memory_error_instead_of_attempting_later_specs( + session: Session, temp_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + paths = [] + for name in ("first.bin", "second.bin"): + path = temp_dir / name + path.write_bytes(b"content") + paths.append(path) + attempted: list[str] = [] + + def _create_record_or_exhaust( + session_arg: Session, + *, + content_id: str, + name: str, + mime_type: str | None, + job_id: str | None, + loader_path: str | None, + tags: list[str], + ) -> Asset: + attempted.append(name) + raise MemoryError("out of memory") + + monkeypatch.setattr("app.assets.scanner.create_record", _create_record_or_exhaust) + + with pytest.raises(MemoryError): + seed_asset_specs(session, [_spec(path) for path in paths]) + + assert attempted == ["first.bin"] + + def test_seed_attempts_remaining_specs_before_propagating_integrity_error( session: Session, temp_dir: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.py index a860e8f5671..913540c8a44 100644 --- a/tests-unit/seeder_test/test_seeder.py +++ b/tests-unit/seeder_test/test_seeder.py @@ -639,6 +639,22 @@ def test_batch_insert_fault_reports_the_specs_committed_before_it( ] +def test_batch_memory_error_stops_the_scan_instead_of_continuing( + scan_seeder: _AssetSeeder, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + with pytest.raises(MemoryError): + _run_faulting_fast_phase( + scan_seeder, + monkeypatch, + tmp_path, + MemoryError("out of memory"), + ) + + assert scan_seeder._errors == [] + + def test_salvage_commit_failure_reports_the_original_batch_fault( scan_seeder: _AssetSeeder, monkeypatch: pytest.MonkeyPatch, From f88d266f3d3c459f0d484fb147143b12cf4f6170 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 16 Sep 2026 20:20:26 -0700 Subject: [PATCH 087/111] docs(assets): document the prune failure response and its None result The route gained a 500 PRUNE_FAILED branch and the seeder method gained a None return, both so a prune that did not run cannot be reported as a clean one. Neither contract was written down. --- app/assets/api/routes.py | 2 ++ app/assets/seeder.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/assets/api/routes.py b/app/assets/api/routes.py index cc512c4dec4..b6fbfe13177 100644 --- a/app/assets/api/routes.py +++ b/app/assets/api/routes.py @@ -1126,6 +1126,8 @@ async def mark_missing_assets(request: web.Request) -> web.Response: Returns: 200 OK with count of marked assets 409 Conflict if a scan is currently running + 500 Internal Server Error with PRUNE_FAILED if the marking failed, so a + prune that did not run is never reported as a completed one """ try: marked = asset_seeder.mark_missing_outside_prefixes() diff --git a/app/assets/seeder.py b/app/assets/seeder.py index f4977377d6f..1b63ca76c9f 100644 --- a/app/assets/seeder.py +++ b/app/assets/seeder.py @@ -455,7 +455,10 @@ def mark_missing_outside_prefixes(self) -> int | None: a full scan of all roots or during maintenance. Returns: - Number of references marked as missing + Number of references marked as missing, or None when the marking + itself failed. Zero and None are deliberately distinct: zero means + nothing was outside the known prefixes, None means the answer is + unknown, so callers must not report a failed prune as a clean one. Raises: ScanInProgressError: If a scan is currently running From f6a464650ee421eb35f0b651f81dc41907cc5860 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 15:10:20 -0700 Subject: [PATCH 088/111] test(assets): assert the surviving spec count after a propagated fault --- .../assets_test/services/test_scanner_seed_resilience.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests-unit/assets_test/services/test_scanner_seed_resilience.py b/tests-unit/assets_test/services/test_scanner_seed_resilience.py index bf3ac14cd17..e64d7c08418 100644 --- a/tests-unit/assets_test/services/test_scanner_seed_resilience.py +++ b/tests-unit/assets_test/services/test_scanner_seed_resilience.py @@ -281,10 +281,11 @@ def _create_record_or_raise( monkeypatch.setattr("app.assets.scanner.create_record", _create_record_or_raise) - _created, error = seed_asset_specs(session, [_spec(path) for path in paths]) + created, error = seed_asset_specs(session, [_spec(path) for path in paths]) session.commit() assert error is unrelated_error + assert created == 2 assert attempted == ["first.bin", "broken.bin", "last.bin"] assert {record.name for record in session.scalars(select(Asset))} == { "first.bin", From 4f3715591469fe7579ee67f69d31bb9cbef010fa Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 15:10:29 -0700 Subject: [PATCH 089/111] docs(db): shorten the lock-ordering comment while keeping its rationale --- app/database/db.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/app/database/db.py b/app/database/db.py index cd684e3375a..deb9ed01712 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -182,11 +182,9 @@ def _init_file_db(db_url): db_path = get_db_path() prepare_file_db_path(db_path) - # Lock BEFORE any of the work below — deliberately diverging from upstream master, whose - # "it would block Alembic" rationale is false (the lock guards a separate `.lock` file, - # not the database Alembic connects to). Only this order makes the legacy import, the - # existence probe deciding whether a backup is taken, revision inspection, backup, upgrade - # and the failure-path restore mutually exclusive between processes. + # Lock before legacy import, migration inspection, backup, upgrade, and failure recovery. + # The separate `.lock` file does not block Alembic; this ordering keeps the sequence + # process-exclusive. _acquire_file_lock(db_path) try: copy_legacy_default_db(db_path) From c0214990a00e79b81d6d976820c8b2b59a78e71f Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 15:51:38 -0700 Subject: [PATCH 090/111] docs(tests): drop the cross-module justification from the import-order comment --- tests-unit/app_test/test_prompt_worker.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests-unit/app_test/test_prompt_worker.py b/tests-unit/app_test/test_prompt_worker.py index f38d9ec90d6..efd4bafeca0 100644 --- a/tests-unit/app_test/test_prompt_worker.py +++ b/tests-unit/app_test/test_prompt_worker.py @@ -4,8 +4,7 @@ from comfy.cli_args import args # Must precede the import: comfy.model_management picks its device at import time, and a CUDA -# build with no driver raises there. Same guard, same placement, as every other test module -# that imports it. +# build with no driver raises there. if not torch.cuda.is_available(): args.cpu = True From 8db7f916b729a224421c922daccbae38c6885d73 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 20:19:47 -0700 Subject: [PATCH 091/111] test(assets): restore the cpu flag after the guarded prompt worker import --- tests-unit/app_test/test_prompt_worker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests-unit/app_test/test_prompt_worker.py b/tests-unit/app_test/test_prompt_worker.py index efd4bafeca0..85732d33396 100644 --- a/tests-unit/app_test/test_prompt_worker.py +++ b/tests-unit/app_test/test_prompt_worker.py @@ -5,11 +5,13 @@ # Must precede the import: comfy.model_management picks its device at import time, and a CUDA # build with no driver raises there. +_original_cpu = args.cpu if not torch.cuda.is_available(): args.cpu = True import app.prompt_worker as prompt_worker_module # noqa: E402 +args.cpu = _original_cpu class LoopEscape(Exception): pass From ca8aba7ab0a2fe5b936050cd32489726922b1c38 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 20:47:32 -0700 Subject: [PATCH 092/111] test(assets): restore the cpu flag even when the prompt worker import fails --- tests-unit/app_test/test_prompt_worker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests-unit/app_test/test_prompt_worker.py b/tests-unit/app_test/test_prompt_worker.py index 85732d33396..1cc79c137ee 100644 --- a/tests-unit/app_test/test_prompt_worker.py +++ b/tests-unit/app_test/test_prompt_worker.py @@ -8,10 +8,10 @@ _original_cpu = args.cpu if not torch.cuda.is_available(): args.cpu = True - -import app.prompt_worker as prompt_worker_module # noqa: E402 - -args.cpu = _original_cpu +try: + import app.prompt_worker as prompt_worker_module +finally: + args.cpu = _original_cpu class LoopEscape(Exception): pass From c1376aa2780e90b4365b3a87602f78162980bcd5 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 20:49:54 -0700 Subject: [PATCH 093/111] Correct the batch-size rationale in the write-transaction docs --- app/assets/docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/docs/README.md b/app/assets/docs/README.md index 0b693523013..05388b4d30b 100644 --- a/app/assets/docs/README.md +++ b/app/assets/docs/README.md @@ -331,7 +331,7 @@ Every database write goes through `run_write_txn`, which opens a dedicated write Background writes are bounded: no scanner transaction covers more than `MAX_WRITE_BATCH` rows, and the scanner checks for pause and cancellation between batches, so a pause takes effect one batch later instead of at the end of the phase. Seeding, reference reconciliation, pruning, and enrichment all chunk their work to that size; the verification and hash-transition drains already write one entry per transaction. -The batch size is 25. A transaction holds the write lock for its whole duration, so the bound is exactly the stall an urgent writer inherits when it arrives mid-batch, and it is also the rollback blast radius when one row in a batch fails. 25 sits just past the point where a larger batch stops buying throughput: measured lock-held medians are 20ms for a 25-row seed batch and 27ms for a 25-row enrichment batch, against a 0.03ms empty-transaction floor, roughly 1ms per row, and 3.8ms for the single-row insert an urgent writer is typically waiting to make. +The batch size is 25. A transaction holds the write lock for its whole duration, so the bound is exactly the stall an urgent writer inherits when it arrives mid-batch, and it is also the rollback blast radius when one row in a batch fails. This is a latency bound rather than a throughput optimisation: measured lock-held medians are 20ms for a 25-row seed batch and 27ms for a 25-row enrichment batch, against 3.8ms for the single-row insert an urgent writer is typically waiting to make. Chunking costs aggregate throughput, because each commit pays a fixed cost and twenty small transactions therefore hold the lock longer in total than one large one. That trade is deliberate: the scanner yields between batches, so the total is spread across gaps an urgent writer can take, where a single large batch holds the lock in one block it cannot. Write callables never touch the filesystem. Stat, hashing, and metadata extraction run in a preflight outside the transaction, and the closure receives the facts that preflight observed; inside the lock it only reads and writes rows. Two invariants make that safe: From 2b24381b6bdbbeb52398671eaf520258dcd5c421 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 21:14:43 -0700 Subject: [PATCH 094/111] Base the batch-size rationale on measured commit cost --- app/assets/docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/docs/README.md b/app/assets/docs/README.md index 05388b4d30b..da77288ce0c 100644 --- a/app/assets/docs/README.md +++ b/app/assets/docs/README.md @@ -331,7 +331,7 @@ Every database write goes through `run_write_txn`, which opens a dedicated write Background writes are bounded: no scanner transaction covers more than `MAX_WRITE_BATCH` rows, and the scanner checks for pause and cancellation between batches, so a pause takes effect one batch later instead of at the end of the phase. Seeding, reference reconciliation, pruning, and enrichment all chunk their work to that size; the verification and hash-transition drains already write one entry per transaction. -The batch size is 25. A transaction holds the write lock for its whole duration, so the bound is exactly the stall an urgent writer inherits when it arrives mid-batch, and it is also the rollback blast radius when one row in a batch fails. This is a latency bound rather than a throughput optimisation: measured lock-held medians are 20ms for a 25-row seed batch and 27ms for a 25-row enrichment batch, against 3.8ms for the single-row insert an urgent writer is typically waiting to make. Chunking costs aggregate throughput, because each commit pays a fixed cost and twenty small transactions therefore hold the lock longer in total than one large one. That trade is deliberate: the scanner yields between batches, so the total is spread across gaps an urgent writer can take, where a single large batch holds the lock in one block it cannot. +The batch size is 25. A transaction holds the write lock for its whole duration, so the batch size sets both the stall an urgent writer inherits when it arrives mid-batch and the rollback blast radius when one row in a batch fails. A commit costs roughly 20ms fixed plus roughly 0.04ms per row, so the batch size trades worst-case hold against total lock occupancy: across a 10,000-file scan, 25-row batches hold the lock about 21ms at a time over 400 commits, where 500-row batches hold it about 39ms over 20. Both sit three orders of magnitude below the point at which a competing writer gives up, so this constant is not what keeps the system correct — moving filesystem work out of the transaction is. 25 is kept for the shorter worst-case hold. Write callables never touch the filesystem. Stat, hashing, and metadata extraction run in a preflight outside the transaction, and the closure receives the facts that preflight observed; inside the lock it only reads and writes rows. Two invariants make that safe: From 98195ca50d08870712d3722defa38f8c91f77689 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 17 Sep 2026 22:19:37 -0700 Subject: [PATCH 095/111] Disable assets instead of exiting when the database fails to initialize --- app/assets/api/routes.py | 6 ++ app/assets/event_log.py | 2 + app/assets/manager.py | 16 +++++- app/database/db.py | 6 +- main.py | 31 +++++----- tests-unit/app_test/test_db_init_locking.py | 56 ++++++++++++++++--- .../assets_test/test_event_log_sites.py | 31 ++++++++++ tests-unit/test_assets_event_log_static.py | 1 + 8 files changed, 122 insertions(+), 27 deletions(-) diff --git a/app/assets/api/routes.py b/app/assets/api/routes.py index f5ed98be5d8..23be921fd82 100644 --- a/app/assets/api/routes.py +++ b/app/assets/api/routes.py @@ -133,6 +133,12 @@ def register_assets_routes( app.add_routes(ROUTES) +def close_assets_feature_gate() -> None: + """Answer 503 from routes that are already registered.""" + global _ASSETS_ENABLED + _ASSETS_ENABLED = False + + def _build_error_response( status: int, code: str, message: str, details: dict | None = None ) -> web.Response: diff --git a/app/assets/event_log.py b/app/assets/event_log.py index 2c8d12feabf..63f77fe8d5e 100644 --- a/app/assets/event_log.py +++ b/app/assets/event_log.py @@ -30,6 +30,7 @@ STAT_SITES = frozenset({"discovery", "enrich", "reference_stat"}) ALLOWED_EVENTS = frozenset({ "assets.enabled", + "assets.disabled", "seeder.scan_started", "seeder.scan_completed", "seeder.scan_failed", @@ -48,6 +49,7 @@ WARNING_LEVEL_EVENTS = frozenset({ "ingest.register_failed", + "assets.disabled", }) diff --git a/app/assets/manager.py b/app/assets/manager.py index a6e8eeab0f9..319ded8a5b6 100644 --- a/app/assets/manager.py +++ b/app/assets/manager.py @@ -4,7 +4,8 @@ from aiohttp import web from app.assets import mode -from app.assets.api.routes import register_assets_routes +from app.assets.api.routes import close_assets_feature_gate, register_assets_routes +from app.assets.event_log import emit, error_type from app.assets.lifecycle import record_hash_mode_transition_intent, run_shutdown, run_startup from app.assets.seeder import ScanPhase, asset_seeder from app.assets.services.ingest import ( @@ -25,6 +26,8 @@ def enabled(self) -> bool: ... def startup(self) -> None: ... + def disable(self, exc: BaseException) -> None: ... + def shutdown(self) -> None: ... def register_routes( @@ -83,6 +86,9 @@ def startup(self) -> None: run_startup(enable_assets=False) record_hash_mode_transition_intent() + def disable(self, exc: BaseException) -> None: + return + def shutdown(self) -> None: _shutdown_assets() @@ -132,16 +138,22 @@ def set_event_sink(self, sink: Callable[[str, dict[str, Any]], None] | None) -> class AssetsEnabled: def __init__(self, args: _ArgsLike) -> None: self._args = args + self._disabled = False @property def enabled(self) -> bool: - return True + return not self._disabled def startup(self) -> None: mode.init(self._args) record_hash_mode_transition_intent() run_startup(enable_assets=True) + def disable(self, exc: BaseException) -> None: + self._disabled = True + close_assets_feature_gate() + emit("assets.disabled", error_type=error_type(exc)) + def shutdown(self) -> None: _shutdown_assets() diff --git a/app/database/db.py b/app/database/db.py index 78386019931..6efe0ffdfc6 100644 --- a/app/database/db.py +++ b/app/database/db.py @@ -47,6 +47,10 @@ T = TypeVar("T") +class WalUnavailableError(RuntimeError): + """WAL journal mode was refused, which SQLite does on network filesystems.""" + + try: from alembic import command from alembic.config import Config @@ -273,7 +277,7 @@ def _configure_runtime_connection(dbapi_connection, db_path): cursor.execute("PRAGMA foreign_keys=ON") journal_mode = cursor.execute("PRAGMA journal_mode=WAL").fetchone()[0] if journal_mode.lower() != "wal": - raise RuntimeError( + raise WalUnavailableError( f"SQLite WAL could not be enabled for database '{db_path}'. " "SQLite WAL is not supported on network filesystems." ) diff --git a/main.py b/main.py index 3c6e41d353a..05acbf3b834 100644 --- a/main.py +++ b/main.py @@ -22,7 +22,7 @@ file_log_outputs = get_file_log_outputs(args.verbose) setup_logger(log_level=console_log_level, file_outputs=file_log_outputs, use_stdout=args.log_stdout) -from app.database.db import dependencies_available, init_db +from app.database.db import WalUnavailableError, dependencies_available, init_db from app.assets.lifecycle import cleanup_temp_filesystem from app.assets.manager import AssetManager, default_asset_manager import itertools @@ -457,33 +457,32 @@ def setup_database(asset_manager): init_db() asset_manager.startup() except Exception as e: - if "database is locked" in str(e): + if isinstance(e, WalUnavailableError): logging.error( - "Database is locked. Another ComfyUI process is already using this database.\n" - "To resolve this, specify a separate database file for this instance:\n" - " --database-url sqlite:///path/to/another.db" + f"{e}\n" + "To resolve this, keep the database on a local disk; your media can stay " + "on the network share:\n" + " --database-url sqlite:///path/on/local/disk.db" ) - sys.exit(1) - if "Could not acquire lock on database" in str(e): + elif "database is locked" in str(e) or "Could not acquire lock on database" in str(e): logging.error( "Database is locked. Another ComfyUI process is already using this database.\n" "To resolve this, specify a separate database file for this instance:\n" " --database-url sqlite:///path/to/another.db" ) - if args.enable_assets: - sys.exit(1) - return - if args.enable_assets: + elif args.enable_assets: logging.error( f"Failed to initialize database: {e}\n" - "The --enable-assets flag requires a working database connection.\n" "To resolve this, try one of the following:\n" " 1. Install the latest requirements: pip install -r requirements.txt\n" - " 2. Specify an alternative database URL: --database-url sqlite:///path/to/your.db\n" - " 3. Use an in-memory database: --database-url sqlite:///:memory:" + " 2. Specify a database on a local disk: --database-url sqlite:///path/to/your.db" ) - sys.exit(1) - logging.error(f"Failed to initialize database. Please ensure you have installed the latest requirements. If the error persists, please report this as in future the database will be required: {e}") + else: + logging.error(f"Failed to initialize database. Please ensure you have installed the latest requirements. If the error persists, please report this as in future the database will be required: {e}") + + if args.enable_assets: + asset_manager.disable(e) + logging.error("Starting with assets disabled; asset endpoints will answer 503.") def start_comfyui(asyncio_loop=None): diff --git a/tests-unit/app_test/test_db_init_locking.py b/tests-unit/app_test/test_db_init_locking.py index b9cfa8c429e..a36866df270 100644 --- a/tests-unit/app_test/test_db_init_locking.py +++ b/tests-unit/app_test/test_db_init_locking.py @@ -118,7 +118,20 @@ def _raise_file_lock(): assert "Failed to initialize database." not in caplog.text -def test_setup_database_exits_for_file_lock_when_assets_are_enabled(monkeypatch, caplog): +class _RecordingManager: + """Stands in for AssetsEnabled so a degrade can be observed without a database.""" + + def __init__(self) -> None: + self.disabled_with: BaseException | None = None + + def startup(self) -> None: + return + + def disable(self, exc: BaseException) -> None: + self.disabled_with = exc + + +def test_setup_database_disables_assets_for_file_lock_instead_of_exiting(monkeypatch, caplog): monkeypatch.setattr(main, "dependencies_available", lambda: True) def _raise_file_lock(): @@ -130,16 +143,19 @@ def _raise_file_lock(): monkeypatch.setattr(main, "init_db", _raise_file_lock) monkeypatch.setattr(main.args, "enable_assets", True) + manager = _RecordingManager() - with caplog.at_level(logging.ERROR), pytest.raises(SystemExit) as error: - main.setup_database(None) + with caplog.at_level(logging.ERROR): + main.setup_database(manager) - assert error.value.code == 1 + assert isinstance(manager.disabled_with, RuntimeError), ( + "a locked database must degrade to assets-off, not stop ComfyUI booting" + ) assert "Database is locked. Another ComfyUI process is already using this database." in caplog.text - assert "The --enable-assets flag requires a working database connection." not in caplog.text + assert "Starting with assets disabled" in caplog.text -def test_setup_database_exits_for_driver_lock_when_assets_are_disabled(monkeypatch, caplog): +def test_setup_database_continues_for_driver_lock_when_assets_are_disabled(monkeypatch, caplog): monkeypatch.setattr(main, "dependencies_available", lambda: True) def _raise_driver_lock(): @@ -148,8 +164,32 @@ def _raise_driver_lock(): monkeypatch.setattr(main, "init_db", _raise_driver_lock) monkeypatch.setattr(main.args, "enable_assets", False) - with caplog.at_level(logging.ERROR), pytest.raises(SystemExit) as error: + with caplog.at_level(logging.ERROR): main.setup_database(None) - assert error.value.code == 1 assert "Database is locked. Another ComfyUI process is already using this database." in caplog.text + + +def test_setup_database_gives_wal_its_own_guidance_and_never_suggests_memory(monkeypatch, caplog): + monkeypatch.setattr(main, "dependencies_available", lambda: True) + + def _raise_wal(): + raise main.WalUnavailableError( + "SQLite WAL could not be enabled for database '/mnt/nas/comfy.db'. " + "SQLite WAL is not supported on network filesystems." + ) + + monkeypatch.setattr(main, "init_db", _raise_wal) + monkeypatch.setattr(main.args, "enable_assets", True) + manager = _RecordingManager() + + with caplog.at_level(logging.ERROR): + main.setup_database(manager) + + assert ":memory:" not in caplog.text, ( + "an in-memory database silently discards the whole asset catalogue on restart; " + "it must never be offered as a remedy for a network filesystem" + ) + assert "keep the database on a local disk" in caplog.text + assert "pip install -r requirements.txt" not in caplog.text + assert isinstance(manager.disabled_with, main.WalUnavailableError) diff --git a/tests-unit/assets_test/test_event_log_sites.py b/tests-unit/assets_test/test_event_log_sites.py index 499713856dd..e06d07e4617 100644 --- a/tests-unit/assets_test/test_event_log_sites.py +++ b/tests-unit/assets_test/test_event_log_sites.py @@ -1,6 +1,8 @@ +import logging import subprocess import sys from pathlib import Path +from types import SimpleNamespace import pytest @@ -64,3 +66,32 @@ def test_noassets_emits_no_enabled_event(tmp_path: Path) -> None: output = run_quick_startup(tmp_path) assert f"{TAG} assets.enabled " not in output + + +def test_disable_announces_itself_on_the_event_channel(caplog) -> None: + """A degrade must reach the monitor, which only reads the tagged channel. + + PromptServer emits assets.enabled during __init__, before setup_database can + fail, so a monitor has already been told assets are up by the time a database + failure degrades them. Without a contradicting event it keeps believing that. + """ + from app.assets.api import routes + from app.assets.manager import AssetsEnabled + + manager = AssetsEnabled(SimpleNamespace(enable_assets=True, enable_asset_hashing=False)) + routes._ASSETS_ENABLED = True + try: + with caplog.at_level(logging.WARNING): + manager.disable(FileNotFoundError(2, "No such file", "/home/alice/models/secret.db")) + + assert manager.enabled is False + assert routes._ASSETS_ENABLED is False, "already-registered routes must start answering 503" + + tagged = [line for line in caplog.text.splitlines() if f"{TAG} assets.disabled" in line] + assert len(tagged) == 1, "the monitor needs exactly one disabled event" + assert "error_type=FileNotFoundError" in tagged[0] + assert "/home/alice" not in tagged[0], ( + "OSError embeds its path in str(); the event must carry only the class name" + ) + finally: + routes._ASSETS_ENABLED = False diff --git a/tests-unit/test_assets_event_log_static.py b/tests-unit/test_assets_event_log_static.py index c7712127590..85edbb3cd8c 100644 --- a/tests-unit/test_assets_event_log_static.py +++ b/tests-unit/test_assets_event_log_static.py @@ -45,6 +45,7 @@ class CallSite(NamedTuple): ( # todo 10 - seeder lifecycle + the single assets.enabled site CallSite("server.py", "__init__", "assets.enabled"), + CallSite("app/assets/manager.py", "disable", "assets.disabled"), CallSite("app/assets/seeder.py", "_run_scan", "seeder.scan_started"), CallSite("app/assets/seeder.py", "_run_scan", "seeder.scan_completed"), CallSite("app/assets/seeder.py", "_run_scan", "seeder.scan_failed"), From 0aaeaa4eae8bcba4d8c8c6e90e5b8fe02a3aa370 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Fri, 18 Sep 2026 00:20:42 -0700 Subject: [PATCH 096/111] Stop scans and ingest when assets are disabled, and bound the transition retry --- app/assets/manager.py | 9 +++ app/assets/services/hash_mode_state.py | 9 ++- .../services/test_transition_drain.py | 57 +++++++++++++++++++ .../assets_test/test_event_log_sites.py | 32 +++++++++++ 4 files changed, 102 insertions(+), 5 deletions(-) diff --git a/app/assets/manager.py b/app/assets/manager.py index 319ded8a5b6..74b40c314d8 100644 --- a/app/assets/manager.py +++ b/app/assets/manager.py @@ -151,6 +151,7 @@ def startup(self) -> None: def disable(self, exc: BaseException) -> None: self._disabled = True + asset_seeder.disable() close_assets_feature_gate() emit("assets.disabled", error_type=error_type(exc)) @@ -163,6 +164,8 @@ def register_routes( register_assets_routes(app, user_manager) def ensure_scan_started(self) -> None: + if self._disabled: + return None asset_seeder.start(roots=("models", "input", "output")) def pause_background_scan(self) -> None: @@ -189,6 +192,8 @@ def register_upload( *, content_written: bool, ) -> UploadAssetView | None: + if self._disabled: + return None try: tag = upload_type if upload_type in ("input", "output") else "input" tags = [tag] + get_known_subfolder_tags(subfolder) @@ -218,11 +223,15 @@ def register_upload( def register_executed_output( self, abs_path: str, job_id: str | None ) -> RegisteredAsset | None: + if self._disabled: + return None return ingest_register_executed_output(abs_path, job_id) def register_cached_output( self, abs_path: str, job_id: str | None ) -> RegisteredAsset | None: + if self._disabled: + return None return ingest_register_cached_output(abs_path, job_id) def set_event_sink(self, sink: Callable[[str, dict[str, Any]], None] | None) -> None: diff --git a/app/assets/services/hash_mode_state.py b/app/assets/services/hash_mode_state.py index 476d8c32f31..a7a2b9f584f 100644 --- a/app/assets/services/hash_mode_state.py +++ b/app/assets/services/hash_mode_state.py @@ -145,11 +145,10 @@ def _apply(session: Session) -> str: content = session.get(AssetContent, content_id) if content is None or content.is_missing or content.path != entry.path: return "drop" - if content.size_bytes != size_bytes or content.mtime_ns != mtime_ns: - return "retry" - if preparation == "retry" or ( - snapshot is None and preparation != "gone" - ): + row_changed = ( + content.size_bytes != size_bytes or content.mtime_ns != mtime_ns + ) + if row_changed or preparation == "retry": if entry.ticks + 1 < _MAX_VERIFY_ATTEMPTS: return "retry" content.hash = None diff --git a/tests-unit/assets_test/services/test_transition_drain.py b/tests-unit/assets_test/services/test_transition_drain.py index cdfc4ef24d4..e40b136b5c2 100644 --- a/tests-unit/assets_test/services/test_transition_drain.py +++ b/tests-unit/assets_test/services/test_transition_drain.py @@ -4,8 +4,11 @@ import folder_paths import pytest from blake3 import blake3 +import sqlalchemy as sa from sqlalchemy import select +import app.database.db as db_mod + from app.assets.database.models import Asset, AssetContent, AssetTag from app.assets.database.queries.records import create_content, create_record from app.assets.helpers import to_stored_hash @@ -394,6 +397,9 @@ def warnings_naming_the_path() -> list[str]: ) +_MAX_DRAIN_TICKS = 10 + + def test_transition_drain_retires_only_the_unreadable_path_and_hashes_the_healthy_one( session, temp_dir, monkeypatch ): @@ -432,3 +438,54 @@ def denied_for_the_unreadable_path(candidate_path: str): assert session.get(AssetContent, unreadable_id).hash is None assert hash_mode_state.pending_transition_count() == 0 assert read_stored_mode(session) == "on" + + +def test_transition_drain_retires_an_entry_that_keeps_losing_the_row_compare_and_set( + session, temp_dir, monkeypatch, caplog +): + """A CAS mismatch must spend the retirement budget, not requeue forever. + + The mode flip is gated on the queue draining, so an entry that always loses the + compare-and-set would keep the transition pending for the process lifetime and + spend a write transaction on every tick. + """ + path = temp_dir / "always-racing.bin" + content_id, _ = _seed_hashed_row(session, path, b"bytes that keep moving") + write_stored_mode(session, "off") + monkeypatch.setattr(hash_mode_state._mode, "hashing_enabled", lambda: True) + + transition = record_transition_intent(session) + enqueue_transition_work(session, transition) + session.commit() + + # Lose the CAS on every attempt: bump the row's mtime after the preflight reads it. + real_preflight = hash_mode_state._preflight_transition_entry + + def preflight_then_race(entry_path: str): + result = real_preflight(entry_path) + if result is not None: + db_mod.run_write_txn( + lambda s: s.execute( + sa.update(AssetContent) + .where(AssetContent.id == content_id) + .values(mtime_ns=AssetContent.mtime_ns + 1000) + ) + ) + return result + + monkeypatch.setattr(hash_mode_state, "_preflight_transition_entry", preflight_then_race) + + with caplog.at_level(logging.WARNING): + for _ in range(_MAX_DRAIN_TICKS): + drain_transition_queue(session) + session.commit() + session.expire_all() + if hash_mode_state.pending_transition_count() == 0: + break + + assert hash_mode_state.pending_transition_count() == 0, ( + "a permanently racing row must be retired by the attempt budget, not requeued forever" + ) + assert read_stored_mode(session) == "on", ( + "the mode flip is gated on the queue emptying, so a stuck entry wedges it" + ) diff --git a/tests-unit/assets_test/test_event_log_sites.py b/tests-unit/assets_test/test_event_log_sites.py index e06d07e4617..c7d0a0bd779 100644 --- a/tests-unit/assets_test/test_event_log_sites.py +++ b/tests-unit/assets_test/test_event_log_sites.py @@ -95,3 +95,35 @@ def test_disable_announces_itself_on_the_event_channel(caplog) -> None: ) finally: routes._ASSETS_ENABLED = False + + +def test_disable_actually_stops_scanning_and_ingest_not_just_http() -> None: + """A degrade must disarm the write paths, not only the aiohttp routes. + + `enabled` is read once, by PromptServer.__init__, which has already run by the + time setup_database can fail. So flipping it is invisible to every later caller: + the scanner and the three ingest entry points have to be gated directly. + """ + from app.assets import manager as manager_mod + from app.assets.api import routes + from app.assets.manager import AssetsEnabled + + started: list[str] = [] + mgr = AssetsEnabled(SimpleNamespace(enable_assets=True, enable_asset_hashing=False)) + routes._ASSETS_ENABLED = True + seeder = manager_mod.asset_seeder + was_disabled = seeder.is_disabled() + try: + mgr.disable(RuntimeError("database is locked")) + + assert seeder.is_disabled(), ( + "queue_output_scan's existing gate reads the seeder, so disable() must arm it" + ) + mgr.ensure_scan_started() + assert started == [] + assert mgr.register_upload("/tmp/x.png", "x", "input", "", content_written=True) is None + assert mgr.register_executed_output("/tmp/x.png", "job-1") is None + assert mgr.register_cached_output("/tmp/x.png", "job-1") is None + finally: + seeder._disabled = was_disabled + routes._ASSETS_ENABLED = False From b3fbe18b8ea4335ed5f7c58a8b9fb95d7daecf96 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Fri, 18 Sep 2026 01:04:58 -0700 Subject: [PATCH 097/111] Store the published file's own stat and reconcile uploads in one transaction --- app/assets/services/ingest.py | 27 +++++---- .../services/test_cross_volume_upload.py | 41 ++++++++++++++ .../assets_test/services/test_upload_b.py | 55 +++++++++++++++++++ 3 files changed, 109 insertions(+), 14 deletions(-) diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index c2a8cefda70..1319d9f0e30 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -185,7 +185,13 @@ def _guess_upload_mime_type( return guessed or "application/octet-stream" -def _move_temp_to_dest(temp_path: str, dest_abs: str) -> None: +def _move_temp_to_dest(temp_path: str, dest_abs: str) -> os.stat_result: + """Publish the upload and return the destination's own stat. + + A cross-device copy replays the modification time at the destination + filesystem's granularity, so the published file is the only authority on + the facts stored beside its hash. + """ os.makedirs(os.path.dirname(dest_abs), exist_ok=True) try: os.replace(temp_path, dest_abs) @@ -205,11 +211,12 @@ def _move_temp_to_dest(temp_path: str, dest_abs: str) -> None: if os.path.exists(destination_temp): os.unlink(destination_temp) raise - os.unlink(temp_path) - return + _remove_temp_path(temp_path) + return os.stat(dest_abs) raise RuntimeError(f"failed to move uploaded file into place: {error}") from error except Exception as e: raise RuntimeError(f"failed to move uploaded file into place: {e}") from e + return os.stat(dest_abs) def _create_upload_record( @@ -734,14 +741,14 @@ def upload_from_temp_path( content_type = _guess_upload_mime_type( mime_type, client_filename, name, os.path.basename(dest_abs) ) - _move_temp_to_dest(temp_path, dest_abs) + published_stat = _move_temp_to_dest(temp_path, dest_abs) return _create_content_and_upload_record( stored_hash, dest_abs, _ContentFacts( stored_hash, - verified_stat.st_size, - verified_stat.st_mtime_ns, + published_stat.st_size, + published_stat.st_mtime_ns, ), True, _UploadRecordSpec( @@ -801,14 +808,6 @@ def register_file_in_place( digest, verified_stat = _snapshot_hash_with_retry(locator) size_bytes, mtime_ns = verified_stat.st_size, verified_stat.st_mtime_ns stored_hash = to_stored_hash(digest) - def _reconcile_work(session: Session) -> None: - _reconcile_live_content_at_path( - session, - locator, - _ContentFacts(stored_hash, size_bytes, mtime_ns), - content_written=content_written, - ) - run_write_txn(_reconcile_work) return _create_content_and_upload_record( stored_hash, locator, diff --git a/tests-unit/assets_test/services/test_cross_volume_upload.py b/tests-unit/assets_test/services/test_cross_volume_upload.py index ec2d15af33e..4e97593bc5a 100644 --- a/tests-unit/assets_test/services/test_cross_volume_upload.py +++ b/tests-unit/assets_test/services/test_cross_volume_upload.py @@ -129,3 +129,44 @@ def fail_copy(source_path: str, destination_path: str) -> None: assert destination.read_bytes() == b"existing" assert source.read_bytes() == b"upload" assert not list(destination.parent.glob(f".{destination.name}*.tmp")) + + +def test_move_temp_to_dest_reports_the_published_files_own_stat( + tmp_path: Path, monkeypatch +) -> None: + """The destination's stat is authoritative, not the temp file's. + + A cross-device publish replays mtime through copy2, but the destination + filesystem stores it at its own granularity (exFAT 2s, HFS+/ext3 1s, many + NFS/CIFS mounts). The caller pairs these facts with the content hash, and + the upload path later re-stats the destination and requires an exact match, + so reporting the temp file's nanoseconds strands the bytes with no row. + """ + source = tmp_path / "upload.part" + destination = tmp_path / "output" / "upload.bin" + source.write_bytes(b"upload") + coarse_mtime_ns = (source.stat().st_mtime_ns // 2_000_000_000) * 2_000_000_000 + + real_replace = ingest.os.replace + + def fail_cross_volume_move(source_path: str, destination_path: str) -> None: + if source_path == str(source) and destination_path == str(destination): + raise OSError(errno.EXDEV, "Invalid cross-device link") + real_replace(source_path, destination_path) + + real_copy2 = ingest.shutil.copy2 + + def copy_then_coarsen(source_path: str, destination_path: str) -> None: + real_copy2(source_path, destination_path) + ingest.os.utime(destination_path, ns=(coarse_mtime_ns, coarse_mtime_ns)) + + monkeypatch.setattr(ingest.os, "replace", fail_cross_volume_move) + monkeypatch.setattr(ingest.shutil, "copy2", copy_then_coarsen) + + published = ingest._move_temp_to_dest(str(source), str(destination)) + + assert published.st_mtime_ns == destination.stat().st_mtime_ns + assert published.st_mtime_ns == coarse_mtime_ns, ( + "the returned stat must describe the published file, not the temp file" + ) + assert published.st_size == destination.stat().st_size diff --git a/tests-unit/assets_test/services/test_upload_b.py b/tests-unit/assets_test/services/test_upload_b.py index f3623491ad0..86191b46413 100644 --- a/tests-unit/assets_test/services/test_upload_b.py +++ b/tests-unit/assets_test/services/test_upload_b.py @@ -1,3 +1,4 @@ +import errno import os import uuid from contextlib import contextmanager @@ -1394,3 +1395,57 @@ def test_multipart_upload_persists_the_stat_hashing_verified( "the digest, not the pair, feeds hash-mode destination naming" ) assert lookup_for_view(session, content.hash) is not None + + +def test_cross_device_upload_to_a_coarse_mtime_destination_still_lands_a_row( + mock_create_session, monkeypatch +): + """A publish onto a filesystem with coarser mtime must not strand the bytes. + + upload_from_temp_path hashes the temp file, then publishes. Same-device + os.replace keeps the inode so the temp stat still describes the destination, + but a cross-device copy replays mtime at the destination's granularity + (exFAT 2s, HFS+/ext3 1s, many NFS/CIFS mounts). The create path re-stats the + destination and requires an exact match, so pairing the hash with the temp + file's nanoseconds raises UploadUnstableError after the bytes are already + published -- a file on disk with no catalogue row, which is the exact failure + shape this branch exists to remove. + """ + temp_path = _write_temp(b"cross-device-bytes") + output_dir = folder_paths.get_output_directory() + os.makedirs(output_dir, exist_ok=True) + + real_replace = ingest_module.os.replace + real_copy2 = ingest_module.shutil.copy2 + coarse = [] + + def force_cross_device(source_path, destination_path): + if str(source_path) == str(temp_path): + raise OSError(errno.EXDEV, "Invalid cross-device link") + real_replace(source_path, destination_path) + + def copy_then_coarsen(source_path, destination_path): + real_copy2(source_path, destination_path) + stat = ingest_module.os.stat(destination_path) + floored = (stat.st_mtime_ns // 2_000_000_000) * 2_000_000_000 + ingest_module.os.utime(destination_path, ns=(floored, floored)) + coarse.append(floored) + + monkeypatch.setattr(ingest_module.os, "replace", force_cross_device) + monkeypatch.setattr(ingest_module.shutil, "copy2", copy_then_coarsen) + + result = upload_from_temp_path( + temp_path=temp_path, + name="cross.bin", + tags=["output"], + client_filename="cross.bin", + ) + + assert coarse, "the cross-device branch must have run for this test to mean anything" + with mock_create_session() as session: + content = session.get(AssetContent, result.content_id) + assert content is not None, "the upload must be catalogued, not stranded on disk" + assert content.mtime_ns == coarse[0], ( + "stored facts must describe the published file, not the temp file" + ) + assert content.mtime_ns == os.stat(content.path).st_mtime_ns From 91c9e87a2637a99ff22a72307f90e8e057c85a75 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Fri, 18 Sep 2026 01:31:48 -0700 Subject: [PATCH 098/111] Delete unreachable scanner and lookup helpers and correct their documentation --- app/assets/docs/README.md | 2 + app/assets/scanner.py | 68 +------------------ app/assets/scanner_admission.py | 3 - app/assets/scanner_changes.py | 6 +- app/assets/services/hash_mode_state.py | 1 - app/assets/services/ingest.py | 15 ++-- app/assets/services/lookup.py | 14 ---- tests-unit/assets_test/helpers.py | 49 +++++++++++++ tests-unit/assets_test/queries/test_lookup.py | 27 +------- .../services/test_admission_gate.py | 4 +- .../services/test_detection_gate.py | 20 +++--- .../services/test_hash_mode_state.py | 2 +- .../services/test_null_hash_recovery.py | 6 +- .../test_queue_drain_characterization.py | 8 +-- .../services/test_scan_lifecycle.py | 4 +- .../assets_test/services/test_scanner_b.py | 10 +-- .../services/test_scanner_sync_chunking.py | 3 +- .../assets_test/services/test_split_policy.py | 6 +- .../services/test_transition_drain.py | 26 +++---- .../services/test_write_lock_hold_barriers.py | 1 - .../assets_test/test_intended_behaviour.py | 15 ++-- .../test_scanner_ingest_event_log.py | 6 +- 22 files changed, 126 insertions(+), 170 deletions(-) diff --git a/app/assets/docs/README.md b/app/assets/docs/README.md index da77288ce0c..ee290b08a58 100644 --- a/app/assets/docs/README.md +++ b/app/assets/docs/README.md @@ -95,6 +95,8 @@ In hash mode, verify the new bytes. Refresh the existing content row if the hash With hashing off, a change to both modification time and size is handled the same way as a hash difference: the old content is marked missing and new content and a new asset record are created. A modification-time change alone refreshes the existing row's file facts and clears its stored hash (see Hashing modes). +Whenever a scan observes that a content row's bytes changed, it also clears the extracted metadata of every record pointing at that row, because that metadata describes the previous bytes. The record itself survives with its name, user tags and user metadata intact; only the extracted metadata is dropped, and the next enrichment pass re-derives it. A client reading such a record in between sees `metadata` absent rather than stale, and this does not count as an edit, so `updated_at` does not advance. + An asset record never changes from one byte identity to another. Existing history references therefore continue to point to the old, already-missing content instead of silently serving new bytes. ### Deleted path reused diff --git a/app/assets/scanner.py b/app/assets/scanner.py index 43d1475eace..5211705af09 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -173,24 +173,6 @@ def collect_models_files() -> list[str]: return out -def sync_references_with_filesystem( - session, - root: RootType, - collect_existing_paths: bool = False, - progress: _ScanProgress | None = None, - pending_verification_ids: list[str] | None = None, - diagnostics: list[OSError] | None = None, -) -> set[str] | None: - return sync_prefixes_with_filesystem( - session, - get_scan_prefixes_for_root(root), - collect_existing_paths=collect_existing_paths, - progress=progress, - pending_verification_ids=pending_verification_ids, - diagnostics=diagnostics, - ) - - class _ReferenceObservation(NamedTuple): content_id: str @@ -217,8 +199,7 @@ def _catalogued_references( def observe_references_on_filesystem( prefixes: list[str], - progress: _ScanProgress | None = None, - diagnostics: list[_ReferenceDiagnostic] | None = None, + diagnostics: list[_ReferenceDiagnostic], session: Session | None = None, ) -> tuple[list[_ReferenceObservation], set[str]]: """Stat every catalogued reference without holding the writer lease. @@ -242,19 +223,9 @@ def observe_references_on_filesystem( _ReferenceObservation(content_id, path, size_bytes, mtime_ns, None) ) except PermissionError as e: - if diagnostics is None: - _log_scan_error("reference_stat", e) - if progress is not None: - progress.permission_denied += 1 - logging.debug("Permission denied accessing %s", path) - else: - diagnostics.append(_ReferenceDiagnostic(path, e)) + diagnostics.append(_ReferenceDiagnostic(path, e)) except OSError as e: - if diagnostics is None: - _log_scan_error("reference_stat", e) - logging.debug("OSError checking %s: %s", path, e) - else: - diagnostics.append(_ReferenceDiagnostic(path, e)) + diagnostics.append(_ReferenceDiagnostic(path, e)) observations.append( _ReferenceObservation(content_id, path, size_bytes, mtime_ns, None) ) @@ -301,27 +272,6 @@ def apply_reference_observations( ) -def sync_prefixes_with_filesystem( - session: Session, - prefixes: list[str], - collect_existing_paths: bool = False, - progress: _ScanProgress | None = None, - pending_verification_ids: list[str] | None = None, - diagnostics: list[_ReferenceDiagnostic] | None = None, -) -> set[str] | None: - if not prefixes: - return set() if collect_existing_paths else None - - observations, survivors = observe_references_on_filesystem( - prefixes, progress=progress, diagnostics=diagnostics, session=session - ) - apply_reference_observations( - session, observations, pending_verification_ids=pending_verification_ids - ) - - return survivors if collect_existing_paths else None - - def _publish_reference_diagnostics( diagnostics: list[_ReferenceDiagnostic], progress: _ScanProgress | None ) -> None: @@ -483,18 +433,6 @@ def _mark_chunk( return marked_so_far -def mark_contents_missing_outside_prefixes( - session: Session, prefixes: list[str] -) -> int: - contents = session.scalars( - sa.select(AssetContent).where(AssetContent.is_missing.is_(False)) - ) - missing = [content for content in contents if not _is_under_prefixes(content.path, prefixes)] - for content in missing: - mark_content_missing(session, content.id) - return len(missing) - - def collect_paths_for_roots(roots: tuple[RootType, ...]) -> list[str]: """Collect all file paths for the given roots.""" paths: list[str] = [] diff --git a/app/assets/scanner_admission.py b/app/assets/scanner_admission.py index 987e3d24f9e..977fa3edaf3 100644 --- a/app/assets/scanner_admission.py +++ b/app/assets/scanner_admission.py @@ -15,8 +15,6 @@ from dataclasses import dataclass from typing import Final -from sqlalchemy.orm import Session - from app.assets.services.path_utils import compute_loader_path, get_name_and_tags_from_asset_path PARTIAL_DOWNLOAD_EXTENSIONS = frozenset({ @@ -68,7 +66,6 @@ def _two_stat_admit(paths_with_stats: list[tuple[str, os.stat_result]]) -> tuple def tick_watch_list( - _session: Session | None = None, interrupt_check: Callable[[], bool] | None = None, ) -> None: from app.assets.scanner import SeedAssetSpec, insert_asset_specs diff --git a/app/assets/scanner_changes.py b/app/assets/scanner_changes.py index 819a089f36c..53833907876 100644 --- a/app/assets/scanner_changes.py +++ b/app/assets/scanner_changes.py @@ -189,10 +189,11 @@ def detect_content_change( return if content.size_bytes == stat_result.st_size: # User identity rule: a same-size mtime bump (rsync, cloud sync, backup restore) is the - # same file — never split, or the record's tags and metadata are destroyed. + # same file — never split, or the record's user tags and identity are destroyed. # The stored hash goes with the refreshed stat: OFF mode cannot prove the bytes, and a # refreshed stat alone would re-qualify the row to be served under a digest it may no - # longer match. + # longer match. Extracted metadata described the old bytes, so it is cleared alongside + # the hash and the next enrich pass re-derives it. content.size_bytes = stat_result.st_size content.mtime_ns = stat_result.st_mtime_ns content.hash = None @@ -266,7 +267,6 @@ def _apply_pending_verification( def drain_pending_verifications( - _session: Session | None = None, limit: int | None = None, interrupt_check: Callable[[], bool] | None = None, ) -> int: diff --git a/app/assets/services/hash_mode_state.py b/app/assets/services/hash_mode_state.py index a7a2b9f584f..a4a6cb0c727 100644 --- a/app/assets/services/hash_mode_state.py +++ b/app/assets/services/hash_mode_state.py @@ -110,7 +110,6 @@ def _preflight_transition_entry(path: str) -> tuple[str, int, int | None] | None def drain_transition_queue( - _session: Session | None = None, interrupt_check: Callable[[], bool] | None = None, ) -> None: global _off_to_on_transition_in_flight diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index 1319d9f0e30..8ef6bc6c17a 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -87,12 +87,15 @@ def _extract_system_metadata_sync( def _discard_unreferenced_content(session: Session, content_id: str) -> None: """Remove a content row left orphaned by a failed registration. - ``create_content`` inserts inside a SAVEPOINT (``begin_nested``); under - pysqlite that insert survives the enclosing ``rollback`` because pysqlite - has no real nested transaction. When the follow-on ``create_record`` fails - we would otherwise leak an unreferenced content row, so delete it explicitly - once we have confirmed no record points at it. Best-effort: cleanup errors - are logged and swallowed so the original failure is what surfaces. + ``create_content`` inserts inside a SAVEPOINT (``begin_nested``). On the + in-memory and no-WAL engines that insert survives the enclosing + ``rollback``, because pysqlite opens no real transaction, so the follow-on + ``create_record`` failing would leak an unreferenced content row. The WAL + writer engine does undo it, so there the queries below find nothing and only + cost a fresh ``BEGIN IMMEDIATE``; the compensation cannot be skipped on that + basis, because a caller's session may be bound to either kind of engine and + the binding is not knowable from here. Best-effort: cleanup errors are + logged and swallowed so the original failure is what surfaces. """ try: ref_count = session.scalar( diff --git a/app/assets/services/lookup.py b/app/assets/services/lookup.py index 6017d7e5438..1d2ce8379e1 100644 --- a/app/assets/services/lookup.py +++ b/app/assets/services/lookup.py @@ -15,7 +15,6 @@ from sqlalchemy import select, update from sqlalchemy.orm import Session -from app.assets import mode from app.assets.database.models import AssetContent @@ -79,18 +78,5 @@ def claim_qualified_content(session: Session, content_id: str, hash: str) -> boo return result.rowcount == 1 -def refresh_qualified_content(session: Session, content_id: str) -> AssetContent | None: - content = session.get(AssetContent, content_id, populate_existing=True) - if content is None or not _qualifies(content): - return None - return content - - -def lookup_for_from_hash(session: Session, hash: str) -> AssetContent | None: - if not mode.hashing_enabled(): - return None - return next(qualified_content_iterator(session, hash), None) - - def lookup_for_view(session: Session, hash: str) -> AssetContent | None: return next(qualified_content_iterator(session, hash), None) diff --git a/tests-unit/assets_test/helpers.py b/tests-unit/assets_test/helpers.py index e6fd4f23b0a..b3bd3819bf0 100644 --- a/tests-unit/assets_test/helpers.py +++ b/tests-unit/assets_test/helpers.py @@ -13,12 +13,14 @@ import requests from aiohttp import web from aiohttp.test_utils import make_mocked_request +import sqlalchemy as sa from sqlalchemy import Engine, create_engine from sqlalchemy.orm import Session from app.assets import scanner from app.assets.api import routes from app.assets.database.models import Asset, AssetContent +from app.assets.database.queries.records import mark_content_missing from app.assets.database.queries.records import create_content, create_record from app.database.models import Base @@ -203,3 +205,50 @@ def assert_hash_fields_consistent( assert h == expected_hash, ( f"hash must equal expected: got {h!r}, expected {expected_hash!r}" ) + + +def sync_prefixes_in_session( + session: Session, + prefixes: list[str], + collect_existing_paths: bool = False, + progress=None, + pending_verification_ids: list[str] | None = None, + diagnostics=None, +) -> set[str] | None: + """Observe and apply a prefix sync inside one caller-supplied session. + + Production drives this through `sync_root_safely`, which owns its own + chunked transactions. Tests that need to inspect the result inside their own + session compose the same two live calls here rather than production carrying + a synchronous entry point nothing ships against. + """ + if not prefixes: + return set() if collect_existing_paths else None + + collected = diagnostics if diagnostics is not None else [] + observations, survivors = scanner.observe_references_on_filesystem( + prefixes, diagnostics=collected, session=session + ) + scanner.apply_reference_observations( + session, observations, pending_verification_ids=pending_verification_ids + ) + if diagnostics is None: + scanner._publish_reference_diagnostics(collected, progress) + return survivors if collect_existing_paths else None + + +def mark_contents_missing_outside_prefixes_in_session( + session: Session, prefixes: list[str] +) -> int: + """Mark every live content outside `prefixes` missing, in the caller's session.""" + contents = session.scalars( + sa.select(AssetContent).where(AssetContent.is_missing.is_(False)) + ) + missing = [ + content + for content in contents + if not scanner._is_under_prefixes(content.path, prefixes) + ] + for content in missing: + mark_content_missing(session, content.id) + return len(missing) diff --git a/tests-unit/assets_test/queries/test_lookup.py b/tests-unit/assets_test/queries/test_lookup.py index 805100b65b8..524b9c93b68 100644 --- a/tests-unit/assets_test/queries/test_lookup.py +++ b/tests-unit/assets_test/queries/test_lookup.py @@ -12,9 +12,7 @@ from app.assets.services.lookup import ( claim_qualified_content, is_temp_path as _is_temp_path, - lookup_for_from_hash, lookup_for_view, - refresh_qualified_content, ) from app.database.models import Base @@ -45,15 +43,6 @@ def _make_file(tmp_path, name: str, content: bytes = b"bytes") -> str: return str(p) -def test_temp_only_match_from_hash_returns_none(session, tmp_path): - f = _make_file(tmp_path, "f.png") - create_content(session, path=f, hash="abc123") - session.commit() - with patch("app.assets.services.lookup.is_temp_path", return_value=True): - result = lookup_for_from_hash(session, "abc123") - assert result is None - - def test_temp_only_match_view_returns_none(session, tmp_path): f = _make_file(tmp_path, "f2.png") create_content(session, path=f, hash="abc123") @@ -69,18 +58,6 @@ def test_sibling_prefix_not_temp(tmp_path): assert _is_temp_path(str(tmp_path / "temp" / "f.png")) -def test_off_mode_from_hash_returns_none(session, tmp_path): - class FakeArgs: - enable_asset_hashing = False - - mode_module.init(FakeArgs()) - f = _make_file(tmp_path, "f3.png") - create_content(session, path=f, hash="abc123") - session.commit() - result = lookup_for_from_hash(session, "abc123") - assert result is None - - def test_upload_content_lookup_not_gated_on_hashing_flag(session, tmp_path): class FakeArgs: enable_asset_hashing = False @@ -104,7 +81,7 @@ def test_stale_older_newer_live_returns_newer(session, tmp_path): session.execute(update(AssetContent).where(AssetContent.id == c_new.id).values(created_at=new_time)) session.commit() - result = lookup_for_from_hash(session, "xyz") + result = lookup_for_view(session, "xyz") assert result is not None assert result.id == c_new.id @@ -156,7 +133,7 @@ def test_refresh_qualified_content_none_when_file_vanishes(session, tmp_path): os.unlink(f) - assert refresh_qualified_content(session, content.id) is None + assert lookup_for_view(session, content.hash) is None def test_size_mismatch_disqualifies_a_row_that_never_recorded_an_mtime(session, tmp_path): diff --git a/tests-unit/assets_test/services/test_admission_gate.py b/tests-unit/assets_test/services/test_admission_gate.py index a7fca074f78..5f93ac30a02 100644 --- a/tests-unit/assets_test/services/test_admission_gate.py +++ b/tests-unit/assets_test/services/test_admission_gate.py @@ -89,7 +89,7 @@ def test_never_stabilizes_dropped_after_cap(session, temp_dir: Path): target_ns = max(path.stat().st_mtime_ns, previous_target_ns) + 1_000_000 os.utime(path, ns=(target_ns, target_ns)) previous_target_ns = target_ns - tick_watch_list(session) + tick_watch_list() assert _WATCH_LIST == [] assert session.scalars(select(AssetContent)).all() == [] @@ -112,7 +112,7 @@ def test_stable_scan_admission_removes_watch_entry_before_next_tick(session, tem ), patch("app.assets.scanner.seed_asset_specs") as seed_asset_specs, ): - tick_watch_list(session) + tick_watch_list() assert admitted == [str(path)] assert watched == [] diff --git a/tests-unit/assets_test/services/test_detection_gate.py b/tests-unit/assets_test/services/test_detection_gate.py index a0e384d97e3..e66bee1fbcb 100644 --- a/tests-unit/assets_test/services/test_detection_gate.py +++ b/tests-unit/assets_test/services/test_detection_gate.py @@ -10,10 +10,10 @@ from app.assets.scanner import ( clear_pending_verifications, drain_pending_verifications, - sync_prefixes_with_filesystem, ) from app.assets.scanner_changes import queue_pending_verification from app.assets.services.snapshot_hash import snapshot_hash +from assets_test.helpers import sync_prefixes_in_session @pytest.fixture(autouse=True) @@ -63,7 +63,7 @@ def test_off_mode_same_size_touch_does_not_split(session, temp_dir: Path): patch("folder_paths.get_input_directory", return_value=str(input_root)), patch("app.assets.scanner.mode.hashing_enabled", return_value=False), ): - sync_prefixes_with_filesystem(session, [str(input_root)]) + sync_prefixes_in_session(session, [str(input_root)]) session.commit() contents = list(session.scalars(select(AssetContent))) @@ -86,7 +86,7 @@ def test_off_mode_size_change_splits(session, temp_dir: Path): patch("folder_paths.get_input_directory", return_value=str(input_root)), patch("app.assets.scanner.mode.hashing_enabled", return_value=False), ): - sync_prefixes_with_filesystem(session, [str(input_root)]) + sync_prefixes_in_session(session, [str(input_root)]) session.commit() contents = list(session.scalars(select(AssetContent).order_by(AssetContent.created_at))) @@ -107,8 +107,8 @@ def test_hash_mode_touch_refreshes_mtime(session, temp_dir: Path): patch("folder_paths.get_input_directory", return_value=str(input_root)), patch("app.assets.scanner.mode.hashing_enabled", return_value=True), ): - sync_prefixes_with_filesystem(session, [str(input_root)]) - processed = drain_pending_verifications(session) + sync_prefixes_in_session(session, [str(input_root)]) + processed = drain_pending_verifications() session.commit() refreshed = session.get(AssetContent, old_content.id) @@ -133,8 +133,8 @@ def test_hash_mode_real_edit_splits(session, temp_dir: Path): patch("folder_paths.get_input_directory", return_value=str(input_root)), patch("app.assets.scanner.mode.hashing_enabled", return_value=True), ): - sync_prefixes_with_filesystem(session, [str(input_root)]) - drain_pending_verifications(session) + sync_prefixes_in_session(session, [str(input_root)]) + drain_pending_verifications() session.commit() contents = list(session.scalars(select(AssetContent).order_by(AssetContent.created_at))) @@ -158,8 +158,8 @@ def test_old_record_id_resolves_to_missing_content_after_split(session, temp_dir patch("folder_paths.get_input_directory", return_value=str(input_root)), patch("app.assets.scanner.mode.hashing_enabled", return_value=True), ): - sync_prefixes_with_filesystem(session, [str(input_root)]) - drain_pending_verifications(session) + sync_prefixes_in_session(session, [str(input_root)]) + drain_pending_verifications() session.commit() session.expire_all() @@ -186,7 +186,7 @@ def mutate_then_hash(candidate_path: str): monkeypatch.setattr("app.assets.scanner_changes.snapshot_hash", mutate_then_hash) - processed = drain_pending_verifications(session) + processed = drain_pending_verifications() live_content = session.scalar( select(AssetContent).where(AssetContent.is_missing.is_(False)) diff --git a/tests-unit/assets_test/services/test_hash_mode_state.py b/tests-unit/assets_test/services/test_hash_mode_state.py index 372f22783c5..3db3ac6cefe 100644 --- a/tests-unit/assets_test/services/test_hash_mode_state.py +++ b/tests-unit/assets_test/services/test_hash_mode_state.py @@ -21,7 +21,7 @@ def test_absent_row_off_mode_no_transition(session): def test_empty_drain_keeps_off_mode_ready_for_a_later_on_transition(session): write_stored_mode(session, "off") - drain_transition_queue(session) + drain_transition_queue() with patch("app.assets.services.hash_mode_state._mode.hashing_enabled", return_value=True): assert record_transition_intent(session) == "off_to_on" diff --git a/tests-unit/assets_test/services/test_null_hash_recovery.py b/tests-unit/assets_test/services/test_null_hash_recovery.py index 6ae5ff96823..e1bf642e5b2 100644 --- a/tests-unit/assets_test/services/test_null_hash_recovery.py +++ b/tests-unit/assets_test/services/test_null_hash_recovery.py @@ -72,7 +72,7 @@ def test_deleted_null_hash_row_recovers_via_scanner_after_restore( path.unlink() transition = record_transition_intent(session) enqueue_transition_work(session, transition) - drain_transition_queue(session) + drain_transition_queue() session.commit() assert session.get(AssetContent, content_id).is_missing is True, ( "precondition: A1's drain marked the row missing on delete" @@ -113,7 +113,7 @@ def test_different_bytes_restored_at_same_path_does_not_recover_old_row( path.unlink() transition = record_transition_intent(session) enqueue_transition_work(session, transition) - drain_transition_queue(session) + drain_transition_queue() session.commit() assert session.get(AssetContent, content_id).is_missing is True @@ -149,7 +149,7 @@ def test_same_size_different_mtime_restored_at_same_path_does_not_recover_old_ro path.unlink() transition = record_transition_intent(session) enqueue_transition_work(session, transition) - drain_transition_queue(session) + drain_transition_queue() session.commit() assert session.get(AssetContent, content_id).is_missing is True diff --git a/tests-unit/assets_test/services/test_queue_drain_characterization.py b/tests-unit/assets_test/services/test_queue_drain_characterization.py index 95fce722ca3..16baf3adb60 100644 --- a/tests-unit/assets_test/services/test_queue_drain_characterization.py +++ b/tests-unit/assets_test/services/test_queue_drain_characterization.py @@ -55,7 +55,7 @@ def test_pending_verification_requeues_after_hash_oserror(session, temp_dir, mon lambda _path: (_ for _ in ()).throw(PermissionError("denied")), ) - processed = drain_pending_verifications(session) + processed = drain_pending_verifications() assert processed == 0 assert scanner_changes._pending_verification_ids == [content.id] @@ -73,7 +73,7 @@ def test_watch_list_keeps_entries_when_stat_raises(session, temp_dir, monkeypatc ) with pytest.raises(PermissionError): - tick_watch_list(session) + tick_watch_list() assert _WATCH_LIST == [entry] @@ -97,7 +97,7 @@ def test_transition_queue_retries_without_losing_companion_path( lambda _path: (_ for _ in ()).throw(PermissionError("denied")), ) - drain_transition_queue(session) + drain_transition_queue() assert list(_PENDING_QUEUE) == [_PendingEntry(path, ticks=1)] assert _PENDING_PATHS == {path} @@ -120,7 +120,7 @@ def test_transition_queue_exhaustion_clears_companion_and_persists_mode( ) for _ in range(3): - drain_transition_queue(session) + drain_transition_queue() session.commit() assert list(_PENDING_QUEUE) == [] diff --git a/tests-unit/assets_test/services/test_scan_lifecycle.py b/tests-unit/assets_test/services/test_scan_lifecycle.py index a1c7a79a618..e81c412bd0a 100644 --- a/tests-unit/assets_test/services/test_scan_lifecycle.py +++ b/tests-unit/assets_test/services/test_scan_lifecycle.py @@ -8,8 +8,8 @@ build_asset_specs, seed_asset_specs, stat_seed_specs, - sync_prefixes_with_filesystem, ) +from assets_test.helpers import sync_prefixes_in_session def _scan(session, root: Path) -> int: @@ -31,7 +31,7 @@ def test_e2e_scan_seed_detect_prune(session, temp_dir: Path): edited.write_bytes(b"replacement") (root / "partial.part").write_bytes(b"partial") with patch("app.assets.scanner.mode.hashing_enabled", return_value=False): - sync_prefixes_with_filesystem(session, [str(root)]) + sync_prefixes_in_session(session, [str(root)]) _scan(session, root) session.commit() contents = list(session.scalars(select(AssetContent))) diff --git a/tests-unit/assets_test/services/test_scanner_b.py b/tests-unit/assets_test/services/test_scanner_b.py index 9b1393eebc9..97ff75b283e 100644 --- a/tests-unit/assets_test/services/test_scanner_b.py +++ b/tests-unit/assets_test/services/test_scanner_b.py @@ -11,14 +11,16 @@ from app.assets.helpers import to_stored_hash from app.assets.scanner import ( build_asset_specs, - mark_contents_missing_outside_prefixes, seed_asset_specs, stat_seed_specs, - sync_prefixes_with_filesystem, ) from app.assets.services.snapshot_hash import snapshot_hash from ..helpers import enrich_via_prepare_apply +from assets_test.helpers import ( + mark_contents_missing_outside_prefixes_in_session, + sync_prefixes_in_session, +) @dataclass(frozen=True, slots=True) @@ -162,7 +164,7 @@ def test_prune_marks_missing_not_deletes(session, temp_dir: Path): seed_asset_specs(session, specs, stat_seed_specs(specs)) session.commit() - marked = mark_contents_missing_outside_prefixes(session, prefixes=[]) + marked = mark_contents_missing_outside_prefixes_in_session(session, prefixes=[]) session.commit() content = session.scalar(select(AssetContent)) @@ -184,7 +186,7 @@ def test_unhashed_missing_content_gets_tagged(session, temp_dir: Path): session.add(record) session.commit() - sync_prefixes_with_filesystem(session, prefixes=[str(temp_dir)]) + sync_prefixes_in_session(session, prefixes=[str(temp_dir)]) session.commit() session.expire_all() diff --git a/tests-unit/assets_test/services/test_scanner_sync_chunking.py b/tests-unit/assets_test/services/test_scanner_sync_chunking.py index d6c7b82dabd..a06a2d02dc6 100644 --- a/tests-unit/assets_test/services/test_scanner_sync_chunking.py +++ b/tests-unit/assets_test/services/test_scanner_sync_chunking.py @@ -6,6 +6,7 @@ from app.assets import scanner from app.assets.database.models import AssetContent from app.assets.database.queries import create_content, create_record, mark_content_missing +from assets_test.helpers import sync_prefixes_in_session def _seed_reference_observations( @@ -122,7 +123,7 @@ def test_root_sync_matches_single_transaction_and_uses_three_bounded_writes( ) baseline_survivors = db_mod.run_write_txn( - lambda session: scanner.sync_prefixes_with_filesystem( + lambda session: sync_prefixes_in_session( session, [str(tmp_path)], collect_existing_paths=True, diff --git a/tests-unit/assets_test/services/test_split_policy.py b/tests-unit/assets_test/services/test_split_policy.py index cf1e6a3d95a..3a76d221273 100644 --- a/tests-unit/assets_test/services/test_split_policy.py +++ b/tests-unit/assets_test/services/test_split_policy.py @@ -349,7 +349,7 @@ def test_accepted_mtime_bump_is_not_re_detected_by_the_next_scan( detect_content_change(session, content, path.stat(), hashing_is_enabled=True) - assert drain_pending_verifications(session) == 0 + assert drain_pending_verifications() == 0 detect_content_change(session, content, path.stat(), hashing_is_enabled=False) session.commit() @@ -381,7 +381,7 @@ def test_dropped_hash_is_refilled_in_place_by_a_later_hash_mode_pass( assert record_id in _candidates_under(session, temp_dir, compute_hashes=True) enqueue_transition_work(session, "off_to_on") - drain_transition_queue(session) + drain_transition_queue() session.commit() session.expire_all() @@ -475,7 +475,7 @@ def test_transition_drain_split_replacement_has_null_metadata( path.write_bytes(b"different new bytes") enqueue_transition_work(session, "off_to_on") - drain_transition_queue(session) + drain_transition_queue() session.commit() session.expire_all() diff --git a/tests-unit/assets_test/services/test_transition_drain.py b/tests-unit/assets_test/services/test_transition_drain.py index e40b136b5c2..9474f72e79d 100644 --- a/tests-unit/assets_test/services/test_transition_drain.py +++ b/tests-unit/assets_test/services/test_transition_drain.py @@ -52,7 +52,7 @@ def test_off_to_on_transition_hashes_null_rows_and_persists_mode(session, temp_d transition = record_transition_intent(session) enqueue_transition_work(session, transition) session.commit() - drain_transition_queue(session) + drain_transition_queue() session.commit() contents = list(session.scalars(select(AssetContent))) @@ -77,7 +77,7 @@ def test_transition_drain_splits_changed_content(session, temp_dir, monkeypatch) enqueue_transition_work(session, "off_to_on") session.commit() - drain_transition_queue(session) + drain_transition_queue() session.commit() contents = list(session.scalars(select(AssetContent))) @@ -113,7 +113,7 @@ def test_transition_drain_serves_unchanged_content_whose_stored_stat_went_stale( transition = record_transition_intent(session) enqueue_transition_work(session, transition) session.commit() - drain_transition_queue(session) + drain_transition_queue() session.commit() refreshed = session.get(AssetContent, content_id) @@ -151,7 +151,7 @@ def hash_or_raise(candidate_path: str): enqueue_transition_work(session, transition) session.commit() - drain_transition_queue(session) + drain_transition_queue() healthy_content = session.scalar( select(AssetContent).where(AssetContent.path == str(healthy_path)) @@ -180,7 +180,7 @@ def test_transition_drain_marks_deleted_path_missing_and_completes_transition( transition = record_transition_intent(session) enqueue_transition_work(session, transition) session.commit() - drain_transition_queue(session) + drain_transition_queue() session.commit() session.expire_all() @@ -220,7 +220,7 @@ def flaky_stat(candidate_path, *args, **kwargs): transition = record_transition_intent(session) enqueue_transition_work(session, transition) session.commit() - drain_transition_queue(session) + drain_transition_queue() session.commit() assert session.get(AssetContent, content_id).is_missing is False, ( @@ -246,7 +246,7 @@ def test_transition_drain_requeues_unstable_present_file_without_marking_it_miss transition = record_transition_intent(session) enqueue_transition_work(session, transition) session.commit() - drain_transition_queue(session) + drain_transition_queue() session.commit() assert session.get(AssetContent, content_id).is_missing is False, ( @@ -278,7 +278,7 @@ def test_transition_drain_mixes_a_deleted_path_with_a_healthy_one(session, temp_ transition = record_transition_intent(session) enqueue_transition_work(session, transition) session.commit() - drain_transition_queue(session) + drain_transition_queue() session.commit() session.expire_all() @@ -315,7 +315,7 @@ def test_transition_drain_skips_out_of_root_path(session, temp_dir, monkeypatch, with caplog.at_level(logging.WARNING): try: - drain_transition_queue(session) + drain_transition_queue() except ValueError as error: pytest.fail( f"an out-of-root path escaped the drain as {error!r}; setup_database turns that " @@ -359,7 +359,7 @@ def warnings_naming_the_path() -> list[str]: with caplog.at_level(logging.WARNING): for attempt in (1, 2): - drain_transition_queue(session) + drain_transition_queue() session.commit() session.expire_all() assert hash_mode_state.pending_transition_count() == 1, ( @@ -375,7 +375,7 @@ def warnings_naming_the_path() -> list[str]: f"attempt {attempt} is a retry, not a terminal outcome; it must stay quiet" ) - drain_transition_queue(session) + drain_transition_queue() session.commit() session.expire_all() @@ -428,7 +428,7 @@ def denied_for_the_unreadable_path(candidate_path: str): enqueue_transition_work(session, transition) session.commit() for _ in range(3): - drain_transition_queue(session) + drain_transition_queue() session.commit() session.expire_all() @@ -477,7 +477,7 @@ def preflight_then_race(entry_path: str): with caplog.at_level(logging.WARNING): for _ in range(_MAX_DRAIN_TICKS): - drain_transition_queue(session) + drain_transition_queue() session.commit() session.expire_all() if hash_mode_state.pending_transition_count() == 0: diff --git a/tests-unit/assets_test/services/test_write_lock_hold_barriers.py b/tests-unit/assets_test/services/test_write_lock_hold_barriers.py index 32ca0731419..38c9b6d6143 100644 --- a/tests-unit/assets_test/services/test_write_lock_hold_barriers.py +++ b/tests-unit/assets_test/services/test_write_lock_hold_barriers.py @@ -106,7 +106,6 @@ def tracked_work(session): monkeypatch.setattr(ingest, "run_write_txn", track_write_transaction) for name in ( "lookup_for_view", - "refresh_qualified_content", "_file_signature", "_file_signature_matches", ): diff --git a/tests-unit/assets_test/test_intended_behaviour.py b/tests-unit/assets_test/test_intended_behaviour.py index dd4a7a36ff9..dcf29c3be7d 100644 --- a/tests-unit/assets_test/test_intended_behaviour.py +++ b/tests-unit/assets_test/test_intended_behaviour.py @@ -32,7 +32,6 @@ build_asset_specs, seed_asset_specs, stat_seed_specs, - sync_prefixes_with_filesystem, ) from app.assets.scanner_admission import _should_skip_extension from app.assets.scanner_changes import ( @@ -49,10 +48,10 @@ from app.assets.services.file_utils import list_files_recursively from app.assets.services.ingest import register_cached_output, upload_from_temp_path from app.assets.services.lookup import ( - lookup_for_from_hash, lookup_for_view, ) from app.assets.services.snapshot_hash import snapshot_hash +from assets_test.helpers import sync_prefixes_in_session @pytest.fixture @@ -109,7 +108,7 @@ def _seed_content_row(session, path: Path, hash_value: str | None = None): def _scan_pass(session, root: Path) -> int: - survivors = sync_prefixes_with_filesystem( + survivors = sync_prefixes_in_session( session, [str(root)], collect_existing_paths=True ) specs, _tag_pool, _skipped = build_asset_specs( @@ -234,8 +233,8 @@ def test_scenario_6_upload_reuses_content_never_the_record(session, tmp_path): still share one ``AssetContent`` row, in BOTH modes — uploads hash unconditionally (``upload_from_temp_path`` calls ``_snapshot_hash_with_retry`` before it consults anything) and - ``lookup_for_view`` never asks ``mode.hashing_enabled``, unlike - ``lookup_for_from_hash``. What changed is record identity: a re-upload is a + ``lookup_for_view`` never asks ``mode.hashing_enabled``. What changed is + record identity: a re-upload is a new delivery, so it gets a new record carrying the attributes THAT request supplied, instead of silently handing back an older record that never saw them. @@ -520,7 +519,7 @@ def test_scenario_18_edit_during_hash_discard(session, tmp_path): queue_pending_verification(content.id) with _writer_lands_mid_hash(path, b"a-concurrent-writer-was-here"): assert snapshot_hash(str(path)) is None - assert drain_pending_verifications(session) == 0 + assert drain_pending_verifications() == 0 assert content.hash == committed_hash assert content.is_missing is False @@ -528,7 +527,7 @@ def test_scenario_18_edit_during_hash_discard(session, tmp_path): assert [row.id for row in session.scalars(select(AssetContent))] == [content.id] path.write_bytes(committed) - assert drain_pending_verifications(session) == 1 + assert drain_pending_verifications() == 1 finally: clear_pending_verifications() @@ -688,7 +687,7 @@ def previews() -> dict[str, str]: def from_hash(): with patch.object(mode, "hashing_enabled", return_value=True): - return lookup_for_from_hash(session, digest) + return lookup_for_view(session, digest) def serving() -> tuple[bool, object]: with patch( diff --git a/tests-unit/assets_test/test_scanner_ingest_event_log.py b/tests-unit/assets_test/test_scanner_ingest_event_log.py index cf4aafc0b97..06f150654be 100644 --- a/tests-unit/assets_test/test_scanner_ingest_event_log.py +++ b/tests-unit/assets_test/test_scanner_ingest_event_log.py @@ -138,7 +138,11 @@ def deny_stat(*_args, **_kwargs): monkeypatch.setattr(scanner, "os", SimpleNamespace(stat=deny_stat, path=scanner.os.path)) monkeypatch.setattr(scanner, "live_contents_under_prefixes", lambda _session, _prefixes: [content]) - scanner.sync_prefixes_with_filesystem(Mock(), ["/private/assets"], progress=progress) + diagnostics: list = [] + scanner.observe_references_on_filesystem( + ["/private/assets"], diagnostics=diagnostics, session=Mock() + ) + scanner._publish_reference_diagnostics(diagnostics, progress) assert progress.permission_denied == 1 From 848defa03e823f5371318f30e9b86196ebd33556 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Fri, 18 Sep 2026 01:51:07 -0700 Subject: [PATCH 099/111] Attach the seeder spy the disable regression test asserts on --- tests-unit/assets_test/test_event_log_sites.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests-unit/assets_test/test_event_log_sites.py b/tests-unit/assets_test/test_event_log_sites.py index c7d0a0bd779..753e0f52db8 100644 --- a/tests-unit/assets_test/test_event_log_sites.py +++ b/tests-unit/assets_test/test_event_log_sites.py @@ -108,11 +108,13 @@ def test_disable_actually_stops_scanning_and_ingest_not_just_http() -> None: from app.assets.api import routes from app.assets.manager import AssetsEnabled - started: list[str] = [] + started: list[tuple] = [] mgr = AssetsEnabled(SimpleNamespace(enable_assets=True, enable_asset_hashing=False)) routes._ASSETS_ENABLED = True seeder = manager_mod.asset_seeder was_disabled = seeder.is_disabled() + real_start = seeder.start + seeder.start = lambda *a, **kw: started.append((a, kw)) try: mgr.disable(RuntimeError("database is locked")) @@ -120,10 +122,11 @@ def test_disable_actually_stops_scanning_and_ingest_not_just_http() -> None: "queue_output_scan's existing gate reads the seeder, so disable() must arm it" ) mgr.ensure_scan_started() - assert started == [] + assert started == [], "a disabled manager must not start the seeder" assert mgr.register_upload("/tmp/x.png", "x", "input", "", content_written=True) is None assert mgr.register_executed_output("/tmp/x.png", "job-1") is None assert mgr.register_cached_output("/tmp/x.png", "job-1") is None finally: + seeder.start = real_start seeder._disabled = was_disabled routes._ASSETS_ENABLED = False From 7be2922c57cb01d4b678550e45d13954281fe0c3 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Fri, 18 Sep 2026 10:48:47 -0700 Subject: [PATCH 100/111] Remove tests that pass against the behaviour they claim to cover --- ...est_scanner_retry_and_counter_integrity.py | 43 ----------------- .../services/test_scanner_sync_chunking.py | 46 ------------------- tests-unit/assets_test/test_event_log.py | 7 --- 3 files changed, 96 deletions(-) diff --git a/tests-unit/assets_test/services/test_scanner_retry_and_counter_integrity.py b/tests-unit/assets_test/services/test_scanner_retry_and_counter_integrity.py index 8e58dc8086b..84d0d101a54 100644 --- a/tests-unit/assets_test/services/test_scanner_retry_and_counter_integrity.py +++ b/tests-unit/assets_test/services/test_scanner_retry_and_counter_integrity.py @@ -1,8 +1,6 @@ -import logging import os import sqlite3 from pathlib import Path -from types import SimpleNamespace from unittest.mock import patch import pytest @@ -127,47 +125,6 @@ class _HashingOn: scanner_changes.clear_pending_verifications() -def test_scanner_sync_permission_diagnostic_published_exactly_once_after_retry( - db_engine, tmp_path: Path, monkeypatch, session, caplog -): - """A commit-time locked failure that later succeeds must publish the - permission-denied diagnostic exactly once: one counter bump, one emit.""" - path = tmp_path / "unreadable.bin" - path.write_bytes(b"unreadable") - content = AssetContent( - path=str(path), hash=None, size_bytes=path.stat().st_size, mtime_ns=path.stat().st_mtime_ns - ) - session.add(content) - session.flush() - session.commit() - - real_stat = os.stat - - def deny_stat(candidate_path, *args, **kwargs): - if str(candidate_path) == str(path): - raise PermissionError(str(path)) - return real_stat(candidate_path, *args, **kwargs) - - monkeypatch.setattr(scanner, "os", SimpleNamespace(stat=deny_stat, path=scanner.os.path)) - monkeypatch.setattr(db_mod, "WriteSession", _fail_commit_once_then_succeed(db_engine)) - - progress = _ScanState() - with ( - patch("folder_paths.get_input_directory", return_value=str(tmp_path)), - caplog.at_level(logging.INFO), - ): - scanner.sync_root_safely("input", progress) - - assert progress.permission_denied == 1 - stat_failed_lines = [ - r.getMessage() - for r in caplog.records - if r.getMessage().startswith("[assets-event] scanner.stat_failed") - ] - assert stat_failed_lines == [ - "[assets-event] scanner.stat_failed error_type=PermissionError site=reference_stat" - ] - def test_enrichment_batch_failure_rolls_back_every_row_and_counts_each_failure( tmp_path: Path, session, monkeypatch diff --git a/tests-unit/assets_test/services/test_scanner_sync_chunking.py b/tests-unit/assets_test/services/test_scanner_sync_chunking.py index a06a2d02dc6..9b7498003a5 100644 --- a/tests-unit/assets_test/services/test_scanner_sync_chunking.py +++ b/tests-unit/assets_test/services/test_scanner_sync_chunking.py @@ -6,7 +6,6 @@ from app.assets import scanner from app.assets.database.models import AssetContent from app.assets.database.queries import create_content, create_record, mark_content_missing -from assets_test.helpers import sync_prefixes_in_session def _seed_reference_observations( @@ -115,51 +114,6 @@ def record(work): return transactions -def test_root_sync_matches_single_transaction_and_uses_three_bounded_writes( - tmp_path: Path, monkeypatch, session -) -> None: - content_ids, observations, expected_survivors = _seed_reference_observations( - tmp_path, 60 - ) - - baseline_survivors = db_mod.run_write_txn( - lambda session: sync_prefixes_in_session( - session, - [str(tmp_path)], - collect_existing_paths=True, - ) - ) - baseline_states = _content_states(content_ids) - _reset_observed_rows(content_ids, observations) - - monkeypatch.setattr( - scanner, - "observe_references_on_filesystem", - lambda *_args, **_kwargs: (observations, expected_survivors), - ) - transactions = _record_transactions(monkeypatch) - - survivors = scanner.sync_root_safely("input") - - assert len(transactions) == 3 - assert survivors == baseline_survivors == expected_survivors - assert _content_states(content_ids) == baseline_states - - -def test_temp_sync_uses_bounded_write_transactions( - tmp_path: Path, monkeypatch, session -) -> None: - _content_ids, observations, survivors = _seed_reference_observations(tmp_path, 30) - monkeypatch.setattr( - scanner, - "observe_references_on_filesystem", - lambda *_args, **_kwargs: (observations, survivors), - ) - transactions = _record_transactions(monkeypatch) - - scanner.sync_temp_references_safely() - - assert len(transactions) == 2 def test_root_sync_interrupts_between_chunks_and_publishes_committed_ids( diff --git a/tests-unit/assets_test/test_event_log.py b/tests-unit/assets_test/test_event_log.py index 0dee93481fb..f3b3da12039 100644 --- a/tests-unit/assets_test/test_event_log.py +++ b/tests-unit/assets_test/test_event_log.py @@ -93,13 +93,6 @@ def go_to_production_mode(monkeypatch: pytest.MonkeyPatch) -> None: # --- the shared cross-repo fixture ------------------------------------------------- -def test_shared_fixture_file_holds_five_newline_terminated_lines(): - raw = FIXTURE_PATH.read_text(encoding="utf-8") - - assert raw.endswith("\n") - assert len(raw.splitlines()) == 5 - - @pytest.mark.parametrize("line", fixture_lines()) def test_emit_reproduces_each_shared_fixture_line_byte_for_byte(caplog, line): """Given a canonical line, When its fields are re-emitted, Then the bytes match.""" From 9c2b9fee68260a056514488f65ee0b8f44de4037 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Fri, 18 Sep 2026 11:00:43 -0700 Subject: [PATCH 101/111] Drop the write-lock barrier suite that could not observe filesystem access --- .../services/test_ingest_write_txn.py | 167 +---- .../services/test_write_lock_hold_barriers.py | 647 ------------------ 2 files changed, 14 insertions(+), 800 deletions(-) delete mode 100644 tests-unit/assets_test/services/test_write_lock_hold_barriers.py diff --git a/tests-unit/assets_test/services/test_ingest_write_txn.py b/tests-unit/assets_test/services/test_ingest_write_txn.py index 6d7e94335a1..747d89faf38 100644 --- a/tests-unit/assets_test/services/test_ingest_write_txn.py +++ b/tests-unit/assets_test/services/test_ingest_write_txn.py @@ -4,7 +4,6 @@ import logging import sqlite3 import threading -import time from types import SimpleNamespace from unittest.mock import AsyncMock @@ -69,12 +68,16 @@ def fake(*args, **kwargs): return fake -class _FilesystemInsideWriteTxn(BaseException): +class _GuardedHelperInsideWriteTxn(BaseException): """Barrier trip. Derives from BaseException so ``except Exception`` cannot eat it.""" -class _WriteTxnFsBarrier: - """Runs write-transaction callables and fails if one of them touches the disk.""" +class _StubbedWriteTxn: + """Runs write-transaction callables without a database. + + The guards below trip only on the three named helpers, so this is a stub that + happens to assert, not proof that a callable left the filesystem alone. + """ def __init__(self) -> None: self.depth = 0 @@ -89,7 +92,7 @@ def run_write_txn(self, work): def _guard(self, label: str, real): def guarded(*args, **kwargs): if self.depth: - raise _FilesystemInsideWriteTxn(f"{label} ran inside run_write_txn") + raise _GuardedHelperInsideWriteTxn(f"{label} ran inside run_write_txn") return real(*args, **kwargs) return guarded @@ -106,14 +109,14 @@ def install(self, monkeypatch) -> None: @pytest.fixture -def write_txn_fs_barrier(monkeypatch) -> _WriteTxnFsBarrier: - barrier = _WriteTxnFsBarrier() +def stubbed_write_txn(monkeypatch) -> _StubbedWriteTxn: + barrier = _StubbedWriteTxn() barrier.install(monkeypatch) return barrier def test_reused_upload_refuses_after_four_stale_preflights( - write_txn_fs_barrier, monkeypatch + stubbed_write_txn, monkeypatch ) -> None: preflight = SimpleNamespace(signature=object()) prepared = object() @@ -137,7 +140,7 @@ def stale_apply(_session, observed_prepared): def test_settle_destination_refuses_after_four_stale_preflights( - write_txn_fs_barrier, monkeypatch + stubbed_write_txn, monkeypatch ) -> None: preflight = SimpleNamespace(signature=object()) prepared = SimpleNamespace(facts=None) @@ -160,7 +163,7 @@ def stale_apply(_session, observed_prepared): def test_cached_registration_gives_up_after_four_stale_preflights( - write_txn_fs_barrier, monkeypatch, caplog + stubbed_write_txn, monkeypatch, caplog ) -> None: path = _output_path("cached-always-stale.bin") with open(path, "wb") as file: @@ -209,7 +212,7 @@ def stale_apply(_session, _preflight, *_args): @pytest.mark.asyncio async def test_upload_route_reports_an_unsettleable_destination_as_unstable( - write_txn_fs_barrier, monkeypatch, tmp_path + stubbed_write_txn, monkeypatch, tmp_path ) -> None: temp_path = tmp_path / "unstable-destination.bin" temp_path.write_bytes(b"upload bytes") @@ -1203,149 +1206,7 @@ def rewrite_then_assert(signature): os.unlink(path) -def test_cached_registration_metadata_extraction_does_not_hold_the_write_lock( - file_database, monkeypatch -) -> None: - path = _output_path("cached-barrier-no-sibling.bin") - with open(path, "wb") as file: - file.write(b"cached barrier bytes") - try: - with db_mod.WriteSession() as session: - content = create_content(session, path, size_bytes=os.path.getsize(path)) - session.commit() - content_id = content.id - - real_extract = ingest._extract_system_metadata_sync - entered = threading.Event() - release = threading.Event() - monkeypatch.setattr( - ingest, - "_extract_system_metadata_sync", - _blocking_fake(entered, release, real_extract), - ) - - result: dict[str, object] = {} - - def _register() -> None: - result["registered"] = ingest.register_cached_output(path) - - worker = threading.Thread(target=_register) - worker.start() - try: - assert entered.wait(timeout=_BARRIER_TIMEOUT) - started = time.monotonic() - _probe_write() - elapsed = time.monotonic() - started - finally: - release.set() - worker.join(timeout=_BARRIER_TIMEOUT) - assert not worker.is_alive() - - assert elapsed < _PROBE_BUDGET_SECONDS - registered = result["registered"] - assert registered is not None - assert registered.content_id == content_id - with db_mod.Session() as session: - record = session.get(ingest.Asset, registered.id) - assert record is not None - assert record.content_id == content_id - finally: - os.unlink(path) - - -def test_upload_settle_hashing_does_not_hold_the_write_lock( - file_database, monkeypatch -) -> None: - path = _output_path("settle-barrier.bin") - with open(path, "wb") as file: - file.write(b"incumbent barrier bytes") - try: - stat = os.stat(path) - with db_mod.WriteSession() as session: - content = create_content( - session, path, size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns - ) - session.commit() - content_id = content.id - real_snapshot_hash = ingest.snapshot_hash - entered = threading.Event() - release = threading.Event() - monkeypatch.setattr( - ingest, - "snapshot_hash", - _blocking_fake(entered, release, real_snapshot_hash), - ) - - def _settle() -> None: - ingest._settle_destination_before_write(path) - - worker = threading.Thread(target=_settle) - worker.start() - try: - assert entered.wait(timeout=_BARRIER_TIMEOUT) - started = time.monotonic() - _probe_write() - elapsed = time.monotonic() - started - finally: - release.set() - worker.join(timeout=_BARRIER_TIMEOUT) - assert not worker.is_alive() - - assert elapsed < _PROBE_BUDGET_SECONDS - - expected_digest, _expected_stat = real_snapshot_hash(path) - with db_mod.Session() as session: - settled = session.get(ingest.AssetContent, content_id) - assert settled is not None - assert settled.hash == ingest.to_stored_hash(expected_digest) - assert settled.is_missing is False - finally: - os.unlink(path) - - -def test_create_record_metadata_extraction_does_not_hold_the_write_lock( - file_database, monkeypatch -) -> None: - path = _output_path("create-record-barrier.bin") - with open(path, "wb") as file: - file.write(b"brand new record bytes") - try: - real_extract = ingest._extract_system_metadata_sync - entered = threading.Event() - release = threading.Event() - monkeypatch.setattr( - ingest, - "_extract_system_metadata_sync", - _blocking_fake(entered, release, real_extract), - ) - - result: dict[str, object] = {} - - def _register() -> None: - result["upload"] = ingest.register_file_in_place( - path, "create-record-barrier.bin", ["output"] - ) - - worker = threading.Thread(target=_register) - worker.start() - try: - assert entered.wait(timeout=_BARRIER_TIMEOUT) - started = time.monotonic() - _probe_write() - elapsed = time.monotonic() - started - finally: - release.set() - worker.join(timeout=_BARRIER_TIMEOUT) - assert not worker.is_alive() - - assert elapsed < _PROBE_BUDGET_SECONDS - upload_result = result["upload"] - assert upload_result is not None - assert upload_result.created_new is True - assert upload_result.ref.file_path == path - finally: - os.unlink(path) def test_unsettled_new_upload_persists_nothing_rather_than_mixing_facts( diff --git a/tests-unit/assets_test/services/test_write_lock_hold_barriers.py b/tests-unit/assets_test/services/test_write_lock_hold_barriers.py deleted file mode 100644 index 38c9b6d6143..00000000000 --- a/tests-unit/assets_test/services/test_write_lock_hold_barriers.py +++ /dev/null @@ -1,647 +0,0 @@ -import os -import threading -import uuid - -import folder_paths -import pytest -from sqlalchemy import text -from sqlalchemy.exc import OperationalError - -import app.assets.mode as mode_module -import app.database.db as db_mod -from app.assets import scanner -from app.assets import scanner_changes -from app.assets.database.queries.records import create_content, create_record -from app.assets.services import hash_mode_state -from app.assets.services import ingest - -_BARRIER_TIMEOUT = 5 -_PROBE_LOCK_DEADLINE_SECONDS = 0.5 -_PROBE_BUSY_TIMEOUT_MS = 250 -_LEASE_HELD = "writer lease was held across out-of-transaction work" - - -@pytest.fixture -def file_database(tmp_path, monkeypatch): - database_path = str(tmp_path / "assets.db") - monkeypatch.setattr(db_mod.args, "enable_assets", True) - monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") - monkeypatch.setattr(db_mod, "Session", None) - monkeypatch.setattr(db_mod, "_db_lock", None) - if hasattr(db_mod, "WriteSession"): - monkeypatch.setattr(db_mod, "WriteSession", None) - db_mod.init_db() - yield database_path - for factory in (db_mod.Session, getattr(db_mod, "WriteSession", None)): - if factory is not None: - factory.kw["bind"].dispose() - db_mod._db_lock.release(force=True) - - -@pytest.fixture -def hashing_on(): - class FakeArgs: - enable_asset_hashing = True - - mode_module.init(FakeArgs()) - yield - mode_module.init(None) - - -def _probe_write() -> None: - name = f"probe-{uuid.uuid4().hex}" - db_mod.run_write_txn( - lambda session: session.execute( - text("INSERT INTO tags (name) VALUES (:name)"), {"name": name} - ) - ) - - -@pytest.fixture -def impatient_probe(monkeypatch): - """Let a probe surface a held lease as an error instead of waiting out the real deadline. - - An unheld lease is acquired on the first attempt, so these shortened deadlines are - only ever reached when the lease really is held. - """ - monkeypatch.setattr( - db_mod, "_WRITE_TXN_LOCK_RETRY_DEADLINE_SECONDS", _PROBE_LOCK_DEADLINE_SECONDS - ) - monkeypatch.setattr(db_mod, "_SQLITE_BUSY_TIMEOUT_MS", _PROBE_BUSY_TIMEOUT_MS) - - -def _probe_write_outcome() -> Exception | None: - try: - _probe_write() - except OperationalError as exc: - return exc - return None - - -def _blocking_fake(entered: threading.Event, release: threading.Event, real_fn): - def fake(*args, **kwargs): - entered.set() - release.wait() - return real_fn(*args, **kwargs) - - return fake - - -class _IngestWriteTxnFsTracker: - def __init__(self, monkeypatch) -> None: - self.calls: list[str] = [] - self.inside = False - real_run_write_txn = ingest.run_write_txn - - def track_write_transaction(work): - def tracked_work(session): - self.inside = True - try: - return work(session) - finally: - self.inside = False - - return real_run_write_txn(tracked_work) - - monkeypatch.setattr(ingest, "run_write_txn", track_write_transaction) - for name in ( - "lookup_for_view", - "_file_signature", - "_file_signature_matches", - ): - if hasattr(ingest, name): - real = getattr(ingest, name) - monkeypatch.setattr(ingest, name, self._track(name, real)) - monkeypatch.setattr(ingest.os, "stat", self._track("os.stat", os.stat)) - monkeypatch.setattr( - ingest.os.path, - "isfile", - self._track("os.path.isfile", os.path.isfile), - ) - - def _track(self, label: str, real): - def tracked(*args, **kwargs): - if self.inside: - self.calls.append(label) - return real(*args, **kwargs) - - return tracked - - -@pytest.fixture -def ingest_write_txn_fs_tracker(monkeypatch) -> _IngestWriteTxnFsTracker: - return _IngestWriteTxnFsTracker(monkeypatch) - - -def test_reused_upload_does_not_touch_the_filesystem_inside_write_transaction( - file_database, tmp_path, ingest_write_txn_fs_tracker -) -> None: - existing_path = tmp_path / "reuse-existing.bin" - existing_path.write_bytes(b"shared bytes") - stat_result = existing_path.stat() - digest, _ = ingest._snapshot_hash_with_retry(str(existing_path)) - stored_hash = ingest.to_stored_hash(digest) - - db_mod.run_write_txn( - lambda session: create_content( - session, - str(existing_path), - hash=stored_hash, - size_bytes=stat_result.st_size, - mtime_ns=stat_result.st_mtime_ns, - ) - ) - upload_path = tmp_path / "reuse-upload.part" - upload_path.write_bytes(b"shared bytes") - - result = ingest.upload_from_temp_path(str(upload_path), name="reuse.bin") - - assert result.content_id is not None - assert ingest_write_txn_fs_tracker.calls == [] - - -def test_new_upload_does_not_touch_the_filesystem_inside_write_transaction( - file_database, tmp_path, monkeypatch, ingest_write_txn_fs_tracker -) -> None: - upload_path = tmp_path / "new-upload.part" - upload_path.write_bytes(b"new bytes") - destination = tmp_path / "new-upload.bin" - monkeypatch.setattr( - ingest, - "_hash_mode_dest_path", - lambda *_args: str(destination), - ) - - result = ingest.upload_from_temp_path( - str(upload_path), - name="new-upload.bin", - tags=["output"], - ) - - assert result.ref.file_path == str(destination) - assert ingest_write_txn_fs_tracker.calls == [] - - -def test_cached_registration_does_not_touch_the_filesystem_inside_write_transaction( - file_database, ingest_write_txn_fs_tracker -) -> None: - path = os.path.join(folder_paths.get_output_directory(), "cached-output.bin") - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "wb") as file: - file.write(b"cached bytes") - stat_result = os.stat(path) - db_mod.run_write_txn( - lambda session: create_content( - session, - path, - size_bytes=stat_result.st_size, - mtime_ns=stat_result.st_mtime_ns, - ) - ) - - try: - result = ingest.register_cached_output(path) - finally: - os.unlink(path) - - assert result is not None - assert ingest_write_txn_fs_tracker.calls == [] - - -def test_settle_destination_does_not_touch_the_filesystem_inside_write_transaction( - file_database, tmp_path, ingest_write_txn_fs_tracker -) -> None: - path = tmp_path / "settle-output.bin" - path.write_bytes(b"incumbent bytes") - stat_result = path.stat() - db_mod.run_write_txn( - lambda session: create_content( - session, - str(path), - size_bytes=stat_result.st_size, - mtime_ns=stat_result.st_mtime_ns, - ) - ) - - ingest._settle_destination_before_write(str(path)) - - assert ingest_write_txn_fs_tracker.calls == [] - - -def test_register_file_in_place_does_not_touch_the_filesystem_inside_write_transaction( - file_database, tmp_path, ingest_write_txn_fs_tracker -) -> None: - path = tmp_path / "register-in-place.bin" - path.write_bytes(b"in-place bytes") - - result = ingest.register_file_in_place(str(path), path.name, ["output"]) - - assert result.ref.file_path == str(path) - assert ingest_write_txn_fs_tracker.calls == [] - - -def test_seed_recovery_hashing_does_not_hold_the_write_lock( - file_database, impatient_probe, hashing_on, tmp_path, monkeypatch -): - path = tmp_path / "recoverable.bin" - path.write_bytes(b"recoverable bytes") - - entered = threading.Event() - release = threading.Event() - monkeypatch.setattr( - scanner_changes, - "snapshot_hash", - _blocking_fake(entered, release, scanner_changes.snapshot_hash), - ) - - stat = path.stat() - spec: scanner.SeedAssetSpec = { - "abs_path": str(path), - "size_bytes": stat.st_size, - "mtime_ns": stat.st_mtime_ns, - "info_name": "recoverable.bin", - "tags": [], - "fname": None, - "metadata": None, - "mime_type": None, - "job_id": None, - } - - result: dict[str, int] = {} - - def _seed() -> None: - result["created"] = scanner.insert_asset_specs([spec], set()) - - worker = threading.Thread(target=_seed) - worker.start() - try: - assert entered.wait(timeout=_BARRIER_TIMEOUT) - probe_error = _probe_write_outcome() - finally: - release.set() - worker.join(timeout=_BARRIER_TIMEOUT) - assert not worker.is_alive() - - assert probe_error is None, _LEASE_HELD - assert result["created"] == 1 - - -def test_pending_verification_hashing_does_not_hold_the_write_lock( - file_database, impatient_probe, tmp_path, monkeypatch -): - path = tmp_path / "verify-me.bin" - path.write_bytes(b"verify me") - stat = path.stat() - - def _seed(session): - content = create_content( - session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns - ) - create_record(session, content.id, "verify-me.bin") - return content.id - - content_id = db_mod.run_write_txn(_seed) - - scanner_changes.clear_pending_verifications() - scanner_changes.queue_pending_verification(content_id) - - entered = threading.Event() - release = threading.Event() - monkeypatch.setattr( - scanner_changes, - "snapshot_hash", - _blocking_fake(entered, release, scanner_changes.snapshot_hash), - ) - - result: dict[str, int] = {} - - def _drain() -> None: - result["processed"] = scanner_changes.drain_pending_verifications() - - worker = threading.Thread(target=_drain) - worker.start() - try: - assert entered.wait(timeout=_BARRIER_TIMEOUT) - probe_error = _probe_write_outcome() - finally: - release.set() - worker.join(timeout=_BARRIER_TIMEOUT) - assert not worker.is_alive() - scanner_changes.clear_pending_verifications() - - assert probe_error is None, _LEASE_HELD - assert result["processed"] == 1 - - -def test_transition_hashing_does_not_hold_the_write_lock( - file_database, impatient_probe, tmp_path, monkeypatch -): - path = tmp_path / "transition-me.bin" - path.write_bytes(b"transition me") - stat = path.stat() - - db_mod.run_write_txn( - lambda session: create_content( - session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns - ) - ) - - hash_mode_state.clear_transition_queue() - hash_mode_state._PENDING_QUEUE.append(hash_mode_state._PendingEntry(str(path))) - hash_mode_state._PENDING_PATHS.add(str(path)) - - entered = threading.Event() - release = threading.Event() - monkeypatch.setattr( - hash_mode_state, - "snapshot_hash", - _blocking_fake(entered, release, hash_mode_state.snapshot_hash), - ) - - result: dict[str, bool] = {} - - def _drain() -> None: - hash_mode_state.drain_transition_queue() - result["done"] = True - - worker = threading.Thread(target=_drain) - worker.start() - try: - assert entered.wait(timeout=_BARRIER_TIMEOUT) - probe_error = _probe_write_outcome() - finally: - release.set() - worker.join(timeout=_BARRIER_TIMEOUT) - assert not worker.is_alive() - hash_mode_state.clear_transition_queue() - - assert probe_error is None, _LEASE_HELD - assert result.get("done") is True - - -def test_enrichment_hashing_does_not_hold_the_write_lock( - file_database, impatient_probe, tmp_path, monkeypatch -): - path = tmp_path / "enrich-hash.bin" - path.write_bytes(b"enrich me via hash") - stat = path.stat() - - def _seed(session): - content = create_content( - session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns - ) - record = create_record(session, content.id, "enrich-hash.bin") - return content.id, record.id - - content_id, record_id = db_mod.run_write_txn(_seed) - - row = scanner.UnenrichedContent( - content_id, - record_id, - str(path), - True, - observed_size_bytes=stat.st_size, - observed_mtime_ns=stat.st_mtime_ns, - ) - - entered = threading.Event() - release = threading.Event() - monkeypatch.setattr( - scanner, - "snapshot_hash", - _blocking_fake(entered, release, scanner.snapshot_hash), - ) - - result: dict[str, object] = {} - - def _enrich() -> None: - result["outcome"] = scanner.enrich_assets_batch( - [row], extract_metadata=False, compute_hash=True - ) - - worker = threading.Thread(target=_enrich) - worker.start() - try: - assert entered.wait(timeout=_BARRIER_TIMEOUT) - probe_error = _probe_write_outcome() - finally: - release.set() - worker.join(timeout=_BARRIER_TIMEOUT) - assert not worker.is_alive() - - assert probe_error is None, _LEASE_HELD - assert result["outcome"] == (1, []) - - -def test_enrichment_metadata_extraction_does_not_hold_the_write_lock( - file_database, impatient_probe, tmp_path, monkeypatch -): - path = tmp_path / "enrich-metadata.bin" - path.write_bytes(b"enrich me via metadata") - stat = path.stat() - - def _seed(session): - content = create_content( - session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns - ) - record = create_record(session, content.id, "enrich-metadata.bin") - return content.id, record.id - - content_id, record_id = db_mod.run_write_txn(_seed) - - row = scanner.UnenrichedContent( - content_id, - record_id, - str(path), - False, - observed_size_bytes=stat.st_size, - observed_mtime_ns=stat.st_mtime_ns, - ) - - entered = threading.Event() - release = threading.Event() - monkeypatch.setattr( - scanner, - "extract_file_metadata", - _blocking_fake(entered, release, scanner.extract_file_metadata), - ) - - result: dict[str, object] = {} - - def _enrich() -> None: - result["outcome"] = scanner.enrich_assets_batch( - [row], extract_metadata=True, compute_hash=False - ) - - worker = threading.Thread(target=_enrich) - worker.start() - try: - assert entered.wait(timeout=_BARRIER_TIMEOUT) - probe_error = _probe_write_outcome() - finally: - release.set() - worker.join(timeout=_BARRIER_TIMEOUT) - assert not worker.is_alive() - - assert probe_error is None, _LEASE_HELD - assert result["outcome"] == (1, []) - - -def test_enrichment_apply_does_not_stat_inside_write_transaction( - file_database, tmp_path, monkeypatch -): - path = tmp_path / "enrich-without-in-transaction-stat.bin" - path.write_bytes(b"prepared before the write") - stat_result = path.stat() - - def seed(session): - content = create_content( - session, - str(path), - size_bytes=stat_result.st_size, - mtime_ns=stat_result.st_mtime_ns, - ) - record = create_record(session, content.id, path.name) - return content.id, record.id - - content_id, record_id = db_mod.run_write_txn(seed) - row = scanner.UnenrichedContent( - content_id, - record_id, - str(path), - observed_size_bytes=stat_result.st_size, - observed_mtime_ns=stat_result.st_mtime_ns, - ) - real_run_write_txn = scanner.run_write_txn - real_stat = scanner.os.stat - inside = False - - def track_write_transaction(work): - def tracked_work(session): - nonlocal inside - inside = True - try: - return work(session) - finally: - inside = False - - return real_run_write_txn(tracked_work) - - def reject_in_transaction_stat(*args, **kwargs): - if inside: - raise AssertionError("enrichment stat ran inside the write transaction") - return real_stat(*args, **kwargs) - - monkeypatch.setattr(scanner, "run_write_txn", track_write_transaction) - monkeypatch.setattr(scanner.os, "stat", reject_in_transaction_stat) - - result = scanner.enrich_assets_batch( - [row], extract_metadata=True, compute_hash=False - ) - - assert result == (1, []) - - -def test_scanner_reference_stat_walk_does_not_hold_the_write_lock( - file_database, impatient_probe, tmp_path, monkeypatch -): - root = tmp_path / "models" - root.mkdir() - path = root / "catalogued.bin" - path.write_bytes(b"catalogued bytes") - stat = path.stat() - - def _seed(session): - content = create_content( - session, str(path), size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns - ) - create_record(session, content.id, "catalogued.bin") - - db_mod.run_write_txn(_seed) - - entered = threading.Event() - release = threading.Event() - real_stat = scanner.os.stat - - def blocking_stat(target, *args, **kwargs): - if str(target) == str(path): - entered.set() - assert release.wait(timeout=_BARRIER_TIMEOUT) - return real_stat(target, *args, **kwargs) - - monkeypatch.setattr(scanner, "get_scan_prefixes_for_root", lambda _root: [str(root)]) - monkeypatch.setattr(scanner.os, "stat", blocking_stat) - - survivors: dict[str, set[str]] = {} - - def _scan() -> None: - survivors["found"] = scanner.sync_root_safely("models") - - worker = threading.Thread(target=_scan) - worker.start() - try: - assert entered.wait(timeout=_BARRIER_TIMEOUT) - probe_error = _probe_write_outcome() - finally: - release.set() - worker.join(timeout=_BARRIER_TIMEOUT) - assert not worker.is_alive() - - assert probe_error is None, _LEASE_HELD - assert survivors["found"] == {str(path)} - - -def test_download_hash_resolution_does_not_hold_the_write_lock( - file_database, impatient_probe, tmp_path, monkeypatch -): - from app.assets.services import asset_management, lookup - - path = tmp_path / "servable.bin" - path.write_bytes(b"servable bytes") - stat = path.stat() - digest = "b" * 64 - stored_hash = f"blake3:{digest}" - - def _seed(session): - content = create_content( - session, - str(path), - hash=stored_hash, - size_bytes=stat.st_size, - mtime_ns=stat.st_mtime_ns, - ) - create_record(session, content.id, "servable.bin") - - db_mod.run_write_txn(_seed) - - entered = threading.Event() - release = threading.Event() - real_stat = lookup.os.stat - - def blocking_stat(target, *args, **kwargs): - if str(target) == str(path): - entered.set() - assert release.wait(timeout=_BARRIER_TIMEOUT) - return real_stat(target, *args, **kwargs) - - monkeypatch.setattr(lookup.os, "stat", blocking_stat) - monkeypatch.setattr(lookup, "is_temp_path", lambda _path: False) - - resolved: dict[str, object] = {} - - def _resolve() -> None: - resolved["result"] = asset_management.resolve_hash_to_path(stored_hash) - - worker = threading.Thread(target=_resolve) - worker.start() - try: - assert entered.wait(timeout=_BARRIER_TIMEOUT) - probe_error = _probe_write_outcome() - finally: - release.set() - worker.join(timeout=_BARRIER_TIMEOUT) - assert not worker.is_alive() - - assert probe_error is None, _LEASE_HELD - assert resolved["result"] is not None - assert resolved["result"].abs_path == str(path) From 104a98c76ded5b99e32b03d327d8eb1e2e9fe386 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Fri, 18 Sep 2026 11:14:11 -0700 Subject: [PATCH 102/111] Parametrise the ingest refusal and registration-failure contracts --- .../services/test_ingest_write_txn.py | 211 +++++++++--------- 1 file changed, 109 insertions(+), 102 deletions(-) diff --git a/tests-unit/assets_test/services/test_ingest_write_txn.py b/tests-unit/assets_test/services/test_ingest_write_txn.py index 747d89faf38..5e9d4339455 100644 --- a/tests-unit/assets_test/services/test_ingest_write_txn.py +++ b/tests-unit/assets_test/services/test_ingest_write_txn.py @@ -115,85 +115,100 @@ def stubbed_write_txn(monkeypatch) -> _StubbedWriteTxn: return barrier -def test_reused_upload_refuses_after_four_stale_preflights( - stubbed_write_txn, monkeypatch -) -> None: +def _reuse_path(monkeypatch, _tmp): preflight = SimpleNamespace(signature=object()) prepared = object() - attempts: list[object] = [] - - monkeypatch.setattr(ingest, "_preflight_upload_record", lambda *_args: preflight) - monkeypatch.setattr(ingest, "_prepare_upload_record", lambda _preflight: prepared) - monkeypatch.setattr(ingest, "_assert_signature_current", lambda _signature: None) - - def stale_apply(_session, observed_prepared): - attempts.append(observed_prepared) - raise ingest._PreflightStale - - monkeypatch.setattr(ingest, "_apply_reused_upload_record", stale_apply) - + monkeypatch.setattr(ingest, "_preflight_upload_record", lambda *_a: preflight) + monkeypatch.setattr(ingest, "_prepare_upload_record", lambda _p: prepared) spec = ingest._UploadRecordSpec("asset", [], None, {}, None) - with pytest.raises(ingest.UploadUnstableError): - ingest._reuse_qualified_content("blake3:hash", spec) - - assert attempts == [prepared, prepared, prepared, prepared] + return ( + "_apply_reused_upload_record", + prepared, + lambda: ingest._reuse_qualified_content("blake3:hash", spec), + ) -def test_settle_destination_refuses_after_four_stale_preflights( - stubbed_write_txn, monkeypatch -) -> None: +def _settle_path(monkeypatch, _tmp): preflight = SimpleNamespace(signature=object()) prepared = SimpleNamespace(facts=None) - attempts: list[object] = [] - - monkeypatch.setattr(ingest, "_preflight_settle_target", lambda _dest: preflight) - monkeypatch.setattr(ingest, "_prepare_settle_target", lambda _preflight: prepared) - monkeypatch.setattr(ingest, "_assert_signature_current", lambda _signature: None) - - def stale_apply(_session, observed_prepared): - attempts.append(observed_prepared) - raise ingest._PreflightStale - - monkeypatch.setattr(ingest, "_apply_settle_target", stale_apply) - - with pytest.raises(ingest.UploadUnstableError): - ingest._settle_destination_before_write(_output_path("settle-unstable.bin")) - - assert attempts == [prepared, prepared, prepared, prepared] + monkeypatch.setattr(ingest, "_preflight_settle_target", lambda _d: preflight) + monkeypatch.setattr(ingest, "_prepare_settle_target", lambda _p: prepared) + return ( + "_apply_settle_target", + prepared, + lambda: ingest._settle_destination_before_write(_output_path("settle-unstable.bin")), + ) -def test_cached_registration_gives_up_after_four_stale_preflights( - stubbed_write_txn, monkeypatch, caplog -) -> None: +def _cached_path(monkeypatch, _tmp): path = _output_path("cached-always-stale.bin") with open(path, "wb") as file: file.write(b"output") - preflight = SimpleNamespace( content_id="content-always-stale", sibling_id=None, sibling_metadata=None, signature=None, ) - attempts: list[str] = [] + monkeypatch.setattr(ingest, "_preflight_cached_registration", lambda _l: preflight) + return "_apply_cached_registration", None, lambda: ingest.register_cached_output(path) + + +@pytest.mark.parametrize( + ("build_path", "raises"), + [ + pytest.param(_reuse_path, True, id="reused-upload"), + pytest.param(_settle_path, True, id="settle-destination"), + pytest.param(_cached_path, False, id="cached-registration"), + ], +) +def test_ingest_paths_refuse_after_four_stale_preflights( + stubbed_write_txn, monkeypatch, tmp_path, build_path, raises +) -> None: + """A preflight that never settles is refused, not persisted from mixed facts.""" + monkeypatch.setattr(ingest, "_assert_signature_current", lambda _s: None) + apply_name, expected_prepared, invoke = build_path(monkeypatch, tmp_path) + attempts: list[object] = [] + def stale_apply(_session, observed, *_args): + attempts.append(observed) + raise ingest._PreflightStale + + monkeypatch.setattr(ingest, apply_name, stale_apply) + + if raises: + with pytest.raises(ingest.UploadUnstableError): + invoke() + else: + assert invoke() is None + + assert len(attempts) == 4, "the retry budget is four attempts, then refusal" + if expected_prepared is not None: + assert attempts == [expected_prepared] * 4 + + +def test_cached_registration_refusal_is_an_outcome_not_a_crash( + stubbed_write_txn, monkeypatch, caplog +) -> None: + path = _output_path("cached-refusal-logging.bin") + with open(path, "wb") as file: + file.write(b"output") monkeypatch.setattr( - ingest, "_preflight_cached_registration", lambda _locator: preflight + ingest, + "_preflight_cached_registration", + lambda _l: SimpleNamespace( + content_id="c", sibling_id=None, sibling_metadata=None, signature=None + ), ) def stale_apply(_session, _preflight, *_args): - attempts.append("apply") raise ingest._PreflightStale monkeypatch.setattr(ingest, "_apply_cached_registration", stale_apply) - try: - with caplog.at_level(logging.INFO): - assert ingest.register_cached_output(path) is None - finally: - os.unlink(path) + with caplog.at_level(logging.INFO): + assert ingest.register_cached_output(path) is None - assert len(attempts) == 4 warnings = [ r.getMessage() for r in caplog.records @@ -865,39 +880,55 @@ def stale_once(_signature): assert checks == 2 -def test_executed_registration_reports_exhausted_locked_retries(monkeypatch, caplog) -> None: - path = _output_path("executed-locked-retries.bin") - with open(path, "wb") as file: - file.write(b"output") - - def exhausted_retries(_work): - raise OperationalError("INSERT", {}, sqlite3.OperationalError("database is locked")) - - monkeypatch.setattr(ingest, "run_write_txn", exhausted_retries) - try: - with caplog.at_level(logging.INFO): - assert ingest.register_executed_output(path, job_id="job-locked") is None - assert _registration_failure_event(caplog) == ( - "[assets-event] ingest.register_failed error_type=OperationalError job_id=job-locked output_kind=executed" - ) - finally: - os.unlink(path) - - -def test_executed_registration_reports_non_retryable_write_failure(monkeypatch, caplog) -> None: - path = _output_path("executed-non-retryable.bin") +def _raise_locked(_work): + raise OperationalError("INSERT", {}, sqlite3.OperationalError("database is locked")) + + +def _raise_integrity(_work): + raise IntegrityError("INSERT", {}, sqlite3.IntegrityError("constraint failed")) + + +@pytest.mark.parametrize( + ("failure", "kind", "job_id", "expected"), + [ + pytest.param( + _raise_locked, "executed", "job-locked", + "error_type=OperationalError job_id=job-locked output_kind=executed", + id="executed-exhausted-lock-retries", + ), + pytest.param( + _raise_integrity, "executed", "job-integrity", + "error_type=IntegrityError job_id=job-integrity output_kind=executed", + id="executed-non-retryable-write", + ), + pytest.param( + _raise_integrity, "cached", None, + "error_type=IntegrityError output_kind=cached", + id="cached-terminal-write", + ), + ], +) +def test_registration_failure_is_reported_and_never_raised( + mock_create_session, monkeypatch, caplog, failure, kind, job_id, expected +) -> None: + """A terminal write failure yields no asset id and one structured event.""" + path = _output_path(f"registration-failure-{kind}-{job_id}.bin") with open(path, "wb") as file: file.write(b"output") + if kind == "cached": + with mock_create_session() as session: + create_content(session, path, size_bytes=6) + session.commit() - def non_retryable_failure(_work): - raise IntegrityError("INSERT", {}, sqlite3.IntegrityError("constraint failed")) - - monkeypatch.setattr(ingest, "run_write_txn", non_retryable_failure) + monkeypatch.setattr(ingest, "run_write_txn", failure) + register = ( + ingest.register_executed_output if kind == "executed" else ingest.register_cached_output + ) try: with caplog.at_level(logging.INFO): - assert ingest.register_executed_output(path, job_id="job-integrity") is None + assert register(path, job_id=job_id) is None assert _registration_failure_event(caplog) == ( - "[assets-event] ingest.register_failed error_type=IntegrityError job_id=job-integrity output_kind=executed" + f"[assets-event] ingest.register_failed {expected}" ) finally: os.unlink(path) @@ -913,30 +944,6 @@ def test_executed_registration_reports_preflight_os_error(monkeypatch, caplog) - ) -def test_cached_registration_reports_terminal_write_failure( - mock_create_session, monkeypatch, caplog -) -> None: - path = _output_path("cached-terminal-failure.bin") - with open(path, "wb") as file: - file.write(b"output") - with mock_create_session() as session: - create_content(session, path, size_bytes=6) - session.commit() - - def non_retryable_failure(_work): - raise IntegrityError("INSERT", {}, sqlite3.IntegrityError("constraint failed")) - - monkeypatch.setattr(ingest, "run_write_txn", non_retryable_failure) - try: - with caplog.at_level(logging.INFO): - assert ingest.register_cached_output(path, job_id=None) is None - assert _registration_failure_event(caplog) == ( - "[assets-event] ingest.register_failed error_type=IntegrityError output_kind=cached" - ) - finally: - os.unlink(path) - - def test_cached_registration_apply_rejects_changed_content_row_with_sibling( mock_create_session, tmp_path ) -> None: From 6001ca454be08c8494ef16178b873ed6f5aab671 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Fri, 18 Sep 2026 16:38:37 -0700 Subject: [PATCH 103/111] Explain why chunking survives a long lock hold that one transaction would not --- app/assets/docs/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/assets/docs/README.md b/app/assets/docs/README.md index ee290b08a58..d9bab4bd0bb 100644 --- a/app/assets/docs/README.md +++ b/app/assets/docs/README.md @@ -333,7 +333,9 @@ Every database write goes through `run_write_txn`, which opens a dedicated write Background writes are bounded: no scanner transaction covers more than `MAX_WRITE_BATCH` rows, and the scanner checks for pause and cancellation between batches, so a pause takes effect one batch later instead of at the end of the phase. Seeding, reference reconciliation, pruning, and enrichment all chunk their work to that size; the verification and hash-transition drains already write one entry per transaction. -The batch size is 25. A transaction holds the write lock for its whole duration, so the batch size sets both the stall an urgent writer inherits when it arrives mid-batch and the rollback blast radius when one row in a batch fails. A commit costs roughly 20ms fixed plus roughly 0.04ms per row, so the batch size trades worst-case hold against total lock occupancy: across a 10,000-file scan, 25-row batches hold the lock about 21ms at a time over 400 commits, where 500-row batches hold it about 39ms over 20. Both sit three orders of magnitude below the point at which a competing writer gives up, so this constant is not what keeps the system correct — moving filesystem work out of the transaction is. 25 is kept for the shorter worst-case hold. +The batch size is 25. A transaction holds the write lock for its whole duration, so the batch size sets both the stall an urgent writer inherits when it arrives mid-batch and the rollback blast radius when one row in a batch fails. + +Chunking matters more than its arithmetic suggests. A commit costs roughly 20ms fixed plus roughly 0.04ms per row, so smaller batches hold the lock for less time each but hold it more often, and the total time spent holding it goes up. What that accounting misses is that a competing writer does not care about the total; it cares whether it can ever acquire. Because the scanner releases and reacquires between chunks, a blocked writer gets a fresh attempt in every gap, and the bounded retry rides over a long series of short holds instead of timing out against one long one. Measured against deliberately slowed storage, writers have waited a full minute across many chunk boundaries and still succeeded, where a single hold of that length would have failed. Chunking turns one fatal wait into many survivable ones. Write callables never touch the filesystem. Stat, hashing, and metadata extraction run in a preflight outside the transaction, and the closure receives the facts that preflight observed; inside the lock it only reads and writes rows. Two invariants make that safe: From 4667fb4f84fa1bd87a11dc3d1f36312d6ef356bd Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Fri, 18 Sep 2026 18:44:28 -0700 Subject: [PATCH 104/111] Correct how a write lock conflict actually fails --- app/assets/docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/docs/README.md b/app/assets/docs/README.md index d9bab4bd0bb..7f0484a3836 100644 --- a/app/assets/docs/README.md +++ b/app/assets/docs/README.md @@ -350,7 +350,7 @@ A persistently unstable upload is refused rather than hashed under the lock: whe The asset database is SQLite in WAL journal mode. Reader connections use a 30-second busy timeout. Writer connections poll `BEGIN IMMEDIATE` in Python, so each lock wait is capped at 30 seconds while the overall retry deadline remains 60 seconds. Several paths still contend for the single writer lock: a non-deduplicated upload writes its bytes and mints a delivery record, while a deduplicated upload reuses existing content and mints only the record; a same-path write whose hash has changed retires the old content and inserts new content, while a same-path write whose hash matches refreshes the existing record in place; execution outputs register per-emission during the generation loop; a background enrichment pass fills hashes and metadata in bounded batches; hash-serves write access time to every record sharing the served content; and the upload dedup claim mints its record once a preflight has already re-checked the file and extracted its metadata outside the lock. -A session that opens as a reader and only later attempts a write can be refused instantly if another connection already holds the write lock, before the busy timeout ever gets a chance to wait. Every writing session avoids this by going through `run_write_txn`, which opens a dedicated writer session and issues `BEGIN IMMEDIATE` up front, declaring writer intent before any statement runs. A write that still finds the database locked retries up to 5 attempts total, sleeping 0.05s/0.1s/0.2s/0.4s between attempts (each jittered ±50%) and giving up once 60 seconds have elapsed since the first attempt; only a locked-database `OperationalError` is retried, every other exception propagates immediately. Those 5 attempts and their backoff table are reachable only when the lock error surfaces from inside the callback, after `BEGIN IMMEDIATE` has already succeeded. Contention at `BEGIN IMMEDIATE` itself behaves differently: each attempt polls for up to 30 seconds before yielding, so the 60-second deadline admits about two attempts and the backoff sleeps are negligible beside the polling. Either way the bound is the same 60-second deadline, after which a locked-database `OperationalError` is raised. Contributors must not open a writing session with `create_session()` directly. `run_write_txn` is the only sanctioned path for writes, and `tests-unit/assets_test/test_write_session_convention.py` enforces that automatically. +When a writer holds the lock for longer than the busy timeout, a competing writer waits the timeout out and then fails. That is the failure this system is built to avoid, and it is not specific to a session that reads before it writes: a write with no preceding read fails the same way. Every writing session goes through `run_write_txn`, which opens a dedicated writer session and issues `BEGIN IMMEDIATE` up front. Declaring write intent before any statement runs lets the writer queue for the lock and retry, rather than discovering the conflict partway through a unit of work it would have to abandon. A write that still finds the database locked retries up to 5 attempts total, sleeping 0.05s/0.1s/0.2s/0.4s between attempts (each jittered ±50%) and giving up once 60 seconds have elapsed since the first attempt; only a locked-database `OperationalError` is retried, every other exception propagates immediately. Those 5 attempts and their backoff table are reachable only when the lock error surfaces from inside the callback, after `BEGIN IMMEDIATE` has already succeeded. Contention at `BEGIN IMMEDIATE` itself behaves differently: each attempt polls for up to 30 seconds before yielding, so the 60-second deadline admits about two attempts and the backoff sleeps are negligible beside the polling. Either way the bound is the same 60-second deadline, after which a locked-database `OperationalError` is raised. Contributors must not open a writing session with `create_session()` directly. `run_write_txn` is the only sanctioned path for writes, and `tests-unit/assets_test/test_write_session_convention.py` enforces that automatically. WAL journal mode leaves `-wal` and `-shm` sidecar files next to the main database file; treat all three as one unit for backups and copies. WAL is not supported on network filesystems, so pointing `--database-url` at a network-mounted path fails at startup with a `RuntimeError` rather than silently falling back to a less safe journal mode. From 4be78b1cc5a8cee083ab555734f899a657471ae0 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Sat, 19 Sep 2026 15:39:23 -0700 Subject: [PATCH 105/111] Remove test guards that hid renames and trim the batch-size rationale --- app/assets/docs/README.md | 2 +- tests-unit/app_test/test_db_promotion.py | 13 +-------- tests-unit/app_test/test_db_write_txn.py | 12 +++------ .../test_queue_drain_characterization.py | 27 ------------------- .../assets_test/test_event_log_sites.py | 10 +++---- 5 files changed, 9 insertions(+), 55 deletions(-) diff --git a/app/assets/docs/README.md b/app/assets/docs/README.md index 7f0484a3836..ea62a67a2c9 100644 --- a/app/assets/docs/README.md +++ b/app/assets/docs/README.md @@ -335,7 +335,7 @@ Background writes are bounded: no scanner transaction covers more than `MAX_WRIT The batch size is 25. A transaction holds the write lock for its whole duration, so the batch size sets both the stall an urgent writer inherits when it arrives mid-batch and the rollback blast radius when one row in a batch fails. -Chunking matters more than its arithmetic suggests. A commit costs roughly 20ms fixed plus roughly 0.04ms per row, so smaller batches hold the lock for less time each but hold it more often, and the total time spent holding it goes up. What that accounting misses is that a competing writer does not care about the total; it cares whether it can ever acquire. Because the scanner releases and reacquires between chunks, a blocked writer gets a fresh attempt in every gap, and the bounded retry rides over a long series of short holds instead of timing out against one long one. Measured against deliberately slowed storage, writers have waited a full minute across many chunk boundaries and still succeeded, where a single hold of that length would have failed. Chunking turns one fatal wait into many survivable ones. +Chunking is what lets a writer wait out a busy scanner. The scanner releases the lock between chunks and a blocked writer re-attempts in every gap, so the bounded retry crosses many short holds where a single hold of the same total length would exceed its deadline. Under deliberately slowed storage, writers have blocked for a minute across chunk boundaries and still committed. Write callables never touch the filesystem. Stat, hashing, and metadata extraction run in a preflight outside the transaction, and the closure receives the facts that preflight observed; inside the lock it only reads and writes rows. Two invariants make that safe: diff --git a/tests-unit/app_test/test_db_promotion.py b/tests-unit/app_test/test_db_promotion.py index 01748d4ad0c..fd0890bb1da 100644 --- a/tests-unit/app_test/test_db_promotion.py +++ b/tests-unit/app_test/test_db_promotion.py @@ -5,17 +5,6 @@ from sqlalchemy import event, text import app.database.db as db_mod -from app.database.db import create_session - - -def _invoke_writer(work): - def _legacy(legacy_work): - with create_session() as session: - result = legacy_work(session) - session.commit() - return result - - return db_mod.run_write_txn(work) if hasattr(db_mod, "run_write_txn") else _legacy(work) def test_write_transaction_waits_for_held_writer_before_select_then_mutate(tmp_path, monkeypatch): @@ -58,7 +47,7 @@ def select_then_mutate(session): return "written" started_at = time.monotonic() - result = _invoke_writer(select_then_mutate) + result = db_mod.run_write_txn(select_then_mutate) elapsed = time.monotonic() - started_at finally: holder.join(timeout=5) diff --git a/tests-unit/app_test/test_db_write_txn.py b/tests-unit/app_test/test_db_write_txn.py index 424080a047d..763ca7acfe3 100644 --- a/tests-unit/app_test/test_db_write_txn.py +++ b/tests-unit/app_test/test_db_write_txn.py @@ -30,8 +30,7 @@ def file_database(tmp_path, monkeypatch): monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") monkeypatch.setattr(db_mod, "Session", None) monkeypatch.setattr(db_mod, "_db_lock", None) - if hasattr(db_mod, "WriteSession"): - monkeypatch.setattr(db_mod, "WriteSession", None) + monkeypatch.setattr(db_mod, "WriteSession", None) db_mod.init_db() yield database_path _dispose_runtime_engines() @@ -43,8 +42,7 @@ def memory_database(monkeypatch): monkeypatch.setattr(db_mod.args, "enable_assets", True) monkeypatch.setattr(db_mod.args, "database_url", "sqlite:///:memory:") monkeypatch.setattr(db_mod, "Session", None) - if hasattr(db_mod, "WriteSession"): - monkeypatch.setattr(db_mod, "WriteSession", None) + monkeypatch.setattr(db_mod, "WriteSession", None) db_mod.init_db() yield _dispose_runtime_engines() @@ -86,8 +84,7 @@ def _migrate_crash_style_database(database_path: str, monkeypatch) -> None: monkeypatch.setattr(db_mod.args, "enable_assets", True) monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") monkeypatch.setattr(db_mod, "Session", None) - if hasattr(db_mod, "WriteSession"): - monkeypatch.setattr(db_mod, "WriteSession", None) + monkeypatch.setattr(db_mod, "WriteSession", None) db_mod._migrate_and_bind(f"sqlite:///{database_path}", database_path, db_exists=True) @@ -554,8 +551,7 @@ def fail_upgrade(_config, _target_revision): monkeypatch.setattr(db_mod.args, "enable_assets", True) monkeypatch.setattr(db_mod.args, "database_url", f"sqlite:///{database_path}") monkeypatch.setattr(db_mod, "Session", None) - if hasattr(db_mod, "WriteSession"): - monkeypatch.setattr(db_mod, "WriteSession", None) + monkeypatch.setattr(db_mod, "WriteSession", None) with pytest.raises(RuntimeError, match="upgrade failure"): db_mod._migrate_and_bind(f"sqlite:///{database_path}", database_path, db_exists=True) diff --git a/tests-unit/assets_test/services/test_queue_drain_characterization.py b/tests-unit/assets_test/services/test_queue_drain_characterization.py index 16baf3adb60..44f703bb7a7 100644 --- a/tests-unit/assets_test/services/test_queue_drain_characterization.py +++ b/tests-unit/assets_test/services/test_queue_drain_characterization.py @@ -15,8 +15,6 @@ _PendingEntry, clear_transition_queue, drain_transition_queue, - read_stored_mode, - write_stored_mode, ) @@ -102,28 +100,3 @@ def test_transition_queue_retries_without_losing_companion_path( assert list(_PENDING_QUEUE) == [_PendingEntry(path, ticks=1)] assert _PENDING_PATHS == {path} assert hash_mode_state._off_to_on_transition_in_flight is True - - -def test_transition_queue_exhaustion_clears_companion_and_persists_mode( - session, monkeypatch -): - path = "/unreadable/exhausted.bin" - _PENDING_QUEUE.append(_PendingEntry(path)) - _PENDING_PATHS.add(path) - hash_mode_state._off_to_on_transition_in_flight = True - write_stored_mode(session, "off") - session.commit() - monkeypatch.setattr( - hash_mode_state, - "snapshot_hash", - lambda _path: (_ for _ in ()).throw(PermissionError("denied")), - ) - - for _ in range(3): - drain_transition_queue() - session.commit() - - assert list(_PENDING_QUEUE) == [] - assert _PENDING_PATHS == set() - assert hash_mode_state._off_to_on_transition_in_flight is False - assert read_stored_mode(session) == "on" diff --git a/tests-unit/assets_test/test_event_log_sites.py b/tests-unit/assets_test/test_event_log_sites.py index 753e0f52db8..df5e8677c40 100644 --- a/tests-unit/assets_test/test_event_log_sites.py +++ b/tests-unit/assets_test/test_event_log_sites.py @@ -6,7 +6,10 @@ import pytest +from app.assets import manager as manager_mod +from app.assets.api import routes from app.assets.event_log import TAG +from app.assets.manager import AssetsEnabled STARTUP_SCRIPT = ( @@ -75,9 +78,6 @@ def test_disable_announces_itself_on_the_event_channel(caplog) -> None: fail, so a monitor has already been told assets are up by the time a database failure degrades them. Without a contradicting event it keeps believing that. """ - from app.assets.api import routes - from app.assets.manager import AssetsEnabled - manager = AssetsEnabled(SimpleNamespace(enable_assets=True, enable_asset_hashing=False)) routes._ASSETS_ENABLED = True try: @@ -104,10 +104,6 @@ def test_disable_actually_stops_scanning_and_ingest_not_just_http() -> None: time setup_database can fail. So flipping it is invisible to every later caller: the scanner and the three ingest entry points have to be gated directly. """ - from app.assets import manager as manager_mod - from app.assets.api import routes - from app.assets.manager import AssetsEnabled - started: list[tuple] = [] mgr = AssetsEnabled(SimpleNamespace(enable_assets=True, enable_asset_hashing=False)) routes._ASSETS_ENABLED = True From 61662a06955a4165e3efce8f484d9f445fee2a62 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Sat, 19 Sep 2026 15:49:37 -0700 Subject: [PATCH 106/111] Keep retryable filesystem errors inside the upload retry loops --- app/assets/scanner.py | 13 ++++++------- app/assets/scanner_changes.py | 11 ----------- app/assets/services/ingest.py | 13 +++++++++++-- .../services/test_scanner_seed_resilience.py | 13 ++++++++++--- 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index 5211705af09..b76562251e0 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -34,9 +34,7 @@ drain_pending_verifications, is_path_under_prefixes, live_contents_under_prefixes, - pending_recovery_count, prepare_missing_content_recovery, - queue_pending_recovery, queue_pending_verification, recover_missing_content_from_preparation, ) @@ -63,7 +61,6 @@ __all__ = [ "clear_pending_verifications", "drain_pending_verifications", - "pending_recovery_count", ] @@ -555,7 +552,11 @@ def seed_asset_specs( if prepared_recoveries is not None and mode.hashing_enabled(): prepared = prepared_recoveries.get(path) if prepared is None: - logging.warning("Skipping vanished asset during scan: %s", path) + logging.warning( + "Skipping asset whose recovery hash could not be prepared " + "during scan: %s", + path, + ) continue recovery = recover_missing_content_from_preparation( session, @@ -629,9 +630,7 @@ def _work(sess: Session) -> tuple[int, list[str]]: ) return created, pending_recovery_paths - created, pending_recovery_paths = run_write_txn(_work) - for path in pending_recovery_paths: - queue_pending_recovery(path) + created, _ = run_write_txn(_work) return created diff --git a/app/assets/scanner_changes.py b/app/assets/scanner_changes.py index 53833907876..0b0a5bc558c 100644 --- a/app/assets/scanner_changes.py +++ b/app/assets/scanner_changes.py @@ -28,7 +28,6 @@ from app.database.db import create_session, run_write_txn _pending_verification_ids: list[str] = [] -_pending_recovery_paths: list[str] = [] class PreparedRecovery(NamedTuple): @@ -52,7 +51,6 @@ def prepare_missing_content_recovery(path: str, stat_result: os.stat_result) -> def clear_pending_verifications() -> None: _pending_verification_ids.clear() - _pending_recovery_paths.clear() def queue_pending_verification(content_id: str) -> None: @@ -60,15 +58,6 @@ def queue_pending_verification(content_id: str) -> None: _pending_verification_ids.append(content_id) -def queue_pending_recovery(path: str) -> None: - if path not in _pending_recovery_paths: - _pending_recovery_paths.append(path) - - -def pending_recovery_count() -> int: - return len(_pending_recovery_paths) - - def recover_missing_content_from_preparation( session: Session, path: str, diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index 8ef6bc6c17a..38a1104ce04 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -462,7 +462,13 @@ def _preflight_upload_record( raise ValueError( f"preview_id {spec.preview_id!r} does not reference an existing asset" ) - signature = _file_signature(path) + try: + signature = _file_signature(path) + except OSError: + # No row backs the fallback path, so a missing file there is permanent, not stale. + if content_id is None: + raise + raise _PreflightStale from None if content_id is not None and ( signature.size_bytes, signature.mtime_ns if row_mtime_ns is not None else None, @@ -557,7 +563,10 @@ def _preflight_settle_target(dest_abs: str) -> _SettleTargetPreflight | None: ).first() if existing is None: return None - signature = _file_signature(dest_abs) + try: + signature = _file_signature(dest_abs) + except OSError: + return None if ( existing.hash is not None and existing.size_bytes == signature.size_bytes diff --git a/tests-unit/assets_test/services/test_scanner_seed_resilience.py b/tests-unit/assets_test/services/test_scanner_seed_resilience.py index 285003de436..c9963592061 100644 --- a/tests-unit/assets_test/services/test_scanner_seed_resilience.py +++ b/tests-unit/assets_test/services/test_scanner_seed_resilience.py @@ -92,8 +92,14 @@ def _hash_or_raise(candidate_path: str) -> str | None: @pytest.mark.parametrize( - "delete_path", - [_delete_before_restat, _delete_during_recovery], + ("delete_path", "expected_message"), + [ + (_delete_before_restat, "Skipping vanished asset during scan: {path}"), + ( + _delete_during_recovery, + "Skipping asset whose recovery hash could not be prepared during scan: {path}", + ), + ], ids=["before-restat", "during-recovery"], ) def test_seed_logs_once_for_each_vanished_path( @@ -102,6 +108,7 @@ def test_seed_logs_once_for_each_vanished_path( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, delete_path: Callable[[pytest.MonkeyPatch, Path], None], + expected_message: str, ) -> None: specs, vanished_path = _specs_with_vanished_path(temp_dir) delete_path(monkeypatch, vanished_path) @@ -115,7 +122,7 @@ def test_seed_logs_once_for_each_vanished_path( for record in caplog.records if str(vanished_path) in record.getMessage() ] - assert messages == [f"Skipping vanished asset during scan: {vanished_path}"] + assert messages == [expected_message.format(path=vanished_path)] def test_seed_isolates_a_poisoned_spec_and_persists_the_specs_around_it( From 20bcd61d4ba9aac858a489c69523123fe7790768 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Sat, 19 Sep 2026 15:59:42 -0700 Subject: [PATCH 107/111] Leave a row alone when the hash-mode transition loses its compare-and-set --- app/assets/services/hash_mode_state.py | 9 +++++---- tests-unit/assets_test/services/test_transition_drain.py | 6 +++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/app/assets/services/hash_mode_state.py b/app/assets/services/hash_mode_state.py index a4a6cb0c727..0042a63e8a7 100644 --- a/app/assets/services/hash_mode_state.py +++ b/app/assets/services/hash_mode_state.py @@ -144,10 +144,11 @@ def _apply(session: Session) -> str: content = session.get(AssetContent, content_id) if content is None or content.is_missing or content.path != entry.path: return "drop" - row_changed = ( - content.size_bytes != size_bytes or content.mtime_ns != mtime_ns - ) - if row_changed or preparation == "retry": + if content.size_bytes != size_bytes or content.mtime_ns != mtime_ns: + if entry.ticks + 1 < _MAX_VERIFY_ATTEMPTS: + return "retry" + return "drop" + if preparation == "retry": if entry.ticks + 1 < _MAX_VERIFY_ATTEMPTS: return "retry" content.hash = None diff --git a/tests-unit/assets_test/services/test_transition_drain.py b/tests-unit/assets_test/services/test_transition_drain.py index 9474f72e79d..9486aa2601e 100644 --- a/tests-unit/assets_test/services/test_transition_drain.py +++ b/tests-unit/assets_test/services/test_transition_drain.py @@ -450,7 +450,7 @@ def test_transition_drain_retires_an_entry_that_keeps_losing_the_row_compare_and spend a write transaction on every tick. """ path = temp_dir / "always-racing.bin" - content_id, _ = _seed_hashed_row(session, path, b"bytes that keep moving") + content_id, original_hash = _seed_hashed_row(session, path, b"bytes that keep moving") write_stored_mode(session, "off") monkeypatch.setattr(hash_mode_state._mode, "hashing_enabled", lambda: True) @@ -489,3 +489,7 @@ def preflight_then_race(entry_path: str): assert read_stored_mode(session) == "on", ( "the mode flip is gated on the queue emptying, so a stuck entry wedges it" ) + retired = session.get(AssetContent, content_id) + assert retired.hash == original_hash, ( + "losing the compare-and-set means another writer owns this row; its hash must survive" + ) From f2292dcb0c86e1346d7c37f521f4e2512b3083d5 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Sat, 19 Sep 2026 17:29:47 -0700 Subject: [PATCH 108/111] Remove the last getattr-spelled compatibility guard --- tests-unit/app_test/test_db_write_txn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests-unit/app_test/test_db_write_txn.py b/tests-unit/app_test/test_db_write_txn.py index 763ca7acfe3..d7b1eedb493 100644 --- a/tests-unit/app_test/test_db_write_txn.py +++ b/tests-unit/app_test/test_db_write_txn.py @@ -18,7 +18,7 @@ def _dispose_runtime_engines(): - for session_factory in (db_mod.Session, getattr(db_mod, "WriteSession", None)): + for session_factory in (db_mod.Session, db_mod.WriteSession): if session_factory is not None: session_factory.kw["bind"].dispose() From 6f12cb5d45a4f0cf6cd297075768172f40cae055 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Mon, 21 Sep 2026 10:39:11 -0700 Subject: [PATCH 109/111] Retry a failed destination stat instead of skipping settlement --- app/assets/services/ingest.py | 3 +- .../assets_test/services/test_upload_b.py | 105 ++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py index 38a1104ce04..9f95528ce8f 100644 --- a/app/assets/services/ingest.py +++ b/app/assets/services/ingest.py @@ -566,7 +566,8 @@ def _preflight_settle_target(dest_abs: str) -> _SettleTargetPreflight | None: try: signature = _file_signature(dest_abs) except OSError: - return None + # isfile already saw the file, so a failed stat is stale, not absent. + raise _PreflightStale from None if ( existing.hash is not None and existing.size_bytes == signature.size_bytes diff --git a/tests-unit/assets_test/services/test_upload_b.py b/tests-unit/assets_test/services/test_upload_b.py index 86191b46413..81b1cce17ad 100644 --- a/tests-unit/assets_test/services/test_upload_b.py +++ b/tests-unit/assets_test/services/test_upload_b.py @@ -656,6 +656,111 @@ def test_upload_unhashed_row_at_destination_is_not_retired( os.unlink(path) +_SETTLE_ATTEMPTS = 4 + + +def _stat_failing_signature(path: str, failures: int): + """Fail the first ``failures`` stats of ``path``, leaving every other call alone.""" + real_file_signature = ingest_module._file_signature + stats = 0 + + def _signature(candidate: str): + nonlocal stats + if candidate == path: + stats += 1 + if stats <= failures: + raise OSError(errno.EIO, "injected stat failure") + return real_file_signature(candidate) + + return _signature + + +def test_upload_retries_a_failed_destination_stat_and_adopts_the_incumbent( + mock_create_session, hashing_on, monkeypatch +): + payload = b"upload-transient-stat-bytes" + probe = _write_temp(payload) + digest = _digest_of(probe) + os.unlink(probe) + stored_hash = to_stored_hash(digest) + output_dir = folder_paths.get_output_directory() + os.makedirs(output_dir, exist_ok=True) + dest = os.path.join(output_dir, f"{digest}.bin") + temp = _write_temp(payload) + try: + with open(dest, "wb") as file: + file.write(payload) + with mock_create_session() as session: + content_id, record_id = _seed_live_content(session, dest, None) + monkeypatch.setattr( + ingest_module, "_file_signature", _stat_failing_signature(dest, failures=1) + ) + + result = upload_from_temp_path( + temp_path=temp, name="up.bin", tags=["output"], client_filename="up.bin" + ) + + assert result.asset.hash == stored_hash + assert result.content_id == content_id, ( + "the retried stat settles the destination, so the incumbent row carries " + "the hash this upload looks up and is reused rather than replaced" + ) + with mock_create_session() as session: + content = session.get(AssetContent, content_id) + assert content is not None + assert content.is_missing is False + assert content.hash == stored_hash + assert _is_missing_tagged(session, record_id) is False + finally: + for path in (temp, dest): + if os.path.exists(path): + os.unlink(path) + + +def test_upload_refuses_publication_when_the_destination_stat_never_recovers( + mock_create_session, hashing_on, monkeypatch +): + payload = b"upload-unreadable-destination" + probe = _write_temp(payload) + digest = _digest_of(probe) + os.unlink(probe) + output_dir = folder_paths.get_output_directory() + os.makedirs(output_dir, exist_ok=True) + dest = os.path.join(output_dir, f"{digest}.bin") + incumbent = b"Y" * len(payload) + temp = _write_temp(payload) + try: + with open(dest, "wb") as file: + file.write(incumbent) + with mock_create_session() as session: + content_id, record_id = _seed_live_content(session, dest, None) + monkeypatch.setattr( + ingest_module, + "_file_signature", + _stat_failing_signature(dest, failures=_SETTLE_ATTEMPTS), + ) + + with pytest.raises(UploadUnstableError): + upload_from_temp_path( + temp_path=temp, name="up.bin", tags=["output"], client_filename="up.bin" + ) + + assert open(dest, "rb").read() == incumbent, ( + "publication is refused before the move, so the bytes the incumbent row " + "describes are still on disk" + ) + with mock_create_session() as session: + content = session.get(AssetContent, content_id) + assert content is not None + assert content.is_missing is False + assert content.hash is None + assert _is_missing_tagged(session, record_id) is False + finally: + for path in (temp, dest): + if os.path.exists(path): + os.unlink(path) + + def test_upload_unhashed_destination_holding_other_bytes_is_retired( mock_create_session, hashing_on ): From 9d71c9b7b14029418612298303f192033dce77d0 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Mon, 21 Sep 2026 13:37:30 -0700 Subject: [PATCH 110/111] Fix paused asset reference sync --- app/assets/seeder.py | 6 +-- .../services/test_scanner_sync_chunking.py | 41 ++++++++++++++++++- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/app/assets/seeder.py b/app/assets/seeder.py index 16cc1d27a55..e6b308417a2 100644 --- a/app/assets/seeder.py +++ b/app/assets/seeder.py @@ -628,7 +628,7 @@ def _run_scan(self) -> None: all_prefixes = get_owned_prefixes() marked = mark_missing_outside_prefixes_safely( all_prefixes, - interrupt_check=lambda: self._is_paused_or_cancelled( + interrupt_check=lambda: self._check_pause_and_cancel( _ScanStage.PRUNING ), ) @@ -641,7 +641,7 @@ def _run_scan(self) -> None: logging.info("Marked %d refs as missing before scan", marked) sync_temp_references_safely( scan_state, - interrupt_check=lambda: self._is_paused_or_cancelled( + interrupt_check=lambda: self._check_pause_and_cancel( _ScanStage.PRUNING ), ) @@ -805,7 +805,7 @@ def _run_fast_phase(self, roots: tuple[RootType, ...]) -> tuple[int, int, int]: sync_root_safely( r, scan_state, - interrupt_check=lambda: self._is_paused_or_cancelled( + interrupt_check=lambda: self._check_pause_and_cancel( _ScanStage.FAST_SCAN ), ) diff --git a/tests-unit/assets_test/services/test_scanner_sync_chunking.py b/tests-unit/assets_test/services/test_scanner_sync_chunking.py index 9b7498003a5..b45ca456094 100644 --- a/tests-unit/assets_test/services/test_scanner_sync_chunking.py +++ b/tests-unit/assets_test/services/test_scanner_sync_chunking.py @@ -3,9 +3,10 @@ import sqlalchemy as sa import app.database.db as db_mod -from app.assets import scanner +from app.assets import scanner, seeder as seeder_module from app.assets.database.models import AssetContent from app.assets.database.queries import create_content, create_record, mark_content_missing +from app.assets.seeder import State, _AssetSeeder, _ScanState def _seed_reference_observations( @@ -156,6 +157,44 @@ def interrupt_after_first_chunk() -> bool: assert published == content_ids[: scanner.MAX_WRITE_BATCH] +def test_seeder_resumes_root_sync_chunks_after_pause( + tmp_path: Path, monkeypatch, session +) -> None: + content_ids, observations, survivors = _seed_reference_observations(tmp_path, 60) + monkeypatch.setattr( + scanner, + "observe_references_on_filesystem", + lambda *_args, **_kwargs: (observations, survivors), + ) + seeder = _AssetSeeder() + seeder._state = State.RUNNING + seeder._scan_state = _ScanState() + events: list[str] = [] + seeder.set_event_sink(lambda event_type, _data: events.append(event_type)) + + def pause_before_root_sync(root, progress, interrupt_check=None): + assert seeder.pause() + return scanner.sync_root_safely(root, progress, interrupt_check) + + def resume_on_wait(timeout=None): + if not seeder._run_gate.is_set(): + assert seeder.resume() + return True + + monkeypatch.setattr(seeder_module, "sync_root_safely", pause_before_root_sync) + monkeypatch.setattr(seeder._run_gate, "wait", resume_on_wait) + monkeypatch.setattr(seeder_module, "collect_paths_for_roots", lambda _roots: []) + monkeypatch.setattr(seeder_module, "tick_watch_list", lambda **_kwargs: None) + + seeder._run_fast_phase(("input",)) + + assert _content_states(content_ids) == [ + (None, observation.stat_result.st_mtime_ns, False) + for observation in observations + ] + assert events.count("assets.seed.paused") == 1 + + def test_root_sync_later_chunk_failure_publishes_only_prior_commits( tmp_path: Path, monkeypatch, session ) -> None: From 1adfd4b5751763181ec2f948bcb081cf8e2bfb46 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Mon, 21 Sep 2026 13:40:09 -0700 Subject: [PATCH 111/111] Reconcile the merge's cross-PR API changes in tests and the watch list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The conflict resolution changed four signatures/returns that call sites outside the conflict hunks still used the old spelling of. Auto-merge kept whichever side a given hunk came from, so these are merge fallout, not new behaviour: - seed_asset_specs now takes #16358's pre-lock stats/prepared-recovery arguments AND returns #16393's (created, batch_fault). Test call sites and the seed_with_recovery helper widened to match. - insert_asset_specs returns (created, batch_fault) and commits through run_write_txn, so tests that bound the fixture engine at scanner's create_session bind it at app.database.db.WriteSession instead. - enrich_assets_batch returns #16393's third element (consumed). - tick_watch_list takes #16358's interrupt_check instead of a caller session. Two behavioural picks where the PRs genuinely disagree, both resolved toward #16393 (the later, explicitly hardening change), with the superseded #16358 tests updated rather than deleted: - An unreadable watched file is dropped with scanner.watch_stat_failed instead of propagating out of the tick. #16358's queue-drain *characterization* test documented the old behaviour; it now documents the new one. - tick_watch_list pops each entry before anything that can fail, so a stat, spec-build or seed fault cannot wedge the list — but an exception escaping insert_asset_specs puts the entry back, because a failed write transaction is the database's fault and not the file's. That keeps #16358's terminal-commit-failure test passing; #16393's escaping-fault test asserted the dropped entry and now asserts it is retained. tests-unit/{assets_test,seeder_test,app_test}: 862 passed, 72 skipped. --- app/assets/scanner.py | 1 + app/assets/scanner_admission.py | 9 ++++- .../services/test_admission_gate.py | 18 ++++----- .../services/test_enrichment_batching.py | 4 +- .../test_queue_drain_characterization.py | 13 +++++-- ...est_scanner_retry_and_counter_integrity.py | 4 +- .../services/test_scanner_seed_resilience.py | 14 ++++--- .../services/test_transition_settle.py | 25 +++++++------ .../assets_test/test_write_contention_e2e.py | 4 +- tests-unit/seeder_test/test_seeder.py | 37 ++++++++++--------- 10 files changed, 76 insertions(+), 53 deletions(-) diff --git a/app/assets/scanner.py b/app/assets/scanner.py index 3d4ed945b61..49016234039 100644 --- a/app/assets/scanner.py +++ b/app/assets/scanner.py @@ -56,6 +56,7 @@ get_comfy_models_folders, get_name_and_tags_from_asset_path, ) +from app.assets.services.ingest import _discard_unreferenced_content from app.assets.services.snapshot_hash import snapshot_hash from app.database.db import create_session, run_write_txn diff --git a/app/assets/scanner_admission.py b/app/assets/scanner_admission.py index 8a2b38f6d92..306ef401927 100644 --- a/app/assets/scanner_admission.py +++ b/app/assets/scanner_admission.py @@ -108,7 +108,14 @@ def tick_watch_list( ) emit("scanner.watch_spec_failed", error_type=error_type(exc)) continue - _created, seed_error = insert_asset_specs([spec], set(spec["tags"])) + try: + _created, seed_error = insert_asset_specs([spec], set(spec["tags"])) + except BaseException: + # A fault that escapes the write transaction is the database's, + # not this file's: the seed never happened, so the entry goes + # back rather than being retired by a lock the next tick may get. + _WATCH_LIST.insert(0, entry) + raise if seed_error is not None: logging.warning( "Dropping watched asset after seeding failed: %s", entry.path diff --git a/tests-unit/assets_test/services/test_admission_gate.py b/tests-unit/assets_test/services/test_admission_gate.py index bc4fdc85fb8..6ddcd68b47d 100644 --- a/tests-unit/assets_test/services/test_admission_gate.py +++ b/tests-unit/assets_test/services/test_admission_gate.py @@ -121,7 +121,7 @@ def _stat(path: str): monkeypatch.setattr(scanner_admission, "os", SimpleNamespace(stat=_stat)) with caplog.at_level(logging.INFO): - tick_watch_list(session) + tick_watch_list() session.commit() persisted_paths = set(session.scalars(select(AssetContent.path)).all()) @@ -151,7 +151,7 @@ def test_seed_failure_does_not_stop_watch_list_drain( _WATCH_LIST[:] = [_WatchEntry(str(path), path.stat()) for path in paths] attempted: list[str] = [] - def seed_or_return_error(_session, specs) -> tuple[int, Exception | None]: + def seed_or_return_error(_session, specs, *_args, **_kwargs) -> tuple[int, Exception | None]: path = specs[0]["abs_path"] attempted.append(path) if path == str(paths[0]): @@ -162,7 +162,7 @@ def seed_or_return_error(_session, specs) -> tuple[int, Exception | None]: monkeypatch.setattr("app.assets.scanner.seed_asset_specs", seed_or_return_error) with caplog.at_level(logging.INFO): - tick_watch_list(session) + tick_watch_list() assert attempted == [str(path) for path in paths] assert _WATCH_LIST == [] @@ -212,7 +212,7 @@ def _name_and_tags(path: str) -> tuple[str, list[str]]: ) with caplog.at_level(logging.INFO): - tick_watch_list(session) + tick_watch_list() session.commit() assert set(session.scalars(select(AssetContent.path)).all()) == {str(stable_path)} @@ -239,7 +239,7 @@ def test_unexpected_fault_mid_drain_leaves_unvisited_entries_on_the_watch_list( path.write_bytes(path.name.encode()) _WATCH_LIST[:] = [_WatchEntry(str(path), path.stat()) for path in paths] - def seed_or_explode(_session, specs) -> tuple[int, Exception | None]: + def seed_or_explode(_session, specs, *_args, **_kwargs) -> tuple[int, Exception | None]: if specs[0]["abs_path"] == str(paths[1]): raise MemoryError("forced unrecoverable fault") return 1, None @@ -248,9 +248,9 @@ def seed_or_explode(_session, specs) -> tuple[int, Exception | None]: monkeypatch.setattr("app.assets.scanner.seed_asset_specs", seed_or_explode) with pytest.raises(MemoryError, match="^forced unrecoverable fault$"): - tick_watch_list(session) + tick_watch_list() - assert [entry.path for entry in _WATCH_LIST] == [str(paths[2])] + assert [entry.path for entry in _WATCH_LIST] == [str(paths[1]), str(paths[2])] def test_stable_scan_admission_removes_watch_entry_before_next_tick(session, temp_dir: Path, monkeypatch): @@ -337,10 +337,10 @@ def test_watch_list_interrupts_between_entries(temp_dir: Path, monkeypatch): interrupted = threading.Event() inserted: list[str] = [] - def record_insert(specs, _tags) -> int: + def record_insert(specs, _tags) -> tuple[int, Exception | None]: inserted.append(specs[0]["abs_path"]) interrupted.set() - return 1 + return 1, None monkeypatch.setattr("app.assets.scanner.insert_asset_specs", record_insert) monkeypatch.setattr( diff --git a/tests-unit/assets_test/services/test_enrichment_batching.py b/tests-unit/assets_test/services/test_enrichment_batching.py index f11f5d812a4..3701ac72617 100644 --- a/tests-unit/assets_test/services/test_enrichment_batching.py +++ b/tests-unit/assets_test/services/test_enrichment_batching.py @@ -50,7 +50,7 @@ def test_enrichment_applies_when_unchanged_row_is_behind_disk( stored_mtime_ns=disk_stat.st_mtime_ns - 5_000_000_000, ) - enriched, failed_ids = scanner.enrich_assets_batch( + enriched, failed_ids, _consumed = scanner.enrich_assets_batch( [row], extract_metadata=True, compute_hash=False ) @@ -87,7 +87,7 @@ def change_row(write_session) -> None: db_mod.run_write_txn(change_row) monkeypatch.setattr(scanner, "_prepare_enrichment", lambda *_args: prepared) - enriched, failed_ids = scanner.enrich_assets_batch( + enriched, failed_ids, _consumed = scanner.enrich_assets_batch( [row], extract_metadata=True, compute_hash=False ) diff --git a/tests-unit/assets_test/services/test_queue_drain_characterization.py b/tests-unit/assets_test/services/test_queue_drain_characterization.py index 44f703bb7a7..28ba40df7b6 100644 --- a/tests-unit/assets_test/services/test_queue_drain_characterization.py +++ b/tests-unit/assets_test/services/test_queue_drain_characterization.py @@ -59,7 +59,13 @@ def test_pending_verification_requeues_after_hash_oserror(session, temp_dir, mon assert scanner_changes._pending_verification_ids == [content.id] -def test_watch_list_keeps_entries_when_stat_raises(session, temp_dir, monkeypatch): +def test_watch_list_drops_entries_when_stat_raises(session, temp_dir, monkeypatch): + """#16393 replaced this path's characterized behaviour. + + A stat error used to propagate out of the tick and leave the entry queued, + which re-attempted the same unreadable file on every tick forever. It is now + absorbed: the entry is dropped and the failure is reported as telemetry. + """ path = temp_dir / "watched.bin" path.write_bytes(b"watched") entry = _WatchEntry(str(path), path.stat()) @@ -70,10 +76,9 @@ def test_watch_list_keeps_entries_when_stat_raises(session, temp_dir, monkeypatc lambda *_args, **_kwargs: (_ for _ in ()).throw(PermissionError("denied")), ) - with pytest.raises(PermissionError): - tick_watch_list() + tick_watch_list() - assert _WATCH_LIST == [entry] + assert _WATCH_LIST == [] def test_transition_queue_retries_without_losing_companion_path( diff --git a/tests-unit/assets_test/services/test_scanner_retry_and_counter_integrity.py b/tests-unit/assets_test/services/test_scanner_retry_and_counter_integrity.py index 84d0d101a54..58036e4556e 100644 --- a/tests-unit/assets_test/services/test_scanner_retry_and_counter_integrity.py +++ b/tests-unit/assets_test/services/test_scanner_retry_and_counter_integrity.py @@ -139,7 +139,7 @@ def test_enrichment_batch_failure_rolls_back_every_row_and_counts_each_failure( ) progress = _ScanState() - enriched, failed_ids = scanner.enrich_assets_batch( + enriched, failed_ids, _consumed = scanner.enrich_assets_batch( rows, extract_metadata=False, compute_hash=True, @@ -166,7 +166,7 @@ def test_enrichment_later_batch_failure_preserves_first_batch( ) monkeypatch.setattr(scanner, "run_write_txn", failing_run_write_txn) - enriched, failed_ids = scanner.enrich_assets_batch( + enriched, failed_ids, _consumed = scanner.enrich_assets_batch( rows, extract_metadata=False, compute_hash=True ) diff --git a/tests-unit/assets_test/services/test_scanner_seed_resilience.py b/tests-unit/assets_test/services/test_scanner_seed_resilience.py index e8e1afbb175..c401769d74d 100644 --- a/tests-unit/assets_test/services/test_scanner_seed_resilience.py +++ b/tests-unit/assets_test/services/test_scanner_seed_resilience.py @@ -184,7 +184,7 @@ def _create_content_or_conflict( _create_content_or_conflict, ) - created, error = seed_asset_specs(session, specs) + created, error = seed_asset_specs(session, specs, stat_seed_specs(specs)) session.commit() assert error is None @@ -221,7 +221,8 @@ def _create_record_or_raise( monkeypatch.setattr("app.assets.scanner.create_record", _create_record_or_raise) - _created, error = seed_asset_specs(session, [_spec(path)]) + specs = [_spec(path)] + _created, error = seed_asset_specs(session, specs, stat_seed_specs(specs)) assert error is unrelated_error @@ -251,8 +252,9 @@ def _create_record_or_exhaust( monkeypatch.setattr("app.assets.scanner.create_record", _create_record_or_exhaust) + specs = [_spec(path) for path in paths] with pytest.raises(MemoryError): - seed_asset_specs(session, [_spec(path) for path in paths]) + seed_asset_specs(session, specs, stat_seed_specs(specs)) assert attempted == ["first.bin"] @@ -363,8 +365,9 @@ def test_seed_skips_negative_fresh_mtime_with_warning_and_telemetry( pre_epoch_ns = -315_547_200_000_000_000 os.utime(paths[1], ns=(pre_epoch_ns, pre_epoch_ns)) + specs = [_spec(path) for path in paths] with caplog.at_level(logging.INFO): - created, error = seed_asset_specs(session, [_spec(path) for path in paths]) + created, error = seed_asset_specs(session, specs, stat_seed_specs(specs)) session.commit() assert error is None @@ -395,8 +398,9 @@ def test_seed_emits_one_invalid_mtime_event_for_a_whole_batch_of_pre_epoch_files _ = path.write_bytes(path.name.encode()) os.utime(path, ns=(pre_epoch_ns, pre_epoch_ns)) + specs = [_spec(path) for path in paths] with caplog.at_level(logging.INFO): - created, error = seed_asset_specs(session, [_spec(path) for path in paths]) + created, error = seed_asset_specs(session, specs, stat_seed_specs(specs)) session.commit() assert error is None diff --git a/tests-unit/assets_test/services/test_transition_settle.py b/tests-unit/assets_test/services/test_transition_settle.py index 40cf0be9dbd..e704e9b8554 100644 --- a/tests-unit/assets_test/services/test_transition_settle.py +++ b/tests-unit/assets_test/services/test_transition_settle.py @@ -181,7 +181,6 @@ def enrich_batch(rows, **_kwargs): asset_seeder._cancel_event.clear() with ( - patch("app.assets.seeder.create_session", _create_session), patch("app.assets.scanner.create_session", _create_session), patch("app.assets.seeder.enrich_assets_batch", enrich_batch), ): @@ -224,8 +223,7 @@ def enrich_batch(rows, **_kwargs): ) monkeypatch.setattr(seeder_module, "enrich_assets_batch", enrich_batch) - with patch("app.assets.seeder.create_session", _create_session): - cancelled, enriched = asset_seeder._run_enrich_phase(("input",)) + cancelled, enriched = asset_seeder._run_enrich_phase(("input",)) assert cancelled is False assert enriched == 1 @@ -289,13 +287,13 @@ def get_candidates( asset_seeder._run_gate.clear() return candidates - def enrich_asset(*_args, **kwargs) -> bool: + def prepare_enrichment(row, *_args, **_kwargs): nonlocal interruption_triggered - attempted.append(kwargs["record_id"]) + attempted.append(row.record_id) if len(attempted) == interrupt_after and not interruption_triggered: interruption_triggered = True asset_seeder._run_gate.clear() - return True + return scanner._PreparedEnrichment(row, None, None, None, None, False) result: list[tuple[bool, int]] = [] errors: list[BaseException] = [] @@ -309,12 +307,17 @@ def run_enrich_phase() -> None: monkeypatch.setattr( seeder_module, "get_unenriched_assets_for_roots", get_candidates ) - monkeypatch.setattr(scanner, "enrich_asset", enrich_asset) + monkeypatch.setattr(scanner, "_prepare_enrichment", prepare_enrichment) + # These rows are synthetic, so the batch apply is stubbed and its write + # transaction short-circuited rather than bound to a real writer engine. + monkeypatch.setattr( + scanner, + "_apply_enrichments", + lambda _session, prepared: [item.row.record_id for item in prepared], + ) + monkeypatch.setattr(scanner, "run_write_txn", lambda work: work(None)) - with ( - patch("app.assets.seeder.create_session", _create_session), - patch("app.assets.scanner.create_session", _create_session), - ): + with patch("app.assets.scanner.create_session", _create_session): worker = threading.Thread(target=run_enrich_phase, daemon=True) worker.start() try: diff --git a/tests-unit/assets_test/test_write_contention_e2e.py b/tests-unit/assets_test/test_write_contention_e2e.py index 705fed2a9bc..c664792104c 100644 --- a/tests-unit/assets_test/test_write_contention_e2e.py +++ b/tests-unit/assets_test/test_write_contention_e2e.py @@ -118,7 +118,7 @@ def observe_registration_lock(error): monkeypatch.setattr(scanner, "_apply_enrichments", block_first_scanner_write) monkeypatch.setattr(scanner, "run_write_txn", count_scanner_writes) monkeypatch.setattr(db_mod, "_is_retryable_lock_error", observe_registration_lock) - scanner_result: dict[str, tuple[int, list[str]]] = {} + scanner_result: dict[str, tuple[int, list[str], int]] = {} registration_result: dict[str, RegisteredAsset | None] = {} def enrich_scanner_rows() -> None: @@ -154,7 +154,7 @@ def register_output() -> None: assert not scanner_worker.is_alive() assert not registration_worker.is_alive() - assert scanner_result["value"] == (len(scanner_rows), []) + assert scanner_result["value"] == (len(scanner_rows), [], len(scanner_rows)) assert scanner_writes == 2 result = registration_result["value"] assert isinstance(result, RegisteredAsset) diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.py index 030d5e1c5f1..bb94abb8f0b 100644 --- a/tests-unit/seeder_test/test_seeder.py +++ b/tests-unit/seeder_test/test_seeder.py @@ -1,7 +1,6 @@ import logging import re import threading -from contextlib import contextmanager, nullcontext from pathlib import Path from unittest.mock import Mock @@ -92,7 +91,9 @@ def _configure_fast_phase( specs: list[SeedAssetSpec], ) -> None: monkeypatch.setattr( - seeder_module, "sync_root_safely", lambda _root, _progress: set() + seeder_module, + "sync_root_safely", + lambda _root, _progress, interrupt_check=None: set(), ) monkeypatch.setattr( seeder_module, "collect_paths_for_roots", lambda _roots: [str(path) for path in paths] @@ -102,9 +103,7 @@ def _configure_fast_phase( "build_asset_specs", lambda *_args, **_kwargs: (specs, set(), 0), ) - watch_session = Mock() - monkeypatch.setattr(seeder_module, "create_session", lambda: nullcontext(watch_session)) - monkeypatch.setattr(seeder_module, "tick_watch_list", lambda _session: None) + monkeypatch.setattr(seeder_module, "tick_watch_list", lambda **_kwargs: None) def _run_faulting_fast_phase( @@ -121,15 +120,15 @@ def _run_faulting_fast_phase( path.write_bytes(path.name.encode()) specs = [_seed_spec(path) for path in paths] - @contextmanager - def database_session(): - with Session(engine) as session: - if commit_failure is not None: - session.connection().exec_driver_sql("BEGIN") - monkeypatch.setattr( - session, "commit", Mock(side_effect=commit_failure) - ) - yield session + def write_session() -> Session: + session = Session(engine) + if commit_failure is not None: + # A real transaction, so the rollback run_write_txn does after the + # failed commit actually discards the batch; pysqlite would otherwise + # leave the savepoint inserts behind. + session.connection().exec_driver_sql("BEGIN") + monkeypatch.setattr(session, "commit", Mock(side_effect=commit_failure)) + return session def create_record_or_raise( session: Session, @@ -153,7 +152,7 @@ def create_record_or_raise( tags=tags, ) - monkeypatch.setattr(scanner_module, "create_session", database_session) + monkeypatch.setattr("app.database.db.WriteSession", write_session) monkeypatch.setattr(scanner_module, "create_record", create_record_or_raise) monkeypatch.setattr(scanner_module.mode, "hashing_enabled", lambda: False) _configure_fast_phase(monkeypatch, paths, specs) @@ -727,10 +726,14 @@ def test_scan_prune_failure_completes_without_type_error( scan_seeder._phase = ScanPhase.FAST monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: []) monkeypatch.setattr( - seeder_module, "mark_missing_outside_prefixes_safely", lambda _prefixes: None + seeder_module, + "mark_missing_outside_prefixes_safely", + lambda _prefixes, interrupt_check=None: None, ) monkeypatch.setattr( - seeder_module, "sync_temp_references_safely", lambda _progress: None + seeder_module, + "sync_temp_references_safely", + lambda _progress, interrupt_check=None: None, ) monkeypatch.setattr(scan_seeder, "_run_fast_phase", lambda _roots: (0, 0, 0))