From 18e6d6cd25c5d43143713eb2973c478f311707de Mon Sep 17 00:00:00 2001 From: sarahmccuan Date: Sat, 22 Aug 2026 17:31:25 -0500 Subject: [PATCH 1/2] change volume to named volume rather than host bind mount --- docker-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 20a8741bb..55f912755 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,7 +52,8 @@ services: image: mariadb:12.1 restart: unless-stopped volumes: - - ./lwt_db_data:/var/lib/mysql + # A named volume, NOT a host bind mount + - lwt_db_data:/var/lib/mysql nlp: build: @@ -71,3 +72,4 @@ services: volumes: nlp_voices: + lwt_db_data: From 43311de0c9a8d51b4e36901d4fb7375f88daeba0 Mon Sep 17 00:00:00 2001 From: sarahmccuan Date: Sun, 23 Aug 2026 18:10:59 -0500 Subject: [PATCH 2/2] memory fix x1 --- .../Application/UseCases/FindSimilarTerms.php | 132 +++++++++++++----- .../Infrastructure/Database/Connection.php | 23 +++ .../Database/PreparedStatement.php | 90 ++++++++++++ .../Infrastructure/Database/QueryBuilder.php | 17 +++ 4 files changed, 226 insertions(+), 36 deletions(-) diff --git a/src/Modules/Vocabulary/Application/UseCases/FindSimilarTerms.php b/src/Modules/Vocabulary/Application/UseCases/FindSimilarTerms.php index fb139b30c..af95c8bdb 100644 --- a/src/Modules/Vocabulary/Application/UseCases/FindSimilarTerms.php +++ b/src/Modules/Vocabulary/Application/UseCases/FindSimilarTerms.php @@ -19,6 +19,7 @@ use Lwt\Shared\Infrastructure\Database\QueryBuilder; use Lwt\Shared\Infrastructure\Database\Settings; +use Lwt\Modules\Vocabulary\Application\Services\LetterPairProfile; use Lwt\Modules\Vocabulary\Application\Services\SimilarityCalculator; use Lwt\Modules\Vocabulary\Domain\LemmatizerInterface; use Lwt\Modules\Vocabulary\Infrastructure\Lemmatizers\DictionaryLemmatizer; @@ -89,33 +90,42 @@ public function execute( float $minRanking, float $phoneticWeight = 0.3 ): array { + if ($maxCount <= 0) { + return []; + } + $comparedTermLc = mb_strtolower($comparedTerm, 'UTF-8'); - // Fetch words with their status for weighting + $lemmaLc = $this->resolveLemma($languageId, $comparedTermLc); + $term = $this->calculator->profile($comparedTermLc); + + // Score all rows as they arrive. Only keep > threshold. + $pool = []; $rows = QueryBuilder::table('words') ->select(['WoID', 'WoTextLC', 'WoStatus', 'WoLemmaLC']) ->where('WoLgID', '=', $languageId) ->where('WoTextLC', '<>', $comparedTermLc) - ->getPrepared(); + ->eachPrepared(); - $candidates = []; foreach ($rows as $record) { - $candidates[] = [ - 'id' => (int) $record["WoID"], - 'textLc' => (string) $record["WoTextLC"], - 'status' => (int) $record["WoStatus"], - 'lemmaLc' => (string) ($record["WoLemmaLC"] ?? ''), - ]; + $entry = $this->poolEntry( + [ + 'id' => (int) $record["WoID"], + 'textLc' => (string) $record["WoTextLC"], + 'status' => (int) $record["WoStatus"], + 'lemmaLc' => (string) ($record["WoLemmaLC"] ?? ''), + ], + $term, + $minRanking, + $phoneticWeight, + $lemmaLc + ); + if ($entry !== null) { + $pool[] = $entry; + } } - return $this->rankByCoverage( - $candidates, - $comparedTermLc, - $maxCount, - $minRanking, - $phoneticWeight, - $this->resolveLemma($languageId, $comparedTermLc) - ); + return $this->selectByCoverage($pool, $term, $maxCount, $phoneticWeight); } /** @@ -221,29 +231,79 @@ public function rankByCoverage( $pool = []; foreach ($candidates as $candidate) { - $profile = $this->calculator->profile($candidate['textLc']); - $baseSimilarity = $this->calculator->getResidualCombinedRanking( - $profile, - $term, - $phoneticWeight - ); - $isFamily = $this->sharesWordFamily($candidate, $lemmaLc); - - // The threshold reads the unweighted score, as it always has - if (!$isFamily && $baseSimilarity < $minRanking) { - continue; + $entry = $this->poolEntry($candidate, $term, $minRanking, $phoneticWeight, $lemmaLc); + if ($entry !== null) { + $pool[] = $entry; } + } + + return $this->selectByCoverage($pool, $term, $maxCount, $phoneticWeight); + } - $statusWeight = $this->calculator->getStatusWeight($candidate['status']); - $pool[] = [ - 'id' => $candidate['id'], - 'profile' => $profile, - 'family' => $isFamily, - 'weight' => $statusWeight, - 'weighted' => $baseSimilarity * $statusWeight, - ]; + /** + * Score one candidate and admit it to the pool, or turn it away. + * + * Split out so that {@see execute()} can score rows as they stream in + * rather than collecting them all first — the scoring itself is unchanged, + * and both callers share this so the two paths cannot drift apart. + * + * @param array{id: int, textLc: string, status: int, lemmaLc?: string} $candidate Candidate + * @param LetterPairProfile $term Searched term's profile + * @param float $minRanking Minimum (0-1) + * @param float $phoneticWeight Phonetic (0-1) + * @param string $lemmaLc Term's lemma + * + * @return array{id: int, profile: LetterPairProfile, family: bool, weight: float, weighted: float}|null + * The pool entry, or null when the candidate does not qualify + */ + private function poolEntry( + array $candidate, + LetterPairProfile $term, + float $minRanking, + float $phoneticWeight, + string $lemmaLc + ): ?array { + $profile = $this->calculator->profile($candidate['textLc']); + $baseSimilarity = $this->calculator->getResidualCombinedRanking( + $profile, + $term, + $phoneticWeight + ); + $isFamily = $this->sharesWordFamily($candidate, $lemmaLc); + + // The threshold reads the unweighted score, as it always has + if (!$isFamily && $baseSimilarity < $minRanking) { + return null; } + $statusWeight = $this->calculator->getStatusWeight($candidate['status']); + + return [ + 'id' => $candidate['id'], + 'profile' => $profile, + 'family' => $isFamily, + 'weight' => $statusWeight, + 'weighted' => $baseSimilarity * $statusWeight, + ]; + } + + /** + * Pick from an already-scored pool, one at a time, by residual coverage. + * + * @param list $pool + * Candidates that cleared the threshold + * @param LetterPairProfile $term Searched term's profile + * @param int $maxCount Maximum to return + * @param float $phoneticWeight Phonetic (0-1) + * + * @return list Word IDs, most useful first + */ + private function selectByCoverage( + array $pool, + LetterPairProfile $term, + int $maxCount, + float $phoneticWeight + ): array { $remaining = $term; $picked = []; $wanted = min($maxCount, count($pool)); diff --git a/src/Shared/Infrastructure/Database/Connection.php b/src/Shared/Infrastructure/Database/Connection.php index 3c1bc976f..101a9cdb9 100644 --- a/src/Shared/Infrastructure/Database/Connection.php +++ b/src/Shared/Infrastructure/Database/Connection.php @@ -326,6 +326,29 @@ public static function preparedFetchAll(string $sql, array $params = []): array return $stmt->fetchAll(); } + /** + * Execute a parameterized query and yield rows one at a time. + * + * The streaming counterpart of {@see preparedFetchAll()}, for scans whose + * result set is too large to hold in memory. See + * {@see PreparedStatement::fetchEach()} for the constraint this puts on + * the loop body: no other query may run on the connection until the + * generator is finished. + * + * @param string $sql The SQL query with ? placeholders + * @param array $params Parameters to bind (indexed array) + * + * @return \Generator> Rows, one at a time + */ + public static function preparedFetchEach(string $sql, array $params = []): \Generator + { + $stmt = self::prepare($sql); + if (!empty($params)) { + $stmt->bindValues($params); + } + yield from $stmt->fetchEach(); + } + /** * Execute a parameterized query and return the first row. * diff --git a/src/Shared/Infrastructure/Database/PreparedStatement.php b/src/Shared/Infrastructure/Database/PreparedStatement.php index 2f92d795a..42465b190 100644 --- a/src/Shared/Infrastructure/Database/PreparedStatement.php +++ b/src/Shared/Infrastructure/Database/PreparedStatement.php @@ -214,6 +214,96 @@ public function fetchAll(): array return $rows; } + /** + * Execute and yield rows one at a time, without buffering the result set. + * + * `fetchAll()` materialises every row before the caller sees any of them, + * so scanning a large table costs memory proportional to the whole result + * — enough to exhaust the memory limit on a big vocabulary. This pulls + * rows from the server on demand and holds only the current one, which is + * what lets a caller scan a table it could never fit in memory. + * + * The result set stays open on the connection for the life of the + * generator, and mysqli permits no other query on that connection until it + * is done. So the loop body must not run queries of its own: gather + * anything else the loop needs before iteration starts. + * + * @return \Generator> Rows, one at a time + * + * @throws DatabaseException If execution fails + * + * @psalm-suppress MixedAssignment Column values are mixed, as in fetchAll() + */ + public function fetchEach(): \Generator + { + $params = empty($this->boundParams) ? null : $this->boundParams; + if (!$this->stmt->execute($params)) { + throw new DatabaseException( + 'Failed to execute statement: ' . $this->stmt->error, + 0, + null, + $this->sql, + $this->stmt->errno + ); + } + + $meta = $this->stmt->result_metadata(); + if ($meta === false) { + // For queries that don't return a result set + return; + } + + $columns = []; + foreach ($meta->fetch_fields() as $field) { + $columns[] = $field->name; + } + $meta->free(); + + // bind_result() binds by reference and rewrites the same slots on every + // fetch(), so each row is copied out before it leaves this method. + $current = array_fill(0, count($columns), null); + $this->bindRowSlots($current); + + try { + while ($this->stmt->fetch()) { + $row = []; + foreach ($columns as $slot => $name) { + $row[$name] = $current[$slot]; + } + yield $row; + } + } finally { + // Release the result set even when the caller abandons the + // generator early, or the connection stays unusable. + $this->stmt->free_result(); + } + } + + /** + * Bind each column of the open result set to a slot of the given array. + * + * mysqli's bind_result() takes its destinations by reference, so this has + * to build an array of references into $target — an idiom Psalm cannot + * model, which is what the suppressions below are for. Kept in its own + * method so the suppressions cover only this, and not the fetch loop. + * + * @param array $target Slots rewritten on every fetch() + * + * @psalm-suppress UnsupportedReferenceUsage + * @psalm-suppress UnusedVariable + * @psalm-suppress MixedAssignment + * + * @return void + */ + private function bindRowSlots(array &$target): void + { + $refs = []; + foreach (array_keys($target) as $slot) { + $refs[] = &$target[$slot]; + } + $this->stmt->bind_result(...$refs); + } + /** * Execute and fetch the first row. * diff --git a/src/Shared/Infrastructure/Database/QueryBuilder.php b/src/Shared/Infrastructure/Database/QueryBuilder.php index 9040d7711..d9577fcbb 100644 --- a/src/Shared/Infrastructure/Database/QueryBuilder.php +++ b/src/Shared/Infrastructure/Database/QueryBuilder.php @@ -1032,6 +1032,23 @@ public function getPrepared(): array return Connection::preparedFetchAll($sql, $this->bindings); } + /** + * Execute the query using prepared statements and yield rows one at a time. + * + * The streaming counterpart of {@see getPrepared()}, for scans too large + * to materialise. The query is built and sent when iteration starts, not + * when this is called. No other query may run on the connection until the + * generator is finished — see {@see PreparedStatement::fetchEach()}. + * + * @return \Generator> Rows, one at a time + */ + public function eachPrepared(): \Generator + { + $this->applyUserScope(); + $sql = $this->toSqlPrepared(); + yield from Connection::preparedFetchEach($sql, $this->bindings); + } + /** * Execute the query using prepared statements and return the first result. *