(fix) dictionary too big -> out of memory

docker-compose.yml

@@ -52,7 +52,8 @@ services:
     image: mariadb:12.1
     restart: unless-stopped
     volumes:
-      - ./lwt_db_data:/var/lib/mysql
+      # A named volume, NOT a host bind mount
+      - lwt_db_data:/var/lib/mysql
 
   nlp:
     build:
@@ -71,3 +72,4 @@ services:
 
 volumes:
   nlp_voices:
+  lwt_db_data:

src/Modules/Vocabulary/Application/UseCases/FindSimilarTerms.php

@@ -19,6 +19,7 @@
 
 use Lwt\Shared\Infrastructure\Database\QueryBuilder;
 use Lwt\Shared\Infrastructure\Database\Settings;
+use Lwt\Modules\Vocabulary\Application\Services\LetterPairProfile;
 use Lwt\Modules\Vocabulary\Application\Services\SimilarityCalculator;
 use Lwt\Modules\Vocabulary\Domain\LemmatizerInterface;
 use Lwt\Modules\Vocabulary\Infrastructure\Lemmatizers\DictionaryLemmatizer;
@@ -89,33 +90,42 @@ public function execute(
         float $minRanking,
         float $phoneticWeight = 0.3
     ): array {
+        if ($maxCount <= 0) {
+            return [];
+        }
+
         $comparedTermLc = mb_strtolower($comparedTerm, 'UTF-8');
 
-        // Fetch words with their status for weighting
+        $lemmaLc = $this->resolveLemma($languageId, $comparedTermLc);
+        $term = $this->calculator->profile($comparedTermLc);
+
+        // Score all rows as they arrive. Only keep > threshold.
+        $pool = [];
         $rows = QueryBuilder::table('words')
             ->select(['WoID', 'WoTextLC', 'WoStatus', 'WoLemmaLC'])
             ->where('WoLgID', '=', $languageId)
             ->where('WoTextLC', '<>', $comparedTermLc)
-            ->getPrepared();
+            ->eachPrepared();
 
-        $candidates = [];
         foreach ($rows as $record) {
-            $candidates[] = [
-                'id' => (int) $record["WoID"],
-                'textLc' => (string) $record["WoTextLC"],
-                'status' => (int) $record["WoStatus"],
-                'lemmaLc' => (string) ($record["WoLemmaLC"] ?? ''),
-            ];
+            $entry = $this->poolEntry(
+                [
+                    'id' => (int) $record["WoID"],
+                    'textLc' => (string) $record["WoTextLC"],
+                    'status' => (int) $record["WoStatus"],
+                    'lemmaLc' => (string) ($record["WoLemmaLC"] ?? ''),
+                ],
+                $term,
+                $minRanking,
+                $phoneticWeight,
+                $lemmaLc
+            );
+            if ($entry !== null) {
+                $pool[] = $entry;
+            }
         }
 
-        return $this->rankByCoverage(
-            $candidates,
-            $comparedTermLc,
-            $maxCount,
-            $minRanking,
-            $phoneticWeight,
-            $this->resolveLemma($languageId, $comparedTermLc)
-        );
+        return $this->selectByCoverage($pool, $term, $maxCount, $phoneticWeight);
     }
 
     /**
@@ -221,29 +231,79 @@ public function rankByCoverage(
 
         $pool = [];
         foreach ($candidates as $candidate) {
-            $profile = $this->calculator->profile($candidate['textLc']);
-            $baseSimilarity = $this->calculator->getResidualCombinedRanking(
-                $profile,
-                $term,
-                $phoneticWeight
-            );
-            $isFamily = $this->sharesWordFamily($candidate, $lemmaLc);
-
-            // The threshold reads the unweighted score, as it always has
-            if (!$isFamily && $baseSimilarity < $minRanking) {
-                continue;
+            $entry = $this->poolEntry($candidate, $term, $minRanking, $phoneticWeight, $lemmaLc);
+            if ($entry !== null) {
+                $pool[] = $entry;
             }
+        }
+
+        return $this->selectByCoverage($pool, $term, $maxCount, $phoneticWeight);
+    }
 
-            $statusWeight = $this->calculator->getStatusWeight($candidate['status']);
-            $pool[] = [
-                'id' => $candidate['id'],
-                'profile' => $profile,
-                'family' => $isFamily,
-                'weight' => $statusWeight,
-                'weighted' => $baseSimilarity * $statusWeight,
-            ];
+    /**
+     * Score one candidate and admit it to the pool, or turn it away.
+     *
+     * Split out so that {@see execute()} can score rows as they stream in
+     * rather than collecting them all first — the scoring itself is unchanged,
+     * and both callers share this so the two paths cannot drift apart.
+     *
+     * @param array{id: int, textLc: string, status: int, lemmaLc?: string} $candidate      Candidate
+     * @param LetterPairProfile                                             $term           Searched term's profile
+     * @param float                                                         $minRanking     Minimum (0-1)
+     * @param float                                                         $phoneticWeight Phonetic (0-1)
+     * @param string                                                        $lemmaLc        Term's lemma
+     *
+     * @return array{id: int, profile: LetterPairProfile, family: bool, weight: float, weighted: float}|null
+     *         The pool entry, or null when the candidate does not qualify
+     */
+    private function poolEntry(
+        array $candidate,
+        LetterPairProfile $term,
+        float $minRanking,
+        float $phoneticWeight,
+        string $lemmaLc
+    ): ?array {
+        $profile = $this->calculator->profile($candidate['textLc']);
+        $baseSimilarity = $this->calculator->getResidualCombinedRanking(
+            $profile,
+            $term,
+            $phoneticWeight
+        );
+        $isFamily = $this->sharesWordFamily($candidate, $lemmaLc);
+
+        // The threshold reads the unweighted score, as it always has
+        if (!$isFamily && $baseSimilarity < $minRanking) {
+            return null;
         }
 
+        $statusWeight = $this->calculator->getStatusWeight($candidate['status']);
+
+        return [
+            'id' => $candidate['id'],
+            'profile' => $profile,
+            'family' => $isFamily,
+            'weight' => $statusWeight,
+            'weighted' => $baseSimilarity * $statusWeight,
+        ];
+    }
+
+    /**
+     * Pick from an already-scored pool, one at a time, by residual coverage.
+     *
+     * @param list<array{id: int, profile: LetterPairProfile, family: bool, weight: float, weighted: float}> $pool
+     *        Candidates that cleared the threshold
+     * @param LetterPairProfile $term           Searched term's profile
+     * @param int               $maxCount       Maximum to return
+     * @param float             $phoneticWeight Phonetic (0-1)
+     *
+     * @return list<int> Word IDs, most useful first
+     */
+    private function selectByCoverage(
+        array $pool,
+        LetterPairProfile $term,
+        int $maxCount,
+        float $phoneticWeight
+    ): array {
         $remaining = $term;
         $picked = [];
         $wanted = min($maxCount, count($pool));

src/Shared/Infrastructure/Database/Connection.php

@@ -326,6 +326,29 @@ public static function preparedFetchAll(string $sql, array $params = []): array
         return $stmt->fetchAll();
     }
 
+    /**
+     * Execute a parameterized query and yield rows one at a time.
+     *
+     * The streaming counterpart of {@see preparedFetchAll()}, for scans whose
+     * result set is too large to hold in memory. See
+     * {@see PreparedStatement::fetchEach()} for the constraint this puts on
+     * the loop body: no other query may run on the connection until the
+     * generator is finished.
+     *
+     * @param string            $sql    The SQL query with ? placeholders
+     * @param array<int, mixed> $params Parameters to bind (indexed array)
+     *
+     * @return \Generator<int, array<string, mixed>> Rows, one at a time
+     */
+    public static function preparedFetchEach(string $sql, array $params = []): \Generator
+    {
+        $stmt = self::prepare($sql);
+        if (!empty($params)) {
+            $stmt->bindValues($params);
+        }
+        yield from $stmt->fetchEach();
+    }
+
     /**
      * Execute a parameterized query and return the first row.
      *

src/Shared/Infrastructure/Database/PreparedStatement.php

@@ -214,6 +214,96 @@ public function fetchAll(): array
         return $rows;
     }
 
+    /**
+     * Execute and yield rows one at a time, without buffering the result set.
+     *
+     * `fetchAll()` materialises every row before the caller sees any of them,
+     * so scanning a large table costs memory proportional to the whole result
+     * — enough to exhaust the memory limit on a big vocabulary. This pulls
+     * rows from the server on demand and holds only the current one, which is
+     * what lets a caller scan a table it could never fit in memory.
+     *
+     * The result set stays open on the connection for the life of the
+     * generator, and mysqli permits no other query on that connection until it
+     * is done. So the loop body must not run queries of its own: gather
+     * anything else the loop needs before iteration starts.
+     *
+     * @return \Generator<int, array<string, mixed>> Rows, one at a time
+     *
+     * @throws DatabaseException If execution fails
+     *
+     * @psalm-suppress MixedAssignment Column values are mixed, as in fetchAll()
+     */
+    public function fetchEach(): \Generator
+    {
+        $params = empty($this->boundParams) ? null : $this->boundParams;
+        if (!$this->stmt->execute($params)) {
+            throw new DatabaseException(
+                'Failed to execute statement: ' . $this->stmt->error,
+                0,
+                null,
+                $this->sql,
+                $this->stmt->errno
+            );
+        }
+
+        $meta = $this->stmt->result_metadata();
+        if ($meta === false) {
+            // For queries that don't return a result set
+            return;
+        }
+
+        $columns = [];
+        foreach ($meta->fetch_fields() as $field) {
+            $columns[] = $field->name;
+        }
+        $meta->free();
+
+        // bind_result() binds by reference and rewrites the same slots on every
+        // fetch(), so each row is copied out before it leaves this method.
+        $current = array_fill(0, count($columns), null);
+        $this->bindRowSlots($current);
+
+        try {
+            while ($this->stmt->fetch()) {
+                $row = [];
+                foreach ($columns as $slot => $name) {
+                    $row[$name] = $current[$slot];
+                }
+                yield $row;
+            }
+        } finally {
+            // Release the result set even when the caller abandons the
+            // generator early, or the connection stays unusable.
+            $this->stmt->free_result();
+        }
+    }
+
+    /**
+     * Bind each column of the open result set to a slot of the given array.
+     *
+     * mysqli's bind_result() takes its destinations by reference, so this has
+     * to build an array of references into $target — an idiom Psalm cannot
+     * model, which is what the suppressions below are for. Kept in its own
+     * method so the suppressions cover only this, and not the fetch loop.
+     *
+     * @param array<int, mixed> $target Slots rewritten on every fetch()
+     *
+     * @psalm-suppress UnsupportedReferenceUsage
+     * @psalm-suppress UnusedVariable
+     * @psalm-suppress MixedAssignment
+     *
+     * @return void
+     */
+    private function bindRowSlots(array &$target): void
+    {
+        $refs = [];
+        foreach (array_keys($target) as $slot) {
+            $refs[] = &$target[$slot];
+        }
+        $this->stmt->bind_result(...$refs);
+    }
+
     /**
      * Execute and fetch the first row.
      *

src/Shared/Infrastructure/Database/QueryBuilder.php

@@ -1032,6 +1032,23 @@ public function getPrepared(): array
         return Connection::preparedFetchAll($sql, $this->bindings);
     }
 
+    /**
+     * Execute the query using prepared statements and yield rows one at a time.
+     *
+     * The streaming counterpart of {@see getPrepared()}, for scans too large
+     * to materialise. The query is built and sent when iteration starts, not
+     * when this is called. No other query may run on the connection until the
+     * generator is finished — see {@see PreparedStatement::fetchEach()}.
+     *
+     * @return \Generator<int, array<string, mixed>> Rows, one at a time
+     */
+    public function eachPrepared(): \Generator
+    {
+        $this->applyUserScope();
+        $sql = $this->toSqlPrepared();
+        yield from Connection::preparedFetchEach($sql, $this->bindings);
+    }
+
     /**
      * Execute the query using prepared statements and return the first result.
      *