fix(language): make the parser setting real, and Chinese use jieba (#278)

CHANGELOG.md

@@ -86,6 +86,22 @@ ones are marked like "v1.0.0-fork".
   unchanged; the work behind them no longer grows with the size of your
   vocabulary.
 
+* **Chinese texts could not be read** (#278). A Chinese language created from
+  the built-in preset produced a text with no clickable words at all — nothing
+  to look up, nothing to track. Chinese and Japanese now come out of the preset
+  already pointed at a real tokenizer: **jieba** segments Chinese into words
+  rather than characters, and Japanese asks for MeCab as it always claimed to.
+  Where that tokenizer is not installed the language falls back to
+  character-by-character parsing, so it stays readable either way.
+
+* **Choosing a parser in the language form did nothing** (#278). The Parser
+  Type menu wrote its value to the database and no part of the parsing pipeline
+  ever read it — picking "Jieba (Chinese)" or "MeCab Python" parsed exactly as
+  before. The setting is now honoured. It also lists the parsers from
+  `config/parsers.php`, which it never did, so jieba and MeCab Python appear on
+  an install that has them. A language that names no parser — every language
+  that exists today — parses exactly as it did.
+
 ## [3.4.2-fork] - 2026-08-16
 
 ### Fixed

locale/de/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "Der Text wird diese Größe haben",
     "form.section_text_processing": "Textverarbeitung",
     "form.parser_type": "Parser-Typ",
+    "form.parser_automatic": "Automatisch",
     "form.parser_unavailable": " (nicht verfügbar)",
     "form.character_substitutions": "Zeichenersetzungen",
     "form.character_substitutions_help": "Zeichen vor dem Parsen ersetzen (Format: von=zu, getrennt durch |)",

locale/en/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "Text will be this size",
     "form.section_text_processing": "Text Processing",
     "form.parser_type": "Parser Type",
+    "form.parser_automatic": "Automatic",
     "form.parser_unavailable": " (unavailable)",
     "form.character_substitutions": "Character Substitutions",
     "form.character_substitutions_help": "Replace characters before parsing (format: from=to, separated by |)",

locale/es/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "El texto tendrá este tamaño",
     "form.section_text_processing": "Procesamiento de texto",
     "form.parser_type": "Tipo de analizador",
+    "form.parser_automatic": "Automático",
     "form.parser_unavailable": " (no disponible)",
     "form.character_substitutions": "Sustituciones de caracteres",
     "form.character_substitutions_help": "Reemplaza caracteres antes del análisis (formato: de=a, separados por |)",

locale/fr/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "Le texte aura cette taille",
     "form.section_text_processing": "Traitement du texte",
     "form.parser_type": "Type d'analyseur",
+    "form.parser_automatic": "Automatique",
     "form.parser_unavailable": " (indisponible)",
     "form.character_substitutions": "Substitutions de caractères",
     "form.character_substitutions_help": "Remplacer les caractères avant l'analyse (format : de=à, séparés par |)",

locale/it/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "Il testo avrà questa dimensione",
     "form.section_text_processing": "Elaborazione del testo",
     "form.parser_type": "Tipo di parser",
+    "form.parser_automatic": "Automatico",
     "form.parser_unavailable": " (non disponibile)",
     "form.character_substitutions": "Sostituzioni di caratteri",
     "form.character_substitutions_help": "Sostituisci i caratteri prima dell'analisi (formato: da=a, separati da |)",

locale/ja/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "テキストはこのサイズになります",
     "form.section_text_processing": "テキスト処理",
     "form.parser_type": "パーサータイプ",
+    "form.parser_automatic": "自動",
     "form.parser_unavailable": "（利用不可）",
     "form.character_substitutions": "文字置換",
     "form.character_substitutions_help": "解析前に文字を置換します（形式：from=to、|で区切り）",

locale/pt/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "O texto terá este tamanho",
     "form.section_text_processing": "Processamento de Texto",
     "form.parser_type": "Tipo de Analisador",
+    "form.parser_automatic": "Automático",
     "form.parser_unavailable": " (indisponível)",
     "form.character_substitutions": "Substituições de Caracteres",
     "form.character_substitutions_help": "Substituir caracteres antes da análise (formato: de=para, separados por |)",

locale/ru/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "Текст будет такого размера",
     "form.section_text_processing": "Обработка текста",
     "form.parser_type": "Тип парсера",
+    "form.parser_automatic": "Автоматически",
     "form.parser_unavailable": " (недоступно)",
     "form.character_substitutions": "Замены символов",
     "form.character_substitutions_help": "Заменять символы перед парсингом (формат: from=to, через |)",

locale/zh/language.json

@@ -79,6 +79,7 @@
     "form.text_size_example": "文本将以此大小显示",
     "form.section_text_processing": "文本处理",
     "form.parser_type": "解析器类型",
+    "form.parser_automatic": "自动",
     "form.parser_unavailable": "（不可用）",
     "form.character_substitutions": "字符替换",
     "form.character_substitutions_help": "解析前替换字符（格式：from=to，用 | 分隔）",

src/Modules/Admin/Application/Services/TtsService.php

@@ -57,7 +57,7 @@ public function __construct(?LanguageFacade $languageService = null)
      *
      * @param int   $lgId       Language ID
      * @param array<string, array{
-     *     0: string, 1: string, 2: bool, 3: string, 4: string, 5: bool, 6: bool, 7: bool
+     *     0: string, 1: string, 2: bool, 3: string, 4: string, 5: bool, 6: bool, 7: bool, 8: string
      * }> $langArray Languages array from langdefs
      *
      * @return string Two-letter language code
@@ -94,7 +94,7 @@ public function getLanguageIdFromCode(string $code, array $langArray): int
      * Get language options for TTS form.
      *
      * @param array<string, array{
-     *     0: string, 1: string, 2: bool, 3: string, 4: string, 5: bool, 6: bool, 7: bool
+     *     0: string, 1: string, 2: bool, 3: string, 4: string, 5: bool, 6: bool, 7: bool, 8: string
      * }> $langArray Languages array from langdefs
      *
      * @return string HTML-formatted options string
@@ -117,7 +117,7 @@ public function getLanguageOptions(array $langArray): string
      * Get current language code for TTS settings.
      *
      * @param array<string, array{
-     *     0: string, 1: string, 2: bool, 3: string, 4: string, 5: bool, 6: bool, 7: bool
+     *     0: string, 1: string, 2: bool, 3: string, 4: string, 5: bool, 6: bool, 7: bool, 8: string
      * }> $langArray Languages array from langdefs
      *
      * @return string Current language code

src/Modules/Language/Application/LanguageFacade.php

@@ -354,7 +354,7 @@ public function getLanguageName($id): string
      *
      * @param int   $id             Language ID
      * @param array<string, array{0: string, 1: string, 2: bool, 3: string,
-     *               4: string, 5: bool, 6: bool, 7: bool}> $languagesTable
+     *               4: string, 5: bool, 6: bool, 7: bool, 8: string}> $languagesTable
      *               Table of languages, usually from LanguagePresets::getAll()
      *
      * @return string Two-letter code (e.g., BCP 47) or empty string

src/Modules/Language/Application/UseCases/GetLanguageCode.php

@@ -63,7 +63,7 @@ public function getLanguageName($id): string
      *
      * @param int   $id             Language ID
      * @param array<string, array{0: string, 1: string, 2: bool, 3: string,
-     *               4: string, 5: bool, 6: bool, 7: bool}> $languagesTable
+     *               4: string, 5: bool, 6: bool, 7: bool, 8: string}> $languagesTable
      *               Table of languages, usually from LanguagePresets::getAll()
      *
      * @return string Two-letter code (e.g., BCP 47) or empty string

src/Modules/Language/Http/LanguageApiHandler.php

@@ -482,7 +482,8 @@ public function formatGetDefinitions(): array
                 'sentSplRegExp' => $def[4],
                 'makeCharacterWord' => $def[5],
                 'removeSpaces' => $def[6],
-                'rightToLeft' => $def[7]
+                'rightToLeft' => $def[7],
+                'parserType' => $def[8]
             ];
         }
 

src/Modules/Language/Infrastructure/Data/langdefs.json

@@ -58,7 +58,7 @@
         "makeCharacterWord": true,
         "removeSpaces": true,
         "rightToLeft": false,
-        "parserType": "character"
+        "parserType": "jieba"
     },
     "Chinese (Traditional)": {
         "glosbeIso": "zh",
@@ -69,7 +69,7 @@
         "makeCharacterWord": true,
         "removeSpaces": true,
         "rightToLeft": false,
-        "parserType": "character"
+        "parserType": "jieba"
     },
     "Croatian": {
         "glosbeIso": "hr",

src/Modules/Language/Infrastructure/Parser/ParserRegistry.php

@@ -42,11 +42,16 @@ class ParserRegistry
     /**
      * Create a new parser registry with default parsers.
      *
-     * @param ExternalParserLoader|null $externalLoader Optional loader for external parsers
+     * Builds its own loader when none is given, so that a registry constructed
+     * directly still sees the parsers in config/parsers.php. It did not before,
+     * which is why jieba never reached the language form's parser list even on
+     * an install where it was installed and working.
+     *
+     * @param ExternalParserLoader|null $externalLoader Loader for external parsers
      */
     public function __construct(?ExternalParserLoader $externalLoader = null)
     {
-        $this->externalLoader = $externalLoader;
+        $this->externalLoader = $externalLoader ?? new ExternalParserLoader();
         $this->registerDefaultParsers();
         $this->registerExternalParsers();
     }
@@ -176,6 +181,40 @@ public function getDefaultType(): string
         return self::DEFAULT_PARSER;
     }
 
+    /**
+     * The parser a language deliberately asked for, if any.
+     *
+     * Only an explicit, non-default `LgParserType` counts. The legacy signals
+     * resolveParserTypeFromRow() also understands — the MECAB magic word and
+     * the split-each-character flag — are deliberately ignored here: the
+     * pipeline has always handled those itself, and every language predating
+     * this field carries no parser type at all, so returning null for them
+     * keeps their parsing byte-identical.
+     *
+     * @param array<string, mixed> $row Database row with Lg* prefixed columns
+     *
+     * @return ParserInterface|null The chosen parser, or null to leave the
+     *         language on the built-in pipeline
+     */
+    public function getOptedInParserFromRow(array $row): ?ParserInterface
+    {
+        $type = trim((string) ($row['LgParserType'] ?? ''));
+        if ($type === '' || $type === self::DEFAULT_PARSER) {
+            return null;
+        }
+
+        $parser = $this->get($type);
+        if ($parser === null || !$parser->isAvailable()) {
+            // An unavailable parser must not drop the language onto the regex
+            // parser: for the CJK languages that ask for jieba or mecab, that
+            // yields a text with no words at all. The built-in pipeline still
+            // honours their split-each-character setting, so fall back to it.
+            return null;
+        }
+
+        return $parser;
+    }
+
     /**
      * Resolve the parser type for a language.
      *

src/Modules/Language/Views/form.php

@@ -432,6 +432,7 @@ class="input"
                     <div class="control">
                         <div class="select is-fullwidth">
                             <select name="LgParserType" id="LgParserType" x-model="lang.parserType">
+                                <option value=""><?php echo __('language.form.parser_automatic'); ?></option>
                                 <?php foreach ($parserInfo as $type => $info) :
                                     $infoAvailable = isset($info['available']) && $info['available'];
                                     $infoName = isset($info['name']) && is_string($info['name']) ? $info['name'] : '';

src/Shared/Infrastructure/Database/TextParsing.php

@@ -18,6 +18,8 @@
 
 namespace Lwt\Shared\Infrastructure\Database;
 
+use Lwt\Modules\Language\Domain\Parser\ParserConfig;
+use Lwt\Modules\Language\Infrastructure\Parser\ParserRegistry;
 use Lwt\Shared\Infrastructure\Exception\DatabaseException;
 
 /**
@@ -80,10 +82,13 @@ public static function parseAndDisplayPreview(string $text, int $lid): void
         if ($pre === null) {
             return;
         }
-        [$ptext, $isMecab] = $pre;
+        [$ptext, $isMecab, $record] = $pre;
 
         // Preview HTML is shown before word splitting.
-        if ($isMecab) {
+        $tokens = self::tokenizeWithOptedInParser($ptext, $record);
+        if ($tokens !== null) {
+            StandardTextParser::echoPreview($ptext, $lid);
+        } elseif ($isMecab) {
             JapaneseTextParser::displayJapanesePreview($ptext);
             $tokens = JapaneseTextParser::tokenize($ptext);
         } else {
@@ -157,20 +162,104 @@ private static function tokenize(string $text, int $lid): array
         if ($pre === null) {
             return [];
         }
-        [$ptext, $isMecab] = $pre;
+        [$ptext, $isMecab, $record] = $pre;
+
+        $opted = self::tokenizeWithOptedInParser($ptext, $record);
+        if ($opted !== null) {
+            return $opted;
+        }
+
         return $isMecab
             ? JapaneseTextParser::tokenize($ptext)
             : StandardTextParser::tokenize($ptext, $lid);
     }
 
+    /**
+     * Tokenize with the parser the language explicitly selected, if any.
+     *
+     * The parser registry has been able to describe jieba, MeCab and any
+     * configured external tokenizer for some time, but nothing consulted it:
+     * `LgParserType` was written by the language form and read by no one, so
+     * choosing "Jieba (Chinese)" changed nothing about how a text was parsed.
+     * This is where that setting takes effect.
+     *
+     * Only a deliberate choice routes here. A language with no parser type —
+     * which is every language created before the field meant anything — stays
+     * on the built-in pipeline, as does one that asked for a parser the server
+     * cannot run. Nothing changes for an install that has not opted in.
+     *
+     * @param string               $ptext  Preprocessed text (substitutions applied)
+     * @param array<string, mixed> $record The language row
+     *
+     * @return ParsedToken[]|null Tokens, or null to use the built-in pipeline
+     */
+    private static function tokenizeWithOptedInParser(string $ptext, array $record): ?array
+    {
+        // Cheap guard before building a registry, which reads the external
+        // parser config: almost every language names no parser at all.
+        if (trim((string) ($record['LgParserType'] ?? '')) === '') {
+            return null;
+        }
+
+        $parser = (new ParserRegistry())->getOptedInParserFromRow($record);
+        if ($parser === null) {
+            return null;
+        }
+
+        $config = ParserConfig::fromDatabaseRow($record);
+
+        try {
+            $result = $parser->parse($ptext, $config);
+        } catch (\RuntimeException $e) {
+            // A tokenizer that dies mid-import must not take the text with it
+            error_log('LWT: parser "' . $parser->getType() . '" failed: ' . $e->getMessage());
+            return null;
+        }
+
+        return self::adaptTokens($result->getTokens());
+    }
+
+    /**
+     * Convert parser-module tokens into the shape the persistence layer takes.
+     *
+     * The two differ only in bookkeeping: the module numbers sentences from
+     * zero and restarts the token order inside each one, while the persistence
+     * layer numbers sentences from one and wants a single running order across
+     * the whole text.
+     *
+     * @param array<int, \Lwt\Modules\Language\Domain\Parser\Token> $tokens Parser tokens
+     *
+     * @return ParsedToken[]
+     */
+    private static function adaptTokens(array $tokens): array
+    {
+        $adapted = [];
+        $order = 0;
+        foreach ($tokens as $token) {
+            $text = $token->getText();
+            if ($text === '') {
+                continue;
+            }
+            $order++;
+            $adapted[] = new ParsedToken(
+                $token->getSentenceIndex() + 1,
+                $order,
+                $token->isWord() ? 1 : 0,
+                $text
+            );
+        }
+        return $adapted;
+    }
+
     /**
      * Apply the language's text preprocessing (escaping + character
      * substitutions) and report whether it uses the MeCab parser.
      *
      * @param string $text Raw text
      * @param int    $lid  Language ID
      *
-     * @return array{0: string, 1: bool}|null [preprocessed text, isMecab] or null if language missing
+     * @return array{0: string, 1: bool, 2: array<string, mixed>}|null
+     *         [preprocessed text, isMecab, language row] or null if missing
      */
     private static function preprocess(string $text, int $lid): ?array
     {
@@ -195,6 +284,6 @@ private static function preprocess(string $text, int $lid): ?array
             }
         }
 
-        return [$text, 'MECAB' === strtoupper(trim($termchar))];
+        return [$text, 'MECAB' === strtoupper(trim($termchar)), $record];
     }
 }