feat(text): say so when a text parses into nothing (#278)

CHANGELOG.md

@@ -77,6 +77,15 @@ ones are marked like "v1.0.0-fork".
 
 ### Fixed
 
+* **A text that parses into nothing now says so** (#278). When a language's
+  *Word Characters* setting does not match the script of its texts, parsing
+  does not fail — it succeeds and produces nothing. The text saves, opens and
+  shows every character, and simply refuses to respond to any click, with
+  nothing anywhere to explain why. The reading view and the check-text page now
+  say what happened and link straight to the language's settings. The test is a
+  word-to-character ratio rather than a plain zero, so it also catches a text
+  where only a few stray tokens matched.
+
 * **Adding a term failed outright on a large vocabulary** (#277). Opening the
   term editor read every term of the language into memory to look for similar
   ones — affordable for a vocabulary built by hand, fatal for one seeded from a

locale/de/text.json

@@ -215,5 +215,10 @@
     "modal.lookup": "Nachschlagen:",
     "modal.dict1": "Wb 1",
     "modal.dict2": "Wb 2",
-    "modal.translate": "Übersetzen"
+    "modal.translate": "Übersetzen",
+
+    "parse_warning.no_words": "Aus diesem Text konnten keine Wörter gebildet werden.",
+    "parse_warning.almost_no_words": "Fast nichts in diesem Text konnte in Wörter umgewandelt werden.",
+    "parse_warning.check_language": "Die Einstellung „Wortzeichen“ der Sprache passt nicht zu diesem Text, daher lässt sich darin nichts anklicken oder lernen.",
+    "parse_warning.edit_language": "Spracheinstellungen prüfen"
 }

locale/en/text.json

@@ -215,5 +215,10 @@
     "modal.lookup": "Lookup:",
     "modal.dict1": "Dict 1",
     "modal.dict2": "Dict 2",
-    "modal.translate": "Translate"
+    "modal.translate": "Translate",
+
+    "parse_warning.no_words": "None of this text could be turned into words.",
+    "parse_warning.almost_no_words": "Almost none of this text could be turned into words.",
+    "parse_warning.check_language": "The language's Word Characters setting does not match this text, so nothing in it can be clicked or learned.",
+    "parse_warning.edit_language": "Check the language settings"
 }

locale/es/text.json

@@ -215,5 +215,10 @@
     "modal.lookup": "Buscar:",
     "modal.dict1": "Dicc. 1",
     "modal.dict2": "Dicc. 2",
-    "modal.translate": "Traducir"
+    "modal.translate": "Traducir",
+
+    "parse_warning.no_words": "No se pudo convertir en palabras ninguna parte de este texto.",
+    "parse_warning.almost_no_words": "Casi nada de este texto se pudo convertir en palabras.",
+    "parse_warning.check_language": "La configuración «Caracteres de palabra» del idioma no coincide con este texto, por lo que no se puede hacer clic ni aprender nada en él.",
+    "parse_warning.edit_language": "Revisar la configuración del idioma"
 }

locale/fr/text.json

@@ -215,5 +215,10 @@
     "modal.lookup": "Recherche :",
     "modal.dict1": "Dict 1",
     "modal.dict2": "Dict 2",
-    "modal.translate": "Traduire"
+    "modal.translate": "Traduire",
+
+    "parse_warning.no_words": "Aucun mot n'a pu être extrait de ce texte.",
+    "parse_warning.almost_no_words": "Presque aucun mot n'a pu être extrait de ce texte.",
+    "parse_warning.check_language": "Le réglage « Caractères de mot » de la langue ne correspond pas à ce texte : rien n'y est cliquable ni apprenable.",
+    "parse_warning.edit_language": "Vérifier les réglages de la langue"
 }

locale/it/text.json

@@ -215,5 +215,10 @@
     "modal.lookup": "Cerca:",
     "modal.dict1": "Diz 1",
     "modal.dict2": "Diz 2",
-    "modal.translate": "Traduci"
+    "modal.translate": "Traduci",
+
+    "parse_warning.no_words": "Non è stato possibile ricavare alcuna parola da questo testo.",
+    "parse_warning.almost_no_words": "Quasi nulla di questo testo è stato convertito in parole.",
+    "parse_warning.check_language": "L'impostazione \"Caratteri di parola\" della lingua non corrisponde a questo testo, quindi non è possibile cliccare né imparare nulla.",
+    "parse_warning.edit_language": "Controlla le impostazioni della lingua"
 }

locale/ja/text.json

@@ -215,5 +215,10 @@
     "modal.lookup": "検索：",
     "modal.dict1": "辞書1",
     "modal.dict2": "辞書2",
-    "modal.translate": "翻訳"
+    "modal.translate": "翻訳",
+
+    "parse_warning.no_words": "このテキストからは単語を1つも抽出できませんでした。",
+    "parse_warning.almost_no_words": "このテキストからはほとんど単語を抽出できませんでした。",
+    "parse_warning.check_language": "この言語の「単語を構成する文字」の設定がこのテキストと一致していないため、クリックも学習もできません。",
+    "parse_warning.edit_language": "言語設定を確認"
 }

locale/pt/text.json

@@ -215,5 +215,10 @@
     "modal.lookup": "Consultar:",
     "modal.dict1": "Dic. 1",
     "modal.dict2": "Dic. 2",
-    "modal.translate": "Traduzir"
+    "modal.translate": "Traduzir",
+
+    "parse_warning.no_words": "Não foi possível extrair nenhuma palavra deste texto.",
+    "parse_warning.almost_no_words": "Quase nada deste texto pôde ser convertido em palavras.",
+    "parse_warning.check_language": "A definição \"Caracteres de palavra\" do idioma não corresponde a este texto, por isso nada nele pode ser clicado ou aprendido.",
+    "parse_warning.edit_language": "Verificar as definições do idioma"
 }

locale/ru/text.json

@@ -215,5 +215,10 @@
     "modal.lookup": "Поиск:",
     "modal.dict1": "Словарь 1",
     "modal.dict2": "Словарь 2",
-    "modal.translate": "Перевести"
+    "modal.translate": "Перевести",
+
+    "parse_warning.no_words": "Из этого текста не удалось выделить ни одного слова.",
+    "parse_warning.almost_no_words": "Почти ничего из этого текста не удалось выделить в слова.",
+    "parse_warning.check_language": "Настройка языка «Символы слова» не соответствует этому тексту, поэтому в нём ничего нельзя выбрать или выучить.",
+    "parse_warning.edit_language": "Проверить настройки языка"
 }

locale/zh/text.json

@@ -203,5 +203,10 @@
     "modal.translate": "翻译",
     "flash.text_too_long": "文本过长，必须小于 65000 字节",
     "flash.error_prefix": "错误：{message}",
-    "flash.error_creating_book": "创建书籍时出错：{error}"
+    "flash.error_creating_book": "创建书籍时出错：{error}",
+
+    "parse_warning.no_words": "无法从这段文本中提取出任何单词。",
+    "parse_warning.almost_no_words": "几乎无法从这段文本中提取出单词。",
+    "parse_warning.check_language": "该语言的“单词字符”设置与此文本不匹配，因此其中的内容无法点击或学习。",
+    "parse_warning.edit_language": "检查语言设置"
 }

src/Modules/Text/Application/UseCases/ParseText.php

@@ -17,6 +17,7 @@
 
 namespace Lwt\Modules\Text\Application\UseCases;
 
+use Lwt\Modules\Text\Domain\ParseCoverage;
 use Lwt\Shared\Infrastructure\Database\Connection;
 use Lwt\Shared\Infrastructure\Database\TextParsing;
 use Lwt\Shared\Infrastructure\Database\UserScopedQuery;
@@ -37,7 +38,8 @@ class ParseText
      * @param string $text       Text content to parse
      * @param int    $languageId Language ID
      *
-     * @return array{sentences: int, words: int, unknownPercent: float, preview: string}
+     * @return array{sentences: int, words: int, unknownPercent: float, preview: string,
+     *         warning: string} `warning` is a ParseCoverage verdict, 'ok' when fine
      */
     public function execute(string $text, int $languageId): array
     {
@@ -57,18 +59,24 @@ public function execute(string $text, int $languageId): array
                 'sentences' => 0,
                 'words' => 0,
                 'unknownPercent' => 100.0,
-                'preview' => 'Language not found'
+                'preview' => 'Language not found',
+                'warning' => ParseCoverage::OK
             ];
         }
 
         // Parse text (preview only, no save)
         $result = TextParsing::checkText($text, $languageId);
 
+        $words = $result['words'] ?? 0;
+
         return [
             'sentences' => $result['sentences'] ?? 0,
-            'words' => $result['words'] ?? 0,
+            'words' => $words,
             'unknownPercent' => $result['unknownPercent'] ?? 100.0,
-            'preview' => $result['preview'] ?? ''
+            'preview' => $result['preview'] ?? '',
+            // A language whose word characters do not fit the script parses
+            // successfully into nothing; say so rather than report a zero
+            'warning' => ParseCoverage::assess($words, mb_strlen($text, 'UTF-8'))
         ];
     }
 

src/Modules/Text/Domain/ParseCoverage.php

@@ -0,0 +1,110 @@
+<?php
+
+/**
+ * Verdict on whether a parse produced anything learnable.
+ *
+ * PHP version 8.1
+ *
+ * @category Lwt
+ * @package  Lwt\Modules\Text\Domain
+ * @author   HugoFara <git@hugofara.net>
+ * @license  Unlicense <http://unlicense.org/>
+ * @link     https://hugofara.github.io/lwt/developer/api
+ * @since    3.4.3
+ */
+
+declare(strict_types=1);
+
+namespace Lwt\Modules\Text\Domain;
+
+/**
+ * How much of a text the parser managed to turn into words.
+ *
+ * A language whose word characters do not match the script of its texts does
+ * not fail: it parses successfully into nothing. The text saves, opens and
+ * renders — every character is there — but not one of them can be clicked,
+ * looked up or learned, and nothing anywhere says why. That is what a reader
+ * meets when a Chinese language is left on the Latin defaults, and it is
+ * indistinguishable from a text that has simply gone inert.
+ *
+ * This is the one place that decides a parse came out empty, so that the
+ * reading view, the check-text page and the API all agree on when to say so.
+ *
+ * @since 3.4.3
+ */
+final class ParseCoverage
+{
+    /**
+     * The parse produced words; nothing to report.
+     */
+    public const OK = 'ok';
+
+    /**
+     * Not one word came out of the text.
+     */
+    public const NO_WORDS = 'no_words';
+
+    /**
+     * Words came out, but far too few for the text to be readable.
+     */
+    public const ALMOST_NO_WORDS = 'almost_no_words';
+
+    /**
+     * Shortest text worth judging on its word density.
+     *
+     * A handful of characters can legitimately hold a single word, so the
+     * density test only applies once there is enough text to be sure.
+     */
+    private const DENSITY_MIN_CHARACTERS = 200;
+
+    /**
+     * Fewest words per character a real language ever produces.
+     *
+     * One word per fifty characters. Prose in a Latin script runs nearer one
+     * per six, and a character-split script nearer one per one, so anything
+     * under this is a language that matched a few stray tokens — digits, or a
+     * Latin fragment in a non-Latin text — and missed the rest.
+     */
+    private const DENSITY_FLOOR = 0.02;
+
+    /**
+     * Judge a parse by what it produced.
+     *
+     * @param int $wordCount      Words the parse produced
+     * @param int $characterCount Characters in the text that was parsed
+     *
+     * @return self::OK|self::NO_WORDS|self::ALMOST_NO_WORDS
+     */
+    public static function assess(int $wordCount, int $characterCount): string
+    {
+        if ($characterCount <= 0) {
+            // An empty text is empty; that is not a parsing problem
+            return self::OK;
+        }
+
+        if ($wordCount <= 0) {
+            return self::NO_WORDS;
+        }
+
+        if (
+            $characterCount >= self::DENSITY_MIN_CHARACTERS
+            && $wordCount / $characterCount < self::DENSITY_FLOOR
+        ) {
+            return self::ALMOST_NO_WORDS;
+        }
+
+        return self::OK;
+    }
+
+    /**
+     * Whether a verdict is worth telling the reader about.
+     *
+     * @param string $verdict A verdict from assess()
+     *
+     * @return bool
+     */
+    public static function isWarning(string $verdict): bool
+    {
+        return $verdict !== self::OK;
+    }
+}

src/Modules/Text/Http/TextTermApiHandler.php

@@ -26,6 +26,7 @@
 use Lwt\Modules\Vocabulary\Application\Services\ExportService;
 use Lwt\Modules\Text\Application\Services\AnnotationService;
 use Lwt\Modules\Tags\Application\TagsFacade;
+use Lwt\Modules\Text\Domain\ParseCoverage;
 use Lwt\Modules\Text\Application\TextFacade;
 use Lwt\Modules\Text\Application\Services\TextScoringService;
 
@@ -216,6 +217,7 @@ public function getWords(int $textId): array
             'modeTrans' => $modeTrans,
             'termDelimiter' => $termDelimiter,
             'readerWidth' => $readerWidth,
+            'parseWarning' => self::parseWarning($words, $langId),
             'annTextSize' => match ($textSize) {
                 100 => 50,
                 150 => 50,
@@ -231,6 +233,45 @@ public function getWords(int $textId): array
         ];
     }
 
+    /**
+     * Tell the reader when a text holds nothing they can act on.
+     *
+     * A language whose word characters do not match the script of its texts
+     * does not fail: it parses successfully into nothing. The page still shows
+     * every character, so the text looks ordinary and simply refuses to
+     * respond to any click, with nothing anywhere to say why (#278).
+     *
+     * @param array<int, array<string, mixed>> $words  The rendered word list
+     * @param int                              $langId Language of the text
+     *
+     * @return array{headline: string, detail: string, linkLabel: string, linkHref: string}|null
+     */
+    private static function parseWarning(array $words, int $langId): ?array
+    {
+        $wordCount = 0;
+        $characters = 0;
+        foreach ($words as $word) {
+            $characters += mb_strlen((string) ($word['text'] ?? ''), 'UTF-8');
+            if (($word['isNotWord'] ?? true) === false) {
+                $wordCount++;
+            }
+        }
+
+        $verdict = ParseCoverage::assess($wordCount, $characters);
+        if (!ParseCoverage::isWarning($verdict)) {
+            return null;
+        }
+
+        return [
+            'headline' => $verdict === ParseCoverage::NO_WORDS
+                ? __('text.parse_warning.no_words')
+                : __('text.parse_warning.almost_no_words'),
+            'detail' => __('text.parse_warning.check_language'),
+            'linkLabel' => __('text.parse_warning.edit_language'),
+            'linkHref' => '/languages/' . $langId . '/edit',
+        ];
+    }
+
     /**
      * Format response for getting text words.
      *