From ee9491b2c04eca83285565df3ddce33fd96e02cf Mon Sep 17 00:00:00 2001 From: HugoFara Date: Sun, 23 Aug 2026 14:23:25 +0200 Subject: [PATCH] feat(text): say so when a text parses into nothing (#278) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A language whose word characters do not match the script of its texts does not fail. It parses successfully into nothing: the text saves, opens, and renders every character, and not one of them can be clicked, looked up or learned. TokenPersistence::save() takes the empty token list and returns silently, so a Chinese text on a Latin language stores a row, a sentence, and zero text items. From the outside that is indistinguishable from an ordinary text that has gone inert, which is what the reporter met and had no way to diagnose. ParseCoverage is the one place that decides a parse came out empty, so the reading view, the check-text page and the API agree on when to speak up. The test is a word-to-character ratio, not a plain zero: a non-Latin text usually matches a few stray tokens — a digit, a Latin fragment — and a zero test would let those through. The floor is one word per fifty characters, well under the one per six of Latin prose or the one per one of a character-split script, and it only applies to texts long enough to judge. Three surfaces, all of them ones the reader already reaches: - the reading view, where the confusion actually happens, via a parseWarning on the payload that already feeds it; - the check-text page, whose whole purpose is to answer "did my parsing work?" and which until now answered a failed parse with an empty list; - ParseText::execute(), for anything reading the API. Each says what happened and links to the language's settings. The banner is built through the DOM rather than by concatenation: the renderer's escapeHtml() leaves quotes alone, which is fine for text but would let an attribute value break out of the attribute it sits in. Verified against a real database: Chinese on Latin defaults warning=no_words banner shown Chinese configured right warning=ok no banner ordinary English warning=ok no banner --- CHANGELOG.md | 9 ++ locale/de/text.json | 7 +- locale/en/text.json | 7 +- locale/es/text.json | 7 +- locale/fr/text.json | 7 +- locale/it/text.json | 7 +- locale/ja/text.json | 7 +- locale/pt/text.json | 7 +- locale/ru/text.json | 7 +- locale/zh/text.json | 7 +- .../Text/Application/UseCases/ParseText.php | 16 ++- src/Modules/Text/Domain/ParseCoverage.php | 110 ++++++++++++++++++ src/Modules/Text/Http/TextTermApiHandler.php | 41 +++++++ .../Database/TokenPersistence.php | 38 ++++++ src/frontend/js/modules/text/api/texts_api.ts | 15 +++ .../js/modules/text/components/text_reader.ts | 5 +- .../text/pages/reading/text_renderer.ts | 37 +++++- .../modules/vocabulary/stores/word_store.ts | 5 +- .../Modules/Text/Domain/ParseCoverageTest.php | 91 +++++++++++++++ tests/frontend/reading/parse_warning.test.ts | 46 ++++++++ tests/frontend/texts/text_reader.test.ts | 1 + 21 files changed, 460 insertions(+), 17 deletions(-) create mode 100644 src/Modules/Text/Domain/ParseCoverage.php create mode 100644 tests/backend/Modules/Text/Domain/ParseCoverageTest.php create mode 100644 tests/frontend/reading/parse_warning.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index c944083a5..28f9775f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,15 @@ ones are marked like "v1.0.0-fork". ### Fixed +* **A text that parses into nothing now says so** (#278). When a language's + *Word Characters* setting does not match the script of its texts, parsing + does not fail — it succeeds and produces nothing. The text saves, opens and + shows every character, and simply refuses to respond to any click, with + nothing anywhere to explain why. The reading view and the check-text page now + say what happened and link straight to the language's settings. The test is a + word-to-character ratio rather than a plain zero, so it also catches a text + where only a few stray tokens matched. + * **Adding a term failed outright on a large vocabulary** (#277). Opening the term editor read every term of the language into memory to look for similar ones — affordable for a vocabulary built by hand, fatal for one seeded from a diff --git a/locale/de/text.json b/locale/de/text.json index 466128dd9..e949b247e 100644 --- a/locale/de/text.json +++ b/locale/de/text.json @@ -215,5 +215,10 @@ "modal.lookup": "Nachschlagen:", "modal.dict1": "Wb 1", "modal.dict2": "Wb 2", - "modal.translate": "Übersetzen" + "modal.translate": "Übersetzen", + + "parse_warning.no_words": "Aus diesem Text konnten keine Wörter gebildet werden.", + "parse_warning.almost_no_words": "Fast nichts in diesem Text konnte in Wörter umgewandelt werden.", + "parse_warning.check_language": "Die Einstellung „Wortzeichen“ der Sprache passt nicht zu diesem Text, daher lässt sich darin nichts anklicken oder lernen.", + "parse_warning.edit_language": "Spracheinstellungen prüfen" } diff --git a/locale/en/text.json b/locale/en/text.json index 1d9552ba9..79ed75c0f 100644 --- a/locale/en/text.json +++ b/locale/en/text.json @@ -215,5 +215,10 @@ "modal.lookup": "Lookup:", "modal.dict1": "Dict 1", "modal.dict2": "Dict 2", - "modal.translate": "Translate" + "modal.translate": "Translate", + + "parse_warning.no_words": "None of this text could be turned into words.", + "parse_warning.almost_no_words": "Almost none of this text could be turned into words.", + "parse_warning.check_language": "The language's Word Characters setting does not match this text, so nothing in it can be clicked or learned.", + "parse_warning.edit_language": "Check the language settings" } diff --git a/locale/es/text.json b/locale/es/text.json index f91ba40f2..94315f2d0 100644 --- a/locale/es/text.json +++ b/locale/es/text.json @@ -215,5 +215,10 @@ "modal.lookup": "Buscar:", "modal.dict1": "Dicc. 1", "modal.dict2": "Dicc. 2", - "modal.translate": "Traducir" + "modal.translate": "Traducir", + + "parse_warning.no_words": "No se pudo convertir en palabras ninguna parte de este texto.", + "parse_warning.almost_no_words": "Casi nada de este texto se pudo convertir en palabras.", + "parse_warning.check_language": "La configuración «Caracteres de palabra» del idioma no coincide con este texto, por lo que no se puede hacer clic ni aprender nada en él.", + "parse_warning.edit_language": "Revisar la configuración del idioma" } diff --git a/locale/fr/text.json b/locale/fr/text.json index 7a2ba5648..2bbe75eb7 100644 --- a/locale/fr/text.json +++ b/locale/fr/text.json @@ -215,5 +215,10 @@ "modal.lookup": "Recherche :", "modal.dict1": "Dict 1", "modal.dict2": "Dict 2", - "modal.translate": "Traduire" + "modal.translate": "Traduire", + + "parse_warning.no_words": "Aucun mot n'a pu être extrait de ce texte.", + "parse_warning.almost_no_words": "Presque aucun mot n'a pu être extrait de ce texte.", + "parse_warning.check_language": "Le réglage « Caractères de mot » de la langue ne correspond pas à ce texte : rien n'y est cliquable ni apprenable.", + "parse_warning.edit_language": "Vérifier les réglages de la langue" } diff --git a/locale/it/text.json b/locale/it/text.json index d45d86ebc..3cb89e1b0 100644 --- a/locale/it/text.json +++ b/locale/it/text.json @@ -215,5 +215,10 @@ "modal.lookup": "Cerca:", "modal.dict1": "Diz 1", "modal.dict2": "Diz 2", - "modal.translate": "Traduci" + "modal.translate": "Traduci", + + "parse_warning.no_words": "Non è stato possibile ricavare alcuna parola da questo testo.", + "parse_warning.almost_no_words": "Quasi nulla di questo testo è stato convertito in parole.", + "parse_warning.check_language": "L'impostazione \"Caratteri di parola\" della lingua non corrisponde a questo testo, quindi non è possibile cliccare né imparare nulla.", + "parse_warning.edit_language": "Controlla le impostazioni della lingua" } diff --git a/locale/ja/text.json b/locale/ja/text.json index 640e7b72c..9f67453d1 100644 --- a/locale/ja/text.json +++ b/locale/ja/text.json @@ -215,5 +215,10 @@ "modal.lookup": "検索：", "modal.dict1": "辞書1", "modal.dict2": "辞書2", - "modal.translate": "翻訳" + "modal.translate": "翻訳", + + "parse_warning.no_words": "このテキストからは単語を1つも抽出できませんでした。", + "parse_warning.almost_no_words": "このテキストからはほとんど単語を抽出できませんでした。", + "parse_warning.check_language": "この言語の「単語を構成する文字」の設定がこのテキストと一致していないため、クリックも学習もできません。", + "parse_warning.edit_language": "言語設定を確認" } diff --git a/locale/pt/text.json b/locale/pt/text.json index 6d0ac88c9..7ae4c3e6c 100644 --- a/locale/pt/text.json +++ b/locale/pt/text.json @@ -215,5 +215,10 @@ "modal.lookup": "Consultar:", "modal.dict1": "Dic. 1", "modal.dict2": "Dic. 2", - "modal.translate": "Traduzir" + "modal.translate": "Traduzir", + + "parse_warning.no_words": "Não foi possível extrair nenhuma palavra deste texto.", + "parse_warning.almost_no_words": "Quase nada deste texto pôde ser convertido em palavras.", + "parse_warning.check_language": "A definição \"Caracteres de palavra\" do idioma não corresponde a este texto, por isso nada nele pode ser clicado ou aprendido.", + "parse_warning.edit_language": "Verificar as definições do idioma" } diff --git a/locale/ru/text.json b/locale/ru/text.json index 20c59cb64..54f8ce94b 100644 --- a/locale/ru/text.json +++ b/locale/ru/text.json @@ -215,5 +215,10 @@ "modal.lookup": "Поиск:", "modal.dict1": "Словарь 1", "modal.dict2": "Словарь 2", - "modal.translate": "Перевести" + "modal.translate": "Перевести", + + "parse_warning.no_words": "Из этого текста не удалось выделить ни одного слова.", + "parse_warning.almost_no_words": "Почти ничего из этого текста не удалось выделить в слова.", + "parse_warning.check_language": "Настройка языка «Символы слова» не соответствует этому тексту, поэтому в нём ничего нельзя выбрать или выучить.", + "parse_warning.edit_language": "Проверить настройки языка" } diff --git a/locale/zh/text.json b/locale/zh/text.json index ad7b03d57..900434479 100644 --- a/locale/zh/text.json +++ b/locale/zh/text.json @@ -203,5 +203,10 @@ "modal.translate": "翻译", "flash.text_too_long": "文本过长，必须小于 65000 字节", "flash.error_prefix": "错误：{message}", - "flash.error_creating_book": "创建书籍时出错：{error}" + "flash.error_creating_book": "创建书籍时出错：{error}", + + "parse_warning.no_words": "无法从这段文本中提取出任何单词。", + "parse_warning.almost_no_words": "几乎无法从这段文本中提取出单词。", + "parse_warning.check_language": "该语言的“单词字符”设置与此文本不匹配，因此其中的内容无法点击或学习。", + "parse_warning.edit_language": "检查语言设置" } diff --git a/src/Modules/Text/Application/UseCases/ParseText.php b/src/Modules/Text/Application/UseCases/ParseText.php index 570239ee2..1f85ca377 100644 --- a/src/Modules/Text/Application/UseCases/ParseText.php +++ b/src/Modules/Text/Application/UseCases/ParseText.php @@ -17,6 +17,7 @@ namespace Lwt\Modules\Text\Application\UseCases; +use Lwt\Modules\Text\Domain\ParseCoverage; use Lwt\Shared\Infrastructure\Database\Connection; use Lwt\Shared\Infrastructure\Database\TextParsing; use Lwt\Shared\Infrastructure\Database\UserScopedQuery; @@ -37,7 +38,8 @@ class ParseText * @param string $text Text content to parse * @param int $languageId Language ID * - * @return array{sentences: int, words: int, unknownPercent: float, preview: string} + * @return array{sentences: int, words: int, unknownPercent: float, preview: string, + * warning: string} `warning` is a ParseCoverage verdict, 'ok' when fine */ public function execute(string $text, int $languageId): array { @@ -57,18 +59,24 @@ public function execute(string $text, int $languageId): array 'sentences' => 0, 'words' => 0, 'unknownPercent' => 100.0, - 'preview' => 'Language not found' + 'preview' => 'Language not found', + 'warning' => ParseCoverage::OK ]; } // Parse text (preview only, no save) $result = TextParsing::checkText($text, $languageId); + $words = $result['words'] ?? 0; + return [ 'sentences' => $result['sentences'] ?? 0, - 'words' => $result['words'] ?? 0, + 'words' => $words, 'unknownPercent' => $result['unknownPercent'] ?? 100.0, - 'preview' => $result['preview'] ?? '' + 'preview' => $result['preview'] ?? '', + // A language whose word characters do not fit the script parses + // successfully into nothing; say so rather than report a zero + 'warning' => ParseCoverage::assess($words, mb_strlen($text, 'UTF-8')) ]; } diff --git a/src/Modules/Text/Domain/ParseCoverage.php b/src/Modules/Text/Domain/ParseCoverage.php new file mode 100644 index 000000000..65fa07270 --- /dev/null +++ b/src/Modules/Text/Domain/ParseCoverage.php @@ -0,0 +1,110 @@ + + * @license Unlicense + * @link https://hugofara.github.io/lwt/developer/api + * @since 3.4.3 + */ + +declare(strict_types=1); + +namespace Lwt\Modules\Text\Domain; + +/** + * How much of a text the parser managed to turn into words. + * + * A language whose word characters do not match the script of its texts does + * not fail: it parses successfully into nothing. The text saves, opens and + * renders — every character is there — but not one of them can be clicked, + * looked up or learned, and nothing anywhere says why. That is what a reader + * meets when a Chinese language is left on the Latin defaults, and it is + * indistinguishable from a text that has simply gone inert. + * + * This is the one place that decides a parse came out empty, so that the + * reading view, the check-text page and the API all agree on when to say so. + * + * @since 3.4.3 + */ +final class ParseCoverage +{ + /** + * The parse produced words; nothing to report. + */ + public const OK = 'ok'; + + /** + * Not one word came out of the text. + */ + public const NO_WORDS = 'no_words'; + + /** + * Words came out, but far too few for the text to be readable. + */ + public const ALMOST_NO_WORDS = 'almost_no_words'; + + /** + * Shortest text worth judging on its word density. + * + * A handful of characters can legitimately hold a single word, so the + * density test only applies once there is enough text to be sure. + */ + private const DENSITY_MIN_CHARACTERS = 200; + + /** + * Fewest words per character a real language ever produces. + * + * One word per fifty characters. Prose in a Latin script runs nearer one + * per six, and a character-split script nearer one per one, so anything + * under this is a language that matched a few stray tokens — digits, or a + * Latin fragment in a non-Latin text — and missed the rest. + */ + private const DENSITY_FLOOR = 0.02; + + /** + * Judge a parse by what it produced. + * + * @param int $wordCount Words the parse produced + * @param int $characterCount Characters in the text that was parsed + * + * @return self::OK|self::NO_WORDS|self::ALMOST_NO_WORDS + */ + public static function assess(int $wordCount, int $characterCount): string + { + if ($characterCount <= 0) { + // An empty text is empty; that is not a parsing problem + return self::OK; + } + + if ($wordCount <= 0) { + return self::NO_WORDS; + } + + if ( + $characterCount >= self::DENSITY_MIN_CHARACTERS + && $wordCount / $characterCount < self::DENSITY_FLOOR + ) { + return self::ALMOST_NO_WORDS; + } + + return self::OK; + } + + /** + * Whether a verdict is worth telling the reader about. + * + * @param string $verdict A verdict from assess() + * + * @return bool + */ + public static function isWarning(string $verdict): bool + { + return $verdict !== self::OK; + } +} diff --git a/src/Modules/Text/Http/TextTermApiHandler.php b/src/Modules/Text/Http/TextTermApiHandler.php index ffc6746e9..7164ee442 100644 --- a/src/Modules/Text/Http/TextTermApiHandler.php +++ b/src/Modules/Text/Http/TextTermApiHandler.php @@ -26,6 +26,7 @@ use Lwt\Modules\Vocabulary\Application\Services\ExportService; use Lwt\Modules\Text\Application\Services\AnnotationService; use Lwt\Modules\Tags\Application\TagsFacade; +use Lwt\Modules\Text\Domain\ParseCoverage; use Lwt\Modules\Text\Application\TextFacade; use Lwt\Modules\Text\Application\Services\TextScoringService; @@ -216,6 +217,7 @@ public function getWords(int $textId): array 'modeTrans' => $modeTrans, 'termDelimiter' => $termDelimiter, 'readerWidth' => $readerWidth, + 'parseWarning' => self::parseWarning($words, $langId), 'annTextSize' => match ($textSize) { 100 => 50, 150 => 50, @@ -231,6 +233,45 @@ public function getWords(int $textId): array ]; } + /** + * Tell the reader when a text holds nothing they can act on. + * + * A language whose word characters do not match the script of its texts + * does not fail: it parses successfully into nothing. The page still shows + * every character, so the text looks ordinary and simply refuses to + * respond to any click, with nothing anywhere to say why (#278). + * + * @param array> $words The rendered word list + * @param int $langId Language of the text + * + * @return array{headline: string, detail: string, linkLabel: string, linkHref: string}|null + */ + private static function parseWarning(array $words, int $langId): ?array + { + $wordCount = 0; + $characters = 0; + foreach ($words as $word) { + $characters += mb_strlen((string) ($word['text'] ?? ''), 'UTF-8'); + if (($word['isNotWord'] ?? true) === false) { + $wordCount++; + } + } + + $verdict = ParseCoverage::assess($wordCount, $characters); + if (!ParseCoverage::isWarning($verdict)) { + return null; + } + + return [ + 'headline' => $verdict === ParseCoverage::NO_WORDS + ? __('text.parse_warning.no_words') + : __('text.parse_warning.almost_no_words'), + 'detail' => __('text.parse_warning.check_language'), + 'linkLabel' => __('text.parse_warning.edit_language'), + 'linkHref' => '/languages/' . $langId . '/edit', + ]; + } + /** * Format response for getting text words. * diff --git a/src/Shared/Infrastructure/Database/TokenPersistence.php b/src/Shared/Infrastructure/Database/TokenPersistence.php index 8667b49db..f99619d2f 100644 --- a/src/Shared/Infrastructure/Database/TokenPersistence.php +++ b/src/Shared/Infrastructure/Database/TokenPersistence.php @@ -18,6 +18,8 @@ namespace Lwt\Shared\Infrastructure\Database; +use Lwt\Modules\Text\Domain\ParseCoverage; + /** * Turns a parsed token stream into `sentences` and `word_occurrences` rows, * detecting multi-word expressions along the way — all in PHP. @@ -152,6 +154,8 @@ public static function echoCheckValid(array $tokens, int $lid): void $nonWordCounts[$lc] = ($nonWordCounts[$lc] ?? 0) + 1; } } + self::echoParseWarning($tokens, $wordCounts); + $single = self::singleWordTerms($lid, array_keys($wordCounts)); $wo = []; foreach ($wordCounts as $lc => $cnt) { @@ -173,6 +177,40 @@ public static function echoCheckValid(array $tokens, int $lid): void echo ''; } + /** + * Warn on the check-text page when the parse produced nothing learnable. + * + * This page exists to answer "did my parsing work?", and until now it + * answered a failed parse with an empty list of words and no explanation. + * + * @param ParsedToken[] $tokens Tokens for the whole text + * @param array $wordCounts Word tokens by lowercase form + * + * @return void + */ + private static function echoParseWarning(array $tokens, array $wordCounts): void + { + $characters = 0; + foreach ($tokens as $t) { + $characters += \mb_strlen($t->text, 'UTF-8'); + } + + $verdict = ParseCoverage::assess(array_sum($wordCounts), $characters); + if (!ParseCoverage::isWarning($verdict)) { + return; + } + + $headline = $verdict === ParseCoverage::NO_WORDS + ? __('text.parse_warning.no_words') + : __('text.parse_warning.almost_no_words'); + + echo '
' + . \htmlspecialchars($headline, ENT_QUOTES, 'UTF-8') + . ' ' + . \htmlspecialchars(__('text.parse_warning.check_language'), ENT_QUOTES, 'UTF-8') + . '
'; + } + /** * Echo the multi-word statistics JSON for the check-text preview. * diff --git a/src/frontend/js/modules/text/api/texts_api.ts b/src/frontend/js/modules/text/api/texts_api.ts index 62bed08fd..591c84d5b 100644 --- a/src/frontend/js/modules/text/api/texts_api.ts +++ b/src/frontend/js/modules/text/api/texts_api.ts @@ -38,6 +38,21 @@ export interface TextReadingConfig { annTextSize: number; // Reader layout settings readerWidth: number; + /** + * Set when the text parsed into (almost) no words, which happens when the + * language's word characters do not match its script. Null when fine. + */ + parseWarning: ParseWarning | null; +} + +/** + * Why a text came out with nothing the reader can click, and where to fix it. + */ +export interface ParseWarning { + headline: string; + detail: string; + linkLabel: string; + linkHref: string; } /** diff --git a/src/frontend/js/modules/text/components/text_reader.ts b/src/frontend/js/modules/text/components/text_reader.ts index b697de258..f714dc04f 100644 --- a/src/frontend/js/modules/text/components/text_reader.ts +++ b/src/frontend/js/modules/text/components/text_reader.ts @@ -10,7 +10,8 @@ import Alpine from 'alpinejs'; import type { WordStoreState } from '@modules/vocabulary/stores/word_store'; -import { renderText, updateWordStatusInDOM, type RenderSettings } from '../pages/reading/text_renderer'; +import { renderText, updateWordStatusInDOM, renderParseWarning, type RenderSettings } + from '../pages/reading/text_renderer'; import { setupMultiWordSelection } from '../pages/reading/text_multiword_selection'; import { TextsApi } from '@modules/text/api/texts_api'; import { SettingsApi } from '@modules/admin/api/settings_api'; @@ -168,7 +169,7 @@ export function textReaderData(): TextReaderData { const settings = this.getRenderSettings(); const html = renderText(this.store.words, settings); - container.innerHTML = html; + container.innerHTML = renderParseWarning(this.store.parseWarning) + html; // Apply RTL styling if needed if (this.store.rightToLeft) { diff --git a/src/frontend/js/modules/text/pages/reading/text_renderer.ts b/src/frontend/js/modules/text/pages/reading/text_renderer.ts index 4976cd258..dde9b86b4 100644 --- a/src/frontend/js/modules/text/pages/reading/text_renderer.ts +++ b/src/frontend/js/modules/text/pages/reading/text_renderer.ts @@ -9,7 +9,7 @@ */ import type { WordData } from '@modules/vocabulary/stores/word_store'; -import type { MultiWordRef } from '@modules/text/api/texts_api'; +import type { MultiWordRef, ParseWarning } from '@modules/text/api/texts_api'; import { parseInlineMarkdown } from '@shared/utils/inline_markdown'; /** @@ -271,6 +271,41 @@ function escapeAttr(text: string): string { * Words and adjacent punctuation are wrapped together to prevent line breaks. * Multi-word expressions are wrapped in mw-group spans with connected underlines. */ +/** + * Banner shown above a text that parsed into (almost) nothing. + * + * Such a text still displays every character, so without this it reads as an + * ordinary text that has inexplicably stopped responding to clicks. + * + * Built through the DOM rather than by concatenation: escapeHtml() leaves + * quotes alone, which is fine for text but would let an attribute value break + * out of the attribute it sits in. + * + * @param warning What the server found wrong, or null when the parse was fine + * @returns HTML for the banner, or an empty string + */ +export function renderParseWarning(warning: ParseWarning | null): string { + if (!warning) return ''; + + const box = document.createElement('div'); + box.className = 'notification is-warning is-light'; + + const message = document.createElement('p'); + const headline = document.createElement('strong'); + headline.textContent = warning.headline; + message.append(headline, ` ${warning.detail}`); + + const linkLine = document.createElement('p'); + const link = document.createElement('a'); + // setAttribute, not .href, so the path stays relative in the markup + link.setAttribute('href', warning.linkHref); + link.textContent = warning.linkLabel; + linkLine.append(link); + + box.append(message, linkLine); + return box.outerHTML; +} + export function renderText(words: WordData[], settings: RenderSettings): string { if (words.length === 0) return ''; diff --git a/src/frontend/js/modules/vocabulary/stores/word_store.ts b/src/frontend/js/modules/vocabulary/stores/word_store.ts index 1814c84e0..c4dc340c7 100644 --- a/src/frontend/js/modules/vocabulary/stores/word_store.ts +++ b/src/frontend/js/modules/vocabulary/stores/word_store.ts @@ -10,7 +10,7 @@ import Alpine from 'alpinejs'; import { TermsApi } from '@modules/vocabulary/api/terms_api'; -import { TextsApi, type TextWord, type TextReadingConfig, type DictLinks, type MultiWordRef } from '@modules/text/api/texts_api'; +import { TextsApi, type TextWord, type TextReadingConfig, type DictLinks, type MultiWordRef, type ParseWarning } from '@modules/text/api/texts_api'; import { injectTextStyles, generateParagraphStyles } from '@modules/text/pages/reading/text_styles'; import { renderText, updateWordStatusInDOM, updateWordTranslationInDOM, type RenderSettings } from '@modules/text/pages/reading/text_renderer'; @@ -57,6 +57,7 @@ export interface WordStoreState { rightToLeft: boolean; textSize: number; removeSpaces: boolean; + parseWarning: ParseWarning | null; dictLinks: DictLinks; // Annotation/display settings @@ -126,6 +127,7 @@ function createWordStore(): WordStoreState { rightToLeft: false, textSize: 100, removeSpaces: false, + parseWarning: null, dictLinks: { dict1: '', dict2: '', @@ -231,6 +233,7 @@ function createWordStore(): WordStoreState { this.rightToLeft = config.rightToLeft; this.textSize = config.textSize; this.removeSpaces = config.removeSpaces ?? false; + this.parseWarning = config.parseWarning ?? null; this.dictLinks = config.dictLinks; // Annotation/display settings diff --git a/tests/backend/Modules/Text/Domain/ParseCoverageTest.php b/tests/backend/Modules/Text/Domain/ParseCoverageTest.php new file mode 100644 index 000000000..dc7555810 --- /dev/null +++ b/tests/backend/Modules/Text/Domain/ParseCoverageTest.php @@ -0,0 +1,91 @@ + + */ + +declare(strict_types=1); + +namespace Lwt\Tests\Modules\Text\Domain; + +use Lwt\Modules\Text\Domain\ParseCoverage; +use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\TestCase; + +/** + * When a parse counts as having produced nothing learnable (#278). + */ +#[CoversClass(ParseCoverage::class)] +class ParseCoverageTest extends TestCase +{ + public function testNoWordsAtAllIsTheReportedCase(): void + { + // The reported bug: a Chinese text on a Latin language parses into + // a readable page where nothing can be clicked + $this->assertSame(ParseCoverage::NO_WORDS, ParseCoverage::assess(0, 420)); + } + + public function testAFewStrayTokensInALongTextStillCounts(): void + { + // A digit or a Latin fragment inside a non-Latin text matches, the + // rest does not; a plain zero test would miss this + $this->assertSame(ParseCoverage::ALMOST_NO_WORDS, ParseCoverage::assess(3, 400)); + } + + /** + * @param int $words Words the parse produced + * @param int $characters Characters in the text + */ + #[DataProvider('healthyParses')] + public function testARealLanguageIsNeverWarnedAbout(int $words, int $characters): void + { + $this->assertSame(ParseCoverage::OK, ParseCoverage::assess($words, $characters)); + } + + /** + * Word-to-character ratios that real languages actually produce. + * + * @return array + */ + public static function healthyParses(): array + { + return [ + 'English prose, ~1 word per 6 characters' => [70, 420], + 'German compounds, ~1 per 12' => [35, 420], + 'character-split Chinese, ~1 per 1' => [400, 420], + 'jieba-segmented Chinese, ~1 per 2' => [200, 420], + 'a sparse but plausible text, 1 per 20' => [21, 420], + ]; + } + + public function testAShortTextIsNotJudgedOnDensity(): void + { + // One word in a title or a caption is not a broken parse + $this->assertSame(ParseCoverage::OK, ParseCoverage::assess(1, 199)); + } + + public function testAShortTextWithNoWordsIsStillReported(): void + { + $this->assertSame(ParseCoverage::NO_WORDS, ParseCoverage::assess(0, 20)); + } + + public function testAnEmptyTextIsNotAParsingProblem(): void + { + $this->assertSame(ParseCoverage::OK, ParseCoverage::assess(0, 0)); + $this->assertSame(ParseCoverage::OK, ParseCoverage::assess(0, -1)); + } + + public function testOnlyTheOkVerdictIsSilent(): void + { + $this->assertFalse(ParseCoverage::isWarning(ParseCoverage::OK)); + $this->assertTrue(ParseCoverage::isWarning(ParseCoverage::NO_WORDS)); + $this->assertTrue(ParseCoverage::isWarning(ParseCoverage::ALMOST_NO_WORDS)); + } +} diff --git a/tests/frontend/reading/parse_warning.test.ts b/tests/frontend/reading/parse_warning.test.ts new file mode 100644 index 000000000..7e17a59e3 --- /dev/null +++ b/tests/frontend/reading/parse_warning.test.ts @@ -0,0 +1,46 @@ +/** + * Tests for the banner shown when a text parses into (almost) nothing (#278). + */ +import { describe, it, expect } from 'vitest'; +import { renderParseWarning } from '../../../src/frontend/js/modules/text/pages/reading/text_renderer'; + +const warning = { + headline: 'None of this text could be turned into words.', + detail: "The language's Word Characters setting does not match this text.", + linkLabel: 'Check the language settings', + linkHref: '/languages/7/edit' +}; + +describe('renderParseWarning', () => { + it('renders nothing when the parse was fine', () => { + expect(renderParseWarning(null)).toBe(''); + }); + + it('shows what went wrong and where to fix it', () => { + const html = renderParseWarning(warning); + + expect(html).toContain('notification is-warning'); + expect(html).toContain('None of this text could be turned into words.'); + expect(html).toContain('does not match this text'); + expect(html).toContain('href="/languages/7/edit"'); + expect(html).toContain('Check the language settings'); + }); + + it('escapes the server-supplied text', () => { + const html = renderParseWarning({ + ...warning, + headline: '', + linkHref: '/languages/1/edit"onmouseover="alert(1)' + }); + + expect(html).not.toContain('