diff --git a/ly/lex/html.py b/ly/lex/html.py index 7def595..c0875e9 100644 --- a/ly/lex/html.py +++ b/ly/lex/html.py @@ -96,7 +96,7 @@ class StringSQEnd(String, _token.StringEnd, _token.Leaver): class EntityRef(_token.Character): - rx = r"\&(#\d+|#[xX][0-9A-Fa-f]+|[A-Za-z_:][\w.:_-]*);" + rx = r"\&(#[0-9]+|#[xX][0-9A-Fa-f]+|[A-Za-z_:][\w.:_-]*);" class LilyPondTag(Tag): diff --git a/ly/lex/lilypond.py b/ly/lex/lilypond.py index 65034d7..67e6e5d 100644 --- a/ly/lex/lilypond.py +++ b/ly/lex/lilypond.py @@ -45,7 +45,7 @@ re_duration = rf"(\\(maxima|longa|breve){re_identifier_end}|(1|2|4|8|16|32|64|128|256|512|1024|2048)(?!\d))" re_dot = r"\." -re_scaling = r"\*[\t ]*\d+(/\d+)?" +re_scaling = r"\*[\t ]*[0-9]+(/[0-9]+)?" @@ -72,15 +72,15 @@ class Value(_token.Item, _token.Numeric): class DecimalValue(Value): - rx = r"-?\d+(\.\d+)?" + rx = r"-?[0-9]+(\.[0-9]+)?" class IntegerValue(DecimalValue): - rx = r"\d+" + rx = r"[0-9]+" class Fraction(Value): - rx = r"\d+/\d+" + rx = r"[0-9]+/[0-9]+" class Delimiter(_token.Token): @@ -289,11 +289,11 @@ class ScriptAbbreviation(Articulation, _token.Leaver): class Fingering(Articulation, _token.Leaver): - rx = r"\d+" + rx = r"[0-9]+" class StringNumber(Articulation): - rx = r"\\\d+" + rx = r"\\[0-9]+" class Slur(_token.Token): @@ -379,7 +379,7 @@ class ChordSeparator(ChordItem): class ChordStepNumber(ChordItem): - rx = r"\d+[-+]?" + rx = r"[0-9]+[-+]?" class DotChord(ChordItem): @@ -571,7 +571,7 @@ def update_state(self, state): class TempoSeparator(Delimiter): - rx = r"[-~](?=\s*\d)" + rx = r"[-~](?=\s*[0-9])" class Partial(Command): @@ -740,7 +740,7 @@ class FigureBracket(Figure): class FigureStep(Figure): """A step figure number or the underscore.""" - rx = r"_|\d+" + rx = r"_|[0-9]+" class FigureAccidental(Figure): diff --git a/ly/lex/scheme.py b/ly/lex/scheme.py index f8d3e5a..78abb41 100644 --- a/ly/lex/scheme.py +++ b/ly/lex/scheme.py @@ -140,18 +140,18 @@ def test_match(cls, match): class Number(_token.Item, _token.Numeric): rx = (r"(" - r"-?\d+|" + r"-?[0-9]+|" r"#(b[0-1]+|o[0-7]+|x[0-9a-fA-F]+)|" r"[-+]inf.0|[-+]?nan.0" r")(?=$|[)\s])") class Fraction(Number): - rx = r"-?\d+/\d+(?=$|[)\s])" + rx = r"-?[0-9]+/[0-9]+(?=$|[)\s])" class Float(Number): - rx = r"-?((\d+(\.\d*)|\.\d+)(E\d+)?)(?=$|[)\s])" + rx = r"-?(([0-9]+(\.[0-9]*)|\.[0-9]+)(E[0-9]+)?)(?=$|[)\s])" class VectorStart(OpenParen): diff --git a/tests/test_lex.py b/tests/test_lex.py new file mode 100644 index 0000000..66ba613 --- /dev/null +++ b/tests/test_lex.py @@ -0,0 +1,61 @@ +"""Tests for the lexers.""" +import ly.document +import ly.lex._token +import ly.lex.html +import ly.lex.lilypond + + +DIGIT_TOKENS = (ly.lex._token.Numeric, + ly.lex.lilypond.Fingering, + ly.lex.lilypond.StringNumber, + ly.lex.lilypond.ChordStepNumber, + ly.lex.lilypond.FigureStep, + ly.lex.lilypond.TempoSeparator, + ly.lex.lilypond.Scaling) + +TO_FULLWIDTH = str.maketrans('0123456789', '0123456789') + +SOURCES = [ + '#(display 42)', + '#(x 1/2)', + '#(x 1.5)', + '\\tempo 4 = 60 - 80', + '\\time 3/4', + 'c4', + 'c-4', + '\\override NoteHead.font-size = 2', + '\\chords { c1:5 }', + '\\figures { <1> }', + '\\score { { c1*2 } }', + '\\score { \\new TabStaff { c4\\4 } }', +] + + +def digit_tokens(text, mode=None): + """Return the text of every token the lexer only reads as such because of + the digits in it.""" + doc = ly.document.Document(text, mode) + return [str(t) for block in doc + for t in doc.tokens(block) if isinstance(t, DIGIT_TOKENS)] + + +def test_ascii_digits_are_read(): + for source in SOURCES: + assert digit_tokens(source), source + + +def test_fullwidth_digits_are_not_read(): + for source in SOURCES: + fullwidth = source.translate(TO_FULLWIDTH) + assert digit_tokens(fullwidth) == [], fullwidth + + +def entity_refs(text): + doc = ly.document.Document(text, 'html') + return [str(t) for block in doc + for t in doc.tokens(block) if isinstance(t, ly.lex.html.EntityRef)] + + +def test_only_ascii_digits_form_a_numeric_entity(): + assert entity_refs('A') == ['A'] + assert entity_refs('A'.translate(TO_FULLWIDTH)) == []