diff --git a/ly/lex/html.py b/ly/lex/html.py
index 7def595..c0875e9 100644
--- a/ly/lex/html.py
+++ b/ly/lex/html.py
@@ -96,7 +96,7 @@ class StringSQEnd(String, _token.StringEnd, _token.Leaver):
class EntityRef(_token.Character):
- rx = r"\&(#\d+|#[xX][0-9A-Fa-f]+|[A-Za-z_:][\w.:_-]*);"
+ rx = r"\&(#[0-9]+|#[xX][0-9A-Fa-f]+|[A-Za-z_:][\w.:_-]*);"
class LilyPondTag(Tag):
diff --git a/ly/lex/lilypond.py b/ly/lex/lilypond.py
index 65034d7..67e6e5d 100644
--- a/ly/lex/lilypond.py
+++ b/ly/lex/lilypond.py
@@ -45,7 +45,7 @@
re_duration = rf"(\\(maxima|longa|breve){re_identifier_end}|(1|2|4|8|16|32|64|128|256|512|1024|2048)(?!\d))"
re_dot = r"\."
-re_scaling = r"\*[\t ]*\d+(/\d+)?"
+re_scaling = r"\*[\t ]*[0-9]+(/[0-9]+)?"
@@ -72,15 +72,15 @@ class Value(_token.Item, _token.Numeric):
class DecimalValue(Value):
- rx = r"-?\d+(\.\d+)?"
+ rx = r"-?[0-9]+(\.[0-9]+)?"
class IntegerValue(DecimalValue):
- rx = r"\d+"
+ rx = r"[0-9]+"
class Fraction(Value):
- rx = r"\d+/\d+"
+ rx = r"[0-9]+/[0-9]+"
class Delimiter(_token.Token):
@@ -289,11 +289,11 @@ class ScriptAbbreviation(Articulation, _token.Leaver):
class Fingering(Articulation, _token.Leaver):
- rx = r"\d+"
+ rx = r"[0-9]+"
class StringNumber(Articulation):
- rx = r"\\\d+"
+ rx = r"\\[0-9]+"
class Slur(_token.Token):
@@ -379,7 +379,7 @@ class ChordSeparator(ChordItem):
class ChordStepNumber(ChordItem):
- rx = r"\d+[-+]?"
+ rx = r"[0-9]+[-+]?"
class DotChord(ChordItem):
@@ -571,7 +571,7 @@ def update_state(self, state):
class TempoSeparator(Delimiter):
- rx = r"[-~](?=\s*\d)"
+ rx = r"[-~](?=\s*[0-9])"
class Partial(Command):
@@ -740,7 +740,7 @@ class FigureBracket(Figure):
class FigureStep(Figure):
"""A step figure number or the underscore."""
- rx = r"_|\d+"
+ rx = r"_|[0-9]+"
class FigureAccidental(Figure):
diff --git a/ly/lex/scheme.py b/ly/lex/scheme.py
index f8d3e5a..78abb41 100644
--- a/ly/lex/scheme.py
+++ b/ly/lex/scheme.py
@@ -140,18 +140,18 @@ def test_match(cls, match):
class Number(_token.Item, _token.Numeric):
rx = (r"("
- r"-?\d+|"
+ r"-?[0-9]+|"
r"#(b[0-1]+|o[0-7]+|x[0-9a-fA-F]+)|"
r"[-+]inf.0|[-+]?nan.0"
r")(?=$|[)\s])")
class Fraction(Number):
- rx = r"-?\d+/\d+(?=$|[)\s])"
+ rx = r"-?[0-9]+/[0-9]+(?=$|[)\s])"
class Float(Number):
- rx = r"-?((\d+(\.\d*)|\.\d+)(E\d+)?)(?=$|[)\s])"
+ rx = r"-?(([0-9]+(\.[0-9]*)|\.[0-9]+)(E[0-9]+)?)(?=$|[)\s])"
class VectorStart(OpenParen):
diff --git a/tests/test_lex.py b/tests/test_lex.py
new file mode 100644
index 0000000..66ba613
--- /dev/null
+++ b/tests/test_lex.py
@@ -0,0 +1,61 @@
+"""Tests for the lexers."""
+import ly.document
+import ly.lex._token
+import ly.lex.html
+import ly.lex.lilypond
+
+
+DIGIT_TOKENS = (ly.lex._token.Numeric,
+ ly.lex.lilypond.Fingering,
+ ly.lex.lilypond.StringNumber,
+ ly.lex.lilypond.ChordStepNumber,
+ ly.lex.lilypond.FigureStep,
+ ly.lex.lilypond.TempoSeparator,
+ ly.lex.lilypond.Scaling)
+
+TO_FULLWIDTH = str.maketrans('0123456789', '0123456789')
+
+SOURCES = [
+ '#(display 42)',
+ '#(x 1/2)',
+ '#(x 1.5)',
+ '\\tempo 4 = 60 - 80',
+ '\\time 3/4',
+ 'c4',
+ 'c-4',
+ '\\override NoteHead.font-size = 2',
+ '\\chords { c1:5 }',
+ '\\figures { <1> }',
+ '\\score { { c1*2 } }',
+ '\\score { \\new TabStaff { c4\\4 } }',
+]
+
+
+def digit_tokens(text, mode=None):
+ """Return the text of every token the lexer only reads as such because of
+ the digits in it."""
+ doc = ly.document.Document(text, mode)
+ return [str(t) for block in doc
+ for t in doc.tokens(block) if isinstance(t, DIGIT_TOKENS)]
+
+
+def test_ascii_digits_are_read():
+ for source in SOURCES:
+ assert digit_tokens(source), source
+
+
+def test_fullwidth_digits_are_not_read():
+ for source in SOURCES:
+ fullwidth = source.translate(TO_FULLWIDTH)
+ assert digit_tokens(fullwidth) == [], fullwidth
+
+
+def entity_refs(text):
+ doc = ly.document.Document(text, 'html')
+ return [str(t) for block in doc
+ for t in doc.tokens(block) if isinstance(t, ly.lex.html.EntityRef)]
+
+
+def test_only_ascii_digits_form_a_numeric_entity():
+ assert entity_refs('A') == ['A']
+ assert entity_refs('A'.translate(TO_FULLWIDTH)) == []