Fix quadratic Doxygen lookahead on single-line declarations

cxxheaderparser/lexer.py

@@ -832,26 +832,35 @@ def get_doxygen_after(self) -> typing.Optional[str]:
         if not tokbuf:
             return None
 
-        # retrieve comments after non-discard elements
+        # retrieve comments after the current declaration boundary. Only scan
+        # the immediate delimiter/comment region; callers must consume any
+        # initializer/expression before asking for trailing comments.
         comments: typing.List[LexToken] = []
-        new_tokbuf = typing.Deque[LexToken]()
+        restore = typing.Deque[LexToken]()
+        seen_boundary = False
 
-        # This is different: we only extract tokens here
         while tokbuf:
             tok = tokbuf.popleft()
-            if tok.type == "NEWLINE":
+            tok_type = tok.type
+            if tok_type == "NEWLINE":
                 break
-            elif tok.type == "WHITESPACE":
-                new_tokbuf.append(tok)
-            elif tok.type in ("COMMENT_SINGLELINE", "COMMENT_MULTILINE"):
+            elif tok_type == "WHITESPACE":
+                restore.append(tok)
+                continue
+            elif tok_type in ("COMMENT_SINGLELINE", "COMMENT_MULTILINE"):
                 comments.append(tok)
+                continue
+            elif comments or seen_boundary:
+                restore.append(tok)
+                break
+
+            restore.append(tok)
+            if tok_type in (",", ";", "}"):
+                seen_boundary = True
             else:
-                new_tokbuf.append(tok)
-                if comments:
-                    break
+                break
 
-        new_tokbuf.extend(tokbuf)
-        self.tokbuf = new_tokbuf
+        tokbuf.extendleft(reversed(restore))
 
         if comments:
             return self._extract_comments(comments)

cxxheaderparser/parser.py

@@ -1324,21 +1324,27 @@ def _parse_enumerator_list(self) -> typing.List[Enumerator]:
             if name_tok.value == "}":
                 break
 
-            if doxygen is None:
-                doxygen = self.lex.get_doxygen_after()
-
             name = name_tok.value
             value = None
             attributes = []
 
-            tok = self._next_token_must_be("}", ",", "=", "DBL_LBRACKET")
-            if tok.type == "DBL_LBRACKET":
+            if doxygen is None:
+                doxygen = self.lex.get_doxygen_after()
+
+            tok = self.lex.token_if("DBL_LBRACKET")
+            if tok:
                 attributes = self._consume_attribute_specifier_seq(tok, record=False)
-                tok = self._next_token_must_be("}", ",", "=")
 
-            if tok.type == "=":
+            if doxygen is None:
+                doxygen = self.lex.get_doxygen_after()
+
+            if self.lex.token_if("="):
                 value = self._create_value(self._consume_value_until([], ",", "}"))
-                tok = self._next_token_must_be("}", ",")
+
+            if doxygen is None:
+                doxygen = self.lex.get_doxygen_after()
+
+            tok = self._next_token_must_be("}", ",")
 
             values.append(Enumerator(name, value, doxygen, attributes))
 

tests/test_doxygen.py

@@ -491,3 +491,238 @@ def test_doxygen_using_alias() -> None:
             ]
         )
     )
+
+
+def test_doxygen_oneline_variables() -> None:
+    content = """
+        int v0; /** v0 */ int v1; /** v1 */
+    """
+
+    data = parse_string(content, cleandoc=True)
+
+    assert data == ParsedData(
+        namespace=NamespaceScope(
+            variables=[
+                Variable(
+                    name=PQName(segments=[NameSpecifier(name="v0")]),
+                    type=Type(
+                        typename=PQName(segments=[FundamentalSpecifier(name="int")])
+                    ),
+                    doxygen="/** v0 */",
+                ),
+                Variable(
+                    name=PQName(segments=[NameSpecifier(name="v1")]),
+                    type=Type(
+                        typename=PQName(segments=[FundamentalSpecifier(name="int")])
+                    ),
+                    doxygen="/** v1 */",
+                ),
+            ]
+        )
+    )
+
+
+def test_doxygen_oneline_class_fields() -> None:
+    content = """
+        class C { int f0; /** f0 */ int f1; /** f1 */ };
+    """
+
+    data = parse_string(content, cleandoc=True)
+
+    assert data == ParsedData(
+        namespace=NamespaceScope(
+            classes=[
+                ClassScope(
+                    class_decl=ClassDecl(
+                        typename=PQName(
+                            segments=[NameSpecifier(name="C")], classkey="class"
+                        )
+                    ),
+                    fields=[
+                        Field(
+                            access="private",
+                            type=Type(
+                                typename=PQName(
+                                    segments=[FundamentalSpecifier(name="int")]
+                                )
+                            ),
+                            name="f0",
+                            doxygen="/** f0 */",
+                        ),
+                        Field(
+                            access="private",
+                            type=Type(
+                                typename=PQName(
+                                    segments=[FundamentalSpecifier(name="int")]
+                                )
+                            ),
+                            name="f1",
+                            doxygen="/** f1 */",
+                        ),
+                    ],
+                )
+            ]
+        )
+    )
+
+
+def test_doxygen_oneline_comma_decls() -> None:
+    content = """
+        int c0, /** c0 */ c1; /** c1 */
+    """
+
+    data = parse_string(content, cleandoc=True)
+
+    assert data == ParsedData(
+        namespace=NamespaceScope(
+            variables=[
+                Variable(
+                    name=PQName(segments=[NameSpecifier(name="c0")]),
+                    type=Type(
+                        typename=PQName(segments=[FundamentalSpecifier(name="int")])
+                    ),
+                    doxygen="/** c0 */",
+                ),
+                Variable(
+                    name=PQName(segments=[NameSpecifier(name="c1")]),
+                    type=Type(
+                        typename=PQName(segments=[FundamentalSpecifier(name="int")])
+                    ),
+                    doxygen="/** c1 */",
+                ),
+            ]
+        )
+    )
+
+
+def test_doxygen_oneline_enum_values() -> None:
+    content = """
+        enum E { E0, /** E0 */ E1 /** E1 */ };
+    """
+
+    data = parse_string(content, cleandoc=True)
+
+    assert data == ParsedData(
+        namespace=NamespaceScope(
+            enums=[
+                EnumDecl(
+                    typename=PQName(
+                        segments=[NameSpecifier(name="E")], classkey="enum"
+                    ),
+                    values=[
+                        Enumerator(name="E0", doxygen="/** E0 */"),
+                        Enumerator(name="E1", doxygen="/** E1 */"),
+                    ],
+                )
+            ]
+        )
+    )
+
+
+def test_doxygen_enum_value_with_template_comma() -> None:
+    content = """
+        enum E { E0 = Foo<int, int>::value, /** E0 */ E1 };
+    """
+
+    data = parse_string(content, cleandoc=True)
+
+    assert data == ParsedData(
+        namespace=NamespaceScope(
+            enums=[
+                EnumDecl(
+                    typename=PQName(
+                        segments=[NameSpecifier(name="E")], classkey="enum"
+                    ),
+                    values=[
+                        Enumerator(
+                            name="E0",
+                            value=Value(
+                                tokens=[
+                                    Token(value="Foo"),
+                                    Token(value="<"),
+                                    Token(value="int"),
+                                    Token(value=","),
+                                    Token(value="int"),
+                                    Token(value=">"),
+                                    Token(value="::"),
+                                    Token(value="value"),
+                                ]
+                            ),
+                            doxygen="/** E0 */",
+                        ),
+                        Enumerator(name="E1"),
+                    ],
+                )
+            ]
+        )
+    )
+
+
+def test_doxygen_enum_value_with_less_than_expression() -> None:
+    content = """
+        enum E { E0 = (1 < 2), /** E0 */ E1 };
+    """
+
+    data = parse_string(content, cleandoc=True)
+
+    assert data == ParsedData(
+        namespace=NamespaceScope(
+            enums=[
+                EnumDecl(
+                    typename=PQName(
+                        segments=[NameSpecifier(name="E")], classkey="enum"
+                    ),
+                    values=[
+                        Enumerator(
+                            name="E0",
+                            value=Value(
+                                tokens=[
+                                    Token(value="("),
+                                    Token(value="1"),
+                                    Token(value="<"),
+                                    Token(value="2"),
+                                    Token(value=")"),
+                                ]
+                            ),
+                            doxygen="/** E0 */",
+                        ),
+                        Enumerator(name="E1"),
+                    ],
+                )
+            ]
+        )
+    )
+
+
+def test_doxygen_enum_value_with_greater_than_expression() -> None:
+    content = """
+        enum E { E0 = 2 > 1, /** E0 */ E1 };
+    """
+
+    data = parse_string(content, cleandoc=True)
+
+    assert data == ParsedData(
+        namespace=NamespaceScope(
+            enums=[
+                EnumDecl(
+                    typename=PQName(
+                        segments=[NameSpecifier(name="E")], classkey="enum"
+                    ),
+                    values=[
+                        Enumerator(
+                            name="E0",
+                            value=Value(
+                                tokens=[
+                                    Token(value="2"),
+                                    Token(value=">"),
+                                    Token(value="1"),
+                                ]
+                            ),
+                            doxygen="/** E0 */",
+                        ),
+                        Enumerator(name="E1"),
+                    ],
+                )
+            ]
+        )
+    )

tests/test_doxygen_performance.py

@@ -0,0 +1,53 @@
+import collections
+
+import pytest
+
+import cxxheaderparser.lexer as lexer
+from cxxheaderparser.simple import parse_string
+
+
+class CountingDeque(collections.deque):
+    popleft_count = 0
+    restore_count = 0
+
+    def __class_getitem__(cls, item):
+        return cls
+
+    def popleft(self):
+        type(self).popleft_count += 1
+        return super().popleft()
+
+    def extend(self, values):
+        values = list(values)
+        type(self).restore_count += len(values)
+        return super().extend(values)
+
+    def extendleft(self, values):
+        values = list(values)
+        type(self).restore_count += len(values)
+        return super().extendleft(values)
+
+    @classmethod
+    def reset_counts(cls) -> None:
+        cls.popleft_count = 0
+        cls.restore_count = 0
+
+
+def _count_token_buffer_touches(
+    declaration_count: int, monkeypatch: pytest.MonkeyPatch
+) -> int:
+    monkeypatch.setattr(lexer.typing, "Deque", CountingDeque)
+    CountingDeque.reset_counts()
+
+    parse_string("int x;" * declaration_count)
+
+    return CountingDeque.popleft_count + CountingDeque.restore_count
+
+
+def test_doxygen_after_lookahead_is_not_quadratic(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    small = _count_token_buffer_touches(120, monkeypatch)
+    large = _count_token_buffer_touches(240, monkeypatch)
+
+    assert large < small * 3