diff --git a/cxxheaderparser/lexer.py b/cxxheaderparser/lexer.py index 748a186..5818a09 100644 --- a/cxxheaderparser/lexer.py +++ b/cxxheaderparser/lexer.py @@ -832,26 +832,35 @@ def get_doxygen_after(self) -> typing.Optional[str]: if not tokbuf: return None - # retrieve comments after non-discard elements + # retrieve comments after the current declaration boundary. Only scan + # the immediate delimiter/comment region; callers must consume any + # initializer/expression before asking for trailing comments. comments: typing.List[LexToken] = [] - new_tokbuf = typing.Deque[LexToken]() + restore = typing.Deque[LexToken]() + seen_boundary = False - # This is different: we only extract tokens here while tokbuf: tok = tokbuf.popleft() - if tok.type == "NEWLINE": + tok_type = tok.type + if tok_type == "NEWLINE": break - elif tok.type == "WHITESPACE": - new_tokbuf.append(tok) - elif tok.type in ("COMMENT_SINGLELINE", "COMMENT_MULTILINE"): + elif tok_type == "WHITESPACE": + restore.append(tok) + continue + elif tok_type in ("COMMENT_SINGLELINE", "COMMENT_MULTILINE"): comments.append(tok) + continue + elif comments or seen_boundary: + restore.append(tok) + break + + restore.append(tok) + if tok_type in (",", ";", "}"): + seen_boundary = True else: - new_tokbuf.append(tok) - if comments: - break + break - new_tokbuf.extend(tokbuf) - self.tokbuf = new_tokbuf + tokbuf.extendleft(reversed(restore)) if comments: return self._extract_comments(comments) diff --git a/cxxheaderparser/parser.py b/cxxheaderparser/parser.py index c5f0fbe..46efef5 100644 --- a/cxxheaderparser/parser.py +++ b/cxxheaderparser/parser.py @@ -1324,21 +1324,27 @@ def _parse_enumerator_list(self) -> typing.List[Enumerator]: if name_tok.value == "}": break - if doxygen is None: - doxygen = self.lex.get_doxygen_after() - name = name_tok.value value = None attributes = [] - tok = self._next_token_must_be("}", ",", "=", "DBL_LBRACKET") - if tok.type == "DBL_LBRACKET": + if doxygen is None: + doxygen = self.lex.get_doxygen_after() + + tok = self.lex.token_if("DBL_LBRACKET") + if tok: attributes = self._consume_attribute_specifier_seq(tok, record=False) - tok = self._next_token_must_be("}", ",", "=") - if tok.type == "=": + if doxygen is None: + doxygen = self.lex.get_doxygen_after() + + if self.lex.token_if("="): value = self._create_value(self._consume_value_until([], ",", "}")) - tok = self._next_token_must_be("}", ",") + + if doxygen is None: + doxygen = self.lex.get_doxygen_after() + + tok = self._next_token_must_be("}", ",") values.append(Enumerator(name, value, doxygen, attributes)) diff --git a/tests/test_doxygen.py b/tests/test_doxygen.py index a6aa64b..a13e963 100644 --- a/tests/test_doxygen.py +++ b/tests/test_doxygen.py @@ -491,3 +491,238 @@ def test_doxygen_using_alias() -> None: ] ) ) + + +def test_doxygen_oneline_variables() -> None: + content = """ + int v0; /** v0 */ int v1; /** v1 */ + """ + + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="v0")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + doxygen="/** v0 */", + ), + Variable( + name=PQName(segments=[NameSpecifier(name="v1")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + doxygen="/** v1 */", + ), + ] + ) + ) + + +def test_doxygen_oneline_class_fields() -> None: + content = """ + class C { int f0; /** f0 */ int f1; /** f1 */ }; + """ + + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="C")], classkey="class" + ) + ), + fields=[ + Field( + access="private", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="f0", + doxygen="/** f0 */", + ), + Field( + access="private", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="f1", + doxygen="/** f1 */", + ), + ], + ) + ] + ) + ) + + +def test_doxygen_oneline_comma_decls() -> None: + content = """ + int c0, /** c0 */ c1; /** c1 */ + """ + + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="c0")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + doxygen="/** c0 */", + ), + Variable( + name=PQName(segments=[NameSpecifier(name="c1")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + doxygen="/** c1 */", + ), + ] + ) + ) + + +def test_doxygen_oneline_enum_values() -> None: + content = """ + enum E { E0, /** E0 */ E1 /** E1 */ }; + """ + + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ), + values=[ + Enumerator(name="E0", doxygen="/** E0 */"), + Enumerator(name="E1", doxygen="/** E1 */"), + ], + ) + ] + ) + ) + + +def test_doxygen_enum_value_with_template_comma() -> None: + content = """ + enum E { E0 = Foo::value, /** E0 */ E1 }; + """ + + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ), + values=[ + Enumerator( + name="E0", + value=Value( + tokens=[ + Token(value="Foo"), + Token(value="<"), + Token(value="int"), + Token(value=","), + Token(value="int"), + Token(value=">"), + Token(value="::"), + Token(value="value"), + ] + ), + doxygen="/** E0 */", + ), + Enumerator(name="E1"), + ], + ) + ] + ) + ) + + +def test_doxygen_enum_value_with_less_than_expression() -> None: + content = """ + enum E { E0 = (1 < 2), /** E0 */ E1 }; + """ + + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ), + values=[ + Enumerator( + name="E0", + value=Value( + tokens=[ + Token(value="("), + Token(value="1"), + Token(value="<"), + Token(value="2"), + Token(value=")"), + ] + ), + doxygen="/** E0 */", + ), + Enumerator(name="E1"), + ], + ) + ] + ) + ) + + +def test_doxygen_enum_value_with_greater_than_expression() -> None: + content = """ + enum E { E0 = 2 > 1, /** E0 */ E1 }; + """ + + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ), + values=[ + Enumerator( + name="E0", + value=Value( + tokens=[ + Token(value="2"), + Token(value=">"), + Token(value="1"), + ] + ), + doxygen="/** E0 */", + ), + Enumerator(name="E1"), + ], + ) + ] + ) + ) diff --git a/tests/test_doxygen_performance.py b/tests/test_doxygen_performance.py new file mode 100644 index 0000000..ec422ac --- /dev/null +++ b/tests/test_doxygen_performance.py @@ -0,0 +1,53 @@ +import collections + +import pytest + +import cxxheaderparser.lexer as lexer +from cxxheaderparser.simple import parse_string + + +class CountingDeque(collections.deque): + popleft_count = 0 + restore_count = 0 + + def __class_getitem__(cls, item): + return cls + + def popleft(self): + type(self).popleft_count += 1 + return super().popleft() + + def extend(self, values): + values = list(values) + type(self).restore_count += len(values) + return super().extend(values) + + def extendleft(self, values): + values = list(values) + type(self).restore_count += len(values) + return super().extendleft(values) + + @classmethod + def reset_counts(cls) -> None: + cls.popleft_count = 0 + cls.restore_count = 0 + + +def _count_token_buffer_touches( + declaration_count: int, monkeypatch: pytest.MonkeyPatch +) -> int: + monkeypatch.setattr(lexer.typing, "Deque", CountingDeque) + CountingDeque.reset_counts() + + parse_string("int x;" * declaration_count) + + return CountingDeque.popleft_count + CountingDeque.restore_count + + +def test_doxygen_after_lookahead_is_not_quadratic( + monkeypatch: pytest.MonkeyPatch, +) -> None: + small = _count_token_buffer_touches(120, monkeypatch) + large = _count_token_buffer_touches(240, monkeypatch) + + assert large < small * 3