From 1bf37df4de5756393888c0713cec4b792b1e1767 Mon Sep 17 00:00:00 2001 From: Adam Layne Date: Tue, 9 Jun 2026 15:22:10 -0700 Subject: [PATCH] feat: optional progress callback for parse and unparse ReqIF files coming from real-world tools can contain hundreds of thousands of spec objects, and parsing or unparsing them takes long enough that embedding applications need progress feedback. Following the convention of urllib.request.urlretrieve(reporthook=...), ReqIFParser.parse/parse_from_string and ReqIFUnparser.unparse accept an optional callback: def progress(section: str, items_done: int, items_total: int) -> None The callback is invoked once per direct child of each REQ-IF-CONTENT container (DATATYPES, SPEC-TYPES, SPECIFICATIONS, SPEC-RELATIONS, SPEC-OBJECTS, SPEC-RELATION-GROUPS), with the container tag as the section name. ReqIFZParser.parse and ReqIFZUnparser.unparse accept the same callback and report at the archive member level: one call per member, with the member's filename as the section name. When no callback is passed, behavior and cost are unchanged. A README section documents the callback for library users. --- README.md | 34 ++++++++ reqif/parser.py | 59 ++++++++++---- reqif/progress.py | 53 +++++++++++++ reqif/unparser.py | 51 +++++++++--- tests/unit/reqif/test_progress.py | 124 ++++++++++++++++++++++++++++++ 5 files changed, 298 insertions(+), 23 deletions(-) create mode 100644 reqif/progress.py create mode 100644 tests/unit/reqif/test_progress.py diff --git a/README.md b/README.md index fe008b8..6833dc4 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,40 @@ with open(output_file_path, "w", encoding="UTF-8") as output_file: The contents of `reqif_xml_output` should be the same as the contents of the `input_file`. +### Reporting parsing/unparsing progress + +ReqIF files produced by real-world tools can contain hundreds of thousands of +spec objects, and parsing or unparsing them can take long enough that an +application wants to show progress to its user. `ReqIFParser.parse`, +`ReqIFParser.parse_from_string`, and `ReqIFUnparser.unparse` accept an +optional `progress` callback, following the convention of +[`urllib.request.urlretrieve(reporthook=...)`](https://docs.python.org/3/library/urllib.request.html#urllib.request.urlretrieve): + +```py +from reqif.parser import ReqIFParser + +def print_progress(section: str, items_done: int, items_total: int) -> None: + if items_done % 1000 == 0 or items_done == items_total: + print(f"{section}: {items_done}/{items_total}") + +reqif_bundle = ReqIFParser.parse("input.reqif", progress=print_progress) +``` + +`section` is the ReqIF container tag being processed (`"DATATYPES"`, +`"SPEC-TYPES"`, `"SPECIFICATIONS"`, `"SPEC-RELATIONS"`, `"SPEC-OBJECTS"`, or +`"SPEC-RELATION-GROUPS"`), `items_done` is the number of the container's +direct children processed so far, and `items_total` is the container's total +number of direct children. The callback is invoked once per child, so a +callback that displays progress should throttle its own output, as in the +example above. + +When no callback is passed, the parsing and unparsing behavior is unchanged. + +The `.reqifz` entry points (`ReqIFZParser.parse`, `ReqIFZUnparser.unparse`) +accept the same callback but report progress at the archive member level: +one call per processed member, with the member's filename as the `section` +argument and the archive's total number of members as `items_total`. + ### Logging The `reqif` library logs its diagnostic messages (for example, warnings about diff --git a/reqif/parser.py b/reqif/parser.py index 3513929..87d0314 100644 --- a/reqif/parser.py +++ b/reqif/parser.py @@ -62,6 +62,7 @@ from reqif.parsers.specification_parser import ( ReqIFSpecificationParser, ) +from reqif.progress import ReqIFProgressCallback, track_progress from reqif.reqif_bundle import ReqIFBundle, ReqIFZBundle logger = logging.getLogger(__name__) @@ -69,13 +70,19 @@ class ReqIFParser: @staticmethod - def parse(input_path: str) -> ReqIFBundle: + def parse( + input_path: str, + progress: Optional[ReqIFProgressCallback] = None, + ) -> ReqIFBundle: with open(input_path, "r", encoding="UTF-8") as file: content = file.read() - return ReqIFParser.parse_from_string(content) + return ReqIFParser.parse_from_string(content, progress=progress) @staticmethod - def parse_from_string(reqif_content: str) -> ReqIFBundle: + def parse_from_string( + reqif_content: str, + progress: Optional[ReqIFProgressCallback] = None, + ) -> ReqIFBundle: # LXML used to produce this error on an empty content string, but now # it simply raises an XMLSyntaxError with no message. # FIXME: No time to investigate/report this now. @@ -91,11 +98,13 @@ def parse_from_string(reqif_content: str) -> ReqIFBundle: raise ReqIFXMLParsingError(str(exception)) from None # Build ReqIF bundle. - reqif_bundle = ReqIFParser._parse_reqif(xml_reqif_root) + reqif_bundle = ReqIFParser._parse_reqif(xml_reqif_root, progress=progress) return reqif_bundle @staticmethod - def _parse_reqif(xml_reqif) -> ReqIFBundle: + def _parse_reqif( + xml_reqif, progress: Optional[ReqIFProgressCallback] = None + ) -> ReqIFBundle: docinfo: DocInfo = xml_reqif.docinfo # There should be a better way of detecting if the whole @@ -218,7 +227,9 @@ def _parse_reqif(xml_reqif) -> ReqIFBundle: reqif_content, lookup, content_exceptions, - ) = ReqIFParser._parse_reqif_content(xml_req_if_content) + ) = ReqIFParser._parse_reqif_content( + xml_req_if_content, progress=progress + ) core_content = ReqIFCoreContent(req_if_content=reqif_content) exceptions.extend(content_exceptions) else: @@ -242,6 +253,7 @@ def _parse_reqif(xml_reqif) -> ReqIFBundle: @staticmethod def _parse_reqif_content( xml_req_if_content, + progress: Optional[ReqIFProgressCallback] = None, ) -> Tuple[ReqIFReqIFContent, ReqIFObjectLookup, List[ReqIFSchemaError]]: assert xml_req_if_content is not None assert xml_req_if_content.tag == "REQ-IF-CONTENT" @@ -252,7 +264,7 @@ def _parse_reqif_content( xml_data_types = xml_req_if_content.find("DATATYPES") if xml_data_types is not None: data_types = [] - for xml_data_type in list(xml_data_types): + for xml_data_type in track_progress(xml_data_types, "DATATYPES", progress): data_type = DataTypeParser.parse(xml_data_type) data_types.append(data_type) data_types_lookup[data_type.identifier] = data_type @@ -271,7 +283,9 @@ def _parse_reqif_content( xml_spec_types = xml_req_if_content.find("SPEC-TYPES") if xml_spec_types is not None: spec_types = [] - for xml_spec_object_type_xml in list(xml_spec_types): + for xml_spec_object_type_xml in track_progress( + xml_spec_types, "SPEC-TYPES", progress + ): spec_type: Union[ ReqIFSpecObjectType, ReqIFSpecRelationType, @@ -299,7 +313,9 @@ def _parse_reqif_content( xml_specifications = xml_req_if_content.find("SPECIFICATIONS") if xml_specifications is not None: specifications = [] - for xml_specification in xml_specifications: + for xml_specification in track_progress( + xml_specifications, "SPECIFICATIONS", progress + ): specification = ReqIFSpecificationParser.parse(xml_specification) specifications.append(specification) @@ -310,7 +326,9 @@ def _parse_reqif_content( if xml_spec_relations is not None: spec_relations = [] - for xml_spec_relation in xml_spec_relations: + for xml_spec_relation in track_progress( + xml_spec_relations, "SPEC-RELATIONS", progress + ): try: spec_relation = SpecRelationParser.parse(xml_spec_relation) spec_relations.append(spec_relation) @@ -333,7 +351,9 @@ def _parse_reqif_content( xml_spec_objects = xml_req_if_content.find("SPEC-OBJECTS") if xml_spec_objects is not None: spec_objects = [] - for xml_spec_object in xml_spec_objects: + for xml_spec_object in track_progress( + xml_spec_objects, "SPEC-OBJECTS", progress + ): if lxml_is_comment_node(xml_spec_object): continue spec_object = SpecObjectParser.parse(xml_spec_object) @@ -347,7 +367,9 @@ def _parse_reqif_content( spec_relation_groups = [] if len(xml_spec_relation_groups) != 0: spec_relation_groups = [] - for xml_relation_group in xml_spec_relation_groups: + for xml_relation_group in track_progress( + xml_spec_relation_groups, "SPEC-RELATION-GROUPS", progress + ): relation_group = ReqIFRelationGroupParser.parse(xml_relation_group) spec_relation_groups.append(relation_group) @@ -370,12 +392,19 @@ def _parse_reqif_content( class ReqIFZParser: @staticmethod - def parse(input_path: str) -> ReqIFZBundle: + def parse( + input_path: str, + progress: Optional[ReqIFProgressCallback] = None, + ) -> ReqIFZBundle: try: with zipfile.ZipFile(input_path) as zip_file: attachments = {} reqif_bundles: Dict[str, ReqIFBundle] = {} - for filename in zip_file.namelist(): + # The progress over a ReqIFz archive is reported at the + # archive member level: one call per processed member, with + # the member's filename as the section name. + filenames = zip_file.namelist() + for file_index, filename in enumerate(filenames): if os.path.splitext(filename)[1] in [".reqif", ".xml"]: with zip_file.open(filename) as file: content = file.read().decode(encoding="UTF-8") @@ -385,6 +414,8 @@ def parse(input_path: str) -> ReqIFZBundle: else: with zip_file.open(filename) as file: attachments[filename] = file.read() + if progress is not None: + progress(filename, file_index + 1, len(filenames)) return ReqIFZBundle(reqif_bundles, attachments) diff --git a/reqif/progress.py b/reqif/progress.py new file mode 100644 index 0000000..a3436fe --- /dev/null +++ b/reqif/progress.py @@ -0,0 +1,53 @@ +""" +Optional progress reporting for parsing and unparsing. + +ReqIF files coming from real-world tools can contain hundreds of thousands +of spec objects, and parsing or unparsing them takes long enough that +embedding applications need progress feedback. Following the convention of +urllib.request.urlretrieve(reporthook=...), the parse and unparse entry +points accept an optional callback: + + def progress(section: str, items_done: int, items_total: int) -> None + +``section`` is the ReqIF container tag being processed (for example, +"SPEC-OBJECTS"), ``items_done`` is the number of the container's direct +children processed so far, and ``items_total`` is the container's total +number of direct children. The callback is invoked once per child, so +callers that display progress should throttle their own output. + +When no callback is passed, no callback-related work is done at all. +""" + +from typing import Callable, Iterator, Optional, Protocol, TypeVar + +ReqIFProgressCallback = Callable[[str, int, int], None] + +_T_co = TypeVar("_T_co", covariant=True) + + +class SizedIterable(Protocol[_T_co]): + """Anything supporting both len() and iteration, e.g. lists and lxml + elements.""" + + def __len__(self) -> int: ... + + def __iter__(self) -> Iterator[_T_co]: ... + + +def track_progress( + items: "SizedIterable[_T_co]", + section: str, + progress: Optional[ReqIFProgressCallback], +) -> Iterator[_T_co]: + """ + Iterate ``items``, reporting each consumed item to ``progress``. + + With ``progress=None``, this is a plain pass-through iteration. + """ + if progress is None: + yield from items + return + items_total = len(items) + for index, item in enumerate(items): + yield item + progress(section, index + 1, items_total) diff --git a/reqif/unparser.py b/reqif/unparser.py index 2abc96d..667aac3 100644 --- a/reqif/unparser.py +++ b/reqif/unparser.py @@ -1,5 +1,5 @@ import io -from typing import List +from typing import List, Optional from zipfile import ZIP_DEFLATED, ZipFile from reqif.models.reqif_namespace_info import ReqIFNamespaceInfo @@ -25,12 +25,16 @@ SpecificationTypeParser, ) from reqif.parsers.specification_parser import ReqIFSpecificationParser +from reqif.progress import ReqIFProgressCallback, track_progress from reqif.reqif_bundle import ReqIFBundle, ReqIFZBundle class ReqIFUnparser: @staticmethod - def unparse(bundle: ReqIFBundle) -> str: + def unparse( + bundle: ReqIFBundle, + progress: Optional[ReqIFProgressCallback] = None, + ) -> str: reqif_xml_output = '\n' reqif_xml_output += ReqIFUnparser.unparse_namespace_info(bundle.namespace_info) @@ -46,13 +50,17 @@ def unparse(bundle: ReqIFBundle) -> str: if reqif_content.data_types is not None: reqif_xml_output += " \n" - for data_type in reqif_content.data_types: + for data_type in track_progress( + reqif_content.data_types, "DATATYPES", progress + ): reqif_xml_output += DataTypeParser.unparse(data_type) reqif_xml_output += " \n" if reqif_content.spec_types is not None: reqif_xml_output += " \n" - for spec_type in reqif_content.spec_types: + for spec_type in track_progress( + reqif_content.spec_types, "SPEC-TYPES", progress + ): if isinstance(spec_type, ReqIFSpecObjectType): reqif_xml_output += SpecObjectTypeParser.unparse(spec_type) elif isinstance(spec_type, ReqIFSpecRelationType): @@ -72,7 +80,9 @@ def unparse(bundle: ReqIFBundle) -> str: if reqif_content.spec_objects is not None: reqif_xml_output += " \n" - for spec_object in reqif_content.spec_objects: + for spec_object in track_progress( + reqif_content.spec_objects, "SPEC-OBJECTS", progress + ): reqif_xml_output += SpecObjectParser.unparse(spec_object) reqif_xml_output += " \n" @@ -80,7 +90,9 @@ def unparse(bundle: ReqIFBundle) -> str: if reqif_content.spec_relations is not None: reqif_xml_output += " \n" - for spec_relation in reqif_content.spec_relations: + for spec_relation in track_progress( + reqif_content.spec_relations, "SPEC-RELATIONS", progress + ): reqif_xml_output += SpecRelationParser.unparse(spec_relation) reqif_xml_output += " \n" @@ -88,7 +100,9 @@ def unparse(bundle: ReqIFBundle) -> str: if reqif_content.specifications is not None: reqif_xml_output += " \n" - for specification in reqif_content.specifications: + for specification in track_progress( + reqif_content.specifications, "SPECIFICATIONS", progress + ): reqif_xml_output += ReqIFSpecificationParser.unparse( specification ) @@ -98,7 +112,11 @@ def unparse(bundle: ReqIFBundle) -> str: if reqif_content.spec_relation_groups is not None: reqif_xml_output += " \n" - for spec_relation_group in reqif_content.spec_relation_groups: + for spec_relation_group in track_progress( + reqif_content.spec_relation_groups, + "SPEC-RELATION-GROUPS", + progress, + ): reqif_xml_output += ReqIFRelationGroupParser.unparse( spec_relation_group ) @@ -158,22 +176,37 @@ def unparse_namespace_info(namespace_info: ReqIFNamespaceInfo) -> str: class ReqIFZUnparser: @staticmethod - def unparse(bundle: ReqIFZBundle) -> bytes: + def unparse( + bundle: ReqIFZBundle, + progress: Optional[ReqIFProgressCallback] = None, + ) -> bytes: """ Based on: Python in-memory zip library https://stackoverflow.com/a/44946732/598057 """ + # The progress over a ReqIFz archive is reported at the archive + # member level: one call per written member, with the member's + # filename as the section name. + files_total = len(bundle.reqif_bundles) + len(bundle.attachments) + files_done = 0 + zip_buffer = io.BytesIO() with ZipFile(zip_buffer, "a", ZIP_DEFLATED) as zip_file: # First write, the ReqIF files themselves. for filename_, reqif_bundle_ in bundle.reqif_bundles.items(): reqif_string_ = ReqIFUnparser.unparse(reqif_bundle_) zip_file.writestr(filename_, reqif_string_) + files_done += 1 + if progress is not None: + progress(filename_, files_done, files_total) # Then write the attachments. for attachment, attachment_bytes in bundle.attachments.items(): zip_file.writestr(attachment, attachment_bytes) + files_done += 1 + if progress is not None: + progress(attachment, files_done, files_total) return zip_buffer.getvalue() diff --git a/tests/unit/reqif/test_progress.py b/tests/unit/reqif/test_progress.py new file mode 100644 index 0000000..d3e5ecc --- /dev/null +++ b/tests/unit/reqif/test_progress.py @@ -0,0 +1,124 @@ +import zipfile +from typing import List, Tuple + +from reqif.parser import ReqIFParser, ReqIFZParser +from reqif.unparser import ReqIFUnparser, ReqIFZUnparser + +REQIF_WITH_CONTENT = """\ + + + + + + + t + + + + t + + + + t + + + + + + +""" + + +class ProgressRecorder: + def __init__(self): + self.calls: List[Tuple[str, int, int]] = [] + + def __call__(self, section: str, items_done: int, items_total: int): + self.calls.append((section, items_done, items_total)) + + +def test_01_parse_reports_progress(): + recorder = ProgressRecorder() + + ReqIFParser.parse_from_string(REQIF_WITH_CONTENT, progress=recorder) + + assert recorder.calls == [ + ("SPEC-OBJECTS", 1, 3), + ("SPEC-OBJECTS", 2, 3), + ("SPEC-OBJECTS", 3, 3), + ] + + +def test_02_parse_without_progress_reports_nothing(): + bundle = ReqIFParser.parse_from_string(REQIF_WITH_CONTENT) + assert len(bundle.core_content.req_if_content.spec_objects) == 3 + + +def test_03_unparse_reports_progress(): + bundle = ReqIFParser.parse_from_string(REQIF_WITH_CONTENT) + + recorder = ProgressRecorder() + output = ReqIFUnparser.unparse(bundle, progress=recorder) + + assert recorder.calls == [ + ("SPEC-OBJECTS", 1, 3), + ("SPEC-OBJECTS", 2, 3), + ("SPEC-OBJECTS", 3, 3), + ] + assert output.count(" is skipped by the parser but must + # still be counted, so that items_done reaches items_total. + reqif_with_comment = REQIF_WITH_CONTENT.replace( + " ", + " \n ", + ) + + recorder = ProgressRecorder() + ReqIFParser.parse_from_string(reqif_with_comment, progress=recorder) + + assert recorder.calls == [ + ("SPEC-OBJECTS", 1, 4), + ("SPEC-OBJECTS", 2, 4), + ("SPEC-OBJECTS", 3, 4), + ("SPEC-OBJECTS", 4, 4), + ] + + +def test_05_reqifz_parse_reports_file_level_progress(tmp_path): + reqifz_path = tmp_path / "sample.reqifz" + with zipfile.ZipFile(reqifz_path, "w") as zip_file: + zip_file.writestr("first.reqif", REQIF_WITH_CONTENT) + zip_file.writestr("attachment.txt", "Some attachment.") + + recorder = ProgressRecorder() + reqifz_bundle = ReqIFZParser.parse(str(reqifz_path), progress=recorder) + + assert recorder.calls == [ + ("first.reqif", 1, 2), + ("attachment.txt", 2, 2), + ] + assert len(reqifz_bundle.reqif_bundles) == 1 + assert len(reqifz_bundle.attachments) == 1 + + +def test_06_reqifz_unparse_reports_file_level_progress(tmp_path): + reqifz_path = tmp_path / "sample.reqifz" + with zipfile.ZipFile(reqifz_path, "w") as zip_file: + zip_file.writestr("first.reqif", REQIF_WITH_CONTENT) + zip_file.writestr("attachment.txt", "Some attachment.") + reqifz_bundle = ReqIFZParser.parse(str(reqifz_path)) + + recorder = ProgressRecorder() + output_bytes = ReqIFZUnparser.unparse(reqifz_bundle, progress=recorder) + + assert recorder.calls == [ + ("first.reqif", 1, 2), + ("attachment.txt", 2, 2), + ] + # The callback must not affect the unparse output. + assert output_bytes == ReqIFZUnparser.unparse(reqifz_bundle)