feat: optional progress callback for parse and unparse

README.md

@@ -67,6 +67,40 @@ with open(output_file_path, "w", encoding="UTF-8") as output_file:
 The contents of `reqif_xml_output` should be the same as the contents of the 
 `input_file`.
 
+### Reporting parsing/unparsing progress
+
+ReqIF files produced by real-world tools can contain hundreds of thousands of
+spec objects, and parsing or unparsing them can take long enough that an
+application wants to show progress to its user. `ReqIFParser.parse`,
+`ReqIFParser.parse_from_string`, and `ReqIFUnparser.unparse` accept an
+optional `progress` callback, following the convention of
+[`urllib.request.urlretrieve(reporthook=...)`](https://docs.python.org/3/library/urllib.request.html#urllib.request.urlretrieve):
+
+```py
+from reqif.parser import ReqIFParser
+
+def print_progress(section: str, items_done: int, items_total: int) -> None:
+    if items_done % 1000 == 0 or items_done == items_total:
+        print(f"{section}: {items_done}/{items_total}")
+
+reqif_bundle = ReqIFParser.parse("input.reqif", progress=print_progress)
+```
+
+`section` is the ReqIF container tag being processed (`"DATATYPES"`,
+`"SPEC-TYPES"`, `"SPECIFICATIONS"`, `"SPEC-RELATIONS"`, `"SPEC-OBJECTS"`, or
+`"SPEC-RELATION-GROUPS"`), `items_done` is the number of the container's
+direct children processed so far, and `items_total` is the container's total
+number of direct children. The callback is invoked once per child, so a
+callback that displays progress should throttle its own output, as in the
+example above.
+
+When no callback is passed, the parsing and unparsing behavior is unchanged.
+
+The `.reqifz` entry points (`ReqIFZParser.parse`, `ReqIFZUnparser.unparse`)
+accept the same callback but report progress at the archive member level:
+one call per processed member, with the member's filename as the `section`
+argument and the archive's total number of members as `items_total`.
+
 ### Logging
 
 The `reqif` library logs its diagnostic messages (for example, warnings about

reqif/parser.py

@@ -62,20 +62,27 @@
 from reqif.parsers.specification_parser import (
     ReqIFSpecificationParser,
 )
+from reqif.progress import ReqIFProgressCallback, track_progress
 from reqif.reqif_bundle import ReqIFBundle, ReqIFZBundle
 
 logger = logging.getLogger(__name__)
 
 
 class ReqIFParser:
     @staticmethod
-    def parse(input_path: str) -> ReqIFBundle:
+    def parse(
+        input_path: str,
+        progress: Optional[ReqIFProgressCallback] = None,
+    ) -> ReqIFBundle:
         with open(input_path, "r", encoding="UTF-8") as file:
             content = file.read()
-        return ReqIFParser.parse_from_string(content)
+        return ReqIFParser.parse_from_string(content, progress=progress)
 
     @staticmethod
-    def parse_from_string(reqif_content: str) -> ReqIFBundle:
+    def parse_from_string(
+        reqif_content: str,
+        progress: Optional[ReqIFProgressCallback] = None,
+    ) -> ReqIFBundle:
         # LXML used to produce this error on an empty content string, but now
         # it simply raises an XMLSyntaxError with no message.
         # FIXME: No time to investigate/report this now.
@@ -91,11 +98,13 @@ def parse_from_string(reqif_content: str) -> ReqIFBundle:
             raise ReqIFXMLParsingError(str(exception)) from None
 
         # Build ReqIF bundle.
-        reqif_bundle = ReqIFParser._parse_reqif(xml_reqif_root)
+        reqif_bundle = ReqIFParser._parse_reqif(xml_reqif_root, progress=progress)
         return reqif_bundle
 
     @staticmethod
-    def _parse_reqif(xml_reqif) -> ReqIFBundle:
+    def _parse_reqif(
+        xml_reqif, progress: Optional[ReqIFProgressCallback] = None
+    ) -> ReqIFBundle:
         docinfo: DocInfo = xml_reqif.docinfo
 
         # There should be a better way of detecting if the whole
@@ -218,7 +227,9 @@ def _parse_reqif(xml_reqif) -> ReqIFBundle:
                     reqif_content,
                     lookup,
                     content_exceptions,
-                ) = ReqIFParser._parse_reqif_content(xml_req_if_content)
+                ) = ReqIFParser._parse_reqif_content(
+                    xml_req_if_content, progress=progress
+                )
                 core_content = ReqIFCoreContent(req_if_content=reqif_content)
                 exceptions.extend(content_exceptions)
             else:
@@ -242,6 +253,7 @@ def _parse_reqif(xml_reqif) -> ReqIFBundle:
     @staticmethod
     def _parse_reqif_content(
         xml_req_if_content,
+        progress: Optional[ReqIFProgressCallback] = None,
     ) -> Tuple[ReqIFReqIFContent, ReqIFObjectLookup, List[ReqIFSchemaError]]:
         assert xml_req_if_content is not None
         assert xml_req_if_content.tag == "REQ-IF-CONTENT"
@@ -252,7 +264,7 @@ def _parse_reqif_content(
         xml_data_types = xml_req_if_content.find("DATATYPES")
         if xml_data_types is not None:
             data_types = []
-            for xml_data_type in list(xml_data_types):
+            for xml_data_type in track_progress(xml_data_types, "DATATYPES", progress):
                 data_type = DataTypeParser.parse(xml_data_type)
                 data_types.append(data_type)
                 data_types_lookup[data_type.identifier] = data_type
@@ -271,7 +283,9 @@ def _parse_reqif_content(
         xml_spec_types = xml_req_if_content.find("SPEC-TYPES")
         if xml_spec_types is not None:
             spec_types = []
-            for xml_spec_object_type_xml in list(xml_spec_types):
+            for xml_spec_object_type_xml in track_progress(
+                xml_spec_types, "SPEC-TYPES", progress
+            ):
                 spec_type: Union[
                     ReqIFSpecObjectType,
                     ReqIFSpecRelationType,
@@ -299,7 +313,9 @@ def _parse_reqif_content(
         xml_specifications = xml_req_if_content.find("SPECIFICATIONS")
         if xml_specifications is not None:
             specifications = []
-            for xml_specification in xml_specifications:
+            for xml_specification in track_progress(
+                xml_specifications, "SPECIFICATIONS", progress
+            ):
                 specification = ReqIFSpecificationParser.parse(xml_specification)
                 specifications.append(specification)
 
@@ -310,7 +326,9 @@ def _parse_reqif_content(
         if xml_spec_relations is not None:
             spec_relations = []
 
-            for xml_spec_relation in xml_spec_relations:
+            for xml_spec_relation in track_progress(
+                xml_spec_relations, "SPEC-RELATIONS", progress
+            ):
                 try:
                     spec_relation = SpecRelationParser.parse(xml_spec_relation)
                     spec_relations.append(spec_relation)
@@ -333,7 +351,9 @@ def _parse_reqif_content(
         xml_spec_objects = xml_req_if_content.find("SPEC-OBJECTS")
         if xml_spec_objects is not None:
             spec_objects = []
-            for xml_spec_object in xml_spec_objects:
+            for xml_spec_object in track_progress(
+                xml_spec_objects, "SPEC-OBJECTS", progress
+            ):
                 if lxml_is_comment_node(xml_spec_object):
                     continue
                 spec_object = SpecObjectParser.parse(xml_spec_object)
@@ -347,7 +367,9 @@ def _parse_reqif_content(
             spec_relation_groups = []
             if len(xml_spec_relation_groups) != 0:
                 spec_relation_groups = []
-                for xml_relation_group in xml_spec_relation_groups:
+                for xml_relation_group in track_progress(
+                    xml_spec_relation_groups, "SPEC-RELATION-GROUPS", progress
+                ):
                     relation_group = ReqIFRelationGroupParser.parse(xml_relation_group)
                     spec_relation_groups.append(relation_group)
 
@@ -370,12 +392,19 @@ def _parse_reqif_content(
 
 class ReqIFZParser:
     @staticmethod
-    def parse(input_path: str) -> ReqIFZBundle:
+    def parse(
+        input_path: str,
+        progress: Optional[ReqIFProgressCallback] = None,
+    ) -> ReqIFZBundle:
         try:
             with zipfile.ZipFile(input_path) as zip_file:
                 attachments = {}
                 reqif_bundles: Dict[str, ReqIFBundle] = {}
-                for filename in zip_file.namelist():
+                # The progress over a ReqIFz archive is reported at the
+                # archive member level: one call per processed member, with
+                # the member's filename as the section name.
+                filenames = zip_file.namelist()
+                for file_index, filename in enumerate(filenames):
                     if os.path.splitext(filename)[1] in [".reqif", ".xml"]:
                         with zip_file.open(filename) as file:
                             content = file.read().decode(encoding="UTF-8")
@@ -385,6 +414,8 @@ def parse(input_path: str) -> ReqIFZBundle:
                     else:
                         with zip_file.open(filename) as file:
                             attachments[filename] = file.read()
+                    if progress is not None:
+                        progress(filename, file_index + 1, len(filenames))
 
                 return ReqIFZBundle(reqif_bundles, attachments)
 

reqif/progress.py

@@ -0,0 +1,53 @@
+"""
+Optional progress reporting for parsing and unparsing.
+
+ReqIF files coming from real-world tools can contain hundreds of thousands
+of spec objects, and parsing or unparsing them takes long enough that
+embedding applications need progress feedback. Following the convention of
+urllib.request.urlretrieve(reporthook=...), the parse and unparse entry
+points accept an optional callback:
+
+    def progress(section: str, items_done: int, items_total: int) -> None
+
+``section`` is the ReqIF container tag being processed (for example,
+"SPEC-OBJECTS"), ``items_done`` is the number of the container's direct
+children processed so far, and ``items_total`` is the container's total
+number of direct children. The callback is invoked once per child, so
+callers that display progress should throttle their own output.
+
+When no callback is passed, no callback-related work is done at all.
+"""
+
+from typing import Callable, Iterator, Optional, Protocol, TypeVar
+
+ReqIFProgressCallback = Callable[[str, int, int], None]
+
+_T_co = TypeVar("_T_co", covariant=True)
+
+
+class SizedIterable(Protocol[_T_co]):
+    """Anything supporting both len() and iteration, e.g. lists and lxml
+    elements."""
+
+    def __len__(self) -> int: ...
+
+    def __iter__(self) -> Iterator[_T_co]: ...
+
+
+def track_progress(
+    items: "SizedIterable[_T_co]",
+    section: str,
+    progress: Optional[ReqIFProgressCallback],
+) -> Iterator[_T_co]:
+    """
+    Iterate ``items``, reporting each consumed item to ``progress``.
+
+    With ``progress=None``, this is a plain pass-through iteration.
+    """
+    if progress is None:
+        yield from items
+        return
+    items_total = len(items)
+    for index, item in enumerate(items):
+        yield item
+        progress(section, index + 1, items_total)

reqif/unparser.py

@@ -1,5 +1,5 @@
 import io
-from typing import List
+from typing import List, Optional
 from zipfile import ZIP_DEFLATED, ZipFile
 
 from reqif.models.reqif_namespace_info import ReqIFNamespaceInfo
@@ -25,12 +25,16 @@
     SpecificationTypeParser,
 )
 from reqif.parsers.specification_parser import ReqIFSpecificationParser
+from reqif.progress import ReqIFProgressCallback, track_progress
 from reqif.reqif_bundle import ReqIFBundle, ReqIFZBundle
 
 
 class ReqIFUnparser:
     @staticmethod
-    def unparse(bundle: ReqIFBundle) -> str:
+    def unparse(
+        bundle: ReqIFBundle,
+        progress: Optional[ReqIFProgressCallback] = None,
+    ) -> str:
         reqif_xml_output = '<?xml version="1.0" encoding="UTF-8"?>\n'
 
         reqif_xml_output += ReqIFUnparser.unparse_namespace_info(bundle.namespace_info)
@@ -46,13 +50,17 @@ def unparse(bundle: ReqIFBundle) -> str:
 
                 if reqif_content.data_types is not None:
                     reqif_xml_output += "      <DATATYPES>\n"
-                    for data_type in reqif_content.data_types:
+                    for data_type in track_progress(
+                        reqif_content.data_types, "DATATYPES", progress
+                    ):
                         reqif_xml_output += DataTypeParser.unparse(data_type)
                     reqif_xml_output += "      </DATATYPES>\n"
 
                 if reqif_content.spec_types is not None:
                     reqif_xml_output += "      <SPEC-TYPES>\n"
-                    for spec_type in reqif_content.spec_types:
+                    for spec_type in track_progress(
+                        reqif_content.spec_types, "SPEC-TYPES", progress
+                    ):
                         if isinstance(spec_type, ReqIFSpecObjectType):
                             reqif_xml_output += SpecObjectTypeParser.unparse(spec_type)
                         elif isinstance(spec_type, ReqIFSpecRelationType):
@@ -72,23 +80,29 @@ def unparse(bundle: ReqIFBundle) -> str:
                 if reqif_content.spec_objects is not None:
                     reqif_xml_output += "      <SPEC-OBJECTS>\n"
 
-                    for spec_object in reqif_content.spec_objects:
+                    for spec_object in track_progress(
+                        reqif_content.spec_objects, "SPEC-OBJECTS", progress
+                    ):
                         reqif_xml_output += SpecObjectParser.unparse(spec_object)
 
                     reqif_xml_output += "      </SPEC-OBJECTS>\n"
 
                 if reqif_content.spec_relations is not None:
                     reqif_xml_output += "      <SPEC-RELATIONS>\n"
 
-                    for spec_relation in reqif_content.spec_relations:
+                    for spec_relation in track_progress(
+                        reqif_content.spec_relations, "SPEC-RELATIONS", progress
+                    ):
                         reqif_xml_output += SpecRelationParser.unparse(spec_relation)
 
                     reqif_xml_output += "      </SPEC-RELATIONS>\n"
 
                 if reqif_content.specifications is not None:
                     reqif_xml_output += "      <SPECIFICATIONS>\n"
 
-                    for specification in reqif_content.specifications:
+                    for specification in track_progress(
+                        reqif_content.specifications, "SPECIFICATIONS", progress
+                    ):
                         reqif_xml_output += ReqIFSpecificationParser.unparse(
                             specification
                         )
@@ -98,7 +112,11 @@ def unparse(bundle: ReqIFBundle) -> str:
                 if reqif_content.spec_relation_groups is not None:
                     reqif_xml_output += "      <SPEC-RELATION-GROUPS>\n"
 
-                    for spec_relation_group in reqif_content.spec_relation_groups:
+                    for spec_relation_group in track_progress(
+                        reqif_content.spec_relation_groups,
+                        "SPEC-RELATION-GROUPS",
+                        progress,
+                    ):
                         reqif_xml_output += ReqIFRelationGroupParser.unparse(
                             spec_relation_group
                         )
@@ -158,22 +176,37 @@ def unparse_namespace_info(namespace_info: ReqIFNamespaceInfo) -> str:
 
 class ReqIFZUnparser:
     @staticmethod
-    def unparse(bundle: ReqIFZBundle) -> bytes:
+    def unparse(
+        bundle: ReqIFZBundle,
+        progress: Optional[ReqIFProgressCallback] = None,
+    ) -> bytes:
         """
         Based on:
         Python in-memory zip library
         https://stackoverflow.com/a/44946732/598057
         """
 
+        # The progress over a ReqIFz archive is reported at the archive
+        # member level: one call per written member, with the member's
+        # filename as the section name.
+        files_total = len(bundle.reqif_bundles) + len(bundle.attachments)
+        files_done = 0
+
         zip_buffer = io.BytesIO()
         with ZipFile(zip_buffer, "a", ZIP_DEFLATED) as zip_file:
             # First write, the ReqIF files themselves.
             for filename_, reqif_bundle_ in bundle.reqif_bundles.items():
                 reqif_string_ = ReqIFUnparser.unparse(reqif_bundle_)
                 zip_file.writestr(filename_, reqif_string_)
+                files_done += 1
+                if progress is not None:
+                    progress(filename_, files_done, files_total)
 
             # Then write the attachments.
             for attachment, attachment_bytes in bundle.attachments.items():
                 zip_file.writestr(attachment, attachment_bytes)
+                files_done += 1
+                if progress is not None:
+                    progress(attachment, files_done, files_total)
 
         return zip_buffer.getvalue()