diff --git a/README.md b/README.md
index fe008b8..6833dc4 100644
--- a/README.md
+++ b/README.md
@@ -67,6 +67,40 @@ with open(output_file_path, "w", encoding="UTF-8") as output_file:
The contents of `reqif_xml_output` should be the same as the contents of the
`input_file`.
+### Reporting parsing/unparsing progress
+
+ReqIF files produced by real-world tools can contain hundreds of thousands of
+spec objects, and parsing or unparsing them can take long enough that an
+application wants to show progress to its user. `ReqIFParser.parse`,
+`ReqIFParser.parse_from_string`, and `ReqIFUnparser.unparse` accept an
+optional `progress` callback, following the convention of
+[`urllib.request.urlretrieve(reporthook=...)`](https://docs.python.org/3/library/urllib.request.html#urllib.request.urlretrieve):
+
+```py
+from reqif.parser import ReqIFParser
+
+def print_progress(section: str, items_done: int, items_total: int) -> None:
+ if items_done % 1000 == 0 or items_done == items_total:
+ print(f"{section}: {items_done}/{items_total}")
+
+reqif_bundle = ReqIFParser.parse("input.reqif", progress=print_progress)
+```
+
+`section` is the ReqIF container tag being processed (`"DATATYPES"`,
+`"SPEC-TYPES"`, `"SPECIFICATIONS"`, `"SPEC-RELATIONS"`, `"SPEC-OBJECTS"`, or
+`"SPEC-RELATION-GROUPS"`), `items_done` is the number of the container's
+direct children processed so far, and `items_total` is the container's total
+number of direct children. The callback is invoked once per child, so a
+callback that displays progress should throttle its own output, as in the
+example above.
+
+When no callback is passed, the parsing and unparsing behavior is unchanged.
+
+The `.reqifz` entry points (`ReqIFZParser.parse`, `ReqIFZUnparser.unparse`)
+accept the same callback but report progress at the archive member level:
+one call per processed member, with the member's filename as the `section`
+argument and the archive's total number of members as `items_total`.
+
### Logging
The `reqif` library logs its diagnostic messages (for example, warnings about
diff --git a/reqif/parser.py b/reqif/parser.py
index 3513929..87d0314 100644
--- a/reqif/parser.py
+++ b/reqif/parser.py
@@ -62,6 +62,7 @@
from reqif.parsers.specification_parser import (
ReqIFSpecificationParser,
)
+from reqif.progress import ReqIFProgressCallback, track_progress
from reqif.reqif_bundle import ReqIFBundle, ReqIFZBundle
logger = logging.getLogger(__name__)
@@ -69,13 +70,19 @@
class ReqIFParser:
@staticmethod
- def parse(input_path: str) -> ReqIFBundle:
+ def parse(
+ input_path: str,
+ progress: Optional[ReqIFProgressCallback] = None,
+ ) -> ReqIFBundle:
with open(input_path, "r", encoding="UTF-8") as file:
content = file.read()
- return ReqIFParser.parse_from_string(content)
+ return ReqIFParser.parse_from_string(content, progress=progress)
@staticmethod
- def parse_from_string(reqif_content: str) -> ReqIFBundle:
+ def parse_from_string(
+ reqif_content: str,
+ progress: Optional[ReqIFProgressCallback] = None,
+ ) -> ReqIFBundle:
# LXML used to produce this error on an empty content string, but now
# it simply raises an XMLSyntaxError with no message.
# FIXME: No time to investigate/report this now.
@@ -91,11 +98,13 @@ def parse_from_string(reqif_content: str) -> ReqIFBundle:
raise ReqIFXMLParsingError(str(exception)) from None
# Build ReqIF bundle.
- reqif_bundle = ReqIFParser._parse_reqif(xml_reqif_root)
+ reqif_bundle = ReqIFParser._parse_reqif(xml_reqif_root, progress=progress)
return reqif_bundle
@staticmethod
- def _parse_reqif(xml_reqif) -> ReqIFBundle:
+ def _parse_reqif(
+ xml_reqif, progress: Optional[ReqIFProgressCallback] = None
+ ) -> ReqIFBundle:
docinfo: DocInfo = xml_reqif.docinfo
# There should be a better way of detecting if the whole
@@ -218,7 +227,9 @@ def _parse_reqif(xml_reqif) -> ReqIFBundle:
reqif_content,
lookup,
content_exceptions,
- ) = ReqIFParser._parse_reqif_content(xml_req_if_content)
+ ) = ReqIFParser._parse_reqif_content(
+ xml_req_if_content, progress=progress
+ )
core_content = ReqIFCoreContent(req_if_content=reqif_content)
exceptions.extend(content_exceptions)
else:
@@ -242,6 +253,7 @@ def _parse_reqif(xml_reqif) -> ReqIFBundle:
@staticmethod
def _parse_reqif_content(
xml_req_if_content,
+ progress: Optional[ReqIFProgressCallback] = None,
) -> Tuple[ReqIFReqIFContent, ReqIFObjectLookup, List[ReqIFSchemaError]]:
assert xml_req_if_content is not None
assert xml_req_if_content.tag == "REQ-IF-CONTENT"
@@ -252,7 +264,7 @@ def _parse_reqif_content(
xml_data_types = xml_req_if_content.find("DATATYPES")
if xml_data_types is not None:
data_types = []
- for xml_data_type in list(xml_data_types):
+ for xml_data_type in track_progress(xml_data_types, "DATATYPES", progress):
data_type = DataTypeParser.parse(xml_data_type)
data_types.append(data_type)
data_types_lookup[data_type.identifier] = data_type
@@ -271,7 +283,9 @@ def _parse_reqif_content(
xml_spec_types = xml_req_if_content.find("SPEC-TYPES")
if xml_spec_types is not None:
spec_types = []
- for xml_spec_object_type_xml in list(xml_spec_types):
+ for xml_spec_object_type_xml in track_progress(
+ xml_spec_types, "SPEC-TYPES", progress
+ ):
spec_type: Union[
ReqIFSpecObjectType,
ReqIFSpecRelationType,
@@ -299,7 +313,9 @@ def _parse_reqif_content(
xml_specifications = xml_req_if_content.find("SPECIFICATIONS")
if xml_specifications is not None:
specifications = []
- for xml_specification in xml_specifications:
+ for xml_specification in track_progress(
+ xml_specifications, "SPECIFICATIONS", progress
+ ):
specification = ReqIFSpecificationParser.parse(xml_specification)
specifications.append(specification)
@@ -310,7 +326,9 @@ def _parse_reqif_content(
if xml_spec_relations is not None:
spec_relations = []
- for xml_spec_relation in xml_spec_relations:
+ for xml_spec_relation in track_progress(
+ xml_spec_relations, "SPEC-RELATIONS", progress
+ ):
try:
spec_relation = SpecRelationParser.parse(xml_spec_relation)
spec_relations.append(spec_relation)
@@ -333,7 +351,9 @@ def _parse_reqif_content(
xml_spec_objects = xml_req_if_content.find("SPEC-OBJECTS")
if xml_spec_objects is not None:
spec_objects = []
- for xml_spec_object in xml_spec_objects:
+ for xml_spec_object in track_progress(
+ xml_spec_objects, "SPEC-OBJECTS", progress
+ ):
if lxml_is_comment_node(xml_spec_object):
continue
spec_object = SpecObjectParser.parse(xml_spec_object)
@@ -347,7 +367,9 @@ def _parse_reqif_content(
spec_relation_groups = []
if len(xml_spec_relation_groups) != 0:
spec_relation_groups = []
- for xml_relation_group in xml_spec_relation_groups:
+ for xml_relation_group in track_progress(
+ xml_spec_relation_groups, "SPEC-RELATION-GROUPS", progress
+ ):
relation_group = ReqIFRelationGroupParser.parse(xml_relation_group)
spec_relation_groups.append(relation_group)
@@ -370,12 +392,19 @@ def _parse_reqif_content(
class ReqIFZParser:
@staticmethod
- def parse(input_path: str) -> ReqIFZBundle:
+ def parse(
+ input_path: str,
+ progress: Optional[ReqIFProgressCallback] = None,
+ ) -> ReqIFZBundle:
try:
with zipfile.ZipFile(input_path) as zip_file:
attachments = {}
reqif_bundles: Dict[str, ReqIFBundle] = {}
- for filename in zip_file.namelist():
+ # The progress over a ReqIFz archive is reported at the
+ # archive member level: one call per processed member, with
+ # the member's filename as the section name.
+ filenames = zip_file.namelist()
+ for file_index, filename in enumerate(filenames):
if os.path.splitext(filename)[1] in [".reqif", ".xml"]:
with zip_file.open(filename) as file:
content = file.read().decode(encoding="UTF-8")
@@ -385,6 +414,8 @@ def parse(input_path: str) -> ReqIFZBundle:
else:
with zip_file.open(filename) as file:
attachments[filename] = file.read()
+ if progress is not None:
+ progress(filename, file_index + 1, len(filenames))
return ReqIFZBundle(reqif_bundles, attachments)
diff --git a/reqif/progress.py b/reqif/progress.py
new file mode 100644
index 0000000..a3436fe
--- /dev/null
+++ b/reqif/progress.py
@@ -0,0 +1,53 @@
+"""
+Optional progress reporting for parsing and unparsing.
+
+ReqIF files coming from real-world tools can contain hundreds of thousands
+of spec objects, and parsing or unparsing them takes long enough that
+embedding applications need progress feedback. Following the convention of
+urllib.request.urlretrieve(reporthook=...), the parse and unparse entry
+points accept an optional callback:
+
+ def progress(section: str, items_done: int, items_total: int) -> None
+
+``section`` is the ReqIF container tag being processed (for example,
+"SPEC-OBJECTS"), ``items_done`` is the number of the container's direct
+children processed so far, and ``items_total`` is the container's total
+number of direct children. The callback is invoked once per child, so
+callers that display progress should throttle their own output.
+
+When no callback is passed, no callback-related work is done at all.
+"""
+
+from typing import Callable, Iterator, Optional, Protocol, TypeVar
+
+ReqIFProgressCallback = Callable[[str, int, int], None]
+
+_T_co = TypeVar("_T_co", covariant=True)
+
+
+class SizedIterable(Protocol[_T_co]):
+ """Anything supporting both len() and iteration, e.g. lists and lxml
+ elements."""
+
+ def __len__(self) -> int: ...
+
+ def __iter__(self) -> Iterator[_T_co]: ...
+
+
+def track_progress(
+ items: "SizedIterable[_T_co]",
+ section: str,
+ progress: Optional[ReqIFProgressCallback],
+) -> Iterator[_T_co]:
+ """
+ Iterate ``items``, reporting each consumed item to ``progress``.
+
+ With ``progress=None``, this is a plain pass-through iteration.
+ """
+ if progress is None:
+ yield from items
+ return
+ items_total = len(items)
+ for index, item in enumerate(items):
+ yield item
+ progress(section, index + 1, items_total)
diff --git a/reqif/unparser.py b/reqif/unparser.py
index 2abc96d..667aac3 100644
--- a/reqif/unparser.py
+++ b/reqif/unparser.py
@@ -1,5 +1,5 @@
import io
-from typing import List
+from typing import List, Optional
from zipfile import ZIP_DEFLATED, ZipFile
from reqif.models.reqif_namespace_info import ReqIFNamespaceInfo
@@ -25,12 +25,16 @@
SpecificationTypeParser,
)
from reqif.parsers.specification_parser import ReqIFSpecificationParser
+from reqif.progress import ReqIFProgressCallback, track_progress
from reqif.reqif_bundle import ReqIFBundle, ReqIFZBundle
class ReqIFUnparser:
@staticmethod
- def unparse(bundle: ReqIFBundle) -> str:
+ def unparse(
+ bundle: ReqIFBundle,
+ progress: Optional[ReqIFProgressCallback] = None,
+ ) -> str:
reqif_xml_output = '\n'
reqif_xml_output += ReqIFUnparser.unparse_namespace_info(bundle.namespace_info)
@@ -46,13 +50,17 @@ def unparse(bundle: ReqIFBundle) -> str:
if reqif_content.data_types is not None:
reqif_xml_output += " \n"
- for data_type in reqif_content.data_types:
+ for data_type in track_progress(
+ reqif_content.data_types, "DATATYPES", progress
+ ):
reqif_xml_output += DataTypeParser.unparse(data_type)
reqif_xml_output += " \n"
if reqif_content.spec_types is not None:
reqif_xml_output += " \n"
- for spec_type in reqif_content.spec_types:
+ for spec_type in track_progress(
+ reqif_content.spec_types, "SPEC-TYPES", progress
+ ):
if isinstance(spec_type, ReqIFSpecObjectType):
reqif_xml_output += SpecObjectTypeParser.unparse(spec_type)
elif isinstance(spec_type, ReqIFSpecRelationType):
@@ -72,7 +80,9 @@ def unparse(bundle: ReqIFBundle) -> str:
if reqif_content.spec_objects is not None:
reqif_xml_output += " \n"
- for spec_object in reqif_content.spec_objects:
+ for spec_object in track_progress(
+ reqif_content.spec_objects, "SPEC-OBJECTS", progress
+ ):
reqif_xml_output += SpecObjectParser.unparse(spec_object)
reqif_xml_output += " \n"
@@ -80,7 +90,9 @@ def unparse(bundle: ReqIFBundle) -> str:
if reqif_content.spec_relations is not None:
reqif_xml_output += " \n"
- for spec_relation in reqif_content.spec_relations:
+ for spec_relation in track_progress(
+ reqif_content.spec_relations, "SPEC-RELATIONS", progress
+ ):
reqif_xml_output += SpecRelationParser.unparse(spec_relation)
reqif_xml_output += " \n"
@@ -88,7 +100,9 @@ def unparse(bundle: ReqIFBundle) -> str:
if reqif_content.specifications is not None:
reqif_xml_output += " \n"
- for specification in reqif_content.specifications:
+ for specification in track_progress(
+ reqif_content.specifications, "SPECIFICATIONS", progress
+ ):
reqif_xml_output += ReqIFSpecificationParser.unparse(
specification
)
@@ -98,7 +112,11 @@ def unparse(bundle: ReqIFBundle) -> str:
if reqif_content.spec_relation_groups is not None:
reqif_xml_output += " \n"
- for spec_relation_group in reqif_content.spec_relation_groups:
+ for spec_relation_group in track_progress(
+ reqif_content.spec_relation_groups,
+ "SPEC-RELATION-GROUPS",
+ progress,
+ ):
reqif_xml_output += ReqIFRelationGroupParser.unparse(
spec_relation_group
)
@@ -158,22 +176,37 @@ def unparse_namespace_info(namespace_info: ReqIFNamespaceInfo) -> str:
class ReqIFZUnparser:
@staticmethod
- def unparse(bundle: ReqIFZBundle) -> bytes:
+ def unparse(
+ bundle: ReqIFZBundle,
+ progress: Optional[ReqIFProgressCallback] = None,
+ ) -> bytes:
"""
Based on:
Python in-memory zip library
https://stackoverflow.com/a/44946732/598057
"""
+ # The progress over a ReqIFz archive is reported at the archive
+ # member level: one call per written member, with the member's
+ # filename as the section name.
+ files_total = len(bundle.reqif_bundles) + len(bundle.attachments)
+ files_done = 0
+
zip_buffer = io.BytesIO()
with ZipFile(zip_buffer, "a", ZIP_DEFLATED) as zip_file:
# First write, the ReqIF files themselves.
for filename_, reqif_bundle_ in bundle.reqif_bundles.items():
reqif_string_ = ReqIFUnparser.unparse(reqif_bundle_)
zip_file.writestr(filename_, reqif_string_)
+ files_done += 1
+ if progress is not None:
+ progress(filename_, files_done, files_total)
# Then write the attachments.
for attachment, attachment_bytes in bundle.attachments.items():
zip_file.writestr(attachment, attachment_bytes)
+ files_done += 1
+ if progress is not None:
+ progress(attachment, files_done, files_total)
return zip_buffer.getvalue()
diff --git a/tests/unit/reqif/test_progress.py b/tests/unit/reqif/test_progress.py
new file mode 100644
index 0000000..d3e5ecc
--- /dev/null
+++ b/tests/unit/reqif/test_progress.py
@@ -0,0 +1,124 @@
+import zipfile
+from typing import List, Tuple
+
+from reqif.parser import ReqIFParser, ReqIFZParser
+from reqif.unparser import ReqIFUnparser, ReqIFZUnparser
+
+REQIF_WITH_CONTENT = """\
+
+
+
+
+
+
+ t
+
+
+
+ t
+
+
+
+ t
+
+
+
+
+
+
+"""
+
+
+class ProgressRecorder:
+ def __init__(self):
+ self.calls: List[Tuple[str, int, int]] = []
+
+ def __call__(self, section: str, items_done: int, items_total: int):
+ self.calls.append((section, items_done, items_total))
+
+
+def test_01_parse_reports_progress():
+ recorder = ProgressRecorder()
+
+ ReqIFParser.parse_from_string(REQIF_WITH_CONTENT, progress=recorder)
+
+ assert recorder.calls == [
+ ("SPEC-OBJECTS", 1, 3),
+ ("SPEC-OBJECTS", 2, 3),
+ ("SPEC-OBJECTS", 3, 3),
+ ]
+
+
+def test_02_parse_without_progress_reports_nothing():
+ bundle = ReqIFParser.parse_from_string(REQIF_WITH_CONTENT)
+ assert len(bundle.core_content.req_if_content.spec_objects) == 3
+
+
+def test_03_unparse_reports_progress():
+ bundle = ReqIFParser.parse_from_string(REQIF_WITH_CONTENT)
+
+ recorder = ProgressRecorder()
+ output = ReqIFUnparser.unparse(bundle, progress=recorder)
+
+ assert recorder.calls == [
+ ("SPEC-OBJECTS", 1, 3),
+ ("SPEC-OBJECTS", 2, 3),
+ ("SPEC-OBJECTS", 3, 3),
+ ]
+ assert output.count(" is skipped by the parser but must
+ # still be counted, so that items_done reaches items_total.
+ reqif_with_comment = REQIF_WITH_CONTENT.replace(
+ " ",
+ " \n ",
+ )
+
+ recorder = ProgressRecorder()
+ ReqIFParser.parse_from_string(reqif_with_comment, progress=recorder)
+
+ assert recorder.calls == [
+ ("SPEC-OBJECTS", 1, 4),
+ ("SPEC-OBJECTS", 2, 4),
+ ("SPEC-OBJECTS", 3, 4),
+ ("SPEC-OBJECTS", 4, 4),
+ ]
+
+
+def test_05_reqifz_parse_reports_file_level_progress(tmp_path):
+ reqifz_path = tmp_path / "sample.reqifz"
+ with zipfile.ZipFile(reqifz_path, "w") as zip_file:
+ zip_file.writestr("first.reqif", REQIF_WITH_CONTENT)
+ zip_file.writestr("attachment.txt", "Some attachment.")
+
+ recorder = ProgressRecorder()
+ reqifz_bundle = ReqIFZParser.parse(str(reqifz_path), progress=recorder)
+
+ assert recorder.calls == [
+ ("first.reqif", 1, 2),
+ ("attachment.txt", 2, 2),
+ ]
+ assert len(reqifz_bundle.reqif_bundles) == 1
+ assert len(reqifz_bundle.attachments) == 1
+
+
+def test_06_reqifz_unparse_reports_file_level_progress(tmp_path):
+ reqifz_path = tmp_path / "sample.reqifz"
+ with zipfile.ZipFile(reqifz_path, "w") as zip_file:
+ zip_file.writestr("first.reqif", REQIF_WITH_CONTENT)
+ zip_file.writestr("attachment.txt", "Some attachment.")
+ reqifz_bundle = ReqIFZParser.parse(str(reqifz_path))
+
+ recorder = ProgressRecorder()
+ output_bytes = ReqIFZUnparser.unparse(reqifz_bundle, progress=recorder)
+
+ assert recorder.calls == [
+ ("first.reqif", 1, 2),
+ ("attachment.txt", 2, 2),
+ ]
+ # The callback must not affect the unparse output.
+ assert output_bytes == ReqIFZUnparser.unparse(reqifz_bundle)