diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 99e9c54f..50d5a077 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -1,6 +1,10 @@ name: Run Tests -on: [pull_request, push, workflow_dispatch] +on: + push: + branches: [master] + pull_request: + workflow_dispatch: jobs: test: @@ -9,7 +13,7 @@ jobs: fail-fast: true matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - python-version: ["3.12", "3.13"] + python-version: ["3.12", "3.13", "3.14"] steps: - name: Checkout source @@ -27,5 +31,32 @@ jobs: pip install -r requirements.txt pip install -r test/requirements.txt + - name: Run tests + run: pytest + + test-legacy: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + os: ["ubuntu-latest", "windows-latest"] + python-version: ["3.10", "3.11"] + + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + + - name: Install + run: | + pip install -ve .[all] + pip install -r requirements.txt + pip install -r test/requirements.txt + - name: Run tests run: pytest \ No newline at end of file diff --git a/.gitignore b/.gitignore index 55335b83..7e91e28b 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,8 @@ MANIFEST bench/ pypi/ caterpillarapi.h +.python-version +.benchmarks/ # PyInstaller # Usually these files are written by a python script from a template diff --git a/CHANGELOG.md b/CHANGELOG.md index 26683f58..a497f4bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,59 @@ # Changelog +## [2.9.0] - Python 3.14 Compatibility and Generic Templates + +### Added + +- Python 3.14-compatible conditional field metadata: + `with If(condition) as when:`, `f[..., when]`, `Start(when)` / + `End(when)`, explicit `ElseIf(previous, condition)` chains, and + `Else(previous)` fallback branches. +- `Branch`, `When`, and `Otherwise` for conditional variants of one attribute, + including branch-local `f[...]` options such as byte order. +- Python generic templates based on normal `typing.TypeVar` and `Generic` + classes. Templates now support direct `Template[uint8]` specialization, + partial specialization, specialization caching, and runtime `__origin__` / + `__args__` metadata. +- `field_of()` for carrying generic template field metadata through `f[...]`, + including sequence length, offset, switch options, byte order, bit width, and + conditions. +- `Padded`, `PrePad`, and `PostPad` wrappers for explicit before/after padding, + dynamic padding lengths, repeated fill patterns, strict validation, and slash + syntax such as `uint8 / PrePad(1) / PostPad(2)`. +- A `pytest-benchmark` benchmark suite covering the `examples/comparison` workload. + +### Changes + +- `derive()` now handles generic templates, returns already materialized struct + classes unchanged when no type arguments are supplied, supports keyword + defaults in legacy templates, and evaluates deferred annotations while legacy + template variables are still available. +- Native C atoms (`Repeated`, `Conditional`, `AtOffset`, and `Switch`) now accept + Python struct classes and other `__struct__` containers, improving Python/C + interoperability and `c_Context` coverage. +- `PyStructFormattedField` now handles prefixed sequence unpacking and validates + empty sequence packing after processing fixed or prefixed lengths. +- `Lazy` now resolves objects that expose `__struct__`, allowing lazy factories + to return decorated model classes or struct containers. + +### Fixes + +- `Field` rejects negative sequence lengths at definition time and uses a faster + no-option unpack path while preserving default fallback and exception wrapping + behavior. +- `Compressed` copies compression and decompression keyword arguments before + resolving context lambdas, preventing cross-call mutation. +- Fixed-length `CString` unpacking now performs exact reads and rejects + truncated input. +- Padding validation now covers strict greedy fill patterns and rejects + unsupported prefixed padding lengths. +- C extension fixes include safer reference handling and error propagation, + length-info forwarding to `__pack_many__` / `__unpack_many__`, `__bits__` + lookup on Python atoms, repeated atom unpack detection, `AtOffset.keep_pos` + behavior, switch type aggregation, arch/endian rich comparison errors, and + writable offset/whence setters. + + ## [2.8.0] - Extended Syntax ### Added diff --git a/README.md b/README.md index cca6c8ba..2f760c81 100644 --- a/README.md +++ b/README.md @@ -124,10 +124,10 @@ assert data_le != data_be ``` > [!NOTE] -> Python 3.14 breaks `with` statements in class definitions since `__annotations__` are added at the end -> of a class definition. Therefore, `Digest` and conditional statements **ARE NOT SUPPORTED** using the `with` syntax in Python 3.14+. -> As of version `2.4.5` the `Digest` class has a counterpart (`DigestField`), which can be used to manually specify a digest without -> the need of a `ẁith` statement. +> Python 3.14 changes when class-body `__annotations__` are available. Digest context managers still need the explicit +> `DigestField` form on Python 3.14+, but conditional `with` blocks are supported through explicit metadata: +> `with If(condition) as when:` plus `f[..., when]` for one field or `Start(when)` / `End(when)` for a block. +> For conditional variants of one attribute, use `Branch(When(...), Otherwise(...))`. This library offers extensive functionality beyond basic struct definitions. For further details on its powerful features, explore the official [documentation](https://matrixeditor.github.io/caterpillar/), @@ -177,4 +177,4 @@ to these approaches. ## License -Distributed under the GNU General Public License (V3). See [License](LICENSE) for more information. \ No newline at end of file +Distributed under the GNU General Public License (V3). See [License](LICENSE) for more information. diff --git a/benchmark/test_comparison_example.py b/benchmark/test_comparison_example.py new file mode 100644 index 00000000..c540cfb9 --- /dev/null +++ b/benchmark/test_comparison_example.py @@ -0,0 +1,338 @@ +# Copyright (C) MatrixEditor 2023-2026 +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# pyright: basic +import pytest + +import caterpillar +from caterpillar.context import O_CONTEXT_FACTORY + +try: + from examples.comparison import comparison_1_caterpillar as caterpillar_default + + pytestmark = pytest.mark.benchmark + +except ImportError: + caterpillar_default = None + + +NATIVE_ONLY = pytest.mark.skipif( + not caterpillar.native_support(), reason="native extension unavailable" +) +HAVE_EXAMPLES = pytest.mark.skipif( + caterpillar_default is None, reason="examples unavailable" +) + +ROUNDS = 10 +ITEM_COUNT = 1000 + + +def _make_default_items(count: int): + return [ + caterpillar_default.Item( + i & 0xFF, + (i * 17) & 0xFFFFFF, + caterpillar_default.Flags(bool(i & 1), i & 0x07), + [i & 0xFF, (i + 1) & 0xFF, (i + 2) & 0xFF], + "...", + "...", + ) + for i in range(count) + ] + + +if caterpillar_default is not None: + DEFAULT_ITEMS = _make_default_items(ITEM_COUNT) + COMPARISON_RAW = caterpillar_default.pack(DEFAULT_ITEMS, caterpillar_default.Format) + + +def _bench(benchmark, fn, validate=None): + result = benchmark.pedantic(fn, rounds=ROUNDS, iterations=10) + if validate is not None: + validate(result) + return result + + +def _assert_caterpillar_items(items): + assert len(items) == ITEM_COUNT + assert items[0].name1 == "..." + assert items[-1].fixedarray1 == [ + (ITEM_COUNT - 1) & 0xFF, + ITEM_COUNT & 0xFF, + (ITEM_COUNT + 1) & 0xFF, + ] + + +def _assert_raw(data): + assert data == COMPARISON_RAW + + +def _assert_mapping_count(obj): + assert obj["count"] == ITEM_COUNT + + +def _assert_attr_count(obj): + assert obj.count == ITEM_COUNT + + +def _assert_bytes(data): + assert isinstance(data, bytes | bytearray) + + +def _assert_hachoir_count(fields): + assert fields[0].value == ITEM_COUNT + + +@HAVE_EXAMPLES +def test_bench_caterpillar_unpack(benchmark): + _bench( + benchmark, + lambda: caterpillar_default.unpack(caterpillar_default.Format, COMPARISON_RAW), + _assert_caterpillar_items, + ) + + +@HAVE_EXAMPLES +def test_bench_caterpillar_pack(benchmark): + _bench( + benchmark, + lambda: caterpillar_default.pack(DEFAULT_ITEMS, caterpillar_default.Format), + _assert_raw, + ) + + +@HAVE_EXAMPLES +@NATIVE_ONLY +def test_bench_caterpillar_c_context_default_unpack(benchmark): + from caterpillar.c import c_Context + + old_factory = O_CONTEXT_FACTORY.value + O_CONTEXT_FACTORY.value = c_Context + try: + _bench( + benchmark, + lambda: caterpillar_default.unpack( + caterpillar_default.Format, COMPARISON_RAW + ), + _assert_caterpillar_items, + ) + finally: + O_CONTEXT_FACTORY.value = old_factory + + +@HAVE_EXAMPLES +@NATIVE_ONLY +def test_bench_caterpillar_c_context_default_pack(benchmark): + from caterpillar.c import c_Context + + old_factory = O_CONTEXT_FACTORY.value + O_CONTEXT_FACTORY.value = c_Context + try: + _bench( + benchmark, + lambda: caterpillar_default.pack(DEFAULT_ITEMS, caterpillar_default.Format), + _assert_raw, + ) + finally: + O_CONTEXT_FACTORY.value = old_factory + + +@HAVE_EXAMPLES +@NATIVE_ONLY +def test_bench_caterpillar_c_classes_unpack(benchmark): + from examples.comparison import comparison_1_caterpillar_c as caterpillar_c + + _bench( + benchmark, + lambda: caterpillar_c.unpack(caterpillar_c.Format, COMPARISON_RAW), + _assert_caterpillar_items, + ) + + +@HAVE_EXAMPLES +@NATIVE_ONLY +def test_bench_caterpillar_c_classes_pack(benchmark): + from examples.comparison import comparison_1_caterpillar_c as caterpillar_c + + c_items = [ + caterpillar_c.Item( + i & 0xFF, + (i * 17) & 0xFFFFFF, + caterpillar_c.Flags(bool(i & 1), i & 0x07), + [i & 0xFF, (i + 1) & 0xFF, (i + 2) & 0xFF], + "...", + "...", + ) + for i in range(ITEM_COUNT) + ] + + _bench( + benchmark, + lambda: caterpillar_c.pack(c_items, caterpillar_c.Format), + _assert_raw, + ) + + +@HAVE_EXAMPLES +@NATIVE_ONLY +def test_bench_caterpillar_c_context_unpack(benchmark): + from caterpillar.c import c_Context + from examples.comparison import comparison_1_caterpillar_c as caterpillar_c + + old_factory = O_CONTEXT_FACTORY.value + O_CONTEXT_FACTORY.value = c_Context + try: + _bench( + benchmark, + lambda: caterpillar_c.unpack(caterpillar_c.Format, COMPARISON_RAW), + _assert_caterpillar_items, + ) + finally: + O_CONTEXT_FACTORY.value = old_factory + + +@HAVE_EXAMPLES +@NATIVE_ONLY +def test_bench_caterpillar_c_context_pack(benchmark): + from caterpillar.c import c_Context + from examples.comparison import comparison_1_caterpillar_c as caterpillar_c + + c_items = [ + caterpillar_c.Item( + i & 0xFF, + (i * 17) & 0xFFFFFF, + caterpillar_c.Flags(bool(i & 1), i & 0x07), + [i & 0xFF, (i + 1) & 0xFF, (i + 2) & 0xFF], + "...", + "...", + ) + for i in range(ITEM_COUNT) + ] + + old_factory = O_CONTEXT_FACTORY.value + O_CONTEXT_FACTORY.value = c_Context + try: + _bench( + benchmark, + lambda: caterpillar_c.pack(c_items, caterpillar_c.Format), + _assert_raw, + ) + finally: + O_CONTEXT_FACTORY.value = old_factory + + +@HAVE_EXAMPLES +def test_bench_construct_parse(benchmark): + construct_comparison = pytest.importorskip( + "examples.comparison.comparison_1_construct" + ) + + _bench( + benchmark, + lambda: construct_comparison.d.parse(COMPARISON_RAW), + _assert_mapping_count, + ) + + +@HAVE_EXAMPLES +def test_bench_construct_build(benchmark): + construct_comparison = pytest.importorskip( + "examples.comparison.comparison_1_construct" + ) + obj = construct_comparison.d.parse(COMPARISON_RAW) + + _bench( + benchmark, + lambda: construct_comparison.d.build(obj), + _assert_raw, + ) + + +@HAVE_EXAMPLES +def test_bench_construct_compiled_parse(benchmark): + construct_comparison = pytest.importorskip( + "examples.comparison.comparison_1_construct" + ) + + _bench( + benchmark, + lambda: construct_comparison.d_compiled.parse(COMPARISON_RAW), + _assert_mapping_count, + ) + + +@HAVE_EXAMPLES +def test_bench_construct_compiled_build(benchmark): + construct_comparison = pytest.importorskip( + "examples.comparison.comparison_1_construct" + ) + obj = construct_comparison.d.parse(COMPARISON_RAW) + + _bench( + benchmark, + lambda: construct_comparison.d_compiled.build(obj), + _assert_raw, + ) + + +@HAVE_EXAMPLES +def test_bench_kaitai_parse(benchmark): + kaitai_comparison = pytest.importorskip("examples.comparison.comparison_1_kaitai") + + _bench( + benchmark, + lambda: kaitai_comparison.Comparison1Kaitai.from_bytes(COMPARISON_RAW), + _assert_attr_count, + ) + + +@HAVE_EXAMPLES +def test_bench_hachoir_parse(benchmark): + hachoir_comparison = pytest.importorskip("examples.comparison.comparison_1_hachoir") + hachoir_stream = pytest.importorskip("hachoir.stream") + + _bench( + benchmark, + lambda: list( + hachoir_comparison.Format(hachoir_stream.StringInputStream(COMPARISON_RAW)) + ), + _assert_hachoir_count, + ) + + +@HAVE_EXAMPLES +def test_bench_mrcrowbar_parse(benchmark): + mrcrowbar_comparison = pytest.importorskip( + "examples.comparison.comparison_1_mrcrowbar" + ) + + _bench( + benchmark, + lambda: mrcrowbar_comparison.Format(COMPARISON_RAW), + _assert_attr_count, + ) + + +@HAVE_EXAMPLES +def test_bench_mrcrowbar_build(benchmark): + mrcrowbar_comparison = pytest.importorskip( + "examples.comparison.comparison_1_mrcrowbar" + ) + obj = mrcrowbar_comparison.Format(COMPARISON_RAW) + + _bench( + benchmark, + obj.export_data, + _assert_bytes, + ) diff --git a/docs/sphinx/source/development/changelog.rst b/docs/sphinx/source/development/changelog.rst index 1e2a0eb3..ee6a617c 100644 --- a/docs/sphinx/source/development/changelog.rst +++ b/docs/sphinx/source/development/changelog.rst @@ -6,6 +6,67 @@ Changelog *More entries will be added in the future.* +.. _changelog_2.9.0: + +[2.9.0] - Python 3.14 Compatibility and Generic Templates +========================================================= + +Added +----- + +- Python 3.14-compatible conditional field metadata: ``with If(condition) as + when:``, ``f[..., when]``, ``Start(when)`` / ``End(when)``, explicit + ``ElseIf(previous, condition)`` chains, and ``Else(previous)`` fallback + branches. +- ``Branch``, ``When``, and ``Otherwise`` for conditional variants of one + attribute, including branch-local ``f[...]`` options such as byte order. +- Python generic templates based on normal ``typing.TypeVar`` and ``Generic`` + classes. Templates now support direct ``Template[uint8]`` specialization, + partial specialization, specialization caching, and runtime ``__origin__`` / + ``__args__`` metadata. +- ``field_of()`` for carrying generic template field metadata through + ``f[...]``, including sequence length, offset, switch options, byte order, bit + width, and conditions. +- ``Padded``, ``PrePad``, and ``PostPad`` wrappers for explicit before/after + padding, dynamic padding lengths, repeated fill patterns, strict validation, + and slash syntax such as ``uint8 / PrePad(1) / PostPad(2)``. +- A ``pytest-benchmark`` benchmark suite covering the ``examples/comparison`` workload. + +Changes +------- + +- ``derive()`` now handles generic templates, returns already materialized + struct classes unchanged when no type arguments are supplied, supports keyword + defaults in legacy templates, and evaluates deferred annotations while legacy + template variables are still available. +- Native C atoms (``Repeated``, ``Conditional``, ``AtOffset``, and ``Switch``) + now accept Python struct classes and other ``__struct__`` containers, + improving Python/C interoperability and ``c_Context`` coverage. +- ``PyStructFormattedField`` now handles prefixed sequence unpacking and + validates empty sequence packing after processing fixed or prefixed lengths. +- ``Lazy`` now resolves objects that expose ``__struct__``, allowing lazy + factories to return decorated model classes or struct containers. + +Fixes +----- + +- ``Field`` rejects negative sequence lengths at definition time and uses a + faster no-option unpack path while preserving default fallback and exception + wrapping behavior. +- ``Compressed`` copies compression and decompression keyword arguments before + resolving context lambdas, preventing cross-call mutation. +- Fixed-length ``CString`` unpacking now performs exact reads and rejects + truncated input. +- Padding validation now covers strict greedy fill patterns and rejects + unsupported prefixed padding lengths. +- C extension fixes include safer reference handling and error propagation, + length-info forwarding to ``__pack_many__`` / ``__unpack_many__``, ``__bits__`` + lookup on Python atoms, repeated atom unpack detection, ``AtOffset.keep_pos`` + behavior, switch type aggregation, arch/endian rich comparison errors, and + writable offset/whence setters. + + + .. _changelog_2.8.0: [2.8.0] - Extended Syntax diff --git a/docs/sphinx/source/library/fields/common.rst b/docs/sphinx/source/library/fields/common.rst index fe00c6d9..02a826bf 100644 --- a/docs/sphinx/source/library/fields/common.rst +++ b/docs/sphinx/source/library/fields/common.rst @@ -278,6 +278,23 @@ Special Structs .. versionchanged:: 2.8.0 Added support for customizable padding objects. +.. autoclass:: caterpillar.fields.Padded + :members: + + Wraps another struct and adds explicit padding bytes before and/or after + it while returning the wrapped struct's value. + + .. versionadded:: 2.9.0 + +.. autofunction:: caterpillar.fields.PrePad + + .. versionadded:: 2.9.0 + +.. autofunction:: caterpillar.fields.PostPad + + .. versionadded:: 2.9.0 + + .. autodata:: caterpillar.fields.Pass See source code for details diff --git a/docs/sphinx/source/library/fields/field_model.rst b/docs/sphinx/source/library/fields/field_model.rst index 00c80914..c3bbc694 100644 --- a/docs/sphinx/source/library/fields/field_model.rst +++ b/docs/sphinx/source/library/fields/field_model.rst @@ -30,9 +30,32 @@ Chains and Conditionals :members: :special-members: +.. autoclass:: caterpillar.fields.ConditionalChain + :members: + :special-members: + .. autoclass:: caterpillar.fields.If :members: .. autoclass:: caterpillar.fields.ElseIf :members: +.. autodata:: caterpillar.fields.Else + +.. autoclass:: caterpillar.fields.Start + :members: + + .. versionadded:: 2.9.0 + +.. autoclass:: caterpillar.fields.End + :members: + +.. autoclass:: caterpillar.fields.Branch + :members: + :special-members: + +.. autoclass:: caterpillar.fields.When + :members: + +.. autoclass:: caterpillar.fields.Otherwise + :members: diff --git a/docs/sphinx/source/library/model/template.rst b/docs/sphinx/source/library/model/template.rst index 5d5f3704..408cbf55 100644 --- a/docs/sphinx/source/library/model/template.rst +++ b/docs/sphinx/source/library/model/template.rst @@ -12,6 +12,9 @@ Templates .. autofunction:: caterpillar.model.istemplate +.. autofunction:: caterpillar.model.field_of + + .. autofunction:: caterpillar.model.template diff --git a/docs/sphinx/source/reference/capi/objects/contextobj.rst b/docs/sphinx/source/reference/capi/objects/contextobj.rst index be6c8e86..607284a6 100644 --- a/docs/sphinx/source/reference/capi/objects/contextobj.rst +++ b/docs/sphinx/source/reference/capi/objects/contextobj.rst @@ -67,4 +67,3 @@ Context Objects .. c:function:: int CpContext_GenericSetAttrString(PyObject *context, const char *path, PyObject *value) String variant of :c:func:`CpContext_GenericSetAttr`. -4 \ No newline at end of file diff --git a/docs/sphinx/source/reference/datamodel/templates.rst b/docs/sphinx/source/reference/datamodel/templates.rst index 8edfe0ca..8be65d8a 100644 --- a/docs/sphinx/source/reference/datamodel/templates.rst +++ b/docs/sphinx/source/reference/datamodel/templates.rst @@ -3,75 +3,195 @@ Templates ========= -A specialized form of structs are *templates*, which are basically generic Python classes. Think of them -as blueprints for your final classes/structs that contain placeholders for actual types. As in C++, a -template needs type arguments, in this case we will name them :class:`~caterpillar.model.TemplateTypeVar`. +.. versionchanged:: 2.9.0 + Added support for Python's builtin :class:`TypeVar`` -Actually, there are two different types of type variables: +Templates are generic model classes that become concrete struct classes after +specialization. A template class stores template metadata on ``__template__`` +and does not store ``__struct__`` until it is specialized. -* Required: - These variables are **required** when creating a new struct based on the template and they - can be used as positional arguments within the type derivation. +Caterpillar supports two template systems: -* Positional: - These arguments are usable only as keyword arguments and are may be optional if a default value - is supplied. +* Python generic templates, the preferred API for new code. +* Legacy ``TemplateTypeVar`` templates, kept for compatibility. -These template type variables can be created using simple variable definitions: +Python Generic Templates +------------------------ ->>> A = TemplateTypeVar("A") +Generic templates use normal Python ``TypeVar`` objects and ``Generic`` bases. -.. important:: - A template class is **not** a struct definition. It specifies a blueprint for the final class. +.. code-block:: python + + from typing import Generic, TypeVar + + from caterpillar.py import template, uint8 + + T = TypeVar("T") + + + @template + class Box(Generic[T]): + value: T + + + ByteBox = Box[uint8] -A template class is defined like a struct, union or bitfield class, but without being a -dataclass nor storing a struct instance. +``Box`` is a template. ``Box[uint8]`` is a concrete struct class. Caterpillar +installs ``__class_getitem__`` on the template class and materializes the +specialization when it is subscripted. + +Specialization performs two different substitutions: + +* In normal Python type positions, Caterpillar field objects are replaced with + their Python value type using ``typeof()``. +* In field metadata positions, Caterpillar keeps the actual field object and + builds a ``Field`` from it. + +For example: .. code-block:: python - >>> @template(A, "B") - ... class FormatTemplate: - ... foo: A - ... bar: B - ... baz: uint32 - ... + @template + class Box(Generic[T]): + value: T + + + ByteBox = Box[uint8] + +``value: T`` is converted to a ``Field(uint8)`` before the generated class is +passed to :class:`~caterpillar.model.Struct`. + -The defined class then can be used to create new classes based on the provided class -structure. For instance, +**Layout Metadata With** ``f[]`` + +``f[]`` is Caterpillar's public spelling for ``typing.Annotated[]``. Use it +when the Python value type and the binary layout metadata must both be present. .. code-block:: python - >>> Format = derive(FormatTemplate, A=uint32, B=uint8) - >>> Format - + from typing import Generic, TypeVar + + from caterpillar.py import f, field_of, template, uint8 + + T = TypeVar("T") + -will return an anonymous class (in this case). Normally, *caterpillar* tries to infer the -variable name from the current module (if :code:`name=...`). In summary, every time -:meth:`~caterpillar.model.derive` is called, a new class will be created if not already -defined. + @template + class Vector(Generic[T]): + values: f[list[T], field_of(T)[2]] -The current implementation materializes the class annotations with -``inspect.get_annotations()`` while the temporary template variables are still -available. This keeps templates compatible with Python versions that defer -annotation evaluation. Template information is then stored on the class using a -special class attribute: :attr:`~class.__template__`. -To support sub-classes of templates, derive a partial template: + ByteVector = Vector[uint8] + +In this example, the Python-facing type becomes ``list[int]`` while the binary +layout metadata becomes a two-element ``Field(uint8)``. + +``field_of(T)`` supports the same layout operators as +:class:`~caterpillar.model.TemplateTypeVar`: sequence length, offset, switch +options, byte order, bit width, and condition. Generic specializations are cached +on the template origin. Repeating the same specialization returns the same class: .. code-block:: python - >>> Format32 = derive(FormatTemplate, A=uint32, partial=True) + assert Box[uint8] is Box[uint8] + +Generated classes store Caterpillar-owned metadata: + +``__origin__`` + The template class that produced the specialization. + +``__args__`` + The concrete specialization arguments. + +Because ``Box[uint8]`` returns a real class at runtime, it is not a +``typing`` generic alias after materialization. Use the metadata above instead +of ``typing.get_origin()`` and ``typing.get_args()`` for runtime inspection. + +Partial Generic Templates +------------------------- + +If a specialization still contains unresolved type variables, Caterpillar keeps +the result as a template. + +.. code-block:: python + + from typing import Generic, TypeVar + + from caterpillar.py import template, uint8, uint16 + + T = TypeVar("T") + U = TypeVar("U") + + + @template + class Pair(Generic[T, U]): + left: T + right: U + + + BytePair = Pair[uint8, U] + ByteWordPair = BytePair[uint16] + +``BytePair`` is a template. ``ByteWordPair`` is a concrete struct class. + +Legacy Template Variables +------------------------- + +Legacy templates use :class:`~caterpillar.model.TemplateTypeVar` or string +names in the decorator. + +.. code-block:: python + + from caterpillar.py import TemplateTypeVar, derive, template, uint8, uint16 + + A = TemplateTypeVar("A") + + + @template(A, "B") + class FormatTemplate: + foo: A + bar: B + + + Format = derive(FormatTemplate, uint8, uint16) + +Legacy templates classify parameters as required or keyword-only defaults: + +* Required parameters are passed positionally or by keyword to + :func:`~caterpillar.model.derive`. +* Keyword defaults are declared in ``@template(T=uint8)`` and may be omitted + from ``derive()``. + +The legacy decorator temporarily injects missing template names into the caller +module while annotations are evaluated. This keeps legacy templates compatible +with deferred annotation evaluation. + +``derive()`` +------------ + +``derive()`` remains available for both template systems. + +For legacy templates, ``derive()`` is the primary specialization API. For +generic templates, direct subscript syntax is preferred, but ``derive()`` can be +used when a name or union option must be supplied explicitly. + +.. code-block:: python + + NamedByteBox = derive(Box, uint8, name="NamedByteBox") + +Passing an already materialized struct class to ``derive()`` without additional +arguments returns that class unchanged. -Again, the resulting class is **not** a struct, but another template class. -Provided replacements are stored in the new template metadata, while missing -required variables must still be supplied by a later non-partial -:func:`~caterpillar.model.derive` call. +Type Checking +------------- -The ``name`` parameter controls the generated class name. Passing ``name=...`` -asks Caterpillar to infer the assignment target when possible; otherwise an -anonymous deterministic name is generated from the replacements. +Static type checkers see generic templates as ordinary Python generic classes. +At runtime, Caterpillar replaces template arguments with concrete binary +layouts. If a project needs precise static typing for field atoms such as +``uint8``, expose typing-only aliases to their Python value types while keeping +the runtime field objects unchanged. .. admonition:: Developer's note - By now, a template won't copy existing field documentation comments. Therefore, you - can't display inherited members using sphinx. \ No newline at end of file + Template specialization is performed once when a concrete class is created. + Pack and unpack operations use the normal ``Struct`` and ``Field`` paths. diff --git a/docs/sphinx/source/tutorial/advanced/conditional.rst b/docs/sphinx/source/tutorial/advanced/conditional.rst index 9257bc5c..80f97521 100644 --- a/docs/sphinx/source/tutorial/advanced/conditional.rst +++ b/docs/sphinx/source/tutorial/advanced/conditional.rst @@ -3,77 +3,165 @@ Conditional Fields ================== -.. warning:: - This feature is not supported in Python 3.14+. +*Conditional fields* allow a struct layout to include or skip fields based on +values that are already available in the parse context. They are useful for +versioned formats, tagged unions, optional trailer data, and protocol flags. -*Conditional fields* allow you to define fields in a struct that are included -or excluded based on certain conditions. This feature is especially useful when -working with versioned formats or optional fields that depend on runtime -conditions. You can easily achieve this using context-based lambdas, which are -built into the library. +Python 3.14 changed when class annotations become visible during class-body +execution. Because of that, Caterpillar supports two conditional styles: -How it works ------------- +- Python <= 3.13 can use the older implicit ``with`` syntax. +- Python >= 3.14 should use explicit conditional metadata. The + ``f[..., when]`` and ``Start(when)`` / ``End(when)`` forms are accepted by + static type checkers because the condition markers live in ``Annotated`` + metadata. -By using the `with` keyword in combination with conditional expressions, you can -bind certain fields to a specific condition. This allows you to include or exclude -fields dynamically, depending on the value of other fields or context. +For one field, the compact explicit form is to bind the condition with +``with If(condition) as when:`` and add ``when`` to the field metadata. -Here's an example demonstrating how to use conditional fields for versioned structs: +.. code-block:: python + :caption: Compact conditional field -.. tab-set:: - :sync-group: syntax + @struct + class Packet: + flag: f[int, uint8] - .. tab-item:: Default Syntax - :sync: default + with If(this.flag == 1) as when: + value: f[int, uint8, when] - .. code-block:: python - :caption: Conditional fields (e.g. for versioned structs) + trailer: f[int, uint8] - @struct - class Format: - version: uint32 - # all following fields will be bound to the condition - with this.version == 1: - header: uint8 +When ``flag`` is not ``1``, ``value`` consumes no bytes and unpacks as ``None``. +When packing, disabled fields write no bytes. - .. tab-item:: Extended Syntax (>=2.8.0) - :sync: extended +``value: when[f[int, uint8]]`` spelling is also supported, but some +static type checkers reject it because ``when`` is a runtime value. Prefer +``f[..., when]`` for new code. - .. code-block:: python - :caption: Conditional fields (e.g. for versioned structs) +Python 3.14+ Changes +-------------------- - @struct - class Format: - version: uint32_t - # all following fields will be bound to the condition - with this.version == 1: - header: uint8_t +Python3.14+ support introduces inline markers make a complete ``with`` block +conditional without adding extra class fields. Add ``Start(when)`` to the first +real field and ``End(when)`` to the last real field. + +.. code-block:: python + :caption: Type-checker-friendly inline marker block + + @struct + class Packet: + flag: f[int, uint8] + + with If(this.flag == 1) as when: + first: f[int, uint8, Start(when)] + second: f[int, uint16] + third: f[int, uint8, End(when)] + + trailer: f[int, uint8] + +When ``flag`` is not ``1``, all three fields consume no bytes and unpack as +``None``. The ``when`` alias is removed from the final struct class. + +Invisible marker-field spelling is also available: + +.. code-block:: python + :caption: Compatibility marker fields + + @struct + class Packet: + flag: f[int, uint8] -Key Concepts ------------- + with If(this.flag == 1) as when: + _: f[None, when] = Invisible() + value: f[int, uint8] + _end: f[None, End(when)] = Invisible() -1. **`with` and Conditionals**: - The :code:`with` keyword is used to define a block of fields that should only be - included if the condition evaluates to :code:`True`. In the example above, the - fields inside :code:`with this.version == 1` are included only when the :code:`version` - field has a value of :code:`1`. + trailer: f[int, uint8] -2. **`ElseIf` for Multiple Conditions**: - For multiple conditions, use :code:`ElseIf` rather than :code:`Else`. The :code:`ElseIf` - construct ensures that the next condition is checked only if the previous - one was false. This is safer and more predictable than using a generic - :code:`Else` clause, which could introduce unintended side effects by executing - under unanticipated conditions. +Marker blocks follow these rules: +- Every ``Start(when)`` block must end with ``End(when)``. +- ``End`` must close the currently active marker. +- Use a unique alias for each Python 3.14 marker block. +- A marker block must contain at least one real field. +- Do not define the same field name in multiple marker branches. Use + :class:`~caterpillar.fields.Branch` for same-field conditional variants. -Example: Versioned Struct -^^^^^^^^^^^^^^^^^^^^^^^^^ +Multiple Branches +----------------- -Conditional fields are particularly useful when dealing with versioned structs, -where the structure of the data may change based on the version number or other -factors. For example: +Use ``ElseIf(previous, condition)`` and ``Else(previous)`` to build explicit +branch chains on Python 3.14+. Each branch receives the marker returned by the +previous branch. +.. code-block:: python + :caption: If / else-if / else marker chain + + @struct + class Packet: + tag: f[int, uint8] + + with If(this.tag == 1) as first: + small: f[int, uint8, first] + + with ElseIf(first, this.tag == 2) as second: + medium: f[int, uint16, second] + + with Else(second) as fallback: + raw: f[int, uint8, fallback] + + trailer: f[int, uint8] + +``ElseIf(first, condition)`` is active only when all previous branch conditions +are false and its own condition is true. ``Else(second)`` is active only when all +previous branch conditions are false. ``ElseIf`` cannot be added after ``Else``. + +Same-Field Branches +------------------- + +When multiple conditions should populate the same attribute, use ``Branch`` with +``When`` and optional ``Otherwise`` arms. This is the right spelling for tagged +fields whose binary type changes with a discriminator. + +.. code-block:: python + :caption: Same attribute, different field types + + @struct + class Packet: + tag: f[int, uint8] + value: f[ + int, + Branch( + When(this.tag == 1, uint8), + When(this.tag == 2, uint16), + Otherwise(uint8), + ), + ] + trailer: f[int, uint8] + +Branch arms can carry normal field options by using ``f[...]`` inside the arm: + +.. code-block:: python + :caption: Branch arm with local options + + @struct(order=LittleEndian) + class Packet: + tag: f[int, uint8] + value: f[ + int, + Branch( + When(this.tag == 1, f[int, uint16, BigEndian]), + Otherwise(uint16), + ), + ] + +In this example, ``value`` is big-endian only when ``tag == 1``. The fallback arm +uses the struct's surrounding byte order. + +Legacy Syntax on Python <= 3.13 +------------------------------- + +Before Python 3.14, Caterpillar can still use the implicit class-body syntax. .. tab-set:: :sync-group: syntax @@ -82,52 +170,34 @@ factors. For example: :sync: default .. code-block:: python - :caption: Conditional fields (e.g. for versioned structs) @struct class Format: version: uint32 - # all following fields will be bound to the condition + with this.version == 1: length: uint8 - extra: uint8 data: Bytes(this.length) - # Use else-if over 'Else' alone + with ElseIf(this.version == 2): name: CString(16) - data: Prefixed(uint8) .. tab-item:: Extended Syntax (>=2.8.0) :sync: extended .. code-block:: python - :caption: Conditional fields (e.g. for versioned structs) @struct class Format: version: uint32_t - # all following fields will be bound to the condition + with this.version == 1: length: uint8_t - extra: uint8_t data: f[bytes, Bytes(this.length)] - # Use else-if over 'Else' alone + with ElseIf(this.version == 2): name: f[str, CString(16)] - data: f[bytes, Prefixed(uint8)] - -Best Practices ---------------- - -- **Avoid Using `Else`**: - It is **strongly recommended** to **avoid** using :code:`Else` for conditional field - inclusion, as it can introduce unintended behavior if not properly managed. - Instead, always use :code:`ElseIf` with an inverted condition to ensure more - predictable and controlled struct parsing. - - -.. note:: - When using conditional fields, it's essential to remember that the struct's - layout can change dynamically depending on the conditions. This flexibility - makes it possible to define complex, version-dependent data structures. \ No newline at end of file +On Python 3.14+, implicit ``with If(condition):``, ``with ElseIf(condition):``, +and ``with Else:`` blocks raise a clear exception. Use the explicit marker +syntax shown above. diff --git a/docs/sphinx/source/tutorial/advanced/templates.rst b/docs/sphinx/source/tutorial/advanced/templates.rst index ef35d686..4c2a7fdd 100644 --- a/docs/sphinx/source/tutorial/advanced/templates.rst +++ b/docs/sphinx/source/tutorial/advanced/templates.rst @@ -3,67 +3,185 @@ Templates ========= -Yes, you read that correctly: *Caterpillar* supports class templates, similar -to the concept in C++. Templates allow you to create generic structures that -can be tailored with specific types when they are derived. +.. versionchanged:: 2.9.0 + Added support for Python's :class:`Generic` and :class:`TypeVar` types. -If you'd like more details about the design decisions and limitations regarding -templates, refer to the :ref:`ref-templates` section in the data model description. +Templates are blueprints for binary structures. They let you describe a layout +once and specialize it with different Caterpillar field types later. -Defining a Template -------------------- +The preferred form uses Python generics. Legacy ``TemplateTypeVar`` templates +are still supported and are covered at the end of this page. -You can define templates using a special syntax. First, you create **template type variables**, -and then you use them within your class definition. These templates can be instantiated -later with specific types or values. +If you'd like the implementation details, see :ref:`ref-templates`. + +Generic Templates +----------------- + +Define a template as a normal generic Python class and decorate it with +``@template``. A plain type variable annotation is enough. .. code-block:: python - :caption: A simple template definition + :caption: A generic scalar template - A = TemplateTypeVar("A") + from typing import Generic, TypeVar - @template(A, "B") # <-- either strings or global variables - class FormatTemplate: - foo: A[uint8::] # <-- prefixed generic array - bar: B # <-- this won't throw an exception, because - # 'B' is created temporarily. + from caterpillar.py import pack, template, uint8, uint16 + + T = TypeVar("T") + + + @template + class Box(Generic[T]): + value: T + + + ByteBox = Box[uint8] + WordBox = Box[uint16] + + assert pack(ByteBox(7)) == b"\x07" + assert pack(WordBox(0x0203)) == b"\x03\x02" + +``Box`` is not a struct by itself. ``Box[uint8]`` materializes a concrete +struct class and caches it, so repeated uses of the same specialization return +the same class. + +Multiple Type Variables +----------------------- + +Templates can use more than one type variable. The order in ``Generic[...]`` +defines the positional specialization order. + +.. code-block:: python + :caption: Multiple template parameters + + from typing import Generic, TypeVar + + from caterpillar.py import BigEndian, pack, template, uint8, uint16 + + T = TypeVar("T") + U = TypeVar("U") + + + @template + class Pair(Generic[T, U]): + left: T + right: U + + + ByteWordPair = Pair[uint8, uint16] + obj = ByteWordPair(1, 0x0203) + + assert pack(obj, order=BigEndian) == b"\x01\x02\x03" + +Fields With Layout Metadata +--------------------------- + +Use direct ``T`` annotations for simple scalar fields. When a field needs +Caterpillar-specific layout metadata, keep the Python type in the first +position of ``f[]`` and put the template field marker in the metadata position. + +.. code-block:: python + :caption: Repeated template field + + from typing import Generic, TypeVar + + from caterpillar.py import f, field_of, pack, template, uint8 + + T = TypeVar("T") -- :code:`A` is a **template type variable** of an unknown generic type -- The class :code:`FormatTemplate` uses :code:`A` and:code: `B` as type parameters, but these types will be defined when the template is derived into a specific class. -- :code:`A[uint8::]` creates a **generic array** prefixed with the type :code:`uint8`. -- :code:`B` is used as a type in the field :code:`bar`. This type will be specified when the template is instantiated, and the library ensures no exceptions are thrown because :code:`B` is set dynamically during class definition. -Deriving a Template -------------------- + @template + class Vector(Generic[T]): + values: f[list[T], field_of(T)[2]] -Once you have defined a template, you can create specific classes (known as -**derivations**) by providing the template with concrete types. This allows -you to reuse the logic from the template with different type combinations, -e.g. creating specialized structs. + + ByteVector = Vector[uint8] + + assert pack(ByteVector([3, 4])) == b"\x03\x04" + +``field_of(T)`` supports the same layout operators as normal fields, including +sequence length, offset, switch mappings, byte order, bit width, and condition +markers. For example, ``field_of(T)[2]`` becomes a two-element field after +specialization. + +Partial Templates +----------------- + +A specialization that still contains unresolved type variables remains a +template. This is useful for fixing only part of a layout. + +.. code-block:: python + :caption: Partial generic specialization + + from typing import Generic, TypeVar + + from caterpillar.py import BigEndian, pack, template, uint8, uint16 + + T = TypeVar("T") + U = TypeVar("U") + + + @template + class Pair(Generic[T, U]): + left: T + right: U + + + BytePair = Pair[uint8, U] + ByteWordPair = BytePair[uint16] + + assert pack(ByteWordPair(1, 0x0203), order=BigEndian) == b"\x01\x02\x03" + +Subclassing Specializations +--------------------------- + +Template specializations are normal struct classes and can be used as bases for +other structs. .. code-block:: python - :caption: Creating template derivations + :caption: Extending a specialization + + from typing import Generic, TypeVar + + from caterpillar.py import BigEndian, f, pack, struct, template, uint8, uint16 + + T = TypeVar("T") + + + @template + class Box(Generic[T]): + value: T + @struct - class Format32(derive(FormatTemplate, A=uint32, B=uint32)): # <- Derived Struct - baz: uint32 + class Packet(Box[uint8]): + tail: f[int, uint16] -*Sub-templates* or partial templates allow you to create smaller, more focused templates based on an -existing template. When you derive a sub-template, only some of the template -parameters are provided initially, and others are deferred. + + assert pack(Packet(1, 0x0203), order=BigEndian) == b"\x01\x02\x03" + +Legacy Templates +---------------- + +The older ``TemplateTypeVar`` style is still supported. Use it when maintaining +existing code that already depends on ``derive()``. .. code-block:: python - :caption: Creating a sub-template + :caption: Legacy template definition - # template sub-classes are allowed as well - FormatSubTemplate = derive(FormatTemplate, A=uint8, partial=True) # <- Derived Template + from caterpillar.py import TemplateTypeVar, derive, template, uint8, uint16 -.. note:: + A = TemplateTypeVar("A") + + + @template(A, "B") + class FormatTemplate: + foo: A + bar: B - While you can pass **keyword arguments** to define template parameters, you can also use - **positional arguments** if the original template decorator defines them in that way. -.. warning:: + Format = derive(FormatTemplate, uint8, uint16) - Currently, there are some **limitations** with the template type variables, and **extended support** - for this feature will be part of future enhancements in this project. +``derive()`` remains available as an explicit escape hatch for generic +templates as well, but direct subscript syntax is the preferred form for new +code. diff --git a/examples/comparison/comparison_1_caterpillar.py b/examples/comparison/comparison_1_caterpillar.py index 5146729f..abc7d210 100644 --- a/examples/comparison/comparison_1_caterpillar.py +++ b/examples/comparison/comparison_1_caterpillar.py @@ -36,7 +36,7 @@ class Item: # # Time goes down from 0.0119 to 0.0099 for unpacking # and from 0.0094 to 0.0082 for packing - name2: f[bytes, Prefixed(uint8, encoding="utf-8")] + name2: f[str, Prefixed(uint8, encoding="utf-8")] if typing.TYPE_CHECKING: diff --git a/examples/comparison/comparison_1_caterpillar_c.py b/examples/comparison/comparison_1_caterpillar_c.py new file mode 100644 index 00000000..1fac1bc9 --- /dev/null +++ b/examples/comparison/comparison_1_caterpillar_c.py @@ -0,0 +1,63 @@ +import typing + +import caterpillar +from caterpillar.context import O_CONTEXT_FACTORY +from caterpillar.fields import Prefixed, uint8, uint32 +from caterpillar.py import Field, f +from caterpillar.shortcuts import bitfield, pack, struct, unpack +from caterpillar.types import cstr_t, int1_t, int3_t, uint24_t, uint8_t + +if not caterpillar.native_support(): + raise RuntimeError("The native Caterpillar extension is required for this example") + +from caterpillar.c import LITTLE_ENDIAN, Repeated, c_Context + + +@bitfield(order=LITTLE_ENDIAN) +class Flags: + bool1: int1_t + num4: int3_t + # padding is generated automatically + + +@struct(order=LITTLE_ENDIAN) +class Item: + num1: uint8_t + num2: uint24_t + flags: Flags + fixedarray1: f[list[int], uint8[3]] + name1: cstr_t + name2: f[str, Prefixed(uint8, encoding="utf-8")] + + if typing.TYPE_CHECKING: + + def __class_getitem__(cls, length) -> Field: ... + + +Format = Repeated(Item, slice(LITTLE_ENDIAN + uint32, None, None)) + + +if __name__ == "__main__": + import sys + import timeit + + try: + from rich import print + except ImportError: + pass + + with open(sys.argv[1], "rb") as fp: + data = fp.read() + + old_factory = O_CONTEXT_FACTORY.value + O_CONTEXT_FACTORY.value = c_Context + try: + obj = unpack(Format, data) + time = timeit.timeit(lambda: unpack(Format, data), number=1000) / 1000 + print("[bold]Timeit measurements:[/]") + print(f"[bold]unpack[/] {time:.10f} sec/call") + + ptime = timeit.timeit(lambda: pack(obj, Format), number=1000) / 1000 + print(f"[bold]pack[/] {ptime:.10f} sec/call") + finally: + O_CONTEXT_FACTORY.value = old_factory diff --git a/examples/template_example.py b/examples/template_example.py index 697cdb7e..8ea97e6e 100644 --- a/examples/template_example.py +++ b/examples/template_example.py @@ -1,17 +1,19 @@ -# type: ignore +# dtype: ignore +from typing import TypeVar, Generic + from caterpillar.py import ( + f, + field_of, struct, set_struct_flags, S_REPLACE_TYPES, uint8, - uint16, - TemplateTypeVar, template, derive, pack, this, ) -from caterpillar.types import uint8_t +from caterpillar.types import uint16_t, uint8_t set_struct_flags(S_REPLACE_TYPES) @@ -24,15 +26,18 @@ class BaseFormat: f1: uint8_t -A = TemplateTypeVar("A") -B = TemplateTypeVar("B") +A = TypeVar("A") +B = TypeVar("B") -@template(A, B) -class FormatTemplate(BaseFormat): +@template +class FormatTemplate(Generic[A, B], BaseFormat): """Template class doc-comment""" - f2: A[this.f1] + # Use the field_of method to apply special operators on a type var + # --> these will be applied to the field later on + + f2: f[list[A], field_of(A)[this.f1]] """Template field doc-comment""" #: inline template field comment @@ -40,17 +45,19 @@ class FormatTemplate(BaseFormat): #: anonymous generated partial template -Format8 = derive(FormatTemplate, uint8, partial=True) +# Format8 = derive(FormatTemplate, uint8, partial=True) +# or direct approach +Format8 = FormatTemplate[uint8_t, B] @struct -class Format(derive(Format8, B=uint8)): +class Format(Format8[uint8_t]): #: inline comment f4: uint8_t -#: inline data comment -Format16 = derive(FormatTemplate, uint16, uint16, name=...) +# Direct specialization via [] is also possible +Format16 = FormatTemplate[uint16_t, uint16_t] if __name__ == "__main__": # Format(f1: int, f2: List, f3: int, f4: int) diff --git a/pypi/README.md b/pypi/README.md index 6e97a3b3..e2dfab7a 100644 --- a/pypi/README.md +++ b/pypi/README.md @@ -124,10 +124,10 @@ assert data_le != data_be ``` > [!NOTE] -> Python 3.14 breaks `with` statements in class definitions since `__annotations__` are added at the end -> of a class definition. Therefore, `Digest` and conditional statements **ARE NOT SUPPORTED** using the `with` syntax in Python 3.14+. -> As of version `2.4.5` the `Digest` class has a counterpart (`DigestField`), which can be used to manually specify a digest without -> the need of a `ẁith` statement. +> Python 3.14 changes when class-body `__annotations__` are available. Digest context managers still need the explicit +> `DigestField` form on Python 3.14+, but conditional `with` blocks are supported through explicit metadata: +> `with If(condition) as when:` plus `f[..., when]` for one field or `Start(when)` / `End(when)` for a block. +> For conditional variants of one attribute, use `Branch(When(...), Otherwise(...))`. This library offers extensive functionality beyond basic struct handling. For further details on its powerful features, explore the official [documentation](https://matrixeditor.github.io/caterpillar/), diff --git a/pypi/pyproject.toml b/pypi/pyproject.toml index ec9cacf3..3f50269b 100644 --- a/pypi/pyproject.toml +++ b/pypi/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "caterpillar-py" -version = "2.8.2" +version = "2.9.0" requires-python = ">=3.10" description="Library to pack and unpack structurized binary data." authors = [{ name = "MatrixEditor" }] diff --git a/pyproject.toml b/pyproject.toml index bb2ad7e2..b0af3671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ cmake.source-dir = "." [project] name = "caterpillar" -version = "2.8.2" +version = "2.9.0" requires-python = ">=3.10" description = "Library to pack and unpack structurized binary data." authors = [{ name = "MatrixEditor" }] @@ -25,6 +25,7 @@ classifiers = [ 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', ] # To ensure compatibility dependencies = ["typing-extensions"] @@ -38,3 +39,25 @@ dependencies = ["typing-extensions"] lzo = ["lzallright"] crypt = ["cryptography"] all = ["lzallright", "cryptography"] + +[dependency-groups] +bench = [ + "construct>=2.10.70", + "hachoir>=3.3.0", + "kaitaistruct>=0.11", + "mrcrowbar>=0.9.0", + "pytest>=9.1.1", + "pytest-benchmark>=5.2.3", +] +dev = [ + "black>=26.5.1", + "pytest>=9.1.1", + "tox>=4.56.1", +] +docs = [ + "breathe>=4.36.0", + "pydata-sphinx-theme>=0.19.0", + "sphinx>=8.1.3", + "sphinx-copybutton>=0.5.2", + "sphinx-design>=0.6.1", +] diff --git a/src/caterpillar/__init__.py b/src/caterpillar/__init__.py index a593b64d..7a5c4ae1 100644 --- a/src/caterpillar/__init__.py +++ b/src/caterpillar/__init__.py @@ -14,7 +14,7 @@ # along with this program. If not, see . import warnings -__version__ = "2.8.2" +__version__ = "2.9.0" __release__ = "2.8" __author__ = "MatrixEditor" diff --git a/src/caterpillar/abc.py b/src/caterpillar/abc.py index 7143f3cd..3ff0cba9 100755 --- a/src/caterpillar/abc.py +++ b/src/caterpillar/abc.py @@ -16,13 +16,7 @@ from collections.abc import Iterable, Collection from io import IOBase from types import EllipsisType, NoneType -from typing import ( - Any, - Callable, - Protocol, - runtime_checkable, - TYPE_CHECKING -) +from typing import Any, Callable, Protocol, runtime_checkable, TYPE_CHECKING from typing_extensions import Buffer, Final, Literal, TypeVar, overload, override @@ -574,6 +568,9 @@ def __call__(self, **kwargs: Any) -> _ContextLike: ... +_AnnotationT = str | bytes | type | _ActionLike | _StructLike | Any + + __all__ = [ "_ContextLike", "_ContextLambda", @@ -595,4 +592,5 @@ def __call__(self, **kwargs: Any) -> _ContextLike: "_ActionLike", "_SwitchOptionsT", "_LengthT", + "_AnnotationT", ] diff --git a/src/caterpillar/context.py b/src/caterpillar/context.py index 1ae32ae6..2fc186b5 100755 --- a/src/caterpillar/context.py +++ b/src/caterpillar/context.py @@ -21,7 +21,16 @@ import warnings from typing import Annotated, Callable, Any, Generic, Protocol, get_args, get_origin -from typing_extensions import Buffer, Final, Literal, Self, Sized, overload, override, TypeVar +from typing_extensions import ( + Buffer, + Final, + Literal, + Self, + Sized, + overload, + override, + TypeVar, +) from types import FrameType, TracebackType from dataclasses import dataclass @@ -38,6 +47,7 @@ _EndianLike, _ArchLike, ) +from caterpillar.shared import iscond if typing.TYPE_CHECKING: from caterpillar.fields._base import Field @@ -201,6 +211,7 @@ def __getitem__(self, key: Literal["_is_seq"], /) -> bool: ... def __getitem__(self, key: Literal["_pos"], /) -> int: ... @overload def __getitem__(self, key: str, /) -> Any: ... + __getitem__ = dict.__getitem__ @@ -403,15 +414,15 @@ class Format: :type condition: Union[_ContextLambda, bool] """ - __slots__: tuple[str, str, str, str] = ( + __slots__: tuple[str, ...] = ( "func", "annotations", "namelist", "depth", ) - def __init__(self, condition: _ContextLambda[bool], depth: int = 2): - self.func: _ContextLambda[bool] = condition + def __init__(self, condition: _ContextLambda[bool] | bool, depth: int = 2): + self.func: _ContextLambda[bool] | bool = condition self.annotations: dict[str, Any] = {} self.namelist: list[str] = list() self.depth: int = depth @@ -453,11 +464,15 @@ def __exit__( # modify newly created fields field: Field | Any = self.annotations[name] is_annotated = get_origin(field) is Annotated - annotated_type = extra_options = None + annotated_type = None + extra_options = () if is_annotated: # annotated_type = field.__origin__ # field, *extra_options = field.__metadata__ - annotated_type, field, *extra_options = get_args(field) + args = get_args(field) + if any(iscond(metadata) for metadata in args[1:]): + continue + annotated_type, field, *extra_options = args if not isinstance(field, Field): # create a field (other attributes will be modified later) @@ -484,7 +499,7 @@ def __exit__( # rebuild the annotated field self.annotations[name] = ( # Python 3.10 does not allow *extra_options - Annotated[annotated_type, field, extra_options] + Annotated[(annotated_type, field, *extra_options)] if is_annotated else field ) diff --git a/src/caterpillar/fields/__init__.py b/src/caterpillar/fields/__init__.py index e000dba1..e4728042 100644 --- a/src/caterpillar/fields/__init__.py +++ b/src/caterpillar/fields/__init__.py @@ -58,6 +58,9 @@ align, Lazy, ENUM_STRICT, + Padded, + PostPad, + PrePad, ) from .varint import VarInt, VARINT_LSB, vint from .compression import ( @@ -82,7 +85,17 @@ RelativePointer, uintptr_fn, ) -from .conditional import ConditionalChain, If, Else, ElseIf +from .conditional import ( + ConditionalChain, + If, + Else, + ElseIf, + When, + Branch, + End, + Start, + Otherwise, +) from .hook import IOHook from .digest import ( Digest, @@ -134,136 +147,144 @@ ) __all__ = [ - "Digest", + "Adler_Algo", + "Adler_Field", + "Adler", "Algorithm", - "Md5", - "Sha1", - "Sha2_256", - "Sha2_224", - "Sha2_384", - "Sha2_512", - "Sha3_224", - "Sha3_256", - "Sha3_384", - "Sha3_512", + "align", + "Aligned", + "And", + "AsLengthRef", + "boolean", + "Branch", + "Bytes", + "Bz2Compressed", + "Chain", + "char", + "Compressed", + "Computed", + "ConditionalChain", + "Const", + "ConstBytes", + "ConstString", + "Crc32_Algo", + "Crc32_Field", "Crc32", - "Adler", - "HMAC", + "CString", + "CTX_DIGEST_ALGO", + "CTX_DIGEST_HOOK", + "CTX_DIGEST_OBJ", + "CTX_DIGEST", + "DEFAULT_OPTION", + "Digest", "DigestField", "DigestFieldAction", + "double", + "Else", + "ElseIf", + "Encrypted", + "End", + "ENUM_STRICT", + "Enum", + "Field", + "FieldMixin", + "FieldStruct", + "float16", + "float32", + "float64", + "get_args", + "get_kwargs", + "HMAC", + "HMACAlgorithm", + "If", + "Int", + "int16", + "int24", + "int32", + "int64", + "int8", + "intptr_fn", + "intptr", + "INVALID_DEFAULT", + "IOHook", + "IPv4Address", + "IPv6Address", + "KeyCipher", + "Lazy", + "LZMACompressed", + "LZOCompressed", + "MAC", + "MACAddress", "Md5_Algo", "Md5_Field", + "Md5", + "Memory", + "offintptr", + "offuintptr", + "Operator", + "Or", + "Otherwise", + "Padded", + "padding", + "Padding", + "Pass", + "pointer", + "Pointer", + "PostPad", + "Prefixed", + "PrePad", + "psize", + "pssize", + "PTR_STRICT", + "PyStructFormattedField", + "relative_pointer", + "RelativePointer", "Sha1_Algo", "Sha1_Field", - "Sha2_256_Algo", - "Sha2_256_Field", + "Sha1", "Sha2_224_Algo", "Sha2_224_Field", + "Sha2_224", + "Sha2_256_Algo", + "Sha2_256_Field", + "Sha2_256", "Sha2_384_Algo", "Sha2_384_Field", + "Sha2_384", "Sha2_512_Algo", "Sha2_512_Field", + "Sha2_512", "Sha3_224_Algo", "Sha3_224_Field", + "Sha3_224", "Sha3_256_Algo", "Sha3_256_Field", + "Sha3_256", "Sha3_384_Algo", "Sha3_384_Field", + "Sha3_384", "Sha3_512_Algo", "Sha3_512_Field", - "Crc32_Algo", - "Crc32_Field", - "Adler_Algo", - "Adler_Field", - "CTX_DIGEST", - "CTX_DIGEST_ALGO", - "CTX_DIGEST_HOOK", - "CTX_DIGEST_OBJ", - "HMACAlgorithm", - "uintptr", - "intptr", - "offintptr", - "offuintptr", - "Pointer", - "pointer", - "intptr_fn", - "PTR_STRICT", - "relative_pointer", - "RelativePointer", - "uintptr_fn", - "Compressed", - "ZLibCompressed", - "Bz2Compressed", - "LZMACompressed", - "LZOCompressed", - "PyStructFormattedField", - "Transformer", - "Const", - "ConstBytes", - "ConstString", - "Enum", + "Sha3_512", + "singleton", + "Start", "String", - "Bytes", - "Memory", - "Computed", - "Pass", - "CString", - "Prefixed", - "Int", + "Timestamp", + "Transformer", "UInt", - "padding", - "char", - "boolean", - "int8", - "uint8", - "int16", "uint16", - "int24", "uint24", - "int32", "uint32", - "int64", "uint64", - "pssize", - "psize", - "float16", - "float32", - "float64", - "double", - "void_ptr", + "uint8", + "uintptr_fn", + "uintptr", "Uuid", - "Aligned", - "align", - "Lazy", - "ENUM_STRICT", - "Field", - "INVALID_DEFAULT", - "DEFAULT_OPTION", - "singleton", - "FieldMixin", - "FieldStruct", - "Chain", - "Operator", - "get_args", - "get_kwargs", - "VarInt", "VARINT_LSB", + "VarInt", "vint", - "Encrypted", + "void_ptr", + "When", "Xor", - "Or", - "And", - "KeyCipher", - "MAC", - "MACAddress", - "IPv4Address", - "IPv6Address", - "ConditionalChain", - "If", - "Else", - "ElseIf", - "IOHook", - "Padding", - "AsLengthRef", - "Timestamp", + "ZLibCompressed", ] diff --git a/src/caterpillar/fields/_base.py b/src/caterpillar/fields/_base.py index 9fb66b7d..94802fad 100755 --- a/src/caterpillar/fields/_base.py +++ b/src/caterpillar/fields/_base.py @@ -229,6 +229,8 @@ def amount(self) -> _LengthT | None: @amount.setter def amount(self, value: _LengthT | None): + if isinstance(value, int) and value < 0: + raise ValueError(f"Sequence length must be non-negative - got {value!r}") self.__amount = value self._amount_is_lambda = callable(value) self._is_seq = self._amount_is_lambda or value is not None @@ -248,7 +250,9 @@ def options(self, value: _SwitchOptionsT | None): self.__options = value self._switch_is_lambda = callable(value) self._switch_has_default = ( - bool(value) and not self._switch_is_lambda and DEFAULT_OPTION in value # pyright: ignore[reportOperatorIssue] + bool(value) + and not self._switch_is_lambda + and DEFAULT_OPTION in value # pyright: ignore[reportOperatorIssue] ) @property @@ -511,6 +515,26 @@ def __unpack__(self, context: _ContextLike) -> _OT: :type context: _ContextLike :return: the parsed data """ + # fast path for default structs + if ( + not self._has_cond + and not self._is_lambda + and self._keep_pos + and not self._has_offset + and self.__options is None + ): + context[CTX_SEQ] = self._is_seq + context[CTX_FIELD] = self + try: + return self.__struct.__unpack__(context) + except Exception as exc: + if not isinstance(exc, StructException): + exc = StructException(str(exc), context) + value = self.default + if value is INVALID_DEFAULT or isinstance(exc, ValidationError): + raise exc + return value + stream: _StreamType = context[CTX_STREAM] if self._has_cond and not self.is_enabled(context): # Disabled fields or context lambdas won't pack any data diff --git a/src/caterpillar/fields/common.py b/src/caterpillar/fields/common.py index a9aaf48b..0e661ad0 100755 --- a/src/caterpillar/fields/common.py +++ b/src/caterpillar/fields/common.py @@ -72,6 +72,7 @@ _NATIVE_ONLY_FORMATS: Final[frozenset[str]] = frozenset({"n", "N", "P"}) + class PyStructFormattedField(FieldStruct[_IT, _IT]): """ A field class representing a binary format using format characters (e.g., 'i', 'I', etc.). @@ -186,10 +187,9 @@ def pack_seq(self, seq: Collection[_IT], context: _ContextLike) -> None: """ field = context.get(CTX_FIELD) target_length = len(seq) - if target_length == 0: - return # nothing to do - if not field: + if target_length == 0: + return # nothing to do # just pack directly # WE LOSE SIZE CHECKING HERE! ch = (self.__byteorder__ or O_DEFAULT_ENDIAN.value or LittleEndian).ch @@ -198,8 +198,10 @@ def pack_seq(self, seq: Collection[_IT], context: _ContextLike) -> None: length = field.length(context) if type(length) is _PrefixedType: context[CTX_SEQ] = False - length.start.__pack__(len(seq), context) - context[CTX_SEQ] = True + try: + length.start.__pack__(len(seq), context) + finally: + context[CTX_SEQ] = True elif length is not Ellipsis: if length != target_length: raise ValueError( @@ -207,6 +209,8 @@ def pack_seq(self, seq: Collection[_IT], context: _ContextLike) -> None: + f"{target_length} elements were provided!" ) + if target_length == 0: + return # nothing to do struct_ = self._cached(field.order.ch, target_length) context[CTX_STREAM].write(struct_.pack(*seq)) @@ -248,6 +252,20 @@ def unpack_seq(self, context: _ContextLike) -> Collection[_IT]: # only possible when a Field has been configured field = context[CTX_FIELD] length = field.length(context) + if type(length) is _PrefixedType: + context[CTX_SEQ] = False + field.amount = 1 + try: + new_length = length.start.__unpack__(context) + finally: + field.amount = length + context[CTX_SEQ] = True + length = new_length + if not isinstance(length, int): + raise InvalidValueError( + f"Prefix struct returned non-integer: {length!r}", context + ) + if length == 0: return [] # maybe add factory @@ -1058,7 +1076,9 @@ def unpack_single(self, context: _ContextLike) -> _MemoryOT: if size is Ellipsis: return memoryview(stream.read()) - return memoryview(read_exact(context, size, "Memory field")) # pyright: ignore[reportReturnType] + return memoryview( + read_exact(context, size, "Memory field") + ) # pyright: ignore[reportReturnType] class Bytes(Memory[bytes, bytes]): @@ -1316,7 +1336,7 @@ def unpack_single(self, context: _ContextLike) -> str: value = bytes(data) else: length = self.__size__(context) - value: bytes = context[CTX_STREAM].read(length) + value = read_exact(context, length, "CString") encoding: str = self.encoding(context) if self._encoding_is_lambda else self.encoding # pyright: ignore[reportCallIssue, reportAssignmentType] return value.rstrip(self._raw_pad).decode(encoding) @@ -2093,7 +2113,7 @@ class Lazy(FieldStruct[_IT, _OT]): when the field is accessed. """ - def __init__(self, struct: Callable[[], _StructLike[_IT, _OT]]) -> None: + def __init__(self, struct: Callable[[], _StructLike[_IT, _OT] | type]) -> None: if not callable(struct): raise TypeError(f"struct must be a callable - got {struct!r}") @@ -2109,7 +2129,8 @@ def struct(self) -> _StructLike[_IT, _OT]: :return: The underlying struct. :rtype: _StructLike """ - return self.struct_fn() + struct = self.struct_fn() + return getstruct(struct, struct) # pyright: ignore[reportReturnType] def __bits__(self) -> int: """ @@ -2439,3 +2460,235 @@ def decode(self, parsed: _TimestampT, context: _ContextLike) -> datetime.datetim tz = self.tz if self.tz is not None else datetime.timezone.utc dt = datetime.datetime.fromtimestamp(float(parsed), tz) return dt if self.tz is not None else dt.replace(tzinfo=None) + + +def _normalize_fill(fill: Any) -> bytes: + match fill: + case int(): + if not 0 <= fill <= 255: + raise ValueError(f"Fill byte must be in range 0-255 - got {fill!r}") + value = bytes([fill]) + case Buffer(): + value = bytes(fill) + case _: + raise TypeError( + f"Fill must be a bytes-like object or integer - got {fill!r}" + ) + if not value: + raise ValueError("fill pattern must be at least one byte") + return value + + +class Padded(FieldStruct[_IT, _OT]): + """ + A wrapper that adds explicit padding bytes before or after a target struct. + + Padding lengths are byte counts and may be static integers or context + lambdas. Multi-byte fill patterns are repeated and truncated to exactly the + requested byte length. + """ + + __slots__: tuple[str, ...] = ( + "struct", + "before", + "after", + "_before_fill", + "_after_fill", + "_before_strict", + "_after_strict", + ) + + def __init__( + self, + struct: _StructLike[_IT, _OT] | type, + *, + before: int | _ContextLambda[int] = 0, + after: int | _ContextLambda[int] = 0, + fill: Buffer | int = 0x00, + strict: bool = False, + ) -> None: + self.struct: _StructLike[_IT, _OT] = ( + getstruct(struct) or struct + ) # pyright: ignore[reportAttributeAccessIssue] + self.before: int | _ContextLambda[int] = before + self.after: int | _ContextLambda[int] = after + fill_bytes = _normalize_fill(fill) + self._before_fill: bytes = fill_bytes + self._after_fill: bytes = fill_bytes + self._before_strict: bool = strict + self._after_strict: bool = strict + + @classmethod + def new( + cls, + struct: _StructLike[_IT, _OT] | type, + *, + before: int | _ContextLambda[int] = 0, + after: int | _ContextLambda[int] = 0, + before_fill: bytes, + after_fill: bytes, + before_strict: bool, + after_strict: bool, + ) -> Self: + obj = cls(struct, before=before, after=after) + obj._before_fill = before_fill + obj._after_fill = after_fill + obj._before_strict = before_strict + obj._after_strict = after_strict + return obj + + def __type__(self) -> type | str | None: + return self.struct.__type__() + + def __size__(self, context: _ContextLike) -> int: + if callable(self.before) or callable(self.after): + raise DynamicSizeError( + "Padded fields with dynamic padding don't have a fixed size" + ) + before = self._resolve_length(self.before, context) + after = self._resolve_length(self.after, context) + return before + self.struct.__size__(context) + after + + def _fill_bytes(self, fill: bytes, length: int) -> bytes: + if length == 0: + return b"" + return (fill * ((length + len(fill) - 1) // len(fill)))[:length] + + def _resolve_length( + self, length: int | _ContextLambda[int], context: _ContextLike + ) -> int: + value = length(context) if callable(length) else length + if not isinstance(value, int): + raise ValueError( + f"Padding length must resolve to an integer - got {value!r}" + ) + if value < 0: + raise ValueError(f"Padding length must be non-negative - got {value!r}") + return value + + def _read_padding( + self, length: int, fill: bytes, strict: bool, context: _ContextLike + ) -> None: + data = read_exact(context, length, "Padded") + expected = self._fill_bytes(fill, length) + if strict and data != expected: + raise ValidationError( + "Parsed padding does not match fill pattern:\n" + + f"- parsed: {data.hex()}h\n" + + f"- fill : {expected.hex()}h", + context, + ) + + def _write_padding(self, length: int, fill: bytes, context: _ContextLike) -> None: + context[CTX_STREAM].write(self._fill_bytes(fill, length)) + + @override + def unpack_single(self, context: _ContextLike) -> _OT: + before = self._resolve_length(self.before, context) + self._read_padding(before, self._before_fill, self._before_strict, context) + obj = self.struct.__unpack__(context) + after = self._resolve_length(self.after, context) + self._read_padding(after, self._after_fill, self._after_strict, context) + return obj + + @override + def pack_single(self, obj: _IT, context: _ContextLike) -> None: + before = self._resolve_length(self.before, context) + self._write_padding(before, self._before_fill, context) + self.struct.__pack__(obj, context) + after = self._resolve_length(self.after, context) + self._write_padding(after, self._after_fill, context) + + +class _PadSpec: + __slots__: tuple[str, ...] = ("length", "fill", "strict", "side") + + def __init__( + self, + side: str, + length: int | _ContextLambda[int], + *, + fill: Buffer | int = 0x00, + strict: bool = False, + ) -> None: + self.side: str = side + self.length: int | _ContextLambda[int] = length + self.fill: bytes = _normalize_fill(fill) + self.strict: bool = strict + + def __call__(self, struct: _StructLike[_IT, _OT] | type) -> Padded[_IT, _OT]: + target = getstruct(struct) or struct + if isinstance(target, Padded): + return self._merge(target) + + if self.side == "before": + return Padded.new( + target, # pyright: ignore[reportArgumentType] + before=self.length, + before_fill=self.fill, + after_fill=b"\x00", + before_strict=self.strict, + after_strict=False, + ) + return Padded.new( + target, # pyright: ignore[reportArgumentType] + after=self.length, + before_fill=b"\x00", + after_fill=self.fill, + before_strict=False, + after_strict=self.strict, + ) + + def _merge(self, struct: Padded[_IT, _OT]) -> Padded[_IT, _OT]: + if self.side == "before": + return Padded.new( + struct.struct, + before=self.length, + after=struct.after, + before_fill=self.fill, + after_fill=struct._after_fill, + before_strict=self.strict, + after_strict=struct._after_strict, + ) + return Padded.new( + struct.struct, + before=struct.before, + after=self.length, + before_fill=struct._before_fill, + after_fill=self.fill, + before_strict=struct._before_strict, + after_strict=self.strict, + ) + + def __rtruediv__(self, struct: _StructLike[_IT, _OT] | type) -> Padded[_IT, _OT]: + return self(struct) + + +def PrePad( + length: int | _ContextLambda[int], + *, + fill: Buffer | int = 0x00, + strict: bool = False, +) -> _PadSpec: + """ + Return a compact padding spec that adds bytes before a target struct. + + The returned object can be called directly with a struct or used with the + slash syntax, e.g. ``PrePad(2)(uint8)`` or ``uint8 / PrePad(2)``. + """ + return _PadSpec("before", length, fill=fill, strict=strict) + + +def PostPad( + length: int | _ContextLambda[int], + *, + fill: Buffer | int = 0x00, + strict: bool = False, +) -> _PadSpec: + """ + Return a compact padding spec that adds bytes after a target struct. + + The returned object can be called directly with a struct or used with the + slash syntax, e.g. ``PostPad(2)(uint8)`` or ``uint8 / PostPad(2)``. + """ + return _PadSpec("after", length, fill=fill, strict=strict) diff --git a/src/caterpillar/fields/compression.py b/src/caterpillar/fields/compression.py index a387cc30..9c1ec17a 100755 --- a/src/caterpillar/fields/compression.py +++ b/src/caterpillar/fields/compression.py @@ -106,7 +106,9 @@ def encode(self, obj: bytes, context: _ContextLike) -> bytes: :return: The compressed data. :rtype: bytes """ - return self.compressor.compress(obj, **get_kwargs(self.comp_args, context)) + return self.compressor.compress( + obj, **get_kwargs(self.comp_args.copy(), context) + ) @override def decode(self, parsed: bytes, context: _ContextLike) -> bytes: @@ -121,7 +123,7 @@ def decode(self, parsed: bytes, context: _ContextLike) -> bytes: :rtype: bytes """ return self.compressor.decompress( - parsed, **get_kwargs(self.decomp_args, context) + parsed, **get_kwargs(self.decomp_args.copy(), context) ) diff --git a/src/caterpillar/fields/conditional.py b/src/caterpillar/fields/conditional.py index 8500c73f..3de21686 100755 --- a/src/caterpillar/fields/conditional.py +++ b/src/caterpillar/fields/conditional.py @@ -12,15 +12,28 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# pyright: reportPrivateUsage=false +# pyright: reportPrivateUsage=false, reportExplicitAny=false, reportUnreachable=false +# pyright: reportAny=false +import operator +import sys + from types import TracebackType from typing import Annotated, Any, get_args, get_origin -from typing_extensions import override, Self - -from caterpillar.context import ConditionContext -from caterpillar.exception import ValidationError -from caterpillar.shared import typeof, constval -from caterpillar.abc import _ContextLambda, _ContextLike, _StructLike +from typing_extensions import Final, override, Self + +from caterpillar.context import BinaryExpression, ConditionContext, UnaryExpression +from caterpillar.exception import StructException, ValidationError +from caterpillar.registry import to_struct +from caterpillar.shared import iscond, iscondend, iscondstart, typeof, constval +from caterpillar.abc import ( + _ArchLike, + _ContextLambda, + _ContextLike, + _EndianLike, + _OptionLike, + _StructLike, + _AnnotationT, +) from ._base import Field @@ -97,12 +110,495 @@ def __size__(self, context: _ContextLike) -> int: return struct.__size__(context) if struct else 0 +class _ConditionalAnnotation: + __slots__: tuple[str, ...] = ("condition", "branch_conditions", "is_last") + + __conditional__: bool = True + + def __init__( + self, + condition: _ContextLambda[bool] | bool, + branch_conditions: tuple[_ContextLambda[bool] | bool, ...] | None = None, + *, + is_last: bool = False, + ) -> None: + self.condition: _ContextLambda[bool] | bool = condition + self.branch_conditions: ( + tuple[_ContextLambda[bool] | bool, ...] | tuple[_ContextLambda[bool] | bool] + ) = branch_conditions or (condition,) + self.is_last: bool = is_last + + def __getitem__(self, annotation: _AnnotationT) -> _AnnotationT: + return _ConditionalAnnotation.apply_condition(annotation, self.condition) + + @staticmethod + def apply_condition( + annotation: _AnnotationT, condition: _ContextLambda[bool] | bool + ) -> _AnnotationT: + annotated_type = None + extra_options = () + field = annotation + is_annotated = get_origin(annotation) is Annotated + if is_annotated: + annotated_type, field, *extra_options = get_args(annotation) + + if not isinstance(field, Field): + struct_obj = to_struct(field) + if not isinstance(struct_obj, Field): + struct_obj = Field(struct_obj) + struct_obj.condition = condition + field = struct_obj + elif field.has_condition(): + field.condition = BinaryExpression( + operator.and_, field.condition, condition + ) + else: + field //= condition # pyright: ignore[reportUnknownVariableType] + + return ( + Annotated[(annotated_type, field, *extra_options)] + if is_annotated + else field + ) + + +class Start: + """Start marker for an inline explicit conditional block. + + Use this as metadata on the first real field in a Python 3.14+ conditional + block: + + .. code-block:: python + + with If(this.flag == 1) as when: + first: f[int, uint8, Start(when)] + second: f[int, uint8] + last: f[int, uint8, End(when)] + + Unlike the older invisible marker-field spelling, this does not add a + synthetic field to the class body. + + .. versionadded: 2.9.0 + """ + + __slots__: tuple[str, ...] = ("marker", "condition", "branch_conditions", "is_last") + + __conditional__: bool = True + __conditional_start__: bool = True + + def __init__(self, marker: "_MarkerT") -> None: + condition = getattr(marker, "condition", getattr(marker, "func", None)) + if condition is None: + raise StructException( + "Start() requires a conditional marker returned by If." + ) + self.marker: _MarkerT = marker + self.condition: _ContextLambda[bool] | bool = condition + self.branch_conditions: tuple[_ContextLambda[bool] | bool, ...] = getattr( + marker, "branch_conditions", (condition,) + ) + self.is_last: bool = getattr(marker, "is_last", False) + + +class End: + """End marker for an explicit conditional block. + + Use this as metadata on the last real field in a Python 3.14+ inline marker + block: + + .. code-block:: python + + with If(this.flag == 1) as when: + first: f[int, uint8, Start(when)] + second: f[int, uint8] + last: f[int, uint8, End(when)] + + The older invisible marker-field spelling is still supported: + + .. code-block:: python + + with If(this.flag == 1) as when: + _: f[None, when] = Invisible() + value: f[int, uint8] + _end: f[None, End(when)] = Invisible() + + Invisible marker fields are removed from the final struct model. + + .. versionadded: 2.9.0 + """ + + __slots__: tuple[str, ...] = ("marker", "condition") + + __conditional__: bool = True + __conditional_end__: bool = True + + def __init__(self, marker: "_MarkerT") -> None: + condition = getattr(marker, "condition", getattr(marker, "func", None)) + if condition is None: + raise StructException("End() requires a conditional marker returned by If.") + self.marker: _MarkerT = marker + self.condition: _ContextLambda[bool] | bool = condition + + +_MarkerT = _ConditionalAnnotation | ConditionContext | Start | End + + +def and_cond( + left: _ContextLambda[bool] | bool, right: _ContextLambda[bool] | bool +) -> _ContextLambda[bool] | bool: + if left is True: + return right + if right is True: + return left + if left is False or right is False: + return False + return BinaryExpression(operator.and_, left, right) + + +def or_cond( + left: _ContextLambda[bool] | bool, right: _ContextLambda[bool] | bool +) -> _ContextLambda[bool] | bool: + if left is True or right is True: + return True + if left is False: + return right + if right is False: + return left + return BinaryExpression(operator.or_, left, right) + + +def not_cond( + condition: _ContextLambda[bool] | bool, +) -> _ContextLambda[bool] | bool: + if condition is True: + return False + if condition is False: + return True + return UnaryExpression("not", operator.not_, condition) + + +def any_cond( + conditions: tuple[_ContextLambda[bool] | bool, ...], +) -> _ContextLambda[bool] | bool: + result: _ContextLambda[bool] | bool = False + for condition in conditions: + result = or_cond(result, condition) + return result + + +def _previous_branch_conditions( + marker: ConditionContext | _ConditionalAnnotation, +) -> tuple[_ContextLambda[bool] | bool, ...]: + if iscondend(marker): + raise StructException( + "Conditional branch requires a start marker, not End(...)." + ) + conditions = getattr(marker, "branch_conditions", None) + if not conditions: + raise StructException( + "Conditional branch requires a marker returned by 'with If(...) as name'." + ) + return conditions + + +def ensure_not_last( + marker: ConditionContext | _ConditionalAnnotation, label: str +) -> None: + if getattr(marker, "is_last", False): + raise StructException(f"{label} cannot be added after Else.") + + +def chain_cond( + previous: ConditionContext, condition: _ContextLambda[bool] | bool +) -> tuple[_ContextLambda[bool] | bool, tuple[_ContextLambda[bool] | bool, ...]]: + ensure_not_last(previous, "ElseIf") + previous_conditions = _previous_branch_conditions(previous) + effective = and_cond(not_cond(any_cond(previous_conditions)), condition) + return effective, (*previous_conditions, condition) + + +def else_cond( + previous: ConditionContext, +) -> tuple[_ContextLambda[bool] | bool, tuple[_ContextLambda[bool] | bool, ...]]: + ensure_not_last(previous, "Else") + previous_conditions = _previous_branch_conditions(previous) + return not_cond(any_cond(previous_conditions)), previous_conditions + + +def _split_conditional_metadata( + annotation: Any, +) -> tuple[Any, list[Any], bool]: + if get_origin(annotation) is not Annotated: + return annotation, [], False + + args = get_args(annotation) + if not args: + return annotation, [], False + + annotated_type, *metadata = args + markers = [value for value in metadata if iscond(value)] + if not markers: + return annotation, [], False + + kept_metadata = [value for value in metadata if not iscond(value)] + marker_field = annotated_type in (None, type(None)) and not kept_metadata + cleaned = ( + Annotated[(annotated_type, *kept_metadata)] if kept_metadata else annotated_type + ) + return cleaned, markers, marker_field + + +def _open_marker( + model: type, + name: str, + marker: Any, + active: list[list[Any]], + seen_starts: set[int], +) -> None: + target: _MarkerT = getattr(marker, "marker", marker) + marker_id = id(target) + if sys.version_info >= (3, 14) and marker_id in seen_starts: + raise StructException( + f"Conditional marker {name!r} in {model!r} reuses a marker " + + "alias. Use a unique alias for each Python 3.14 marker block." + ) + seen_starts.add(marker_id) + active.append([target, 0]) + + +def _close_marker( + model: type, name: str, marker: _MarkerT, active: list[list[Any]] +) -> None: + if not active: + raise StructException( + f"Conditional end marker {name!r} in {model!r} has no " + + "matching start marker." + ) + expected = getattr(marker, "marker", marker) + current, field_count = active[-1] + if current is not expected: + raise StructException( + f"Conditional end marker {name!r} in {model!r} does not " + + "match the active conditional marker." + ) + if field_count == 0: + raise StructException( + f"Conditional marker ending at {name!r} in {model!r} does " + + "not contain any fields. Repeated field names in marker " + + "branches are not supported; use Branch(...) for same-field " + + "conditional chains." + ) + _ = active.pop() + + +def apply_conditional_markers( + model: type, annotations: dict[str, _AnnotationT] +) -> tuple[dict[str, Any], set[str]]: + updated: dict[str, Any] = {} + removed: set[str] = set() + active: list[list[Any]] = [] + seen_starts: set[int] = set() + + for name, annotation in annotations.items(): + annotation, markers, marker_field = _split_conditional_metadata(annotation) + if marker_field: + removed.add(name) + for marker in markers: + if iscondend(marker): + _close_marker(model, name, marker, active) + else: + _open_marker(model, name, marker, active, seen_starts) + continue + + start_markers = [marker for marker in markers if iscondstart(marker)] + end_markers = [marker for marker in markers if iscondend(marker)] + field_markers = [ + marker + for marker in markers + if not iscondstart(marker) and not iscondend(marker) + ] + + for marker in start_markers: + _open_marker(model, name, marker, active, seen_starts) + + if sys.version_info >= (3, 14): + for entry in active: + conditional = entry[0] + annotation = _ConditionalAnnotation.apply_condition( + annotation, conditional.condition + ) + for marker in field_markers: + annotation = _ConditionalAnnotation.apply_condition( + annotation, marker.condition + ) + for entry in active: + entry[1] += 1 + updated[name] = annotation + + for marker in end_markers: + _close_marker(model, name, marker, active) + + if active: + raise StructException( + f"Conditional marker in {model!r} is missing an End(when) marker." + ) + + return updated, removed + + +class When: + """One conditional arm for :class:`Branch`. + + :param condition: Context expression controlling this arm. + :param annotation: Field annotation or struct selected when the condition + evaluates to true. + + .. versionadded:: 2.9.0 + """ + + __slots__: tuple[str, ...] = ("condition", "annotation") + + def __init__( + self, condition: _ContextLambda[bool] | bool, annotation: _AnnotationT + ) -> None: + self.condition: _ContextLambda[bool] | bool = condition + self.annotation: _AnnotationT = annotation + + +class Otherwise: + """Fallback arm for :class:`Branch`. + + The fallback arm is selected when no earlier :class:`When` condition matched. + A branch can contain at most one fallback arm, and it must appear last. + + .. versionadded:: 2.9.0 + """ + + __slots__: tuple[str, ...] = ("annotation",) + + def __init__(self, annotation: _AnnotationT) -> None: + self.annotation: _AnnotationT = annotation + + +class Branch: + """Conditional field chain for one attribute. + + Use this when several conditions should decode or encode the same Python + attribute with different field definitions. + + .. code-block:: python + + @struct + class Packet: + tag: f[int, uint8] + value: f[ + int, + Branch( + When(this.tag == 1, uint8), + When(this.tag == 2, uint16), + Otherwise(uint8), + ), + ] + + Arm annotations can use ``f[...]`` to carry local options such as byte order. + + .. versionadded:: 2.9.0 + """ + + __slots__: tuple[str, ...] = ("chain",) + + def __init__(self, *arms: When | Otherwise) -> None: + if not arms: + raise StructException( + "Branch() requires at least one When or Otherwise arm." + ) + + chain: ConditionalChain | None = None + seen_otherwise = False + for arm in arms: + match arm: + case When(): + if seen_otherwise: + raise StructException( + "When(...) cannot appear after Otherwise(...)." + ) + condition = arm.condition + annotation = arm.annotation + case Otherwise(): + if seen_otherwise: + raise StructException( + "Branch() can only contain one Otherwise(...)." + ) + seen_otherwise = True + condition = None + annotation = arm.annotation + case _: # pyright: ignore[reportUnnecessaryComparison] + raise StructException( + f"Unsupported Branch arm {arm!r}; expected When or Otherwise." + ) + + # create the struct + options: list[_OptionLike | _EndianLike | _ArchLike] = [] + if get_origin(annotation) is Annotated: + _, annotation, *options = get_args(annotation) + + struct_obj = ( + annotation if isinstance(annotation, Field) else to_struct(annotation) + ) + if options: + # This way we make sure no options are left out + if not isinstance(struct_obj, Field): + struct_obj = Field(struct_obj) + + for option in options: + match option: + case _ArchLike(): + struct_obj.arch = option + case _EndianLike(): + struct_obj.order = option + case _OptionLike(): + struct_obj.add_flag(option) + case _: # pyright: ignore[reportUnnecessaryComparison] + raise ValidationError( + f"Could not add branch option: unsupported type ({type(option)})" + ) + + if chain is None: + chain = ConditionalChain( + struct_obj, True if condition is None else condition + ) + if condition is None: + chain.conditions[-1] = None + else: + chain.add(struct_obj, condition) # pyright: ignore[reportArgumentType] + + if chain is None: + raise ValueError("Invalid number of arguments provided") + + self.chain: ConditionalChain = chain + + def __type__(self) -> type | str | None: + return self.chain.__type__() + + def __unpack__(self, context: _ContextLike) -> object: + return self.chain.__unpack__(context) + + def __pack__(self, obj: object, context: _ContextLike) -> None: + self.chain.__pack__(obj, context) + + def __size__(self, context: _ContextLike) -> int: + return self.chain.__size__(context) + + class If(ConditionContext): """If-statement implementation for class definitions. - .. versionchanged:: 2.4.5 + .. versionchanged:: 2.9.0 - Python 3.14 is **not** supported. + Python 3.14+ requires explicit conditional annotations using either + ``with If(condition) as when:`` with ``field: f[type, field, when]`` for + one field, inline ``Start(when)`` / ``End(when)`` metadata for a block, + ``field: when[...]``, or explicit invisible marker fields. .. code-block:: python @@ -113,6 +609,30 @@ class Format: with If(lambda _: GLOBAL_CONSTANT == 33): b: uint8 + Python 3.14+ supports type-checker-friendly per-field metadata: + + .. code-block:: python + + @struct + class Format: + a: uint32 + + with If(lambda _: GLOBAL_CONSTANT == 33) as when: + b: f[int, uint8, when] + + It also supports inline block markers: + + .. code-block:: python + + @struct + class Format: + a: uint32 + + with If(lambda _: GLOBAL_CONSTANT == 33) as when: + b: f[int, uint8, Start(when)] + c: uint8 + d: f[int, uint8, End(when)] + Note that this class will alter the used fields and cover multiple field definitions. In addition, the type annotation will be modified to display the condition as well. @@ -121,18 +641,72 @@ class Format: This class is **not** a struct, but a simple context manager. """ - # As this class essentially does the same as ConditionContext, - # we don't have to implement anything. A simple if-statment - # is using the built-in conditional execution model from the Field - # class. + __slots__: tuple[str, ...] = ("_proxy",) + __conditional__: bool = True + + def __init__(self, condition: _ContextLambda[bool], depth: int = 2): + super().__init__(condition, depth) + self._proxy: _ConditionalAnnotation | None = None + + @property + def condition(self) -> _ContextLambda[bool] | bool: + return self.func + + @property + def branch_conditions(self) -> tuple[_ContextLambda[bool] | bool, ...]: + return (self.func,) + + @property + def is_terminal(self) -> bool: + return False + + @override + def __enter__(self) -> Self | _ConditionalAnnotation: + if sys.version_info >= (3, 14): + self._proxy = _ConditionalAnnotation(self.func, (self.func,)) + return self._proxy + + self._proxy = _ConditionalAnnotation(self.func) + depth = self.depth + self.depth: int = depth + 1 + try: + return super().__enter__() + finally: + self.depth = depth + + def __getitem__(self, annotation: _AnnotationT) -> Any: + if sys.version_info >= (3, 14): + return _ConditionalAnnotation.apply_condition(annotation, self.func) + return annotation + + @override + def __exit__( + self, exc_type: type, exc_value: Exception, traceback: TracebackType + ) -> None: + if sys.version_info >= (3, 14): + frame = self.getframe(2, "Could not exit condition context!") + if not any(value is self._proxy for value in frame.f_locals.values()): + raise StructException( + "Implicit 'with If(condition):' blocks are not supported on " + + "Python 3.14+. Use 'with If(condition) as when:' with " + + "'field: f[type, field, when]' for one field, or " + + "'Start(when)' / 'End(when)' metadata for a block." + ) + self._proxy = None + return None + depth = self.depth + self.depth = depth + 1 + try: + return super().__exit__(exc_type, exc_value, traceback) + finally: + self.depth = depth + # TODO(REVISIT): fix Annotated[...] annotation handling class ElseIf(ConditionContext): """ElseIf-statement implementation for class definitions. - .. versionchanged:: 2.4.3 - - Python 3.14 is **not** supported. + Python <= 3.13 supports the legacy implicit form: .. code-block:: python @@ -145,20 +719,105 @@ class Format: with ElseIf(this.a == 34): ... + + Python 3.14+ requires the explicit marker form: + + .. code-block:: python + + @struct + class Format: + a: uint32 + + with If(this.a == 32) as first: + one: f[int, uint8, first] + + with ElseIf(first, this.a == 34) as second: + two: f[int, uint8, second] """ + __slots__: tuple[str, ...] = ( + "_branch_conditions", + "is_last", + "_marker_mode", + "_proxy", + ) + + def __init__(self, *args: ConditionContext | _ContextLambda[bool] | bool) -> None: + self._proxy: _ConditionalAnnotation | None = None + match len(args): + case 1: + if isinstance(args[0], ConditionContext): + raise TypeError("ElseIf requires a condition!") + + super().__init__(args[0]) + self._branch_conditions = (self.func,) + self.is_last: bool = False + self._marker_mode: bool = False + case 2: + previous, condition = args + ensure_not_last(previous, "ElseIf") + previous_conditions = _previous_branch_conditions(previous) + effective = and_cond(not_cond(any_cond(previous_conditions)), condition) + branch_conditions = (*previous_conditions, condition) + super().__init__(effective) + self._branch_conditions: tuple[_ContextLambda[bool] | bool, ...] = ( + branch_conditions # pyright: ignore[reportAttributeAccessIssue] + ) + self.is_last = False + self._marker_mode = True + case _: + raise TypeError( + "ElseIf expects either ElseIf(condition) or ElseIf(previous, condition)." + ) + + @property + def condition(self) -> _ContextLambda[bool] | bool: + return self.func + + @property + def branch_conditions(self) -> tuple[_ContextLambda[bool] | bool, ...]: + return self._branch_conditions + + @property + def is_terminal(self) -> bool: + return self.is_last + @override - def __enter__(self) -> Self: + def __enter__(self) -> Self | _ConditionalAnnotation: + if self._marker_mode: + self._proxy = _ConditionalAnnotation( + self.func, + self.branch_conditions, + is_last=self.is_terminal, + ) + return self._proxy + if sys.version_info >= (3, 14): + raise StructException( + "Implicit 'with ElseIf(condition):' blocks are not supported on " + + "Python 3.14+. Use 'with ElseIf(previous, condition) as when:' " + + "alongside 'f[..., when]' for one field or Start/End metadata for a block." + ) self.depth: int = 3 super().__enter__() # pyright: ignore[reportUnusedCallResult] self.depth = 2 # We have to copy all variables here as we want to # provide the possibility to re-define some fields. - self.annotations: dict[str, Any] = self.annotations.copy() + self.annotations: dict[str, _AnnotationT] = self.annotations.copy() return self @override def __exit__(self, exc_type: type, exc_value: Exception, traceback: TracebackType): + if self._marker_mode: + frame = self.getframe(2, "Could not exit condition context!") + if not any(value is self._proxy for value in frame.f_locals.values()): + raise StructException( + "Python 3.14+ ElseIf blocks require " + + "'with ElseIf(previous, condition) as when:' with " + + "'f[..., when]' for one field or Start/End metadata for a block." + ) + self._proxy = None + return None + # fmt: off # we have to inspect no only new names but also defined ones frame = self.getframe(self.depth, "Could not enter condition context!") @@ -167,7 +826,7 @@ def __exit__(self, exc_type: type, exc_value: Exception, traceback: TracebackTyp # inspect defined fields for name in set(annotations) & set(self.namelist): new_field = annotations[name] - field: _StructLike = self.annotations[name] # pyright: ignore[reportAny] + field: _AnnotationT = self.annotations[name] is_annotated = get_origin(field) is Annotated if is_annotated: # annotated_type = field.__origin__ @@ -209,4 +868,81 @@ def __exit__(self, exc_type: type, exc_value: Exception, traceback: TracebackTyp # REVISIT: There is one case where 'ELSE' is not applicable and will cause # a field to be present at all times. This problem exists if we add fields # into an else-branch without a previously defined field. -Else = ElseIf(lambda context: True) +class _ElseBranch(ConditionContext): + __slots__: tuple[str, ...] = ("_branch_conditions", "_is_last", "_proxy") + + def __init__(self, previous: ConditionContext) -> None: + effective, branch_conditions = else_cond(previous) + super().__init__(effective) + self._branch_conditions: tuple[_ContextLambda[bool] | bool, ...] = ( + branch_conditions + ) + self._is_last: bool = True + self._proxy: _ConditionalAnnotation | None = None + + @property + def condition(self) -> _ContextLambda[bool] | bool: + return self.func + + @property + def branch_conditions(self) -> tuple[_ContextLambda[bool] | bool, ...]: + return self._branch_conditions + + @property + def is_last(self) -> bool: + return self._is_last + + def __enter__(self) -> _ConditionalAnnotation: + self._proxy = _ConditionalAnnotation( + self.func, + self.branch_conditions, + is_last=self.is_last, + ) + return self._proxy + + def __exit__( + self, exc_type: type, exc_value: Exception, traceback: TracebackType + ) -> None: + frame = self.getframe(2, "Could not exit condition context!") + if not any(value is self._proxy for value in frame.f_locals.values()): + raise StructException( + "Python 3.14+ Else blocks require 'with Else(previous) as when:' " + + "with 'f[..., when]' for one field or Start/End metadata for a block." + ) + self._proxy = None + return None + + +class _Else: + """Else marker factory. + + .. versionchanged:: 2.9.0 + Python <= 3.13 supports ``with Else:`` for legacy condition blocks. Python + 3.14+ requires ``with Else(previous) as when:`` with ``f[..., when]`` for + one field or ``Start(when)`` / ``End(when)`` metadata for a block. + """ + + __slots__: tuple[str, ...] = ("_legacy",) + + def __init__(self) -> None: + self._legacy: ConditionContext = ElseIf(lambda context: True) + + def __call__(self, previous: ConditionContext) -> _ElseBranch: + return _ElseBranch(previous) + + def __enter__(self) -> ElseIf: + if sys.version_info >= (3, 14): + raise StructException( + "Implicit 'with Else:' blocks are not supported on Python 3.14+. " + + "Use 'with Else(previous) as when:' with 'f[..., when]' for one " + + "field or Start/End metadata for a block." + ) + return self._legacy.__enter__() # pyright: ignore[reportReturnType] + + def __exit__( + self, exc_type: type, exc_value: Exception, traceback: TracebackType + ) -> None: + return self._legacy.__exit__(exc_type, exc_value, traceback) + + +Else: Final[_Else] = _Else() diff --git a/src/caterpillar/include/caterpillar/arch.h b/src/caterpillar/include/caterpillar/arch.h index 4640adf8..8f7633f3 100644 --- a/src/caterpillar/include/caterpillar/arch.h +++ b/src/caterpillar/include/caterpillar/arch.h @@ -210,11 +210,11 @@ CpEndian_SetEndian(PyObject* op, CpEndianObject* endian) return NULL; } PyObject* ret = PyObject_CallOneArg(attr, (PyObject*)endian); + Py_DECREF(attr); if (!ret) { return NULL; } - Py_DECREF(attr); return ret; } -#endif \ No newline at end of file +#endif diff --git a/src/caterpillar/include/caterpillar/atoms/builtin/conditional.h b/src/caterpillar/include/caterpillar/atoms/builtin/conditional.h index ec995860..61eb33db 100644 --- a/src/caterpillar/include/caterpillar/atoms/builtin/conditional.h +++ b/src/caterpillar/include/caterpillar/atoms/builtin/conditional.h @@ -64,6 +64,7 @@ CpConditionalAtom_SetAtom(PyObject* pObj, PyObject* pAtom) CpConditionalAtomObject* self = _Cp_CAST(CpConditionalAtomObject*, pObj); if (!pAtom) { PyErr_SetString(PyExc_ValueError, "Atom cannot be null"); + return -1; } Py_XSETREF(self->m_atom, Py_NewRef(pAtom)); @@ -76,10 +77,11 @@ CpConditionalAtom_SetCondition(PyObject* pObj, PyObject* pCondition) CpConditionalAtomObject* self = _Cp_CAST(CpConditionalAtomObject*, pObj); if (!pCondition) { PyErr_SetString(PyExc_ValueError, "Condition cannot be null"); + return -1; } Py_XSETREF(self->m_condition, Py_NewRef(pCondition)); return 0; } -#endif \ No newline at end of file +#endif diff --git a/src/caterpillar/include/caterpillar/atoms/builtin/offset.h b/src/caterpillar/include/caterpillar/atoms/builtin/offset.h index f8081756..c0f9a35e 100644 --- a/src/caterpillar/include/caterpillar/atoms/builtin/offset.h +++ b/src/caterpillar/include/caterpillar/atoms/builtin/offset.h @@ -78,21 +78,40 @@ CpOffsetAtom_WhenceAsLong(PyObject* pObj) static inline int CpOffsetAtom_SetAtom(PyObject* pObj, PyObject* pAtom) { - _Cp_CAST(CpOffsetAtomObject*, pObj)->m_atom = Py_NewRef(pAtom); + CpOffsetAtomObject* self = _Cp_CAST(CpOffsetAtomObject*, pObj); + if (!pAtom) { + PyErr_SetString(PyExc_ValueError, "Atom cannot be null"); + return -1; + } + + Py_XSETREF(self->m_atom, Py_NewRef(pAtom)); return 0; } static inline int CpOffsetAtom_SetOffset(PyObject* pObj, PyObject* pOffset) { - _Cp_CAST(CpOffsetAtomObject*, pObj)->m_offset = Py_NewRef(pOffset); + CpOffsetAtomObject* self = _Cp_CAST(CpOffsetAtomObject*, pObj); + if (!pOffset) { + PyErr_SetString(PyExc_ValueError, "Offset cannot be null"); + return -1; + } + + Py_XSETREF(self->m_offset, Py_NewRef(pOffset)); + self->s_is_number = PyNumber_Check(self->m_offset); return 0; } static inline int CpOffsetAtom_SetWhence(PyObject* pObj, PyObject* pWhence) { - _Cp_CAST(CpOffsetAtomObject*, pObj)->m_whence = Py_NewRef(pWhence); + CpOffsetAtomObject* self = _Cp_CAST(CpOffsetAtomObject*, pObj); + if (!pWhence) { + PyErr_SetString(PyExc_ValueError, "Whence cannot be null"); + return -1; + } + + Py_XSETREF(self->m_whence, Py_NewRef(pWhence)); return 0; } @@ -108,4 +127,4 @@ CpOffsetAtom_SetKeepPosition(PyObject* pObj, int keep_pos) _Cp_CAST(CpOffsetAtomObject*, pObj)->s_keep_pos = keep_pos; } -#endif \ No newline at end of file +#endif diff --git a/src/caterpillar/include/caterpillar/atoms/builtin/repeated.h b/src/caterpillar/include/caterpillar/atoms/builtin/repeated.h index a4770038..02d3f0ac 100644 --- a/src/caterpillar/include/caterpillar/atoms/builtin/repeated.h +++ b/src/caterpillar/include/caterpillar/atoms/builtin/repeated.h @@ -68,6 +68,7 @@ CpRepeatedAtom_SetAtom(PyObject* pObj, PyObject* pAtom) CpRepeatedAtomObject* self = _Cp_CAST(CpRepeatedAtomObject*, pObj); if (!pAtom) { PyErr_SetString(PyExc_ValueError, "Atom cannot be null"); + return -1; } Py_XSETREF(self->m_atom, Py_NewRef(pAtom)); @@ -80,10 +81,11 @@ CpRepeatedAtom_SetLength(PyObject* pObj, PyObject* pLength) CpRepeatedAtomObject* self = _Cp_CAST(CpRepeatedAtomObject*, pObj); if (!pLength) { PyErr_SetString(PyExc_ValueError, "Length cannot be null"); + return -1; } Py_XSETREF(self->m_length, Py_NewRef(pLength)); return 0; } -#endif \ No newline at end of file +#endif diff --git a/src/caterpillar/include/caterpillar/atoms/builtin/switch.h b/src/caterpillar/include/caterpillar/atoms/builtin/switch.h index be9330ab..1ba08285 100644 --- a/src/caterpillar/include/caterpillar/atoms/builtin/switch.h +++ b/src/caterpillar/include/caterpillar/atoms/builtin/switch.h @@ -71,6 +71,7 @@ CpSwitchAtom_SetCases(PyObject* pObj, PyObject* pCases) CpSwitchAtomObject* self = _Cp_CAST(CpSwitchAtomObject*, pObj); if (!pCases) { PyErr_SetString(PyExc_ValueError, "Cases cannot be null"); + return -1; } Py_XSETREF(self->m_cases, Py_NewRef(pCases)); @@ -84,10 +85,11 @@ CpSwitchAtom_SetAtom(PyObject* pObj, PyObject* pAtom) CpSwitchAtomObject* self = _Cp_CAST(CpSwitchAtomObject*, pObj); if (!pAtom) { PyErr_SetString(PyExc_ValueError, "Atom cannot be null"); + return -1; } Py_XSETREF(self->m_atom, Py_NewRef(pAtom)); return 0; } -#endif \ No newline at end of file +#endif diff --git a/src/caterpillar/include/caterpillar/caterpillarapi.h.in b/src/caterpillar/include/caterpillar/caterpillarapi.h.in index 235d1ea4..d8ec7077 100644 --- a/src/caterpillar/include/caterpillar/caterpillarapi.h.in +++ b/src/caterpillar/include/caterpillar/caterpillarapi.h.in @@ -73,17 +73,20 @@ _import_caterpillar(void) PyObject* c_api = PyObject_GetAttrString(caterpillar, "_C_API"); if (c_api == NULL) { PyErr_SetString(PyExc_AttributeError, "_C_API not found"); + Py_DECREF(caterpillar); return -1; } if (!PyCapsule_CheckExact(c_api)) { PyErr_SetString(PyExc_TypeError, "_C_API is not a capsule"); Py_DECREF(c_api); + Py_DECREF(caterpillar); return -1; } Cp_API = (void**)PyCapsule_GetPointer(c_api, NULL); Py_DECREF(c_api); + Py_DECREF(caterpillar); if (Cp_API == NULL) { PyErr_SetString(PyExc_AttributeError, "_C_API is NULL pointer"); return -1; @@ -128,4 +131,4 @@ Cp_ImportCpAPI(void) } #endif // _CPMODULE -#endif // CATERPILLAR_API_H \ No newline at end of file +#endif // CATERPILLAR_API_H diff --git a/src/caterpillar/include/caterpillar/context.h b/src/caterpillar/include/caterpillar/context.h index 67df4457..57afab75 100644 --- a/src/caterpillar/include/caterpillar/context.h +++ b/src/caterpillar/include/caterpillar/context.h @@ -74,11 +74,14 @@ CpContext_GetDict(PyObject* obj) static inline int CpContext_COPYITEM(PyObject* pContext, PyObject* pSrc, PyObject* pKey) { + int result = 0; PyObject* nValue = CpContext_ITEM(pSrc, pKey); if (!nValue) { return -1; } - return CpContext_SETITEM(pContext, pKey, nValue); + result = CpContext_SETITEM(pContext, pKey, nValue); + Py_DECREF(nValue); + return result; } #define CpContext_IO(context, state) \ @@ -87,4 +90,4 @@ CpContext_COPYITEM(PyObject* pContext, PyObject* pSrc, PyObject* pKey) #define CpContext_SETIO(context, state, pIO) \ CpContext_SETITEM(context, state->str__context_io, pIO) -#endif \ No newline at end of file +#endif diff --git a/src/caterpillar/model/__init__.py b/src/caterpillar/model/__init__.py index d9bf8ca3..e10431c7 100644 --- a/src/caterpillar/model/__init__.py +++ b/src/caterpillar/model/__init__.py @@ -39,7 +39,14 @@ bitfield_factory, BitfieldDefMixin, ) -from ._template import istemplate, template, TemplateTypeVar, derive +from ._template import ( + istemplate, + template, + TemplateTypeVar, + derive, + field_of, + TemplateFieldRef, +) from .provider import ( unpack, unpack_file, @@ -50,38 +57,40 @@ ) __all__ = [ - "Sequence", - "RemoveField", - "Struct", - "struct", - "UnionHook", - "union", - "unpack", - "unpack_file", - "pack", - "pack_into", - "pack_file", - "sizeof", - "Bitfield", + "bitfield_factory", "bitfield", - "BitfieldGroup", - "issigned", - "getbits", - "istemplate", - "template", - "TemplateTypeVar", - "derive", - "NewGroup", - "EndGroup", - "SetAlignment", + "Bitfield", + "BitfieldDefMixin", "BitfieldEntry", + "BitfieldGroup", "BitfieldValueFactory", - "EnumFactory", "CharFactory", "DEFAULT_ALIGNMENT", + "derive", + "EndGroup", + "EnumFactory", + "field_of", + "getbits", "Invisible", - "StructDefMixin", + "issigned", + "istemplate", + "NewGroup", + "pack_file", + "pack_into", + "pack", + "RemoveField", + "Sequence", + "SetAlignment", + "sizeof", "struct_factory", - "bitfield_factory", - "BitfieldDefMixin", + "struct", + "Struct", + "StructDefMixin", + "template", + "TemplateFieldRef", + "TemplateTypeVar", + "union", + "UnionHook", + "unpack_file", + "unpack", ] diff --git a/src/caterpillar/model/_struct.py b/src/caterpillar/model/_struct.py index a1df5dc2..fbcab816 100755 --- a/src/caterpillar/model/_struct.py +++ b/src/caterpillar/model/_struct.py @@ -29,7 +29,8 @@ Buffer, ) -from caterpillar.shared import getstruct, hasstruct, ATTR_STRUCT +from caterpillar.fields.conditional import apply_conditional_markers +from caterpillar.shared import getstruct, hasstruct, ATTR_STRUCT, iscond from caterpillar.exception import InvalidValueError from caterpillar.options import ( S_EVAL_ANNOTATIONS, @@ -138,7 +139,19 @@ def _prepare_fields(self) -> dict[str, Any]: eval_str: bool = self.has_option(S_EVAL_ANNOTATIONS) # The why is described in detail here: https://docs.python.org/3/howto/annotations.html - return inspect.get_annotations(self.model, eval_str=eval_str) + annotations = inspect.get_annotations(self.model, eval_str=eval_str) + annotations, marker_names = apply_conditional_markers(self.model, annotations) + if marker_names: + self.model.__annotations__ = annotations + for name in marker_names: + default = getattr(self.model, name, None) + if isinstance(default, dc.Field): + delattr(self.model, name) + + for name, value in list(vars(self.model).items()): + if iscond(value): + delattr(self.model, name) + return annotations @override def _set_default(self, name: str, value: Any) -> None: diff --git a/src/caterpillar/model/_template.py b/src/caterpillar/model/_template.py index 614a39fc..81959102 100755 --- a/src/caterpillar/model/_template.py +++ b/src/caterpillar/model/_template.py @@ -1,4 +1,5 @@ # pylint: disable=protected-access +# pyright: reportAny=false, reportExplicitAny=false # Copyright (C) MatrixEditor 2023-2026 # # This program is free software: you can redistribute it and/or modify @@ -17,26 +18,38 @@ import inspect import types import dataclasses + +from hashlib import md5 from types import ModuleType -from typing import Any, Callable, TypeVar, overload -from typing_extensions import override +from typing import ( + Annotated, + Any, + Callable, + Generic, + TypeVar, + get_args, + get_origin, +) +from typing_extensions import overload, override, dataclass_transform from caterpillar.fields import Field, INVALID_DEFAULT -from caterpillar.model import Struct +from caterpillar.model import Invisible, Struct from caterpillar.options import S_UNION -from caterpillar.shared import ATTR_TEMPLATE +from caterpillar.shared import ATTR_TEMPLATE, hasstruct, typeof from caterpillar.abc import ( _LengthT, _StructLike, _ContextLambda, _GreedyType, - _ContextLike, _SwitchLambda, _EndianLike, _ArchLike, ) +_TYPEVAR_TYPE: type[TypeVar] = type(TypeVar("_CaterpillarTemplateTypeVar")) + + class TemplateTypeVar: """Template type variable. @@ -105,9 +118,77 @@ def to_field( default: Any = INVALID_DEFAULT, ) -> Field: # REVISIT: what about flags? + if get_origin(struct) is Annotated: + return struct + return Field(struct, arch=arch, default=default, **self.field_kwds) +class TemplateFieldRef: + """Field metadata for Python ``TypeVar`` based templates. + + .. versionadded:: 2.9.0 + """ + + param: TypeVar + field_kwds: dict[str, Any] + + def __init__(self, param: TypeVar, **field_kwds: Any) -> None: + self.param = param + self.field_kwds = field_kwds or {} + + @override + def __repr__(self) -> str: + name = getattr(self.param, "__name__", repr(self.param)) + count = self.field_kwds.get("amount") + if not count: + return f"field_of({name})" + return f"field_of({name})[{count}]" + + def __getitem__(self, amount: _LengthT) -> "TemplateFieldRef": + return TemplateFieldRef(self.param, amount=amount, **self.field_kwds) + + def __rshift__( + self, switch: _SwitchLambda | dict[str, _StructLike] + ) -> "TemplateFieldRef": + return TemplateFieldRef(self.param, options=switch, **self.field_kwds) + + def __matmul__(self, offset: int | _ContextLambda[int]) -> "TemplateFieldRef": + return TemplateFieldRef(self.param, offset=offset, **self.field_kwds) + + def __set_byteorder__(self, order: _EndianLike) -> "TemplateFieldRef": + return TemplateFieldRef(self.param, order=order, **self.field_kwds) + + def __rsub__(self, bits: int | _ContextLambda[int]) -> "TemplateFieldRef": + return TemplateFieldRef(self.param, bits=bits, **self.field_kwds) + + def __floordiv__( + self, condition: bool | _ContextLambda[bool] + ) -> "TemplateFieldRef": + return TemplateFieldRef(self.param, condition=condition, **self.field_kwds) + + def with_param(self, param: TypeVar) -> "TemplateFieldRef": + return TemplateFieldRef(param, **self.field_kwds) + + def to_field( + self, + struct: _StructLike | _ContextLambda, + default: Any = INVALID_DEFAULT, + ) -> Field: + if get_origin(struct) is Annotated: + _, struct, *_ = get_args(struct) + + return Field(struct, default=default, **self.field_kwds) + + +def field_of(param: Any) -> TemplateFieldRef: + """Create layout metadata for a Python ``TypeVar`` template field. + + .. versionadded:: 2.9.0 + """ + return TemplateFieldRef(param) + + def get_caller_module(frame: int = 1) -> str: try: # Direct call to retrieve the module name @@ -145,6 +226,12 @@ def add_positional(self, name: str, default: Any | None = None) -> None: self.positional_tys[name] = default +@dataclasses.dataclass +class GenericTemplateInfo: + parameters: tuple[TypeVar, ...] + cache: dict[tuple[str, ...], type] = dataclasses.field(default_factory=dict) + + def istemplate(obj: object) -> bool: """Return true if the object is a template.""" return hasattr(obj, ATTR_TEMPLATE) @@ -153,8 +240,164 @@ def istemplate(obj: object) -> bool: _TemplateModelT = TypeVar("_TemplateModelT") +def _is_typevar(value: Any) -> bool: + return isinstance(value, _TYPEVAR_TYPE) + + +def _replace_typevars(value: Any, bindings: dict[Any, Any]) -> Any: + if _is_typevar(value): + replacement = bindings.get(value, value) + if _is_typevar(replacement): + return replacement + return typeof(replacement) if not isinstance(replacement, type) else replacement + + origin = get_origin(value) + if origin is None: + return value + + args = get_args(value) + if not args: + return value + + if origin is Annotated: + type_hint, *metadata = args + return Annotated[ + ( + _replace_typevars(type_hint, bindings), + *(_replace_typevars(item, bindings) for item in metadata), + ) + ] + + replaced_args = tuple(_replace_typevars(arg, bindings) for arg in args) + try: + return origin[replaced_args] + except TypeError: + return value + + +def _resolve_generic_marker( + marker: Any, + bindings: dict[Any, Any], + default: Any, +) -> Any: + if isinstance(marker, TemplateFieldRef): + replacement = bindings.get(marker.param, marker.param) + if _is_typevar(replacement): + return marker.with_param(replacement) + return marker.to_field(replacement, default=default) + + if _is_typevar(marker): + replacement = bindings.get(marker, marker) + if _is_typevar(replacement): + return replacement + + if get_origin(replacement) is Annotated: + return replacement + + return Field(replacement, default=default) + + return _replace_typevars(marker, bindings) + + +def _specialize_generic_annotation( + annotation: Any, + bindings: dict[Any, Any], + default: Any, +) -> Any: + if get_origin(annotation) is Annotated: + type_hint, marker, *extra = get_args(annotation) + return Annotated[ + ( + _replace_typevars(type_hint, bindings), + _resolve_generic_marker(marker, bindings, default), + *(_replace_typevars(item, bindings) for item in extra), + ) + ] + return _resolve_generic_marker(annotation, bindings, default) + + +def _generic_name(origin: type, args: tuple[Any, ...], partial: bool) -> str: + suffix = get_mangled_name(origin, {str(arg): "" for arg in args}) + template_suffix = "Partial" if partial else "Struct" + return f"{suffix}_{template_suffix}" + + +def _create_generic_template( + cls: type[_TemplateModelT], + parameters: tuple[Any, ...] | None = None, +) -> type[_TemplateModelT]: + params = tuple(parameters or getattr(cls, "__parameters__", ())) + if not params: + raise TypeError("Generic template class needs at least one type parameter") + + def class_getitem(template_cls: type, args: Any) -> type: + return _derive_generic_template( + template_cls, + *((args,) if not isinstance(args, tuple) else args), + allow_partial=True, + module_name=template_cls.__module__, + ) + + setattr(cls, ATTR_TEMPLATE, GenericTemplateInfo(params)) + setattr(cls, "__class_getitem__", classmethod(class_getitem)) + return cls + + +def _collect_generic_args( + info: GenericTemplateInfo, + tys_args: tuple[Any, ...], + tys_kwargs: dict[str, Any], + partial: bool, +) -> tuple[Any, ...]: + if len(tys_args) > len(info.parameters): + raise ValueError( + f"Expected max. {len(info.parameters)} positional arguments - got {len(tys_args)}!" + ) + + args: list[Any | None] = [None] * len(info.parameters) + for index, value in enumerate(tys_args): + args[index] = value + + param_names = {param.__name__: index for index, param in enumerate(info.parameters)} + for name, value in tys_kwargs.items(): + if name not in param_names: + raise ValueError(f"Unknown type argument: {name!r}") + index = param_names[name] + if args[index] is not None: + raise ValueError(f"Type argument {name!r} already defined!") + args[index] = value + + for index, value in enumerate(args): + if value is not None: + continue + if partial: + args[index] = info.parameters[index] + continue + name = info.parameters[index].__name__ + raise ValueError(f"Missing required type argument: {name!r}") + + return tuple(args) + + +@overload +@dataclass_transform(field_specifiers=(dataclasses.field, Invisible)) def template( - *args: str | TemplateTypeVar, **kwargs: str | TemplateTypeVar + cls: str | TemplateTypeVar | TypeVar | None = None, + *args: str | TemplateTypeVar | type[_TemplateModelT] | TypeVar, + **kwargs: str | TemplateTypeVar | TypeVar, +) -> Callable[[type[_TemplateModelT]], type[_TemplateModelT]]: ... +@overload +@dataclass_transform(field_specifiers=(dataclasses.field, Invisible)) +def template( + cls: type[_TemplateModelT], + *args: str | TemplateTypeVar | type[_TemplateModelT] | TypeVar, + **kwargs: str | TemplateTypeVar | TypeVar, +) -> type[_TemplateModelT]: ... +@dataclass_transform(field_specifiers=(dataclasses.field, Invisible)) +def template( + cls: str | TemplateTypeVar | type[_TemplateModelT] | TypeVar | None = None, + *args: str | TemplateTypeVar | type[_TemplateModelT] | TypeVar, + **kwargs: str | TemplateTypeVar | TypeVar, ) -> Callable[[type[_TemplateModelT]], type[_TemplateModelT]]: """ Defines required template type variables if necessary and prepares @@ -163,6 +406,12 @@ def template( :return: a wrapper function that will be called with the class instance :rtype: Callable[[type], type] """ + if isinstance(cls, type): + return _create_generic_template(cls) + + if cls is not None: + args = (cls,) + args + if len(args) == 0 and len(kwargs) == 0: raise ValueError("Template class needs at least one template type var") @@ -187,7 +436,7 @@ def template( setattr(module, var.name, var) disposable.append(var.name) - for name, value in kwargs: + for name, value in kwargs.items(): # ellipsis indicates no default value if isinstance(value, _GreedyType): value = None @@ -200,7 +449,7 @@ def template( # the class will get special attributes that identify it as # a template class def create_template_class(cls: type[_TemplateModelT]) -> type[_TemplateModelT]: - cls.__annotations__ = inspect.get_annotations(cls) + cls.__annotations__ = inspect.get_annotations(cls, eval_str=True) for name in disposable: # Only temporary template vars will be removed delattr(module, name) @@ -214,12 +463,87 @@ def get_mangled_name(model_ty: type, annotations: dict[str, Any]) -> str: ty_name = model_ty.__name__ parts: list[str] = [] for name, value in annotations.items(): - parts.append(str(hash(f"{name}{value!r}"))) + parts.append(str(f"{name}{value!r}")) - hex_name = format(hash("".join(parts)), "X").replace("-", "_") + hex_name = md5("".join(parts).encode()).hexdigest() return f"_{hex_name}{ty_name}" +def _derive_generic_template( + template_ty: type, + *tys_args: TypeVar | str, + partial: bool = False, + allow_partial: bool = False, + name: str | _GreedyType | None = None, + union: bool = False, + module_name: str | None = None, + **tys_kwargs: Any, +) -> type: + info: GenericTemplateInfo = getattr(template_ty, ATTR_TEMPLATE) + args = _collect_generic_args( + info, + tys_args, + tys_kwargs, + partial=partial or allow_partial, + ) + remaining = tuple(arg for arg in args if _is_typevar(arg)) + is_partial = bool(remaining) + if is_partial and not (partial or allow_partial): + raise ValueError(f"Missing required type argument: {remaining[0].__name__!r}") + + key = tuple(repr(arg) for arg in args) + cached = info.cache.get(key) + if cached is not None and name is None and not union: + return cached + should_cache = name is None and not union + + if isinstance(name, _GreedyType): + name = None + + class_name = name or _generic_name(template_ty, args, is_partial) + module = module_name or template_ty.__module__ + bindings = dict(zip(info.parameters, args)) + annotations = inspect.get_annotations(template_ty, eval_str=True) + new_annotations = { + field_name: _specialize_generic_annotation( + annotation, + bindings, + getattr(template_ty, field_name, INVALID_DEFAULT), + ) + for field_name, annotation in annotations.items() + } + + def body(namespace: dict[str, Any]) -> None: + namespace["__module__"] = module + namespace["__annotations__"] = new_annotations + namespace["__origin__"] = template_ty + namespace["__args__"] = args + + bases = tuple(base for base in template_ty.__bases__ if base is not Generic) + if remaining: + bases = bases + ( + Generic[remaining[0]] if len(remaining) == 1 else Generic[remaining], + ) + bases = bases or (object,) + + new_ty = types.new_class( + class_name, + bases, + {}, + body, + ) + if is_partial: + _create_generic_template(new_ty, remaining) + if should_cache: + info.cache[key] = new_ty + return new_ty + + struct_ty = Struct(new_ty, options={} if not union else {S_UNION}).model + if should_cache: + info.cache[key] = struct_ty + return struct_ty + + def derive( template_ty: type, *tys_args: _StructLike, @@ -239,17 +563,43 @@ def derive( :return: the derived type :rtype: type """ - if len(tys_args) == 0 and len(tys_kwargs) == 0: - raise ValueError( - ( - "To derive a class from a template class at least one " - "type argument must be given!" - ) - ) + if hasstruct(template_ty) and not tys_args and not tys_kwargs: + return template_ty + if not istemplate(template_ty): raise TypeError(f"{template_ty.__name__} is not a template class!") info: TemplateInfo = getattr(template_ty, ATTR_TEMPLATE) + if isinstance(info, GenericTemplateInfo): + if isinstance(name, _GreedyType): + try: + frame = sys._getframe(1) + context = inspect.getframeinfo(frame).code_context[0] + if context.count("=") != 0: + parts = context.split(" = ") + if len(parts) >= 2: + name = parts[0] + except (AttributeError, KeyError, IndexError, TypeError): + pass + return _derive_generic_template( + template_ty, + *tys_args, + partial=partial, + name=name, + union=union, + module_name=get_caller_module(2), + **tys_kwargs, + ) + + if len(tys_args) == 0 and len(tys_kwargs) == 0: + has_defaults = any(value is not None for value in info.positional_tys.values()) + if not has_defaults: + raise ValueError( + ( + "To derive a class from a template class at least one " + "type argument must be given!" + ) + ) if len(tys_args) > len(info.required_tys): raise ValueError( f"Expected max. {len(info.required_tys)} positional arguments - got {len(tys_args)}!" diff --git a/src/caterpillar/py.py b/src/caterpillar/py.py index d24fabf3..fade08f6 100644 --- a/src/caterpillar/py.py +++ b/src/caterpillar/py.py @@ -136,71 +136,204 @@ ) __all__ = [ - "ExprMixin", - "WithoutContextVar", "AARCH64", + "Action", + "Adler_Algo", + "Adler_Field", + "Adler", + "Algorithm", + "align", + "Aligned", "AMD", "AMD64", + "And", + "annotation_registry", + "Arch", "ARM", "ARM64", - "Arch", + "AsLengthRef", + "ATTR_ACTION_PACK", + "ATTR_ACTION_UNPACK", + "ATTR_BITS", + "ATTR_BYTEORDER", + "ATTR_PACK", + "ATTR_SIGNED", + "ATTR_STRUCT", + "ATTR_TEMPLATE", + "ATTR_TYPE", + "ATTR_UNPACK", + "B_GROUP_END", + "B_GROUP_KEEP", + "B_GROUP_NEW", + "B_NO_AUTO_BOOL", + "B_OVERWRITE_ALIGNMENT", "BigEndian", - "ByteOrder", - "Dynamic", - "DynByteOrder", - "LittleEndian", - "MIPS", - "MIPS64", - "Native", - "NetEndian", - "PowerPC", - "PowerPC64", - "RISC_V", - "RISC_V64", - "SPARC", - "SPARC64", - "SysNative", - "system_arch", - "x86", - "x86_64", "BinaryExpression", + "bitfield_factory", + "bitfield", + "Bitfield", + "BitfieldEntry", + "BitfieldGroup", + "BitfieldValueFactory", + "boolean", + "Branch", + "ByteOrder", + "Bytes", + "Bz2Compressed", + "Chain", + "char", + "CharFactory", + "Compressed", + "Computed", + "ConditionalChain", + "ConditionContext", + "Const", + "ConstBytes", + "ConstString", + "constval", + "Context", + "ContextLength", + "ContextPath", + "Crc32_Algo", + "Crc32_Field", + "Crc32", + "CString", "CTX_ARCH", + "CTX_DIGEST_ALGO", + "CTX_DIGEST_HOOK", + "CTX_DIGEST_OBJ", + "CTX_DIGEST", "CTX_FIELD", "CTX_INDEX", "CTX_OBJECT", "CTX_OFFSETS", + "CTX_ORDER", "CTX_PARENT", "CTX_PATH", "CTX_POS", + "CTX_ROOT", "CTX_SEQ", "CTX_STREAM", "CTX_VALUE", - "ConditionContext", - "Context", - "ContextLength", - "ContextPath", - "UnaryExpression", "ctx", - "parent", - "this", + "DEFAULT_ALIGNMENT", + "DEFAULT_OPTION", "DelegationError", + "derive", + "Digest", + "DigestField", + "DigestFieldAction", + "double", + "Dynamic", "DynamicSizeError", - "InvalidValueError", - "OptionError", - "Stop", - "StreamError", - "StructException", - "ValidationError", + "DynByteOrder", + "Else", + "ElseIf", + "Encrypted", + "End", + "EndGroup", + "ENUM_STRICT", + "Enum", + "EnumFactory", + "ExprMixin", "F_DYNAMIC", "F_KEEP_POSITION", "F_OFFSET_OVERRIDE", "F_SEQUENTIAL", + "f", + "field_of", + "Field", + "FieldMixin", + "FieldStruct", "Flag", + "float16", + "float32", + "float64", + "get_args", + "get_flag", + "get_flags", + "get_kwargs", + "getbits", + "getstruct", "GLOBAL_BITFIELD_FLAGS", "GLOBAL_FIELD_FLAGS", "GLOBAL_STRUCT_OPTIONS", "GLOBAL_UNION_OPTIONS", + "has_flag", + "hasstruct", + "HMAC", + "HMACAlgorithm", + "If", + "Int", + "int16", + "int24", + "int32", + "int64", + "int8", + "intptr_fn", + "intptr", + "INVALID_DEFAULT", + "InvalidValueError", + "Invisible", + "IOHook", + "IPv4Address", + "IPv6Address", + "iseof", + "issigned", + "istemplate", + "KeyCipher", + "Lazy", + "LittleEndian", + "LZMACompressed", + "LZOCompressed", + "MAC", + "MACAddress", + "Md5_Algo", + "Md5_Field", + "Md5", + "Memory", + "MIPS", + "MIPS64", + "MODE_PACK", + "MODE_UNPACK", + "Native", + "NetEndian", + "NewGroup", "O_ARRAY_FACTORY", + "O_CONTEXT_FACTORY", + "offintptr", + "offuintptr", + "Operator", + "OptionError", + "Or", + "Otherwise", + "pack_file", + "pack_into", + "pack_seq", + "pack", + "Padded", + "padding", + "Padding", + "parent", + "parentctx", + "Pass", + "pointer", + "Pointer", + "PostPad", + "PowerPC", + "PowerPC64", + "Prefixed", + "PrePad", + "psize", + "pssize", + "PTR_STRICT", + "PyStructFormattedField", + "relative_pointer", + "RelativePointer", + "RemoveField", + "RISC_V", + "RISC_V64", + "root", "S_ADD_BYTES", "S_DISCARD_CONST", "S_DISCARD_UNNAMED", @@ -208,209 +341,86 @@ "S_REPLACE_TYPES", "S_SLOTS", "S_UNION", - "get_flag", - "get_flags", - "has_flag", + "Sequence", "set_field_flags", "set_struct_flags", "set_union_flags", - "TypeConverter", - "annotation_registry", - "to_struct", - "ATTR_ACTION_PACK", - "ATTR_STRUCT", - "Action", - "iseof", - "pack_seq", - "unpack_seq", - "ATTR_ACTION_UNPACK", - "ATTR_BITS", - "ATTR_BYTEORDER", - "ATTR_SIGNED", - "ATTR_TEMPLATE", - "ATTR_TYPE", - "getstruct", - "hasstruct", - "MODE_PACK", - "MODE_UNPACK", - "typeof", - "Sequence", - "RemoveField", - "Struct", - "struct", - "UnionHook", - "union", - "unpack", - "unpack_file", - "pack", - "pack_into", - "pack_file", - "sizeof", - "Bitfield", - "bitfield", - "BitfieldGroup", - "issigned", - "getbits", - "istemplate", - "template", - "TemplateTypeVar", - "derive", - "NewGroup", - "EndGroup", "SetAlignment", - "BitfieldEntry", - "BitfieldValueFactory", - "EnumFactory", - "CharFactory", - "DEFAULT_ALIGNMENT", - "Digest", - "Algorithm", - "Md5", - "Sha1", - "Sha2_256", - "Sha2_224", - "Sha2_384", - "Sha2_512", - "Sha3_224", - "Sha3_256", - "Sha3_384", - "Sha3_512", - "Crc32", - "Adler", - "HMAC", - "DigestField", - "DigestFieldAction", - "Md5_Algo", - "Md5_Field", + "SetContextVar", "Sha1_Algo", "Sha1_Field", - "Sha2_256_Algo", - "Sha2_256_Field", + "Sha1", "Sha2_224_Algo", "Sha2_224_Field", + "Sha2_224", + "Sha2_256_Algo", + "Sha2_256_Field", + "Sha2_256", "Sha2_384_Algo", "Sha2_384_Field", + "Sha2_384", "Sha2_512_Algo", "Sha2_512_Field", + "Sha2_512", "Sha3_224_Algo", "Sha3_224_Field", + "Sha3_224", "Sha3_256_Algo", "Sha3_256_Field", + "Sha3_256", "Sha3_384_Algo", "Sha3_384_Field", + "Sha3_384", "Sha3_512_Algo", "Sha3_512_Field", - "Crc32_Algo", - "Crc32_Field", - "Adler_Algo", - "Adler_Field", - "CTX_DIGEST", - "CTX_DIGEST_ALGO", - "CTX_DIGEST_HOOK", - "CTX_DIGEST_OBJ", - "HMACAlgorithm", - "uintptr", - "intptr", - "offintptr", - "offuintptr", - "Pointer", - "pointer", - "intptr_fn", - "PTR_STRICT", - "relative_pointer", - "RelativePointer", - "uintptr_fn", - "Compressed", - "ZLibCompressed", - "Bz2Compressed", - "LZMACompressed", - "LZOCompressed", - "PyStructFormattedField", - "Transformer", - "Const", - "ConstBytes", - "ConstString", - "Enum", + "Sha3_512", + "singleton", + "sizeof", + "SPARC", + "SPARC64", + "Start", + "Stop", + "StreamError", "String", - "Bytes", - "Memory", - "Computed", - "Pass", - "CString", - "Prefixed", - "Int", + "struct_factory", + "struct", + "Struct", + "StructDefMixin", + "StructException", + "SysNative", + "system_arch", + "template", + "TemplateFieldRef", + "TemplateTypeVar", + "this", + "to_struct", + "Transformer", + "TypeConverter", + "typeof", "UInt", - "padding", - "char", - "boolean", - "int8", - "uint8", - "int16", "uint16", - "int24", "uint24", - "int32", "uint32", - "int64", "uint64", - "pssize", - "psize", - "float16", - "float32", - "float64", - "double", - "void_ptr", + "uint8", + "uintptr_fn", + "uintptr", + "UnaryExpression", + "union", + "UnionHook", + "unpack_file", + "unpack_seq", + "unpack", "Uuid", - "Aligned", - "align", - "Lazy", - "ENUM_STRICT", - "Field", - "INVALID_DEFAULT", - "DEFAULT_OPTION", - "singleton", - "FieldMixin", - "FieldStruct", - "Chain", - "Operator", - "get_args", - "get_kwargs", - "VarInt", + "ValidationError", "VARINT_LSB", + "VarInt", "vint", - "Encrypted", + "void_ptr", + "When", + "WithoutContextVar", + "x86_64", + "x86", "Xor", - "Or", - "And", - "KeyCipher", - "MAC", - "MACAddress", - "IPv4Address", - "IPv6Address", - "ConditionalChain", - "If", - "Else", - "ElseIf", - "IOHook", - "CTX_ROOT", - "CTX_ORDER", - "root", - "B_GROUP_END", - "B_GROUP_KEEP", - "B_GROUP_NEW", - "B_NO_AUTO_BOOL", - "B_OVERWRITE_ALIGNMENT", - "constval", - "f", - "Padding", - "Invisible", - "O_CONTEXT_FACTORY", - "SetContextVar", - "ATTR_PACK", - "ATTR_UNPACK", - "StructDefMixin", - "AsLengthRef", - "struct_factory", - "parentctx", - "bitfield_factory", + "ZLibCompressed", ] diff --git a/src/caterpillar/shared.py b/src/caterpillar/shared.py index 1374f4b0..986a8b60 100644 --- a/src/caterpillar/shared.py +++ b/src/caterpillar/shared.py @@ -142,6 +142,24 @@ .. versionadded:: 2.4.0 """ +ATTR_CONDITIONAL: Final[str] = "__conditional__" +"""Attribute indicating a conditional object. + +.. versionadded:: 2.9.0 +""" + +ATTR_CONDITIONAL_START: Final[str] = "__conditional_start__" +"""Attribute indicating the start of a conditional block. + +.. versionadded:: 2.9.0 +""" + +ATTR_CONDITIONAL_END: Final[str] = "__conditional_end__" +"""Attribute indicating then end of a conditional block. + +.. versionadded:: 2.9.0 +""" + def constval(value: _OT) -> "_ContextLambda[_OT]": """Returns a lambda that returns a constant value when invoked. @@ -261,6 +279,18 @@ def hasstruct(obj: object) -> TypeIs[_ContainsStruct]: return hasattr(obj.__class__ if not isinstance(obj, type) else obj, ATTR_STRUCT) +def iscond(obj: object) -> bool: + return getattr(obj, ATTR_CONDITIONAL, False) is True + + +def iscondstart(obj: object) -> bool: + return getattr(obj, ATTR_CONDITIONAL_START, False) is True + + +def iscondend(obj: object) -> bool: + return getattr(obj, ATTR_CONDITIONAL_END, False) is True + + @overload def getstruct( obj: type[_IT], @@ -492,10 +522,12 @@ def from_file( """ from caterpillar.model import unpack_file - return unpack_file( # pyright: ignore[reportCallIssue, reportUnknownVariableType] - self, # pyright: ignore[reportArgumentType] - filename, - order=order, - arch=arch, - **kwargs, + return ( + unpack_file( # pyright: ignore[reportCallIssue, reportUnknownVariableType] + self, # pyright: ignore[reportArgumentType] + filename, + order=order, + arch=arch, + **kwargs, + ) ) diff --git a/src/ccaterpillar/arch.c b/src/ccaterpillar/arch.c index 35774861..312ab12a 100644 --- a/src/ccaterpillar/arch.c +++ b/src/ccaterpillar/arch.c @@ -16,6 +16,7 @@ cp_arch_new(PyTypeObject* type, PyObject* args, PyObject* kw) self->name = PyUnicode_FromString(""); if (!self->name) { + Py_DECREF(self); return NULL; } self->pointer_size = 0; @@ -59,7 +60,11 @@ cp_arch_repr(CpArchObject* self) static PyObject* cp_arch_richcmp(CpArchObject* self, PyObject* other, int op) { - if (!PyObject_IsInstance(other, (PyObject*)&CpArch_Type)) { + int is_instance = PyObject_IsInstance(other, (PyObject*)&CpArch_Type); + if (is_instance < 0) { + return NULL; + } + if (!is_instance) { Py_RETURN_NOTIMPLEMENTED; } return PyObject_RichCompare(self->name, ((CpArchObject*)other)->name, op); @@ -135,6 +140,7 @@ cp_endian_new(PyTypeObject* type, PyObject* args, PyObject* kw) self->name = PyUnicode_FromString(""); if (!self->name) { + Py_DECREF(self); return NULL; } self->id = 0; @@ -189,7 +195,11 @@ cp_endian_repr(CpEndianObject* self) static PyObject* cp_endian_richcmp(CpEndianObject* self, PyObject* other, int op) { - if (!PyObject_IsInstance(other, (PyObject*)&CpEndian_Type)) { + int is_instance = PyObject_IsInstance(other, (PyObject*)&CpEndian_Type); + if (is_instance < 0) { + return NULL; + } + if (!is_instance) { Py_RETURN_NOTIMPLEMENTED; } @@ -290,4 +300,4 @@ cp_arch__mod_init(PyObject* m, _modulestate* state) CpModuleState_AddObject( cp_arch__host, "HOST_ARCH", -1, CpArch_New("", sizeof(void*) * 8)); return 0; -} \ No newline at end of file +} diff --git a/src/ccaterpillar/atom.c b/src/ccaterpillar/atom.c index aac702e4..81b29521 100644 --- a/src/ccaterpillar/atom.c +++ b/src/ccaterpillar/atom.c @@ -138,14 +138,14 @@ static PyObject* cp_atom_type(CpAtomObject* self) { return self->ob_type ? self->ob_type(_Cp_CAST(PyObject*, self)) - : Py_NotImplemented; + : Py_NewRef(Py_NotImplemented); } static PyObject* cp_atom_bits(CpAtomObject* self) { return self->ob_bits ? self->ob_bits(_Cp_CAST(PyObject*, self)) - : Py_NotImplemented; + : Py_NewRef(Py_NotImplemented); } static PyObject* @@ -239,9 +239,10 @@ CpAtom_PackMany(PyObject* pAtom, if (CpAtom_Check(pAtom)) { func = _Cp_CAST(CpAtomObject*, pAtom)->ob_pack_many; if (!func) { - PyErr_Format(PyExc_NotImplementedError, - "The atom of type '%s' cannot be packed (missing __pack__)", - Py_TYPE(pAtom)->tp_name); + PyErr_Format( + PyExc_NotImplementedError, + "The atom of type '%s' cannot be packed (missing __pack_many__)", + Py_TYPE(pAtom)->tp_name); return -1; } return func(pAtom, pObj, pContext, pLengthInfo); @@ -249,7 +250,7 @@ CpAtom_PackMany(PyObject* pAtom, state = get_global_module_state(); nResult = PyObject_CallMethodObjArgs( - pAtom, state->str__pack_many, pObj, pContext, NULL); + pAtom, state->str__pack_many, pObj, pContext, pLengthInfo, NULL); if (!nResult) { return -1; } @@ -303,7 +304,7 @@ CpAtom_UnpackMany(PyObject* pAtom, PyObject* pContext, PyObject* pLengthInfo) } else { state = get_global_module_state(); nResult = PyObject_CallMethodObjArgs( - pAtom, state->str__unpack_many, pContext, NULL); + pAtom, state->str__unpack_many, pContext, pLengthInfo, NULL); } return nResult; } @@ -313,7 +314,7 @@ PyObject* CpAtom_BitsOf(PyObject* pAtom) { _modulestate* state = NULL; - PyObject* nResult = NULL; + PyObject *nResult = NULL, *nBits = NULL; bitsfunc func = NULL; if (CpAtom_Check(pAtom)) { @@ -327,9 +328,18 @@ CpAtom_BitsOf(PyObject* pAtom) nResult = func(pAtom); } else { state = get_global_module_state(); - nResult = PyObject_CallMethodObjArgs(pAtom, state->str__bits, NULL); + _Cp_AssignCheck(nBits, PyObject_GetAttr(pAtom, state->str__bits), error); + if (PyCallable_Check(nBits)) { + nResult = PyObject_CallNoArgs(nBits); + Py_DECREF(nBits); + } else { + nResult = nBits; + } } return nResult; + +error: + return NULL; } /*CpAPI*/ @@ -430,4 +440,4 @@ cp_atom__mod_init(PyObject* m, _modulestate* state) _CACHED_STRING(state, str__bits, CpAtom_Bits_STR, -1); _CACHED_STRING(state, str__struct, "__struct__", -1); return 0; -} \ No newline at end of file +} diff --git a/src/ccaterpillar/atoms/builtin/conditional.c b/src/ccaterpillar/atoms/builtin/conditional.c index 34c81ff0..4fc0f5fa 100644 --- a/src/ccaterpillar/atoms/builtin/conditional.c +++ b/src/ccaterpillar/atoms/builtin/conditional.c @@ -39,13 +39,19 @@ cp_conditionalatom_init(CpConditionalAtomObject* self, PyObject* kw) { static char* kwlist[] = { "atom", "condition", NULL }; - PyObject *atom = NULL, *condition = NULL; + PyObject *atom = NULL, *condition = NULL, *nAtom = NULL; if (!PyArg_ParseTupleAndKeywords(args, kw, "OO", kwlist, &atom, &condition)) { return -1; } - _Cp_SetObj(self->m_atom, atom); + _Cp_AssignCheck(nAtom, Cp_GetStruct(atom), error); + Py_XSETREF(self->m_atom, nAtom); + nAtom = NULL; _Cp_SetObj(self->m_condition, condition); return 0; + +error: + Py_XDECREF(nAtom); + return -1; } _CpEndian_ImplSetByteorder(CpConditionalAtomObject, @@ -75,7 +81,7 @@ cp_conditionalatom_eval_with_context(PyObject* self, return NULL; } - return result ? Py_True : Py_False; + return PyBool_FromLong(result); } /*Public API*/ @@ -137,7 +143,7 @@ CpConditionalAtom_Unpack(PyObject* pAtom, PyObject* pContext) return enabled ? CpAtom_Unpack(_Cp_CAST(CpConditionalAtomObject*, pAtom)->m_atom, pContext) - : Py_None; + : Py_NewRef(Py_None); } /*CpAPI*/ @@ -228,4 +234,4 @@ cp_conditional__mod_init(PyObject* m, _modulestate* state) { CpModule_AddObject(CpConditionalAtom_NAME, &CpConditionalAtom_Type, -1); return 0; -} \ No newline at end of file +} diff --git a/src/ccaterpillar/atoms/builtin/offset.c b/src/ccaterpillar/atoms/builtin/offset.c index 88cee078..41c147ee 100644 --- a/src/ccaterpillar/atoms/builtin/offset.c +++ b/src/ccaterpillar/atoms/builtin/offset.c @@ -40,25 +40,31 @@ static int cp_offsetatom_init(CpOffsetAtomObject* self, PyObject* args, PyObject* kw) { static char* kwlist[] = { "atom", "offset", "whence", "keep_pos", NULL }; - PyObject *bAtom = NULL, *bOffset = NULL; + PyObject *bAtom = NULL, *bOffset = NULL, *nAtom = NULL, *nWhence = NULL; int whence = PY_SEEK_SET, keepPos = false; if (!PyArg_ParseTupleAndKeywords( args, kw, "OO|ip", kwlist, &bAtom, &bOffset, &whence, &keepPos)) { return -1; } - _Cp_SetObj(self->m_atom, bAtom); + _Cp_AssignCheck(nAtom, Cp_GetStruct(bAtom), error); + Py_XSETREF(self->m_atom, nAtom); + nAtom = NULL; _Cp_SetObj(self->m_offset, bOffset); if (whence < PY_SEEK_SET || whence > PY_SEEK_END) { PyErr_SetString(PyExc_ValueError, "invalid whence"); goto error; } - _Cp_AssignCheck(self->m_whence, PyLong_FromLong(whence), error); + _Cp_AssignCheck(nWhence, PyLong_FromLong(whence), error); + Py_XSETREF(self->m_whence, nWhence); + nWhence = NULL; self->s_keep_pos = keepPos; self->s_is_number = PyNumber_Check(self->m_offset); return 0; error: + Py_XDECREF(nAtom); + Py_XDECREF(nWhence); return -1; } @@ -140,6 +146,7 @@ CpOffsetAtom_EvalOffset(PyObject* pAtom, PyObject* pContext) Py_CLEAR(nResult); success: + Py_XDECREF(nOffset); return nResult; } @@ -148,7 +155,10 @@ int CpOffsetAtom_Pack(PyObject* pAtom, PyObject* pObj, PyObject* pContext) { PyObject *nFallbackOffset = NULL, *nTmp = NULL, *nIO = NULL, *nTmpIO = NULL, - *nOffset = NULL, *nRoot = NULL, *nOffsets = NULL, *nBuffer = NULL; + *nOffset = NULL, *nRoot = NULL, *nOffsets = NULL, *nBuffer = NULL, + *nTargetOffset = NULL, *nSeekSet = NULL, *nKeepOffset = NULL, + *nBufferLen = NULL; + Py_ssize_t bufferLen = 0; int result = 0; CpOffsetAtomObject* self = _Cp_CAST(CpOffsetAtomObject*, pAtom); _modulestate* state = get_global_module_state(); @@ -157,8 +167,8 @@ CpOffsetAtom_Pack(PyObject* pAtom, PyObject* pObj, PyObject* pContext) _Cp_AssignCheck(nFallbackOffset, CpContextIO_Tell(pContext), error); _Cp_AssignCheck(nOffset, CpOffsetAtom_EvalOffset(pAtom, pContext), error); _Cp_AssignCheck( - nTmp, CpContextIO_Seek(pContext, nOffset, self->m_whence), error); - Py_CLEAR(nTmp); + nTargetOffset, CpContextIO_Seek(pContext, nOffset, self->m_whence), error); + _Cp_AssignCheck(nSeekSet, PyLong_FromLong(PY_SEEK_SET), error); _Cp_AssignCheck(nIO, CpContext_IO(pContext, state), error); _Cp_AssignCheck(nTmpIO, CpObject_CreateNoArgs(CpBytesIO_Type), error); @@ -174,21 +184,34 @@ CpOffsetAtom_Pack(PyObject* pAtom, PyObject* pObj, PyObject* pContext) // finally, seek back _Cp_AssignCheck( - nTmp, CpContextIO_Seek(pContext, nFallbackOffset, self->m_whence), error); - Py_CLEAR(nTmp); - - _Cp_AssignCheck(nRoot, CpContext_GetRoot(pContext), error); + nRoot, CpContext_GetRoot(pContext), error); _Cp_AssignCheck( nOffsets, CpContext_ITEM(nRoot, state->str__context_offsets), error); _Cp_AssignCheck( nBuffer, PyObject_CallMethodNoArgs(nTmpIO, state->str__io_getvalue), error); - if (PyObject_SetItem(nOffsets, nOffset, nBuffer) < 0) { + if (PyObject_SetItem(nOffsets, nTargetOffset, nBuffer) < 0) { goto error; } if (CpContext_SETIO(pContext, state, nIO) < 0) { goto error; } + + if (self->s_keep_pos) { + bufferLen = PyObject_Length(nBuffer); + if (bufferLen < 0) { + goto error; + } + _Cp_AssignCheck(nBufferLen, PyLong_FromSsize_t(bufferLen), error); + _Cp_AssignCheck(nKeepOffset, + PyNumber_Add(nTargetOffset, nBufferLen), + error); + _Cp_AssignCheck( + nTmp, CpContextIO_Seek(pContext, nKeepOffset, nSeekSet), error); + } else { + _Cp_AssignCheck( + nTmp, CpContextIO_Seek(pContext, nFallbackOffset, nSeekSet), error); + } goto success; error: @@ -203,6 +226,10 @@ CpOffsetAtom_Pack(PyObject* pAtom, PyObject* pObj, PyObject* pContext) Py_XDECREF(nRoot); Py_XDECREF(nOffsets); Py_XDECREF(nBuffer); + Py_XDECREF(nTargetOffset); + Py_XDECREF(nSeekSet); + Py_XDECREF(nKeepOffset); + Py_XDECREF(nBufferLen); return result; } @@ -211,18 +238,21 @@ PyObject* CpOffsetAtom_Unpack(PyObject* pAtom, PyObject* pContext) { PyObject *nResult = NULL, *nOffset = NULL, *nTmp = NULL, - *nFallbackOffset = NULL; + *nFallbackOffset = NULL, *nSeekSet = NULL; CpOffsetAtomObject* self = _Cp_CAST(CpOffsetAtomObject*, pAtom); _Cp_AssignCheck(nFallbackOffset, CpContextIO_Tell(pContext), error); _Cp_AssignCheck(nOffset, CpOffsetAtom_EvalOffset(pAtom, pContext), error); + _Cp_AssignCheck(nSeekSet, PyLong_FromLong(PY_SEEK_SET), error); _Cp_AssignCheck( nTmp, CpContextIO_Seek(pContext, nOffset, self->m_whence), error); Py_CLEAR(nTmp); _Cp_AssignCheck(nResult, CpAtom_Unpack(self->m_atom, pContext), error); - _Cp_AssignCheck( - nTmp, CpContextIO_Seek(pContext, nFallbackOffset, self->m_whence), error); + if (!self->s_keep_pos) { + _Cp_AssignCheck( + nTmp, CpContextIO_Seek(pContext, nFallbackOffset, nSeekSet), error); + } goto success; error: @@ -232,6 +262,7 @@ CpOffsetAtom_Unpack(PyObject* pAtom, PyObject* pContext) Py_XDECREF(nTmp); Py_XDECREF(nOffset); Py_XDECREF(nFallbackOffset); + Py_XDECREF(nSeekSet); return nResult; } @@ -244,6 +275,7 @@ CpOffsetAtom_TypeOf(PyObject* pAtom) /*type*/ static PyMemberDef CpOffsetAtom_Members[] = { + { "atom", T_OBJECT, offsetof(CpOffsetAtomObject, m_atom), READONLY }, { "offset", T_OBJECT, offsetof(CpOffsetAtomObject, m_offset), 0 }, { "whence", T_OBJECT, offsetof(CpOffsetAtomObject, m_whence), 0 }, { "is_number", T_BOOL, offsetof(CpOffsetAtomObject, s_is_number), READONLY }, @@ -305,4 +337,4 @@ cp_offset__mod_init(PyObject* m, _modulestate* state) { CpModule_AddObject(CpOffsetAtom_NAME, &CpOffsetAtom_Type, -1); return 0; -} \ No newline at end of file +} diff --git a/src/ccaterpillar/atoms/builtin/repeated.c b/src/ccaterpillar/atoms/builtin/repeated.c index ae118d7a..9f0eacf2 100644 --- a/src/ccaterpillar/atoms/builtin/repeated.c +++ b/src/ccaterpillar/atoms/builtin/repeated.c @@ -18,12 +18,16 @@ cp_repeatedatom_new(PyTypeObject* type, PyObject* args, PyObject* kw) CpBuiltinAtom_ATOM(self).ob_unpack_many = NULL; CpBuiltinAtom_ATOM(self).ob_type = CpRepeatedAtom_TypeOf; CpBuiltinAtom_ATOM(self).ob_size = CpRepeatedAtom_Size; + self->m_atom = NULL; + self->m_length = NULL; return _Cp_CAST(PyObject*, self); } static void cp_repeatedatom_dealloc(CpRepeatedAtomObject* self) { + Py_CLEAR(self->m_atom); + Py_CLEAR(self->m_length); Py_TYPE(self)->tp_free((PyObject*)self); } @@ -31,15 +35,21 @@ static int cp_repeatedatom_init(CpRepeatedAtomObject* self, PyObject* args, PyObject* kw) { static char* kwlist[] = { "atom", "length", NULL }; - PyObject *atom = NULL, *length = NULL; + PyObject *atom = NULL, *length = NULL, *nAtom = NULL; if (!PyArg_ParseTupleAndKeywords(args, kw, "OO", kwlist, &atom, &length)) { return -1; } - _Cp_SetObj(self->m_atom, atom); + _Cp_AssignCheck(nAtom, Cp_GetStruct(atom), error); + Py_XSETREF(self->m_atom, nAtom); + nAtom = NULL; _Cp_SetObj(self->m_length, length); return 0; + +error: + Py_XDECREF(nAtom); + return -1; } static PyObject* @@ -236,11 +246,7 @@ CpRepeatedAtom_TypeOf(PyObject* pAtom) PyObject* CpRepeatedAtom_Bits(PyObject* pAtom) { - PyObject *nResult = NULL, *nLength = NULL, *nAtomBits = NULL, - *nBitsSize = PyLong_FromLong(8); - if (!nBitsSize) { - goto error; - } + PyObject *nResult = NULL, *nLength = NULL, *nAtomBits = NULL; _Cp_AssignCheck(nLength, CpRepeatedAtom_GetLength(pAtom, NULL), error); if (!PyNumber_Check(nLength)) { PyErr_SetString(PyExc_ValueError, "length is not a number!"); @@ -251,10 +257,6 @@ CpRepeatedAtom_Bits(PyObject* pAtom) CpAtom_BitsOf(_Cp_CAST(CpRepeatedAtomObject*, pAtom)->m_atom), error); _Cp_AssignCheck(nResult, PyNumber_Multiply(nLength, nAtomBits), error); - Py_XSETREF(nResult, PyNumber_Multiply(nResult, nBitsSize)); - if (!nResult) { - goto error; - } goto success; error: @@ -263,7 +265,6 @@ CpRepeatedAtom_Bits(PyObject* pAtom) success: Py_XDECREF(nLength); Py_XDECREF(nAtomBits); - Py_XDECREF(nBitsSize); return nResult; } @@ -349,6 +350,7 @@ CpRepeatedAtom_Pack(PyObject* pAtom, PyObject* pObj, PyObject* pContext) nRaisedException, PyExc_NotImplementedError))) { // Make sure this method continues to pack the given object PyErr_Clear(); + result = 0; } else { if (result < 0 && nRaisedException) { // This call steals a reference to exc, which must be a valid exception. @@ -381,8 +383,7 @@ CpRepeatedAtom_Pack(PyObject* pAtom, PyObject* pObj, PyObject* pContext) goto error; // _parent - if (CpContext_SETITEM( - nSeqContext, state->str__context_parent, Py_NewRef(pContext)) < 0) + if (CpContext_SETITEM(nSeqContext, state->str__context_parent, pContext) < 0) goto error; // _io @@ -391,9 +392,11 @@ CpRepeatedAtom_Pack(PyObject* pAtom, PyObject* pObj, PyObject* pContext) } // _length - if (CpContext_SETITEM(nSeqContext, - state->str__context_length, - CpLengthInfo_LengthAsLong(nLengthInfo)) < 0) + Py_XSETREF(nTmpObj, CpLengthInfo_LengthAsLong(nLengthInfo)); + if (!nTmpObj) { + goto error; + } + if (CpContext_SETITEM(nSeqContext, state->str__context_length, nTmpObj) < 0) goto error; // _field @@ -478,7 +481,7 @@ CpRepeatedAtom_Unpack(PyObject* pAtom, PyObject* pContext) bool hasUnpackMany = false; CpRepeatedAtomObject* self = _Cp_CAST(CpRepeatedAtomObject*, pAtom); - hasUnpackMany = CpAtom_HasPackMany(self->m_atom); + hasUnpackMany = CpAtom_HasUnpackMany(self->m_atom); _Cp_AssignCheck(nLength, CpRepeatedAtom_GetLength(pAtom, pContext), error); _Cp_AssignCheck(nLengthInfo, CpLengthInfo_New(0, false), error); if (_CpUnpack_EvalLength( @@ -503,6 +506,8 @@ CpRepeatedAtom_Unpack(PyObject* pAtom, PyObject* pContext) PyErr_SetRaisedException(nRaisedException); nRaisedException = NULL; } + nResult = nTmpObj; + nTmpObj = NULL; goto success; } } @@ -516,16 +521,23 @@ CpRepeatedAtom_Unpack(PyObject* pAtom, PyObject* pContext) nBasePath, CpContext_ITEM(pContext, state->str__context_path), error); // _root - CpContext_SETITEM( - nSeqContext, state->str__context_root, CpContext_GetRoot(pContext)); + Py_XSETREF(nTmpObj, CpContext_GetRoot(pContext)); + if (!nTmpObj || + CpContext_SETITEM(nSeqContext, state->str__context_root, nTmpObj) < 0) { + goto error; + } // _parent - CpContext_SETITEM(nSeqContext, state->str__context_parent, pContext); + if (CpContext_SETITEM(nSeqContext, state->str__context_parent, pContext) < 0) + goto error; // _length - CpContext_SETITEM(nSeqContext, state->str__context_length, nLength); + if (CpContext_SETITEM(nSeqContext, state->str__context_length, nLength) < 0) + goto error; // _obj - CpContext_COPYITEM(nSeqContext, pContext, state->str__context_obj); + if (CpContext_COPYITEM(nSeqContext, pContext, state->str__context_obj) < 0) + PyErr_Clear(); // _is_seq - CpContext_SETITEM(nSeqContext, state->str__context_is_seq, Py_False); + if (CpContext_SETITEM(nSeqContext, state->str__context_is_seq, Py_False) < 0) + goto error; // _field if (CpContext_COPYITEM(nSeqContext, pContext, state->str__context_field) < 0) { @@ -533,9 +545,11 @@ CpRepeatedAtom_Unpack(PyObject* pAtom, PyObject* pContext) PyErr_Clear(); } // _io - CpContext_COPYITEM(nSeqContext, pContext, state->str__context_io); + if (CpContext_COPYITEM(nSeqContext, pContext, state->str__context_io) < 0) + goto error; // _lst - CpContext_SETITEM(nSeqContext, state->str__context_list, nSeq); + if (CpContext_SETITEM(nSeqContext, state->str__context_list, nSeq) < 0) + goto error; while (CpLengthInfo_IsGreedy(nLengthInfo) || index < CpLengthInfo_Length(nLengthInfo)) { @@ -590,7 +604,7 @@ CpRepeatedAtom_Unpack(PyObject* pAtom, PyObject* pContext) Py_XDECREF(nTmpIndex); Py_XDECREF(nTmpObj); Py_XDECREF(nRaisedException); - // Py_XDECREF(nSeq); // new ref is stored in nResult + Py_XDECREF(nSeq); return nResult; } diff --git a/src/ccaterpillar/atoms/builtin/switch.c b/src/ccaterpillar/atoms/builtin/switch.c index 81165ac8..349da533 100644 --- a/src/ccaterpillar/atoms/builtin/switch.c +++ b/src/ccaterpillar/atoms/builtin/switch.c @@ -39,14 +39,20 @@ static int cp_switchatom_init(CpSwitchAtomObject* self, PyObject* args, PyObject* kw) { static char* kwlist[] = { "atom", "cases", NULL }; - PyObject *atom = NULL, *cases = NULL; + PyObject *atom = NULL, *cases = NULL, *nAtom = NULL; if (!PyArg_ParseTupleAndKeywords(args, kw, "OO", kwlist, &atom, &cases)) { return -1; } - _Cp_SetObj(self->m_atom, atom); + _Cp_AssignCheck(nAtom, Cp_GetStruct(atom), error); + Py_XSETREF(self->m_atom, nAtom); + nAtom = NULL; _Cp_SetObj(self->m_cases, cases); - self->s_callable = PyCallable_Check(atom); + self->s_callable = PyCallable_Check(self->m_atom); return 0; + +error: + Py_XDECREF(nAtom); + return -1; } _CpEndian_ImplSetByteorder(CpSwitchAtomObject, switchatom, self->m_atom); @@ -207,6 +213,9 @@ CpSwitchAtom_TypeOf(PyObject* pAtom) } Py_XSETREF(nResult, PyNumber_Or(nResult, nTmpAtomType)); + if (!nResult) { + goto error; + } } goto success; @@ -216,6 +225,8 @@ CpSwitchAtom_TypeOf(PyObject* pAtom) success: Py_XDECREF(nAtomType); Py_XDECREF(nCasesValues); + Py_XDECREF(nTmpAtomType); + Py_XDECREF(nTmpAtom); return nResult; } @@ -269,4 +280,4 @@ cp_switch__mod_init(PyObject* m, _modulestate* state) { CpModule_AddObject(CpSwitchAtom_NAME, &CpSwitchAtom_Type, -1); return 0; -} \ No newline at end of file +} diff --git a/src/ccaterpillar/context.c b/src/ccaterpillar/context.c index cf49da85..a006bb67 100644 --- a/src/ccaterpillar/context.c +++ b/src/ccaterpillar/context.c @@ -7,12 +7,15 @@ static int cp_context_init(CpContextObject* self, PyObject* args, PyObject* kw) { - return PyDict_Type.tp_init((PyObject*)&self->m_dict, args, kw) < 0; + return PyDict_Type.tp_init((PyObject*)&self->m_dict, args, kw); } static int cp_context__setattr__(CpContextObject* self, char* name, PyObject* value) { + if (!value) { + return PyDict_DelItemString((PyObject*)&self->m_dict, name); + } return PyDict_SetItemString((PyObject*)&self->m_dict, name, value); } @@ -106,7 +109,6 @@ PyObject* CpContext_GenericGetAttr(PyObject* context, PyObject* path) { _modulestate* state = get_global_module_state(); - PyObject* str = PyObject_Repr(path); // Names starting with 'n' contain a NEW reference to a // Python object, whereas variables starting with 'b' // store a borrowed reference. @@ -265,8 +267,7 @@ CpContext_GenericSetAttr(PyObject* pContext, PyObject* pPath, PyObject* pValue) // if length is one, set the attribute directly if (PyList_Size(nElemets) == 1) { - if (PyObject_SetItem( - pContext, pPath, pValue ? pValue : Py_NewRef(Py_None)) < 0) + if (PyObject_SetItem(pContext, pPath, pValue ? pValue : Py_None) < 0) goto error; } else { bNewPath = PyList_GetItem(nElemets, 0); @@ -281,8 +282,7 @@ CpContext_GenericSetAttr(PyObject* pContext, PyObject* pPath, PyObject* pValue) if (!bTarget) goto error; - if (PyObject_SetAttr(nObj, bTarget, pValue ? pValue : Py_NewRef(Py_None)) < - 0) + if (PyObject_SetAttr(nObj, bTarget, pValue ? pValue : Py_None) < 0) goto error; } goto success; @@ -534,4 +534,4 @@ cp_context__mod_clear(PyObject* m, _modulestate* state) Py_CLEAR(state->str__io_tell); Py_CLEAR(state->str__context_offsets); Py_CLEAR(state->str__io_getvalue); -} \ No newline at end of file +} diff --git a/src/ccaterpillar/lengthinfo.c b/src/ccaterpillar/lengthinfo.c index 67b75fc9..89f2eb5e 100644 --- a/src/ccaterpillar/lengthinfo.c +++ b/src/ccaterpillar/lengthinfo.c @@ -64,6 +64,7 @@ PyTypeObject CpLengthInfo_Type = { .tp_new = (newfunc)cp_lengthinfo_new, .tp_init = (initproc)cp_lengthinfo_init, .tp_repr = (reprfunc)cp_lengthinfo_repr, + .tp_members = CpLengthInfo_Members, .tp_doc = NULL, }; @@ -85,4 +86,4 @@ cp_lengthinfo__mod_init(PyObject* m, _modulestate* state) { CpModule_AddObject(CpLengthInfo_NAME, &CpLengthInfo_Type, -1); return 0; -} \ No newline at end of file +} diff --git a/src/ccaterpillar/module.c.in b/src/ccaterpillar/module.c.in index ef989195..d5553525 100644 --- a/src/ccaterpillar/module.c.in +++ b/src/ccaterpillar/module.c.in @@ -41,7 +41,7 @@ PyModuleDef CpModule = { PyMODINIT_FUNC PyInit__C(void) { - PyObject *m = NULL, *c_api = NULL, *nDict = NULL; + PyObject *m = NULL, *c_api = NULL; _modulestate* state = NULL; m = PyState_FindModule(&CpModule); @@ -77,13 +77,10 @@ PyInit__C(void) goto err; } - if ((nDict = PyDict_New(), !nDict)) { - goto err; - } - - if (PyDict_SetItemString(nDict, "_C_API", c_api) < 0) { + if (PyModule_AddObject(m, "_C_API", c_api) < 0) { goto err; } + c_api = NULL; goto success; err: @@ -93,10 +90,9 @@ err: Py_XSETREF(m, NULL); success: - Py_XDECREF(nDict); Py_XDECREF(c_api); return m; #undef SETUP_TYPES #undef ADD_OBJECTS -} \ No newline at end of file +} diff --git a/src/ccaterpillar/option.c b/src/ccaterpillar/option.c index 426b6493..117ab3b8 100644 --- a/src/ccaterpillar/option.c +++ b/src/ccaterpillar/option.c @@ -15,6 +15,7 @@ cp_option_new(PyTypeObject* type, PyObject* args, PyObject* kw) } if ((self->name = PyUnicode_FromString(""), !self->name)) { + Py_DECREF(self); return NULL; } @@ -45,14 +46,21 @@ static PyObject* cp_option_richcmp(CpOptionObject* self, PyObject* other, int op) { static const char* _NameAttr = "name"; - if (!PyObject_IsInstance(other, (PyObject*)&CpOption_Type)) { + int is_instance = PyObject_IsInstance(other, (PyObject*)&CpOption_Type); + if (is_instance < 0) { + return NULL; + } + if (!is_instance) { // check if name is equal to this object's name if (PyObject_HasAttrString(other, _NameAttr)) { PyObject* otherName = PyObject_GetAttrString(other, _NameAttr); + PyObject* result = NULL; if (!otherName) { return NULL; } - return PyObject_RichCompare(self->name, otherName, op); + result = PyObject_RichCompare(self->name, otherName, op); + Py_DECREF(otherName); + return result; } return Py_NewRef(Py_False); } @@ -139,4 +147,4 @@ cp_option__mod_init(PyObject* m, _modulestate* state) { CpModule_AddObject(CpOption_NAME, &CpOption_Type, -1); return 0; -} \ No newline at end of file +} diff --git a/src/ccaterpillar/private.h.in b/src/ccaterpillar/private.h.in index 6c3889de..d893801a 100644 --- a/src/ccaterpillar/private.h.in +++ b/src/ccaterpillar/private.h.in @@ -44,7 +44,6 @@ Py_INCREF(value); \ if (PyModule_AddObject(m, name, (PyObject*)(value)) < 0) { \ Py_DECREF(value); \ - Py_DECREF(m); \ PyErr_SetString(PyExc_RuntimeError, "unable to add '" name "' to module"); \ return (ret); \ } @@ -92,7 +91,11 @@ if (!ret) { \ return NULL; \ } \ - _Cp_SetObj(field, ret); \ + if (ret == field) { \ + Py_DECREF(ret); \ + } else { \ + Py_XSETREF(field, ret); \ + } \ return Py_NewRef((PyObject*)self); \ } @@ -118,4 +121,4 @@ shared__mod_clear(PyObject* m, _modulestate* state); #undef _CpDef_ModFn -#endif // __CP_PRIVATE_H \ No newline at end of file +#endif // __CP_PRIVATE_H diff --git a/src/ccaterpillar/pyproject.toml b/src/ccaterpillar/pyproject.toml index 7de72f75..d94c441d 100644 --- a/src/ccaterpillar/pyproject.toml +++ b/src/ccaterpillar/pyproject.toml @@ -17,7 +17,7 @@ CP_ENABLE_NATIVE = "1" [project] name = "caterpillar" -version = "2.8.2" +version = "2.9.0" requires-python = ">=3.12" description = "Library to pack and unpack structurized binary data." readme = "../../README.md" diff --git a/src/ccaterpillar/shared.c b/src/ccaterpillar/shared.c index 259eab35..4e21b654 100644 --- a/src/ccaterpillar/shared.c +++ b/src/ccaterpillar/shared.c @@ -26,6 +26,7 @@ Cp_FactoryNew(PyObject* pFactoryReference) } else { nResult = PyObject_CallNoArgs(nFactory); } + Py_XDECREF(nFactory); return nResult; } @@ -126,6 +127,7 @@ shared__mod_clear(PyObject* m, _modulestate* state) Py_CLEAR(Cp_ArrayFactory); Py_CLEAR(Cp_ContextFactory); Py_CLEAR(Cp_DefaultOption); + Py_CLEAR(CpBytesIO_Type); } int @@ -163,7 +165,8 @@ shared__mod_init(PyObject* m, _modulestate* state) return 0; err: + Py_XDECREF(nTmpMod); return -1; #undef _IMPORT_ATTR -} \ No newline at end of file +} diff --git a/test/_Py/fields/test_py_compressed.py b/test/_Py/fields/test_py_compressed.py index 74f6157e..a98f5fdb 100644 --- a/test/_Py/fields/test_py_compressed.py +++ b/test/_Py/fields/test_py_compressed.py @@ -5,7 +5,7 @@ import pytest -from caterpillar.py import Bytes, pack, unpack +from caterpillar.py import Bytes, pack, root, unpack from caterpillar.fields.compression import ( Bz2Compressed, Compressed, @@ -72,3 +72,23 @@ def test_compression_kwargs_are_passed_to_algorithm(): assert unpack(store, stored) == payload assert unpack(shrink, shrunk) == payload + +class PrefixCodec: + def compress(self, data: bytes, *, prefix: bytes) -> bytes: + return prefix + data + + def decompress(self, data: bytes, *, prefix: bytes) -> bytes: + assert data.startswith(prefix) + return data[len(prefix) :] + + +def test_context_lambda_kwargs_are_recomputed_per_op(): + field = Compressed( + PrefixCodec(), + Bytes(...), + comp_kwargs={"prefix": root.prefix}, + decomp_kwargs={"prefix": root.prefix}, + ) + + assert pack(b"a", field, prefix=b"x") == b"xa" + assert pack(b"a", field, prefix=b"y") == b"ya" diff --git a/test/_Py/fields/test_py_if.py b/test/_Py/fields/test_py_if.py index acf33ce3..db618b4d 100644 --- a/test/_Py/fields/test_py_if.py +++ b/test/_Py/fields/test_py_if.py @@ -1,8 +1,32 @@ +# pyright: basic +import sys + import pytest -from caterpillar.py import Else, If, f, pack, struct, this, uint8, uint16, unpack +from caterpillar.py import ( + BigEndian, + Branch, + Else, + ElseIf, + End, + If, + Invisible, + LittleEndian, + Start, + StructException, + When, + Otherwise, + f, + pack, + struct, + this, + uint8, + uint16, + unpack, +) +# <3.14 SYNTAX: def define_optional_byte(): @struct class OptionalByte: @@ -14,8 +38,56 @@ class OptionalByte: return OptionalByte -@pytest.mark.xfail( - reason="If cannot access class __annotations__ on Python 3.14;" +# >=3.14 SYNTAX +def define_explicit_optional_byte(): + @struct + class OptionalByte: + flag: f[int, uint8] + # With python3.14 it is required to assign the condition + # manually or define block start end block end. The conditional + # context allows direct [] (getitem) and indirect using Start/End (see below) + with If(this.flag == 1) as when: + value: when[f[int, uint8]] # pyright: ignore[reportInvalidTypeForm] + trailer: f[int, uint8] + + return OptionalByte + + +# >=3.14 SYNTAX +def define_marker_optional_byte(): + @struct + class OptionalByte: + flag: f[int, uint8] + # Example with explicit block boundaries: We have to manually + # define an attribute for the block's scope start and end. It + # is recommended to use Invisible() on them to hide them from + # the constructor. + with If(this.flag == 1) as when: + _: f[None, when] = Invisible() + value: f[int, uint8] + _end: f[None, End(when)] = Invisible() + trailer: f[int, uint8] + + return OptionalByte + + +# >=3.14 SYNTAX +def define_inline_optional_byte(): + @struct + class OptionalByte: + flag: f[int, uint8] + # markers and conditonal contexts can be applied as extra options + # in the annotation too. + with If(this.flag == 1) as when: + value: f[int, uint8, when] + trailer: f[int, uint8] + + return OptionalByte + + +@pytest.mark.skipif( + sys.version_info >= (3, 14), + reason="Implicit 'with If(condition):' blocks are not supported on 3.14", ) def test_if_unpacks_true_branch(): OptionalByte = define_optional_byte() @@ -27,10 +99,11 @@ def test_if_unpacks_true_branch(): assert decoded.trailer == 0xFF -@pytest.mark.xfail( - reason="If cannot access class __annotations__ on Python 3.14;" +@pytest.mark.skipif( + sys.version_info >= (3, 14), + reason="Implicit 'with If(condition):' blocks are not supported on 3.14", ) -def test_if_false_branch_consumes_no_bytes_and_returns_default_none(): +def test_if_false_branch_consumes_no_bytes(): OptionalByte = define_optional_byte() decoded = unpack(OptionalByte, b"\x00\xff") @@ -40,13 +113,515 @@ def test_if_false_branch_consumes_no_bytes_and_returns_default_none(): assert decoded.trailer == 0xFF -@pytest.mark.xfail( - reason="If cannot access class __annotations__ on Python 3.14;" +@pytest.mark.skipif( + sys.version_info >= (3, 14), + reason="Implicit 'with If(condition):' blocks are not supported on 3.14", ) -def test_if_pack_false_branch_writes_nothing_for_disabled_field(): +def test_if_pack_false_branch_writes_nothing(): OptionalByte = define_optional_byte() - assert pack(OptionalByte(flag=0, value=0xAA, trailer=0xFF), OptionalByte) == b"\x00\xff" - assert pack(OptionalByte(flag=1, value=0xAA, trailer=0xFF), OptionalByte) == b"\x01\xaa\xff" + assert ( + pack(OptionalByte(flag=0, value=0xAA, trailer=0xFF), OptionalByte) + == b"\x00\xff" + ) + assert ( + pack(OptionalByte(flag=1, value=0xAA, trailer=0xFF), OptionalByte) + == b"\x01\xaa\xff" + ) + + +def test_explicit_if_alias(): + OptionalByte = define_explicit_optional_byte() + + decoded = unpack(OptionalByte, b"\x01\xaa\xff") + + assert not hasattr(OptionalByte, "when") + assert decoded.flag == 1 + assert decoded.value == 0xAA + assert decoded.trailer == 0xFF + + +def test_explicit_if_alias_false_branch(): + OptionalByte = define_explicit_optional_byte() + + decoded = unpack(OptionalByte, b"\x00\xff") + + assert decoded.flag == 0 + assert decoded.value is None + assert decoded.trailer == 0xFF + assert pack(OptionalByte(flag=0, value=0xAA, trailer=0xFF)) == b"\x00\xff" + + +def test_explicit_if_alias_preserves_annotated_options(): + @struct(order=LittleEndian) + class Packet: + flag: f[int, uint8] + with If(this.flag == 1) as when: + # type checker will scream here unfortunately, + # use f[int, uint16, BigEndian, when] instead + value: when[f[int, uint16, BigEndian]] + trailer: f[int, uint8] + + assert pack(Packet(flag=1, value=0x0102, trailer=0xFF)) == b"\x01\x01\x02\xff" + assert unpack(Packet, b"\x01\x01\x02\xff") == Packet( + flag=1, value=0x0102, trailer=0xFF + ) + assert pack(Packet(flag=0, value=0x0102, trailer=0xFF)) == b"\x00\xff" + + +def test_marker_if_alias_unpacks_true_branch(): + OptionalByte = define_marker_optional_byte() + + decoded = unpack(OptionalByte, b"\x01\xaa\xff") + + assert not hasattr(OptionalByte, "when") + assert not hasattr(OptionalByte, "_") + assert not hasattr(OptionalByte, "_end") + assert decoded.flag == 1 + assert decoded.value == 0xAA + assert decoded.trailer == 0xFF + + +def test_marker_if_alias_false_branch(): + OptionalByte = define_marker_optional_byte() + + decoded = unpack(OptionalByte, b"\x00\xff") + + assert decoded.flag == 0 + assert decoded.value is None + assert decoded.trailer == 0xFF + assert pack(OptionalByte(flag=0, value=0xAA, trailer=0xFF)) == b"\x00\xff" + + +@pytest.mark.skipif( + sys.version_info < (3, 14), reason="inline conditional metadata is a 3.14 path" +) +def test_inline_metadata_if_alias_unpacks_true_branch_without_marker_fields(): + OptionalByte = define_inline_optional_byte() + + decoded = unpack(OptionalByte, b"\x01\xaa\xff") + + assert not hasattr(OptionalByte, "when") + assert decoded.flag == 1 + assert decoded.value == 0xAA + assert decoded.trailer == 0xFF + + +@pytest.mark.skipif( + sys.version_info < (3, 14), reason="inline conditional metadata is a 3.14 path" +) +def test_inline_metadata_if_alias_false_branch_consumes_no_bytes(): + OptionalByte = define_inline_optional_byte() + + decoded = unpack(OptionalByte, b"\x00\xff") + + assert decoded.flag == 0 + assert decoded.value is None + assert decoded.trailer == 0xFF + assert pack(OptionalByte(flag=0, value=0xAA, trailer=0xFF)) == b"\x00\xff" + + +@pytest.mark.skipif( + sys.version_info < (3, 14), reason="inline conditional metadata is a 3.14 path" +) +def test_inline_start_end_cover_multi_field_block_without_marker_fields(): + @struct + class Packet: + flag: f[int, uint8] + with If(this.flag == 1) as when: + first: f[int, uint8, Start(when)] + second: f[int, uint16] + third: f[int, uint8, End(when)] + trailer: f[int, uint8] + + assert not hasattr(Packet, "when") + + decoded = unpack(Packet, b"\x01\xa1\x03\x02\xa3\xff") + assert ( + decoded.flag, + decoded.first, + decoded.second, + decoded.third, + decoded.trailer, + ) == (1, 0xA1, 0x0203, 0xA3, 0xFF) + + decoded = unpack(Packet, b"\x00\xff") + assert ( + decoded.flag, + decoded.first, + decoded.second, + decoded.third, + decoded.trailer, + ) == (0, None, None, None, 0xFF) + assert pack(Packet(flag=0, first=1, second=2, third=3, trailer=0xFF)) == b"\x00\xff" + +@pytest.mark.skipif( + sys.version_info < (3, 14), reason="inline conditional metadata is a 3.14 path" +) +def test_inline_start_requires_end_marker(): + with pytest.raises(StructException, match="missing an End\\(when\\) marker"): + + @struct + class Packet: + flag: f[int, uint8] + with If(this.flag == 1) as when: + value: f[int, uint8, Start(when)] + trailer: f[int, uint8] + + +@pytest.mark.skipif( + sys.version_info < (3, 14), reason="inline conditional metadata is a 3.14 path" +) +def test_inline_end_requires_start_marker(): + with pytest.raises(StructException, match="has no matching start marker"): + + @struct + class Packet: + flag: f[int, uint8] + with If(this.flag == 1) as when: + value: f[int, uint8, End(when)] + trailer: f[int, uint8] + + +def test_marker_if_alias_preserves_annotated_options(): + @struct(order=LittleEndian) + class Packet: + flag: f[int, uint8] + with If(this.flag == 1) as when: + _: f[None, when] = Invisible() + value: f[int, uint16, BigEndian] + _end: f[None, End(when)] = Invisible() + trailer: f[int, uint8] + + assert pack(Packet(flag=1, value=0x0102, trailer=0xFF)) == b"\x01\x01\x02\xff" + assert unpack(Packet, b"\x01\x01\x02\xff") == Packet( + flag=1, value=0x0102, trailer=0xFF + ) + assert pack(Packet(flag=0, value=0x0102, trailer=0xFF)) == b"\x00\xff" + + +def test_marker_multiple_if_blocks_are_independent(): + @struct + class Packet: + flag: f[int, uint8] + with If(this.flag == 1) as when1: + _: f[None, when1] = Invisible() + first: f[int, uint8] + _end: f[None, End(when1)] = Invisible() + with If(this.flag == 2) as when2: + _2: f[None, when2] = Invisible() + second: f[int, uint8] + _end2: f[None, End(when2)] = Invisible() + trailer: f[int, uint8] + + decoded = unpack(Packet, b"\x01\xa1\xff") + assert (decoded.flag, decoded.first, decoded.second, decoded.trailer) == ( + 1, + 0xA1, + None, + 0xFF, + ) + decoded = unpack(Packet, b"\x02\xb2\xff") + assert (decoded.flag, decoded.first, decoded.second, decoded.trailer) == ( + 2, + None, + 0xB2, + 0xFF, + ) + decoded = unpack(Packet, b"\x00\xff") + assert (decoded.flag, decoded.first, decoded.second, decoded.trailer) == ( + 0, + None, + None, + 0xFF, + ) + + +@pytest.mark.skipif( + sys.version_info < (3, 14), + reason="lazy annotations make repeated marker aliases observable", +) +def test_py314_rejects_reused_marker_alias(): + with pytest.raises(StructException, match="reuses a marker alias"): + + @struct + class Packet: + flag: f[int, uint8] + with If(this.flag == 1) as when: + _: f[None, when] = Invisible() + first: f[int, uint8] + _end: f[None, End(when)] = Invisible() + with If(this.flag == 2) as when: + _2: f[None, when] = Invisible() + second: f[int, uint8] + _end2: f[None, End(when)] = Invisible() + trailer: f[int, uint8] + + +def test_marker_nested_if_blocks_combine_conditions(): + @struct + class Packet: + flag: f[int, uint8] + kind: f[int, uint8] + with If(this.flag == 1) as outer: + _: f[None, outer] = Invisible() + with If(this.kind == 2) as inner: + _1: f[None, inner] = Invisible() + value: f[int, uint8] + _end1: f[None, End(inner)] = Invisible() + _end: f[None, End(outer)] = Invisible() + trailer: f[int, uint8] + + decoded = unpack(Packet, b"\x00\x02\xff") + assert (decoded.flag, decoded.kind, decoded.value, decoded.trailer) == ( + 0, + 2, + None, + 0xFF, + ) + decoded = unpack(Packet, b"\x01\x02\xaa\xff") + assert (decoded.flag, decoded.kind, decoded.value, decoded.trailer) == ( + 1, + 2, + 0xAA, + 0xFF, + ) + decoded = unpack(Packet, b"\x01\x03\xff") + assert (decoded.flag, decoded.kind, decoded.value, decoded.trailer) == ( + 1, + 3, + None, + 0xFF, + ) + + +@pytest.mark.skipif( + sys.version_info < (3, 14), + reason="duplicate marker behavior differs before lazy annotations", +) +def test_py314_marker_if_rejects_duplicate_field_names_with_branch_guidance(): + with pytest.raises(StructException, match="use Branch"): + + @struct + class Packet: + tag: f[int, uint8] + with If(this.tag == 1) as when1: + _: f[None, when1] = Invisible() + value: f[int, uint8] + _end: f[None, End(when1)] = Invisible() + with If(this.tag == 2) as when2: + _2: f[None, when2] = Invisible() + value: f[int, uint16] + _end2: f[None, End(when2)] = Invisible() + trailer: f[int, uint8] + + +@pytest.mark.skipif( + sys.version_info < (3, 14), reason="explicit marker ElseIf is a 3.14 path" +) +def test_py314_marker_elseif_and_else_chain_distinct_fields(): + @struct + class Packet: + tag: f[int, uint8] + with If(this.tag == 1) as first: + _: f[None, first] = Invisible() + a: f[int, uint8] + _end: f[None, End(first)] = Invisible() + with ElseIf(first, this.tag == 2) as second: + _2: f[None, second] = Invisible() + b: f[int, uint8] + _end2: f[None, End(second)] = Invisible() + with Else(second) as fallback: + _3: f[None, fallback] = Invisible() + c: f[int, uint8] + _end3: f[None, End(fallback)] = Invisible() + trailer: f[int, uint8] + + decoded = unpack(Packet, b"\x01\xa1\xff") + assert (decoded.tag, decoded.a, decoded.b, decoded.c, decoded.trailer) == ( + 1, + 0xA1, + None, + None, + 0xFF, + ) + decoded = unpack(Packet, b"\x02\xb2\xff") + assert (decoded.tag, decoded.a, decoded.b, decoded.c, decoded.trailer) == ( + 2, + None, + 0xB2, + None, + 0xFF, + ) + decoded = unpack(Packet, b"\x03\xc3\xff") + assert (decoded.tag, decoded.a, decoded.b, decoded.c, decoded.trailer) == ( + 3, + None, + None, + 0xC3, + 0xFF, + ) + + +@pytest.mark.skipif( + sys.version_info < (3, 14), reason="inline conditional metadata is a 3.14 path" +) +def test_py314_inline_elseif_and_else_chain_distinct_fields(): + @struct + class Packet: + tag: f[int, uint8] + with If(this.tag == 1) as first: + a: f[int, uint8, first] + with ElseIf(first, this.tag == 2) as second: + b: f[int, uint8, second] + with Else(second) as fallback: + c: f[int, uint8, fallback] + trailer: f[int, uint8] + + decoded = unpack(Packet, b"\x01\xa1\xff") + assert (decoded.tag, decoded.a, decoded.b, decoded.c, decoded.trailer) == ( + 1, + 0xA1, + None, + None, + 0xFF, + ) + decoded = unpack(Packet, b"\x02\xb2\xff") + assert (decoded.tag, decoded.a, decoded.b, decoded.c, decoded.trailer) == ( + 2, + None, + 0xB2, + None, + 0xFF, + ) + decoded = unpack(Packet, b"\x03\xc3\xff") + assert (decoded.tag, decoded.a, decoded.b, decoded.c, decoded.trailer) == ( + 3, + None, + None, + 0xC3, + 0xFF, + ) + + +@pytest.mark.skipif( + sys.version_info < (3, 14), reason="explicit marker ElseIf is a 3.14 path" +) +def test_py314_marker_rejects_elseif_after_else(): + with pytest.raises(StructException, match="after Else"): + + @struct + class Packet: + tag: f[int, uint8] + with If(this.tag == 1) as first: + _: f[None, first] = Invisible() + a: f[int, uint8] + _end: f[None, End(first)] = Invisible() + with Else(first) as fallback: + _2: f[None, fallback] = Invisible() + b: f[int, uint8] + _end2: f[None, End(fallback)] = Invisible() + with ElseIf(fallback, this.tag == 3) as third: + _3: f[None, third] = Invisible() + c: f[int, uint8] + _end3: f[None, End(third)] = Invisible() + + +@pytest.mark.skipif( + sys.version_info < (3, 14), reason="legacy ElseIf still works before 3.14" +) +def test_py314_legacy_elseif_fails_with_clear_message(): + with pytest.raises(StructException, match="ElseIf\\(previous, condition\\)"): + + @struct + class Packet: + tag: f[int, uint8] + with If(this.tag == 1) as when: + _: f[None, when] = Invisible() + value: f[int, uint8] + _end: f[None, End(when)] = Invisible() + with ElseIf(this.tag == 2): + other: f[int, uint8] + trailer: f[int, uint8] + + +def test_branch_same_field_chain_unpack_and_pack(): + @struct + class Packet: + tag: f[int, uint8] + value: f[ + int, + Branch( + When(this.tag == 1, uint8), + When(this.tag == 2, uint16), + Otherwise(uint8), + ), + ] + trailer: f[int, uint8] + + decoded = unpack(Packet, b"\x01\xaa\xff") + assert (decoded.tag, decoded.value, decoded.trailer) == (1, 0xAA, 0xFF) + decoded = unpack(Packet, b"\x02\x02\x01\xff") + assert (decoded.tag, decoded.value, decoded.trailer) == (2, 0x0102, 0xFF) + decoded = unpack(Packet, b"\x03\xcc\xff") + assert (decoded.tag, decoded.value, decoded.trailer) == (3, 0xCC, 0xFF) + + assert pack(Packet(tag=1, value=0xAA, trailer=0xFF)) == b"\x01\xaa\xff" + assert pack(Packet(tag=2, value=0x0102, trailer=0xFF)) == b"\x02\x02\x01\xff" + assert pack(Packet(tag=3, value=0xCC, trailer=0xFF)) == b"\x03\xcc\xff" + + +def test_branch_arm_preserves_annotated_options(): + @struct(order=LittleEndian) + class Packet: + tag: f[int, uint8] + value: f[ + int, + Branch( + When(this.tag == 1, f[int, uint16, BigEndian]), + Otherwise(uint16), + ), + ] + trailer: f[int, uint8] + + assert pack(Packet(tag=1, value=0x0102, trailer=0xFF)) == b"\x01\x01\x02\xff" + assert pack(Packet(tag=0, value=0x0102, trailer=0xFF)) == b"\x00\x02\x01\xff" + + +def test_marker_if_requires_end_marker(): + with pytest.raises(StructException, match="missing an End\\(when\\) marker"): + + @struct + class Packet: + flag: f[int, uint8] + with If(this.flag == 1) as when: + _: f[None, when] = Invisible() + value: f[int, uint8] + trailer: f[int, uint8] + + +def test_marker_if_rejects_end_without_start_marker(): + with pytest.raises(StructException, match="has no matching start marker"): + + @struct + class Packet: + flag: f[int, uint8] + with If(this.flag == 1) as when: + _end: f[None, End(when)] = Invisible() + value: f[int, uint8] + trailer: f[int, uint8] + + +@pytest.mark.skipif( + sys.version_info < (3, 14), reason="implicit If still uses legacy annotations" +) +def test_implicit_if_on_python314_requires_explicit_alias(): + with pytest.raises(StructException, match="with If\\(condition\\) as when"): + @struct + class Packet: + flag: f[int, uint8] + with If(this.flag == 1): + value: f[int, uint8] + trailer: f[int, uint8] diff --git a/test/_Py/fields/test_py_padding.py b/test/_Py/fields/test_py_padding.py index f8ca9cdf..87dd61d8 100644 --- a/test/_Py/fields/test_py_padding.py +++ b/test/_Py/fields/test_py_padding.py @@ -1,6 +1,26 @@ +import io + import pytest -from caterpillar.py import ValidationError, constval, padding, pack, unpack, Padding +from caterpillar.py import ( + DynamicSizeError, + Padded, + PostPad, + PrePad, + ValidationError, + constval, + f, + padding, + pack, + sizeof, + struct, + this, + uint8, + uint16, + unpack, + Padding, +) +from caterpillar.types import uint8_t def test_py_padding(): @@ -44,5 +64,77 @@ def test_py_padding_custom_unpack_strict(): def test_py_padding_context_length(): - pad = padding[constval(10)] # always 10 + pad = padding[constval(10)] # always 10 assert pack(None, pad) == b"\x00" * 10 + + +def test_py_padded_static_pre_and_post_roundtrip(): + field = Padded(uint8, before=2, after=3) + + assert pack(0xAA, field) == b"\x00\x00\xaa\x00\x00\x00" + assert unpack(field, b"\x00\x00\xaa\x00\x00\x00") == 0xAA + + +def test_py_padded_custom_fill_repeats_and_truncates_to_byte_length(): + field = Padded(uint8, before=5, after=3, fill=b"AB") + + assert pack(0xCC, field) == b"ABABA\xccABA" + assert unpack(field, b"12345\xcc678") == 0xCC + + +def test_py_padded_strict_validates_fill(): + field = Padded(uint8, before=2, after=2, fill=0xFF, strict=True) + + assert unpack(field, b"\xff\xff\x01\xff\xff") == 1 + with pytest.raises(ValidationError): + _ = unpack(field, b"\xff\x00\x01\xff\xff") + with pytest.raises(ValidationError): + _ = unpack(field, b"\xff\xff\x01\xff") + + +def test_py_padded_context_length(): + @struct + class Format: + before: uint8_t + after: uint8_t + value: f[int, Padded(uint8, before=this.before, after=this.after, fill=0x50)] + + obj = Format(2, 3, 0xAA) + data = b"\x02\x03PP\xaaPPP" + + assert pack(obj) == data + assert unpack(Format, data) == obj + + +def test_py_padded_invalid_resolved_lengths(): + with pytest.raises(ValueError): + _ = pack(1, Padded(uint8, before=-1)) + + with pytest.raises(ValueError): + _ = pack(1, Padded(uint8, after=constval("bad"))) + + +def test_py_padded_sizeof_static_and_dynamic(): + assert sizeof(Padded(uint16, before=2, after=4)) == 8 + + with pytest.raises(DynamicSizeError): + _ = sizeof(Padded(uint16, after=this.amount)) + + +def test_py_padded_prepad_postpad(): + direct = PostPad(2, fill=0xCC)(PrePad(1, fill=0xAA)(uint8)) + slash = uint8 / PrePad(1, fill=0xAA) / PostPad(2, fill=0xCC) + + assert pack(0x11, direct) == b"\xaa\x11\xcc\xcc" + assert pack(0x11, slash) == b"\xaa\x11\xcc\xcc" + assert unpack(direct, b"\xaa\x11\xcc\xcc") == 0x11 + assert unpack(slash, b"\xaa\x11\xcc\xcc") == 0x11 + + +def test_py_padded_preserves_stream_position(): + field = uint8 / PostPad(2) + stream = io.BytesIO(b"\x01\x00\x00\xff") + + assert unpack(field, stream) == 1 + assert stream.tell() == 3 + assert stream.read() == b"\xff" diff --git a/test/_Py/model/test_template.py b/test/_Py/model/test_template.py new file mode 100644 index 00000000..1d1398f0 --- /dev/null +++ b/test/_Py/model/test_template.py @@ -0,0 +1,104 @@ +from typing_extensions import TypeVar, Generic + +from caterpillar.py import ( + BigEndian, + derive, + f, + field_of, + struct, + template, + uint16, + uint8, + pack, + unpack, +) +from caterpillar.types import uint8_t + + +def test_generic_template_direct_typevar(): + T = TypeVar("T") + + @template + class Box(Generic[T]): + value: T + + # When we use the struct candidate directly, type checkers won't be able to + # infer the right type, since these structs are of type 'PyStructFormattesField' + # and not 'int'. Use the f[int, > expression instead. + ByteBox = Box[uint8] + + assert ByteBox is Box[uint8] + assert ByteBox.__origin__ is Box + assert ByteBox.__args__ == (uint8,) + assert pack(ByteBox(7)) == b"\x07" + assert unpack(ByteBox, b"\x08") == ByteBox(8) + assert pack([ByteBox(1), ByteBox(2)], ByteBox[2]) == b"\x01\x02" + + +def test_generic_template_direct_typevar_annotated_field(): + T = TypeVar("T") + + @template + class Box(Generic[T]): + value: T + + ByteBox = Box[uint8_t] + + assert ByteBox is Box[uint8_t] + assert ByteBox.__origin__ is Box + assert ByteBox.__args__ == (uint8_t,) + assert pack(ByteBox(7)) == b"\x07" + assert unpack(ByteBox, b"\x08") == ByteBox(8) + assert pack([ByteBox(1), ByteBox(2)], ByteBox[2]) == b"\x01\x02" + + +def test_generic_template_multiple_typevars(): + T = TypeVar("T") + U = TypeVar("U") + + @template + class Pair(Generic[T, U]): + left: T + right: U + + ByteWordPair = Pair[uint8, uint16] + obj = ByteWordPair(1, 0x0203) + + assert pack(obj, order=BigEndian) == b"\x01\x02\x03" + assert unpack(ByteWordPair, b"\x01\x02\x03", order=BigEndian) == obj + + +def test_generic_template_field_of(): + T = TypeVar("T") + + @template + class Vector(Generic[T]): + values: f[list[T], field_of(T)[2]] + + # hint: use the *_t types directly when you use type checkers + ByteVector = Vector[uint8_t] + obj = ByteVector([3, 4]) + + assert ByteVector.__annotations__["values"].__args__[0] == list[object] + assert pack(obj) == b"\x03\x04" + assert unpack(ByteVector, b"\x05\x06") == ByteVector([5, 6]) + + +def test_generic_template_derive(): + T = TypeVar("T") + + @template + class Box(Generic[T]): + value: T + + ByteBox = derive(Box, uint8, name="DerivedGenericByteBoxForTest") + assert derive(ByteBox) is ByteBox + + @struct + class Packet(ByteBox): + tail: f[int, uint16] + + obj = Packet(1, 0x0203) + + assert pack(obj, order=BigEndian) == b"\x01\x02\x03" + assert unpack(Packet, b"\x04\x05\x06", order=BigEndian) == Packet(4, 0x0506)