Support zarr v3

.github/scripts/serialization/objects.py

@@ -17,6 +17,11 @@
     the full state (traces or spike trains, properties, annotations, probe) to disk.
     Targets the on-disk encoding axis: property and annotation preservation, and the
     probe representation. "binary" is recording-only, hence "numpy_folder" for sortings.
+
+Additionally, the `check_extra_data` kwarg can be used to store additional information in a JSON file alongside the
+fixture, which is then passed to the check function. This is used for the SortingAnalyzer extension data keys,
+which are computed and stored in the fixture JSON to verify that they are reloaded correctly.
+
 """
 from packaging.version import parse
 from spikeinterface import __version__ as si_version
@@ -31,6 +36,7 @@
     "binary": "_binary",
     "numpy_folder": "_numpy_folder",
     "zarr": ".zarr",
+    "binary_folder": "_binary_folder",
 }
 
 
@@ -49,7 +55,7 @@ def _build_noise_generator_recording():
     return NoiseGeneratorRecording(num_channels=4, sampling_frequency=30000.0, durations=[1.0, 1.5], seed=0)
 
 
-def _check_noise_generator_recording(rec):
+def _check_noise_generator_recording(rec, check_extra_data=None):
     assert type(rec).__name__ == "NoiseGeneratorRecording", type(rec).__name__
     assert rec.get_num_channels() == 4
     assert rec.get_num_segments() == 2
@@ -62,7 +68,7 @@ def _build_mock_recording():
     return generate_recording(num_channels=4, durations=[1.0], sampling_frequency=30000.0, seed=0)
 
 
-def _check_mock_recording(rec):
+def _check_mock_recording(rec, check_extra_data=None):
     from spikeinterface.core import BaseRecording
 
     assert isinstance(rec, BaseRecording), type(rec).__name__
@@ -83,7 +89,7 @@ def _build_recording_with_properties():
     return rec
 
 
-def _check_recording_with_properties(rec):
+def _check_recording_with_properties(rec, check_extra_data=None):
     assert rec.get_num_channels() == 4
     assert list(rec.get_property("quality")) == ["good", "good", "bad", "good"]
     assert rec.get_annotation("experimenter") == "test"
@@ -104,7 +110,7 @@ def _build_recording_with_probe():
     return rec_with_probe
 
 
-def _check_recording_with_probe(rec):
+def _check_recording_with_probe(rec, check_extra_data=None):
     import numpy as np
 
     assert rec.get_num_channels() == 8
@@ -137,7 +143,7 @@ def _build_recording_with_interleaved_probes():
     return rec_with_probe
 
 
-def _check_recording_with_interleaved_probes(rec):
+def _check_recording_with_interleaved_probes(rec, check_extra_data=None):
     import numpy as np
 
     assert rec.get_num_channels() == 8
@@ -170,7 +176,7 @@ def _build_preprocessed_chain():
     return common_reference(scale(rec, gain=2.0))
 
 
-def _check_preprocessed_chain(rec):
+def _check_preprocessed_chain(rec, check_extra_data=None):
     # The outer wrapper and the recursive parent chain must both reload (the kwargs
     # embed the parent recording dict, so this exercises recursive deserialization).
     assert type(rec).__name__ == "CommonReferenceRecording", type(rec).__name__
@@ -184,7 +190,7 @@ def _build_sorting():
     return generate_sorting(num_units=5, sampling_frequency=30000.0, durations=[1.0])
 
 
-def _check_sorting(sorting):
+def _check_sorting(sorting, check_extra_data=None):
     assert sorting.get_num_units() == 5, sorting.get_num_units()
     assert sorting.get_num_segments() == 1
     spike_train = sorting.get_unit_spike_train(sorting.unit_ids[0], segment_index=0)
@@ -201,12 +207,41 @@ def _build_sorting_with_properties():
     return sorting
 
 
-def _check_sorting_with_properties(sorting):
+def _check_sorting_with_properties(sorting, check_extra_data=None):
     assert sorting.get_num_units() == 4
     assert list(sorting.get_property("quality")) == ["good", "good", "bad", "good"]
     assert sorting.get_annotation("experimenter") == "test"
 
 
+def _build_sorting_analyzer_with_extensions():
+    from spikeinterface.core import generate_ground_truth_recording, create_sorting_analyzer
+    recording, sorting = generate_ground_truth_recording(durations=[10, 5], num_channels=16, num_units=5, seed=0)
+    analyzer = create_sorting_analyzer(sorting, recording)
+    extensions = analyzer.get_computable_extensions()
+    analyzer.compute(extensions, n_jobs=-1)
+    # for SortingAnalyzer, we also save a JSON file with extension as
+    # keys and date entries as values to check that everything is reloaded correctly
+    check_extra_data = {}
+    for ext_name in analyzer.extensions:
+        ext = analyzer.get_extension(ext_name)
+        check_extra_data[ext_name] = list(ext.data.keys())
+    return analyzer, check_extra_data
+
+
+def _check_sorting_analyzer_with_extensions(analyzer, check_extra_data=None):
+    extensions = analyzer.get_saved_extension_names()
+    for extension in extensions:
+        extension = analyzer.get_extension(extension)  # just check it loads without error
+        ext_data = extension.get_data()
+        assert ext_data is not None, f"extension {extension} returned None data"
+    if check_extra_data is not None:
+        # Check that the extension data keys match what was saved in the fixture JSON.
+        for ext_name, expected_keys in check_extra_data.items():
+            ext = analyzer.get_extension(ext_name)
+            actual_keys = set(ext.data.keys())
+            assert actual_keys == set(expected_keys), f"extension {ext_name} keys mismatch: {actual_keys} != {set(expected_keys)}"
+
+
 OBJECTS = [
     {
         "id": "noise_generator_recording",
@@ -256,4 +291,10 @@ def _check_sorting_with_properties(sorting):
         "check": _check_sorting_with_properties,
         "formats": ["numpy_folder", "zarr"],
     },
+    {
+        "id": "sorting_analyzer_with_extensions",
+        "build": _build_sorting_analyzer_with_extensions,
+        "check": _check_sorting_analyzer_with_extensions,  # no check; the extensions are computed and stored, but the analyzer itself is not serialized
+        "formats": ["binary_folder", "zarr"],
+    },
 ]

.github/scripts/serialization/serialize_objects.py

@@ -12,31 +12,52 @@
 """
 
 import sys
+import json
 from pathlib import Path
 
-import spikeinterface
+from spikeinterface import __version__ as si_version
+from spikeinterface import SortingAnalyzer
+from spikeinterface.core.base import BaseExtractor
+
 
 sys.path.insert(0, str(Path(__file__).parent))
 from objects import OBJECTS, FIXTURE_SUFFIX  # noqa: E402
 
 out_dir = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("serialization_fixtures")
 out_dir.mkdir(parents=True, exist_ok=True)  # do not rmtree an arbitrary path; overwrite in place
 
-print(f"Generating serialization fixtures with spikeinterface {spikeinterface.__version__}")
+print(f"Generating serialization fixtures with spikeinterface {si_version}")
 for entry in OBJECTS:
-    obj = entry["build"]()
+    objs = entry["build"]()
+    if isinstance(objs, tuple) and len(objs) == 2 and isinstance(objs[1], dict):
+        obj, check_extra_data = objs
+    else:
+        obj = objs
+        check_extra_data = None
     for fmt in entry["formats"]:
         dest = out_dir / f"{entry['id']}{FIXTURE_SUFFIX[fmt]}"
-        if fmt == "json":
-            obj.dump_to_json(dest)
-        elif fmt == "pickle":
-            obj.dump_to_pickle(dest)
-        elif fmt == "binary":
-            obj.save(folder=dest, format="binary", overwrite=True)
-        elif fmt == "numpy_folder":
-            obj.save(folder=dest, format="numpy_folder", overwrite=True)
-        elif fmt == "zarr":
-            obj.save(folder=dest, format="zarr", overwrite=True)
+        if isinstance(obj, BaseExtractor):
+            if fmt == "json":
+                obj.dump_to_json(dest)
+            elif fmt == "pickle":
+                obj.dump_to_pickle(dest)
+            elif fmt == "binary":
+                obj.save(folder=dest, format="binary", overwrite=True)
+            elif fmt == "numpy_folder":
+                obj.save(folder=dest, format="numpy_folder", overwrite=True)
+            elif fmt == "zarr":
+                obj.save(folder=dest, format="zarr", overwrite=True)
+        elif isinstance(obj, SortingAnalyzer):
+            if dest.is_dir():
+                import shutil
+                shutil.rmtree(dest)
+            obj.save_as(folder=dest, format=fmt)
+
         print(f"  wrote {dest.name} ({fmt})")
+    if check_extra_data is not None:
+        json_dest = out_dir / f"{entry['id']}.json"
+        with open(json_dest, "w") as f:
+            json.dump(check_extra_data, f)
+        print(f"  wrote {json_dest.name} (extra data)")
 
 print(f"Fixtures written to: {out_dir.resolve()}")

.github/scripts/serialization/test_cross_version_compatibility.py

@@ -9,6 +9,7 @@
 """
 
 import os
+import json
 from pathlib import Path
 
 import pytest
@@ -27,4 +28,8 @@ def test_load_old_serialized_object(entry, fmt):
     fixture = FIXTURES_DIR / f"{entry['id']}{FIXTURE_SUFFIX[fmt]}"
     assert fixture.exists(), f"missing fixture {fixture}"
     obj = load(fixture)
-    entry["check"](obj)
+    check_extra_data = None
+    if (FIXTURES_DIR / f"{entry['id']}.json").exists():
+        with open(FIXTURES_DIR / f"{entry['id']}.json", "r") as f:
+            check_extra_data = json.load(f)
+    entry["check"](obj, check_extra_data=check_extra_data)

.github/workflows/all-tests.yml

@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.13"]  # Lower and higher versions we support
+        python-version: ["3.11", "3.13"]  # Lower and higher versions we support
         os: [macos-latest, windows-latest, ubuntu-latest]
     steps:
       - uses: actions/checkout@v6

.github/workflows/caches_cron_job.yml

@@ -19,10 +19,10 @@ jobs:
     steps:
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - uses: astral-sh/setup-uv@v7
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           enable-cache: true
           ignore-nothing-to-cache: true # some runs do not require building a cache (ie macOS) this says this is okay
       - name: Create the directory to store the data

.github/workflows/core-test.yml

@@ -22,10 +22,10 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - uses: astral-sh/setup-uv@v7
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           enable-cache: true
       - name: Install dependencies
         run: |

.github/workflows/cross_version_serialization.yml

@@ -8,7 +8,9 @@ name: Cross-version serialization
 # run in an isolated uv environment (nothing committed). The version list is computed
 # from PyPI (latest patch of each minor at or above MIN_VERSION_TO_TEST): on a pull
 # request only the latest released minor is tested; the full matrix runs weekly and on
-# manual dispatch.
+# manual dispatch. The pseudo-version "dev" is always included: it generates the fixtures
+# with the branch under test, so a round-trip break (write and read with the same code)
+# is caught independently of any release.
 
 on:
   pull_request:
@@ -43,7 +45,7 @@ jobs:
     steps:
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - id: set
         env:
           GITHUB_EVENT_NAME: ${{ github.event_name }}
@@ -71,7 +73,8 @@ jobs:
           if os.environ.get('GITHUB_EVENT_NAME') == 'pull_request':
               minor_releases = minor_releases[-1:]
 
-          print(json.dumps([str(v) for v in minor_releases]))
+          # always round-trip against the branch itself as well
+          print(json.dumps([str(v) for v in minor_releases] + ['dev']))
           ")
 
           echo "list=$versions_to_test" >> "$GITHUB_OUTPUT"
@@ -93,21 +96,33 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: Set up uv
         uses: astral-sh/setup-uv@v7
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           enable-cache: false
 
       - name: Generate fixtures with spikeinterface ${{ matrix.si-version }}
+        env:
+          SI_VERSION: ${{ matrix.si-version }}
         run: |
-          uv run --isolated --no-project --python 3.10 \
-            --with "spikeinterface[core]==${{ matrix.si-version }}" \
+          # "dev" means the checked-out branch; anything else is a release from PyPI.
+          # [full] is needed because the SortingAnalyzer fixture computes every
+          # extension (scikit-learn, numba, ...).
+          if [ "$SI_VERSION" = "dev" ]; then
+            spec=".[full]"
+          else
+            spec="spikeinterface[full]==$SI_VERSION"
+          fi
+          uv run --isolated --no-project --python 3.11 \
+            --with "$spec" \
             python .github/scripts/serialization/serialize_objects.py "$SI_SERIALIZATION_FIXTURES_DIR"
 
       - name: Load fixtures with the branch
         run: |
-          uv pip install --system -e . --group test-core
+          # [full] here too: loading the analyzer extensions unpickles e.g. the
+          # scikit-learn PCA models written at generation time
+          uv pip install --system -e ".[full]" --group test-core
           pytest .github/scripts/serialization/test_cross_version_compatibility.py -v

.github/workflows/deepinterpolation.yml

@@ -1,10 +1,8 @@
 name: Testing deepinterpolation
 
+# Manual only — deepinterpolation requires Python 3.11, incompatible with 3.11+ required by Zarr 3.0.0+
 on:
-  pull_request:
-    types: [synchronize, opened, reopened]
-    branches:
-      - main
+  workflow_dispatch:
 
 concurrency: # Cancel previous workflows on the same pull request
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -22,10 +20,10 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - uses: astral-sh/setup-uv@v7
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           enable-cache: false
       - name: Get changed files
         id: changed-files

.github/workflows/test_containers_docker.yml

@@ -14,10 +14,10 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: '3.11'
       - uses: astral-sh/setup-uv@v7
         with:
-          python-version: "3.10"
+          python-version: '3.11'
           enable-cache: true
       - name: Python version
         run: python --version

.github/workflows/test_containers_singularity.yml

@@ -15,10 +15,10 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: '3.11'
       - uses: astral-sh/setup-uv@v7
         with:
-          python-version: "3.10"
+          python-version: '3.11'
           enable-cache: true
       - uses: eWaterCycle/setup-singularity@v7
         with:

.github/workflows/test_imports.yml

@@ -22,10 +22,10 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - uses: astral-sh/setup-uv@v7
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           enable-cache: true
       - name: Install Spikeinterface with only core dependencies
         run: |