ENH: Out-of-core architecture rewrite and filter optimizations

CMakeLists.txt

@@ -53,6 +53,14 @@ option(SIMPLNX_DOWNLOAD_TEST_FILES "Download the test files" ON)
 # ------------------------------------------------------------------------------
 option(SIMPLNX_WRITE_TEST_OUTPUT "Write unit test output files" OFF)
 
+# ------------------------------------------------------------------------------
+# Controls which algorithm paths are exercised by dual-dispatch unit tests.
+#   0 (Both)       - tests run with forceOoc=false AND forceOoc=true (default)
+#   1 (OocOnly)    - tests run with forceOoc=true only (use for OOC builds)
+#   2 (InCoreOnly) - tests run with forceOoc=false only (quick validation)
+# ------------------------------------------------------------------------------
+set(SIMPLNX_TEST_ALGORITHM_PATH "0" CACHE STRING "Algorithm paths to test: 0=Both, 1=OocOnly, 2=InCoreOnly")
+
 # ------------------------------------------------------------------------------
 # Is the SimplnxCore Plugin enabled [DEFAULT=ON]
 # ------------------------------------------------------------------------------
@@ -245,6 +253,9 @@ if(MSVC)
   target_compile_options(simplnx
     PRIVATE
       /MP
+      # Dream3dIO.cpp exceeds the default COMDAT section limit (C1128) in Debug
+      # after the out-of-core resolver refactor; /bigobj raises the limit.
+      /bigobj
   )
 endif()
 
@@ -269,6 +280,7 @@ if(SIMPLNX_ENABLE_MULTICORE)
   target_link_libraries(simplnx PUBLIC TBB::tbb)
 endif()
 
+
 target_link_libraries(simplnx
   PUBLIC
     fmt::fmt
@@ -301,6 +313,11 @@ if(SIMPLNX_ENABLE_LINK_FILESYSTEM)
 endif()
 
 set(SIMPLNX_GENERATED_DIR ${simplnx_BINARY_DIR}/generated)
+
+configure_file(
+  ${simplnx_SOURCE_DIR}/cmake/SimplnxConfig.hpp.in
+  ${SIMPLNX_GENERATED_DIR}/simplnx/Common/SimplnxConfig.hpp
+  @ONLY)
 set(SIMPLNX_GENERATED_HEADER_DIR ${simplnx_BINARY_DIR}/generated/simplnx)
 set(SIMPLNX_EXPORT_HEADER ${SIMPLNX_GENERATED_HEADER_DIR}/simplnx_export.hpp)
 
@@ -359,6 +376,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Common/Constants.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/DataTypeUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/DataVector.hpp
+  ${SIMPLNX_SOURCE_DIR}/Common/Extent.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/EulerAngle.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/Filesystem.hpp
   ${SIMPLNX_SOURCE_DIR}/Common/IteratorUtility.hpp
@@ -388,6 +406,8 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/DataIOCollection.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/IDataFactory.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/IDataIOManager.hpp
+  ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/IDataStoreFormatResolver.hpp
+  ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/InMemoryFormatResolver.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/IOConstants.hpp
 
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/HDF5/DataIOManager.hpp
@@ -473,6 +493,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/DataStructure/DynamicListArray.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/EmptyDataStore.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/EmptyListStore.hpp
+  ${SIMPLNX_SOURCE_DIR}/DataStructure/EmptyStringStore.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IArray.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IDataArray.hpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IDataStore.hpp
@@ -548,12 +569,14 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Plugin/PluginLoader.hpp
 
   ${SIMPLNX_SOURCE_DIR}/Utilities/ArrayCreationUtilities.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/SliceBufferedTransfer.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/AlignSections.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/ArrayThreshold.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataArrayUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataGroupUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataObjectUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataStoreUtilities.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/AlgorithmDispatch.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/FilePathGenerator.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/ColorTableUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/FileUtilities.hpp
@@ -564,6 +587,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Utilities/HistogramUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MaskCompareUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryUtilities.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryBudgetManager.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MessageHelper.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/StringUtilities.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/StringInterpretationUtilities.hpp
@@ -576,6 +600,7 @@ set(SIMPLNX_HDRS
   ${SIMPLNX_SOURCE_DIR}/Utilities/SamplingUtils.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/SegmentFeatures.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/TimeUtilities.hpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/UnionFind.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/TooltipGenerator.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/TooltipRowItem.hpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/OStreamUtilities.hpp
@@ -648,6 +673,7 @@ set(SIMPLNX_SRCS
 
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/DataIOCollection.cpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/IDataIOManager.cpp
+  ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/IDataStoreFormatResolver.cpp
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/Generic/CoreDataIOManager.cpp
 
   ${SIMPLNX_SOURCE_DIR}/DataStructure/IO/HDF5/DataIOManager.cpp
@@ -779,6 +805,7 @@ set(SIMPLNX_SRCS
   ${SIMPLNX_SOURCE_DIR}/Utilities/DataStoreUtilities.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MaskCompareUtilities.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryUtilities.cpp
+  ${SIMPLNX_SOURCE_DIR}/Utilities/MemoryBudgetManager.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/MessageHelper.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/IParallelAlgorithm.cpp
   ${SIMPLNX_SOURCE_DIR}/Utilities/ParallelDataAlgorithm.cpp

cmake/Plugin.cmake

@@ -355,6 +355,11 @@ function(create_simplnx_plugin_unit_test)
     ${${ARGS_PLUGIN_NAME}UnitTest_SRCS}
   )
 
+  # Record every plugin unit-test target on a global property so consumer builds
+  # (which include simplnx via add_subdirectory) can attach additional sources or
+  # settings to the test executables without simplnx knowing about the consumer.
+  set_property(GLOBAL APPEND PROPERTY SIMPLNX_UNIT_TEST_TARGETS ${UNIT_TEST_TARGET})
+
   target_link_libraries(${UNIT_TEST_TARGET}
     PRIVATE
       simplnx
@@ -389,6 +394,7 @@ function(create_simplnx_plugin_unit_test)
   target_compile_definitions(${UNIT_TEST_TARGET}
     PRIVATE
       SIMPLNX_BUILD_DIR="$<TARGET_FILE_DIR:simplnx_test>"
+      SIMPLNX_TEST_ALGORITHM_PATH=${SIMPLNX_TEST_ALGORITHM_PATH}
   )
 
   target_compile_options(${UNIT_TEST_TARGET}

cmake/SimplnxConfig.hpp.in

@@ -0,0 +1,12 @@
+#pragma once
+
+// This header is generated from cmake/SimplnxConfig.hpp.in by configure_file().
+// It carries compile-time configuration into every translation unit that links
+// simplnx, including MOC, via simplnx's PUBLIC generated include directory.
+// Delivering values through a header (not per-target compile definitions) keeps
+// them identical across all consumers, which is required for ODR safety when a
+// macro gates inline code in public headers.
+//
+// Out-of-core support is no longer a compile-time switch in core simplnx; OOC is
+// provided at runtime by a registered IO manager (see DataIOCollection). This
+// header is retained for future compile-time configuration values.

src/Plugins/ITKImageProcessing/src/ITKImageProcessing/Common/ITKArrayHelper.hpp

@@ -840,7 +840,7 @@ Result<OutputActions> DataCheck(const DataStructure& dataStructure, const DataPa
   const auto& inputArray = dataStructure.getDataRefAs<IDataArray>(inputArrayPath);
   const auto& inputDataStore = inputArray.getIDataStoreRef();
 
-  if(!inputArray.getDataFormat().empty())
+  if(inputArray.getStoreType() == IDataStore::StoreType::OutOfCore)
   {
     return MakeErrorResult<OutputActions>(Constants::k_OutOfCoreDataNotSupported,
                                           fmt::format("Input Array '{}' utilizes out-of-core data. This is not supported within ITK filters.", inputArrayPath.toString()));
@@ -862,7 +862,7 @@ Result<detail::ITKFilterFunctorResult_t<FilterCreationFunctorT>> Execute(DataStr
 
   using ResultT = detail::ITKFilterFunctorResult_t<FilterCreationFunctorT>;
 
-  if(!inputArray.getDataFormat().empty())
+  if(inputArray.getStoreType() == IDataStore::StoreType::OutOfCore)
   {
     return MakeErrorResult(Constants::k_OutOfCoreDataNotSupported, fmt::format("Input Array '{}' utilizes out-of-core data. This is not supported within ITK filters.", inputArrayPath.toString()));
   }

src/Plugins/ITKImageProcessing/test/ITKTestBase.cpp

@@ -30,7 +30,7 @@ std::string ComputeMD5HashTyped(const IDataArray& outputDataArray)
   usize arraySize = dataArray.getSize();
 
   MD5 md5;
-  if(outputDataArray.getDataFormat().empty())
+  if(outputDataArray.getIDataStoreRef().getStoreType() != IDataStore::StoreType::OutOfCore)
   {
     const T* dataPtr = dataArray.template getIDataStoreRefAs<DataStore<T>>().data();
     md5.update(reinterpret_cast<const uint8*>(dataPtr), arraySize * sizeof(T));
@@ -135,47 +135,7 @@ namespace ITKTestBase
 bool IsArrayInMemory(DataStructure& dataStructure, const DataPath& outputDataPath)
 {
   const auto& outputDataArray = dataStructure.getDataRefAs<IDataArray>(outputDataPath);
-  DataType outputDataType = outputDataArray.getDataType();
-
-  switch(outputDataType)
-  {
-  case DataType::float32: {
-    return dynamic_cast<const DataArray<float32>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::float64: {
-    return dynamic_cast<const DataArray<float64>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int8: {
-    return dynamic_cast<const DataArray<int8>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint8: {
-    return dynamic_cast<const DataArray<uint8>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int16: {
-    return dynamic_cast<const DataArray<int16>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint16: {
-    return dynamic_cast<const DataArray<uint16>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int32: {
-    return dynamic_cast<const DataArray<int32>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint32: {
-    return dynamic_cast<const DataArray<uint32>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::int64: {
-    return dynamic_cast<const DataArray<int64>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::uint64: {
-    return dynamic_cast<const DataArray<uint64>&>(outputDataArray).getDataFormat().empty();
-  }
-  case DataType::boolean: {
-    [[fallthrough]];
-  }
-  default: {
-    return {};
-  }
-  }
+  return outputDataArray.getIDataStoreRef().getStoreType() != IDataStore::StoreType::OutOfCore;
 }
 //------------------------------------------------------------------------------
 std::string ComputeMd5Hash(DataStructure& dataStructure, const DataPath& outputDataPath)

src/Plugins/OrientationAnalysis/CMakeLists.txt

@@ -9,6 +9,7 @@ else()
 endif()
 
 
+
 # ------------------------------------------------------------------------------
 # EbsdLib needs install rules for creating packages
 # ------------------------------------------------------------------------------
@@ -173,6 +174,8 @@ set(filter_algorithms
   AlignSectionsMisorientation
   AlignSectionsMutualInformation
   BadDataNeighborOrientationCheck
+  BadDataNeighborOrientationCheckScanline
+  BadDataNeighborOrientationCheckWorklist
   CAxisSegmentFeatures
   ComputeAvgCAxes
   ComputeAvgOrientations
@@ -187,12 +190,17 @@ set(filter_algorithms
   ComputeFZQuaternions
   ComputeGBCD
   ComputeGBCDMetricBased
-  ComputeGBCDPoleFigure
+  ComputeGBCDPoleFigureDirect
+  ComputeGBCDPoleFigureScanline
   ComputeGBPDMetricBased
   ComputeIPFColors
+  ComputeIPFColorsDirect
+  ComputeIPFColorsScanline
   ComputeKernelAvgMisorientations
   ComputeMisorientations
   ComputeQuaternionConjugate
+  ComputeQuaternionConjugateDirect
+  ComputeQuaternionConjugateScanline
   ComputeSchmids
   ComputeShapes
   ComputeShapesTriangleGeom
@@ -455,5 +463,3 @@ if(${PLUGIN_NAME}_INSTALL_DATA_FILES)
     COMPONENT Applications
   )
 endif()
-
-

src/Plugins/OrientationAnalysis/docs/AlignSectionsMisorientationFilter.md

@@ -45,6 +45,20 @@ In this new structure, what follows is what the created structures represent:
 In previous versions a file would have been produced instead. If you wish to recreate this, you can write the Attribute Matrix as a CSV/Text file.
 
 
+## Algorithm
+
+### In-Core Path
+
+For each pair of adjacent Z-sections, the algorithm computes the misorientation between voxels across the section boundary. It tests candidate X-Y shifts to find the shift that minimizes the total misorientation between the two sections. All voxel comparisons use direct array indexing with `operator[]`.
+
+### Out-of-Core Path
+
+Reads pairs of adjacent Z-slices into local memory buffers using `copyIntoBuffer()`. All misorientation comparisons for a given slice pair operate entirely on the in-memory buffers. This converts what would otherwise be random cross-slice element access into sequential bulk reads, avoiding chunk thrashing when data is stored on disk in compressed chunks.
+
+### Performance
+
+The OOC optimization matters most for large datasets that exceed available RAM. By reading entire Z-slices in bulk rather than accessing individual voxels across slice boundaries, the algorithm avoids repeatedly decompressing the same disk chunks. For in-memory datasets, the two paths produce identical results with negligible overhead difference.
+
 % Auto generated parameter table will be inserted here
 
 ## Example Pipelines

src/Plugins/OrientationAnalysis/docs/AlignSectionsMutualInformationFilter.md

@@ -49,6 +49,20 @@ In this new structure, what follows is what the created structures represent:
 
 In previous versions a file would have been produced instead. If you wish to recreate this, you can write the Attribute Matrix as a CSV/Text file.
 
+## Algorithm
+
+### In-Core Path
+
+Aligns Z-sections by maximizing the mutual information of orientation data between adjacent slices. The algorithm segments each slice into temporary features, bins the orientations, and computes joint and marginal histograms to evaluate mutual information at each candidate shift position. All orientation and feature ID data is accessed through direct array indexing with `operator[]`.
+
+### Out-of-Core Path
+
+Reads the orientation and phase data for each pair of adjacent Z-slices into local memory buffers using `copyIntoBuffer()` before computing histograms. This eliminates per-voxel out-of-core reads during the histogram binning and mutual information calculation, replacing them with two sequential bulk reads per slice pair.
+
+### Performance
+
+The OOC optimization matters most for large datasets that exceed available RAM. Histogram computation requires visiting every voxel in both slices multiple times (once per candidate shift), so eliminating per-element OOC access prevents repeated decompression of the same disk chunks. For in-memory datasets, the two paths produce identical results with negligible overhead difference.
+
 % Auto generated parameter table will be inserted here
 
 ## References

src/Plugins/OrientationAnalysis/docs/BadDataNeighborOrientationCheckFilter.md

@@ -47,6 +47,32 @@ The filter allocates a temporary `int32` neighbor-count array sized to the total
 (4 bytes per voxel). For a 1-billion-voxel dataset, that is approximately 4 GB of additional
 working memory during execution. This memory is released when the filter finishes.
 
+### Implementation Notes
+
+The filter selects between two implementations at runtime based on how the input arrays are stored.
+
+#### In-Core Path (BadDataNeighborOrientationCheckWorklist)
+
+When all arrays reside in contiguous in-memory storage, the algorithm uses a two-phase worklist approach:
+
+1. **Phase 1 (Initial count)**: A single linear scan counts matching good face-neighbors for every bad voxel, storing the count in a per-voxel array (the neighbor-count array described under *Memory Considerations*).
+2. **Phase 2 (Worklist propagation)**: For each level, a deque is seeded with all bad voxels meeting the threshold. As each voxel is flipped, its still-bad neighbors' counts are incremented. If a neighbor's count now meets the threshold, it is enqueued. This breadth-first flood-fill processes each voxel at most once per level, achieving O(flipped) amortized cost.
+
+#### Out-of-Core Path (BadDataNeighborOrientationCheckScanline)
+
+When any of the quaternion, mask, or phase arrays are backed by chunked (OOC) disk storage, the algorithm uses a 3-slice rolling window over the Z axis:
+
+1. Three Z-slices of quaternions, phases, and mask data are maintained in memory (previous, current, next).
+2. For each bad voxel in the current slice, the count of matching good face-neighbors is recomputed on-the-fly using the rolling window buffers.
+3. If a voxel is flipped, the mask change is written back to the OOC store per-slice via `copyFromBuffer()`.
+4. The window shifts forward one Z-slice at a time, with only one new slice loaded per step.
+
+This approach trades recomputation (no persistent neighbor-count array) for strictly sequential I/O that avoids the random-access chunk thrashing that would occur with the worklist variant's BFS pattern.
+
+#### Performance
+
+The in-core worklist variant is significantly faster for datasets that fit in RAM because each voxel is processed at most once per level (O(flipped) cost vs. O(N * passes) for the scanline variant). The OOC scanline variant is slower in absolute terms but avoids catastrophic performance degradation on disk-backed datasets where the worklist's random access pattern would trigger continuous chunk load/evict cycles. Memory usage is O(N) for the worklist variant vs. O(3 * sliceSize) for the scanline variant.
+
 ## Example Data
 
 | Example Input Image                                                       | Example Output Image                                                                                                                          |

src/Plugins/OrientationAnalysis/docs/CAxisSegmentFeaturesFilter.md

@@ -47,6 +47,22 @@ is to still use the 6 face neighbors ("Face Only") in order to stay consistent w
 |:--:|:--:|
 | ![Shared Edges & Points With Disconnected Region - "Face Only"](Images/SegmentFeatures/combination_face_only.png) | ![Shared Edges & Points With Disconnected Region - "All Connected"](Images/SegmentFeatures/combination_all_connected.png) |
 
+## Algorithm
+
+This filter segments voxels into features based on c-axis alignment, targeting hexagonal crystal systems. Voxels whose c-axes (the <001> crystallographic direction) are aligned within a user-defined tolerance are grouped into the same feature.
+
+### In-Core Path
+
+Uses a BFS-style flood fill where seed voxels are compared to their neighbors by computing the c-axis misalignment via direct array access with `operator[]`. Neighbors within tolerance are added to the current feature and queued for further expansion.
+
+### Out-of-Core Path
+
+Adapted to use sequential data access through the `SegmentFeatures` base class OOC support. The base class manages bulk I/O so that the flood-fill algorithm can proceed without triggering per-voxel OOC reads across chunk boundaries.
+
+### Performance
+
+The OOC optimization matters most for large datasets that exceed available RAM. Flood-fill naturally exhibits random access patterns as it grows regions across Z-slices, which can cause severe chunk thrashing with compressed on-disk storage. The OOC path mitigates this by leveraging the base class sequential access strategy. For in-memory datasets, the two paths produce identical results with negligible overhead difference.
+
 % Auto generated parameter table will be inserted here
 
 ## Example Pipelines