Split gotoblas dispatch table by group / operation (1/2)

.gitignore

@@ -120,3 +120,4 @@ CMakeCache.txt
 CMakeFiles/*
 .vscode
 **/__pycache__
+dyn_cores.h

Makefile

@@ -449,7 +449,7 @@ clean ::
 ifeq ($(OSNAME), Darwin)
 	@rm -rf getarch.dSYM getarch_2nd.dSYM
 endif
-	@rm -f Makefile.conf config.h Makefile_kernel.conf config_kernel.h st* *.dylib
+	@rm -f Makefile.conf config.h Makefile_kernel.conf config_kernel.h dyn_cores.h st* *.dylib
 	@rm -f cblas.tmp cblas.tmp2
 	@touch $(NETLIB_LAPACK_DIR)/make.inc
 	@$(MAKE) -C $(NETLIB_LAPACK_DIR) clean

Makefile.system

@@ -867,6 +867,15 @@ override DYNAMIC_ARCH=
 endif
 endif
 
+# dyn_cores.h tells common_param.h which cores this DYNAMIC_ARCH build has.
+ifeq ($(DYNAMIC_ARCH), 1)
+ifndef OPENBLAS_DYN_CORES
+export OPENBLAS_DYN_CORES := $(sort $(DYNAMIC_CORE))
+HASH := \#
+DUMMY := $(shell echo '$(HASH)define OPENBLAS_CORE_LIST(X, arg) $(foreach c,$(OPENBLAS_DYN_CORES),X($(c), arg))' > $(TOPDIR)/dyn_cores.h)
+endif
+endif
+
 ifeq ($(ARCH), ia64)
 NO_BINARY_MODE	= 1
 BINARY_DEFINED	= 1
@@ -1484,6 +1493,9 @@ endif
 
 ifeq ($(DYNAMIC_ARCH), 1)
 CCOMMON_OPT	+= -DDYNAMIC_ARCH
+ifeq ($(C_COMPILER), $(filter $(C_COMPILER),GCC LSB CLANG INTEL))
+CCOMMON_OPT	+= -ffunction-sections -fdata-sections
+endif
 endif
 
 ifeq ($(DYNAMIC_OLDER), 1)

cmake/arch.cmake

@@ -129,9 +129,26 @@ if (DYNAMIC_ARCH)
 	  message (FATAL_ERROR "Your build directory contains a file config_kernel.h, probably from a previous compilation with make. This will conflict with the cmake compilation and cause strange compiler errors - please remove the file before trying again")
   endif ()
 
+  if (EXISTS ${PROJECT_SOURCE_DIR}/dyn_cores.h)
+	  message (FATAL_ERROR "Your source directory contains a file dyn_cores.h, probably from a previous compilation with make. It would be used instead of the one generated for this build - please remove the file before trying again")
+  endif ()
+
   if (NOT DYNAMIC_CORE)
     message (STATUS "DYNAMIC_ARCH is not supported on this architecture, removing from options")
     unset(DYNAMIC_ARCH CACHE)
+  else ()
+    # dyn_cores.h tells common_param.h which cores this build has;
+    # see the equivalent rule in Makefile.system.
+    set(DYN_CORES_LIST ${DYNAMIC_CORE})
+    list(REMOVE_DUPLICATES DYN_CORES_LIST)
+    list(SORT DYN_CORES_LIST)
+    set(DYN_CORES_H "#define OPENBLAS_CORE_LIST(X, arg)")
+    foreach (DCORE ${DYN_CORES_LIST})
+      string(APPEND DYN_CORES_H " X(${DCORE}, arg)")
+    endforeach ()
+    file(WRITE ${PROJECT_BINARY_DIR}/dyn_cores.h.tmp "${DYN_CORES_H}\n")
+    configure_file(${PROJECT_BINARY_DIR}/dyn_cores.h.tmp ${PROJECT_BINARY_DIR}/dyn_cores.h COPYONLY)
+    file(REMOVE ${PROJECT_BINARY_DIR}/dyn_cores.h.tmp)
   endif ()
 endif ()
 

cmake/system.cmake

@@ -536,6 +536,9 @@ endif ()
 if (DYNAMIC_ARCH)
   if (X86 OR X86_64 OR ARM64 OR POWER OR RISCV64 OR LOONGARCH64 OR ZARCH)
     set(CCOMMON_OPT "${CCOMMON_OPT} -DDYNAMIC_ARCH")
+    if (CMAKE_C_COMPILER_ID MATCHES "GNU|Clang|Intel" AND NOT MSVC)
+      set(CCOMMON_OPT "${CCOMMON_OPT} -ffunction-sections -fdata-sections")
+    endif ()
     if (DYNAMIC_OLDER)
       set(CCOMMON_OPT "${CCOMMON_OPT} -DDYNAMIC_OLDER")
     endif ()

common_c.h

@@ -501,6 +501,7 @@
 
 #endif
 
+#define CGEMM_SMALL_KERNEL_BASE		FUNC_BASE(cgemm_small_kernel_nn)
 #define CGEMM_SMALL_KERNEL_NN		FUNC_OFFSET(cgemm_small_kernel_nn)
 #define CGEMM_SMALL_KERNEL_NT		FUNC_OFFSET(cgemm_small_kernel_nt)
 #define CGEMM_SMALL_KERNEL_NR		FUNC_OFFSET(cgemm_small_kernel_nr)

common_d.h

@@ -344,6 +344,7 @@
 
 #endif
 
+#define DGEMM_SMALL_KERNEL_BASE		FUNC_BASE(dgemm_small_kernel_nn)
 #define DGEMM_SMALL_KERNEL_NN		FUNC_OFFSET(dgemm_small_kernel_nn)
 #define DGEMM_SMALL_KERNEL_NT		FUNC_OFFSET(dgemm_small_kernel_nt)
 #define DGEMM_SMALL_KERNEL_TN		FUNC_OFFSET(dgemm_small_kernel_tn)

common_macro.h

@@ -697,6 +697,7 @@
 
 #define GEMM_SMALL_MATRIX_PERMIT	DGEMM_SMALL_MATRIX_PERMIT
 
+#define GEMM_SMALL_KERNEL_BASE  DGEMM_SMALL_KERNEL_BASE
 #define GEMM_SMALL_KERNEL_NN    DGEMM_SMALL_KERNEL_NN
 #define GEMM_SMALL_KERNEL_NT    DGEMM_SMALL_KERNEL_NT
 #define GEMM_SMALL_KERNEL_TN    DGEMM_SMALL_KERNEL_TN
@@ -1103,6 +1104,7 @@
 
 #define GEMM_SMALL_MATRIX_PERMIT	SBGEMM_SMALL_MATRIX_PERMIT
 
+#define GEMM_SMALL_KERNEL_BASE  SBGEMM_SMALL_KERNEL_BASE
 #define GEMM_SMALL_KERNEL_NN    SBGEMM_SMALL_KERNEL_NN
 #define GEMM_SMALL_KERNEL_NT    SBGEMM_SMALL_KERNEL_NT
 #define GEMM_SMALL_KERNEL_TN    SBGEMM_SMALL_KERNEL_TN
@@ -1446,6 +1448,7 @@
 
 #define GEMM_SMALL_MATRIX_PERMIT	SGEMM_SMALL_MATRIX_PERMIT
 
+#define GEMM_SMALL_KERNEL_BASE  SGEMM_SMALL_KERNEL_BASE
 #define GEMM_SMALL_KERNEL_NN    SGEMM_SMALL_KERNEL_NN
 #define GEMM_SMALL_KERNEL_NT    SGEMM_SMALL_KERNEL_NT
 #define GEMM_SMALL_KERNEL_TN    SGEMM_SMALL_KERNEL_TN
@@ -2393,6 +2396,7 @@
 
 #define GEMM_SMALL_MATRIX_PERMIT	ZGEMM_SMALL_MATRIX_PERMIT
 
+#define GEMM_SMALL_KERNEL_BASE  ZGEMM_SMALL_KERNEL_BASE
 #define GEMM_SMALL_KERNEL_NN    ZGEMM_SMALL_KERNEL_NN
 #define GEMM_SMALL_KERNEL_NT    ZGEMM_SMALL_KERNEL_NT
 #define GEMM_SMALL_KERNEL_NR    ZGEMM_SMALL_KERNEL_NR
@@ -2913,6 +2917,7 @@
 
 #define GEMM_SMALL_MATRIX_PERMIT	CGEMM_SMALL_MATRIX_PERMIT
 
+#define GEMM_SMALL_KERNEL_BASE  CGEMM_SMALL_KERNEL_BASE
 #define GEMM_SMALL_KERNEL_NN    CGEMM_SMALL_KERNEL_NN
 #define GEMM_SMALL_KERNEL_NT    CGEMM_SMALL_KERNEL_NT
 #define GEMM_SMALL_KERNEL_NR    CGEMM_SMALL_KERNEL_NR

common_param.h

@@ -44,7 +44,19 @@
 
 #ifdef DYNAMIC_ARCH
 
+/* OPENBLAS_CORE_LIST(X, arg) expands to X(CORE, arg) for each core in this
+   build.  dyn_cores.h is generated from DYNAMIC_CORE by the build system. */
+#include "dyn_cores.h"
+
+#define OPENBLAS_CORE_ENUM(core, unused) OPENBLAS_CORE_##core,
+enum { OPENBLAS_CORE_LIST(OPENBLAS_CORE_ENUM, ~) OPENBLAS_NUM_CORES };
+
+#define OPENBLAS_DISPATCH(group)		(openblas_##group##_dispatch[gotoblas->core])
+#define OPENBLAS_DISPATCH_OFFSET(group, func)	(size_t)(&((openblas_##group##_dispatch_t *)NULL)->func)
+#define OPENBLAS_DISPATCH_BASE(group)		((char *)OPENBLAS_DISPATCH(group))
+
 typedef struct {
+  int core;
   int dtb_entries;
   int switch_ratio;
   int divide_rate;
@@ -1322,6 +1334,8 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL
 extern gotoblas_t *gotoblas;
 
 #define FUNC_OFFSET(func)	(size_t)(&((gotoblas_t *)NULL)->func)
+/* The address that FUNC_OFFSET(func) is an offset from. */
+#define FUNC_BASE(func)	((char *)gotoblas)
 
 #define DTB_ENTRIES  gotoblas -> dtb_entries
 #define GEMM_OFFSET_A	gotoblas -> offsetA
@@ -1461,6 +1475,7 @@ extern gotoblas_t *gotoblas;
 #else
 
 #define FUNC_OFFSET(func)	(size_t)(func)
+#define OPENBLAS_DISPATCH_OFFSET(group, func)	(size_t)(func)
 
 #define DTB_ENTRIES  DTB_DEFAULT_ENTRIES
 

common_s.h

@@ -392,6 +392,7 @@
 
 #endif
 
+#define SGEMM_SMALL_KERNEL_BASE		FUNC_BASE(sgemm_small_kernel_nn)
 #define SGEMM_SMALL_KERNEL_NN		FUNC_OFFSET(sgemm_small_kernel_nn)
 #define SGEMM_SMALL_KERNEL_NT		FUNC_OFFSET(sgemm_small_kernel_nt)
 #define SGEMM_SMALL_KERNEL_TN		FUNC_OFFSET(sgemm_small_kernel_tn)

common_sb.h

@@ -45,6 +45,7 @@
 #define SBGEMM_SMALL_MATRIX_PERMIT	gotoblas -> sbgemm_small_matrix_permit
 #endif
 
+#define SBGEMM_SMALL_KERNEL_BASE		FUNC_BASE(sbgemm_small_kernel_nn)
 #define SBGEMM_SMALL_KERNEL_NN		FUNC_OFFSET(sbgemm_small_kernel_nn)
 #define SBGEMM_SMALL_KERNEL_NT		FUNC_OFFSET(sbgemm_small_kernel_nt)
 #define SBGEMM_SMALL_KERNEL_TN		FUNC_OFFSET(sbgemm_small_kernel_tn)

common_z.h

@@ -501,6 +501,7 @@
 
 #endif
 
+#define ZGEMM_SMALL_KERNEL_BASE		FUNC_BASE(zgemm_small_kernel_nn)
 #define ZGEMM_SMALL_KERNEL_NN		FUNC_OFFSET(zgemm_small_kernel_nn)
 #define ZGEMM_SMALL_KERNEL_NT		FUNC_OFFSET(zgemm_small_kernel_nt)
 #define ZGEMM_SMALL_KERNEL_NR		FUNC_OFFSET(zgemm_small_kernel_nr)

driver/others/CMakeLists.txt

@@ -51,6 +51,7 @@ GenerateNamedObjects("abs.c" "DOUBLE" "z_abs" 0 "" "" 1)
 GenerateNamedObjects("openblas_get_config.c;openblas_get_parallel.c" "" "" 0 "" "" 1)
 
 if (DYNAMIC_ARCH)
+  list(APPEND COMMON_SOURCES dispatch.c)
   if (ARM64)
     list(APPEND COMMON_SOURCES dynamic_arm64.c)
   elseif (POWER)

driver/others/Makefile

@@ -15,6 +15,7 @@ endif
 # COMMONOBJS	+=  info.$(SUFFIX)
 
 ifeq ($(DYNAMIC_ARCH), 1)
+COMMONOBJS	+=  dispatch.$(SUFFIX)
 ifeq ($(ARCH),arm64)
 COMMONOBJS	+=  dynamic_arm64.$(SUFFIX)
 else
@@ -133,6 +134,9 @@ dynamic%$(SUFFIX) : dynamic%c
 dynamic%$(PSUFFIX) : dynamic%c
 	$(CC) $(PFLAGS) -c $< -o $(@F)
 
+dispatch.$(SUFFIX) : dispatch.c
+	$(CC) $(CFLAGS) -c $< -o $(@F)
+
 parameter.$(SUFFIX) : parameter.c ../../param.h
 	$(CC) $(CFLAGS) -c $< -o $(@F)
 

driver/others/dispatch.c

@@ -0,0 +1,49 @@
+/*****************************************************************************
+Copyright (c) 2026, The OpenBLAS Project
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   1. Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+   3. Neither the name of the OpenBLAS project nor the names of its contributors may
+      be used to endorse or promote products derived from this software
+      without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/* The arrays of per-core kernel tables that OPENBLAS_DISPATCH(group)
+   indexes (see common_param.h), one per kernel group. */
+
+#include "common.h"
+
+#define DISPATCH_EXTERN(core, group) \
+  extern const openblas_##group##_dispatch_t openblas_##group##_dispatch_##core;
+#define DISPATCH_ENTRY(core, group) \
+  [OPENBLAS_CORE_##core] = &openblas_##group##_dispatch_##core,
+
+/* openblas_<group>_dispatch[OPENBLAS_CORE_<CORE>] points to
+   openblas_<group>_dispatch_<CORE>, which kernel/setparam-ref.c defines for
+   that core. */
+#define DEFINE_DISPATCH_TABLE(group) \
+  OPENBLAS_CORE_LIST(DISPATCH_EXTERN, group) \
+  const openblas_##group##_dispatch_t *const openblas_##group##_dispatch[OPENBLAS_NUM_CORES] = { \
+    OPENBLAS_CORE_LIST(DISPATCH_ENTRY, group) \
+  };

interface/gemm.c

@@ -128,7 +128,7 @@ static int (*gemm[])(blas_arg_t *, BLASLONG *, BLASLONG *, IFLOAT *, IFLOAT *, B
 #ifndef DYNAMIC_ARCH
 #define SMALL_KERNEL_ADDR(table, idx) ((void *)(table[idx]))
 #else
-#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)((char *)gotoblas + (size_t)(table[idx]))))
+#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)(GEMM_SMALL_KERNEL_BASE + (size_t)(table[idx]))))
 #endif
 
 

interface/gemm_batch.c

@@ -75,7 +75,7 @@ static int (*gemm[])(blas_arg_t *, BLASLONG *, BLASLONG *, IFLOAT *, IFLOAT *, B
 #ifndef DYNAMIC_ARCH
 #define SMALL_KERNEL_ADDR(table, idx) ((void *)(table[idx]))
 #else
-#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)((char *)gotoblas + (size_t)(table[idx]))))
+#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)(GEMM_SMALL_KERNEL_BASE + (size_t)(table[idx]))))
 #endif
 
 

interface/gemm_batch_strided.c

@@ -75,7 +75,7 @@ static int (*gemm[])(blas_arg_t *, BLASLONG *, BLASLONG *, IFLOAT *, IFLOAT *, B
 #ifndef DYNAMIC_ARCH
 #define SMALL_KERNEL_ADDR(table, idx) ((void *)(table[idx]))
 #else
-#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)((char *)gotoblas + (size_t)(table[idx]))))
+#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)(GEMM_SMALL_KERNEL_BASE + (size_t)(table[idx]))))
 #endif
 
 

kernel/CMakeLists.txt

@@ -1601,7 +1601,7 @@ endif ()
     add_library(kernel${TSUFFIX} OBJECT ${OPENBLAS_SRC})
     set_target_properties(kernel${TSUFFIX} PROPERTIES COMPILE_FLAGS "${KERNEL_DEFINITIONS}")
     get_target_property(KERNEL_INCLUDE_DIRECTORIES kernel${TSUFFIX} INCLUDE_DIRECTORIES)
-    set_target_properties(kernel${TSUFFIX} PROPERTIES INCLUDE_DIRECTORIES "${KERNEL_INCLUDE_DIRECTORIES};${TARGET_CONF_DIR}")
+    set_target_properties(kernel${TSUFFIX} PROPERTIES INCLUDE_DIRECTORIES "${KERNEL_INCLUDE_DIRECTORIES};${TARGET_CONF_DIR};${PROJECT_BINARY_DIR}")
     if (USE_GEMM3M)
       target_compile_definitions(kernel${TSUFFIX} PRIVATE USE_GEMM3M)
     endif()

kernel/setparam-ref.c

@@ -51,6 +51,8 @@ extern char* gotoblas_corename(void);
 static void init_parameter(void);
 
 gotoblas_t TABLE_NAME = {
+  OPENBLAS_CORETS,
+
   DTB_DEFAULT_ENTRIES,
 
   SWITCH_RATIO,