Skip to content

Add ecRad test suite #2637

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
matthias-ibach wants to merge 1 commit into
base: aomp-dev
Choose a base branch
from
Open

Add ecRad test suite #2637

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
255 changes: 254 additions & 1 deletion bin/build_supp.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ $ $AOMP_SUPP/build/cmdlog File with log of all components built
EOF
}

SUPPLEMENTAL_COMPONENTS=${SUPPLEMENTAL_COMPONENTS:-openmpi silo hdf5 fftw ninja xpmem ucx ucc rocmopenmpi hdf5-parallel}
SUPPLEMENTAL_COMPONENTS=${SUPPLEMENTAL_COMPONENTS:-openmpi silo hdf5 netcdf-c netcdf-fortran ecbuild fiat fftw ninja xpmem ucx ucc rocmopenmpi hdf5-parallel}
Comment thread
jplehr marked this conversation as resolved.
Resolved

# rocsmilib and hwloc build invocation moved to build_rocm_sysdeps.sh (libdrm support). rocmsmi is dependent on libdrm and hwloc depends on rocmsmi.
PREREQUISITE_COMPONENTS=${PREREQUISITE_COMPONENTS:-cmake aqlprofile rocm-core}
Expand Down Expand Up @@ -680,6 +680,251 @@ function buildhdf5(){
echo "# $_linkfrom is now symbolic link to $_installdir " >>"$CMDLOGFILE"
}

function buildnetcdfc(){
local _cname="netcdf-c"
local _version=4.10.0
local _installdir=$AOMP_SUPP_INSTALL/$_cname-$_version
local _linkfrom=$AOMP_SUPP/$_cname
local _builddir=$AOMP_SUPP_BUILD/$_cname
BUILD_TYPE=${BUILD_TYPE:-Release}

if checkversion "$_cname" "$_version" "$_linkfrom"; then
return
fi

if [ ! -d "$AOMP_SUPP/hdf5" ] ; then
echo "Info: HDF5 not found at $AOMP_SUPP/hdf5, building it first..."
buildhdf5
fi
local _hdf5_path=$AOMP_SUPP/hdf5

declare -a MYCMAKEOPTS
MYCMAKEOPTS=(-DCMAKE_BUILD_TYPE="$BUILD_TYPE"
-DCMAKE_INSTALL_PREFIX="$_installdir"
-DCMAKE_INSTALL_LIBDIR=lib
-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON
-DCMAKE_C_COMPILER="$AOMP/bin/clang"
-DCMAKE_PREFIX_PATH="$_hdf5_path"
-DBUILD_SHARED_LIBS=ON
-DNETCDF_ENABLE_HDF5=ON
-DNETCDF_ENABLE_CDF5=ON
-DNETCDF_ENABLE_DAP=OFF
-DNETCDF_ENABLE_BYTERANGE=OFF
-DNETCDF_ENABLE_S3=OFF
-DNETCDF_ENABLE_NCZARR=OFF
-DNETCDF_ENABLE_FILTER_SZIP=OFF
-DNETCDF_BUILD_UTILITIES=ON
-DNETCDF_ENABLE_EXAMPLES=OFF
-DNETCDF_ENABLE_TESTS=OFF
-DNETCDF_ENABLE_PARALLEL_TESTS=OFF)

if [ -d "$_builddir" ] ; then
runcmd "rm -rf $_builddir"
fi
runcmd "mkdir -p $_builddir"
runcmd "cd $_builddir"
runcmd "wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v$_version.tar.gz"
runcmd "tar -xzf v$_version.tar.gz"
runcmd "cd netcdf-c-$_version"
if [ -d "$_installdir" ] ; then
runcmd "rm -rf $_installdir"
fi
runcmd "mkdir -p $_installdir"
runcmd "mkdir -p build"
runcmd "cd build"
echo
echo " -----Running cmake ---- "
echo "${AOMP_CMAKE}" "$(shquot "${MYCMAKEOPTS[@]}")" \
"../"

if ! ${AOMP_CMAKE} "${MYCMAKEOPTS[@]}" ../; then
echo "ERROR cmake failed. Cmake flags"
echo " $(shquot "${MYCMAKEOPTS[@]}")"
exit 1
fi
runcmd "make -j${AOMP_JOB_THREADS}"
runcmd "make install"
if [ -L "$_linkfrom" ] ; then
runcmd "rm $_linkfrom"
fi
runcmd "ln -sfr $_installdir $_linkfrom"
echo "# $_linkfrom is now symbolic link to $_installdir " >>"$CMDLOGFILE"
}

function buildnetcdffortran(){
local _cname="netcdf-fortran"
local _version=4.6.2
local _installdir=$AOMP_SUPP_INSTALL/$_cname-$_version
local _linkfrom=$AOMP_SUPP/$_cname
local _builddir=$AOMP_SUPP_BUILD/$_cname
BUILD_TYPE=${BUILD_TYPE:-Release}

if checkversion "$_cname" "$_version" "$_linkfrom"; then
return
fi

if [ ! -d "$AOMP_SUPP/netcdf-c" ] ; then
echo "Info: NetCDF-C not found at $AOMP_SUPP/netcdf-c, building it first..."
buildnetcdfc
fi
local _netcdfc_path=$AOMP_SUPP/netcdf-c
local _hdf5_path=$AOMP_SUPP/hdf5

declare -a MYCMAKEOPTS
MYCMAKEOPTS=(-DCMAKE_BUILD_TYPE="$BUILD_TYPE"
-DCMAKE_INSTALL_PREFIX="$_installdir"
-DCMAKE_INSTALL_LIBDIR=lib
-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON
-DCMAKE_C_COMPILER="$AOMP/bin/clang"
-DCMAKE_Fortran_COMPILER="$AOMP/bin/flang"
-DCMAKE_PREFIX_PATH="$_netcdfc_path;$_hdf5_path"
-DBUILD_SHARED_LIBS=ON
-DENABLE_TESTS=OFF
-DBUILD_EXAMPLES=OFF)

if [ -d "$_builddir" ] ; then
runcmd "rm -rf $_builddir"
fi
runcmd "mkdir -p $_builddir"
runcmd "cd $_builddir"
runcmd "wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v$_version.tar.gz"
runcmd "tar -xzf v$_version.tar.gz"
runcmd "cd netcdf-fortran-$_version"
if [ -d "$_installdir" ] ; then
runcmd "rm -rf $_installdir"
fi
runcmd "mkdir -p $_installdir"
runcmd "mkdir -p build"
runcmd "cd build"
echo
echo " -----Running cmake ---- "
echo "${AOMP_CMAKE}" "$(shquot "${MYCMAKEOPTS[@]}")" \
"../"

if ! ${AOMP_CMAKE} "${MYCMAKEOPTS[@]}" ../; then
echo "ERROR cmake failed. Cmake flags"
echo " $(shquot "${MYCMAKEOPTS[@]}")"
exit 1
fi
runcmd "make -j${AOMP_JOB_THREADS}"
runcmd "make install"
if [ -L "$_linkfrom" ] ; then
runcmd "rm $_linkfrom"
fi
runcmd "ln -sfr $_installdir $_linkfrom"
echo "# $_linkfrom is now symbolic link to $_installdir " >>"$CMDLOGFILE"
}

function buildecbuild(){
local _cname="ecbuild"
local _version=3.16.0
local _installdir=$AOMP_SUPP_INSTALL/$_cname-$_version
local _linkfrom=$AOMP_SUPP/$_cname
local _builddir=$AOMP_SUPP_BUILD/$_cname

if checkversion "$_cname" "$_version" "$_linkfrom"; then
return
fi

declare -a MYCMAKEOPTS
MYCMAKEOPTS=(-DCMAKE_INSTALL_PREFIX="$_installdir"
-DENABLE_TESTS=OFF)

if [ -d "$_builddir" ] ; then
runcmd "rm -rf $_builddir"
fi
runcmd "mkdir -p $_builddir"
runcmd "cd $_builddir"
runcmd "wget https://github.com/ecmwf/ecbuild/archive/refs/tags/$_version.tar.gz"
runcmd "tar -xzf $_version.tar.gz"
runcmd "cd ecbuild-$_version"
if [ -d "$_installdir" ] ; then
runcmd "rm -rf $_installdir"
fi
runcmd "mkdir -p $_installdir"
runcmd "mkdir -p build"
runcmd "cd build"
echo
echo " -----Running cmake ---- "
echo "${AOMP_CMAKE}" "$(shquot "${MYCMAKEOPTS[@]}")" \
"../"

if ! ${AOMP_CMAKE} "${MYCMAKEOPTS[@]}" ../; then
echo "ERROR cmake failed. Cmake flags"
echo " $(shquot "${MYCMAKEOPTS[@]}")"
exit 1
fi
runcmd "make install"
if [ -L "$_linkfrom" ] ; then
runcmd "rm $_linkfrom"
fi
runcmd "ln -sfr $_installdir $_linkfrom"
echo "# $_linkfrom is now symbolic link to $_installdir " >>"$CMDLOGFILE"
}

function buildfiat(){
local _cname="fiat"
local _version=2.0.1
local _installdir=$AOMP_SUPP_INSTALL/$_cname-$_version
local _linkfrom=$AOMP_SUPP/$_cname
local _builddir=$AOMP_SUPP_BUILD/$_cname
BUILD_TYPE=${BUILD_TYPE:-Release}

if checkversion "$_cname" "$_version" "$_linkfrom"; then
return
fi

if [ ! -d "$AOMP_SUPP/ecbuild" ] ; then
echo "Info: ecbuild not found at $AOMP_SUPP/ecbuild, building it first..."
buildecbuild
fi
local _ecbuild_path=$AOMP_SUPP/ecbuild

declare -a MYCMAKEOPTS
MYCMAKEOPTS=(-DCMAKE_BUILD_TYPE="$BUILD_TYPE"

@jplehr jplehr Oct 29, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this a GPU build and if so, for which GPU arch does that build? All of them or the ones in the machine?

@matthias-ibach matthias-ibach Oct 30, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, the build_supp components are CPU-only. ecbuild does not compile anything (only CMake macros) and fiat is built without any --offload-arch. The only GPU build is ecRad in run_ecrad.sh, for the arch detected on the machine.

@jplehr jplehr Oct 30, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are there any concerns when there are multiple GPU archs on that machine? Eg. a CPU with iGPU and another dGPU? Will that still work and pick the correct target (given ROCR_VISIBLE_DEVICES is set?

@matthias-ibach matthias-ibach Oct 30, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That is a good point, which I checked now. The arch comes from setaompgpu (same as for run_qe.sh and run_mglet.sh, I actually based this PR on the PRs where QE and MGLET were added!). It takes the first GPU that rocm_agent_enumerator lists. On an iGPU + dGPU machine it can pick the iGPU arch if that one comes first, ROCR_VISIBLE_DEVICES however will not change that.
Another note is that in run_ecrad.sh I set OMP_TARGET_OFFLOAD=MANDATORY (this was not done for QE and MGLET, so feel free to either keep it or omit it). I thought it might be good if a binary built for the wrong arch aborts (instead of silently just running on the host).

@jplehr jplehr Oct 30, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we make it so that AOMP_GPU takes precedence for which arch to build, when it is set? This would make a couple of places easier in other areas where we might consume these scripts.

@dpalermo dpalermo Oct 30, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any libraries that are built for a specific GPU should be built as part of the test script itself (run_ecrad.sh). The tools & libraries that are built in build_supp.sh are built once and used across a wide variety of gfx targets on different test systems.

@jplehr jplehr Nov 2, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are right! Apologies, I got lost which script I'm commenting on. My comment about AOMP_GPU should be attached to the actual run_* script, not the build_supp.sh.

@matthias-ibach matthias-ibach Nov 2, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dpalermo:
Thank you for making that clear! I think my thought on it was the following your comment. Since ecbuild, fiat and netcdf are not GPU specific I put them in build_supp.sh. The GPU part (ecRad itself) is built in run_ecrad.sh.
Note: another reason why I put ecbuild and fiat into build_supp.sh is that I am working on putting ecTrans also in the CI (coming up soon, already working on it). It needs ecbuild and fiat too (+ LAPACK, also CPU-only), so this way they are only built once and both apps use the same ones.
Is that fine for you, or would you rather have ecbuild and fiat built inside the run scripts, so they would be built once per app?

@jplehr:
Not a problem, now I understand it :) AOMP_GPU should already take precedence (setaompgpu only uses the detected GPU if AOMP_GPU is not set) and then run_ecrad.sh uses GPU_ARCH=${GPU_ARCH:-$AOMP_GPU} (as mentioned I took QE and MGLET as reference).
The only thing that overrides AOMP_GPU is GPU_ARCH, if this is set explicitly.
Should I drop GPU_ARCH from run_ecrad.sh so AOMP_GPU is the only knob, or keep it (like in the MGLET script)?

@jplehr jplehr Nov 2, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry again for causing the confusion. :)

I am not concerned about GPU_ARCH in that it could overwrite AOMP_GPU. Thank you.

@matthias-ibach matthias-ibach Nov 2, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No worries :)
Alright, then I will leave as is! Let me know if there is anything else.

-DCMAKE_INSTALL_PREFIX="$_installdir"
-Decbuild_ROOT="$_ecbuild_path"
-DCMAKE_C_COMPILER="$AOMP/bin/clang"
-DCMAKE_Fortran_COMPILER="$AOMP/bin/flang"
"-DCMAKE_C_FLAGS=-fopenmp -fPIC"
"-DCMAKE_Fortran_FLAGS=-fopenmp -fPIC"
-DENABLE_MPI=OFF
-DENABLE_DR_HOOK_ROCTX=OFF
-DENABLE_TESTS=OFF)

if [ -d "$_builddir" ] ; then
runcmd "rm -rf $_builddir"
fi
runcmd "mkdir -p $_builddir"
runcmd "cd $_builddir"
runcmd "wget https://github.com/ecmwf-ifs/fiat/archive/refs/tags/$_version.tar.gz"
runcmd "tar -xzf $_version.tar.gz"
runcmd "cd fiat-$_version"
if [ -d "$_installdir" ] ; then
runcmd "rm -rf $_installdir"
fi
runcmd "mkdir -p $_installdir"
runcmd "mkdir -p build"
runcmd "cd build"
echo
echo " -----Running cmake ---- "
echo "${AOMP_CMAKE}" "$(shquot "${MYCMAKEOPTS[@]}")" \
"../"

if ! ${AOMP_CMAKE} "${MYCMAKEOPTS[@]}" ../; then
echo "ERROR cmake failed. Cmake flags"
echo " $(shquot "${MYCMAKEOPTS[@]}")"
exit 1
fi
runcmd "make -j${AOMP_JOB_THREADS}"
runcmd "make install"
if [ -L "$_linkfrom" ] ; then
runcmd "rm $_linkfrom"
fi
runcmd "ln -sfr $_installdir $_linkfrom"
echo "# $_linkfrom is now symbolic link to $_installdir " >>"$CMDLOGFILE"
}

function buildsilo(){
local _cname="silo"
local _version=4.11.1
Expand Down Expand Up @@ -916,6 +1161,14 @@ for _component in $_components ; do
buildhdf5
elif [ "$_component" == "hdf5-parallel" ] ; then
buildhdf5 "-parallel"
elif [ "$_component" == "netcdf-c" ] ; then
buildnetcdfc
elif [ "$_component" == "netcdf-fortran" ] ; then
buildnetcdffortran
elif [ "$_component" == "ecbuild" ] ; then
buildecbuild
elif [ "$_component" == "fiat" ] ; then
buildfiat
elif [ "$_component" == "fftw" ] ; then
buildfftw
elif [ "$_component" == "hwloc" ] ; then
Expand Down
Loading
Loading