Extra apps

configuration/cirrus-ex.py

@@ -3,7 +3,6 @@
 from reframe.core.backends import register_launcher
 from reframe.core.launchers import JobLauncher
 
-
 @register_launcher("torchrun")
 class TorchRunLauncher(JobLauncher):
     """Launcher for Torch run"""
@@ -51,7 +50,7 @@ def command(self, job):
                         "PrgEnv-aocc",
                         "PrgEnv-intel"
                     ],
-                    "max_jobs": 64,
+                    "max_jobs": 10,
                     "processor": {
                         "num_cpus": 288,
                         "num_cpus_per_socket": 144,

pyproject.toml

@@ -8,6 +8,7 @@ license = {file = "LICENSE"}
 
 [tool.black]
 line-length = 120
+target-version = ["py311"]
 
 [tool.flake8]
 max-line-length = 120

tests/apps/gromacs/gmx_1400k_atoms.py

@@ -22,6 +22,7 @@ class Gromacs1400kAtomsBase(GromacsBaseCheck):
             "archer2-tds:compute": 128,
             "cirrus:compute": 36,
             "cirrus:compute-gpu": 40,
+            "cirrus-ex:compute": 288,
         },
     )
 
@@ -30,15 +31,26 @@ class Gromacs1400kAtomsBase(GromacsBaseCheck):
     reference = {
         "archer2:compute": {
             "energy": (energy_reference, -0.01, 0.01, "kJ/mol"),
+            "performance": (24.0, -0.1, None, "ns/day"),
         },
         "archer2-tds:compute": {
             "energy": (energy_reference, -0.01, 0.01, "kJ/mol"),
+            "performance": (22.4, -0.1, None, "ns/day"),
         },
         "cirrus:compute": {
             "energy": (energy_reference, -0.01, 0.01, "kJ/mol"),
+            "performance": (5.50, -0.1, None, "ns/day"),
         },
         "cirrus:compute-gpu": {
             "energy": (energy_reference, -0.01, 0.01, "kJ/mol"),
+            "performance": (11.5, -0.05, None, "ns/day"),
+        },
+        "cirrus-ex:compute": {
+            "energy": (energy_reference, -0.01, 0.01, "kJ/mol"),
+            # The performance of this benchmark seems to be very variable,
+            # with performance sometimes dropping below 50 ns/day.
+            # Disabling the performance check for now until we can investigate this further.
+            # "performance": (78.0, -0.1, None, "ns/day"),
         },
     }
 
@@ -47,26 +59,21 @@ class Gromacs1400kAtomsBase(GromacsBaseCheck):
 class GromacsCPUCheck(Gromacs1400kAtomsBase):
     """Gromacs CPU checks"""
 
-    valid_systems = ["archer2:compute", "cirrus:compute"]
+    valid_systems = ["archer2:compute", "cirrus:compute", "cirrus-ex:compute"]
     modules = ["gromacs"]
     descr = Gromacs1400kAtomsBase.descr + " -- CPU"
 
     n_nodes = 4
-    num_tasks = 128
+
     num_cpus_per_task = 1
     env_vars = {"OMP_NUM_THREADS": str(num_cpus_per_task)}
-    reference["archer2:compute"] = {}
-    reference["archer2-tds:compute"] = {}
-    reference["cirrus:compute"] = {}
 
-    reference["archer2:compute"]["performance"] = (24.0, -0.1, None, "ns/day")
-    reference["archer2-tds:compute"]["performance"] = (22.4, -0.1, None, "ns/day")
-    reference["cirrus:compute"]["performance"] = (5.50, -0.1, None, "ns/day")
+    tags = {"applications", "performance", "small"}
 
     @run_before("run")
     def setup_resources(self):
         """sets up number of tasks"""
-        self.num_tasks_per_node = self.cores.get(self.current_partition.fullname, 1)
+        self.num_tasks_per_node = self.current_partition.processor.num_cpus
         self.num_tasks = self.n_nodes * self.num_tasks_per_node
 
 
@@ -89,8 +96,6 @@ class GromacsGPUCheck(Gromacs1400kAtomsBase):
     n_nodes = 1
     num_tasks = None
     num_cpus_per_tasks = None
-    reference["cirrus:compute-gpu"] = {}
-    reference["cirrus:compute-gpu"]["performance"] = (11.5, -0.05, None, "ns/day")
 
     @run_after("setup")
     def setup_gpu_options(self):

tests/apps/lammps/ethanol.py

@@ -32,10 +32,22 @@ class LAMMPSBaseEthanol(LAMMPSBase):
     ethanol_energy_reference = 537394.35
 
     reference = {
-        "cirrus:compute-gpu": {"energy": (ethanol_energy_reference, -0.01, 0.01, "kJ/mol")},
-        "archer2:compute": {"energy": (ethanol_energy_reference, -0.01, 0.01, "kJ/mol")},
-        "cirrus-ex:compute": {"energy": (ethanol_energy_reference, -0.01, 0.01, "kJ/mol")},
-        "archer2-tds:compute": {"energy": (ethanol_energy_reference, -0.01, 0.01, "kJ/mol")},
+        "cirrus:compute-gpu": {
+            "energy": (ethanol_energy_reference, -0.01, 0.01, "kJ/mol"),
+            "performance": (9.4, -0.05, None, "ns/day"),
+        },
+        "archer2:compute": {
+            "energy": (ethanol_energy_reference, -0.01, 0.01, "kJ/mol"),
+            "performance": (11.250, -0.05, None, "ns/day"),
+        },
+        "cirrus-ex:compute": {
+            "energy": (ethanol_energy_reference, -0.01, 0.01, "kJ/mol"),
+            "performance": (26, -0.1, None, "ns/day"),
+        },
+        "archer2-tds:compute": {
+            "energy": (ethanol_energy_reference, -0.01, 0.01, "kJ/mol"),
+            "performance": (11.250, -0.05, None, "ns/day"),
+        },
     }
 
     @performance_function("kJ/mol", perf_key="energy")
@@ -55,15 +67,6 @@ class LAMMPSEthanolCPU(LAMMPSBaseEthanol):
 
     descr = LAMMPSBaseEthanol.descr + " -- CPU"
 
-    reference["archer2-tds:compute"] = {}
-    reference["archer2:compute"] = {}
-    reference["cirrus-ex:compute"] = {}
-
-    reference["archer2:compute"]["performance"] = (11.250, -0.05, None, "ns/day")
-    reference["cirrus-ex:compute"]["performance"] = (27.56, -0.05, None, "ns/day")
-
-    reference["archer2-tds:compute"]["performance"] = (11.250, -0.05, None, "ns/day")
-
     @run_after("init")
     def setup_nnodes(self):
         """sets up number of tasks per node"""
@@ -127,8 +130,6 @@ class LAMMPSEthanolGPU(LAMMPSBaseEthanol):
     n_nodes = 1
     num_tasks = None
     num_cpus_per_task = None
-    reference["cirrus:compute-gpu"] = {}
-    reference["cirrus:compute-gpu"]["performance"] = (9.4, -0.05, None, "ns/day")
 
     @run_after("init")
     def setup_nnodes(self):

tests/apps/lammps/exaalt.py

@@ -1,33 +1,30 @@
 """ReFrame script for lammps dipole test"""
 
-import os
-
 import reframe as rfm
 import reframe.utility.sanity as sn
 
-from lammps_base import BuildLAMMPS, LAMMPSBase
+from lammps_base import LAMMPSBase
 
 
 class LAMMPSBaseExaalt(LAMMPSBase):
     """ReFrame LAMMPS Base class for Exaalt tests"""
 
-    valid_systems = ["archer2:compute"]
-    stream_binary = fixture(BuildLAMMPS, scope="environment")
     num_cpus_per_task = 1
     env_vars = {"OMP_NUM_THREADS": str(num_cpus_per_task)}
-
+    modules = ["lammps"]
     cores = variable(
         dict,
-        value={
-            "archer2:compute": 128,
-        },
+        value={"archer2:compute": 128, "cirrus-ex:compute": 288},
     )
 
     reference = {
         "archer2:compute": {
             "energy": (-8.7467248, -0.001, 0.001, "kJ/mol"),
             "performance": (0.007, -0.1, None, "ns/day"),
         },
+        "cirrus-ex:compute": {
+            "performance": (0.055, -0.15, None, "ns/day"),
+        },
     }
 
     @run_after("init")
@@ -39,7 +36,7 @@ def setup_nnodes(self):
     @run_after("setup")
     def set_executable(self):
         """sets up executable"""
-        self.executable = os.path.join(self.stream_binary.build_system.builddir, "lmp")
+        self.executable = "lmp"
 
     @run_before("run")
     def setup_resources(self):
@@ -62,36 +59,41 @@ def extract_energy(self):
 class LAMMPSExaaltSmall(LAMMPSBaseExaalt):
     """ReFrame LAMMPS small test based on NERSC-10 Exaalt benchmark"""
 
+    valid_systems = ["archer2:compute", "cirrus-ex:compute"]
     descr = "Small performance test using NERSC-10 Exaalt LAMMPS benchmark reference run"
-    tags = {"applications", "performance"}
+    tags = {"applications", "performance", "small"}
     executable_opts = [
         "-in in.snap.test",
         "-var snapdir 2J8_W.SNAP",
-        "-var nx 256",
-        "-var ny 256",
-        "-var nz 256",
-        "-var nsteps 100",
+        "-var nx 128",
+        "-var ny 128",
+        "-var nz 128",
+        "-var nsteps 200",
     ]
 
-    n_nodes = 16
+    n_nodes = 4
     time_limit = "30m"
 
 
 @rfm.simple_test
 class LAMMPSExaaltRef(LAMMPSBaseExaalt):
     """ReFrame LAMMPS largescale test based on NERSC-10 Exaalt benchmark"""
 
+    valid_systems = ["archer2:compute", "cirrus-ex:compute"]
+
+    """ReFrame LAMMPS largescale test based on NERSC-10 Exaalt benchmark"""
+
     descr = "Largescale performance test using NERSC-10 Exaalt LAMMPS benchmark reference run"
     tags = {"largescale", "applications", "performance"}
 
     executable_opts = [
         "-in in.snap.test",
         "-var snapdir 2J8_W.SNAP",
-        "-var nx 1024",
-        "-var ny 1024",
-        "-var nz 1024",
+        "-var nx 512",
+        "-var ny 512",
+        "-var nz 512",
         "-var nsteps 100",
     ]
 
-    n_nodes = 1024
+    n_nodes = 256
     time_limit = "30m"

tests/apps/openfoam/openfoam_org_cavity3d.py

@@ -0,0 +1,65 @@
+"""ReFrame test for OpenFOAM 3D lid-driven cavity."""
+
+import reframe as rfm
+import reframe.utility.sanity as sn
+
+from openfoam_org_base import OpenFOAMBase
+
+
+class OpenFOAMCavity3DBase(OpenFOAMBase):
+    """OpenFOAM 3D lid-driven cavity base test."""
+
+    valid_systems = ["cirrus-ex:compute"]
+
+    num_cpus_per_task = 1
+    time_limit = "20m"
+
+    @run_before("run")
+    def setup_tasks(self):
+        """Set up the number of tasks and tasks per node for the test."""
+        self.num_tasks_per_node = self.current_partition.processor.num_cpus
+        self.num_tasks = self.num_nodes * self.num_tasks_per_node
+
+    @sanity_function
+    def assert_finished(self):
+        """Check that the solver completed successfully."""
+        return sn.all([sn.assert_found("End", self.stdout), sn.assert_found("Finalising parallel run", self.stdout)])
+
+    @performance_function("seconds", perf_key="performance")
+    def extract_perf(self):
+        """Extract the final execution time from the solver output."""
+        return sn.extractsingle(
+            r"ExecutionTime\s+=\s+(?P<time>\d+\.\d+)\s+s\s+ClockTime\s+=\s+\d+\s+s\n\nEnd",
+            self.stdout,
+            "time",
+            float,
+        )
+
+    @run_before("run")
+    def setup_testcase(self):
+        """Prepare to run OpenFoam in parallel, by performance the mesh decomposition and partitioning."""
+
+        self.prerun_cmds = [
+            "cd cavity_3d",
+            "blockMesh",
+            f"cp system/decomposeParDict{self.num_tasks:04.0f}  system/decomposeParDict",
+            "decomposePar -force -fileHandler collated",
+        ]
+
+    modules = ["openfoam-org"]
+
+    @run_before("performance")
+    def set_reference(self):
+        """Set the reference performance for Cirrus."""
+        self.reference["cirrus-ex:compute:performance"] = self.reference_performance_cirrus_ex
+
+
+@rfm.simple_test
+class OpenFOAMCavity3D4Node(OpenFOAMCavity3DBase):
+    """OpenFOAM 3D lid-driven cavity test on four nodes."""
+
+    num_nodes = 4
+    executable = "icoFoam"
+    executable_opts = ["-parallel", "-fileHandler", "collated"]
+
+    reference_performance_cirrus_ex = (78, -0.2, 0.05, "seconds")

tests/apps/openfoam/src/cavity_3d/0/U

@@ -0,0 +1,41 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2212                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       volVectorField;
+    object      U;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+dimensions      [0 1 -1 0 0 0 0];
+
+internalField   uniform (0 0 0);
+
+boundaryField
+{
+    movingWall
+    {
+        type            fixedValue;
+        value           uniform (1 0 0);
+    }
+
+    fixedWalls
+    {
+        type            noSlip;
+    }
+
+    frontAndBack
+    {
+        type            empty;
+    }
+}
+
+
+// ************************************************************************* //