Merge pull request #78 from ecmwf-ifs/develop

Version 1.5.0

.github/scripts/bootstrap-nvhpc.sh

@@ -2,7 +2,8 @@
 set -euo pipefail
 set -x
 
-nvhpc_version=21.9
+# Set nvhpc version to default value if unset
+: "${nvhpc_version:=21.9}"
 
 # Use Atlas' nvhpc installation script
 wget https://raw.githubusercontent.com/ecmwf/atlas/develop/tools/install-nvhpc.sh

.github/scripts/verify-targets.sh

@@ -8,49 +8,57 @@ exit_code=0
 # Build the list of targets
 #
 
-targets=(dwarf-P-cloudMicrophysics-IFSScheme dwarf-cloudsc-fortran)
+targets=(dwarf-P-cloudMicrophysics-IFSScheme dwarf-cloudsc-fortran dwarf-cloudsc-c)
 
-if [[ "$io_library_flag" == "--with-serialbox" ]]
+if [[ "$build_flags" == *"--with-gpu"* ]]
 then
-  targets+=(dwarf-cloudsc-c)
-fi
-
-if [[ "$gpu_flag" == "--with-gpu" ]]
-then
-  targets+=(dwarf-cloudsc-gpu-scc dwarf-cloudsc-gpu-scc-hoist dwarf-cloudsc-gpu-omp-scc-hoist)
-  if [[ "$claw_flag" == "--with-claw" ]]
+  targets+=(dwarf-cloudsc-gpu-scc dwarf-cloudsc-gpu-scc-hoist dwarf-cloudsc-gpu-scc-k-caching)
+  targets+=(dwarf-cloudsc-gpu-omp-scc-hoist)
+  if [[ "$build_flags" == *"--with-claw"* ]]
   then
     targets+=(dwarf-cloudsc-gpu-claw)
   fi
-  if [[ "$cuda_flag" == "--with-cuda" ]]
+  if [[ "$build_flags" == *"--with-cuda"* ]]
   then
     targets+=(dwarf-cloudsc-gpu-scc-cuf dwarf-cloudsc-gpu-scc-cuf-k-caching)
-    targets+=(dwarf-cloudsc-gpu-scc-field)
-  fi
-  if [[ "$cuda_flag" == "--with-cuda" && "$io_library_flag" == "--with-serialbox" ]]
-  then
-      targets+=(dwarf-cloudsc-cuda dwarf-cloudsc-cuda-hoist dwarf-cloudsc-cuda-k-caching)
+    targets+=(dwarf-cloudsc-c-cuda dwarf-cloudsc-c-cuda-hoist dwarf-cloudsc-c-cuda-k-caching)
   fi
 fi
 
-if [[ "$loki_flag" == "--with-loki" ]]
+if [[ "$build_flags" == *"--with-loki"* ]]
 then
   targets+=(dwarf-cloudsc-loki-idem dwarf-cloudsc-loki-sca)
   targets+=(dwarf-cloudsc-loki-scc dwarf-cloudsc-loki-scc-hoist)
-  if [[ "$prec_flag" != "--single-precision" ]]
+  targets+=(dwarf-cloudsc-loki-idem-stack dwarf-cloudsc-loki-scc-stack)
+  if [[ "$build_flags" != *"--single-precision"* ]]
   then
     targets+=(dwarf-cloudsc-loki-c)
   fi
-  if [[ "$claw_flag" == "--with-claw" ]]
+  if [[ "$build_flags" == *"--with-claw"* ]]
   then
     targets+=(dwarf-cloudsc-loki-claw-cpu dwarf-cloudsc-loki-claw-gpu)
   fi
-  if [[ "$cuda_flag" == "--with-cuda" ]]
+  if [[ "$build_flags" == *"--with-cuda"* ]]
   then
     targets+=(dwarf-cloudsc-loki-scc-cuf-hoist dwarf-cloudsc-loki-scc-cuf-parametrise)
   fi
 fi
 
+if [[ "$build_flags" == *"--with-atlas"* ]]
+then
+  targets+=(dwarf-cloudsc-fortran-atlas)
+fi
+
+if [[ "$build_flags" == *"--cloudsc-fortran-pyiface=ON"* ]]
+then
+  targets+=(cloudsc_pyiface.py)
+fi
+
+if [[ "$build_flags" == *"--cloudsc-python-f2py=ON"* ]]
+then
+  targets+=(cloudsc_f2py.py)
+fi
+
 #
 # Verify each target exists
 #
@@ -65,16 +73,4 @@ do
   fi
 done
 
-#
-# Check there aren't any other binaries
-#
-
-if [[ ${#targets[@]} -lt $(ls build/bin | wc -l) ]]
-then
-  exit_code=1
-  echo "::error::Additional targets found in build/bin"
-  echo "::error::Expected targets: ${targets[@]}"
-  echo "::error::Found targets: $(ls -1 build/bin | tr '\n' ' ')"
-fi
-
 exit $exit_code

.github/workflows/build.yml

@@ -17,7 +17,7 @@ on:
 jobs:
   # This workflow contains a single job called "build"
   build:
-    name: Test on ${{ matrix.arch }} ${{ matrix.io_library_flag }} ${{ matrix.mpi_flag }} ${{ matrix.prec_flag }} ${{ matrix.gpu_flag }} ${{ matrix.loki_flag }} ${{ matrix.claw_flag }}
+    name: ${{ matrix.arch }} ${{ matrix.io_library_flag }} ${{ matrix.build_flags }}
 
     # The type of runner that the job will run on
     runs-on: ubuntu-20.04
@@ -27,44 +27,75 @@ jobs:
 
       matrix:
 
-        arch:
-          - github/ubuntu/gnu/9.4.0
+        arch: ['gnu/9.4.0']  # Default arch on Github is GNU 9.4.0 for now
 
         io_library_flag: ['', '--with-serialbox']  # Switch between Serialbox and HDF5
 
-        mpi_flag: ['', '--with-mpi']  # Enable MPI-parallel build
+        build_flags:
+          - ''  # Plain build without any options
+          - '--with-gpu --with-loki --with-atlas' # Enable Loki, Atlas, and GPU variants
+          - '--with-gpu --with-loki --with-atlas --with-mpi' # Enable Loki, Atlas, and GPU variants with MPI
+          - '--single-precision --with-gpu --with-loki --with-atlas --with-mpi' # Enable Loki, and GPU variants with MPI in a single-precision build
 
-        prec_flag: ['', '--single-precision']  # Switch single/double precision
+        pyiface_flag: ['']  # Enable the pyiface variant
 
-        gpu_flag: ['', '--with-gpu']  # GPU-variants enabled
+        python_f2py_flag: ['']  # Enable the f2py variant
 
-        cuda_flag: ['']  # Enable CUDA variants
-
-        loki_flag: ['', '--with-loki']  # Loki source-to-source translation enabled
-
-        claw_flag: ['']  # Flag to enable CLAW-generated variants
+        ctest_exclude_pattern: ['-scc-hoist-']  # Regex to disable CTest tests
 
         include:
+          # Add pyiface build configuration for double precision, non-MPI, HDF5 only
+          - arch: gnu/9.4.0
+            io_library_flag: ''
+            build_flags: '--cloudsc-fortran-pyiface=ON --cloudsc-python-f2py=ON'
+
           # Add nvhpc build configurations with serialbox and HDF5
-          - arch: github/ubuntu/nvhpc/21.9
+          - arch: nvhpc/21.9
+            nvhpc_version: 21.9
+            io_library_flag: ''
+            build_flags: '--with-gpu --with-loki --cmake="ENABLE_ACC=OFF"'
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda-'  # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE
+          - arch: nvhpc/21.9
+            nvhpc_version: 21.9
             io_library_flag: ''
-            mpi_flag: ''
-            prec_flag: ''
-            gpu_flag: '--with-gpu'
-            cuda_flag: '--with-cuda'
-            loki_flag: '--with-loki'
-          - arch: github/ubuntu/nvhpc/21.9
+            build_flags: '--with-gpu --with-loki --with-cuda'
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda-'  # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE
+          - arch: nvhpc/21.9
+            nvhpc_version: 21.9
+            io_library_flag: ''
+            build_flags: '--single-precision --with-gpu --with-loki --with-cuda --with-atlas'
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda'  # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE
+          - arch: nvhpc/21.9
+            nvhpc_version: 21.9
             io_library_flag: '--with-serialbox'
-            mpi_flag: ''
-            prec_flag: ''
-            gpu_flag: '--with-gpu'
-            cuda_flag: '--with-cuda'
-            loki_flag: '--with-loki'
+            build_flags: '--with-gpu --with-loki --with-cuda --with-atlas'
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda'  # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE
+
+          - arch: nvhpc/23.5
+            nvhpc_version: 23.5
+            io_library_flag: ''
+            build_flags: '--with-gpu --with-loki --cmake="ENABLE_ACC=OFF"'
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda-'  # GPU variants don't work on CPU runners, loki-c and loki-sca variant causes SIGFPE
+          - arch: nvhpc/23.5
+            nvhpc_version: 23.5
+            io_library_flag: ''
+            build_flags: '--with-gpu --with-loki --with-cuda'
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda-'  # GPU variants don't work on CPU runners, loki-c and loki-sca variant causes SIGFPE
+          - arch: nvhpc/23.5
+            nvhpc_version: 23.5
+            io_library_flag: ''
+            build_flags: '--single-precision --with-gpu --with-loki --with-cuda --with-atlas'
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda'  # GPU variants don't work on CPU runners, loki-c and loki-sca variant causes SIGFPE
+          - arch: nvhpc/23.5
+            nvhpc_version: 23.5
+            io_library_flag: '--with-serialbox'
+            build_flags: '--with-gpu --with-loki --with-cuda --with-atlas'
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda'  # GPU variants don't work on CPU runners, loki-c and loki-sca variant causes SIGFPE
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       # Installs required packages
       - name: Package installation
@@ -73,12 +104,33 @@ jobs:
 
       # Install MPI
       - name: Install MPI via Apt
-        if: contains( matrix.mpi_flag, 'with-mpi' )
+        if: contains( matrix.build_flags, 'with-mpi' )
         run: sudo apt-get install libopenmpi-dev
 
+      # Free up disk space for nvhpc
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        if: contains( matrix.arch, 'nvhpc' )
+        continue-on-error: true
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: false
+
+          # all of these default to true, but feel free to set to
+          # "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: true
+
       # Install Compiler
       - name: Install nvhpc
         if: contains( matrix.arch, 'nvhpc' )
+        env:
+          nvhpc_version: ${{ matrix.nvhpc_version }}
         run: .github/scripts/bootstrap-nvhpc.sh
 
       # Install HDF5
@@ -88,7 +140,9 @@ jobs:
 
       - name: Install HDF5 from source
         if: contains( matrix.arch, 'nvhpc' ) && ! contains( matrix.io_library_flag, 'with-serialbox' )
-        run: source arch/${{ matrix.arch }}/env.sh && .github/scripts/install-hdf5.sh
+        run: |
+          source arch/github/ubuntu/${{ matrix.arch }}/env.sh
+          FC=pgf90 .github/scripts/install-hdf5.sh
 
       # Install Boost
       - name: Install Boost libraries
@@ -102,27 +156,21 @@ jobs:
       # Build the targets
       - name: Bundle build
         run: |
-          ./cloudsc-bundle build --retry-verbose \
-          --arch=arch/${{ matrix.arch }} ${{ matrix.prec_flag }} \
-          ${{ matrix.mpi_flag }} ${{ matrix.io_library_flag }} ${{ matrix.gpu_flag }} \
-          ${{ matrix.claw_flag}} ${{ matrix.loki_flag }} ${{ matrix.cuda_flag }}
+          ./cloudsc-bundle build --verbose --retry-verbose \
+          --arch=arch/github/ubuntu/${{ matrix.arch }} \
+          ${{ matrix.io_library_flag }} ${{ matrix.build_flags }}
 
       # Verify targets exist
       - name: Verify targets
         env:
           io_library_flag: ${{ matrix.io_library_flag }}
-          prec_flag: ${{ matrix.prec_flag }}
-          gpu_flag: ${{ matrix.gpu_flag }}
-          cuda_flag: ${{ matrix.cuda_flag }}
-          loki_flag: ${{ matrix.loki_flag }}
-          claw_flag: ${{ matrix.claw_flag }}
+          build_flags: ${{ matrix.build_flags }}
         run: .github/scripts/verify-targets.sh
 
-      # Run double-precision targets
-      # (Mind the exclusions inside the script!)
-      - name: Run targets
-        env:
-          mpi_flag: ${{ matrix.mpi_flag }}
-          arch: ${{ matrix.arch }}
-        if: ${{ matrix.prec_flag == '' }}
-        run: .github/scripts/run-targets.sh
+      # Run ctest
+      - name: Run CTest
+        if: ${{ !( contains(matrix.build_flags, '--single-precision') || (contains(matrix.build_flags, '--with-cuda') && contains(matrix.arch, 'nvhpc')) ) }}
+        working-directory: ./build
+        run: |
+          source env.sh
+          ctest -O ctest.log --output-on-failure -E "${{ matrix.ctest_exclude_pattern }}"

AUTHORS.md

@@ -2,6 +2,7 @@
 
 - M. Ahlgrimm (ECMWF)
 - P. Bechtold (ECMWF)
+- S. Brdar (ECMWF)
 - W. Deconinck (ECMWF)
 - R. Forbes (ECMWF)
 - C. Jakob (ECMWF)
@@ -12,7 +13,9 @@
 - L. Lucido (Atos)
 - O. Marsden (ECMWF)
 - G. Mengaldo (ECMWF)
+- A. Morvan (Atos)
 - G. Mozdzynski (ECMWF)
+- A. Nawab (ECMWF)
 - Z. Piotrowski (ECMWF)
 - B. Reuter (ECMWF)
 - D. Salmond (ECMWF)