CLOUDSC HIP (SCC, SCC-HOIST, SCC-K-CACHING)

CMakeLists.txt

@@ -83,7 +83,7 @@ endif()
 ecbuild_add_option( FEATURE SERIALBOX
     DESCRIPTION "Use Serialbox to read input and reference data"
     REQUIRED_PACKAGES "Serialbox"
-    CONDITION NOT HAVE_HDF5
+    CONDITION NOT HAVE_HDF5 OR HAVE_SERIALBOX
     DEFAULT OFF )
 if( HAVE_SERIALBOX )
     list(APPEND CLOUDSC_DEFINITIONS HAVE_SERIALBOX)

arch/eurohpc/lumi/cray-gpu/15.0.1/env.sh

@@ -0,0 +1,51 @@
+# (C) Copyright 1988- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+# Source me to get the correct configure/build/run environment
+
+# Store tracing and disable (module is *way* too verbose)
+{ tracing_=${-//[^x]/}; set +x; } 2>/dev/null
+
+module_load() {
+  echo "+ module load $1"
+  module load $1
+}
+module_unload() {
+  echo "+ module unload $1"
+  module unload $1
+}
+
+# Unload to be certain
+module reset
+
+# Load modules
+module_load PrgEnv-cray/8.3.3
+module_load LUMI/23.03
+# module_load partition/G
+module_load rocm/5.2.3
+module_load cce/15.0.1
+module_load cray-libsci/22.08.1.1
+module_load cray-mpich/8.1.18
+module_load craype/2.7.20
+module_load craype-accel-amd-gfx90a
+module_load buildtools/23.03
+module_load cray-hdf5/1.12.1.5
+module_load cray-python/3.9.12.1
+module_load Boost/1.81.0-cpeCray-23.03
+module_load partition/G
+
+module list
+
+set -x
+
+export CC=cc CXX=CC FC=ftn
+
+# Restore tracing to stored setting
+{ if [[ -n "$tracing_" ]]; then set -x; else set +x; fi } 2>/dev/null
+
+export ECBUILD_TOOLCHAIN="./toolchain.cmake"

arch/eurohpc/lumi/cray-gpu/15.0.1/toolchain.cmake

@@ -0,0 +1,43 @@
+# (C) Copyright 1988- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+####################################################################
+# COMPILER
+####################################################################
+
+set( ECBUILD_FIND_MPI OFF )
+set( ENABLE_USE_STMT_FUNC ON CACHE STRING "" )
+
+####################################################################
+# OpenMP FLAGS
+####################################################################
+
+set( ENABLE_OMP ON CACHE STRING "" )
+set( OpenMP_C_FLAGS   "-homp" CACHE STRING "" )
+set( OpenMP_Fortran_FLAGS   "-homp -hnoacc -hlist=aimd -maxrregcount 64" CACHE STRING "" )
+
+####################################################################
+# OpenACC FLAGS
+####################################################################
+
+set( ENABLE_ACC ON CACHE STRING "" )
+set( OpenACC_C_FLAGS "-hacc" )
+set( OpenACC_CXX_FLAGS "-hacc" )
+set( OpenACC_Fortran_FLAGS "-hacc -h acc_model=deep_copy" )
+
+####################################################################
+# Compiler FLAGS
+####################################################################
+
+# General Flags (add to default)
+set(ECBUILD_Fortran_FLAGS "-hcontiguous")
+set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -hbyteswapio")
+set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -Wl, --as-needed")
+
+set(ECBUILD_Fortran_FLAGS_BIT "-O3 -G2 -haggress -DNDEBUG")
+# set(ECBUILD_Fortran_FLAGS_BIT "-O3 -hfp1 -hscalar3 -hvector3 -G2 -haggress -DNDEBUG")

bundle.yml

@@ -78,6 +78,10 @@ options :
             ENABLE_CLOUDSC_GPU_SCC_CUF=ON
             ENABLE_CLOUDSC_GPU_SCC_CUF_K_CACHING=ON
             BUILD_field_api=ON
+    - with-hip :
+        help: Enable GPU kernel variant based on HIP
+        cmake: >
+            ENABLE_CLOUDSC_HIP=ON 
 
     - with-mpi :
         help  : Enable MPI-parallel kernel

src/CMakeLists.txt

@@ -13,5 +13,6 @@ add_subdirectory(cloudsc_pyiface)
 add_subdirectory(cloudsc_python)
 add_subdirectory(cloudsc_c)
 add_subdirectory(cloudsc_cuda)
+add_subdirectory(cloudsc_hip)
 add_subdirectory(cloudsc_gpu)
 add_subdirectory(cloudsc_loki)

src/cloudsc_hip/CMakeLists.txt

@@ -0,0 +1,168 @@
+# (C) Copyright 1988- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+# Define this dwarf variant as an ECBuild feature
+ecbuild_add_option( FEATURE CLOUDSC_HIP
+	DESCRIPTION "Build the HIP version CLOUDSC using Serialbox" DEFAULT OFF
+        CONDITION Serialbox_FOUND
+)
+
+if( HAVE_CLOUDSC_HIP )
+
+
+    if(NOT DEFINED ROCM_PATH)
+      if(DEFINED ENV{ROCM_PATH})
+        set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to which ROCM has been installed")
+      else()
+        set(ROCM_PATH "/opt/rocm" CACHE PATH "Path to which ROCM has been installed")
+      endif()
+    endif()
+
+    find_package(hip REQUIRED)
+
+    set(CMAKE_C_COMPILER "${ROCM_PATH}/bin/hipcc")
+    set(CMAKE_CXX_COMPILER "${ROCM_PATH}/bin/hipcc")
+    # set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -03 -ffast-math")
+
+    ###### SCC-HIP ####
+    ecbuild_add_library(
+        TARGET dwarf-cloudsc-hip-lib
+    	INSTALL_HEADERS LISTED
+        SOURCES
+    	    cloudsc/yoecldp_c.h
+    	    cloudsc/load_state.h
+            cloudsc/load_state.cpp
+            cloudsc/cloudsc_c.h
+            cloudsc/cloudsc_c.cpp
+            cloudsc/cloudsc_driver.h
+            cloudsc/cloudsc_driver.cpp
+            cloudsc/cloudsc_validate.h
+            cloudsc/cloudsc_validate.cpp
+            cloudsc/mycpu.h
+            cloudsc/mycpu.cpp
+        PUBLIC_INCLUDES
+            $<INSTALL_INTERFACE:include>
+            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>
+        PUBLIC_LIBS
+    	    hip::device
+            Serialbox::Serialbox_C
+            $<${HAVE_OMP}:OpenMP::OpenMP_C>
+    )
+
+    target_include_directories(dwarf-cloudsc-hip-lib PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>)
+    target_link_libraries(dwarf-cloudsc-hip-lib PUBLIC hip::device Serialbox::Serialbox_C $<${HAVE_OMP}:OpenMP::OpenMP_C>)
+
+    target_compile_options(dwarf-cloudsc-hip-lib PRIVATE --offload-arch=gfx90a)
+
+    ecbuild_add_executable(
+        TARGET  dwarf-cloudsc-hip
+        SOURCES dwarf_cloudsc.cpp
+        LIBS dwarf-cloudsc-hip-lib
+    )
+
+    ## 
+
+    ###### SCC-HOIST-HIP ####
+    ecbuild_add_library(
+        TARGET dwarf-cloudsc-hip-hoist-lib
+    	INSTALL_HEADERS LISTED
+        SOURCES
+    	    cloudsc/yoecldp_c.h
+    	    cloudsc/load_state.h
+            cloudsc/load_state.cpp
+            cloudsc/cloudsc_c_hoist.h
+            cloudsc/cloudsc_c_hoist.cpp
+            cloudsc/cloudsc_driver_hoist.h
+            cloudsc/cloudsc_driver_hoist.cpp
+            cloudsc/cloudsc_validate.h
+            cloudsc/cloudsc_validate.cpp
+            cloudsc/mycpu.h
+            cloudsc/mycpu.cpp
+        PUBLIC_INCLUDES
+            $<INSTALL_INTERFACE:include>
+            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>
+        PUBLIC_LIBS
+    	    hip::device
+            Serialbox::Serialbox_C
+            $<${HAVE_OMP}:OpenMP::OpenMP_C>
+    )
+
+    target_include_directories(dwarf-cloudsc-hip-hoist-lib PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>)
+    target_link_libraries(dwarf-cloudsc-hip-hoist-lib PUBLIC hip::device Serialbox::Serialbox_C $<${HAVE_OMP}:OpenMP::OpenMP_C>)
+
+    target_compile_options(dwarf-cloudsc-hip-hoist-lib PRIVATE --offload-arch=gfx90a)
+
+    ecbuild_add_executable(
+        TARGET  dwarf-cloudsc-hip-hoist
+        SOURCES dwarf_cloudsc.cpp
+        LIBS dwarf-cloudsc-hip-hoist-lib
+    )
+
+    ## 
+
+    ###### SCC-K-CACHING-HIP ####
+    ecbuild_add_library(
+        TARGET dwarf-cloudsc-hip-k-caching-lib
+    	INSTALL_HEADERS LISTED
+        SOURCES
+    	    cloudsc/yoecldp_c.h
+    	    cloudsc/load_state.h
+            cloudsc/load_state.cpp
+            cloudsc/cloudsc_c_k_caching.h
+            cloudsc/cloudsc_c_k_caching.cpp
+            cloudsc/cloudsc_driver.h
+            cloudsc/cloudsc_driver.cpp
+            cloudsc/cloudsc_validate.h
+            cloudsc/cloudsc_validate.cpp
+            cloudsc/mycpu.h
+            cloudsc/mycpu.cpp
+        PUBLIC_INCLUDES
+            $<INSTALL_INTERFACE:include>
+            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>
+        PUBLIC_LIBS
+    	    hip::device
+            Serialbox::Serialbox_C
+            $<${HAVE_OMP}:OpenMP::OpenMP_C>
+    )
+
+    target_include_directories(dwarf-cloudsc-hip-k-caching-lib PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>)
+    target_link_libraries(dwarf-cloudsc-hip-k-caching-lib PUBLIC hip::device Serialbox::Serialbox_C $<${HAVE_OMP}:OpenMP::OpenMP_C>)
+
+    target_compile_options(dwarf-cloudsc-hip-k-caching-lib PRIVATE --offload-arch=gfx90a)
+
+    ecbuild_add_executable(
+        TARGET  dwarf-cloudsc-hip-k-caching
+        SOURCES dwarf_cloudsc.cpp
+        LIBS dwarf-cloudsc-hip-k-caching-lib
+    )
+
+    ## 
+
+
+
+    # Create symlink for the input data
+    execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
+        ${CMAKE_CURRENT_SOURCE_DIR}/../../data ${CMAKE_CURRENT_BINARY_DIR}/../../../data )
+
+    ecbuild_add_test(
+        TARGET dwarf-cloudsc-hip-serial
+        COMMAND bin/dwarf-cloudsc-hip
+        ARGS 1 100 16
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../..
+        OMP 1
+    )
+    ecbuild_add_test(
+        TARGET dwarf-cloudsc-hip-omp
+        COMMAND bin/dwarf-cloudsc-hip
+        ARGS 4 100 16
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../..
+        OMP 4
+        CONDITION HAVE_OMP
+    )
+
+endif()