Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

CLOUDSC HIP (SCC, SCC-HOIST, SCC-K-CACHING) #59

Merged
merged 4 commits into from
Dec 5, 2023
Merged

CLOUDSC HIP (SCC, SCC-HOIST, SCC-K-CACHING) #59

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3ef7b5a
Manual (or rather semi-automatically generated) HIP versions of cloud…
MichaelSt98 Aug 31, 2023
7c397fe
Change in load state in order to allow correct validation for nproma …
MichaelSt98 Sep 4, 2023
e3b8876
improving cmake usage for HIP version, minor changes and refactoring
MichaelSt98 Dec 4, 2023
7a617c7
removing multi-threaded HIP test and adding relevant HIP ctests for a…
MichaelSt98 Dec 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,20 @@ if( HAVE_CUDA )
enable_language( CUDA )
endif()

ecbuild_add_option( FEATURE HIP
DESCRIPTION "HIP" DEFAULT OFF
REQUIRED_PACKAGES "hip" )
if ( HAVE_HIP )
if(NOT DEFINED ROCM_PATH)
if(DEFINED ENV{ROCM_PATH})
set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to which ROCM has been installed")
else()
set(ROCM_PATH "/opt/rocm" CACHE PATH "Path to which ROCM has been installed")
endif()
endif()
find_package(hip REQUIRED)
endif()

### OpenMP
ecbuild_add_option( FEATURE OMP
DESCRIPTION "OpenMP" DEFAULT ON
Expand Down
51 changes: 51 additions & 0 deletions arch/eurohpc/lumi/cray-gpu/15.0.1/env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
# (C) Copyright 1988- ECMWF.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
# In applying this licence, ECMWF does not waive the privileges and immunities
# granted to it by virtue of its status as an intergovernmental organisation
# nor does it submit to any jurisdiction.

# Source me to get the correct configure/build/run environment

# Store tracing and disable (module is *way* too verbose)
{ tracing_=${-//[^x]/}; set +x; } 2>/dev/null

module_load() {
echo "+ module load $1"
module load $1
}
module_unload() {
echo "+ module unload $1"
module unload $1
}

# Unload to be certain
module reset

# Load modules
module_load PrgEnv-cray/8.3.3
module_load LUMI/23.03
# module_load partition/G
module_load rocm/5.2.3
module_load cce/15.0.1
module_load cray-libsci/22.08.1.1
module_load cray-mpich/8.1.18
module_load craype/2.7.20
module_load craype-accel-amd-gfx90a
module_load buildtools/23.03
module_load cray-hdf5/1.12.1.5
module_load cray-python/3.9.12.1
module_load Boost/1.81.0-cpeCray-23.03
module_load partition/G

module list

set -x

export CC=cc CXX=CC FC=ftn

# Restore tracing to stored setting
{ if [[ -n "$tracing_" ]]; then set -x; else set +x; fi } 2>/dev/null

export ECBUILD_TOOLCHAIN="./toolchain.cmake"
58 changes: 58 additions & 0 deletions arch/eurohpc/lumi/cray-gpu/15.0.1/toolchain.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# (C) Copyright 1988- ECMWF.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
# In applying this licence, ECMWF does not waive the privileges and immunities
# granted to it by virtue of its status as an intergovernmental organisation
# nor does it submit to any jurisdiction.

####################################################################
# COMPILER
####################################################################

set( ECBUILD_FIND_MPI OFF )
set( ENABLE_USE_STMT_FUNC ON CACHE STRING "" )

####################################################################
# OpenMP FLAGS
####################################################################

set( ENABLE_OMP ON CACHE STRING "" )
set( OpenMP_C_FLAGS "-fopenmp" CACHE STRING "" )
set( OpenMP_CXX_FLAGS "-fopenmp" CACHE STRING "" )
set( OpenMP_Fortran_FLAGS "-fopenmp -hnoacc -hlist=aimd" CACHE STRING "" )

set( OpenMP_C_LIB_NAMES "craymp" )
set( OpenMP_CXX_LIB_NAMES "craymp" )
set( OpenMP_Fortran_LIB_NAMES "craymp" )
set( OpenMP_craymp_LIBRARY "/opt/cray/pe/cce/15.0.1/cce/x86_64/lib/libcraymp.so" )

MichaelSt98 marked this conversation as resolved.
Show resolved Hide resolved
####################################################################
# OpenACC FLAGS
####################################################################

set( ENABLE_ACC ON CACHE STRING "" )
set( OpenACC_C_FLAGS "-hacc" )
set( OpenACC_CXX_FLAGS "-hacc" )
set( OpenACC_Fortran_FLAGS "-hacc -h acc_model=deep_copy" )

####################################################################
# OpenACC FLAGS
####################################################################

set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -03 -ffast-math")
if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
set(CMAKE_HIP_ARCHITECTURES gfx90a)
endif()

####################################################################
# Compiler FLAGS
####################################################################

# General Flags (add to default)
set(ECBUILD_Fortran_FLAGS "-hcontiguous")
set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -hbyteswapio")
set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -Wl, --as-needed")

set(ECBUILD_Fortran_FLAGS_BIT "-O3 -G2 -haggress -DNDEBUG")
# set(ECBUILD_Fortran_FLAGS_BIT "-O3 -hfp1 -hscalar3 -hvector3 -G2 -haggress -DNDEBUG")
5 changes: 5 additions & 0 deletions bundle.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,11 @@ options :
ENABLE_CLOUDSC_GPU_SCC_CUF=ON
ENABLE_CLOUDSC_GPU_SCC_CUF_K_CACHING=ON
BUILD_field_api=ON

- with-hip :
MichaelSt98 marked this conversation as resolved.
Show resolved Hide resolved
help: Enable GPU kernel variant based on HIP
cmake: >
ENABLE_HIP=ON

- with-mpi :
help : Enable MPI-parallel kernel
Expand Down
1 change: 1 addition & 0 deletions src/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,6 @@ add_subdirectory(cloudsc_pyiface)
add_subdirectory(cloudsc_python)
add_subdirectory(cloudsc_c)
add_subdirectory(cloudsc_cuda)
add_subdirectory(cloudsc_hip)
add_subdirectory(cloudsc_gpu)
add_subdirectory(cloudsc_loki)
168 changes: 168 additions & 0 deletions src/cloudsc_hip/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
# (C) Copyright 1988- ECMWF.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
# In applying this licence, ECMWF does not waive the privileges and immunities
# granted to it by virtue of its status as an intergovernmental organisation
# nor does it submit to any jurisdiction.

# Define this dwarf variant as an ECBuild feature
ecbuild_add_option( FEATURE CLOUDSC_HIP
DESCRIPTION "Build the HIP version CLOUDSC using Serialbox" DEFAULT ON
CONDITION Serialbox_FOUND AND HAVE_HIP
)

if( HAVE_CLOUDSC_HIP )

set(CMAKE_C_COMPILER "${ROCM_PATH}/bin/hipcc")
set(CMAKE_CXX_COMPILER "${ROCM_PATH}/bin/hipcc")

###### SCC-HIP ####
ecbuild_add_library(
TARGET dwarf-cloudsc-hip-lib
INSTALL_HEADERS LISTED
SOURCES
cloudsc/yoecldp_c.h
cloudsc/load_state.h
cloudsc/load_state.cpp
cloudsc/cloudsc_c.h
cloudsc/cloudsc_c.cpp
cloudsc/cloudsc_driver.h
cloudsc/cloudsc_driver.cpp
cloudsc/cloudsc_validate.h
cloudsc/cloudsc_validate.cpp
cloudsc/mycpu.h
cloudsc/mycpu.cpp
PUBLIC_INCLUDES
$<INSTALL_INTERFACE:include>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>
PUBLIC_LIBS
hip::device
Serialbox::Serialbox_C
$<${HAVE_OMP}:OpenMP::OpenMP_C>
)

target_include_directories(dwarf-cloudsc-hip-lib PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>)
target_link_libraries(dwarf-cloudsc-hip-lib PUBLIC hip::device Serialbox::Serialbox_C $<${HAVE_OMP}:OpenMP::OpenMP_C>)

if (NOT DEFINED CMAKE_HIP_ARCHITECTURES)
message(WARNING "No HIP architecture is set! ('CMAKE_HIP_ARCHITECTURES' is not defined)")
else()
target_compile_options(dwarf-cloudsc-hip-lib PRIVATE --offload-arch=${CMAKE_HIP_ARCHITECTURES})
endif()

ecbuild_add_executable(
TARGET dwarf-cloudsc-hip
SOURCES dwarf_cloudsc.cpp
LIBS dwarf-cloudsc-hip-lib
)

##

###### SCC-HOIST-HIP ####
ecbuild_add_library(
TARGET dwarf-cloudsc-hip-hoist-lib
INSTALL_HEADERS LISTED
SOURCES
cloudsc/yoecldp_c.h
cloudsc/load_state.h
cloudsc/load_state.cpp
cloudsc/cloudsc_c_hoist.h
cloudsc/cloudsc_c_hoist.cpp
cloudsc/cloudsc_driver_hoist.h
cloudsc/cloudsc_driver_hoist.cpp
cloudsc/cloudsc_validate.h
cloudsc/cloudsc_validate.cpp
cloudsc/mycpu.h
cloudsc/mycpu.cpp
PUBLIC_INCLUDES
$<INSTALL_INTERFACE:include>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>
PUBLIC_LIBS
hip::device
Serialbox::Serialbox_C
$<${HAVE_OMP}:OpenMP::OpenMP_C>
)

target_include_directories(dwarf-cloudsc-hip-hoist-lib PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>)
target_link_libraries(dwarf-cloudsc-hip-hoist-lib PUBLIC hip::device Serialbox::Serialbox_C $<${HAVE_OMP}:OpenMP::OpenMP_C>)

if (NOT DEFINED CMAKE_HIP_ARCHITECTURES)
message(WARNING "No HIP architecture is set! ('CMAKE_HIP_ARCHITECTURES' is not defined)")
else()
target_compile_options(dwarf-cloudsc-hip-hoist-lib PRIVATE --offload-arch=${CMAKE_HIP_ARCHITECTURES})
endif()

ecbuild_add_executable(
TARGET dwarf-cloudsc-hip-hoist
SOURCES dwarf_cloudsc.cpp
LIBS dwarf-cloudsc-hip-hoist-lib
)

##

###### SCC-K-CACHING-HIP ####
ecbuild_add_library(
TARGET dwarf-cloudsc-hip-k-caching-lib
INSTALL_HEADERS LISTED
SOURCES
cloudsc/yoecldp_c.h
cloudsc/load_state.h
cloudsc/load_state.cpp
cloudsc/cloudsc_c_k_caching.h
cloudsc/cloudsc_c_k_caching.cpp
cloudsc/cloudsc_driver.h
cloudsc/cloudsc_driver.cpp
cloudsc/cloudsc_validate.h
cloudsc/cloudsc_validate.cpp
cloudsc/mycpu.h
cloudsc/mycpu.cpp
PUBLIC_INCLUDES
$<INSTALL_INTERFACE:include>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>
PUBLIC_LIBS
hip::device
Serialbox::Serialbox_C
$<${HAVE_OMP}:OpenMP::OpenMP_C>
)

target_include_directories(dwarf-cloudsc-hip-k-caching-lib PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>)
target_link_libraries(dwarf-cloudsc-hip-k-caching-lib PUBLIC hip::device Serialbox::Serialbox_C $<${HAVE_OMP}:OpenMP::OpenMP_C>)

if (NOT DEFINED CMAKE_HIP_ARCHITECTURES)
message(WARNING "No HIP architecture is set! ('CMAKE_HIP_ARCHITECTURES' is not defined)")
else()
target_compile_options(dwarf-cloudsc-hip-k-caching-lib PRIVATE --offload-arch=${CMAKE_HIP_ARCHITECTURES})
endif()

ecbuild_add_executable(
TARGET dwarf-cloudsc-hip-k-caching
SOURCES dwarf_cloudsc.cpp
LIBS dwarf-cloudsc-hip-k-caching-lib
)

##



# Create symlink for the input data
execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
${CMAKE_CURRENT_SOURCE_DIR}/../../data ${CMAKE_CURRENT_BINARY_DIR}/../../../data )

ecbuild_add_test(
TARGET dwarf-cloudsc-hip-serial
COMMAND bin/dwarf-cloudsc-hip
ARGS 1 100 16
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../..
OMP 1
)
ecbuild_add_test(
TARGET dwarf-cloudsc-hip-omp
COMMAND bin/dwarf-cloudsc-hip
ARGS 4 100 16
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../..
OMP 4
CONDITION HAVE_OMP
)
reuterbal marked this conversation as resolved.
Show resolved Hide resolved

endif()
Loading
Loading