From cf2da7672222677f9760f8229c27d6a2450c7080 Mon Sep 17 00:00:00 2001 From: GeorgeGayno-NOAA <52789452+GeorgeGayno-NOAA@users.noreply.github.com> Date: Wed, 21 Aug 2024 08:44:31 -0400 Subject: [PATCH] Use HPC-stack libraries on WCOSS2 (#975) Update to HPC-stack on WCOSS2. Newer versions of these libraries are now used: HDF5 - from 1.10.6 to 1.14.0. NetCDF - from 4.7.4 to 4.92. nemsio - from 2.5.2 to 2.5.4. splib - from 2.3.3 to 2.4.0. ESMF - from 8.4.1 to 8.6.0. Use the stack version of the nccmp utility in the regression tests instead of a local copy. Fixes #877. --- modulefiles/build.wcoss2.intel.lua | 24 ++++++++++--------- reg_tests/chgres_cube/13km.conus.nam.grib2.sh | 2 ++ reg_tests/chgres_cube/13km.conus.rap.grib2.sh | 2 ++ .../chgres_cube/13km.na.gfs.ncei.grib2.sh | 2 ++ reg_tests/chgres_cube/25km.conus.gfs.grib2.sh | 2 ++ .../chgres_cube/25km.conus.gfs.pbgrib2.sh | 2 ++ .../3km.conus.hrrr.gfssdf.grib2.sh | 2 ++ .../3km.conus.hrrr.newsfc.grib2.sh | 2 ++ reg_tests/chgres_cube/driver.wcoss2.sh | 7 +++--- reg_tests/cpld_gridgen/rt.sh | 6 ++++- reg_tests/global_cycle/driver.wcoss2.sh | 4 ++-- reg_tests/grid_gen/driver.wcoss2.sh | 4 ++-- reg_tests/ocnice_prep/rt.sh | 6 ++++- reg_tests/weight_gen/driver.wcoss2.sh | 4 ++-- 14 files changed, 47 insertions(+), 22 deletions(-) diff --git a/modulefiles/build.wcoss2.intel.lua b/modulefiles/build.wcoss2.intel.lua index 585107fc3..6e0659bdd 100644 --- a/modulefiles/build.wcoss2.intel.lua +++ b/modulefiles/build.wcoss2.intel.lua @@ -14,7 +14,7 @@ load(pathJoin("craype", craype_ver)) intel_ver=os.getenv("intel_ver") or "19.1.3.304" load(pathJoin("intel", intel_ver)) -cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.7" +cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.12" load(pathJoin("cray-mpich", cray_mpich_ver)) libjpeg_ver=os.getenv("libjpeg_ver") or "9c" @@ -26,11 +26,11 @@ load(pathJoin("zlib", zlib_ver)) libpng_ver=os.getenv("libpng_ver") or "1.6.37" load(pathJoin("libpng", libpng_ver)) -hdf5_ver=os.getenv("hdf5_ver") or "1.10.6" -load(pathJoin("hdf5", hdf5_ver)) +hdf5_ver=os.getenv("hdf5_ver") or "1.14.0" +load(pathJoin("hdf5-C", hdf5_ver)) -netcdf_ver=os.getenv("netcdf_ver") or "4.7.4" -load(pathJoin("netcdf", netcdf_ver)) +netcdf_ver=os.getenv("netcdf_ver") or "4.9.2" +load(pathJoin("netcdf-C", netcdf_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) @@ -42,13 +42,13 @@ load(pathJoin("bacio", bacio_ver)) w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" load(pathJoin("w3emc", w3emc_ver)) -nemsio_ver=os.getenv("nemsio_ver") or "2.5.2" +nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" load(pathJoin("nemsio", nemsio_ver)) sigio_ver=os.getenv("sigio_ver") or "2.3.2" load(pathJoin("sigio", sigio_ver)) -sp_ver=os.getenv("sp_ver") or "2.3.3" +sp_ver=os.getenv("sp_ver") or "2.4.0" load(pathJoin("sp", sp_ver)) ip_ver=os.getenv("ip_ver") or "4.0.0" @@ -57,20 +57,22 @@ load(pathJoin("ip", ip_ver)) g2_ver=os.getenv("g2_ver") or "3.4.5" load(pathJoin("g2", g2_ver)) --- for mpiexec command +-- Needed for mpiexec command. cray_pals_ver=os.getenv("cray_pals_ver") or "1.2.2" load(pathJoin("cray-pals", cray_pals_ver)) +-- Needed at runtime for nco utilities. udunits_ver=os.getenv("udunits_ver") or "2.2.28" load(pathJoin("udunits", udunits_ver)) +-- Needed at runtime for nco utilities. gsl_ver=os.getenv("gsl_ver") or "2.7" load(pathJoin("gsl", gsl_ver)) -nco_ver=os.getenv("nco_ver") or "4.9.7" +nco_ver=os.getenv("nco_ver") or "5.0.6" load(pathJoin("nco", nco_ver)) -esmf_ver=os.getenv("esmf_ver") or "8.4.1" -load(pathJoin("esmf", esmf_ver)) +esmf_ver=os.getenv("esmf_ver") or "8.6.0" +load(pathJoin("esmf-C", esmf_ver)) whatis("Description: UFS_UTILS build environment") diff --git a/reg_tests/chgres_cube/13km.conus.nam.grib2.sh b/reg_tests/chgres_cube/13km.conus.nam.grib2.sh index a5bd051c4..830d46a62 100755 --- a/reg_tests/chgres_cube/13km.conus.nam.grib2.sh +++ b/reg_tests/chgres_cube/13km.conus.nam.grib2.sh @@ -34,6 +34,8 @@ export TRACERS_INPUT="" export OMP_NUM_THREADS_CH=${OMP_NUM_THREADS:-1} +NCCMP=${NCCMP:-$(which nccmp)} + #----------------------------------------------------------------------------- # Invoke chgres program. #----------------------------------------------------------------------------- diff --git a/reg_tests/chgres_cube/13km.conus.rap.grib2.sh b/reg_tests/chgres_cube/13km.conus.rap.grib2.sh index 4b684b646..4da2468d8 100755 --- a/reg_tests/chgres_cube/13km.conus.rap.grib2.sh +++ b/reg_tests/chgres_cube/13km.conus.rap.grib2.sh @@ -42,6 +42,8 @@ export GEOGRID_FILE_INPUT=${HOMEufs}/fix/am/geo_em.d01.nc_RAPX export OMP_NUM_THREADS_CH=${OMP_NUM_THREADS:-1} +NCCMP=${NCCMP:-$(which nccmp)} + #----------------------------------------------------------------------------- # Invoke chgres program. #----------------------------------------------------------------------------- diff --git a/reg_tests/chgres_cube/13km.na.gfs.ncei.grib2.sh b/reg_tests/chgres_cube/13km.na.gfs.ncei.grib2.sh index c3e0c6f5d..398616b92 100755 --- a/reg_tests/chgres_cube/13km.na.gfs.ncei.grib2.sh +++ b/reg_tests/chgres_cube/13km.na.gfs.ncei.grib2.sh @@ -30,6 +30,8 @@ export CDATE=2019080100 export OMP_NUM_THREADS_CH=${OMP_NUM_THREADS:-1} +NCCMP=${NCCMP:-$(which nccmp)} + #----------------------------------------------------------------------------- # Invoke chgres program. #----------------------------------------------------------------------------- diff --git a/reg_tests/chgres_cube/25km.conus.gfs.grib2.sh b/reg_tests/chgres_cube/25km.conus.gfs.grib2.sh index 2efb252d7..c4bae7747 100755 --- a/reg_tests/chgres_cube/25km.conus.gfs.grib2.sh +++ b/reg_tests/chgres_cube/25km.conus.gfs.grib2.sh @@ -32,6 +32,8 @@ export CDATE=2019110400 export OMP_NUM_THREADS_CH=${OMP_NUM_THREADS:-1} +NCCMP=${NCCMP:-$(which nccmp)} + #----------------------------------------------------------------------------- # Invoke chgres program. #----------------------------------------------------------------------------- diff --git a/reg_tests/chgres_cube/25km.conus.gfs.pbgrib2.sh b/reg_tests/chgres_cube/25km.conus.gfs.pbgrib2.sh index 4b3491516..007ab6b37 100755 --- a/reg_tests/chgres_cube/25km.conus.gfs.pbgrib2.sh +++ b/reg_tests/chgres_cube/25km.conus.gfs.pbgrib2.sh @@ -31,6 +31,8 @@ export CDATE=2021062718 export OMP_NUM_THREADS_CH=${OMP_NUM_THREADS:-1} +NCCMP=${NCCMP:-$(which nccmp)} + #----------------------------------------------------------------------------- # Invoke chgres program. #----------------------------------------------------------------------------- diff --git a/reg_tests/chgres_cube/3km.conus.hrrr.gfssdf.grib2.sh b/reg_tests/chgres_cube/3km.conus.hrrr.gfssdf.grib2.sh index 4f72173d6..db7c041d0 100755 --- a/reg_tests/chgres_cube/3km.conus.hrrr.gfssdf.grib2.sh +++ b/reg_tests/chgres_cube/3km.conus.hrrr.gfssdf.grib2.sh @@ -36,6 +36,8 @@ export GEOGRID_FILE_INPUT=${HOMEufs}/fix/am/geo_em.d01.nc_HRRRX export OMP_NUM_THREADS_CH=${OMP_NUM_THREADS:-1} +NCCMP=${NCCMP:-$(which nccmp)} + #----------------------------------------------------------------------------- # Invoke chgres program. #----------------------------------------------------------------------------- diff --git a/reg_tests/chgres_cube/3km.conus.hrrr.newsfc.grib2.sh b/reg_tests/chgres_cube/3km.conus.hrrr.newsfc.grib2.sh index 86a3d8939..1059779c1 100755 --- a/reg_tests/chgres_cube/3km.conus.hrrr.newsfc.grib2.sh +++ b/reg_tests/chgres_cube/3km.conus.hrrr.newsfc.grib2.sh @@ -42,6 +42,8 @@ export GEOGRID_FILE_INPUT=${HOMEufs}/fix/am/geo_em.d01.nc_HRRRX export OMP_NUM_THREADS_CH=${OMP_NUM_THREADS:-1} +NCCMP=${NCCMP:-$(which nccmp)} + #----------------------------------------------------------------------------- # Invoke chgres program. #----------------------------------------------------------------------------- diff --git a/reg_tests/chgres_cube/driver.wcoss2.sh b/reg_tests/chgres_cube/driver.wcoss2.sh index 4fd7f7750..fd81b4ea9 100755 --- a/reg_tests/chgres_cube/driver.wcoss2.sh +++ b/reg_tests/chgres_cube/driver.wcoss2.sh @@ -29,6 +29,9 @@ compiler=${compiler:-"intel"} source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.$compiler +# load nccmp +module load netcdf +module load nccmp module list export OUTDIR="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" @@ -60,8 +63,6 @@ rm -f $LOG_FILE* $SUM_FILE export OMP_STACKSIZE=1024M -export NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp -#export NCCMP=${NCCMP:-nccmp} rm -fr $OUTDIR this_dir=$PWD @@ -163,7 +164,7 @@ TEST10=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltim LOG_FILE=consistency.log11 export APRUN="mpiexec -n 12 -ppn 12 --cpu-bind core" TEST11=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c96.fv3.netcdf2wam -l select=1:ncpus=12:ompthreads=1:mem=75GB $PWD/c96.fv3.netcdf2wam.sh) + -N c96.fv3.netcdf2wam -l select=1:ncpus=12:ompthreads=1:mem=80GB $PWD/c96.fv3.netcdf2wam.sh) #----------------------------------------------------------------------------- # Initialize CONUS 25-KM USING GFS PGRIB2+BGRIB2 files. diff --git a/reg_tests/cpld_gridgen/rt.sh b/reg_tests/cpld_gridgen/rt.sh index 10ce80fb2..bec67ee68 100755 --- a/reg_tests/cpld_gridgen/rt.sh +++ b/reg_tests/cpld_gridgen/rt.sh @@ -131,7 +131,7 @@ if [[ $target = wcoss2 ]]; then export APRUN="mpiexec -n 1 -ppn 1 --cpu-bind core" QUEUE=${QUEUE:-dev} SBATCH_COMMAND="./cpld_gridgen.sh" - NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp + NCCMP=nccmp elif [[ $target = hera ]]; then STMP=${STMP:-/scratch1/NCEPDEV/stmp4/$USER} export MOM6_FIXDIR=/scratch1/NCEPDEV/global/glopara/fix/mom6/20220805 @@ -217,6 +217,10 @@ fi module use $PATHTR/modulefiles module load build.$target.$compiler +if [[ $target = wcoss2 ]]; then + module load netcdf + module load nccmp +fi module list if [[ $CREATE_BASELINE = true ]]; then diff --git a/reg_tests/global_cycle/driver.wcoss2.sh b/reg_tests/global_cycle/driver.wcoss2.sh index 1655a4bb4..9a8f4deac 100755 --- a/reg_tests/global_cycle/driver.wcoss2.sh +++ b/reg_tests/global_cycle/driver.wcoss2.sh @@ -25,6 +25,8 @@ compiler=${compiler:-"intel"} source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.$compiler +module load netcdf +module load nccmp module list WORK_DIR="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" @@ -56,8 +58,6 @@ export NWPROD=$PWD/../.. reg_dir=$PWD -export NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp - LOG_FILE=consistency.log rm -f ${LOG_FILE}* diff --git a/reg_tests/grid_gen/driver.wcoss2.sh b/reg_tests/grid_gen/driver.wcoss2.sh index 4e381c797..013ae3648 100755 --- a/reg_tests/grid_gen/driver.wcoss2.sh +++ b/reg_tests/grid_gen/driver.wcoss2.sh @@ -24,6 +24,8 @@ compiler=${compiler:-"intel"} source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.$compiler +module load netcdf +module load nccmp module list set -x @@ -44,8 +46,6 @@ if [ "$UPDATE_BASELINE" = "TRUE" ]; then source ../get_hash.sh fi -export NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp - LOG_FILE=consistency.log rm -f ${LOG_FILE} SUM_FILE=summary.log diff --git a/reg_tests/ocnice_prep/rt.sh b/reg_tests/ocnice_prep/rt.sh index 2348c4c40..750897d33 100755 --- a/reg_tests/ocnice_prep/rt.sh +++ b/reg_tests/ocnice_prep/rt.sh @@ -130,8 +130,8 @@ if [[ $target = wcoss2 ]]; then ACCOUNT=${ACCOUNT:-GFS-DEV} export APRUN="mpiexec -n 1 -ppn 1 --cpu-bind core" QUEUE=${QUEUE:-dev} + NCCMP=nccmp SBATCH_COMMAND="./ocnice_prep.sh" - NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp elif [[ $target = hera ]]; then STMP=${STMP:-/scratch1/NCEPDEV/stmp4/$USER} BASELINE_ROOT=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/ocnice_prep/baseline_data @@ -221,6 +221,10 @@ fi module use $PATHTR/modulefiles module load build.$target.$compiler +if [[ $target = wcoss2 ]]; then + module load netcdf + module load nccmp/1.8.9.0 +fi module list if [[ $CREATE_BASELINE = true ]]; then diff --git a/reg_tests/weight_gen/driver.wcoss2.sh b/reg_tests/weight_gen/driver.wcoss2.sh index 584c0fea1..720100d04 100755 --- a/reg_tests/weight_gen/driver.wcoss2.sh +++ b/reg_tests/weight_gen/driver.wcoss2.sh @@ -35,6 +35,8 @@ compiler=${compiler:-"intel"} source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.$compiler +module load netcdf +module load nccmp module list export DATA="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" @@ -54,8 +56,6 @@ fi export HOMEreg=/lfs/h2/emc/nems/noscrub/emc.nems/UFS_UTILS/reg_tests/weight_gen export HOMEufs=$PBS_O_WORKDIR/../.. -export NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp - ./weight_gen.sh exit 0