Merge pull request #20 from ufs-community/develop

Merge upstream develop

.github/PULL_REQUEST_TEMPLATE

@@ -20,7 +20,11 @@ If there are changes to the build or source code, the tests below must be conduc
 - [ ] Compile branch on Hera using GNU.
 - [ ] Compile branch in 'Debug' mode on WCOSS2.
 - [ ] Run unit tests locally on any Tier 1 machine.
-- [ ] Run relevant consistency tests locally on all Tier 1 machine.
+- [ ] Run relevant consistency tests locally on all Tier 1 machines.
+
+Optional test.
+
+- [ ] Run full set of chgres_cube consistency tests on Hera.
 
 Describe any additional tests performed.
 

CMakeLists.txt

@@ -23,7 +23,10 @@ option(GCYCLE "Enable building global_cycle.fd" ON)
 option(FRENCTOOLS "Enable building fre-nctools.fd" ON)
 option(GRIDTOOLS "Enable building grid_tools.fd" ON)
 option(CHGRES "Enable building chgres_cube.fd" ON)
+option(CHGRES_ALL "Build chgres with all input data options." OFF)
 option(OROG_MASK_TOOLS "Enable building orog_mask_tools.fd" ON)
+# OROG_MASK_TOOLS must be ON for OROG_NETCDF_TOOLS to build.
+option(OROG_NETCDF_TOOLS "Enable building orog_netcdf_tools.fd" OFF)
 option(SFC_CLIMO_GEN "Enable building sfc_climo_gen.fd" ON)
 option(VCOORD_GEN "Enable building vcoord_gen.fd" ON)
 option(FVCOMTOOLS "Enable building fvcom_tools.fd" ON)
@@ -38,7 +41,6 @@ option(GFS "Enable building GFS-only utilities" OFF)
 # When building the GFS, the following need not be built
 if(GFS)
   message(STATUS "Building utilities specific to the GFS")
-  set(FRENCTOOLS OFF CACHE BOOL "Disable building fre-nctools.fd" FORCE)
   set(GRIDTOOLS OFF CACHE BOOL "Disable building grid_tools.fd" FORCE)
   set(OROG_MASK_TOOLS OFF CACHE BOOL "Disable building orog_mask_tools.fd" FORCE)
   set(SFC_CLIMO_GEN OFF CACHE BOOL "Disable building sfc_climo_gen.fd" FORCE)
@@ -97,15 +99,20 @@ if(OPENMP)
   find_package(OpenMP REQUIRED COMPONENTS Fortran)
 endif()
 
-find_package(sfcio 1.4.0 REQUIRED)
+if(CHGRES_ALL)
+  find_package(sfcio 1.4.0 REQUIRED)
+endif()
 find_package(w3emc 2.9.0 REQUIRED)
 find_package(bacio 2.4.0 REQUIRED)
-find_package(nemsio 2.5.0 REQUIRED)
-find_package(sigio 2.3.0 REQUIRED)
-find_package(sp 2.3.3 REQUIRED)
+if(CHGRES_ALL OR GBLEVENTS)
+  find_package(nemsio 2.5.0 REQUIRED)
+  find_package(sigio 2.3.0 REQUIRED)
+endif()
 find_package(ip 3.3.3 REQUIRED)
+if(ip_VERSION LESS 5.0)
+  find_package(sp 2.3.3 REQUIRED)
+endif()
 find_package(g2 3.4.3 REQUIRED)
-find_package(sigio 2.3.0 REQUIRED)
 
 # If doxygen documentation we enabled, build it. This must come before
 # adding the source code directories; the main documentation build

README.md

@@ -59,7 +59,7 @@ It also uses the following repositories:
 
 ## Installing
 
-On Orion, Hercules, Jet, Hera and WCOSS2 do the following:
+On Orion, Hercules, Jet, Hera, S4, Gaea and WCOSS2 do the following:
 
 1) Set the 'fixed' directories using the `link_fixdirs.sh`
 script in `./fix`. Usage: `./link_fixdirs.sh $RUN_ENVIR $machine`,

VERSION

@@ -1 +1 @@
-1.12.0
+1.13.0

build_all.sh

@@ -15,10 +15,11 @@ else
   readonly DIR_ROOT=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P)
 fi
 
+source "${DIR_ROOT}/sorc/machine-setup.sh"
+
 # User Options
 target=${target:-"NULL"}
 compiler=${compiler:-"intel"}
-PW_CSP=${PW_CSP:-} # TODO: This is an implementation from EPIC and consistent with the UFS WM build system.
 
 if [[ "$target" == "linux.*" || "$target" == "macosx.*" ]]; then
   unset -f module
@@ -27,17 +28,10 @@ if [[ "$target" == "linux.*" || "$target" == "macosx.*" ]]; then
   set -x
 else
   set +x
-  source "${DIR_ROOT}/sorc/machine-setup.sh"
-  if [[ "${target}" == "noaacloud" ]]; then
-    #TODO: This will need to be revisited once the EPIC supported-stacks come online.
-    #TODO: This is a hack due to how the spack-stack module files are generated; there may be a better way to do this.
-    source /contrib/global-workflow/spack-stack/envs/spack_2021.0.3.env
-  else
-    module use "${DIR_ROOT}/modulefiles"
-  fi
+  module use "${DIR_ROOT}/modulefiles"
   module load "build.$target.$compiler" > /dev/null
   module list
- set -x
+  set -x
 fi
 
 # Ensure the submodules have been initialized.

docs/main.md

@@ -9,6 +9,7 @@ https://github.com/ufs-community/UFS_UTILS.
 
 ## Documentation for Previous Versions of UFS_UTILS
 
+* [UFS_UTILS Version 1.13.0](ver-1.13.0/index.html)
 * [UFS_UTILS Version 1.12.0](ver-1.12.0/index.html)
 * [UFS_UTILS Version 1.11.0](ver-1.11.0/index.html)
 * [UFS_UTILS Version 1.10.0](ver-1.10.0/index.html)

docs/source/chgres_cube.rst

@@ -3,7 +3,7 @@
 Introduction
 ------------
 
-The chgres_cube program creates initial condition files to “coldstart” the forecast model.  The initial conditions are created from either Finite-Volume Sphere (FV3) Global Forecast System (GFS), North American Mesoscale Forecast System (NAM), Rapid Refresh (RAP), or High Resolution Rapid Refresh (HRRR) gridded binary version 2 (GRIB2) data.
+The chgres_cube program creates initial condition files to coldstart the forecast model.  The initial conditions are created from either Finite-Volume Sphere (FV3) Global Forecast System (GFS), North American Mesoscale Forecast System (NAM), Rapid Refresh (RAP), or High Resolution Rapid Refresh (HRRR) gridded binary version 2 (GRIB2) data.
 
 Code structure
 --------------
@@ -55,7 +55,7 @@ Program inputs and outputs for global applications
 
 **Inputs**
 
-Users may create their own global grids, or use the pre-defined files are located `here <https://ftp.emc.ncep.noaa.gov/EIB/UFS/global/fix/fix_fv3_gmted2010.v20191213/>`_.
+Users may create their own global grids, or use the pre-defined files located in the `./CRES directories <https://noaa-nws-global-pds.s3.amazonaws.com/index.html#fix/orog/20231027/>`_. (where CRES is the atmospheric resolution and mxRES is the ocean resolution).
 
       * FV3 mosaic file - (NetCDF format)
 	      * CRES_mosaic.nc
@@ -69,22 +69,22 @@ Users may create their own global grids, or use the pre-defined files are locate
 	      * CRES_grid.tile6.nc
 
       * FV3 orography files - (NetCDF format)
-	      * CRES_oro_data.tile1.nc
-	      * CRES_oro_data.tile2.nc
-	      * CRES_oro_data.tile3.nc
-	      * CRES_oro_data.tile4.nc
-	      * CRES_oro_data.tile5.nc
-	      * CRES_oro_data.tile6.nc
-
-      * FV3 surface climatological files - Located under the `./fix_sfc <https://ftp.emc.ncep.noaa.gov/EIB/UFS/global/fix/fix_fv3_gmted2010.v20191213/C48/fix_sfc>`_ sub-directory.  One file for each tile.  NetCDF format.
-	      * CRES.facsf.tileX.nc (fractional coverage for strong/weak zenith angle dependent albedo)
-	      * CRES.maximum_snow_albedo.tileX.nc (maximum snow albedo)
-	      * CRES.slope_type.tileX.nc (slope type)
-	      * CRES.snowfree_albedo.tileX.nc (snow-free albedo)
-	      * CRES.soil_type.tileX.nc (soil type)
-	      * CRES.subtrate_temperature.tileX.nc (soil substrate temperature)
-	      * CRES.vegetation_greenness.tileX.nc (vegetation greenness)
-	      * CRES.vegetation_type.tileX.nc (vegetation type)
+	      * CRES.mxRES_oro_data.tile1.nc
+	      * CRES.mxRES_oro_data.tile2.nc
+	      * CRES.mxRES_oro_data.tile3.nc
+	      * CRES.mxRES_oro_data.tile4.nc
+	      * CRES.mxRES_oro_data.tile5.nc
+	      * CRES.mxRES_oro_data.tile6.nc
+
+      * FV3 surface climatological files - Located under the `./CRES/sfc <https://noaa-nws-global-pds.s3.amazonaws.com/index.html#fix/orog/20231027/>`_ subdirectories. One file for each tile.  NetCDF format.
+	      * CRES.mxRES.facsf.tileX.nc (fractional coverage for strong/weak zenith angle dependent albedo)
+	      * CRES.mxRES.maximum_snow_albedo.tileX.nc (maximum snow albedo)
+	      * CRES.mxRES.slope_type.tileX.nc (slope type)
+	      * CRES.mxRES.snowfree_albedo.tileX.nc (snow-free albedo)
+	      * CRES.mxRES.soil_type.tileX.nc (soil type)
+	      * CRES.mxRES.subtrate_temperature.tileX.nc (soil substrate temperature)
+	      * CRES.mxRES.vegetation_greenness.tileX.nc (vegetation greenness)
+	      * CRES.mxRES.vegetation_type.tileX.nc (vegetation type)
 
       * FV3 vertical coordinate file.  Text file.  `Located here <https://noaa-ufs-srw-pds.s3.amazonaws.com/index.html#fix/fix_am/>`_.
 	      * global_hyblev.l$LEVS.txt
@@ -93,15 +93,15 @@ Users may create their own global grids, or use the pre-defined files are locate
 
 **Outputs**
 
-      * Atmospheric “coldstart” files.  NetCDF.
+      * Atmospheric coldstart files.  NetCDF.
 	      * out.atm.tile1.nc
 	      * out.atm.tile2.nc
 	      * out.atm.tile3.nc
 	      * out.atm.tile4.nc
 	      * out.atm.tile5.nc
 	      * out.atm.tile6.nc
 
-      * Surface/Near Sea Surface Temperature (NSST) “coldstart” files.  NetCDF
+      * Surface/Near Sea Surface Temperature (NSST) coldstart files.  NetCDF
 	      * out.sfc.tile1.nc
 	      * out.sfc.tile1.nc
 	      * out.sfc.tile1.nc
@@ -264,10 +264,10 @@ The following four sets of files/directories should all be located in the same d
 
 **Outputs**
 
-      * Atmospheric “coldstart” file.  NetCDF.
+      * Atmospheric coldstart file.  NetCDF.
         * out.atm.tile7.nc
 
-      * Surface “coldstart” file.  NetCDF.
+      * Surface coldstart file.  NetCDF.
         * out.sfc.tile7.nc
 
 Where to find FV3GFS, NAM, HRRR, and RAP GRIB2 data for regional applications

docs/source/ufs_utils.rst

@@ -240,11 +240,11 @@ Program inputs and outputs
 
       * The "grid" files (CRES_grid.tile#.nc) containing the geo-reference records for the grid - (NetCDF).  Created by the make_hgrid or regional_esg_grid programs.
       * Global 30-arc-second University of Maryland land cover data.  Used to create the land-sea mask.
-             * landcover30.fixed (unformatted binary). Located here `./fix/fix_orog <https://noaa-ufs-srw-pds.s3.amazonaws.com/index.html#fix/fix_orog/>`_.
+             * landcover.umd.30s.nc (NetCDF). Located here `./fix/fix_orog <https://noaa-ufs-srw-pds.s3.amazonaws.com/index.html#fix/fix_orog/>`_.
       * Global 30-arc-second USGS GMTED2010 orography data.
-             * gmted2010.30sec.int (unformatted binary). Located here `./fix/fix_orog <https://noaa-ufs-srw-pds.s3.amazonaws.com/index.html#fix/fix_orog/>`_.
+             * topography.gmted2010.30s.nc (NetCDF). Located here `./fix/fix_orog <https://noaa-ufs-srw-pds.s3.amazonaws.com/index.html#fix/fix_orog/>`_.
       * 30-arc-second RAMP Antarctic terrain data (Radarsat Antarctic Mapping Project)
-             * thirty.second.antarctic.new.bin (unformatted binary). Located here `./fix/fix_orog <https://noaa-ufs-srw-pds.s3.amazonaws.com/index.html#fix/fix_orog/>`_.
+             * topography.antarctica.ramp.30s.nc (NetCDF). Located here `./fix/fix_orog <https://noaa-ufs-srw-pds.s3.amazonaws.com/index.html#fix/fix_orog/>`_.
 
 **Output data:**  
 
@@ -665,3 +665,73 @@ Run script
 ----------
 
 To run, use the machine-dependent script under ./util/weight_gen
+
+***************************************************
+UFS_UTILS utilities
+***************************************************
+
+gdas_init
+=========
+
+Introduction
+------------
+
+The gdas_init utility is used to create coldstart initial conditions for global cycled and forecast-only experiments using the chgres_cube program.  It has two components: one that pulls the input data required by chgres_cube from HPSS, and one that runs chgres_cube. The utility is only supported on machines with access to HPSS:
+
+     * Hera
+     * Jet
+     * WCOSS2
+     * S4 (Only the chgres_cube step is supported, not the data pull step.)
+
+Location
+--------
+
+Find it here: ./util/gdas_init
+
+Build UFS_UTILS and set 'fixed' directories
+-------------------------------------------
+
+Invoke the build script from the root directory: 
+
+::
+
+  ./build_all.sh
+
+Set the 'fixed' directories using the script in the './fix' subdirectory (where $MACHINE is 'hera', 'jet', 'wcoss2', or 's4'):
+
+::
+
+  ./link_fixdirs.sh emc $MACHINE
+
+Configure for your experiment
+-----------------------------
+
+Edit the variables in the 'config' file for your experiment:
+
+     * **EXTRACT_DIR**  - Directory where data extracted from HPSS is stored.
+     * **EXTRACT_DATA** - Set to 'yes' to extract data from HPSS. If data has been extracted and is located in EXTRACT_DIR, set to 'no'. On 's4' this step can't be run. Instead, the data must be pulled from another machine.
+     * **RUN_CHGRES**   - To run chgres, set to 'yes'.  To extract data only, set to 'no'.
+     * **yy/mm/dd/hh**  - The year/month/day/hour of your desired experiment. Use a four digit year and two digits for month/day/hour. **NOTE:** *The standard build of chgres_cube does NOT support experiments prior to June 12, 2019. To coldstart an experiment prior to these dates, contact a repository manager for assistance.*
+     * **LEVS**         - Number of hybrid levels plus 1.  To run with 127 levels, set LEVS to 128.
+     * **CRES_HIRES**   - Resolution of the hires component of your experiment. Example: C768.
+     * **CRES_ENKF**    - Resolution of the enkf component of the experiments.
+     * **UFS_DIR**      - Location of your cloned UFS_UTILS repository.
+     * **OUTDIR**       - Directory where the coldstart data output from chgres is stored.
+     * **CDUMP**        - When 'gdas', will process gdas and enkf members. When 'gfs', will process gfs member for running free forecast only.
+     * **use_v16retro** - When 'yes', use v16 retrospective parallel data. The retrospective parallel tarballs can be missing or incomplete. So this option may not always work. Contact a UFS_UTILS repository manager if you encounter problems.
+
+Note: This utility selects the ocean resolution in the set_fixed_files.sh script using a default based on the user-selected CRES value. For example, for a cycled experiment with a CRES_HIRES/CRES_ENKF of C384/C192, the ocean resolution defaults to 0.25/0.50-degree. To choose another ocean resolution, the user will need to manually modify the set_fixed_files.sh script.
+
+
+Kick off the utility
+--------------------
+
+Submit the driver script (where $MACHINE is 'hera', 'jet', 'wcoss2', or 's4')
+
+::
+
+  ./driver.$MACHINE.sh 
+
+The standard output will be placed in log files in the current directory. 
+
+The converted output will be found in $OUTDIR, including the needed abias and radstat initial condition files (if CDUMP=gdas). The files will be in the needed directory structure for the global-workflow system, therefore a user can move the contents of their $OUTDIR directly into their $ROTDIR.

driver_scripts/driver_grid.hercules.sh

@@ -154,8 +154,8 @@ fi
 #-----------------------------------------------------------------------
 
 export home_dir=$SLURM_SUBMIT_DIR/..
-export TEMP_DIR=/work/noaa/stmp/$LOGNAME/fv3_grid.$gtype
-export out_dir=/work/noaa/stmp/$LOGNAME/my_grids
+export TEMP_DIR=/work2/noaa/stmp/$LOGNAME/fv3_grid.$gtype
+export out_dir=/work2/noaa/stmp/$LOGNAME/my_grids
 
 #-----------------------------------------------------------------------
 # Should not need to change anything below here.

fix/link_fixdirs.sh

@@ -9,15 +9,15 @@ set -ex
 #               'nco' (copies data).
 #
 #  $machine - is the machine. Choices are:
-#             'wcoss2', 'hera', 'jet', 'orion', 'hercules', 's4'
+#             'wcoss2', 'hera', 'jet', 'orion', 'hercules', 's4', 'gaea'
 
 RUN_ENVIR=${1}
 machine=${2}
 
 if [ $# -lt 2 ]; then
     set +x
     echo '***ERROR*** must specify two arguements: (1) RUN_ENVIR, (2) machine'
-    echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 |  hera  | jet | orion | hercules | s4 )'
+    echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 |  hera  | jet | orion | hercules | s4 | gaea )'
     exit 1
 fi
 
@@ -28,10 +28,10 @@ if [ $RUN_ENVIR != emc -a $RUN_ENVIR != nco ]; then
     exit 1
 fi
 
-if [ $machine != wcoss2 -a $machine != hera -a $machine != jet -a $machine != orion -a $machine != s4 -a $machine != hercules ]; then
+if [ $machine != wcoss2 -a $machine != hera -a $machine != jet -a $machine != orion -a $machine != s4 -a $machine != hercules -a $machine != gaea ]; then
     set +x
     echo '***ERROR*** unsupported machine'
-    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | hera | jet | orion | hercules | s4 )'
+    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | hera | jet | orion | hercules | s4 | gaea )'
     exit 1
 fi
 
@@ -54,6 +54,8 @@ elif [ $machine = "wcoss2" ]; then
     FIX_DIR="/lfs/h2/emc/global/noscrub/emc.global/FIX/fix"
 elif [ $machine = "s4" ]; then
     FIX_DIR="/data/prod/glopara/fix"
+elif [ $machine = "gaea" ]; then
+    FIX_DIR="/gpfs/f5/epic/proj-shared/global/glopara/data/fix"
 fi
 
 am_ver=${am_ver:-20220805}