Update hash of ufs weather model (#146)

* update hash of ufs weather model * update template files * update forecast ex-script * update node ppn for forecast * update hash of ufs-weather-model * clean up forecast script * update ctest scripts * fix unbound variables on ctest * parameterize hpc variables for forecast * update ctest scripts for forecast * change path to jedi on hercules * change path to jedi for hercules * update version of run.ver files
ufs-community · Oct 8, 2024 · d8b7ea8 · d8b7ea8
1 parent 96de8e0
commit d8b7ea8
Show file tree

Hide file tree

Showing 45 changed files with 1,709 additions and 1,551 deletions.
diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua
@@ -31,6 +31,7 @@ load(pathJoin("sp", sp_ver))
 load(pathJoin("w3emc", w3emc_ver))
 load(pathJoin("gftl-shared", gftl_shared_ver))
 load(pathJoin("mapl", mapl_ver))
+load(pathJoin("scotch", scotch_ver))
 load(pathJoin("prod_util", prod_util_ver))
 load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
 

diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua
@@ -31,6 +31,7 @@ load(pathJoin("sp", sp_ver))
 load(pathJoin("w3emc", w3emc_ver))
 load(pathJoin("gftl-shared", gftl_shared_ver))
 load(pathJoin("mapl", mapl_ver))
+load(pathJoin("scotch", scotch_ver))
 load(pathJoin("prod_util", prod_util_ver))
 load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
 
@@ -43,4 +44,4 @@ setenv("FC", "mpiifort")
 setenv("CMAKE_Platform", "hercules.intel")
 
 setenv("EPICHOME", "/work/noaa/epic/UFS_Land-DA_Dev")
-setenv("JEDI_INSTALL", "/work2/noaa/epic/UFS_Land-DA_Dev/jedi_v7_hercules")
+setenv("JEDI_INSTALL", "/work/noaa/epic/UFS_Land-DA_Dev/jedi_v7_hercules")
diff --git a/modulefiles/build_orion_intel.lua b/modulefiles/build_orion_intel.lua
@@ -31,6 +31,7 @@ load(pathJoin("sp", sp_ver))
 load(pathJoin("w3emc", w3emc_ver))
 load(pathJoin("gftl-shared", gftl_shared_ver))
 load(pathJoin("mapl", mapl_ver))
+load(pathJoin("scotch", scotch_ver))
 load(pathJoin("prod_util", prod_util_ver))
 load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
 

diff --git a/modulefiles/tasks/hera/task.plot_stats.lua b/modulefiles/tasks/hera/task.plot_stats.lua
@@ -1,4 +1,4 @@
-prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_unienv"))
 
 load(pathJoin("stack-intel", stack_intel_ver))
 load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))

diff --git a/modulefiles/tasks/hercules b/modulefiles/tasks/hercules
@@ -0,0 +1 @@
+hera
diff --git a/modulefiles/tasks/hercules/task.analysis.lua b/modulefiles/tasks/hercules/task.analysis.lua
diff --git a/modulefiles/tasks/hercules/task.forecast.lua b/modulefiles/tasks/hercules/task.forecast.lua
diff --git a/modulefiles/tasks/hercules/task.plot_stats.lua b/modulefiles/tasks/hercules/task.plot_stats.lua
diff --git a/modulefiles/tasks/hercules/task.post_anal.lua b/modulefiles/tasks/hercules/task.post_anal.lua
diff --git a/modulefiles/tasks/hercules/task.pre_anal.lua b/modulefiles/tasks/hercules/task.pre_anal.lua
diff --git a/modulefiles/tasks/hercules/task.prep_obs.lua b/modulefiles/tasks/hercules/task.prep_obs.lua
diff --git a/modulefiles/tasks/orion b/modulefiles/tasks/orion
@@ -0,0 +1 @@
+hera
diff --git a/modulefiles/tasks/orion/task.analysis.lua b/modulefiles/tasks/orion/task.analysis.lua
diff --git a/modulefiles/tasks/orion/task.forecast.lua b/modulefiles/tasks/orion/task.forecast.lua
diff --git a/modulefiles/tasks/orion/task.plot_stats.lua b/modulefiles/tasks/orion/task.plot_stats.lua
diff --git a/modulefiles/tasks/orion/task.post_anal.lua b/modulefiles/tasks/orion/task.post_anal.lua
diff --git a/modulefiles/tasks/orion/task.pre_anal.lua b/modulefiles/tasks/orion/task.pre_anal.lua
diff --git a/modulefiles/tasks/orion/task.prep_obs.lua b/modulefiles/tasks/orion/task.prep_obs.lua
diff --git a/parm/land_analysis_hera.yaml b/parm/land_analysis_hera.yaml
@@ -17,9 +17,18 @@ workflow:
     WARMSTART_DIR: "/scratch2/NAGAPE/epic/UFS_Land-DA_Dev/inputs/DATA_RESTART"
     ATMOS_FORC: "gswp3"
     RES: "96"
-    FCSTHR: "24"
     NPROCS_ANALYSIS: "6"
-    NPROCS_FORECAST: "7"
+    FCSTHR: "24"
+    DT_ATMOS: "900"
+    DT_RUNSEQ: "3600"
+    NPROCS_FORECAST: "26"
+    NPROCS_FORECAST_ATM: "12"
+    NPROCS_FORECAST_LND: "12"
+    LND_LAYOUT_X: "1"
+    LND_LAYOUT_Y: "2"
+    LND_OUTPUT_FREQ_SEC: "21600"
+    NNODES_FORECAST: "1"
+    NPROCS_PER_NODE: "26"
     OBSDIR: ""
     OBSDIR_SUBDIR: ""
     OBS_TYPES: "GHCN"
@@ -213,12 +222,21 @@ workflow:
         cyc: "&cyc;"
         DAtype: "&DAtype;"
         FCSTHR: "&FCSTHR;"
-        NPROCS_FORECAST: "&NPROCS_FORECAST;"          
+        DT_ATMOS: "&DT_ATMOS;"
+        DT_RUNSEQ: "&DT_RUNSEQ;"
+        NPROCS_FORECAST: "&NPROCS_FORECAST;"
+        NPROCS_FORECAST_ATM: "&NPROCS_FORECAST_ATM;"
+        NPROCS_FORECAST_LND: "&NPROCS_FORECAST_LND;"
+        LND_LAYOUT_X: "&LND_LAYOUT_X;"
+        LND_LAYOUT_Y: "&LND_LAYOUT_Y;"
+        LND_OUTPUT_FREQ_SEC: "&LND_OUTPUT_FREQ_SEC;"
+        NNODES_FORECAST: "&NNODES_FORECAST;"
+        NPROCS_PER_NODE: "&NPROCS_PER_NODE;"
       account: "&ACCOUNT;"
       command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "forecast" "&HOMElandda;" "&MACHINE;"'
       jobname: forecast
-      nodes: "1:ppn=&NPROCS_FORECAST;"
-      walltime: 01:00:00
+      nodes: "&NNODES_FORECAST;:ppn=&NPROCS_PER_NODE;"
+      walltime: 00:30:00
       queue: batch
       join: "&LOGDIR;/forecast&LOGFN_SUFFIX;"
       dependency:

diff --git a/parm/land_analysis_hercules.yaml b/parm/land_analysis_hercules.yaml
@@ -13,13 +13,22 @@ workflow:
     SCHED: "slurm"
     ACCOUNT: "epic"
     EXP_BASEDIR: "/work2/noaa/epic/{USER}/landda_test"
-    JEDI_INSTALL: "/work2/noaa/epic/UFS_Land-DA_Dev/jedi_v7_hercules"
+    JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA_Dev/jedi_v7_hercules"
     WARMSTART_DIR: "/work/noaa/epic/UFS_Land-DA_Dev/inputs/DATA_RESTART"
     ATMOS_FORC: "gswp3"
     RES: "96"
-    FCSTHR: "24"
     NPROCS_ANALYSIS: "6"
-    NPROCS_FORECAST: "7"
+    FCSTHR: "24"
+    DT_ATMOS: "900"
+    DT_RUNSEQ: "3600"
+    NPROCS_FORECAST: "26"
+    NPROCS_FORECAST_ATM: "12"
+    NPROCS_FORECAST_LND: "12"
+    LND_LAYOUT_X: "1"
+    LND_LAYOUT_Y: "2"
+    LND_OUTPUT_FREQ_SEC: "21600"
+    NNODES_FORECAST: "1"
+    NPROCS_PER_NODE: "26"
     OBSDIR: ""
     OBSDIR_SUBDIR: ""
     OBS_TYPES: "GHCN"
@@ -213,12 +222,21 @@ workflow:
         cyc: "&cyc;"
         DAtype: "&DAtype;"
         FCSTHR: "&FCSTHR;"
-        NPROCS_FORECAST: "&NPROCS_FORECAST;"          
+        DT_ATMOS: "&DT_ATMOS;"
+        DT_RUNSEQ: "&DT_RUNSEQ;"
+        NPROCS_FORECAST: "&NPROCS_FORECAST;"
+        NPROCS_FORECAST_ATM: "&NPROCS_FORECAST_ATM;"
+        NPROCS_FORECAST_LND: "&NPROCS_FORECAST_LND;"
+        LND_LAYOUT_X: "&LND_LAYOUT_X;"
+        LND_LAYOUT_Y: "&LND_LAYOUT_Y;"
+        LND_OUTPUT_FREQ_SEC: "&LND_OUTPUT_FREQ_SEC;"
+        NNODES_FORECAST: "&NNODES_FORECAST;"
+        NPROCS_PER_NODE: "&NPROCS_PER_NODE;"
       account: "&ACCOUNT;"
       command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "forecast" "&HOMElandda;" "&MACHINE;"'
       jobname: forecast
-      nodes: "1:ppn=&NPROCS_FORECAST;"
-      walltime: 01:00:00
+      nodes: "&NNODES_FORECAST;:ppn=&NPROCS_PER_NODE;"
+      walltime: 00:30:00
       queue: batch
       join: "&LOGDIR;/forecast&LOGFN_SUFFIX;"
       dependency:

diff --git a/parm/land_analysis_orion.yaml b/parm/land_analysis_orion.yaml
@@ -17,9 +17,18 @@ workflow:
     WARMSTART_DIR: "/work/noaa/epic/UFS_Land-DA_Dev/inputs/DATA_RESTART"
     ATMOS_FORC: "gswp3"
     RES: "96"
-    FCSTHR: "24"
     NPROCS_ANALYSIS: "6"
-    NPROCS_FORECAST: "7"
+    FCSTHR: "24"
+    DT_ATMOS: "900"
+    DT_RUNSEQ: "3600"
+    NPROCS_FORECAST: "26"
+    NPROCS_FORECAST_ATM: "12"
+    NPROCS_FORECAST_LND: "12"
+    LND_LAYOUT_X: "1"
+    LND_LAYOUT_Y: "2"
+    LND_OUTPUT_FREQ_SEC: "21600"
+    NNODES_FORECAST: "1"
+    NPROCS_PER_NODE: "26"
     OBSDIR: ""
     OBSDIR_SUBDIR: ""
     OBS_TYPES: "GHCN"
@@ -213,12 +222,21 @@ workflow:
         cyc: "&cyc;"
         DAtype: "&DAtype;"
         FCSTHR: "&FCSTHR;"
-        NPROCS_FORECAST: "&NPROCS_FORECAST;"          
+        DT_ATMOS: "&DT_ATMOS;"
+        DT_RUNSEQ: "&DT_RUNSEQ;"
+        NPROCS_FORECAST: "&NPROCS_FORECAST;"
+        NPROCS_FORECAST_ATM: "&NPROCS_FORECAST_ATM;"
+        NPROCS_FORECAST_LND: "&NPROCS_FORECAST_LND;"
+        LND_LAYOUT_X: "&LND_LAYOUT_X;"
+        LND_LAYOUT_Y: "&LND_LAYOUT_Y;"
+        LND_OUTPUT_FREQ_SEC: "&LND_OUTPUT_FREQ_SEC;"
+        NNODES_FORECAST: "&NNODES_FORECAST;"
+        NPROCS_PER_NODE: "&NPROCS_PER_NODE;"
       account: "&ACCOUNT;"
       command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "forecast" "&HOMElandda;" "&MACHINE;"'
       jobname: forecast
-      nodes: "1:ppn=&NPROCS_FORECAST;"
-      walltime: 01:00:00
+      nodes: "&NNODES_FORECAST;:ppn=&NPROCS_PER_NODE;"
+      walltime: 00:30:00
       queue: batch
       join: "&LOGDIR;/forecast&LOGFN_SUFFIX;"
       dependency:

diff --git a/parm/templates/template.data_table b/parm/templates/template.data_table
@@ -0,0 +1 @@
+"OCN", "runoff", "runoff", "./INPUT/", "none" ,  1.0
diff --git a/parm/templates/template.datm_in b/parm/templates/template.datm_in
@@ -11,4 +11,5 @@
   nx_global = 144
   ny_global = 96
   restfilm = "null"
+  export_all = .true.
 /
diff --git a/parm/templates/template.diag_table b/parm/templates/template.diag_table
@@ -75,6 +75,7 @@ XXYYYY XXMM XXDD XXHH 0 0
 "gfs_dyn",     "delp",        "dpres",     "fv3_history",    "all",  .false.,  "none",  2
 "gfs_dyn",     "delz",        "delz",      "fv3_history",    "all",  .false.,  "none",  2
 "gfs_dyn",     "w",           "dzdt",      "fv3_history",    "all",  .false.,  "none",  2
+"gfs_dyn",     "omga",        "omga",      "fv3_history",    "all",  .false.,  "none",  2
 "gfs_dyn",     "ice_wat",     "icmr",      "fv3_history",    "all",  .false.,  "none",  2
 "gfs_dyn",     "rainwat",     "rwmr",      "fv3_history",    "all",  .false.,  "none",  2
 "gfs_dyn",     "snowwat",     "snmr",      "fv3_history",    "all",  .false.,  "none",  2