Fix issue on multiple log files with same prefix in plot_stats (#168)

* fix issues on multiple log files

* update detect_platform script

* fix coldstart of forecast

parm/detect_platform.sh

@@ -2,20 +2,25 @@
 #
 # Detect HPC platforms
 #
-if [[ -d /scratch1 ]] ; then
+if [[ -d /scratch2/NAGAPE ]] ; then
   PLATFORM="hera"
-elif [[ -d /jetmon ]] ; then
-  PLATFORM="jet"
-elif [[ -d /work ]]; then
+elif [[ -d /work/noaa ]]; then
   hoststr=$(hostname)
   if [[ "$hoststr" == "hercules"* ]]; then
     PLATFORM="hercules"
   else
     PLATFORM="orion"
   fi
+elif [[ -d /ncrc ]]; then
+  PLATFORM="gaea"
+elif [[ -d /glade ]]; then
+  PLATFORM="derecho"
+elif [[ -d /lfs4/HFIP ]] ; then
+  PLATFORM="jet"
 elif [[ -d /lfs/h2 ]] ; then
   PLATFORM="wcoss2"
 else
   PLATFORM="unknown"
 fi
 MACHINE="${PLATFORM}"
+

parm/templates/template.datm.streams

@@ -10,9 +10,9 @@ yearFirst01:               1999
 yearLast01:                2000
 yearAlign01:               1999
 stream_vectors01:          null
-stream_mesh_file01:        INPUT/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.SCRIP.210520_ESMFmesh.nc
+stream_mesh_file01:        INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.SCRIP.210520_ESMFmesh.nc
 stream_lev_dimname01:      null
-stream_data_files01:       "INPUT/clmforc.GSWP3.c2011.0.5x0.5.Solr.1999-12.nc" "INPUT/clmforc.GSWP3.c2011.0.5x0.5.Solr.2000-01.nc"
+stream_data_files01:       "INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Solr.1999-12.nc" "INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Solr.2000-01.nc"
 stream_data_variables01:   "FSDS Faxa_swdn"
 
 taxmode02:                 limit
@@ -25,9 +25,9 @@ yearFirst02:               1999
 yearLast02:                2000
 yearAlign02:               1999
 stream_vectors02:          null
-stream_mesh_file02:        INPUT/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.SCRIP.210520_ESMFmesh.nc
+stream_mesh_file02:        INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.SCRIP.210520_ESMFmesh.nc
 stream_lev_dimname02:      null
-stream_data_files02:       "INPUT/clmforc.GSWP3.c2011.0.5x0.5.Prec.1999-12.nc" "INPUT/clmforc.GSWP3.c2011.0.5x0.5.Prec.2000-01.nc"
+stream_data_files02:       "INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Prec.1999-12.nc" "INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Prec.2000-01.nc"
 stream_data_variables02:   "PRECTmms Faxa_precn"
 
 taxmode03:                 limit
@@ -40,9 +40,9 @@ yearFirst03:               1999
 yearLast03:                2000
 yearAlign03:               1999
 stream_vectors03:          null
-stream_mesh_file03:        INPUT/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.SCRIP.210520_ESMFmesh.nc
+stream_mesh_file03:        INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.SCRIP.210520_ESMFmesh.nc
 stream_lev_dimname03:      null
-stream_data_files03:       "INPUT/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.1999-12.nc" "INPUT/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.2000-01.nc"
+stream_data_files03:       "INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.1999-12.nc" "INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.2000-01.nc"
 stream_data_variables03:   "TBOT Sa_tbot" "WIND Sa_wind" "QBOT Sa_shum" "PSRF Sa_pbot" "PSRF Sa_pslv" "FLDS Faxa_lwdn"
 
 taxmode04:                 cycle
@@ -55,7 +55,7 @@ yearFirst04:               1
 yearLast04:                1
 yearAlign04:               1
 stream_vectors04:          null
-stream_mesh_file04:        INPUT/topodata_0.9x1.SCRIP.210520_ESMFmesh.nc
+stream_mesh_file04:        INPUT_DATM/topodata_0.9x1.SCRIP.210520_ESMFmesh.nc
 stream_lev_dimname04:      null
-stream_data_files04:       "INPUT/topodata_0.9x1.25_USGS_070110_stream_c151201.nc"
+stream_data_files04:       "INPUT_DATM/topodata_0.9x1.25_USGS_070110_stream_c151201.nc"
 stream_data_variables04:   "TOPO Sa_topo"

parm/templates/template.datm_in

@@ -6,8 +6,8 @@
   flds_presaero = .false.
   flds_wiso = .false.
   iradsw = 1
-  model_maskfile = "INPUT/fv1.9x2.5_141008_ESMFmesh.nc"
-  model_meshfile = "INPUT/fv1.9x2.5_141008_ESMFmesh.nc"
+  model_maskfile = "INPUT_DATM/fv1.9x2.5_141008_ESMFmesh.nc"
+  model_meshfile = "INPUT_DATM/fv1.9x2.5_141008_ESMFmesh.nc"
   nx_global = 144
   ny_global = 96
   restfilm = "null"

parm/templates/template.land_analysis.yaml

@@ -7,7 +7,7 @@ workflow:
   cycledef:
     - attrs:
         group: cycled
-      spec: 200001030000 200001040000 24:00:00
+      spec: 200001040000 200001050000 24:00:00
   entities:
     MACHINE: "{{ machine }}"
     SCHED: "slurm"

parm/templates/template.ufs.configure

@@ -102,7 +102,7 @@ ALLCOMP_attributes::
   ScalarFieldIdxGridNTile = 3
   ScalarFieldIdxNextSwCday = 4 # required for data atmosphere configurations
   ScalarFieldName = cpl_scalars
-  start_type = startup
+  start_type = continue
   restart_dir = ./RESTART/
   case_name = ufs.cpld
   restart_n = 1
@@ -112,7 +112,7 @@ ALLCOMP_attributes::
   stop_n = 24
   stop_option = nhours
   stop_ymd = -999
-  read_restart = .false.
+  read_restart = .true.
   mediator_present = true
   orb_eccen = 1.e36
   orb_iyear = 2000

scripts/exlandda_forecast.sh

@@ -126,9 +126,11 @@ else
 fi
 ls -1 "RESTART/${rfile2}">rpointer.atm
 
+mkdir -p INPUT_DATM
+ln -nsf ${FIXlandda}/DATM_input_data/${ATMOS_FORC}/* INPUT_DATM/.
+
 mkdir -p INPUT
 cd INPUT
-ln -nsf ${FIXlandda}/DATM_input_data/${ATMOS_FORC}/* .
 for itile in {1..6}
 do
   ln -nsf ${FIXlandda}/NOAHMP_IC/ufs-land_C${RES}_init_fields.tile${itile}.nc C${RES}.initial.tile${itile}.nc

ush/plot_analysis_timehistory.py

@@ -8,6 +8,7 @@
 ## History ===============================
 ## V000: 2024/10/14: Chan-Hoo Jeon : Preliminary version
 ## V001: 2024/10/15: Chan-Hoo Jeon : Add wall-clock time plot
+## V002: 2024/10/31: Chan-Hoo Jeon : Fix input log file name issue
 ###################################################################### CHJ #####
 
 import os, sys
@@ -64,11 +65,13 @@ def get_data_analysis(path_data,fn_data_anal_prefix,fn_data_anal_suffix,nprocs_a
     fp_data_anal_prefix = os.path.join(path_data,fn_data_anal_prefix)
     files = []
     for entry in os.scandir(path_data):
-        if entry.is_file() and entry.name.startswith(fn_data_anal_prefix):
+        if entry.is_file() and \
+           entry.name.startswith(fn_data_anal_prefix) and \
+           entry.name.endswith(fn_data_anal_suffix):
             files.append(entry.path)
 
     files.sort()
-    #print("Files=",files)
+    print("Files=",files)
 
     nobs_qc_prefix = "QC SnowDepthGHCN totalSnowDepth"
     wtime_oops_prefix = "OOPS_STATS util::Timers::Total"
@@ -134,12 +137,36 @@ def get_data_analysis(path_data,fn_data_anal_prefix,fn_data_anal_suffix,nprocs_a
                         wtime_oops_file = float(line_split[2])
                         #print("WTIME OOPS AVG=",wtime_oops_file)
 
-        min_val_final.append(min_val_file[-1])
-        max_val_final.append(max_val_file[-1])
-        rms_val_final.append(rms_val_file[-1])
-        nobs_qc_final.append(nobs_qc_file[-1])
-        nobs_in_final.append(nobs_in_file[-1])
-        wtime_oops.append(wtime_oops_file)
+        if not min_val_file:
+            min_val_final.append(None)
+        else:
+            min_val_final.append(min_val_file[-1])
+
+        if not max_val_file:
+            max_val_final.append(None)
+        else:
+            max_val_final.append(max_val_file[-1])
+
+        if not rms_val_file:
+            rms_val_final.append(None)
+        else:
+            rms_val_final.append(rms_val_file[-1])
+
+        if not nobs_qc_file:
+            nobs_qc_final.append(None)
+        else:
+            nobs_qc_final.append(nobs_qc_file[-1])
+
+        if not nobs_in_file:
+            nobs_in_final.append(None)
+        else:
+            nobs_in_final.append(nobs_in_file[-1])
+
+        if not wtime_oops_file:
+            wtime_oops.append(None)
+        else:
+            wtime_oops.append(wtime_oops_file)
+
     # ms to sec 
     wtime_oops = [x * 0.001 for x in wtime_oops]
     tcpu_oops = [x * nprocs_anal for x in wtime_oops]
@@ -193,7 +220,10 @@ def get_data_forecast(path_data,fn_data_fcst_prefix,fn_data_fcst_suffix,nprocs_f
                     wtime_uwm_file = float(line_split)
                     #print("WTIME UFS Weather Model=",wtime_uwm_file)
 
-        wtime_uwm.append(wtime_uwm_file)
+        if not wtime_uwm_file:
+            wtime_uwm.append(None)
+        else:
+            wtime_uwm.append(wtime_uwm_file)
 
     tcpu_uwm = [x * nprocs_fcst for x in wtime_uwm]