From 555b70829e66a5045869fa8df9d5c899af99d473 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:46:34 -0500 Subject: [PATCH] Fix issue on multiple log files with same prefix in plot_stats (#168) * fix issues on multiple log files * update detect_platform script * fix coldstart of forecast --- parm/detect_platform.sh | 13 ++++-- parm/templates/template.datm.streams | 16 ++++---- parm/templates/template.datm_in | 4 +- parm/templates/template.land_analysis.yaml | 2 +- parm/templates/template.ufs.configure | 4 +- scripts/exlandda_forecast.sh | 4 +- ush/plot_analysis_timehistory.py | 48 ++++++++++++++++++---- 7 files changed, 64 insertions(+), 27 deletions(-) diff --git a/parm/detect_platform.sh b/parm/detect_platform.sh index be6ea2f7..42516330 100755 --- a/parm/detect_platform.sh +++ b/parm/detect_platform.sh @@ -2,20 +2,25 @@ # # Detect HPC platforms # -if [[ -d /scratch1 ]] ; then +if [[ -d /scratch2/NAGAPE ]] ; then PLATFORM="hera" -elif [[ -d /jetmon ]] ; then - PLATFORM="jet" -elif [[ -d /work ]]; then +elif [[ -d /work/noaa ]]; then hoststr=$(hostname) if [[ "$hoststr" == "hercules"* ]]; then PLATFORM="hercules" else PLATFORM="orion" fi +elif [[ -d /ncrc ]]; then + PLATFORM="gaea" +elif [[ -d /glade ]]; then + PLATFORM="derecho" +elif [[ -d /lfs4/HFIP ]] ; then + PLATFORM="jet" elif [[ -d /lfs/h2 ]] ; then PLATFORM="wcoss2" else PLATFORM="unknown" fi MACHINE="${PLATFORM}" + diff --git a/parm/templates/template.datm.streams b/parm/templates/template.datm.streams index 6ab531f7..3badd37b 100644 --- a/parm/templates/template.datm.streams +++ b/parm/templates/template.datm.streams @@ -10,9 +10,9 @@ yearFirst01: 1999 yearLast01: 2000 yearAlign01: 1999 stream_vectors01: null -stream_mesh_file01: INPUT/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.SCRIP.210520_ESMFmesh.nc +stream_mesh_file01: INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.SCRIP.210520_ESMFmesh.nc stream_lev_dimname01: null -stream_data_files01: "INPUT/clmforc.GSWP3.c2011.0.5x0.5.Solr.1999-12.nc" "INPUT/clmforc.GSWP3.c2011.0.5x0.5.Solr.2000-01.nc" +stream_data_files01: "INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Solr.1999-12.nc" "INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Solr.2000-01.nc" stream_data_variables01: "FSDS Faxa_swdn" taxmode02: limit @@ -25,9 +25,9 @@ yearFirst02: 1999 yearLast02: 2000 yearAlign02: 1999 stream_vectors02: null -stream_mesh_file02: INPUT/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.SCRIP.210520_ESMFmesh.nc +stream_mesh_file02: INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.SCRIP.210520_ESMFmesh.nc stream_lev_dimname02: null -stream_data_files02: "INPUT/clmforc.GSWP3.c2011.0.5x0.5.Prec.1999-12.nc" "INPUT/clmforc.GSWP3.c2011.0.5x0.5.Prec.2000-01.nc" +stream_data_files02: "INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Prec.1999-12.nc" "INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Prec.2000-01.nc" stream_data_variables02: "PRECTmms Faxa_precn" taxmode03: limit @@ -40,9 +40,9 @@ yearFirst03: 1999 yearLast03: 2000 yearAlign03: 1999 stream_vectors03: null -stream_mesh_file03: INPUT/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.SCRIP.210520_ESMFmesh.nc +stream_mesh_file03: INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.SCRIP.210520_ESMFmesh.nc stream_lev_dimname03: null -stream_data_files03: "INPUT/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.1999-12.nc" "INPUT/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.2000-01.nc" +stream_data_files03: "INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.1999-12.nc" "INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.2000-01.nc" stream_data_variables03: "TBOT Sa_tbot" "WIND Sa_wind" "QBOT Sa_shum" "PSRF Sa_pbot" "PSRF Sa_pslv" "FLDS Faxa_lwdn" taxmode04: cycle @@ -55,7 +55,7 @@ yearFirst04: 1 yearLast04: 1 yearAlign04: 1 stream_vectors04: null -stream_mesh_file04: INPUT/topodata_0.9x1.SCRIP.210520_ESMFmesh.nc +stream_mesh_file04: INPUT_DATM/topodata_0.9x1.SCRIP.210520_ESMFmesh.nc stream_lev_dimname04: null -stream_data_files04: "INPUT/topodata_0.9x1.25_USGS_070110_stream_c151201.nc" +stream_data_files04: "INPUT_DATM/topodata_0.9x1.25_USGS_070110_stream_c151201.nc" stream_data_variables04: "TOPO Sa_topo" diff --git a/parm/templates/template.datm_in b/parm/templates/template.datm_in index 783d965e..4dc30373 100644 --- a/parm/templates/template.datm_in +++ b/parm/templates/template.datm_in @@ -6,8 +6,8 @@ flds_presaero = .false. flds_wiso = .false. iradsw = 1 - model_maskfile = "INPUT/fv1.9x2.5_141008_ESMFmesh.nc" - model_meshfile = "INPUT/fv1.9x2.5_141008_ESMFmesh.nc" + model_maskfile = "INPUT_DATM/fv1.9x2.5_141008_ESMFmesh.nc" + model_meshfile = "INPUT_DATM/fv1.9x2.5_141008_ESMFmesh.nc" nx_global = 144 ny_global = 96 restfilm = "null" diff --git a/parm/templates/template.land_analysis.yaml b/parm/templates/template.land_analysis.yaml index 0bef1b72..e801e33b 100644 --- a/parm/templates/template.land_analysis.yaml +++ b/parm/templates/template.land_analysis.yaml @@ -7,7 +7,7 @@ workflow: cycledef: - attrs: group: cycled - spec: 200001030000 200001040000 24:00:00 + spec: 200001040000 200001050000 24:00:00 entities: MACHINE: "{{ machine }}" SCHED: "slurm" diff --git a/parm/templates/template.ufs.configure b/parm/templates/template.ufs.configure index 792bd679..0d6ec1e3 100644 --- a/parm/templates/template.ufs.configure +++ b/parm/templates/template.ufs.configure @@ -102,7 +102,7 @@ ALLCOMP_attributes:: ScalarFieldIdxGridNTile = 3 ScalarFieldIdxNextSwCday = 4 # required for data atmosphere configurations ScalarFieldName = cpl_scalars - start_type = startup + start_type = continue restart_dir = ./RESTART/ case_name = ufs.cpld restart_n = 1 @@ -112,7 +112,7 @@ ALLCOMP_attributes:: stop_n = 24 stop_option = nhours stop_ymd = -999 - read_restart = .false. + read_restart = .true. mediator_present = true orb_eccen = 1.e36 orb_iyear = 2000 diff --git a/scripts/exlandda_forecast.sh b/scripts/exlandda_forecast.sh index 5a03d8e9..6dbf3d1a 100755 --- a/scripts/exlandda_forecast.sh +++ b/scripts/exlandda_forecast.sh @@ -126,9 +126,11 @@ else fi ls -1 "RESTART/${rfile2}">rpointer.atm +mkdir -p INPUT_DATM +ln -nsf ${FIXlandda}/DATM_input_data/${ATMOS_FORC}/* INPUT_DATM/. + mkdir -p INPUT cd INPUT -ln -nsf ${FIXlandda}/DATM_input_data/${ATMOS_FORC}/* . for itile in {1..6} do ln -nsf ${FIXlandda}/NOAHMP_IC/ufs-land_C${RES}_init_fields.tile${itile}.nc C${RES}.initial.tile${itile}.nc diff --git a/ush/plot_analysis_timehistory.py b/ush/plot_analysis_timehistory.py index daaa4fda..986c8ecd 100755 --- a/ush/plot_analysis_timehistory.py +++ b/ush/plot_analysis_timehistory.py @@ -8,6 +8,7 @@ ## History =============================== ## V000: 2024/10/14: Chan-Hoo Jeon : Preliminary version ## V001: 2024/10/15: Chan-Hoo Jeon : Add wall-clock time plot +## V002: 2024/10/31: Chan-Hoo Jeon : Fix input log file name issue ###################################################################### CHJ ##### import os, sys @@ -64,11 +65,13 @@ def get_data_analysis(path_data,fn_data_anal_prefix,fn_data_anal_suffix,nprocs_a fp_data_anal_prefix = os.path.join(path_data,fn_data_anal_prefix) files = [] for entry in os.scandir(path_data): - if entry.is_file() and entry.name.startswith(fn_data_anal_prefix): + if entry.is_file() and \ + entry.name.startswith(fn_data_anal_prefix) and \ + entry.name.endswith(fn_data_anal_suffix): files.append(entry.path) files.sort() - #print("Files=",files) + print("Files=",files) nobs_qc_prefix = "QC SnowDepthGHCN totalSnowDepth" wtime_oops_prefix = "OOPS_STATS util::Timers::Total" @@ -134,12 +137,36 @@ def get_data_analysis(path_data,fn_data_anal_prefix,fn_data_anal_suffix,nprocs_a wtime_oops_file = float(line_split[2]) #print("WTIME OOPS AVG=",wtime_oops_file) - min_val_final.append(min_val_file[-1]) - max_val_final.append(max_val_file[-1]) - rms_val_final.append(rms_val_file[-1]) - nobs_qc_final.append(nobs_qc_file[-1]) - nobs_in_final.append(nobs_in_file[-1]) - wtime_oops.append(wtime_oops_file) + if not min_val_file: + min_val_final.append(None) + else: + min_val_final.append(min_val_file[-1]) + + if not max_val_file: + max_val_final.append(None) + else: + max_val_final.append(max_val_file[-1]) + + if not rms_val_file: + rms_val_final.append(None) + else: + rms_val_final.append(rms_val_file[-1]) + + if not nobs_qc_file: + nobs_qc_final.append(None) + else: + nobs_qc_final.append(nobs_qc_file[-1]) + + if not nobs_in_file: + nobs_in_final.append(None) + else: + nobs_in_final.append(nobs_in_file[-1]) + + if not wtime_oops_file: + wtime_oops.append(None) + else: + wtime_oops.append(wtime_oops_file) + # ms to sec wtime_oops = [x * 0.001 for x in wtime_oops] tcpu_oops = [x * nprocs_anal for x in wtime_oops] @@ -193,7 +220,10 @@ def get_data_forecast(path_data,fn_data_fcst_prefix,fn_data_fcst_suffix,nprocs_f wtime_uwm_file = float(line_split) #print("WTIME UFS Weather Model=",wtime_uwm_file) - wtime_uwm.append(wtime_uwm_file) + if not wtime_uwm_file: + wtime_uwm.append(None) + else: + wtime_uwm.append(wtime_uwm_file) tcpu_uwm = [x * nprocs_fcst for x in wtime_uwm]