Replace SED command to update templates with fill-jinja python script (…

…#157) * add fill-jinja-template py script * update analysis template * update analysis script * update post_anal template * update forecast templates * fix typo
ufs-community · Oct 22, 2024 · 7ee2e1c · 7ee2e1c
1 parent d8acfb4
commit 7ee2e1c
Show file tree

Hide file tree

Showing 17 changed files with 500 additions and 146 deletions.
diff --git a/modulefiles/tasks/hera/task.analysis.lua b/modulefiles/tasks/hera/task.analysis.lua
@@ -5,6 +5,9 @@ load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 load(pathJoin("stack-python", stack_python_ver))
 
 load(pathJoin("prod_util", prod_util_ver))
+
+load(pathJoin("py-jinja2", py_jinja2_ver))
 load(pathJoin("py-netcdf4", py_netcdf4_ver))
 load(pathJoin("py-numpy", py_numpy_ver))
+load(pathJoin("py-pyyaml", py_pyyaml_ver))
 
diff --git a/modulefiles/tasks/hera/task.forecast.lua b/modulefiles/tasks/hera/task.forecast.lua
@@ -23,6 +23,9 @@ load(pathJoin("w3emc", w3emc_ver))
 load(pathJoin("gftl-shared", gftl_shared_ver))
 load(pathJoin("mapl", mapl_ver))
 load(pathJoin("prod_util", prod_util_ver))
+
+load(pathJoin("py-jinja2", py_jinja2_ver))
 load(pathJoin("py-netcdf4", py_netcdf4_ver))
 load(pathJoin("py-numpy", py_numpy_ver))
+load(pathJoin("py-pyyaml", py_pyyaml_ver))
 
diff --git a/modulefiles/tasks/hera/task.post_anal.lua b/modulefiles/tasks/hera/task.post_anal.lua
@@ -8,6 +8,9 @@ load(pathJoin("hdf5", hdf5_ver))
 load(pathJoin("netcdf-c", netcdf_c_ver))
 load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
 load(pathJoin("prod_util", prod_util_ver))
+
+load(pathJoin("py-jinja2", py_jinja2_ver))
 load(pathJoin("py-netcdf4", py_netcdf4_ver))
 load(pathJoin("py-numpy", py_numpy_ver))
+load(pathJoin("py-pyyaml", py_pyyaml_ver))
 
diff --git a/modulefiles/tasks/hera/task.pre_anal.lua b/modulefiles/tasks/hera/task.pre_anal.lua
@@ -8,3 +8,7 @@ load(pathJoin("hdf5", hdf5_ver))
 load(pathJoin("netcdf-c", netcdf_c_ver))
 load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
 load(pathJoin("prod_util", prod_util_ver))
+
+load(pathJoin("py-jinja2", py_jinja2_ver))
+load(pathJoin("py-pyyaml", py_pyyaml_ver))
+
diff --git a/parm/jedi/GHCN.yaml b/parm/jedi/GHCN.yaml
@@ -7,11 +7,11 @@
       obsdatain:
         engine:
           type: H5File
-          obsfile: GHCN_XXYYYYXXMMXXDDXXHH.nc
+          obsfile: GHCN_{{ yyyymmddhh }}.nc
       obsdataout:
         engine:
           type: H5File
-          obsfile: output/DA/hofx/letkf_hofx_ghcn_XXYYYYXXMMXXDDXXHH.nc
+          obsfile: output/DA/hofx/letkf_hofx_ghcn_{{ yyyymmddhh }}.nc
     obs operator:
       name: Identity
     obs error:

diff --git a/parm/jedi/letkfoi_snow.yaml b/parm/jedi/letkfoi_snow.yaml
@@ -3,45 +3,45 @@ geometry:
     namelist filename: Data/fv3files/fmsmpp.nml
     field table filename: Data/fv3files/field_table
   akbk: Data/fv3files/akbk64.nc4
-  npx: XXREP
-  npy: XXREP
+  npx: {{ resp1 }}
+  npy: {{ resp1 }}
   npz: 64
   field metadata override: gfs-land.yaml
   time invariant fields:
     state fields:
-      datetime: XXYYYP-XXMP-XXDPTXXHP:00:00Z
+      datetime: {{ yyyp }}-{{ mp }}-{{ dp }}T{{ hp }}:00:00Z
       filetype: fms restart
       skip coupler file: true
       state variables: [orog_filt]
-      datapath: XXTPATH
-      filename_orog: XXTSTUB.nc
+      datapath: {{ tpath }}
+      filename_orog: {{ tstub }}.nc
     derived fields: [nominal_surface_pressure]
 
 time window:
-  begin: XXYYYP-XXMP-XXDPTXXHP:00:00Z
+  begin: {{ yyyp }}-{{ mp }}-{{ dp }}T{{ hp }}:00:00Z
   length: PT24H
 
 background:
- date: &date XXYYYY-XXMM-XXDDTXXHH:00:00Z
+ date: &date {{ yyyy }}-{{ mm }}-{{ dd }}T{{ hh }}:00:00Z
  members:
-   - datetime: XXYYYY-XXMM-XXDDTXXHH:00:00Z
+   - datetime: {{ yyyy }}-{{ mm }}-{{ dd }}T{{ hh }}:00:00Z
      filetype: fms restart
      state variables: [snwdph,vtype,slmsk]
      datapath: mem_pos/
-     filename_sfcd: XXYYYYXXMMXXDD.XXHH0000.sfc_data.nc
-     filename_cplr: XXYYYYXXMMXXDD.XXHH0000.coupler.res
-   - datetime: XXYYYY-XXMM-XXDDTXXHH:00:00Z
+     filename_sfcd: {{ yyyymmdd }}.{{ hh }}0000.sfc_data.nc
+     filename_cplr: {{ yyyymmdd }}.{{ hh }}0000.coupler.res
+   - datetime: {{ yyyy }}-{{ mm }}-{{ dd }}T{{ hh }}:00:00Z
      filetype: fms restart
      state variables: [snwdph,vtype,slmsk]
      datapath: mem_neg/
-     filename_sfcd: XXYYYYXXMMXXDD.XXHH0000.sfc_data.nc
-     filename_cplr: XXYYYYXXMMXXDD.XXHH0000.coupler.res
+     filename_sfcd: {{ yyyymmdd }}.{{ hh }}0000.sfc_data.nc
+     filename_cplr: {{ yyyymmdd }}.{{ hh }}0000.coupler.res
 
 driver:
   save posterior mean: false
   save posterior mean increment: true
   save posterior ensemble: false
-  run as observer only: XXHOFX
+  run as observer only: {{ driver_obs_only }}
 
 local ensemble DA:
   solver: LETKF

diff --git a/parm/templates/template.coupler.res b/parm/templates/template.coupler.res
@@ -1,4 +1,4 @@
      2        (Calendar: no_calendar=0, thirty_day_months=1, julian=2, gregorian=3, noleap=4)
-  XXYYYP    XXMP    XXDP   XXHP   0     0        Model start time:   year, month, day, hour, minute, second
-  XXYYYY    XXMM    XXDD   XXHH   0     0        Current model time: year, month, day, hour, minute, second
+  {{ yyyp }}    {{ mp }}    {{ dp }}   {{ hp }}   0     0        Model start time:   year, month, day, hour, minute, second
+  {{ yyyy }}    {{ mm }}    {{ dd }}   {{ hh }}   0     0        Current model time: year, month, day, hour, minute, second
 
diff --git a/parm/templates/template.diag_table b/parm/templates/template.diag_table
@@ -1,5 +1,5 @@
-XXYYYYMMDD.00Z.1760x880.64bit.non-mono
-XXYYYY XXMM XXDD XXHH 0 0
+{{ yyyymmdd }}.00Z.1760x880.64bit.non-mono
+{{ yyyy }} {{ mm }} {{ dd }} {{ hh }} 0 0
 
 "fv3_history",    0,  "hours",  1,  "hours",  "time"
 "fv3_history2d",  0,  "hours",  1,  "hours",  "time"

diff --git a/parm/templates/template.jedi2ufs b/parm/templates/template.jedi2ufs
@@ -8,17 +8,17 @@
 
 ! FV3 resolution and path to oro files for restart/perturbation conversion
 
-  tile_size = XXRES
-  tile_path = "FIXlandda/FV3_fix_tiled/CXXRES/"
-  tile_fstub = "XXTSTUB"
+  tile_size = {{ res }}
+  tile_path = "{{ fix_landda }}/FV3_fix_tiled/C{{ res }}/"
+  tile_fstub = "{{ tstub }}"
 
 !------------------- only restart conversion -------------------
 ! Time stamp for conversion for restart conversion
 
-  restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00"
+  restart_date = "{{ yyyy }}-{{ mm }}-{{ dd }} {{ hh }}:00:00"
 
 ! Path for static file  
-  static_filename = "FIXlandda/static/ufs-land_CXXRES_static_fields.nc"
+  static_filename = "{{ fix_landda }}/static/ufs-land_C{{ res }}_static_fields.nc"
 
 ! Location of vector restart file (vector2tile direction)
 

diff --git a/parm/templates/template.model_configure b/parm/templates/template.model_configure
@@ -1,9 +1,9 @@
-start_year:              XXYYYY
-start_month:             XXMM
-start_day:               XXDD
-start_hour:              XXHH
+start_year:              {{ yyyy }}
+start_month:             {{ mm }}
+start_day:               {{ dd }}
+start_hour:              {{ hh }}
 start_minute:            0
 start_second:            0
-nhours_fcst:             XXFCSTHR
-dt_atmos:                XXDT_ATMOS
+nhours_fcst:             {{ fcsthr }}
+dt_atmos:                {{ dt_atmos }}
 fhrot:                   0
diff --git a/parm/templates/template.ufs.configure b/parm/templates/template.ufs.configure
@@ -14,7 +14,7 @@ EARTH_attributes::
 
 # MED #
 MED_model:                      cmeps
-MED_petlist_bounds:             0 XXNPROCS_ATM_M1
+MED_petlist_bounds:             0 {{ nprocs_atm_m1 }}
 MED_omp_num_threads:            1
 MED_attributes::
   Verbosity = 1
@@ -30,7 +30,7 @@ MED_attributes::
 
 # ATM #
 ATM_model:                      datm
-ATM_petlist_bounds:             0 XXNPROCS_ATM_M1
+ATM_petlist_bounds:             0 {{ nprocs_atm_m1 }}
 ATM_omp_num_threads:            1
 ATM_attributes::
   Verbosity = 0
@@ -39,15 +39,15 @@ ATM_attributes::
 
 # LND #
 LND_model:                      noahmp
-LND_petlist_bounds:             XXNPROCS_FORECAST_ATM XXNPROCS_ATM_LND_M1
+LND_petlist_bounds:             {{ nprocs_forecast_atm }} {{ nprocs_atm_lnd_m1 }}
 LND_omp_num_threads:            1
 LND_attributes::
   Verbosity = 1
   Diagnostic = 0
   mosaic_file = INPUT/C96_mosaic.nc
   input_dir = INPUT/
   ic_type = custom
-  layout = XXLND_LAYOUT_X:XXLND_LAYOUT_Y # need to be consistent with number of PEs (6*Lx*Ly)
+  layout = {{ lnd_layout_x }}:{{ lnd_layout_y }} # need to be consistent with number of PEs (6*Lx*Ly)
   num_soil_levels = 4
   forcing_height = 10
   soil_level_thickness = 0.10:0.30:0.60:1.00
@@ -67,15 +67,15 @@ LND_attributes::
   surface_evap_resistance_option = 1 # not used, it is fixed to 4 in sfc_noahmp_drv.F90
   glacier_option = 1
   surface_thermal_roughness_option = 2
-  output_freq = XXLND_OUTPUT_FREQ_SEC
+  output_freq = {{ lnd_output_freq_sec }}
   restart_freq = -1
   calc_snet = .true.
   initial_albedo = 0.25
 ::
 
 # cold
 runSeq::
-@XXDT_RUNSEQ
+@{{ dt_runseq }}
    MED med_phases_prep_atm
    MED -> ATM :remapMethod=redist
    ATM

diff --git a/parm/templates/template.ufs2jedi b/parm/templates/template.ufs2jedi
@@ -8,17 +8,17 @@
 
 ! FV3 resolution and path to oro files for restart/perturbation conversion
 
-  tile_size = XXRES
-  tile_path = "FIXlandda/FV3_fix_tiled/CXXRES/"
-  tile_fstub = "XXTSTUB"
+  tile_size = {{ res }}
+  tile_path = "{{ fix_landda }}/FV3_fix_tiled/C{{ res }}/"
+  tile_fstub = "{{ tstub }}"
 
 !------------------- only restart conversion -------------------
 ! Time stamp for conversion for restart conversion
 
-  restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00"
+  restart_date = "{{ yyyy }}-{{ mm }}-{{ dd }} {{ hh }}:00:00"
 
 ! Path for static file  
-  static_filename = "FIXlandda/static/ufs-land_CXXRES_static_fields.nc"
+  static_filename = "{{ fix_landda }}/static/ufs-land_C{{ res }}_static_fields.nc"
 
 ! Location of vector restart file (vector2tile direction)
 

diff --git a/scripts/exlandda_analysis.sh b/scripts/exlandda_analysis.sh
@@ -43,17 +43,21 @@ do
 done
 ln -nsf ${COMIN}/obs/*_${YYYY}${MM}${DD}${HH}.nc .
 
-cres_file=${DATA}/${FILEDATE}.coupler.res
-cp ${PARMlandda}/templates/template.coupler.res $cres_file
-
-sed -i -e "s/XXYYYY/${YYYY}/g" $cres_file
-sed -i -e "s/XXMM/${MM}/g" $cres_file
-sed -i -e "s/XXDD/${DD}/g" $cres_file
-sed -i -e "s/XXHH/${HH}/g" $cres_file
-sed -i -e "s/XXYYYP/${YYYP}/g" $cres_file
-sed -i -e "s/XXMP/${MP}/g" $cres_file
-sed -i -e "s/XXDP/${DP}/g" $cres_file
-sed -i -e "s/XXHP/${HP}/g" $cres_file
+# update coupler.res file
+settings="\
+  'yyyy': !!str ${YYYY}
+  'mm': !!str ${MM}
+  'dd': !!str ${DD}
+  'hh': !!str ${HH}
+  'yyyp': !!str ${YYYP}
+  'mp': !!str ${MP}
+  'dp': !!str ${DP}
+  'hp': !!str ${HP}
+" # End of settins variable
+
+fp_template="${PARMlandda}/templates/template.coupler.res"
+fn_namelist="${DATA}/${FILEDATE}.coupler.res"
+${USHlandda}/fill_jinja_template.py -u "${settings}" -t "${fp_template}" -o "${fn_namelist}"
 
 ################################################
 # CREATE BACKGROUND ENSEMBLE (LETKFOI)
@@ -95,11 +99,7 @@ fi
 
 do_DA="YES"
 do_HOFX="NO"
-
-if [[ $do_DA == "NO" && $do_HOFX == "NO" ]]; then 
-  echo "do_landDA:No obs found, not calling JEDI" 
-  exit 0 
-fi
+RESP1=$((RES+1))
 
 mkdir -p output/DA/hofx
 # if yaml is specified by user, use that. Otherwise, build the yaml
@@ -116,20 +116,28 @@ if [[ $do_DA == "YES" ]]; then
     cp ${PARMlandda}/jedi/${YAML_DA} ${DATA}/letkf_land.yaml
   fi
 
-  sed -i -e "s/XXYYYY/${YYYY}/g" letkf_land.yaml
-  sed -i -e "s/XXMM/${MM}/g" letkf_land.yaml
-  sed -i -e "s/XXDD/${DD}/g" letkf_land.yaml
-  sed -i -e "s/XXHH/${HH}/g" letkf_land.yaml
-  sed -i -e "s/XXYYYP/${YYYP}/g" letkf_land.yaml
-  sed -i -e "s/XXMP/${MP}/g" letkf_land.yaml
-  sed -i -e "s/XXDP/${DP}/g" letkf_land.yaml
-  sed -i -e "s/XXHP/${HP}/g" letkf_land.yaml
-  sed -i -e "s/XXTSTUB/${TSTUB}/g" letkf_land.yaml
-  sed -i -e "s#XXTPATH#${TPATH}#g" letkf_land.yaml
-  sed -i -e "s/XXRES/${RES}/g" letkf_land.yaml
-  RESP1=$((RES+1))
-  sed -i -e "s/XXREP/${RESP1}/g" letkf_land.yaml
-  sed -i -e "s/XXHOFX/false/g" letkf_land.yaml  # do DA
+  # update jedi yaml file
+  settings="\
+    'yyyy': !!str ${YYYY}
+    'mm': !!str ${MM}
+    'dd': !!str ${DD}
+    'hh': !!str ${HH}
+    'yyyymmdd': !!str ${PDY}
+    'yyyymmddhh': !!str ${PDY}${cyc}
+    'yyyp': !!str ${YYYP}
+    'mp': !!str ${MP}
+    'dp': !!str ${DP}
+    'hp': !!str ${HP}
+    'tstub': ${TSTUB}
+    'tpath': ${TPATH}
+    'res': ${RES}
+    'resp1': ${RESP1}
+    'driver_obs_only': false
+  " # End of settins variable
+
+  fp_template="${DATA}/letkf_land.yaml"
+  fn_namelist="${DATA}/letkf_land.yaml"
+  ${USHlandda}/fill_jinja_template.py -u "${settings}" -t "${fp_template}" -o "${fn_namelist}"
 fi
 
 if [[ $do_HOFX == "YES" ]]; then 
@@ -145,21 +153,28 @@ if [[ $do_HOFX == "YES" ]]; then
     cp ${PARMlandda}/jedi/${YAML_HOFX} ${DATA}/hofx_land.yaml
   fi
 
-  sed -i -e "s/XXYYYY/${YYYY}/g" hofx_land.yaml
-  sed -i -e "s/XXMM/${MM}/g" hofx_land.yaml
-  sed -i -e "s/XXDD/${DD}/g" hofx_land.yaml
-  sed -i -e "s/XXHH/${HH}/g" hofx_land.yaml
-  sed -i -e "s/XXYYYP/${YYYP}/g" hofx_land.yaml
-  sed -i -e "s/XXMP/${MP}/g" hofx_land.yaml
-  sed -i -e "s/XXDP/${DP}/g" hofx_land.yaml
-  sed -i -e "s/XXHP/${HP}/g" hofx_land.yaml
-  sed -i -e "s#XXTPATH#${TPATH}#g" hofx_land.yaml
-  sed -i -e "s/XXTSTUB/${TSTUB}/g" hofx_land.yaml
-  sed -i -e "s/XXRES/${RES}/g" hofx_land.yaml
-  RESP1=$((RES+1))
-  sed -i -e "s/XXREP/${RESP1}/g" hofx_land.yaml
-  sed -i -e "s/XXHOFX/true/g" hofx_land.yaml  # do HOFX
-
+  # update jedi yaml file
+  settings="\
+    'yyyy': !!str ${YYYY}
+    'mm': !!str ${MM}
+    'dd': !!str ${DD}
+    'hh': !!str ${HH}
+    'yyyymmdd': !!str ${PDY}
+    'yyyymmddhh': !!str ${PDY}${cyc}
+    'yyyp': !!str ${YYYP}
+    'mp': !!str ${MP}
+    'dp': !!str ${DP}
+    'hp': !!str ${HP}
+    'tstub': ${TSTUB}
+    'tpath': ${TPATH}
+    'res': ${RES}
+    'resp1': ${RESP1}
+    'driver_obs_only': true
+  " # End of settins variable
+
+  fp_template="${DATA}/hofx_land.yaml"
+  fn_namelist="${DATA}/hofx_land.yaml"
+  ${USHlandda}/fill_jinja_template.py -u "${settings}" -t "${fp_template}" -o "${fn_namelist}"
 fi
 
 if [[ "$GFSv17" == "NO" ]]; then