Update develop-ref after #2710 (#2731)

.github/update_truth_change_log.txt

@@ -9,3 +9,4 @@
 [20240926_16:50:38 develop] #2701 - #2701 added a new use case (SeriesAnalysis_aggr) that produces new output
 [20241014_19:42:56 develop] dtcenter/MET#2988 - dtcenter/MET#2988 -- see issue #2719 for details -- retry because there was an issue with convert not being available in the ubuntu image that runs the unit tests
 [20241015_17:11:29 develop] dtcenter/MET#2988 - dtcenter/MET#2988 -- see issue #2719 for details -- retry (again) because there was an issue with convert not being available in the ubuntu image that runs the unit tests and METdbLoad use cases were broken
+[20241016_20:39:30 develop] #2710 - #2710 fixed a bug in a use case

.github/workflows/sonarqube.yml

@@ -64,6 +64,9 @@ jobs:
         python3 -m pip install --upgrade pip
         python3 -m pip install -r internal/tests/pytests/requirements.txt
 
+    - name: Install ImageMagick convert
+      run: sudo apt install imagemagick
+
     - name: Run Pytests
       run: coverage run -m pytest internal/tests/pytests
       env:

docs/Contributors_Guide/continuous_integration.rst

@@ -780,6 +780,8 @@ Example::
     met_tool_wrapper/GridStat/GridStat.conf,met_tool_wrapper/GridStat/GridStat_forecast.conf,met_tool_wrapper/GridStat/GridStat_observation.conf
 
 
+.. _cg-ci-dependencies:
+
 dependencies
 """"""""""""
 
@@ -799,6 +801,8 @@ Example::
 Use Case Dependencies
 ^^^^^^^^^^^^^^^^^^^^^
 
+.. _cg-ci-conda-environments:
+
 Conda Environments
 """"""""""""""""""
 

docs/Users_Guide/glossary.rst

@@ -12970,3 +12970,23 @@ METplus Configuration Glossary
      using the -aggr command line argument.
 
      | *Used by:* SeriesAnalysis
+
+   POINT_STAT_POINT_WEIGHT_FLAG
+     Specify the value for 'point_weight_flag' in the MET configuration file for PointStat.
+
+     | *Used by:* PointStat
+
+   ENSEMBLE_STAT_POINT_WEIGHT_FLAG
+     Specify the value for 'point_weight_flag' in the MET configuration file for EnsembleStat.
+
+     | *Used by:* EnsembleStat
+
+   POINT_STAT_OBTYPE_AS_GROUP_VAL_FLAG
+     Specify the value for 'obtype_as_group_val_flag' in the MET configuration file for PointStat.
+
+     | *Used by:* PointStat
+
+   ENSEMBLE_STAT_OBTYPE_AS_GROUP_VAL_FLAG
+     Specify the value for 'obtype_as_group_val_flag' in the MET configuration file for EnsembleStat.
+
+     | *Used by:* EnsembleStat

docs/Users_Guide/wrappers.rst

@@ -233,6 +233,7 @@ METplus Configuration
 | :term:`ENSEMBLE_STAT_CUSTOM_LOOP_LIST`
 | :term:`ENSEMBLE_STAT_SKIP_IF_OUTPUT_EXISTS`
 | :term:`ENSEMBLE_STAT_DESC`
+| :term:`ENSEMBLE_STAT_OBTYPE_AS_GROUP_VAL_FLAG`
 | :term:`ENSEMBLE_STAT_ENS_SSVAR_BIN_SIZE`
 | :term:`ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE`
 | :term:`ENSEMBLE_STAT_CLIMO_CDF_BINS`
@@ -356,6 +357,7 @@ METplus Configuration
 | :term:`ENSEMBLE_STAT_ENS_MEMBER_IDS`
 | :term:`ENSEMBLE_STAT_CONTROL_ID`
 | :term:`ENSEMBLE_STAT_GRID_WEIGHT_FLAG`
+| :term:`ENSEMBLE_STAT_POINT_WEIGHT_FLAG`
 | :term:`ENSEMBLE_STAT_PROB_CAT_THRESH`
 | :term:`ENSEMBLE_STAT_PROB_PCT_THRESH`
 | :term:`ENSEMBLE_STAT_ECLV_POINTS`
@@ -849,6 +851,18 @@ ${METPLUS_OBS_ERROR_FLAG}
    * - :term:`ENSEMBLE_STAT_OBS_ERROR_FLAG`
      - obs_error.flag
 
+${METPLUS_OBTYPE_AS_GROUP_VAL_FLAG}
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. list-table::
+   :widths: 5 5
+   :header-rows: 1
+
+   * - METplus Config(s)
+     - MET Config File
+   * - :term:`ENSEMBLE_STAT_OBTYPE_AS_GROUP_VAL_FLAG`
+     - obtype_as_group_val_flag
+
 ${METPLUS_ENS_SSVAR_BIN_SIZE}
 """""""""""""""""""""""""""""
 
@@ -1091,6 +1105,18 @@ ${METPLUS_GRID_WEIGHT_FLAG}
    * - :term:`ENSEMBLE_STAT_GRID_WEIGHT_FLAG`
      - grid_weight_flag
 
+${METPLUS_POINT_WEIGHT_FLAG}
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. list-table::
+   :widths: 5 5
+   :header-rows: 1
+
+   * - METplus Config(s)
+     - MET Config File
+   * - :term:`ENSEMBLE_STAT_POINT_WEIGHT_FLAG`
+     - point_weight_flag
+
 ${METPLUS_OUTPUT_PREFIX}
 """"""""""""""""""""""""
 
@@ -7070,6 +7096,7 @@ Configuration
 | :term:`POINT_STAT_HIRA_SHAPE`
 | :term:`POINT_STAT_HIRA_PROB_CAT_THRESH`
 | :term:`POINT_STAT_MESSAGE_TYPE_GROUP_MAP`
+| :term:`POINT_STAT_OBTYPE_AS_GROUP_VAL_FLAG`
 | :term:`FCST_POINT_STAT_IS_PROB`
 | :term:`FCST_POINT_STAT_PROB_IN_GRIB_PDS`
 | :term:`FCST_POINT_STAT_WINDOW_BEGIN`
@@ -7097,6 +7124,7 @@ Configuration
 | :term:`POINT_STAT_UGRID_MAX_DISTANCE_KM`
 | :term:`POINT_STAT_UGRID_COORDINATES_FILE`
 | :term:`POINT_STAT_UGRID_CONFIG_FILE`
+| :term:`POINT_STAT_POINT_WEIGHT_FLAG`
 |
 
 .. warning:: **DEPRECATED:**
@@ -7472,7 +7500,19 @@ ${METPLUS_MESSAGE_TYPE_GROUP_MAP}
      - MET Config File
    * - :term:`POINT_STAT_MESSAGE_TYPE_GROUP_MAP`
      - message_type_group_map
-
+
+${METPLUS_OBTYPE_AS_GROUP_VAL_FLAG}
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. list-table::
+   :widths: 5 5
+   :header-rows: 1
+
+   * - METplus Config(s)
+     - MET Config File
+   * - :term:`POINT_STAT_OBTYPE_AS_GROUP_VAL_FLAG`
+     - obtype_as_group_val_flag
+
 ${METPLUS_CLIMO_MEAN_DICT}
 """"""""""""""""""""""""""
 
@@ -7787,6 +7827,18 @@ ${METPLUS_OUTPUT_PREFIX}
    * - :term:`POINT_STAT_OUTPUT_PREFIX`
      - output_prefix
 
+${METPLUS_POINT_WEIGHT_FLAG}
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. list-table::
+   :widths: 5 5
+   :header-rows: 1
+
+   * - METplus Config(s)
+     - MET Config File
+   * - :term:`POINT_STAT_POINT_WEIGHT_FLAG`
+     - point_weight_flag
+
 ${METPLUS_MET_CONFIG_OVERRIDES}
 """""""""""""""""""""""""""""""
 

docs/use_cases/use_case_documentation_template.py

@@ -0,0 +1,213 @@
+“””
+    PointStat: Use Python embedding to calculate temperature terciles
+    =================================================================
+
+    model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds.conf
+
+“””
+##############################################################################
+# .. contents::
+#   :depth: 1
+#   :local:
+#   :backlinks: none
+
+##############################################################################
+# Scientific Objective
+# --------------------
+# [UPDATE_SECTION_CONTENT]
+#
+# To provide statistical information on the forecast hail size compared to
+# the observed hail size from MRMS MESH data. Using objects to verify hail size
+# avoids the “unfair penalty” issue, where a CAM must first generate convection
+# to have any chance of accurately predicting the hail size. In addition, studies
+# have shown that MRMS MESH observed hail sizes do not correlate one-to-one with
+# observed sizes but can only be used to group storms into general categories.
+# Running MODE allows a user to do this.
+
+##############################################################################
+# Version Added
+# -------------
+# [UPDATE_SECTION_CONTENT]
+#
+# METplus version 6.0
+
+##############################################################################
+# Datasets
+# --------
+# [UPDATE_SECTION_CONTENT]
+# **Forecast:** Global Forecast System (GFS) 25km resolution, 2m temperature
+#
+# **Observation:** ECMWF Reanalysis v5 (ERA5) 5 degree resolution, 2m temperature
+#
+# **Climatology:** None
+#
+# **Location:** All of the input data required for this use case can be 
+# found in a sample data tarball. Each use case category will have 
+# one or more sample data tarballs. It is only necessary to download 
+# the tarball with the use case’s dataset and not the entire collection 
+# of sample data. Click here to access the METplus releases page and download sample data 
+# for the appropriate release: https://github.com/dtcenter/METplus/releases
+# This tarball should be unpacked into the directory that you will 
+# set the value of INPUT_BASE. See :ref:`running-metplus` section for more information.
+
+##############################################################################
+# METplus Components
+# ------------------
+# [UPDATE_SECTION_CONTENT]
+#
+# The only tool this use case calls is GridStat. Within GridStat a Python 
+# script is used for ingesting forecast data, once for each year of data of 
+# the CFSv2 ensemble.
+
+##############################################################################
+# METplus Workflow
+# ----------------
+# [UPDATE_SECTION_CONTENT]
+#
+# **Beginning time (INIT_BEG):** 1982-01-01
+# **End time (INIT_END):** 2010-01-02
+# **Increment between beginning and end times (INIT_INCREMENT):** 1 year
+# **Sequence of forecast leads to process (LEAD_SEQ):** None
+#
+# With an increment of 1 year, all January 1st’s from 1982 to 2010 are processed 
+# for a total of 29 years, with 24 members in each ensemble forecast. This use case 
+# initially runs SeriesAnalysis 24 times, once for each member of the CFSv2 ensemble 
+# across the 29 years of data. The resulting 24 outputs are read in by GenEnsProd 
+# which uses the normalize option to normalize each of the ensemble members 
+# relative to its climatology (FBAR) and standard deviation (FSTDEV). The output from 
+# GenEnsProd are 29 files containing the uncalibrated probability forecasts for 
+# the lower tercile of January for each year. The final probability verification 
+# is done across the temporal scale in SeriesAnalysis, and the spatial scale in GridStat.
+
+##############################################################################
+# METplus Configuration
+# ---------------------
+# [UPDATE_SECTION_CONTENT]
+#
+# METplus first loads all of the configuration files found in parm/metplus_config, 
+# then it loads any configuration files passed to METplus via the command line, 
+# i.e. parm/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf
+#
+# .. highlight:: bash
+# .. literalinclude:: ../../../../parm/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf
+#
+
+##############################################################################
+# MET Configuration
+# -----------------
+# [UPDATE_SECTION_CONTENT]
+#
+# METplus sets environment variables based on user settings in the METplus
+# configuration file. See :ref:`How METplus controls MET config file settings<metplus-control-met>` for more details.
+#
+# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!**
+#
+# If there is a setting in the MET configuration file that is currently
+# not supported by METplus you’d like to control, please refer to:
+# :ref:`Overriding Unsupported MET config file settings<met-config-overrides>`
+#
+# .. dropdown:: GridStatConfig_wrapped
+#
+#   .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped
+
+##############################################################################
+# Python Embedding
+# ----------------
+# [UPDATE_SECTION_CONTENT]
+#
+# This use case calls the read_ASCAT_data.py script to read and pass to PointStat 
+# the user-requested variable. The script needs 5 inputs in the following order: 
+# a path to a directory that contains only ASCAT data of the “ascat_YYYYMMDDHHMMSS_*” 
+# string, a start time in YYYYMMDDHHMMSS, an end time in the same format, 
+# a message type to code the variables as, and a variable name to read in. 
+# Currently the script puts the same station ID to each observation, but there is 
+# space in the code describing an alternate method that may be improved upon to 
+# allow different satellites to have their own station IDs. 
+# This code currently ingests all files it finds in the directory, pulls out the 
+# requested variable, and arranges the data in a list of lists following the 
+# 11-column format for point data. This list of lists is passed back 
+# to PointStat for evaluation and the requested statistical output. The location 
+# of the code is 
+# parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds/read_ASCAT_data.py
+#
+# For more information on the basic requirements to utilize Python Embedding in METplus, 
+# please refer to the MET User’s Guide section on `Python embedding <https://met.readthedocs.io/en/latest/Users_Guide/appendixF.html#appendix-f-python-embedding>`_ 
+#
+# .. highlight:: python
+# .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds/read_ASCAT_data.py
+
+##############################################################################
+# User Scripting
+# --------------
+# [UPDATE_SECTION_CONTENT]
+#
+# This use case uses a Python script to perform plotting, which at the time of 
+# this use case creation was not an ability METplus had. Additionally some of 
+# the plotting features used in this script are not currently slated for METplus 
+# analysis suite development.
+# In order to create the plots, the script reads in a yaml file and sets up 
+# the correct environment. Plot parameters (which are hard coded in the script) are set, 
+# and the datasets are read in from the input file. The desired variable fields 
+# are placed into arrays, which are then treated for bad data and squeezed to the 
+# appropriate dimensions. Additional basic math is completed on the resulting arrays 
+# to create the cross spectra values with the results being graphed.
+#
+# .. highlight:: python
+# .. literalinclude:: ../../../../parm/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra/cross_spectra_plot.py
+
+##############################################################################
+# Running METplus
+# ---------------
+# [UPDATE_SECTION_CONTENT]
+#
+# Pass the use case configuration file to the run_metplus.py script along 
+# with any user-specific system configuration files if desired:
+#
+# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds.conf /path/to/user_system.conf
+#
+# See :ref:`running-metplus` for more information.
+
+##############################################################################
+# Expected Output
+# ---------------
+# [UPDATE_SECTION_CONTENT]
+#
+# A successful run will output the following both to the screen and to the logfile::
+#   INFO: METplus has successfully finished running.
+#
+# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. 
+# Output for this use case will be found in 
+# {OUPUT_BASE}/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds 
+# and will contain the following files::
+#
+#  * grid_stat_198201_000000L_19700101_000000V_pairs.nc
+#  * grid_stat_198201_000000L_19700101_000000V_pstd.txt
+#  * grid_stat_198201_000000L_19700101_000000V.stat
+#
+# Each file should contain corresponding statistics for the line type(s) requested.
+# For the netCDF file, five variable fields are present (not including the lat/lon fields). 
+# Those variables are::
+#
+#  * FCST_fcst_ENS_FREQ_lt-0.43_0_0_all_all_FULL(lat, lon)
+#  * OBS_tmp2m_20100101_000000_all_all_FULL(lat, lon)
+#  * CLIMO_MEAN_tmp2m_20100101_000000_all_all_FULL(lat, lon)
+#  * CLIMO_STDEV_tmp2m_20100101_000000_all_all_FULL(lat, lon)
+#  * CLIMO_CDF_tmp2m_20100101_000000_all_all_FULL(lat, lon)
+
+##############################################################################
+# Keywords
+# --------
+# [UPDATE_SECTION_CONTENT]
+#
+# .. note::
+#
+#   * PointStatToolUseCase
+#   * PythonEmbeddingFileUseCase
+#   * GRIB2FileUseCase
+#   * MarineAndCryosphereAppUseCase
+#
+#   Navigate to the :ref:`quick-search` page to discover other similar use cases.
+#
+#
+#
+# sphinx_gallery_thumbnail_path = ‘_static/short-range-MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.png’

internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py

@@ -829,6 +829,11 @@ def test_ensemble_stat_field_info(metplus_config, config_overrides,
                                            'time_interp_method = NEAREST;'
                                            'match_month = TRUE;day_interval = 30;'
                                            'hour_interval = 12;}')}),
+        ({'ENSEMBLE_STAT_POINT_WEIGHT_FLAG': 'SID', },
+         {'METPLUS_POINT_WEIGHT_FLAG': 'point_weight_flag = SID;'}),
+        ({'ENSEMBLE_STAT_OBTYPE_AS_GROUP_VAL_FLAG': 'FALSE', },
+         {'METPLUS_OBTYPE_AS_GROUP_VAL_FLAG': 'obtype_as_group_val_flag = FALSE;'}),
+
     ]
 )
 @pytest.mark.wrapper_c