diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md new file mode 100644 index 0000000000..b93c8438f4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -0,0 +1,68 @@ +--- +name: Documentation +about: Update the documentation +title: 'Documentation: ' +labels: 'alert: NEED ACCOUNT KEY, alert: NEED MORE DEFINITION, alert: NEED CYCLE ASSIGNMENT, type: documentation' +assignees: '' + +--- + +*Replace italics below with details for this issue.* + +## Describe the Task ## +*Provide a description of the task here.* + +### Time Estimate ### +*Estimate the amount of work required here.* +*Issues should represent approximately 1 to 3 days of work.* + +### Sub-Issues ### +Consider breaking the task down into sub-issues. +- [ ] *Add a checkbox for each sub-issue here.* + +### Relevant Deadlines ### +*List relevant project deadlines here or state NONE.* + +### Funding Source ### +*Define the source of funding and account keys here or state NONE.* + +## Define the Metadata ## + +### Assignee ### +- [ ] Select appropriate **assignee** for this issue + +### Labels ### +- [ ] Review default **alert** labels +- [ ] Select **component(s)** +- [ ] Select **priority** +- [ ] Select **requestor(s)** + +### Milestone and Projects ### +- [ ] Select **Milestone** as a **METplus-Wrappers-X.Y.Z** version, **Consider for Next Release**, or **Backlog of Development Ideas** +- [ ] For a **METplus-Wrappers-X.Y.Z** version, select the **METplus-Wrappers-X.Y.Z Development** project + +## Define Related Issue(s) ## +Consider the impact to the other METplus components. +- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdataio](https://github.com/dtcenter/METdataio/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) + +## Task Checklist ## +See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors_Guide/github_workflow.html) for details. +- [ ] Complete the issue definition above, including the **Time Estimate** and **Funding Source**. +- [ ] Fork this repository or create a branch of **develop**. +Branch name: `feature__` +- [ ] Complete the development and test your changes. +- [ ] Add/update log messages for easier debugging. +- [ ] Add/update unit tests. +- [ ] Add/update documentation. +- [ ] Add any new Python packages to the [METplus Components Python Requirements](https://metplus.readthedocs.io/en/develop/Users_Guide/appendixA.html#metplus-components-python-packages) table. +- [ ] For any new datasets, an entry to the [METplus Verification Datasets Guide](https://metplus.readthedocs.io/en/latest/Verification_Datasets/index.html). +- [ ] Push local changes to GitHub. +- [ ] Submit a pull request to merge into **develop**. +Pull request: `feature ` +- [ ] Define the pull request metadata, as permissions allow. +Select: **Reviewer(s)** and **Development** issue +Select: **Milestone** as the next official version +Select: **METplus-Wrappers-X.Y.Z Development** project for development toward the next official release +- [ ] Iterate until the reviewer(s) accept and merge your changes. +- [ ] Delete your fork or branch. +- [ ] Close this issue. diff --git a/.github/ISSUE_TEMPLATE/enhancement_request.md b/.github/ISSUE_TEMPLATE/enhancement_request.md index 479825cabc..d90bbf44de 100644 --- a/.github/ISSUE_TEMPLATE/enhancement_request.md +++ b/.github/ISSUE_TEMPLATE/enhancement_request.md @@ -1,7 +1,7 @@ --- name: Enhancement request about: Improve something that it's currently doing -title: '' +title: 'Enhancement: ' labels: 'alert: NEED ACCOUNT KEY, alert: NEED MORE DEFINITION, alert: NEED CYCLE ASSIGNMENT, type: enhancement' assignees: '' diff --git a/.github/jobs/docker_build_metplus_images.sh b/.github/jobs/docker_build_metplus_images.sh new file mode 100755 index 0000000000..231df2a4ab --- /dev/null +++ b/.github/jobs/docker_build_metplus_images.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# assumes SOURCE_BRANCH is set before calling script + +source "${GITHUB_WORKSPACE}"/.github/jobs/bash_functions.sh + +dockerhub_repo=dtcenter/metplus +dockerhub_repo_analysis=dtcenter/metplus-analysis + +# check if tag is official or bugfix release -- no -betaN or -rcN suffix +is_official=1 +if [[ "${SOURCE_BRANCH}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + is_official=0 +fi + +# remove v prefix +metplus_version=${SOURCE_BRANCH:1} + +# Get MET tag and adjust MET Docker repo if develop +met_tag=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o MET -f "{X}.{Y}-latest" --no-get_dev_version) +echo "$met_tag" + +MET_DOCKER_REPO=met +if [ "$met_tag" == "develop" ]; then + MET_DOCKER_REPO=met-dev +fi + +# get METplus Analysis tool versions +METDATAIO_VERSION=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o METdataio) +METCALCPY_VERSION=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o METcalcpy) +METPLOTPY_VERSION=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o METplotpy) + +# Build metplus image +METPLUS_IMAGE_NAME=${dockerhub_repo}:${metplus_version} +if ! time_command docker build -t "$METPLUS_IMAGE_NAME" \ + --build-arg SOURCE_VERSION="$SOURCE_BRANCH" \ + --build-arg MET_TAG="$met_tag" \ + --build-arg MET_DOCKER_REPO="$MET_DOCKER_REPO" \ + -f "${GITHUB_WORKSPACE}"/internal/scripts/docker/Dockerfile \ + "${GITHUB_WORKSPACE}"; then + exit 1 +fi + +# Build metplus-analysis image +METPLUS_A_IMAGE_NAME=${dockerhub_repo_analysis}:${metplus_version} +if ! time_command docker build -t "$METPLUS_A_IMAGE_NAME" \ + --build-arg METPLUS_BASE_TAG="${metplus_version}" \ + --build-arg METDATAIO_VERSION="${METDATAIO_VERSION}" \ + --build-arg METCALCPY_VERSION="${METCALCPY_VERSION}" \ + --build-arg METPLOTPY_VERSION="${METPLOTPY_VERSION}" \ + -f "${GITHUB_WORKSPACE}"/internal/scripts/docker/Dockerfile.metplus-analysis \ + "${GITHUB_WORKSPACE}"; then + exit 1 +fi + +# if official release, create X.Y-latest tag as well +if [ "${is_official}" == 0 ]; then + LATEST_TAG=$(echo "$metplus_version" | cut -f1,2 -d'.')-latest + docker tag "${METPLUS_IMAGE_NAME}" "${dockerhub_repo}:${LATEST_TAG}" + docker tag "${METPLUS_A_IMAGE_NAME}" "${dockerhub_repo_analysis}:${LATEST_TAG}" + echo LATEST_TAG="${LATEST_TAG}" >> "$GITHUB_OUTPUT" +fi diff --git a/.github/jobs/docker_push_metplus_images.sh b/.github/jobs/docker_push_metplus_images.sh new file mode 100755 index 0000000000..1ae5e87378 --- /dev/null +++ b/.github/jobs/docker_push_metplus_images.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# assumes SOURCE_BRANCH is set before calling script +# assumes latest_tag will be set if pushing an official or bugfix release + +source "${GITHUB_WORKSPACE}"/.github/jobs/bash_functions.sh + +# get names of images to push + +dockerhub_repo=dtcenter/metplus +dockerhub_repo_analysis=dtcenter/metplus-analysis + +# remove v prefix +metplus_version=${SOURCE_BRANCH:1} + +METPLUS_IMAGE_NAME=${dockerhub_repo}:${metplus_version} +METPLUS_A_IMAGE_NAME=${dockerhub_repo_analysis}:${metplus_version} + +# skip docker push if credentials are not set +if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PASSWORD+x} ]; then + echo "DockerHub credentials not set. Skipping docker push" + exit 0 +fi + +echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin + +# push images + +if ! time_command docker push "${METPLUS_IMAGE_NAME}"; then + exit 1 +fi + +if ! time_command docker push "${METPLUS_A_IMAGE_NAME}"; then + exit 1 +fi + +# only push X.Y-latest tag if official or bugfix release +# shellcheck disable=SC2154 +if [ "${LATEST_TAG}" != "" ]; then + if ! time_command docker push "${dockerhub_repo}:${LATEST_TAG}"; then + exit 1 + fi + if ! time_command docker push "${dockerhub_repo_analysis}:${LATEST_TAG}"; then + exit 1 + fi +fi diff --git a/.github/jobs/docker_setup.sh b/.github/jobs/docker_setup.sh index 8cab9ad44d..1554a5d97f 100755 --- a/.github/jobs/docker_setup.sh +++ b/.github/jobs/docker_setup.sh @@ -31,7 +31,8 @@ echo "TIMING: docker pull ${DOCKERHUB_TAG} took `printf '%02d' $(($duration / 60 # set DOCKERFILE_PATH that is used by docker hook script get_met_version export DOCKERFILE_PATH=${GITHUB_WORKSPACE}/internal/scripts/docker/Dockerfile -MET_TAG=`${GITHUB_WORKSPACE}/internal/scripts/docker/hooks/get_met_version` +metplus_version=$(head -n 1 "${GITHUB_WORKSPACE}/metplus/VERSION") +MET_TAG=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o MET -f "{X}.{Y}-latest" --no-get_dev_version) MET_DOCKER_REPO=met-dev if [ "${MET_TAG}" != "develop" ]; then diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 7212019889..1c69828aa4 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -17,6 +17,7 @@ from metplus.util.string_manip import expand_int_string_to_list from docker_utils import VERSION_EXT from metplus import get_metplus_version +from metplus.component_versions import get_component_version # path to METplus install location in Docker METPLUS_DOCKER_LOC = '/metplus/METplus' @@ -39,7 +40,7 @@ ] -def handle_automation_env(host_name, reqs, work_dir): +def handle_automation_env(host_name, reqs): # if no env is specified, use metplus base environment conda_env = METPLUS_BASE_ENV @@ -60,8 +61,9 @@ def handle_automation_env(host_name, reqs, work_dir): conda_env_w_ext = f'{conda_env}{VERSION_EXT}' # start building commands to run before run_metplus.py in Docker - setup_env = [] - setup_env.append(_add_to_bashrc('# BELOW WAS ADDED BY TEST SCRIPT')) + setup_env = [ + _add_to_bashrc('# BELOW WAS ADDED BY TEST SCRIPT') + ] # add conda bin to beginning of PATH python_dir = os.path.join('/usr', 'local', 'conda', 'envs', @@ -76,35 +78,34 @@ def handle_automation_env(host_name, reqs, work_dir): else: py_embed_arg = '' - # get METplus version to determine Externals file to use - # to get METplotpy/METcalcpy/METdataio + # get METplus version to determine which version of + # METplotpy/METcalcpy/METdataio to use # If stable release, get main branch, otherwise get develop - is_stable_release = len(get_metplus_version().split('-')) == 1 - externals_ext = '_stable.cfg' if is_stable_release else '.cfg' + metplus_version = get_metplus_version() # if any metplotpy/metcalcpy keywords are in requirements list, # add command to obtain and install METplotpy and METcalcpy + components = [] if any([item for item in PLOTCALC_KEYWORDS if item in str(reqs).lower()]): - ce_file = os.path.join(work_dir, '.github', 'parm', - f'Externals_metplotcalcpy{externals_ext}') - setup_env.extend(( - f'cd {METPLUS_DOCKER_LOC}', - f'{work_dir}/manage_externals/checkout_externals -e {ce_file}', - f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METplotpy', - f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METcalcpy', - 'cd -', - )) + components.extend(('METplotpy', 'METcalcpy')) # if metdataio is in requirements list, add command to obtain METdataio if 'metdataio' in str(reqs).lower(): - ce_file = os.path.join(work_dir, '.github', 'parm', - f'Externals_metdataio{externals_ext}') + components.append('METdataio') + + setup_env.append(f'cd {METPLUS_DOCKER_LOC}/..') + for component in components: + version = get_component_version(input_component='METplus', + input_version=metplus_version, + output_component=component, + output_format='main_v{X}.{Y}', + get_dev=False) setup_env.extend(( - f'cd {METPLUS_DOCKER_LOC}', - f'{work_dir}/manage_externals/checkout_externals -e {ce_file}', - f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METdataio', - 'cd -', + 'git --version', + f'git clone --single-branch --branch {version} https://github.com/dtcenter/{component}', + f'{python_path} -m pip install --no-deps {METPLUS_DOCKER_LOC}/../{component}', )) + setup_env.append('cd -') # if metplus is in requirements list, # add top of METplus repo to PYTHONPATH so metplus can be imported @@ -118,7 +119,7 @@ def handle_automation_env(host_name, reqs, work_dir): setup_env.extend(( f'echo Using environment: dtcenter/metplus-envs:{conda_env_w_ext}', f'echo cat /usr/local/conda/envs/{conda_env_w_ext}/environments.yml', - f'echo ----------------------------------------', + 'echo ----------------------------------------', f'cat /usr/local/conda/envs/{conda_env_w_ext}/environments.yml', 'echo ----------------------------------------', )) @@ -140,6 +141,24 @@ def main(categories, subset_list, work_dir=None, test_suite = METplusUseCaseSuite() test_suite.add_use_case_groups(categories, subset_list) + for group_name, use_cases_by_req in test_suite.category_groups.items(): + for use_case_by_requirement in use_cases_by_req: + reqs = use_case_by_requirement.requirements + + setup_env, py_embed_arg = handle_automation_env(host_name, reqs) + + use_case_cmds = _get_use_case_cmds(host_name, use_case_by_requirement, work_dir, group_name, py_embed_arg) + + # add commands to set up environment before use case commands + all_commands.append((setup_env, use_case_cmds, reqs)) + + return all_commands + + +def _get_use_case_cmds(host_name, use_case_by_requirement, work_dir, group_name, py_embed_arg): + # use status variable to track if any use cases failed + use_case_cmds = [] + output_top_dir = os.environ.get('METPLUS_TEST_OUTPUT_BASE', '/data/output') # use METPLUS_TEST_SETTINGS_CONF if set @@ -150,58 +169,53 @@ def main(categories, subset_list, work_dir=None, 'parm', 'test_settings.conf') - for group_name, use_cases_by_req in test_suite.category_groups.items(): - for use_case_by_requirement in use_cases_by_req: - reqs = use_case_by_requirement.requirements + if host_name != 'docker': + use_case_cmds.append('status=0') + for use_case in use_case_by_requirement.use_cases: + # add parm/use_cases path to config args if they are conf files + config_args = _get_config_args(use_case.config_args, work_dir, host_name) + + output_base = os.path.join(output_top_dir, + group_name.split('-')[0], + use_case.name) + use_case_cmd = (f"run_metplus.py" + f" {' '.join(config_args)}" + f" {py_embed_arg}{test_settings_conf}" + f" config.OUTPUT_BASE={output_base}") + use_case_cmds.append(use_case_cmd) + # check exit code from use case command and + # set status to non-zero value on error + if host_name != 'docker': + use_case_cmds.append("if [ $? != 0 ]; then status=1; fi") + + # if any use cases failed, force non-zero exit code with false + if host_name != 'docker': + use_case_cmds.append("if [ $status != 0 ]; then false; fi") - setup_env, py_embed_arg = handle_automation_env(host_name, reqs, - work_dir) - - # use status variable to track if any use cases failed - use_case_cmds = [] - if host_name != 'docker': - use_case_cmds.append('status=0') - for use_case in use_case_by_requirement.use_cases: - # add parm/use_cases path to config args if they are conf files - config_args = [] - ci_overrides = None - for config_arg in use_case.config_args: - if config_arg.endswith('.conf'): - config_arg = os.path.join(work_dir, 'parm', - 'use_cases', config_arg) - - # look for CI overrides conf file - override_path = os.path.join(config_arg[0:-5], - 'ci_overrides.conf') - if os.path.exists(override_path): - ci_overrides = override_path - - config_args.append(config_arg) - - # add CI overrides config file if running in docker - if ci_overrides and host_name == 'docker': - config_args.append(ci_overrides) - - output_base = os.path.join(output_top_dir, - group_name.split('-')[0], - use_case.name) - use_case_cmd = (f"run_metplus.py" - f" {' '.join(config_args)}" - f" {py_embed_arg}{test_settings_conf}" - f" config.OUTPUT_BASE={output_base}") - use_case_cmds.append(use_case_cmd) - # check exit code from use case command and - # set status to non-zero value on error - if host_name != 'docker': - use_case_cmds.append("if [ $? != 0 ]; then status=1; fi") - - # if any use cases failed, force non-zero exit code with false - if host_name != 'docker': - use_case_cmds.append("if [ $status != 0 ]; then false; fi") - # add commands to set up environment before use case commands - all_commands.append((setup_env, use_case_cmds, reqs)) + return use_case_cmds - return all_commands + +def _get_config_args(input_config_args, work_dir, host_name): + config_args = [] + ci_overrides = None + for config_arg in input_config_args: + if config_arg.endswith('.conf'): + config_arg = os.path.join(work_dir, 'parm', + 'use_cases', config_arg) + + # look for CI overrides conf file + override_path = os.path.join(config_arg[0:-5], + 'ci_overrides.conf') + if os.path.exists(override_path): + ci_overrides = override_path + + config_args.append(config_arg) + + # add CI overrides config file if running in docker + if ci_overrides and host_name == 'docker': + config_args.append(ci_overrides) + + return config_args def handle_command_line_args(): @@ -232,9 +246,9 @@ def handle_command_line_args(): if __name__ == '__main__': - categories, subset_list, _ = handle_command_line_args() - all_commands = main(categories, subset_list) - for setup_commands, use_case_commands, requirements in all_commands: + input_categories, input_subset_list, _ = handle_command_line_args() + commands = main(input_categories, input_subset_list) + for setup_commands, use_case_commands, requirements in commands: print(f"REQUIREMENTS: {','.join(requirements)}") if setup_commands: command_format = ';\\\n'.join(setup_commands.split(';')) diff --git a/.github/labels/README.md b/.github/labels/README.md index 927671a2bd..634256fca1 100644 --- a/.github/labels/README.md +++ b/.github/labels/README.md @@ -1,54 +1 @@ -METplus Repository Labels -========================= - -The contents of this directory define GitHub issue labels that are common -to all of the METplus-related repositories in the DTCenter organization. - -These labels should be defined consistently across all of the repositories. -However, individual repositories are encouraged to add labels specific to -their codebase. - -Labels are defined for the following categories: -- "alert" flags issues for discussion -- "component" classifies the type of work (customization encouraged) -- "priority" defines the priority level -- "reporting" defines project management reporting requirements -- "requestor" identifies the relevant organization(s) -- "type" corresponds to GitHub issue templates (customization allowed) - -In general, new "alert", "priority", "reporting", and "requestor" labels -should be added to this common location and defined consistently across -all the METplus repositories. - -Each repository should define a relevant set of "component" labels since -they will vary widely depending on the code base. - -New "type" labels should only be added to a repository when a corresponding -GitHub issue template is also added. - -To add a new common label, manually update the common_labels.txt file. - -Sample commands for processing all METplus repos: - -``` -# List of METplus repositories -REPO_LIST="metplus met metplotpy metcalcpy metdataio metviewer \ - metexpress metplus-training metplus-internal \ - metbaseimage"; - -# Build commands to add/update common labels -for repo in ${REPO_LIST}; do - echo $repo; - ./post_patch_labels.sh [user] [auth] $repo common_labels.txt; -done - -# Build commands to delete extra labels -for repo in ${REPO_LIST}; do - echo $repo; - ./delete_labels.sh [user] [auth] $repo; -done -``` - -The resulting commands are written to bash shell scripts in the commands -directory. Those commands should be carefully reviewed before executing them. - +Please see the [GitHub Labels to Organize Issues](https://metplus.readthedocs.io/en/develop/Contributors_Guide/github_workflow.html#github-labels-to-organize-issues) section of the METplus Contributor's Guide. diff --git a/.github/labels/common_labels.txt b/.github/labels/common_labels.txt index 4ea4bd760b..4ed7ffa623 100644 --- a/.github/labels/common_labels.txt +++ b/.github/labels/common_labels.txt @@ -24,6 +24,7 @@ {"name": "reporting: DTC NOAA R2O","color": "fbca04","description": "NOAA Research to Operations DTC Project"} {"name": "reporting: HSUP 1-3a","color": "fbca04","description": "Hurricane Supplemental Project 1-3a"} {"name": "reporting: HSUP 4-1b","color": "fbca04","description": "Hurricane Supplemental Project 4-1b"} +{"name": "reporting: NRL METplus","color": "fbca04","description": "Naval Research Laboratory METplus Project"} {"name": "requestor: Army/ARL","color": "3101c1","description": "United States Army Research Laboratory"} {"name": "requestor: Australian BOM","color": "3101c1","description": "Australian Bureau of Meteorology"} {"name": "requestor: Community","color": "3101c1","description": "General Community"} diff --git a/.github/labels/delete_labels.sh b/.github/labels/delete_labels.sh index 4ff004ebe8..479f28d3b6 100755 --- a/.github/labels/delete_labels.sh +++ b/.github/labels/delete_labels.sh @@ -6,8 +6,7 @@ # - repository name if [[ $# -ne 3 ]]; then - echo "ERROR: `basename $0` ... must specify the GitHub username, authorization key, and repository name." - echo "ERROR: repo list: metplus, met, metdataio, metcalcpy, metplotpy, metviewer, metexpress, metplus-training" + echo "ERROR: `basename $0` ... must specify the GitHub username, authorization key, and METplus repository name." exit 1 else user=$1 diff --git a/.github/labels/get_labels.sh b/.github/labels/get_labels.sh index b6bd6b0c5d..a4fb0844d3 100755 --- a/.github/labels/get_labels.sh +++ b/.github/labels/get_labels.sh @@ -6,8 +6,7 @@ # - repository name if [[ $# -ne 3 ]]; then - echo "ERROR: `basename $0` ... must specify the GitHub username, authorization key, and repository name." - echo "ERROR: repo names: metplus, met, metdataio, metcalcpy, metplotpy, metviewer, metexpress, metplus-training" + echo "ERROR: `basename $0` ... must specify the GitHub username, authorization key, and METplus repository name." exit 1 else user=$1 diff --git a/.github/labels/post_patch_labels.sh b/.github/labels/post_patch_labels.sh index 5c3cc53441..5d5ee3a36d 100755 --- a/.github/labels/post_patch_labels.sh +++ b/.github/labels/post_patch_labels.sh @@ -7,8 +7,7 @@ # - label file if [[ $# -ne 4 ]]; then - echo "ERROR: `basename $0` ... must specify the GitHub username, authorization key, repository name, and label file." - echo "ERROR: repo names: metplus, met, metdataio, metcalcpy, metplotpy, metviewer, metexpress, metplus-training, metplus-internal, metbaseimage" + echo "ERROR: `basename $0` ... must specify the GitHub username, authorization key, METplus repository name, and label file." exit 1 else user=$1 diff --git a/.github/labels/process_labels.sh b/.github/labels/process_labels.sh index a23bf7c90c..a75b8675a4 100755 --- a/.github/labels/process_labels.sh +++ b/.github/labels/process_labels.sh @@ -17,7 +17,7 @@ SCRIPT_DIR=`dirname $0` # List of METplus repositories REPO_LIST="metplus met metplotpy metcalcpy metdataio metviewer \ - metexpress metplus-training"; + metexpress metbaseimage metplus-internal metplus-training"; # Process each repository for REPO in ${REPO_LIST}; do diff --git a/.github/parm/Externals_metdataio.cfg b/.github/parm/Externals_metdataio.cfg deleted file mode 100644 index b7f7544f81..0000000000 --- a/.github/parm/Externals_metdataio.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[METdataio] -local_path = ../METdataio -protocol = git -required = True -repo_url = https://github.com/dtcenter/METdataio -branch = develop - -[externals_description] -schema_version = 1.0.0 diff --git a/.github/parm/Externals_metdataio_stable.cfg b/.github/parm/Externals_metdataio_stable.cfg deleted file mode 100644 index af45402334..0000000000 --- a/.github/parm/Externals_metdataio_stable.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[METdataio] -local_path = ../METdataio -protocol = git -required = True -repo_url = https://github.com/dtcenter/METdataio -branch = main_v2.1 - -[externals_description] -schema_version = 1.0.0 diff --git a/.github/parm/Externals_metplotcalcpy.cfg b/.github/parm/Externals_metplotcalcpy.cfg deleted file mode 100644 index fdf1383555..0000000000 --- a/.github/parm/Externals_metplotcalcpy.cfg +++ /dev/null @@ -1,16 +0,0 @@ -[METcalcpy] -local_path = ../METcalcpy -protocol = git -required = True -repo_url = https://github.com/dtcenter/METcalcpy -branch = develop - -[METplotpy] -local_path = ../METplotpy -protocol = git -required = True -repo_url = https://github.com/dtcenter/METplotpy -branch = develop - -[externals_description] -schema_version = 1.0.0 diff --git a/.github/parm/Externals_metplotcalcpy_stable.cfg b/.github/parm/Externals_metplotcalcpy_stable.cfg deleted file mode 100644 index 5aed1cb69b..0000000000 --- a/.github/parm/Externals_metplotcalcpy_stable.cfg +++ /dev/null @@ -1,16 +0,0 @@ -[METcalcpy] -local_path = ../METcalcpy -protocol = git -required = True -repo_url = https://github.com/dtcenter/METcalcpy -branch = main_v2.1 - -[METplotpy] -local_path = ../METplotpy -protocol = git -required = True -repo_url = https://github.com/dtcenter/METplotpy -branch = main_v2.1 - -[externals_description] -schema_version = 1.0.0 diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 8d12b81db2..7eefb5125a 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -59,10 +59,14 @@ "index_list": "0-1", "run": false }, + { + "category": "fire", + "index_list": "0", + "run": false + }, { "category": "land_surface", "index_list": "0", - "disabled": true, "run": false }, { @@ -112,7 +116,7 @@ }, { "category": "medium_range", - "index_list": "3-5,10", + "index_list": "3-5,10", "run": false }, { diff --git a/.github/update_truth_change_log.txt b/.github/update_truth_change_log.txt index 731c759593..186f170d10 100644 --- a/.github/update_truth_change_log.txt +++ b/.github/update_truth_change_log.txt @@ -10,3 +10,4 @@ [20241014_19:42:56 develop] dtcenter/MET#2988 - dtcenter/MET#2988 -- see issue #2719 for details -- retry because there was an issue with convert not being available in the ubuntu image that runs the unit tests [20241015_17:11:29 develop] dtcenter/MET#2988 - dtcenter/MET#2988 -- see issue #2719 for details -- retry (again) because there was an issue with convert not being available in the ubuntu image that runs the unit tests and METdbLoad use cases were broken [20241016_20:39:30 develop] #2710 - #2710 fixed a bug in a use case +[20241113_21:34:39 develop] #2769 and #2785 - #2769 adds new use case for fire weather, #2785 turns on land_surface use case that was previously reporting differences that was resolved by bugfix dtcenter/MET#2899 diff --git a/.github/workflows/release-docker-images.yml b/.github/workflows/release-docker-images.yml new file mode 100644 index 0000000000..1ebe6ff2c3 --- /dev/null +++ b/.github/workflows/release-docker-images.yml @@ -0,0 +1,34 @@ +name: Create Docker images for release + +on: + release: + types: + - published + +jobs: + build_and_push: + name: Build and Push Images + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Get and check tag name + id: get_tag_name + run: | + SOURCE_BRANCH=${GITHUB_REF#refs/tags/} + if [[ ! "${SOURCE_BRANCH}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then + echo "ERROR: Tag name (${SOURCE_BRANCH}) does not start with vX.Y.Z format" + exit 1 + fi + echo SOURCE_BRANCH=${SOURCE_BRANCH} >> $GITHUB_OUTPUT + - name: Build metplus and metplus-analysis images + id: build_images + run: .github/jobs/docker_build_metplus_images.sh + env: + SOURCE_BRANCH: ${{ steps.get_tag_name.outputs.SOURCE_BRANCH }} + - name: Push metplus and metplus-analysis images + run: .github/jobs/docker_push_metplus_images.sh + env: + SOURCE_BRANCH: ${{ steps.get_tag_name.outputs.SOURCE_BRANCH }} + LATEST_TAG: ${{ steps.build_images.outputs.LATEST_TAG }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} diff --git a/.github/workflows/update_input_data.yml b/.github/workflows/update_input_data.yml index 2f774c68f8..5235a03511 100644 --- a/.github/workflows/update_input_data.yml +++ b/.github/workflows/update_input_data.yml @@ -13,7 +13,7 @@ jobs: - run: | branch_name=$(echo ${{ github.event.inputs.branch || github.ref }} | cut -d/ -f3) echo "branch_name=$branch_name" >> "$GITHUB_ENV" - - uses: dtcenter/metplus-action-data-update@v2 + - uses: dtcenter/metplus-action-data-update@v3 with: docker_name: ${{ secrets.DOCKER_USERNAME }} docker_pass: ${{ secrets.DOCKER_PASSWORD }} diff --git a/docs/Contributors_Guide/add_use_case.rst b/docs/Contributors_Guide/add_use_case.rst index 5f61469bc1..4521669d96 100644 --- a/docs/Contributors_Guide/add_use_case.rst +++ b/docs/Contributors_Guide/add_use_case.rst @@ -48,6 +48,7 @@ one of the following: * climate * clouds * data_assimilation +* fire * extremes * land_surface * marine_and_cryosphere @@ -374,12 +375,13 @@ the text content. These are already provided in the example template. as well as an embedded link (and image) of all Python scripts used in Python Embedding. The links to these scripts will need to be updated by the user. - If no Python embedding is used, that should be noted here. + * If no Python embedding is used, that should be noted here by stating + "This use case does not use Python embedding." * User Scripting * This section should provide a description of any Python scripting that’s used - in the use case. For a common definition, User scripting is any condition where + in the use case. For a common definition, User Scripting is any condition where the evaluation/verification/output processes are being completed inside the Python script, outside of METplus. Essentially, if a Python script is called and a METplus-readable dataset is not passed back to METplus, it is User Scripting. @@ -388,7 +390,8 @@ the text content. These are already provided in the example template. made to use User scripting rather than Python embedding. The end of this section is an embedded link (and image) of all Python scripts used in User Scripting. The links to these scripts will need to be updated by the user. - If no User scripting is used, that should be noted here. + * If no User Scripting is used, that should be noted here by stating + "This use case does not use additional scripts." * Running METplus diff --git a/docs/Contributors_Guide/coding_standards.rst b/docs/Contributors_Guide/coding_standards.rst index 67cee26da1..bfc099be76 100644 --- a/docs/Contributors_Guide/coding_standards.rst +++ b/docs/Contributors_Guide/coding_standards.rst @@ -4,6 +4,9 @@ Coding Standards **************** +Guidelines +========== + * Python code style outlined in `PEP8 `_ * `Doxygen `_, `Python docstrings `_, and @@ -11,3 +14,20 @@ Coding Standards * **NOTE: Please do not use f-strings in the run_metplus.py file so that the Python version check can notify the user of the incorrect version. Using Python 3.5 or earlier will output the SyntaxError from the f-string instead of the useful error message.** Python code analysis tools like **pylint** can be used to check for errors and violations of PEP8 standards. + +Python Package Disclaimer +========================= + +Any external Python packages used in the METplus components are defined in the +requirements.txt file and, if applicable, nco_requirements.txt file in the top +level of the code repositories. It is important that the METplus components only +require Python packages that are allowed on operational high performance +computers (HPCs). The process of approval for some HPCs can take months and +some packages have been denied. + +Proceed with caution and request approval before starting development with a +new Python package. Some general guidelines to consider when choosing new +potential Python packages are: + + * The package would ideally have a version number of 1 or greater. + * Ensure that the package is regularly being maintained. diff --git a/docs/Contributors_Guide/continuous_integration.rst b/docs/Contributors_Guide/continuous_integration.rst index 976a05ae5f..447d2bd9ee 100644 --- a/docs/Contributors_Guide/continuous_integration.rst +++ b/docs/Contributors_Guide/continuous_integration.rst @@ -36,7 +36,6 @@ Many useful actions are provided by GitHub and external collaborators. Developers can also write their own custom actions to perform complex tasks to simplify a workflow. - Testing (testing.yml) --------------------- @@ -44,6 +43,8 @@ This workflow performs a variety of tasks to ensure that changes do not break any existing functionality. See the :ref:`cg-ci-testing-workflow` for more information. +.. _cg-ci-documentation: + Documentation (documentation.yml) --------------------------------- @@ -76,6 +77,96 @@ at the bottom of the workflow summary page when the workflow has completed. .. figure:: figure/ci-doc-artifacts.png +.. _cg-ci-sonarqube: + +SonarQube (sonarqube.yml) +------------------------- + +SonarQube is a static code analysis tool used to enhance software code quality +and security. The METplus team acknowledges the generous funding from the +United States Air Force to support our ongoing use of this important utility. + +The **sonarqube.yml** workflow is defined in all of the METplus component +code repositories. This workflow is triggered by a **pull_request** or +**push** to the **develop** or **main_vX.Y** branch and also by manual +**workflow_dispatch** events. However, changes to documentation only or other +specific infrastructure directories do not trigger this workflow. + +A **sonar-project.properties** file within each repository defines the +configuration of the SonarQube scans for that code base. The SonarQube +workflows for the Python-based METplus components are all very similar while +the logic for the repositories with compiled code differ. + +The SonarQube workflows for the Python-based components (METplus, METplotpy, +METcalcpy, and METdataio) run jobs to: + +* Check out the source code +* Set up a Python environment +* Run Pytests and create a test code coverage report +* Configure the SonarQube properties based on the triggering event +* Run a SonarQube scan job provided by SonarSource +* Run a SonarQube quality gate check job provided by SonarSource + +The quality gate check job pushes the scan results, including code coverage, +to a [SonarQube server](https://needham.rap.ucar.edu/) hosted by the METplus +team. All memebers of the DTCenter GitHub organization can access this server +by logging in with their GitHub credentials. + +The SonarQube scans for MET and METviewer require that the code be built, +which is done inside a Docker container. However, the results of those +Docker-based scans are also pushed to the same SonarQube server. + +The scan results for each repository are stored on the server in a project +whose name matches the code repository name. + +The SonarQube server defines a configurable set of **Quality Gate** acceptance +criteria. For each scan, a reference source is defined and changes are tracked +relative to that reference. For example, a new scan of the **develop** branch +is compared to the previous scan of **develop**, while each pull request scan +is compared to the latest scan of the destination branch, typically +**develop**. + +SonarQube scans report on the following (listed in approximate order of +concern from a security perspective): + +* **Vulnerabilities** for security findings +* **Bugs** for reliability findings +* **Security Hotspots** for security findings to be reviewed +* **Code Smells** for maintainability findings +* Test code **Coverage** percentage (if provided to the scan) +* Code **Duplication** percentage + +For each finding, the SonarQube scan categorizes it by type, shows its +location in the code, and provides detailed information about the reason +for the issue, suggestions on how to fix it, and links to additional +information. + +SonarQube differentiates between **New Code** and **Overall Code** where the +former shows findings flagged only in new files and lines modified in existing +files and the latter shows all findings. Generally speaking, the configurable +**Quality Gate** settings define acceptance criteria based only on findings +in **New Code**. + +The **Quality Gate** check in the SonarQube workflow returns a good status +(green checkmark) if the acceptance criteria is met or bad status (red X) if +not. Ideally, each change to the code base would result in fewer findings, +increased test coverage, and a lower code duplication. Pull requests should +never add new **Vulnerabilities** or **Bugs**, and the submitter should fix +them before their pull request is approved and merged. Introducing new +**Code Smells** is acceptable in certain circumstances. In this case, +developers are strongly encouraged to make additional changes that reduce the +total number of **Code Smells** in the **Overall Code**. While a pull request +can add new **Code Smells** that are not easily fixed, the overall number +should be reduced. + +Developers are encouaraged to manually run the SonarQube workflow with the +GitHub **workflow_dispatch** option and check the results to confirm the +quality of their code before submitting a pull request for review. Developers +are encouraged to describe the SonarQube status of their proposed code changes +in the body of each pull request. Reviewers should not approve pull requests +that introduce new **Vulnerabilities** or **Bugs** or increase the number of +**Code Smells** in the **Overall Code**. + .. _cg-ci-update-truth-data: Update Truth Data (update_truth.yml) @@ -176,7 +267,6 @@ branch name text box blank and select the branch name from the pull-down menu. Verify that the workflow ran successfully and properly obtained the new data by reviewing the log output from the workflow run. - Release Published (release_published.yml) - DEPRECATED ------------------------------------------------------ diff --git a/docs/Contributors_Guide/github_workflow.rst b/docs/Contributors_Guide/github_workflow.rst index ab0be1b3b0..c27e463704 100644 --- a/docs/Contributors_Guide/github_workflow.rst +++ b/docs/Contributors_Guide/github_workflow.rst @@ -297,6 +297,108 @@ required to perform the following steps. - Click the **Link a project** button and find/select this newly created support project. +.. _wo-github-labels: + +GitHub Labels to Organize Issues +================================ + +The METplus system consists of several software components, each with its own GitHub +repository. Each issue in the METplus component repositories is assigned multiple +labels to keep the work well-organized. Each label begins with a prefix to indicate +its category and, generally, all labels of the same category share a common color. +Over 70 labels common to all METplus component repositories fall into the following +categories: + + + - **alert** labels are colored yellow, are added to each new issue by default, but + are intended to be addressed and removed by the time work for the issue begins. + + - **component** labels are colored blue and roughly categorize the work by topic. + Additional repository-specific **component** labels are encouraged. + + - **priority** labels are colored green, yellow, or red to indicate priority level. + + - **reporting** labels are colored light orange and are generally assigned by + project managers to indicate where progress on this work will be reported. + + - **requestor** labels are colored purple and indicate which organization(s) have + requested the work. + + - **required** labels are colored pink and indicate whether the work is actually + required for the development cycle and/or official release to which is has been + assigned. + + - **type** labels are colored tan and differentiate between, for example, bugfixes, + new features, enhancements, and miscellaneous tasks. Generally, each type label + corresponds to a GitHub issue template. Additional repository-specific **type** + labels are supported. + + +In general, new **alert**, **priority**, **reporting**, and **requestor** labels +should be added to this common location and defined consistently across all +METplus repositories. Each repository should define a relevant set of **component** +labels since they will vary widely depending on the code base. New **type** labels +should only be added to a repository when a corresponding GitHub issue template is +also added to its :code:`.github/ISSUE_TEMPLATE` directory. + + +Scripts to manage these common labels are provided in the :code:`.github/labels` +directory of the METplus repository: + + + - :code:`common_labels.txt` defines the name, color, and description for each of the + labels common to all METplus repositories. + + - :code:`process_labels.sh` has two required arguments: the GitHub user name and + authorization token for command line actions. It loops through the METplus + component repositories and runs the following scripts for each. + + - :code:`post_patch_labels.sh` is called by :code:`process_labels.sh` for each + METplus repository. It retrieves the list of existing labels for the specified + repository, compares it to :code:`common_labels.txt` and generates commands to + *add* new labels (i.e. :code:`curl POST`) or *update* existing labels (i.e. + :code:`curl PATCH`). For example, if the label description is modified, the update + command will make that change. + + - :code:`delete_labels.sh` is called by :code:`process_labels.sh` for each + METplus repository. It retrieves the list of existing labels for the specified + repository, compares it to :code:`common_labels.txt`, and generates commands + to *delete* (i.e. :code:`curl DELETE`) any labels that should be removed. + + - :code:`get_labels.sh` retrieves the current set of the labels from the + specified METplus repository. + + +Note that the scripts **DO NOT** actually add, modify, or delete any labels +directly. Instead, they generate the *curl* commands for those actions and +write them to shell scripts in the **commands** sub-directory. After generating +these commands, the user should carefully review them to decide before executing +them: + + + - :code:`commands/post_labels_*_cmd.sh` scripts contain :code:`curl POST` commands + for adding new common labels. Run these after adding a new common label. + + - :code:`commands/delete_labels_*_cmd.sh` scripts contain :code:`curl DELETE` + commands to remove labels which should not exist. Run these after deleting + existing common labels. Proceed with caution since deletions cannot be undone, + and all occurrences of these labels will be removed. + + - :code:`commands/patch_labels_*_cmd.sh` scripts contain :code:`curl PATCH` commands for + updating all existing labels. Run these after modifying the description or color + of existing common labels. + + +The process of adding a new label, for example, consists of the following steps: + + 1. Add a new line in :code:`common_labels.txt` to define it. + + 2. Run :code:`process_labels.sh` to generate commands, supplying your GitHub user name + and access token as arguments. + + 3. Review the actions in the :code:`commands/post_labels_*_cmd.sh` scripts and run them. + + Sequence of Events - Contributing Code ====================================== diff --git a/docs/Contributors_Guide/index.rst b/docs/Contributors_Guide/index.rst index 2dabc2a162..fb321c1f78 100644 --- a/docs/Contributors_Guide/index.rst +++ b/docs/Contributors_Guide/index.rst @@ -7,6 +7,7 @@ Contributor's Guide :numbered: coding_standards + utilities basic_components create_wrapper conda_env diff --git a/docs/Contributors_Guide/utilities.rst b/docs/Contributors_Guide/utilities.rst new file mode 100644 index 0000000000..9d576142a7 --- /dev/null +++ b/docs/Contributors_Guide/utilities.rst @@ -0,0 +1,222 @@ +.. _cg_util: + +********* +Utilities +********* + +.. _cg_util_version: + +Component Versions Script +========================= + +**metplus/component_versions.py** + +This script is used to query a METplus coordinated release component version +number lookup table to determine a corresponding version number for another +METplus component. + +This script can be called directly from a script or the command line. +It returns the version of the requested (output) METplus component. +This functionality can also be imported in a Python script. +See below for examples. + +Usage Statement +--------------- +:: + + usage: component_versions.py [-h] [-i INPUT_COMPONENT] [-v INPUT_VERSION] -o + OUTPUT_COMPONENT [-f OUTPUT_FORMAT] + [--get_dev_version | --no-get_dev_version] + + options: + -h, --help show this help message and exit + -i INPUT_COMPONENT, --input_component INPUT_COMPONENT + Name of METplus component to use to find version. Default + is METplus. + -v INPUT_VERSION, --input_version INPUT_VERSION + version of input_component to search. Default is latest + official release + -o OUTPUT_COMPONENT, --output_component OUTPUT_COMPONENT + name of METplus component to obtain version + -f OUTPUT_FORMAT, --output_format OUTPUT_FORMAT + format to use to output version number.{X}, {Y}, and {Z} + will be replaced with x, y, and z version numbers from + X.Y.Z. {N} will be replaced with development version if + found in the input version, e.g. "-beta3" or "-rc1" + Default is v{X}.{Y}.{Z}{N} + --get_dev_version, --no-get_dev_version + If True, get corresponding -beta or -rc version. If + False, return develop if development version. (default: + True) + +Examples +-------- + +These examples do not include the full path to the script. + +Get MET vX.Y.Z version from METplus release +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + $ component_versions.py -v 5.1.0 -o MET + v11.1.1 + +If the input component is not specified, it will use the METplus version. +The default output format is v{X}.{Y}.{Z}{N}. + +Get MET vX.Y.Z version from coordinated release +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + $ component_versions.py -v 5.1 -o MET + v11.1.1 + +The coordinated release version matches the METplus X.Y version, +so the coordinated release version can also be used as the input version. + +Get MET vX.Y.Z development version from beta release +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + $ component_versions.py -v 6.0.0-beta3 -o MET + v12.0.0-beta3 + +If a beta release version is provided as the input, the output will include +the same beta version. + +Get MET vX.Y.Z development version from -dev version +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + $ component_versions.py -v 6.0.0-beta3-dev -o MET + develop + +If the input version includes -dev, the result will always be *develop*. + +Get MET vX.Y.Z development version from beta version +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + $ component_versions.py -v 6.0.0-beta3 -o MET --no-get_dev_version + develop + +If the *--no-get_dev_version* argument is provided, +an input version that includes -betaN or -rcN will return *develop*. + +Get MET main_vX.Y version from METplus release +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + $ component_versions.py -v 5.1.0 -o MET -f main_v{X}.{Y} + main_v11.1 + +The output format can be specified using the *-f* argument. +{X}, {Y}, {Z}, and {N} will be substituted with values based on the input. + +Get METplotpy main_vX.Y version from METviewer release +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + $ component_versions.py -i METviewer -v 5.1.0 -o METplotpy -f main_v{X}.{Y} + main_v2.1 + +The *-i* argument can be used to specify the input component that corresponds +to the input version number. + +Get METplotpy main_vX.Y version from METviewer main_vX.Y branch +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + $ component_versions.py -i METviewer -v main_v5.1 -o METplotpy -f main_v{X}.{Y} + main_v2.1 + +The input version number can be provided in different formats, +including **main_vX.Y** and **vX.Y.Z**. + +Using Python Function +^^^^^^^^^^^^^^^^^^^^^ +:: + + >>> from metplus.component_versions import get_component_version + >>> version = get_component_version(input_component='METplus', + input_version='6.0.0', + output_component='MET', + output_format='main_v{X}.{Y}', + get_dev=False) + >>> print(version) + main_v12.0 + + +METplus Utils +============= + +These files are found under **metplus/util**. + +Utility scripts used by the METplus Wrappers. + +**MORE INFO COMING SOON** + +config_metplus.py +----------------- + +config_util.py +-------------- + +config_validate.py +------------------ + +constants.py +------------ + +diff_util.py +------------ + +field_util.py +------------- + +met_config.py +------------- + +metplus_check.py +---------------- + +run_util.py +----------- + +string_manip.py +--------------- + +string_template_substitution.py +------------------------------- + +system_util.py +-------------- + +time_looping.py +--------------- + +time_util.py +------------ + +wrapper_init.py +--------------- + +Internal Development Tools +========================== + +These utilities scripts can be found in **internal/scripts/dev_tools**. +They were written to assist with common development tasks. + +Add MET Config Helper +--------------------- + +**internal/scripts/dev_tools/add_met_config_helper.py** + +**MORE INFO COMING SOON** + +Generate Release Notes +---------------------- + +**internal/scripts/dev_tools/generate_release_notes.py** + +**MORE INFO COMING SOON** diff --git a/docs/Release_Guide/index.rst b/docs/Release_Guide/index.rst index 37fb437c19..58a99ce1cb 100644 --- a/docs/Release_Guide/index.rst +++ b/docs/Release_Guide/index.rst @@ -82,6 +82,27 @@ software team and the respective funding institutions. For further inquiries or to report any bugs, please contact our dedicated support team in the `METplus GitHub Discussions Forum `_. + +.. _releaseDependencies: + +******************** +Release Dependencies +******************** + +Software dependencies exist for the METplus components. The METviewer, METplotpy, +METplus wrappers, and METexpress components have dependencies on other METplus +components. + +Before METplotpy is released, METcalcpy needs to have been released. + +Before METviewer is released, METdataio, METcalcpy, and METplotpy need +to have been released. + +Before METplus wrappers is released, MET, METdataio, METcalcpy, METplotpy, and +METviewer all need to have been released. + +Before METexpress is released, METdataio needs to have been released. + ******************** Instructions Summary ******************** diff --git a/docs/Release_Guide/met_bugfix.rst b/docs/Release_Guide/met_bugfix.rst index 6861bc8b76..d70e91427e 100644 --- a/docs/Release_Guide/met_bugfix.rst +++ b/docs/Release_Guide/met_bugfix.rst @@ -15,4 +15,5 @@ Create a new vX.Y.Z bugfix release from the main_vX.Y branch. .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/met/update_dtc_website.rst +.. include:: release_steps/update_version_lookup.rst .. include:: release_steps/finalize_release_on_github_bugfix.rst diff --git a/docs/Release_Guide/met_development.rst b/docs/Release_Guide/met_development.rst index df04a8b900..465c6a2006 100644 --- a/docs/Release_Guide/met_development.rst +++ b/docs/Release_Guide/met_development.rst @@ -7,12 +7,14 @@ Create a new vX.Y.Z-betaN or vX.Y.Z-rcN development release from the develop bra .. include:: release_steps/open_release_issue.rst .. include:: release_steps/clone_project_repository.rst -.. include:: release_steps/checkout_develop_branch.rst +.. include:: release_steps/checkout_source_branch.rst .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/met/update_version_development.rst .. include:: release_steps/update_release_notes_development.rst .. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/merge_release_issue.rst +.. include:: release_steps/met/create_release_reference_branch.rst +.. include:: release_steps/create_release_branch.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/met/update_dtc_website.rst diff --git a/docs/Release_Guide/met_official.rst b/docs/Release_Guide/met_official.rst index 34cefc7b23..3129049dd8 100644 --- a/docs/Release_Guide/met_official.rst +++ b/docs/Release_Guide/met_official.rst @@ -7,7 +7,7 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/open_release_issue.rst .. include:: release_steps/clone_project_repository.rst -.. include:: release_steps/checkout_develop_branch.rst +.. include:: release_steps/checkout_main_branch.rst .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/met/update_version_official.rst .. include:: release_steps/update_release_notes_official.rst @@ -15,8 +15,6 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/rotate_authorship.rst .. include:: release_steps/merge_release_issue.rst .. include:: release_steps/met/update_web_server_data.rst -.. include:: release_steps/met/create_release_reference_branch.rst -.. include:: release_steps/create_release_branch.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/met/update_dtc_website.rst diff --git a/docs/Release_Guide/metcalcpy_bugfix.rst b/docs/Release_Guide/metcalcpy_bugfix.rst index b224fdb209..62d6ade67e 100644 --- a/docs/Release_Guide/metcalcpy_bugfix.rst +++ b/docs/Release_Guide/metcalcpy_bugfix.rst @@ -15,4 +15,5 @@ Create a new vX.Y.Z bugfix release from the main_vX.Y branch. .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/update_dtc_website.rst +.. include:: release_steps/update_version_lookup.rst .. include:: release_steps/finalize_release_on_github_official.rst diff --git a/docs/Release_Guide/metcalcpy_development.rst b/docs/Release_Guide/metcalcpy_development.rst index bf9aaa635a..a70a26311c 100644 --- a/docs/Release_Guide/metcalcpy_development.rst +++ b/docs/Release_Guide/metcalcpy_development.rst @@ -7,12 +7,13 @@ Create a new vX.Y.Z-betaN or vX.Y.Z-rcN development release from the develop bra .. include:: release_steps/open_release_issue.rst .. include:: release_steps/clone_project_repository.rst -.. include:: release_steps/checkout_develop_branch.rst +.. include:: release_steps/checkout_source_branch.rst .. include:: release_steps/create_release_feature_branch.rst -.. include:: release_steps/metcalcpy/update_version.rst +.. include:: release_steps/metcalcpy/update_version_development.rst .. include:: release_steps/update_release_notes_development.rst .. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/merge_release_issue.rst +.. include:: release_steps/create_release_branch.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/update_dtc_website.rst diff --git a/docs/Release_Guide/metcalcpy_official.rst b/docs/Release_Guide/metcalcpy_official.rst index 7a554eebc2..cc1d8b51c6 100644 --- a/docs/Release_Guide/metcalcpy_official.rst +++ b/docs/Release_Guide/metcalcpy_official.rst @@ -7,15 +7,13 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/open_release_issue.rst .. include:: release_steps/clone_project_repository.rst -.. include:: release_steps/checkout_develop_branch.rst +.. include:: release_steps/checkout_main_branch.rst .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/metcalcpy/update_version_official.rst .. include:: release_steps/update_release_notes_official.rst .. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/rotate_authorship.rst .. include:: release_steps/merge_release_issue.rst -.. include:: release_steps/create_release_branch.rst -.. include:: release_steps/push_release_branch.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/update_dtc_website.rst diff --git a/docs/Release_Guide/metdataio_bugfix.rst b/docs/Release_Guide/metdataio_bugfix.rst index 096e9c260a..992e44b48f 100644 --- a/docs/Release_Guide/metdataio_bugfix.rst +++ b/docs/Release_Guide/metdataio_bugfix.rst @@ -18,4 +18,5 @@ Create a new vX.Y.Z bugfix release from the main_vX.Y branch. .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/common/update_dtc_website.rst +.. include:: release_steps/update_version_lookup.rst .. include:: release_steps/finalize_release_on_github_bugfix.rst diff --git a/docs/Release_Guide/metdataio_development.rst b/docs/Release_Guide/metdataio_development.rst index e7f3b4a184..33bc526a24 100644 --- a/docs/Release_Guide/metdataio_development.rst +++ b/docs/Release_Guide/metdataio_development.rst @@ -10,12 +10,13 @@ Create a new vX.Y.Z-betaN or vX.Y.Z-rcN development release from the develop bra .. include:: release_steps/open_release_issue.rst .. include:: release_steps/clone_project_repository.rst -.. include:: release_steps/checkout_develop_branch.rst +.. include:: release_steps/checkout_source_branch.rst .. include:: release_steps/create_release_feature_branch.rst -.. include:: release_steps/metdataio/update_version.rst +.. include:: release_steps/metdataio/update_version_development.rst .. include:: release_steps/update_release_notes_development.rst .. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/merge_release_issue.rst +.. include:: release_steps/create_release_branch.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/common/update_dtc_website.rst diff --git a/docs/Release_Guide/metdataio_official.rst b/docs/Release_Guide/metdataio_official.rst index a3e16ee72e..b53b87729a 100644 --- a/docs/Release_Guide/metdataio_official.rst +++ b/docs/Release_Guide/metdataio_official.rst @@ -10,15 +10,13 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/open_release_issue.rst .. include:: release_steps/clone_project_repository.rst -.. include:: release_steps/checkout_develop_branch.rst +.. include:: release_steps/checkout_main_branch.rst .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/metdataio/update_version_official.rst .. include:: release_steps/update_release_notes_official.rst .. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/rotate_authorship.rst .. include:: release_steps/merge_release_issue.rst -.. include:: release_steps/create_release_branch.rst -.. include:: release_steps/push_release_branch.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/common/update_dtc_website.rst diff --git a/docs/Release_Guide/metexpress_official.rst b/docs/Release_Guide/metexpress_official.rst index 256c143a68..20cfcc1520 100644 --- a/docs/Release_Guide/metexpress_official.rst +++ b/docs/Release_Guide/metexpress_official.rst @@ -18,5 +18,6 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/metexpress/pull_changes_and_merge_to_dev.rst .. include:: release_steps/update_docs_official.rst .. include:: release_steps/metexpress/update_dtc_website.rst +.. include:: release_steps/update_version_lookup.rst .. include:: release_steps/metexpress/announce_release.rst .. include:: release_steps/set_beta_deletion_reminder_official.rst diff --git a/docs/Release_Guide/metplotpy_bugfix.rst b/docs/Release_Guide/metplotpy_bugfix.rst index ec177ee9b9..f9609dcdec 100644 --- a/docs/Release_Guide/metplotpy_bugfix.rst +++ b/docs/Release_Guide/metplotpy_bugfix.rst @@ -14,5 +14,6 @@ Create a new vX.Y.Z bugfix release from the main_vX.Y branch. .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/update_dtc_website.rst +.. include:: release_steps/update_version_lookup.rst .. include:: release_steps/finalize_release_on_github_official.rst diff --git a/docs/Release_Guide/metplotpy_development.rst b/docs/Release_Guide/metplotpy_development.rst index 50ad0368c8..e8aa8a864f 100644 --- a/docs/Release_Guide/metplotpy_development.rst +++ b/docs/Release_Guide/metplotpy_development.rst @@ -7,12 +7,13 @@ Create a new vX.Y.Z-betaN or vX.Y.Z-rcN development release from the develop bra .. include:: release_steps/open_release_issue.rst .. include:: release_steps/clone_project_repository.rst -.. include:: release_steps/checkout_develop_branch.rst +.. include:: release_steps/checkout_source_branch.rst .. include:: release_steps/create_release_feature_branch.rst -.. include:: release_steps/metplotpy/update_version.rst -.. include:: release_steps/merge_release_issue.rst +.. include:: release_steps/metplotpy/update_version_development.rst .. include:: release_steps/update_release_notes_development.rst .. include:: release_steps/update_upgrade_instructions.rst +.. include:: release_steps/merge_release_issue.rst +.. include:: release_steps/create_release_branch.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/update_dtc_website.rst diff --git a/docs/Release_Guide/metplotpy_official.rst b/docs/Release_Guide/metplotpy_official.rst index f16a82c753..47d97cc131 100644 --- a/docs/Release_Guide/metplotpy_official.rst +++ b/docs/Release_Guide/metplotpy_official.rst @@ -7,15 +7,13 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/open_release_issue.rst .. include:: release_steps/clone_project_repository.rst -.. include:: release_steps/checkout_develop_branch.rst +.. include:: release_steps/checkout_main_branch.rst .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/metplotpy/update_version_official.rst .. include:: release_steps/update_release_notes_official.rst .. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/rotate_authorship.rst .. include:: release_steps/merge_release_issue.rst -.. include:: release_steps/create_release_branch.rst -.. include:: release_steps/push_release_branch.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/update_dtc_website.rst diff --git a/docs/Release_Guide/metplus_bugfix.rst b/docs/Release_Guide/metplus_bugfix.rst index f4635dae4c..ecc08da001 100644 --- a/docs/Release_Guide/metplus_bugfix.rst +++ b/docs/Release_Guide/metplus_bugfix.rst @@ -15,4 +15,5 @@ Create a new vX.Y.Z bugfix release from the main_vX.Y branch. .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/metplus/create_release_extra.rst .. include:: release_steps/metplus/update_dtc_website.rst +.. include:: release_steps/update_version_lookup.rst .. include:: release_steps/finalize_release_on_github_official.rst diff --git a/docs/Release_Guide/metplus_development.rst b/docs/Release_Guide/metplus_development.rst index 7754a7759b..be37a5d996 100644 --- a/docs/Release_Guide/metplus_development.rst +++ b/docs/Release_Guide/metplus_development.rst @@ -7,12 +7,13 @@ Create a new vX.Y.Z-betaN or vX.Y.Z-rcN development release from the develop bra .. include:: release_steps/open_release_issue.rst .. include:: release_steps/clone_project_repository.rst -.. include:: release_steps/checkout_develop_branch.rst +.. include:: release_steps/checkout_source_branch.rst .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/metplus/update_version.rst .. include:: release_steps/update_release_notes_development.rst .. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/merge_release_issue.rst +.. include:: release_steps/metplus/create_release_reference_branch.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/metplus/create_release_extra.rst .. include:: release_steps/metplus/update_version_on_develop.rst diff --git a/docs/Release_Guide/metplus_official.rst b/docs/Release_Guide/metplus_official.rst index 4b996155e7..64e91213e3 100644 --- a/docs/Release_Guide/metplus_official.rst +++ b/docs/Release_Guide/metplus_official.rst @@ -7,21 +7,21 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/open_release_issue.rst .. include:: release_steps/clone_project_repository.rst -.. include:: release_steps/checkout_develop_branch.rst +.. include:: release_steps/checkout_main_branch.rst .. include:: release_steps/create_release_feature_branch.rst +.. include:: release_steps/metplus/update_version_official.rst .. include:: release_steps/metplus/update_release_date.rst .. include:: release_steps/update_release_notes_official.rst .. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/rotate_authorship.rst .. include:: release_steps/metplus/update_manage_externals.rst .. include:: release_steps/merge_release_issue.rst -.. include:: release_steps/metplus/create_release_reference_branch.rst -.. include:: release_steps/push_release_branch.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/metplus/create_release_extra.rst .. include:: release_steps/metplus/update_dtc_website.rst .. include:: release_steps/finalize_release_on_github_official.rst -.. include:: release_steps/metplus/update_version_on_develop.rst .. include:: release_steps/update_docs_official.rst +.. include:: release_steps/metplus/add_next_version_to_lookup.rst +.. include:: release_steps/metplus/coordinated_release_announcement.rst .. include:: release_steps/metplus/update_web_server_data.rst .. include:: release_steps/set_beta_deletion_reminder_official.rst diff --git a/docs/Release_Guide/metviewer_bugfix.rst b/docs/Release_Guide/metviewer_bugfix.rst index e8f5a35c1d..39a1705a00 100644 --- a/docs/Release_Guide/metviewer_bugfix.rst +++ b/docs/Release_Guide/metviewer_bugfix.rst @@ -15,4 +15,5 @@ Create a new vX.Y.Z bugfix release from the main_vX.Y branch. .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/update_dtc_website.rst +.. include:: release_steps/update_version_lookup.rst .. include:: release_steps/finalize_release_on_github_official.rst diff --git a/docs/Release_Guide/metviewer_development.rst b/docs/Release_Guide/metviewer_development.rst index b1af85e12e..6f0d07b5c7 100644 --- a/docs/Release_Guide/metviewer_development.rst +++ b/docs/Release_Guide/metviewer_development.rst @@ -7,12 +7,13 @@ Create a new vX.Y.Z-betaN or vX.Y.Z-rcN development release from the develop bra .. include:: release_steps/open_release_issue.rst .. include:: release_steps/clone_project_repository.rst -.. include:: release_steps/checkout_develop_branch.rst +.. include:: release_steps/checkout_source_branch.rst .. include:: release_steps/create_release_feature_branch.rst -.. include:: release_steps/metviewer/update_version.rst +.. include:: release_steps/metviewer/update_version_development.rst .. include:: release_steps/update_release_notes_development.rst .. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/merge_release_issue.rst +.. include:: release_steps/create_release_branch.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/update_dtc_website.rst diff --git a/docs/Release_Guide/metviewer_official.rst b/docs/Release_Guide/metviewer_official.rst index 15a2e385db..a0dbdf45ce 100644 --- a/docs/Release_Guide/metviewer_official.rst +++ b/docs/Release_Guide/metviewer_official.rst @@ -7,15 +7,13 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/open_release_issue.rst .. include:: release_steps/clone_project_repository.rst -.. include:: release_steps/checkout_develop_branch.rst +.. include:: release_steps/checkout_main_branch.rst .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/metviewer/update_version_official.rst .. include:: release_steps/update_release_notes_official.rst .. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/rotate_authorship.rst .. include:: release_steps/merge_release_issue.rst -.. include:: release_steps/create_release_branch.rst -.. include:: release_steps/push_release_branch.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/update_dtc_website.rst diff --git a/docs/Release_Guide/release_steps/checkout_develop_branch.rst b/docs/Release_Guide/release_steps/checkout_develop_branch.rst deleted file mode 100644 index 1f46eb0426..0000000000 --- a/docs/Release_Guide/release_steps/checkout_develop_branch.rst +++ /dev/null @@ -1,8 +0,0 @@ -Checkout the Develop Branch -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -* Checkout the develop branch: - -.. parsed-literal:: - - git checkout develop diff --git a/docs/Release_Guide/release_steps/checkout_main_branch.rst b/docs/Release_Guide/release_steps/checkout_main_branch.rst index 4b53593fe7..830c96d30a 100644 --- a/docs/Release_Guide/release_steps/checkout_main_branch.rst +++ b/docs/Release_Guide/release_steps/checkout_main_branch.rst @@ -2,7 +2,7 @@ Checkout the Main Branch ^^^^^^^^^^^^^^^^^^^^^^^^ Checkout the main branch for the X.Y release. If you are creating -the |projectRepo|-X.Y.Z bugfix release, checkout main_vX.Y. +the |projectRepo|-X.Y.Z official or bugfix release, checkout main_vX.Y. .. parsed-literal:: diff --git a/docs/Release_Guide/release_steps/checkout_source_branch.rst b/docs/Release_Guide/release_steps/checkout_source_branch.rst new file mode 100644 index 0000000000..094a8029fd --- /dev/null +++ b/docs/Release_Guide/release_steps/checkout_source_branch.rst @@ -0,0 +1,18 @@ +Checkout the Source Branch +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* If creating a **beta** (betaN) or **first release candidate** (rc1) release, + checkout the develop branch: + +.. parsed-literal:: + + git checkout develop + +**OR** + +* If creating a **later release candidate** (rc2+) release, + checkout the appropriate main branch: + +.. parsed-literal:: + + git checkout main_vX.Y diff --git a/docs/Release_Guide/release_steps/create_release_branch.rst b/docs/Release_Guide/release_steps/create_release_branch.rst index 737420a9c0..bb92aaa1ff 100644 --- a/docs/Release_Guide/release_steps/create_release_branch.rst +++ b/docs/Release_Guide/release_steps/create_release_branch.rst @@ -1,7 +1,16 @@ Create Release Branch ^^^^^^^^^^^^^^^^^^^^^ -* Create a branch from the develop branch for the new official release and push it to GitHub. +.. note:: + + These instructions only apply when creating the **first release candidate** + (rc1) development release. Skip this section for earlier beta (betaN) or later + release candidate (rc2+) development releases. + + +* Create a new 'main_vX.Y' branch from the develop branch for the upcoming + official release and push it to GitHub. All remaining development for the + upcoming official release occurs on this new 'main_vX.Y' branch. .. parsed-literal:: diff --git a/docs/Release_Guide/release_steps/create_release_on_github.rst b/docs/Release_Guide/release_steps/create_release_on_github.rst index c9818ed653..efbbbb76a8 100644 --- a/docs/Release_Guide/release_steps/create_release_on_github.rst +++ b/docs/Release_Guide/release_steps/create_release_on_github.rst @@ -1,13 +1,22 @@ Create Release on GitHub ^^^^^^^^^^^^^^^^^^^^^^^^ +* Ensure that any and all dependent releases for |projectRepo| + have been created according to :ref:`releaseDependencies`. + * Go to Releases on GitHub and click 'Draft a new release' -* Set the tag based on the version, starting with 'v', i.e. vX.Y.Z-betaN, vX.Y.Z-rcN, or vX.Y.Z +* For 'Choose a tag', create a new tag based on the version, starting with 'v' + (i.e. vX.Y.Z-betaN, vX.Y.Z-rcN, or vX.Y.Z) + +* Define the 'Target' branch as: + + * 'develop' for a beta development release -* Set the branch value to 'develop' for a development release or 'main_vX.Y' for a bugfix or official release + * 'main_vX.Y' for a release candidate, bugfix, or official release -* Choose a release name (i.e. |projectRepo|-X.Y.Z-betaN, |projectRepo|-X.Y.Z-rcN, or |projectRepo|-X.Y.Z) +* Define the 'Release title' based on the repository name and version, *without* a leading 'v' + (i.e. |projectRepo|-X.Y.Z-betaN, |projectRepo|-X.Y.Z-rcN, or |projectRepo|-X.Y.Z) * Add a link to the release notes from the |projectRepo| User's Guide, i.e. https://|projectRepo|.readthedocs.io/en/vX.Y.Z-betaN/Users_Guide/release-notes.html diff --git a/docs/Release_Guide/release_steps/met/create_release_reference_branch.rst b/docs/Release_Guide/release_steps/met/create_release_reference_branch.rst index 5f4f7000cd..c99abd33c3 100644 --- a/docs/Release_Guide/release_steps/met/create_release_reference_branch.rst +++ b/docs/Release_Guide/release_steps/met/create_release_reference_branch.rst @@ -1,9 +1,16 @@ Create Release Reference Branch ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. note:: + + These instructions only apply when creating the **first release candidate** + (rc1) development release. Skip this section for earlier beta (betaN) or later + release candidate (rc2+) development releases. + + * Create a branch from the develop branch for the reference branch for the new official release and push it to GitHub. The branch name should match - the format main_vX.Y-ref where X.Y is the major/minor release number. + the format 'main_vX.Y-ref' where X.Y is the major/minor release number. .. parsed-literal:: @@ -22,14 +29,14 @@ Push Reference Branch to GitHub Pushing this branch to GitHub should trigger the GitHub Actions automation that runs all of the use cases and creates Docker data volumes with the output data. These data will be used to verify that any bugfixes applied to the -main_vX.Y branch does not break any of existing logic. +'main_vX.Y' branch does not break any of existing logic. Monitor GitHub Actions Workflow """"""""""""""""""""""""""""""" Navigate to https://github.com/dtcenter/MET/actions and verify that a -*Testing* workflow was triggered on the *main_vX.Y-ref* branch. +'Testing' workflow was triggered on the 'main_vX.Y-ref' branch. * Wait until the entire workflow has run successfully. The final job entitled - "Create Output Docker Data Volumes" should create Docker data volumes for + 'Create Output Docker Data Volumes' should create Docker data volumes for each use case category on DockerHub (dtcenter/met-dev). diff --git a/docs/Release_Guide/release_steps/met/update_version_bugfix.rst b/docs/Release_Guide/release_steps/met/update_version_bugfix.rst index c432ce3407..224dc35250 100644 --- a/docs/Release_Guide/release_steps/met/update_version_bugfix.rst +++ b/docs/Release_Guide/release_steps/met/update_version_bugfix.rst @@ -3,8 +3,8 @@ Update Version Number * Update the version in the code and documentation: - * Update the *met_version* variable in *src/basic/vx_util/util_constants.h* which defines the version number written to the MET output files. + * Update the 'met_version' variable in 'src/basic/vx_util/util_constants.h' which defines the version number written to the MET output files. - * In *docs/conf.py*, update the *version*, *release_year*, and *release_date* variables for the documentation. + * In 'docs/conf.py', update the 'version', 'release_year', and 'release_date' variables for the documentation. * DO NOT update the version number listed in the MET configuration files, add a new table file, or add a new test header file. diff --git a/docs/Release_Guide/release_steps/met/update_version_development.rst b/docs/Release_Guide/release_steps/met/update_version_development.rst index 82fc66e793..069d9dd926 100644 --- a/docs/Release_Guide/release_steps/met/update_version_development.rst +++ b/docs/Release_Guide/release_steps/met/update_version_development.rst @@ -1,10 +1,11 @@ Update Version Number ^^^^^^^^^^^^^^^^^^^^^ + +.. note:: -* Prior to creating the first development release (e.g. beta1) for a new version, the official release version number should have already been updated. + The software version number should have been updated for the next official release during development toward the **first development release** (e.g. beta1). + If not, follow the :ref:`official release instructions ` to do so prior to creating the first development release. -* If the official release version has not yet been updated (e.g. beta1), follow the :ref:`official release instructions ` to do so. +* If the official release version has already been updated (e.g. beta2+ and rc1+), do the following: -* If the official release version has already been updated (e.g. beta2 and beyond), do the following: - - * In *docs/conf.py*, update the *version*, *release_year*, and *release_date* variables for the documentation. + * In 'docs/conf.py', update the 'version', 'release_year', and 'release_date' variables for the documentation. diff --git a/docs/Release_Guide/release_steps/met/update_version_official.rst b/docs/Release_Guide/release_steps/met/update_version_official.rst index 31f06d31cb..d007a3878c 100644 --- a/docs/Release_Guide/release_steps/met/update_version_official.rst +++ b/docs/Release_Guide/release_steps/met/update_version_official.rst @@ -5,20 +5,28 @@ Update Version Number * Update the version in the code and documentation: - * If necessary, update the *met_version* variable in *src/basic/vx_util/util_constants.h* which defines the version number written to the MET output files. + * If necessary, update the 'met_version' variable in 'src/basic/vx_util/util_constants.h' which defines the version number written to the MET output files. - * In *docs/conf.py*, update the *version*, *release_year*, and *release_date* variables for the documentation. + * In 'docs/conf.py', update the 'version', 'release_year', and 'release_date' variables for the documentation. - * In *docs/Users_Guide/installation.rst*, search for the X.Y version, replacing the current X.Y version with the official X.Y version, if necessary. Pay particular attention to the "Note" about the C++ standard and modify if necessary. The X.Y version number in the "Note" box should NOT change unless the default C++ standard changes. + * In 'docs/Users_Guide/installation.rst', search for the X.Y version, replacing the current X.Y version with the official X.Y version, if necessary. Pay particular attention to the "Note" about the C++ standard and modify if necessary. The X.Y version number in the "Note" box should NOT change unless the default C++ standard changes. + * If necessary, update the MET flowchart: + + * In 'docs/Flowchart', update 'MET_flowchart.pptx' to add a new version for this release. + + * Save and commit a PNG version of the flowchart named 'MET_flowchart_vX.Y.Z.png'. + + * Copy the new flowchart version into 'docs/Users_Guide/figure/overview-figure.png'. + * If necessary, update the version number listed in the MET configuration files: - * Default configuration files in *data/config*. + * Default configuration files in 'data/config'. - * Sample configuration files in *scripts/config*. + * Sample configuration files in 'scripts/config'. - * Test configuration files, searching recursively, in *internal/test_unit/config*. + * Test configuration files, searching recursively, in 'internal/test_unit/config'. - * If necessary, add a new *data/table_files/met_header_columns_VX.Y.txt* defining the columns names for this version. + * If necessary, add a new 'data/table_files/met_header_columns_VX.Y.txt' defining the columns names for this version. - * If necessary, add a new *internal/test_unit/hdr/met_X_Y.hdr* file defining the column names for this version for the test scripts. + * If necessary, add a new 'internal/test_unit/hdr/met_X_Y.hdr' file defining the column names for this version for the test scripts. diff --git a/docs/Release_Guide/release_steps/metcalcpy/update_version.rst b/docs/Release_Guide/release_steps/metcalcpy/update_version.rst deleted file mode 100644 index 2f14f2278f..0000000000 --- a/docs/Release_Guide/release_steps/metcalcpy/update_version.rst +++ /dev/null @@ -1,12 +0,0 @@ -Update Version Number for Release -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Remove **-dev** from the version number: - -* We are naming releases with X.Y.Z format even if Z is 0. (the file containing the version number is located at docs/version - -* In the develop branch, the version should match the upcoming release with -dev added to the end like X.Y.Z-betaN-dev, i.e. 4.0.0-beta1-dev - -* Remove **-dev** from the version number so that it matches the release you are creating. - -* Commit changes and push to GitHub. diff --git a/docs/Release_Guide/release_steps/metcalcpy/update_version_bugfix.rst b/docs/Release_Guide/release_steps/metcalcpy/update_version_bugfix.rst index d4ebf5eb71..247803c919 100644 --- a/docs/Release_Guide/release_steps/metcalcpy/update_version_bugfix.rst +++ b/docs/Release_Guide/release_steps/metcalcpy/update_version_bugfix.rst @@ -1,9 +1,8 @@ -Update Version Number for Release -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Update Version Number +^^^^^^^^^^^^^^^^^^^^^ -Update the version number for the bugfix release: +* Update the version: -* For METcalcpy, the file containing the version number is located at docs/version. -* If the current release is listed as X.Y.Z (major.minor.micro), the bugfix version should be X.Y.Z+1 - (i.e. increment the micro value by 1: 1.1.0 becomes 1.1.1) + * In 'docs/version', update '__version__'. This value is used by setup.py. + * In 'docs/conf.py', update the 'version', 'release_year', and 'release_date' variables for the documentation. diff --git a/docs/Release_Guide/release_steps/metcalcpy/update_version_development.rst b/docs/Release_Guide/release_steps/metcalcpy/update_version_development.rst new file mode 100644 index 0000000000..510d07c91d --- /dev/null +++ b/docs/Release_Guide/release_steps/metcalcpy/update_version_development.rst @@ -0,0 +1,13 @@ +Update Version Number +^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + + The software version number should have been updated for the next official release during development toward the **first development release** (e.g. beta1). + If not, follow the :ref:`official release instructions ` to do so prior to creating the first development release. + +* If the official release version has already been updated (e.g. beta2+ and rc1+), do the following: + + * In 'docs/conf.py', update the 'version', 'release_year', and 'release_date' variables for the documentation. + + * In 'docs/version', update '__version__'. This value is used by setup.py. diff --git a/docs/Release_Guide/release_steps/metcalcpy/update_version_official.rst b/docs/Release_Guide/release_steps/metcalcpy/update_version_official.rst index 2f2796b5c1..e3a477a605 100644 --- a/docs/Release_Guide/release_steps/metcalcpy/update_version_official.rst +++ b/docs/Release_Guide/release_steps/metcalcpy/update_version_official.rst @@ -1,17 +1,11 @@ -Update Version Number for Release -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Update Version Number +^^^^^^^^^^^^^^^^^^^^^ -Remove **-dev** from the version number: +* Update the version: -* We are naming releases with X.Y.Z format even if Z is 0. -* The file containing the version number is located at docs/version. -* In the develop branch, the version should match the upcoming release with -dev added to the end like X.Y.Z-betaN-dev, i.e. 4.0.0-beta1-dev -* Remove **-dev** from the version number so that it matches the release you are creating. + * If necessary, in 'docs/version', update '__version__'. This value is used by setup.py. -Update the version number in the quick search links: + * In 'docs/conf.py', update the 'version', 'release_year', and 'release_date' variables for the documentation. + -* Open the docs/Users_Guide/quicksearch.rst file for editing. -* Replace the word "develop" in all of the links with "vX.Y.Z", replacing the X.Y.Z with the version number. For example, replace "develop" with "v4.0.0". -* Save and close the file. - -In docs/conf.py, update the version, release_year, and release_date variables for the documentation. + diff --git a/docs/Release_Guide/release_steps/metcalcpy/update_version_on_develop.rst b/docs/Release_Guide/release_steps/metcalcpy/update_version_on_develop.rst index 99318b7cdd..99305311a2 100644 --- a/docs/Release_Guide/release_steps/metcalcpy/update_version_on_develop.rst +++ b/docs/Release_Guide/release_steps/metcalcpy/update_version_on_develop.rst @@ -5,16 +5,15 @@ Change METcalcpy/docs/version __version__ value to the next release after this o to the end. Releases will loosely following these names, but are subject to change: -+-------------------+-----------------------+ -| Release Version | New Develop Version | -+===================+=======================+ -| X.Y.Z-beta1 | X.Y.Z-beta2-dev | -+-------------------+-----------------------+ -| X.Y.Z-beta2 | X.Y.Z-beta3-dev | -+-------------------+-----------------------+ -| X.Y.Z-beta3 | X.Y.Z-rc1-dev | -+-------------------+-----------------------+ -| X.Y.Z-rc1 | X.Y.Z-dev | -+-------------------+-----------------------+ -| X.Y.Z | (X+1).0.0-beta1-dev | -+-------------------+-----------------------+ ++-------------------+----------------------------+ +| Release Version | New Develop Version | ++===================+============================+ +| X.Y.Z-beta1 | X.Y.Z-beta2-dev | ++-------------------+----------------------------+ +| X.Y.Z-beta2 | X.Y.Z-beta3-dev | ++-------------------+----------------------------+ +| X.Y.Z-beta3 | X.Y.Z-rc1-dev | ++-------------------+----------------------------+ +| X.Y.Z-rc1 | (X+1).0.0-beta1-dev OR | +| | **X.(Y+1).0-beta1-dev** | ++-------------------+----------------------------+ diff --git a/docs/Release_Guide/release_steps/metdataio/update_version.rst b/docs/Release_Guide/release_steps/metdataio/update_version.rst deleted file mode 100644 index 2f14f2278f..0000000000 --- a/docs/Release_Guide/release_steps/metdataio/update_version.rst +++ /dev/null @@ -1,12 +0,0 @@ -Update Version Number for Release -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Remove **-dev** from the version number: - -* We are naming releases with X.Y.Z format even if Z is 0. (the file containing the version number is located at docs/version - -* In the develop branch, the version should match the upcoming release with -dev added to the end like X.Y.Z-betaN-dev, i.e. 4.0.0-beta1-dev - -* Remove **-dev** from the version number so that it matches the release you are creating. - -* Commit changes and push to GitHub. diff --git a/docs/Release_Guide/release_steps/metdataio/update_version_bugfix.rst b/docs/Release_Guide/release_steps/metdataio/update_version_bugfix.rst index 86120b0378..b9c72c1a28 100644 --- a/docs/Release_Guide/release_steps/metdataio/update_version_bugfix.rst +++ b/docs/Release_Guide/release_steps/metdataio/update_version_bugfix.rst @@ -1,8 +1,10 @@ -Update Version Number for Release -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Update Version Number +^^^^^^^^^^^^^^^^^^^^^ -Update the version number for the bugfix release: +* Update the version: -* For |projectRepo|, the file containing the version number is located at docs/version. -* If the current release is listed as X.Y.Z (major.minor.micro), the bugfix version should be X.Y.Z+1 - (i.e. increment the micro value by 1: 1.1.0 becomes 1.1.1) + * In 'docs/version', update the version number. This value is used by METplus use cases that run METdbLoad and is used by METviewer. + + * In the top level 'pyproject.toml', update the value of 'version'. + + * In 'docs/conf.py', update the 'version', 'release_year', and 'release_date' variables for the documentation. diff --git a/docs/Release_Guide/release_steps/metdataio/update_version_development.rst b/docs/Release_Guide/release_steps/metdataio/update_version_development.rst new file mode 100644 index 0000000000..4f99a3d8dc --- /dev/null +++ b/docs/Release_Guide/release_steps/metdataio/update_version_development.rst @@ -0,0 +1,15 @@ +Update Version Number +^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + + The software version number should have been updated for the next official release during development toward the **first development release** (e.g. beta1). + If not, follow the :ref:`official release instructions ` to do so prior to creating the first development release. + +* If the official release version has already been updated (e.g. beta2+ and rc1+), do the following: + + * In 'docs/conf.py', update the 'version', 'release_year', and 'release_date' variables for the documentation. + + * In 'docs/version', update the version number. This value is used by METplus use cases that run METdbLoad and is used by METviewer. + + * In the top level 'pyproject.toml', update the value of 'version'. diff --git a/docs/Release_Guide/release_steps/metdataio/update_version_official.rst b/docs/Release_Guide/release_steps/metdataio/update_version_official.rst index 442b4cbf6d..2e31cfd456 100644 --- a/docs/Release_Guide/release_steps/metdataio/update_version_official.rst +++ b/docs/Release_Guide/release_steps/metdataio/update_version_official.rst @@ -1,11 +1,11 @@ -Update Version Number for Release -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Update Version Number +^^^^^^^^^^^^^^^^^^^^^ -Remove **-dev** from the version number: +* Update the version: -* We are naming releases with X.Y.Z format even if Z is 0. -* The file containing the version number is located at docs/version. -* In the develop branch, the version should match the upcoming release with -dev added to the end like X.Y.Z-betaN-dev, i.e. 4.0.0-beta1-dev -* Remove **-dev** from the version number so that it matches the release you are creating. + * If necessary, in 'docs/version', update the version number. This value is used by METplus use cases that run METdbLoad and is used by METviewer. -In docs/conf.py, update the version, release_year, and release_date variables for the documentation. + * If necessary, in the top level 'pyproject.toml', update the value of 'version'. + + * In 'docs/conf.py', update the 'version', 'release_year', and 'release_date' variables for the documentation. + diff --git a/docs/Release_Guide/release_steps/metdataio/update_version_on_develop.rst b/docs/Release_Guide/release_steps/metdataio/update_version_on_develop.rst index 11632a7893..ad2d01901f 100644 --- a/docs/Release_Guide/release_steps/metdataio/update_version_on_develop.rst +++ b/docs/Release_Guide/release_steps/metdataio/update_version_on_develop.rst @@ -5,16 +5,15 @@ Change |projectRepo|/docs/version value to the next release after this one with to the end. Releases will loosely follow these names, but are subject to change: -+-------------------+-----------------------+ -| Release Version | New Develop Version | -+===================+=======================+ -| X.Y.Z-beta1 | X.Y.Z-beta2-dev | -+-------------------+-----------------------+ -| X.Y.Z-beta2 | X.Y.Z-beta3-dev | -+-------------------+-----------------------+ -| X.Y.Z-beta3 | X.Y.Z-rc1-dev | -+-------------------+-----------------------+ -| X.Y.Z-rc1 | X.Y.Z-dev | -+-------------------+-----------------------+ -| X.Y.Z | (X+1).0.0-beta1-dev | -+-------------------+-----------------------+ ++-------------------+----------------------------+ +| Release Version | New Develop Version | ++===================+============================+ +| X.Y.Z-beta1 | X.Y.Z-beta2-dev | ++-------------------+----------------------------+ +| X.Y.Z-beta2 | X.Y.Z-beta3-dev | ++-------------------+----------------------------+ +| X.Y.Z-beta3 | X.Y.Z-rc1-dev | ++-------------------+----------------------------+ +| X.Y.Z-rc1 | (X+1).0.0-beta1-dev OR | +| | **X.(Y+1).0-beta1-dev** | ++-------------------+----------------------------+ diff --git a/docs/Release_Guide/release_steps/metplotpy/update_version.rst b/docs/Release_Guide/release_steps/metplotpy/update_version.rst deleted file mode 100644 index 4fe3d30a5e..0000000000 --- a/docs/Release_Guide/release_steps/metplotpy/update_version.rst +++ /dev/null @@ -1,8 +0,0 @@ -Update Version Number for Release -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Create the version number - -- In the METplotpy/docs/version file, assign the version to the '__version__' keyword -- We are naming releases with X.Y.Z format even if Z is 0. -- In the develop branch, the version should match the upcoming release like X.Y.Z-betaN diff --git a/docs/Release_Guide/release_steps/metplotpy/update_version_bugfix.rst b/docs/Release_Guide/release_steps/metplotpy/update_version_bugfix.rst index 69d259dbc1..883f3a93f5 100644 --- a/docs/Release_Guide/release_steps/metplotpy/update_version_bugfix.rst +++ b/docs/Release_Guide/release_steps/metplotpy/update_version_bugfix.rst @@ -1,9 +1,7 @@ -Update Version Number for Release -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Update Version Number +^^^^^^^^^^^^^^^^^^^^^ -Update the version number for the bugfix release: +* Update the version: -* For METplotpy, the file containing the version number is located at docs/version. -* If the current release is listed as X.Y.Z (major.minor.micro), the bugfix version should be X.Y.Z+1 - (i.e. increment the micro value by 1: 1.1.0 becomes 1.1.1) + * In 'docs/conf.py', update the 'version', 'release_year', and 'release_date' variables for the documentation. diff --git a/docs/Release_Guide/release_steps/metplotpy/update_version_development.rst b/docs/Release_Guide/release_steps/metplotpy/update_version_development.rst new file mode 100644 index 0000000000..069d9dd926 --- /dev/null +++ b/docs/Release_Guide/release_steps/metplotpy/update_version_development.rst @@ -0,0 +1,11 @@ +Update Version Number +^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + + The software version number should have been updated for the next official release during development toward the **first development release** (e.g. beta1). + If not, follow the :ref:`official release instructions ` to do so prior to creating the first development release. + +* If the official release version has already been updated (e.g. beta2+ and rc1+), do the following: + + * In 'docs/conf.py', update the 'version', 'release_year', and 'release_date' variables for the documentation. diff --git a/docs/Release_Guide/release_steps/metplotpy/update_version_official.rst b/docs/Release_Guide/release_steps/metplotpy/update_version_official.rst index 2f2796b5c1..883f3a93f5 100644 --- a/docs/Release_Guide/release_steps/metplotpy/update_version_official.rst +++ b/docs/Release_Guide/release_steps/metplotpy/update_version_official.rst @@ -1,17 +1,7 @@ -Update Version Number for Release -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Update Version Number +^^^^^^^^^^^^^^^^^^^^^ -Remove **-dev** from the version number: +* Update the version: -* We are naming releases with X.Y.Z format even if Z is 0. -* The file containing the version number is located at docs/version. -* In the develop branch, the version should match the upcoming release with -dev added to the end like X.Y.Z-betaN-dev, i.e. 4.0.0-beta1-dev -* Remove **-dev** from the version number so that it matches the release you are creating. + * In 'docs/conf.py', update the 'version', 'release_year', and 'release_date' variables for the documentation. -Update the version number in the quick search links: - -* Open the docs/Users_Guide/quicksearch.rst file for editing. -* Replace the word "develop" in all of the links with "vX.Y.Z", replacing the X.Y.Z with the version number. For example, replace "develop" with "v4.0.0". -* Save and close the file. - -In docs/conf.py, update the version, release_year, and release_date variables for the documentation. diff --git a/docs/Release_Guide/release_steps/metplotpy/update_version_on_develop.rst b/docs/Release_Guide/release_steps/metplotpy/update_version_on_develop.rst index 1c382af054..52ff99ea4f 100644 --- a/docs/Release_Guide/release_steps/metplotpy/update_version_on_develop.rst +++ b/docs/Release_Guide/release_steps/metplotpy/update_version_on_develop.rst @@ -5,16 +5,15 @@ Change METplotpy/docs/version __version__ value to the next release after this o to the end. Releases will loosely following these names, but are subject to change: -+-------------------+-----------------------+ -| Release Version | New Develop Version | -+===================+=======================+ -| X.Y.Z-beta1 | X.Y.Z-beta2-dev | -+-------------------+-----------------------+ -| X.Y.Z-beta2 | X.Y.Z-beta3-dev | -+-------------------+-----------------------+ -| X.Y.Z-beta3 | X.Y.Z-rc1-dev | -+-------------------+-----------------------+ -| X.Y.Z-rc1 | X.Y.Z-dev | -+-------------------+-----------------------+ -| X.Y.Z | (X+1).0.0-beta1-dev | -+-------------------+-----------------------+ ++-------------------+----------------------------+ +| Release Version | New Develop Version | ++===================+============================+ +| X.Y.Z-beta1 | X.Y.Z-beta2-dev | ++-------------------+----------------------------+ +| X.Y.Z-beta2 | X.Y.Z-beta3-dev | ++-------------------+----------------------------+ +| X.Y.Z-beta3 | X.Y.Z-rc1-dev | ++-------------------+----------------------------+ +| X.Y.Z-rc1 | (X+1).0.0-beta1-dev OR | +| | **X.(Y+1).0-beta1-dev** | ++-------------------+----------------------------+ diff --git a/docs/Release_Guide/release_steps/metplus/add_next_version_to_lookup.rst b/docs/Release_Guide/release_steps/metplus/add_next_version_to_lookup.rst new file mode 100644 index 0000000000..006d56c45d --- /dev/null +++ b/docs/Release_Guide/release_steps/metplus/add_next_version_to_lookup.rst @@ -0,0 +1,38 @@ +Add Next Version to Lookup Table +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In the develop branch, modify the **metplus/component_versions.py** file to +add an entry for the next coordinated release. + +If the X.0 release was just created, add an entry for the X.1 release. + +If the X.1 release was just created, add an entry for the X+1.0 release. + +Set the appropriate X.Y.0 versions for each component. +Maybe sure to set the Z number to 0. + +Set the version for metexpress to None (not a string). + +For example, if the coordinated 6.0 release was just created, add:: + + '6.1': { + 'metplus': '6.1.0', + 'met': '12.1.0', + 'metplotpy': '3.1.0', + 'metcalcpy': '3.1.0', + 'metdataio': '3.1.0', + 'metviewer': '6.1.0', + 'metexpress': None, + }, + +For example, if the coordinated 6.1 release was just created, add:: + + '7.0': { + 'metplus': '7.0.0', + 'met': '13.0.0', + 'metplotpy': '4.0.0', + 'metcalcpy': '4.0.0', + 'metdataio': '4.0.0', + 'metviewer': '7.0.0', + 'metexpress': None, + }, diff --git a/docs/Release_Guide/release_steps/metplus/coordinated_release_announcement.rst b/docs/Release_Guide/release_steps/metplus/coordinated_release_announcement.rst new file mode 100644 index 0000000000..2c7eb97a3d --- /dev/null +++ b/docs/Release_Guide/release_steps/metplus/coordinated_release_announcement.rst @@ -0,0 +1,7 @@ +Coordinated Release Announcement +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Send an email to the 'metplus-all' Google group indicating the completion of +the coordinated METplus release. This email serves as notification to the +project manager that an announcement can now be sent to the METplus +community. diff --git a/docs/Release_Guide/release_steps/metplus/create_release_reference_branch.rst b/docs/Release_Guide/release_steps/metplus/create_release_reference_branch.rst index c52a3fc716..b4bf001c28 100644 --- a/docs/Release_Guide/release_steps/metplus/create_release_reference_branch.rst +++ b/docs/Release_Guide/release_steps/metplus/create_release_reference_branch.rst @@ -1,6 +1,13 @@ Create Release Reference Branch ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. note:: + + These instructions only apply when creating the **first release candidate** + (rc1) development release. Skip this section for earlier beta (betaN) or later + release candidate (rc2+) development releases. + + * For METplus, the corresponding MET release must be created before starting this step. Specifically, a DockerHub tag on dtcenter/met that is named (X+6).Y-latest must already exist. For example, for METplus 4.1.0, the MET @@ -25,14 +32,17 @@ Update the version number Update the content that should go into the release version but remain unchanged in the develop branch. -Remove **-dev** from the version number: +Note that this step is mandatory because pushing a new main_vX.Y-ref branch +without any new commits will **not** trigger the GitHub Actions workflow to +create the output Docker data volumes. + +Remove **-dev** from the -rc1 version number: * As of METplus 4.0.0, we are naming releases with X.Y.Z format even if Z is 0. * As of METplus v4.0.0, the file containing the version number is located at - **metplus/VERSION** (in earlier releases, the file was located at - docs/version or doc/version). -* In the develop branch, the version should match the upcoming release - with -dev added to the end like X.Y.Z-betaN-dev, i.e. 4.0.0-beta1-dev + **metplus/VERSION**. +* The version should match the upcoming release with the -rc1 suffix, + like X.Y.Z-rc1 * Remove **-dev** from the version number so that it matches the release you are creating. @@ -66,8 +76,9 @@ Navigate to https://github.com/dtcenter/METplus/actions and verify that a Create main branch and push to GitHub """"""""""""""""""""""""""""""""""""" -* After the truth data volumes have been generated, create the main_vX.Y - branch off of the -ref branch. +* After the truth data volumes have been generated, create the 'main_vX.Y' + branch from 'main_vX.Y-ref' branch. All remaining development for the upcoming + official release occurs on this new 'main_vX.Y' branch. :: diff --git a/docs/Release_Guide/release_steps/metplus/update_manage_externals.rst b/docs/Release_Guide/release_steps/metplus/update_manage_externals.rst index adc564a516..d063602d37 100644 --- a/docs/Release_Guide/release_steps/metplus/update_manage_externals.rst +++ b/docs/Release_Guide/release_steps/metplus/update_manage_externals.rst @@ -43,18 +43,3 @@ Update build_components/Externals_stable.cfg Ensure the *tag* for each component is correct. It should match the format **vX.Y.Z** where X.Y.Z is the version of that component. For example, MET should be **v11.0.0** for METplus 5.0.0. - - -Update .github/parm/Externals_metdataio_stable.cfg -"""""""""""""""""""""""""""""""""""""""""""""""""" - -Ensure the *branch* value is correct. It should match the format -**main_vX.Y** where X.Y is the version of that component. -For example, METdataio should be **main_v2.0** for METplus 5.0.0. - -Update .github/parm/Externals_metplotcalcpy_stable.cfg -"""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Ensure the *branch* for each component is correct. It should match the format -**main_vX.Y** where X.Y is the version of that component. -For example, METplotpy and METcalcpy should be **main_v2.0** for METplus 5.0.0. diff --git a/docs/Release_Guide/release_steps/metplus/update_version.rst b/docs/Release_Guide/release_steps/metplus/update_version.rst index c62860d3f5..3f1955d112 100644 --- a/docs/Release_Guide/release_steps/metplus/update_version.rst +++ b/docs/Release_Guide/release_steps/metplus/update_version.rst @@ -1,11 +1,20 @@ Update Version Number for Release ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. note:: + + These instructions only apply when creating a **beta** + (betaN) development release. Skip this section for + release candidate (rcN) development releases. + Remove **-dev** from the version number: -* As of METplus v4.0.0-beta2, we are naming releases with X.Y.Z format even if Z is 0. ( As of METplus v4.0.0-beta4, the file containing the version number is located at metplus/VERSION (in earlier releases, the file was located at docs/version or doc/version) +* As of METplus v4.0.0-beta2, we are naming releases with X.Y.Z format even if Z is 0. + As of METplus v4.0.0-beta4, the file containing the version number is located at metplus/VERSION + (in earlier releases, the file was located at docs/version or doc/version). -* In the develop branch, the version should match the upcoming release with -dev added to the end like X.Y.Z-betaN-dev, i.e. 4.0.0-beta1-dev +* In the develop branch, the version should match the upcoming release with -dev + added to the end like X.Y.Z-betaN-dev, i.e. 4.0.0-beta1-dev * Remove **-dev** from the version number so that it matches the release you are creating. diff --git a/docs/Release_Guide/release_steps/metplus/update_version_official.rst b/docs/Release_Guide/release_steps/metplus/update_version_official.rst new file mode 100644 index 0000000000..360f05a6e1 --- /dev/null +++ b/docs/Release_Guide/release_steps/metplus/update_version_official.rst @@ -0,0 +1,7 @@ +Update Version Number for Official Release +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Update the version number for the official release: + +* The *VERSION* file is located in the **metplus** subdirectory. +* Remove the -rcN suffix so the version matches the format X.Y.0 diff --git a/docs/Release_Guide/release_steps/metplus/update_version_on_develop.rst b/docs/Release_Guide/release_steps/metplus/update_version_on_develop.rst index a34a1dc016..a5abe5c009 100644 --- a/docs/Release_Guide/release_steps/metplus/update_version_on_develop.rst +++ b/docs/Release_Guide/release_steps/metplus/update_version_on_develop.rst @@ -5,16 +5,15 @@ Switch to the develop branch and change metplus/VERSION value to the next release after this one with -dev added to the end. Releases will loosely following these names, but are subject to change: -+-------------------+-----------------------+ -| Release Version | New Develop Version | -+===================+=======================+ -| X.Y.Z-beta1 | X.Y.Z-beta2-dev | -+-------------------+-----------------------+ -| X.Y.Z-beta2 | X.Y.Z-beta3-dev | -+-------------------+-----------------------+ -| X.Y.Z-beta3 | X.Y.Z-rc1-dev | -+-------------------+-----------------------+ -| X.Y.Z-rc1 | X.Y.Z-dev | -+-------------------+-----------------------+ -| X.Y.Z | (X+1).0.0-beta1-dev | -+-------------------+-----------------------+ ++-------------------+----------------------------+ +| Release Version | New Develop Version | ++===================+============================+ +| X.Y.Z-beta1 | X.Y.Z-beta2-dev | ++-------------------+----------------------------+ +| X.Y.Z-beta2 | X.Y.Z-beta3-dev | ++-------------------+----------------------------+ +| X.Y.Z-beta3 | X.Y.Z-rc1-dev | ++-------------------+----------------------------+ +| X.Y.Z-rc1 | (X+1).0.0-beta1-dev OR | +| | **X.(Y+1).0-beta1-dev** | ++-------------------+----------------------------+ diff --git a/docs/Release_Guide/release_steps/metviewer/update_version.rst b/docs/Release_Guide/release_steps/metviewer/update_version.rst deleted file mode 100644 index a184af6f36..0000000000 --- a/docs/Release_Guide/release_steps/metviewer/update_version.rst +++ /dev/null @@ -1,10 +0,0 @@ -Update Version Number for Release -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Create the version number - -- In the METviewer/docs/version file, assign the version to the '__version__' keyword -- In the METviewer/build.xml file, assign the version in the 'dist' target -- In the METviewer/vebapp/metviewer/metviewer1.jsp file, assign the version to
and -- We are naming releases with X.Y.Z format even if Z is 0. -- In the develop branch, the version should match the upcoming release like X.Y.Z-betaN diff --git a/docs/Release_Guide/release_steps/metviewer/update_version_bugfix.rst b/docs/Release_Guide/release_steps/metviewer/update_version_bugfix.rst index 7c1b59ec5c..f7f5d93d97 100644 --- a/docs/Release_Guide/release_steps/metviewer/update_version_bugfix.rst +++ b/docs/Release_Guide/release_steps/metviewer/update_version_bugfix.rst @@ -1,12 +1,17 @@ -Update Version Number for Release -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Update Version Number +^^^^^^^^^^^^^^^^^^^^^ -Update the version number for the bugfix release: +* Update the version: + + * In the top level file 'build.xml', update the version value for the attribute name 'Specification-Version' under the 'dist' target name. + + * In the 'webapp/metviewer/metviewer1.jsp' file: + + * Update the 'TITLE' version value under 'HEAD'. + + * Update the version value in the :code:`span id="release"` section. + + * In 'docs/conf.py', update the 'version', 'release_year', and 'release_date' variables for the documentation. -* For METviewer, the file containing the version number is located at docs/version. -* If the current release is listed as X.Y.Z (major.minor.micro), the bugfix version should be X.Y.Z+1 - (i.e. increment the micro value by 1: 1.1.0 becomes 1.1.1) -* In the METviewer/build.xml file, assign the version in the 'dist' target -* In the METviewer/vebapp/metviewer/metviewer1.jsp file, assign the version to <div id='release'> and <title> diff --git a/docs/Release_Guide/release_steps/metviewer/update_version_development.rst b/docs/Release_Guide/release_steps/metviewer/update_version_development.rst new file mode 100644 index 0000000000..c3c51d1421 --- /dev/null +++ b/docs/Release_Guide/release_steps/metviewer/update_version_development.rst @@ -0,0 +1,19 @@ +Update Version Number +^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + + The software version number should have been updated for the next official release during development toward the **first development release** (e.g. beta1). + If not, follow the :ref:`official release instructions <update_version_official>` to do so prior to creating the first development release. + +* If the official release version has already been updated (e.g. beta2+ and rc1+), do the following: + + * In 'docs/conf.py', update the 'version', 'release_year', and 'release_date' variables for the documentation. + + * In the top level file 'build.xml', update the version value for the attribute name 'Specification-Version' under the 'dist' target name. + + * In the 'webapp/metviewer/metviewer1.jsp' file: + + * Update the 'TITLE' version value under 'HEAD'. + + * Update the version value in the :code:`span id="release"` section. diff --git a/docs/Release_Guide/release_steps/metviewer/update_version_official.rst b/docs/Release_Guide/release_steps/metviewer/update_version_official.rst index cfc694bfb1..778469e6ea 100644 --- a/docs/Release_Guide/release_steps/metviewer/update_version_official.rst +++ b/docs/Release_Guide/release_steps/metviewer/update_version_official.rst @@ -1,13 +1,15 @@ -Update Version Number for Release -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Update Version Number +^^^^^^^^^^^^^^^^^^^^^ -Remove **-dev** from the version number: +* Update the version: -* We are naming releases with X.Y.Z format even if Z is 0. -* The file containing the version number is located at docs/version. -* In the METviewer/build.xml file, assign the version in the 'dist' target -* In the METviewer/vebapp/metviewer/metviewer1.jsp file, assign the version to <div id='release'> and <title> + * If necessary, in the top level file 'build.xml', update the version value for the attribute name 'Specification-Version' under the 'dist' target name. -* Save and close the file. + * If necessary, in the 'webapp/metviewer/metviewer1.jsp' file: + + * Update the 'TITLE' version value under 'HEAD'. + + * Update the version value in the :code:`span id="release"` section. + + * In 'docs/conf.py', update the 'version', 'release_year', and 'release_date' variables for the documentation. -In docs/conf.py, update the version, release_year, and release_date variables for the documentation. diff --git a/docs/Release_Guide/release_steps/metviewer/update_version_on_develop.rst b/docs/Release_Guide/release_steps/metviewer/update_version_on_develop.rst index 8171373ec4..10a869a166 100644 --- a/docs/Release_Guide/release_steps/metviewer/update_version_on_develop.rst +++ b/docs/Release_Guide/release_steps/metviewer/update_version_on_develop.rst @@ -5,16 +5,15 @@ Change METviewer/docs/version __version__ value to the next release after this o to the end. Releases will loosely following these names, but are subject to change: -+-------------------+-----------------------+ -| Release Version | New Develop Version | -+===================+=======================+ -| X.Y.Z-beta1 | X.Y.Z-beta2-dev | -+-------------------+-----------------------+ -| X.Y.Z-beta2 | X.Y.Z-beta3-dev | -+-------------------+-----------------------+ -| X.Y.Z-beta3 | X.Y.Z-rc1-dev | -+-------------------+-----------------------+ -| X.Y.Z-rc1 | X.Y.Z-dev | -+-------------------+-----------------------+ -| X.Y.Z | (X+1).0.0-beta1-dev | -+-------------------+-----------------------+ ++-------------------+----------------------------+ +| Release Version | New Develop Version | ++===================+============================+ +| X.Y.Z-beta1 | X.Y.Z-beta2-dev | ++-------------------+----------------------------+ +| X.Y.Z-beta2 | X.Y.Z-beta3-dev | ++-------------------+----------------------------+ +| X.Y.Z-beta3 | X.Y.Z-rc1-dev | ++-------------------+----------------------------+ +| X.Y.Z-rc1 | (X+1).0.0-beta1-dev OR | +| | **X.(Y+1).0-beta1-dev** | ++-------------------+----------------------------+ diff --git a/docs/Release_Guide/release_steps/push_release_branch.rst b/docs/Release_Guide/release_steps/push_release_branch.rst deleted file mode 100644 index 589670b333..0000000000 --- a/docs/Release_Guide/release_steps/push_release_branch.rst +++ /dev/null @@ -1,8 +0,0 @@ -Push Release Branch to GitHub -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Push the release branch to GitHub:: - - release_branch=main_vX.Y - - git push origin $release_branch diff --git a/docs/Release_Guide/release_steps/update_release_notes_development.rst b/docs/Release_Guide/release_steps/update_release_notes_development.rst index a49edc7f92..928b332d0f 100644 --- a/docs/Release_Guide/release_steps/update_release_notes_development.rst +++ b/docs/Release_Guide/release_steps/update_release_notes_development.rst @@ -30,7 +30,9 @@ release. Open the following URL in a browser: * If you are creating a betaX release, add a new betaX section above the betaX-1 release. -* For the METplus repository, update the **development timeline**. +* For the METplus repository, update the **development timeline** under the + `METplus Release Information <https://metplus.readthedocs.io/en/develop/Users_Guide/release-notes.html#metplus-release-information>`_ + section. * If you are creating a beta1 release, add development timeline information with approximate dates for planned development cycles. diff --git a/docs/Release_Guide/release_steps/update_release_notes_official.rst b/docs/Release_Guide/release_steps/update_release_notes_official.rst index 5da7a676db..b13a3438f4 100644 --- a/docs/Release_Guide/release_steps/update_release_notes_official.rst +++ b/docs/Release_Guide/release_steps/update_release_notes_official.rst @@ -26,7 +26,8 @@ release. Open the following URL in a browser: * When creating the official release, combine the beta release sections into one section (i.e. "Version X.Y.Z release notes (YYYYMMDD)"). -* For the METplus repository, remove the **development timeline** information - prior to creating the official release. +* For the METplus repository, remove the **development timeline** information under the + `METplus Release Information <https://metplus.readthedocs.io/en/develop/Users_Guide/release-notes.html#metplus-release-information>`_ + section prior to creating the official release. * Commit changes and push to GitHub. diff --git a/docs/Release_Guide/release_steps/update_version_lookup.rst b/docs/Release_Guide/release_steps/update_version_lookup.rst new file mode 100644 index 0000000000..d465b1ee2a --- /dev/null +++ b/docs/Release_Guide/release_steps/update_version_lookup.rst @@ -0,0 +1,46 @@ +Update Version Lookup Table +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Modify the version lookup table in the METplus repository to include the +correct version. + +.. dropdown:: Instructions + + * Clone the METplus repository. + + Using SSH: + + .. parsed-literal:: + + git clone git@github.com:dtcenter/METplus + + Using HTTP: + + .. parsed-literal:: + + git clone https://github.com/dtcenter/METplus + + * Enter the METplus repository directory: + + .. parsed-literal:: + + cd METplus + + * Checkout the develop branch + + .. parsed-literal:: + + git checkout develop + + * Create a branch off of develop to update. + Include the name of the repository and version in the name. + + .. parsed-literal:: + + git checkout -b update_version_vX.Y.Z_repo + + * Open **metplus/component_versions.py** and increment the version for the + appropriate |projectRepo| entry. + + * Commit change, push to GitHub, and create a pull request to merge change + into the **develop** branch. diff --git a/docs/Users_Guide/existing_builds.rst b/docs/Users_Guide/existing_builds.rst new file mode 100644 index 0000000000..4c1f55ff65 --- /dev/null +++ b/docs/Users_Guide/existing_builds.rst @@ -0,0 +1,561 @@ + +Existing Builds +=============== + +The METplus team supports the installation of the **METplus software components** +on several operational research high performance computing platforms. This +includes installations at NCAR, NOAA, and other select community machines. +Pre-built METplus images on **DockerHub** are also provided. + +The **NCAR/RAL Common Installation** location under the **NCAR Machines** +drop down menu is only updated after official releases. No developmental releases +are installed in a common space on RAL machines. + +Please submit a new discussion in the +`METplus Components Discussion <https://github.com/dtcenter/METplus/discussions>`_ +forum if no instructions exist for the current release on a supported +platform and the release is needed on that platform. + +Select from the list below for instructions on using existing builds of +the METplus components' software packages. Please note that the commands to +load the METplus components assume the user is using bash. If an installation +is needed on a machine not listed here, please follow the +`installation instructions <https://met.readthedocs.io/en/latest/Users_Guide/installation.html>`_ +in the +`MET User’s Guide <https://met.readthedocs.io/en/latest>`_. + +.. dropdown:: NCAR machines + + .. dropdown:: CASPER + + | **NCAR MACHINE CASPER** (see + `Casper Information <https://ncar-hpc-docs.readthedocs.io/en/latest/compute-systems/casper/>`_) + | *Last Updated: October 30, 2024* + + * METplus-6.0.0-beta6 + + * METplus-6.0.0-beta6 Installation: + + * /glade/work/dtcrt/METplus/casper/components/METplus/installation + + * METplus-6.0 Sample Data: + + * /glade/work/dtcrt/METplus/data/components/METplus/METplus-6.0_sample_data + + * Users should create a file like + /glade/work/dtcrt/METplus/casper/components/METplus/installations/casper.dtcrt.conf + to set a personalized INPUT_BASE and OUTPUT_BASE. + + * To set up the environment run: + + .. code-block:: ini + + export TOP_DIR=/glade/work/dtcrt/METplus/casper/components + module use $TOP_DIR/METplus/installations/modulefiles + module load metplus/6.0.0-beta6 + + * MET-12.0.0-beta6 + + * MODULES: + + .. code-block:: ini + + export TOP_DIR=/glade/work/dtcrt/METplus/casper/components + module use $TOP_DIR/MET/installations/modulefiles + module load met/12.0.0-beta6 + + * METdataio-3.0.0-beta6 + + * MODULES: + + .. code-block:: ini + + export TOP_DIR=/glade/work/dtcrt/METplus/casper/components + module use $TOP_DIR/METdataio/installations/modulefiles + module load metdataio/3.0.0-beta6 + METcalcpy-3.0.0-beta6 + MODULES: + + * METcalcpy-3.0.0-beta6 + + * MODULES: + + .. code-block:: ini + + export TOP_DIR=/glade/work/dtcrt/METplus/casper/components + module use $TOP_DIR/METcalcpy/installations/modulefiles + module load metcalcpy/3.0.0-beta6 + + * METplotpy-3.0.0-beta6 + + * MODULES: + + .. code-block:: ini + + export TOP_DIR=/glade/work/dtcrt/METplus/casper/components + module use $TOP_DIR/METplotpy/installations/modulefiles + module load metplotpy/3.0.0-beta6 + + .. dropdown:: DERECHO + + .. warning:: + Users are encouraged to **run METplus on Casper** or submit to + the **develop queue on Derecho**. Submitting serial METplus jobs + to the main queue on Derecho may incur **up to 128 times** more charges + than necessary. Please see this + `Derecho Job-submission queues and charges <https://ncar-hpc-docs.readthedocs.io/en/latest/pbs/charging/#job-submission-queues-and-charges>`_ summary. + + | **NCAR MACHINE DERECHO** See `Derecho Information <https://ncar-hpc-docs.readthedocs.io/en/latest/compute-systems/derecho/>`_ + | *Last Updated:* + + * **MET-12.0.0-beta3** + + * MODULES: + + * module use + /glade/work/dtcrt/METplus/derecho/components/MET/installations/modulefiles + * **Installation coming soon** + + * **METplus-6.0.0-beta3** + + * METplus-6.0.0-beta3 Installation: + + * /glade/work/dtcrt/METplus/derecho/components/METplus/installations/METplus-6.0.0-beta3 + + * METplus-6.0 Sample Data: + + * /glade/work/dtcrt/METplus/data/components/METplus/METplus-6.0_sample_data + + * To set up the environment run: Users should create a file like + /glade/work/dtcrt/METplus/derecho/components/METplus/installations/derecho.dtcrt.conf + to set a personalized INPUT_BASE and OUTPUT_BASE. + + * module use /glade/work/dtcrt/METplus/derecho/components/METplus/installations/modulefiles + * **Installation coming soon** + + * **METcalcpy-3.0.0-beta3 / METplotpy-3.0.0-beta3** + + * MODULES: + + * module use + /glade/work/dtcrt/METplus/derecho/components/METcalcpy/installations/modulefiles + * module load metcalcpy/3.0.0-beta3 + * module use /glade/work/dtcrt/METplus/derecho/components/METplotpy/installations/modulefiles + * module load metplotpy/3.0.0-beta3 + + * **METdataio-3.0.0-beta3** + + * MODULES: + * module use /glade/work/dtcrt/METplus/derecho/components/METdataio/installations/modulefiles + * module load metdataio/3.0.0-beta3 + + .. dropdown:: NCAR/RAL Common Installation + + | **NCAR RAL MACHINES (STANDARD LOCATION)** + | *Last Updated:* + + * **METv12.0.0** + + * MET BUILD: + + * **METplus-12.0.0** + + * METplus INSTALLATION: Add text here + + .. dropdown:: NCAR/RAL Internal Development + + | **NCAR RAL MACHINES SENECA** + | **MET-12.0.0-beta1** + | *Last Updated:* + + * MET BUILD: + + | **NCAR RAL MACHINES KIOWA** + | **MET-12.0.0-beta1** + | *Last Updated:* + + * MET BUILD: + + | **NCAR RAL MACHINES MOHAWK** + | **METviewer-6.0.0-beta1** + | *Last Updated:* + + * LOCATION: + * URL: + +.. dropdown:: NOAA machines + + .. dropdown:: WCOSS2 + + | **NOAA machines Dogwood and Cactus (WCOSS2 - Cray)** + | *Last updated: September 19, 2024* + + * **MET v12.0.0-beta5 / METplus v6.0.0-beta5 / METplus Analysis Tools v3.0.0-beta5** + + * MODULES: + + .. code-block:: ini + + module reset + module use /apps/dev/modulefiles/ + module load ve/evs/2.0 + module use /apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304 + export HPC_OPT=/apps/ops/para/libs + module load gsl/2.7 + module load netcdf/4.7.4 + module load met/12.0.0-beta5 + module load metplus/6.0.0-beta5 + module load METplotpy/3.0.0-beta5 + module load METdataio/3.0.0-beta5 + module load METcalcpy/3.0.0-beta5 + + + .. dropdown:: HERA + + | **NOAA MACHINE HERA** + | *Last updated: October 21, 2024* + + * **METplus-6.0.0-beta6** + + * METplus-6.0.0-beta6 Installation + + * /contrib/METplus/METplus-6.0.0-beta6 + + * METplus-6.0 Sample Data + + * /scratch1/BMC/dtc/METplus/METplus-6.0_sample_data + + * Users should create a file like + /scratch1/BMC/dtc/METplus/hera.role-metplus.conf + to set a personalized INPUT_BASE and OUTPUT_BASE. + + * To use METplus run: + + .. code-block:: ini + + module load intel/2022.1.2 + module use /contrib/METplus/modulefiles + module load metplus/6.0.0-beta6 + + * **MET-12.0.0-beta6** + + * MODULES: + + .. code-block:: ini + + module load intel/2022.1.2 + module use -a /contrib/met/modulefiles/ + module load met/12.0.0-beta6 + + * **METcalcpy-3.0.0-beta6 / METplotpy-3.0.0-beta6** + + * MODULES: + + .. code-block:: ini + + module load intel/2022.1.2 + module use /contrib/METcalcpy/modulefiles + module load metcalcpy/3.0.0-beta6 + module use /contrib/METplotpy/modulefiles + module load metplotpy/3.0.0-beta6 + + * **METdataio-3.0.0-beta6** + + * MODULES: + + .. code-block:: ini + + module load intel/2022.1.2 + module use /contrib/METdataio/modulefiles + module load metdataio/3.0.0-beta6 + + .. dropdown:: HERCULES + + | **NOAA MACHINE HERCULES (MANAGED BY MSU)** + | *Last updated:* + + * **MET-12.0.0-beta3** + + * MODULES: + + * module load contrib + * module load intel-oneapi-compilers/2022.2.1 + * module load met/12.0.0-beta3 + + * **METplus-6.0.0-beta3** + + * METplus-6.0.0-beta3 Installation + * METplus-6.0 Sample Data + + * /work/noaa/ovp/jprestop/METplus/METplus-6.0_sample_data + + * To use METplus run: + + * module load contrib + * module load metplus/6.0.0-beta3 + * Users should create a file like + /work/noaa/ovp/user_name/METplus/hercules.user_name.conf + to set a personalized INPUT_BASE and OUTPUT_BASE. + + * **METcalcpy-3.0.0-beta3 / METplotpy-3.0.0-beta3** + + * MODULES: + + * module load contrib + * module load intel-oneapi-compilers/2022.2.1 + * module load metcalcpy/3.0.0-beta3 + * module load metplotpy/3.0.0-beta3 + + * PIP INSTALL: + + * python -m pip install --user tornado + * python -m pip install --user plotly + * python -m pip install --user kaleido + * python -m pip install --user xarray + * python -m pip install --user netcdf4 + * python -m pip install --user h5netcdf + + * **METdataio-3.0.0-beta3** + + * MODULES: + + * module load contrib + * module load intel-oneapi-compilers/2022.2.1 + * module load metdataio/3.0.0-beta3 + + + .. dropdown:: ORION + + | **NOAA MACHINE ORION (MANAGED BY MSU)** + | *Last updated: October 30, 2024* + + * **METplus-6.0.0-beta6** + + * METplus-6.0 Sample Data + + * /work/noaa/ovp/jprestop/METplus/METplus-6.0_sample_data + + * To use METplus run: Users should create a file like /work/noaa/ovp/user_name/METplus/orion.role-ovp.conf + to set a personalized INPUT_BASE and OUTPUT_BASE. + + .. code-block:: ini + + module load contrib + module load metplus/6.0.0-beta6 + + * **MET-12.0.0-beta6** + + * MODULES: + + .. code-block:: ini + + module load contrib + module load intel-oneapi-compilers/2022.2.1 + module load met/12.0.0-beta6 + + * **METcalcpy-3.0.0-beta6 / METplotpy-3.0.0-beta6** + + * MODULES: + + .. code-block:: ini + + module load contrib + module load intel-oneapi-compilers/2022.2.1 + module use /apps/contrib/modulefiles + module load metcalcpy/3.0.0-beta6 + module load metplotpy/3.0.0-beta6 + + * PIP INSTALL + + .. code-block:: ini + + python -m pip install --user tornado + python -m pip install --user plotly + python -m pip install --user kaleido + python -m pip install --user xarray + python -m pip install --user netcdf4 + python -m pip install --user h5netcdf + + * **METdataio-3.0.0-beta6** + + * MODULES: + + .. code-block:: ini + + module load contrib + module load intel-oneapi-compilers/2022.2.1 + module use /apps/contrib/modulefiles + module load metdataio/3.0.0-beta6 + + .. dropdown:: JET + + | **NOAA MACHINE JET** + | *Last updated: October 23, 2024* + + * **METplus-6.0.0-beta6** + + * METplus-6.0.0-beta6 Installation + + * /contrib/met/METplus/METplus-6.0.0-beta6 + + * METplus-6.0 Sample Data + + * /lfs5/HFIP/dtc-hurr/METplus/sample_data/METplus-6.0_sample_data + + * To use METplus run: Create a like /lfs5/HFIP/dtc-hurr/METplus/jet.role-metplus.conf + to set a personalized INPUT_BASE and OUTPUT_BASE. + + .. code-block:: ini + + module load intel/2022.1.2 + module load nco/4.9.1 + module load wgrib/1.8.1.0b + module load wgrib2/3.1.2_wmo + module load R/4.0.2 + module use /contrib/met/modulefiles + module load met/12.0.0-beta6 + module use /contrib/met/METplus/modulefiles + module load metplus/6.0.0-beta6 + + * **METv12.0.0-beta6** + + * MODULES: + + .. code-block:: ini + + module load intel/2022.1.2 + module load contrib + module use /contrib/met/modulefiles + module load met/12.0.0-beta6 + + * **METcalcpy-3.0.0-beta6 / METplotpy-3.0.0-beta6** + + * MODULES: + + .. code-block:: ini + + module load intel/2022.1.2 + module use /contrib/met/METcalcpy/modulefiles + module load metcalcpy/3.0.0-beta6 + module use /contrib/met/METplotpy/modulefiles + module load metplotpy/3.0.0-beta6 + + * **METdataio-3.0.0-beta6** + + * MODULES: + + .. code-block:: ini + + module load intel/2022.1.2 + module use /contrib/met/METdataio/modulefiles + module load metdataio/3.0.0-beta6 + + .. dropdown:: GAEA + + | **NOAA MACHINE GAEA** + | *Last Updated: July 16, 2024* + + * **METplus-6.0.0-beta5** + + * METplus-6.0.0-beta5 Installation + + * /usw/met/METplus/METplus-6.0.0-beta5 + + * METplus-6.0 Sample Data + + * /ncrc/proj/nggps_psd/user_name/projects/METplus/sample_data/METplus-6.0_sample_data + + * To use METplus run: Users should create a file like + /gpfs/f5/esrl/proj-shared/user_name/projects/METplus/gaea.metplus.conf + to set a personalized INPUT_BASE and OUTPUT_BASE. + + .. code-block:: ini + + module unload cray-libsci/23.02.1.1 + module load intel-oneapi/2022.0.2 + module use /usw/met/METplus/modulefiles + module load metplus/6.0.0-beta5 + + * **MET-12.0.0-beta5** + + * MODULES: + + .. code-block:: ini + + module unload cray-libsci/23.02.1.1 + module load intel-oneapi/2022.0.2 + module use -a /usw/met/modulefiles/ + module load met/12.0.0-beta5 + + + * **METcalcpy-3.0.0-beta5 / METplotpy-3.0.0-beta5** + + * MODULES: + + .. code-block:: ini + + module unload cray-libsci/23.02.1.1 + module load intel-oneapi/2022.0.2 + module use /usw/met/METcalcpy/modulefiles + module load metcalcpy/3.0.0-beta5 + module use /usw/met/METplotpy/modulefiles + module load metplotpy/3.0.0-beta5 + + * **METdataio-3.0.0-beta5** + + * MODULES: + + .. code-block:: ini + + module unload cray-libsci/23.02.1.1 + module load intel-oneapi/2022.0.2 + module use /usw/met/METdataio/modulefiles + module load metdataio/3.0.0-beta5 + +.. dropdown:: Community machines + + .. dropdown:: FRONTERA + + | **TEXAS ADVANCED COMPUTING CENTER (TACC) FRONTERA** + | *Last Updated:* + + * **MET-12.0.0-beta1** + + * MODULES: + + * **METplus-6.0.0-beta1** + + * METplus-6.0.0-beta1 Installation + * METplus-6.0 Sample Data + * To set up the environment run: + * Users should create a file like /work2/06612/tg859120/frontera/METplus/frontera.user_name.conf + to set a personalized INPUT_BASE and OUTPUT_BASE. + +.. dropdown:: Docker Hub + + | **MET** + | *Last Updated:* + + .. code-block:: ini + + docker pull dtcenter/met:12.0.0-beta1 + + `dtcenter/met Docker Hub <https://hub.docker.com/r/dtcenter/met>`_ + + | **METplus** + | *Last Updated:* + + .. code-block:: ini + + docker pull dtcenter/metplus:6.0.0-beta1 + + `dtcenter/metplus Docker Hub <https://hub.docker.com/r/dtcenter/metplus>`_ + +.. dropdown:: AWS + + | **METviewer v6.0.0-beta1** + | *Last Updated:* + + * LOCATION: + * URL: + diff --git a/docs/Users_Guide/getting_started.rst b/docs/Users_Guide/getting_started.rst index 4d25c4f28d..258fe225a1 100644 --- a/docs/Users_Guide/getting_started.rst +++ b/docs/Users_Guide/getting_started.rst @@ -377,11 +377,13 @@ the path to the METplus log file that was generated. * Review the :ref:`metplus_final.conf<metplus_final_conf>` file to see all of the settings that were used in the use case. +Containers +========== .. _metplus-docker: METplus in Docker -================= +----------------- METplus is available on DockerHub. The METplus Docker image includes all of the MET executables from the corresponding METplus Coordinated Release and @@ -400,7 +402,7 @@ on DockerHub. .. _docker-sample-input: Sample Input Data ------------------ +^^^^^^^^^^^^^^^^^ Sample input data for all of the use cases provided with the METplus wrappers are also available on DockerHub. These data are found in the @@ -426,7 +428,7 @@ The input data will be available inside the container under /data/input/METplus_Data. Running METplus in Docker -------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^ The run_metplus.py script is in the user's path inside the container. The use case configuration files can be found under @@ -451,3 +453,78 @@ the corresponding category can be run:: Please note that use cases that have additional Python package dependencies may not run successfully unless those packages are installed inside the container or obtained elsewhere. + + +.. _metplus-apptainer: + +METplus in Apptainer +-------------------- + +Apptainer (formerly Singularity) can also be used to run METplus. +Images can be pulled from DockerHub using Apptainer commands. + +If running on the NCAR Casper cluster, be sure to first load the Apptainer module via:: + + module load apptainer + +Navigate to a working directory and pull an image from DockerHub, e.g.:: + + apptainer pull docker://dtcenter/metplus:5.1-latest + +This will create a *.sif* file in the current directory, +e.g. **metplus_5.1-latest.sif**. + +See :ref:`metplus-docker` for information about available images on DockerHub. + +.. _apptainer-sample-input: + +Sample Input Data +^^^^^^^^^^^^^^^^^ + +The Docker data volumes that contain sample input data provided on DockerHub are +not compatible with Apptainer. Therefore, sample input data must be obtained from from the METplus Data website. +Navigate to the `METplus Data <https://dtcenter.ucar.edu/dfiles/code/METplus/METplus_Data>`_ +website. Next, navigate to the directory that corresponds to the vX.Y version that +will be run, e.g. `v5.1 <https://dtcenter.ucar.edu/dfiles/code/METplus/METplus_Data/v5.1>`_ +The names of sample data tar files include the corresponding use case category. + +Download one or more of the sample data tar files and uncompress them into +a local directory. The directory that contains the uncompressed sample data +will need to be mounted to the container to make the data available inside it. + +The **APPTAINER_BIND** environment variable can be used to mount directories to +the container. The local directory and the container directory will be separated +by a colon (:). + +Mount the local directory (./input_data in this example) to the +/data/input/METplus_Data directory inside the container:: + + export APPTAINER_BIND="./input_data:/data/input/METplus_Data" + + +Running METplus in Apptainer +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The METplus wrappers inside the container come pre-configured so that +:ref:`sys_conf_input_base` is set to /data/input/METplus_Data. +Mounting the local data directory to this path will allow the use cases +to be run without the need to modify the default configuration. + +The :ref:`sys_conf_output_base` value is set to /data/output by default. +This directory can also be mounted to a local directory so the output from +a use case run can be accessed locally. Multiple directory mounts can be specified +by separating them by comma:: + + export APPTAINER_BIND="./input_data:/data/input/METplus_Data,./output_data:/data/output" + +In this example, the directory **./output_data** will be populated with the +output from the use case. + +To run a use case, run the **apptainer exec** command with the **.sif** +file that was created by the **apptainer pull** command and the call to the +**run_metplus.py** script as arguments:: + + apptainer exec metplus_5.1-latest.sif /metplus/METplus/ush/run_metplus.py /metplus/METplus/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf + +Note that the full path to the run_metplus.py script and the configuration file for the use case you are running must be provided. + diff --git a/docs/Users_Guide/quicksearch.rst b/docs/Users_Guide/quicksearch.rst index c51d262d3d..80e1aef693 100644 --- a/docs/Users_Guide/quicksearch.rst +++ b/docs/Users_Guide/quicksearch.rst @@ -76,14 +76,15 @@ Use Cases by Application: | `Air Quality and Composition <../search.html?q=AirQualityAndCompAppUseCase&check_keywords=yes&area=default>`_ | `Climate <../search.html?q=ClimateAppUseCase&check_keywords=yes&area=default>`_ | `Clouds <../search.html?q=CloudsAppUseCase&check_keywords=yes&area=default>`_ - | `Short Range <../search.html?q=ShortRangeAppUseCase&check_keywords=yes&area=default>`_ | `Data Assimilation <../search.html?q=DataAssimilationAppUseCase&check_keywords=yes&area=default>`_ | `Ensemble <../search.html?q=EnsembleAppUseCase&check_keywords=yes&area=default>`_ + | `Fire <../search.html?q=FireAppUseCase&check_keywords=yes&area=default>`_ | `Land Surface <../search.html?q=LandSurfaceAppUseCase&check_keywords=yes&area=default>`_ | `Marine and Cryosphere <../search.html?q=MarineAndCryosphereAppUseCase&check_keywords=yes&area=default>`_ | `Medium Range <../search.html?q=MediumRangeAppUseCase&check_keywords=yes&area=default>`_ | `PBL <../search.html?q=PBLAppUseCase&check_keywords=yes&area=default>`_ | `Precipitation <../search.html?q=PrecipitationAppUseCase&check_keywords=yes&area=default>`_ + | `Short Range <../search.html?q=ShortRangeAppUseCase&check_keywords=yes&area=default>`_ | `Space Weather <../search.html?q=SpaceWeatherAppUseCase&check_keywords=yes&area=default>`_ | `Subseasonal to Seasonal <../search.html?q=S2SAppUseCase&check_keywords=yes&area=default>`_ | `Subseasonal to Seasonal: Madden-Julian Oscillation <../search.html?q=S2SMJOAppUseCase&check_keywords=yes&area=default>`_ @@ -95,14 +96,15 @@ Use Cases by Application: | **Air Quality and Composition**: *AirQualityAndCompAppUseCase* | **Climate**: *ClimateAppUseCase* | **Clouds**: *CloudsAppUseCase* - | **Short Range**: *ShortRangeAppUseCase* | **Data Assimilation**: *DataAssimilationAppUseCase* | **Ensemble**: *EnsembleAppUseCase* + | **Fire**: *FireAppUseCase* | **LandSurface**: *LandSurfaceAppUseCase* | **Marine and Cryosphere**: *MarineAndCryosphereAppUseCase* | **Medium Range**: *MediumRangeAppUseCase* | **PBL**: *PBLAppUseCase* | **Precipitation**: *PrecipitationAppUseCase* + | **Short Range**: *ShortRangeAppUseCase* | **Space Weather**: *SpaceWeatherAppUseCase* | **Subseasonal to Seasonal**: *S2SAppUseCase* | **Subseasonal to Seasonal: Madden-Julian Oscillation**: *S2SMJOAppUseCase* diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index 2c90caf871..4f4e0182a7 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -19,10 +19,12 @@ is broken down into the following development cycles for each component: 3. **Beta3** releases for the METplus components occurred around 2024-02-08. 4. **Beta4** releases for the METplus components occurred around 2024-04-17. 5. **Beta5** releases for the METplus components occurred around 2024-07-10. -6. **Beta6** releases are tentatively scheduled for 2024-10-16. +6. **Beta6** releases for the METplus components occurred around 2024-10-18. 7. **Release Candidate 1** releases are tentatively scheduled for 2024-11-13. 8. **Official Release** releases are tentatively scheduled for 2024-12-11. +.. include:: existing_builds.rst + .. _components-release-notes: METplus Components Release Note Links @@ -43,6 +45,65 @@ METplus Wrappers Release Notes When applicable, release notes are followed by the `GitHub issue <https://github.com/dtcenter/METplus/issues>`__ number which describes the bugfix, enhancement, or new feature. +Important issues are listed **in bold** for emphasis. + +METplus Version 6.0.0 beta6 Release Notes (2024-10-18) +------------------------------------------------------ + + .. dropdown:: Enhancement + + * Support for setting Point2Grid MET config variables + (`#2540 <https://github.com/dtcenter/METplus/issues/2540>`_) + * Support processing groups of forecast leads + (`#2612 <https://github.com/dtcenter/METplus/issues/2612>`_) + * Support separate climatology datasets for both the forecast and observation inputs + (`#2622 <https://github.com/dtcenter/METplus/issues/2622>`_) + * Support the new `-aggr` command line option in SeriesAnalysis wrapper + (`#2651 <https://github.com/dtcenter/METplus/issues/2651>`_) + * **Deprecate master_metplus.py** + (`#2714 <https://github.com/dtcenter/METplus/issues/2714>`_) + * Support for setting point_weight_flag and obtype_as_group_val_flag in PointStat and EnsembleStat + (`#2727 <https://github.com/dtcenter/METplus/issues/2727>`_) + + .. dropdown:: Bugfix + + * ASCII2NC file window bad default value and redundant initialization of wrappers + (`#2520 <https://github.com/dtcenter/METplus/issues/2520>`_) + * Inconsistent RUN_ID values when using instances + (`#2596 <https://github.com/dtcenter/METplus/issues/2596>`_) + * Fix GridStat_SeriesAnalysis _fcstNMME_obsCPC _seasonal_forecast use cases with poorly configured climatology settings + (`#2695 <https://github.com/dtcenter/METplus/issues/2695>`_) + * Improve SeriesAnalysis field info generation with regards to time + (`#2705 <https://github.com/dtcenter/METplus/issues/2705>`_) + * Clean up existing use cases wrt SonarQube + (`#2710 <https://github.com/dtcenter/METplus/issues/2710>`_) + + .. dropdown:: New Wrapper + + NONE + + .. dropdown:: New Use Case + + * Multivariate MODE for RRFS + (`#2647 <https://github.com/dtcenter/METplus/issues/2647>`_) + + .. dropdown:: Documentation + + * Develop an RST template for use cases + (`#918 <https://github.com/dtcenter/METplus/issues/918>`_) + * Add information to Contributor's Guide for adding new use cases that utilize METplotpy/METcalcpy/METdataio + (`#1882 <https://github.com/dtcenter/METplus/issues/1882>`_) + * Update Release Guide for MET releases to update version numbers in the installation.rst in the MET User's Guide + (`#2452 <https://github.com/dtcenter/METplus/issues/2452>`_) + * Update Documentation Overview and Conventions + (`#2489 <https://github.com/dtcenter/METplus/issues/2489>`_) + * Update the User Support section in the Contributor's Guide + (`#2679 <https://github.com/dtcenter/METplus/issues/2679>`_) + + .. dropdown:: Internal + + NONE + METplus Version 6.0.0 Beta 5 Release Notes (2024-07-10) ------------------------------------------------------- diff --git a/docs/_static/fire-GridStat_fcstWRF_obsMMA_fire_perimeter.png b/docs/_static/fire-GridStat_fcstWRF_obsMMA_fire_perimeter.png new file mode 100644 index 0000000000..6382e41649 Binary files /dev/null and b/docs/_static/fire-GridStat_fcstWRF_obsMMA_fire_perimeter.png differ diff --git a/docs/use_cases/met_tool_wrapper/TCDiag/TCDiag.py b/docs/use_cases/met_tool_wrapper/TCDiag/TCDiag.py index 7b2b1791ac..2a95b5f8b6 100644 --- a/docs/use_cases/met_tool_wrapper/TCDiag/TCDiag.py +++ b/docs/use_cases/met_tool_wrapper/TCDiag/TCDiag.py @@ -131,8 +131,9 @@ # Output for this use case will be found in met_tool_wrapper/TCDiag (relative to **OUTPUT_BASE**) # and will contain the following files: # -# * tc_diag_AL032023_GFSO_2023062012_cyl_grid_nest.nc -# * tc_diag_AL032023_GFSO_2023062012_cyl_grid_parent.nc +# * sal032023_gfso_doper_2023062012_cyl_grid_parent.nc +# * sal032023_gfso_doper_2023062012_diag.nc +# * sal032023_gfso_doper_2023062012_diag.dat # ############################################################################## diff --git a/docs/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter.py b/docs/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter.py new file mode 100644 index 0000000000..f1fe08fdd9 --- /dev/null +++ b/docs/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter.py @@ -0,0 +1,218 @@ +""" +GridStat: WRF and MMA Fire Perimeter +==================================== + +model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter.conf + +""" +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + +############################################################################## +# Scientific Objective +# -------------------- +# +# This use case demonstrates the use of GridStat to evaluate the performance of the +# fire spread forecast from the WRF-Fire model for the 416 fire in Colorado in 2018. +# Using available fire perimeter observations and WRF-Fire forecasts, +# contingency statistics are produced to evaluate the forecast performance +# relative to the observed fire. + +############################################################################## +# Version Added +# ------------- +# +# METplus version 6.0 + +############################################################################## +# Datasets +# -------- +# +# +# **Forecast:** WRF Fire +# +# **Observations:** Multimission Aircraft (MMA) +# +# **Location:** All of the input data required for this use case can be +# found in a sample data tarball. Each use case category will have +# one or more sample data tarballs. It is only necessary to download +# the tarball with the use case’s dataset and not the entire collection +# of sample data. Click here to access the METplus releases page and download sample data +# for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will +# set the value of INPUT_BASE. See :ref:`running-metplus` section for more information. +# + + +############################################################################## +# METplus Components +# ------------------ +# +# This use case uses the UserScript wrapper to run a Python script to that +# converts KML fire perimeter files to the poly line format that can be read by +# MET. Then it runs GenVxMask to create gridded MET NetCDF files from the poly +# files. Then it runs GridStat to process the WRF fire forecast files and the +# observation mask files. +# + +############################################################################## +# METplus Workflow +# ---------------- +# +# **Beginning time (INIT_BEG):** 2018-06-01 at 16Z +# +# **End time (INIT_END):** 2018-06-01 at 16Z +# +# **Increment between beginning and end times (INIT_INCREMENT):** None +# +# **Sequence of forecast leads to process (LEAD_SEQ):** 4 hour, 23 hour, 32 hour +# +# This use case processes 3 forecast leads initialized at 16Z on June 1, 2018, running 3 times. +# First, the UserScript tool is called. This tool calls a Python script to convert the +# kml shapefiles to polylines. Then, GenVxMask is run to convert the shapefile +# produced by UserScript to a netCDF mask, providing the observation input for GridStat. +# Finally, GridStat is called to compare the WRF-Fire forecast of fire area to the +# observation mask created using the GenVxMask tool. The GridStat call uses Python Embedding +# in order to read the WRF-Fire subgrid into GridStat. + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads the default configuration file, +# then it loads any configuration files passed to METplus via the command line +# e.g. parm/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter.conf +# + +############################################################################## +# MET Configuration +# ----------------- +# +# METplus sets environment variables based on user settings in the METplus +# configuration file. See :ref:`How METplus controls MET config file settings<metplus-control-met>` for more details. +# +# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** +# +# If there is a setting in the MET configuration file that is currently +# not supported by METplus you’d like to control, please refer to: +# :ref:`Overriding Unsupported MET config file settings<met-config-overrides>` +# +# .. dropdown:: GridStatConfig_wrapped +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped +# + +############################################################################## +# Python Embedding +# ---------------- +# +# This use case uses a Python embedding script to read the WRF-Fire forecast into GridStat. +# The script hard codes settings directly from WRF-Fire for the WRF-Fire netCDF variable name (FIRE_AREA) +# as well as the format of the WRF-Fire forecast file template. The input directory and valid time +# are read from the METplus script. Once the file(s) are found, the script sets attributes for +# initiation and valid times, variable name, level, and units in MET format and defines a grid +# based on the input WRF-Fire netCDF file's subgrid. The data from the FIRE_AREA variable and MET attributes +# are then printed to be read in by GridStat. +# +# .. dropdown:: parm/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter/read_wrfout_fire.py +# +# .. highlight:: python +# .. literalinclude:: ../../../../parm/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter/read_wrfout_fire.py +# +# For more information on the basic requirements to utilize Python Embedding in METplus, +# please refer to the MET User’s Guide section on `Python embedding <https://met.readthedocs.io/en/latest/Users_Guide/appendixF.html#appendix-f-python-embedding>`_ + +############################################################################## +# User Scripting +# -------------- +# +# This use case calls a Python script to read MMA fire perimeter .kml files and convert them +# into a poly line file that can be read by GenVxMask. The script hard codes the filename template +# for the .kml files and a valid time format. This valid time format is provided by the METplus configuration file. +# The script the sets up a variable for the previous valid time and an output file path. +# If a .kml file is not found for the current valid time, the script searches for a .kml file +# from the previous hours. Once a file is found, the Python script parses the input file to find +# the set of coordinates that define the fire perimeter. These coordinates are then written to +# a text file (.poly file) in the order longitude, latitude, elevation for each point. +# This .poly file is then ready to be read in by the GenVxMask tool. +# +# .. dropdown:: parm/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter/find_and_read_fire_perim_poly.py +# +# .. highlight:: python +# .. literalinclude:: ../../../../parm/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter/find_and_read_fire_perim_poly.py +# + + + +############################################################################## +# Running METplus +# --------------- +# +# Pass the use case configuration file to the run_metplus.py script +# along with any user-specific system configuration files if desired:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter.conf /path/to/user_system.conf +# +# See :ref:`running-metplus` for more information. +# + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# There are three groups of outputs. First, three polyline files resulting from the UserScript tool +# +# * poly/fire_perim_20180601_20.poly +# * poly/fire_perim_20180602_15.poly +# * poly/fire_perim_20180603_00.poly +# +# Second, three netCDF files resulting from the GenVxMask tool containing fire perimeter observations +# +# * mask/fire_perim_20180601_20_mask.nc +# * mask/fire_perim_20180602_15_mask.nc +# * mask/fire_perim_20180603_00_mask.nc +# +# Finally, six files from the GridStat tool run with Python Embedding. The .stat files contain the +# CTC and CTS line types for the FIRE_AREA variable at the given lead time (two lines total per stat file). +# The netCDF files contain the following five fields: lat (latitude), lon (longitude), +# FCST_FIRE_AREA_Z0_FULL (fire spread area from the WRF-Fire forecast), OBS_FIRE_PERIM_all_all_FULL (fire spread area from the .kml observations), +# and DIFF_FIRE_AREA_Z0_FIRE_PERIM_all_all_FULL (both the forecast and observed fire spread areas, including overlaps and differences). +# +# * grid_stat/2018060120/grid_stat_040000L_20180601_200000V.stat +# * grid_stat/2018060120/grid_stat_040000L_20180601_200000V_pairs.nc +# * grid_stat/2018060215/grid_stat_230000L_20180602_150000V.stat +# * grid_stat/2018060215/grid_stat_230000L_20180602_150000V_pairs.nc +# * grid_stat/2018060300/grid_stat_320000L_20180603_000000V.stat +# * grid_stat/2018060300/grid_stat_320000L_20180603_000000V_pairs.nc +# + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * UserScriptUseCase +# * GenVxMaskToolUseCase +# * GridStatToolUseCase +# * PythonEmbeddingFileUseCase +# * GRIB2FileUseCase +# * FireAppUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# +# +# sphinx_gallery_thumbnail_path = '_static/fire-GridStat_fcstWRF_obsMMA_fire_perimeter.png' diff --git a/docs/use_cases/model_applications/fire/README.rst b/docs/use_cases/model_applications/fire/README.rst new file mode 100644 index 0000000000..c9ec7c6bba --- /dev/null +++ b/docs/use_cases/model_applications/fire/README.rst @@ -0,0 +1,3 @@ +Fire +---- +Verification of fire weather-related atmospheric parameters and fire spread models diff --git a/docs/use_cases/use_case_documentation_template.py b/docs/use_cases/use_case_documentation_template.py index 99e1ce8e23..3429b3116c 100644 --- a/docs/use_cases/use_case_documentation_template.py +++ b/docs/use_cases/use_case_documentation_template.py @@ -1,10 +1,10 @@ -“”” - PointStat: Use Python embedding to calculate temperature terciles - ================================================================= +""" +PointStat: Use Python embedding to calculate temperature terciles +================================================================= - model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds.conf +model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds.conf -“”” +""" ############################################################################## # .. contents:: # :depth: 1 @@ -65,8 +65,11 @@ # [UPDATE_SECTION_CONTENT] # # **Beginning time (INIT_BEG):** 1982-01-01 +# # **End time (INIT_END):** 2010-01-02 +# # **Increment between beginning and end times (INIT_INCREMENT):** 1 year +# # **Sequence of forecast leads to process (LEAD_SEQ):** None # # With an increment of 1 year, all January 1st’s from 1982 to 2010 are processed @@ -90,7 +93,6 @@ # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf -# ############################################################################## # MET Configuration @@ -128,13 +130,14 @@ # 11-column format for point data. This list of lists is passed back # to PointStat for evaluation and the requested statistical output. The location # of the code is -# parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds/read_ASCAT_data.py +# +# .. dropdown:: parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds/read_ASCAT_data.py # +# .. highlight:: python +# .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds/read_ASCAT_data.py +# # For more information on the basic requirements to utilize Python Embedding in METplus, # please refer to the MET User’s Guide section on `Python embedding <https://met.readthedocs.io/en/latest/Users_Guide/appendixF.html#appendix-f-python-embedding>`_ -# -# .. highlight:: python -# .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds/read_ASCAT_data.py ############################################################################## # User Scripting @@ -152,8 +155,10 @@ # appropriate dimensions. Additional basic math is completed on the resulting arrays # to create the cross spectra values with the results being graphed. # -# .. highlight:: python -# .. literalinclude:: ../../../../parm/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra/cross_spectra_plot.py +# .. dropdown:: parm/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra/cross_spectra_plot.py +# +# .. highlight:: python +# .. literalinclude:: ../../../../parm/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra/cross_spectra_plot.py ############################################################################## # Running METplus @@ -161,9 +166,9 @@ # [UPDATE_SECTION_CONTENT] # # Pass the use case configuration file to the run_metplus.py script along -# with any user-specific system configuration files if desired: +# with any user-specific system configuration files if desired:: # -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds.conf /path/to/user_system.conf +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds.conf /path/to/user_system.conf # # See :ref:`running-metplus` for more information. @@ -173,11 +178,12 @@ # [UPDATE_SECTION_CONTENT] # # A successful run will output the following both to the screen and to the logfile:: +# # INFO: METplus has successfully finished running. # # Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. # Output for this use case will be found in -# {OUPUT_BASE}/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds +# {OUTPUT_BASE}/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds # and will contain the following files:: # # * grid_stat_198201_000000L_19700101_000000V_pairs.nc diff --git a/internal/scripts/dev_tools/generate_release_notes.py b/internal/scripts/dev_tools/generate_release_notes.py new file mode 100755 index 0000000000..224228c9f1 --- /dev/null +++ b/internal/scripts/dev_tools/generate_release_notes.py @@ -0,0 +1,129 @@ +#! /usr/bin/env python3 + +import sys +import os + +from github import Github +from datetime import datetime, timezone + +GITHUB_ORG = 'dtcenter' + +CATEGORIES = ( + 'Enhancement', + 'Bugfix', + 'New Wrapper', + 'New Use Case', + 'Documentation', + 'Internal', +) + + +def main(dev_name, dev_start_date, dev_end_date=datetime.today(), repo_name='METplus'): + token = os.getenv('GITHUB_TOKEN') + if not token: + print("ERROR: Must set GITHUB_TOKEN environment variable") + sys.exit(1) + + all_issues = get_all_issues_since_dev_start(token, repo_name, dev_start_date) + issues_by_category = get_issues_by_category(all_issues) + + print_banner('ADD THIS TO docs/Users_Guide/release-notes.rst') + + print_header(repo_name, dev_name, dev_end_date) + print_issues_by_category(repo_name, issues_by_category) + + print_banner('ADD THIS TO METplus Coordinated Release Acceptance Testing') + + print_release_testing(repo_name, dev_name, all_issues) + + +def get_all_issues_since_dev_start(token, repo_name, dev_start_date): + print(f"Finding issues in {GITHUB_ORG}/{repo_name} that were closed after {dev_start_date.strftime('%Y-%m-%d')}...") + github_obj = Github(token) + org = github_obj.get_organization(GITHUB_ORG) + repo = org.get_repo(repo_name) + + all_issues = repo.get_issues(state='closed', since=dev_start_date) + all_issues = [issue for issue in all_issues if issue.pull_request is None] + all_issues = [issue for issue in all_issues if not issue.title.startswith('Update Truth')] + all_issues.sort(key=lambda x: x.number) + return all_issues + + +def print_banner(msg): + print(f"\n{'*' * len(msg)}\n{msg}\n{'*' * len(msg)}\n") + + +def print_header(repo_name, dev_name, dev_end_date): + dev_fmt = dev_name.replace('-', '').replace('beta', 'Beta ').replace('rc', 'RC ') + header = f"{repo_name} Version {dev_fmt} Release Notes ({dev_end_date.strftime('%Y-%m-%d')})" + print(header) + print('-' * len(header)) + + +def print_issues_by_category(repo_name, issues_by_category): + for category, issues in issues_by_category.items(): + print() + if category != 'none': + print(f" .. dropdown:: {category}\n") + else: + print('COULD NOT PARSE CATEGORY FROM THESE:\n') + if issues is None: + print(' NONE') + continue + for issue in issues: + title = issue.title.removeprefix(category).lstrip(' :') + num = issue.number + print(f" * {title}") + print(f" (`#{num} <https://github.com/{GITHUB_ORG}/{repo_name}/issues/{num}>`_)") + + +def get_issues_by_category(all_issues): + issues_by_category = dict.fromkeys(CATEGORIES) + issues_by_category['none'] = [] + for issue in all_issues: + found_cat = False + for category in CATEGORIES: + if issue.title.startswith(f"{category}:"): + if issues_by_category[category] is None: + issues_by_category[category] = [] + issues_by_category[category].append(issue) + found_cat = True + break + if not found_cat: + if issues_by_category['none'] is None: + issues_by_category['none'] = [] + issues_by_category['none'].append(issue) + return issues_by_category + + +def print_release_testing(repo_name, dev_name, all_issues): + dev_info = dev_name.split('-')[1] + for issue in all_issues: + num = issue.number + print(f"| **OPEN** || [#{num}](https://github.com/{GITHUB_ORG}/{repo_name}/issues/{num}) | {dev_info} |||") + + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('dev_name', + help="name of development cycle, e.g. 6.0.0-beta6") + parser.add_argument('start_date', + help="YYYYMMDD date when development cycle started") + parser.add_argument('-r', '--repo', + default='METplus', + help="Repository to parse, default is METplus") + args = parser.parse_args() + + ymd = args.start_date + try: + year = int(ymd[:4]) + month = int(ymd[4:6]) + day = int(ymd[6:8]) + except ValueError: + print('ERROR: Argument must be YYYYMMDD') + sys.exit(1) + + start_date = datetime(year, month, day, tzinfo=timezone.utc) + main(repo_name=args.repo, dev_name=args.dev_name, dev_start_date=start_date) diff --git a/internal/scripts/docker/Dockerfile.metplus-analysis b/internal/scripts/docker/Dockerfile.metplus-analysis new file mode 100644 index 0000000000..5104d9cc36 --- /dev/null +++ b/internal/scripts/docker/Dockerfile.metplus-analysis @@ -0,0 +1,16 @@ +ARG METPLUS_BASE_REPO=metplus +ARG METPLUS_BASE_TAG + +FROM dtcenter/${METPLUS_BASE_REPO}:${METPLUS_BASE_TAG} +LABEL org.opencontainers.image.authors="mccabe@ucar.edu" + +ARG METDATAIO_VERSION +ARG METCALCPY_VERSION +ARG METPLOTPY_VERSION + +RUN git clone --branch "$METDATAIO_VERSION" https://github.com/dtcenter/METdataio \ + && pip install -r METdataio/requirements.txt && pip install METdataio/. \ + && git clone --branch "$METCALCPY_VERSION" https://github.com/dtcenter/METcalcpy \ + && pip install -r METcalcpy/requirements.txt && pip install METcalcpy/. \ + && git clone --branch "$METPLOTPY_VERSION" https://github.com/dtcenter/METplotpy \ + && pip install -r METplotpy/requirements.txt && pip install METplotpy/. diff --git a/internal/scripts/docker/hooks/build b/internal/scripts/docker/hooks/build deleted file mode 100644 index 472551c107..0000000000 --- a/internal/scripts/docker/hooks/build +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# get version, use develop or X+6.Y for MET_TAG -met_tag=`$(dirname $DOCKERFILE_PATH)/hooks/get_met_version` - -echo $met_tag - -MET_DOCKER_REPO=met -if [ "$met_tag" == "develop" ]; then - MET_DOCKER_REPO=met-dev -fi - -docker build -t $IMAGE_NAME \ - --build-arg SOURCE_VERSION=$SOURCE_BRANCH \ - --build-arg MET_TAG=$met_tag \ - --build-arg MET_DOCKER_REPO=$MET_DOCKER_REPO \ - . diff --git a/internal/scripts/docker/hooks/get_met_version b/internal/scripts/docker/hooks/get_met_version deleted file mode 100755 index f580b55cb9..0000000000 --- a/internal/scripts/docker/hooks/get_met_version +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# get version, use develop or X+6.Y for MET_BRANCH -version_file=$(dirname $DOCKERFILE_PATH)/../../../metplus/VERSION - -if cat $version_file | egrep -q '^[0-9.]+$'; then - let major=$(cut -d '.' -f1 $version_file)+6 - minor=$(cut -d '.' -f2 $version_file ) - met_branch=$major"."$minor"-latest" -else - met_branch=develop -fi - -echo $met_branch diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index ab3a8ae341..a87edc24ea 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -4,6 +4,7 @@ sonar.projectName=METplus sonar.projectVersion=SONAR_PROJECT_VERSION sonar.branch.name=SONAR_BRANCH_NAME sonar.sources=docs,internal,manage_externals,metplus,parm,ush +sonar.exclusions=metplus/scripts/** sonar.coverage.exclusions=internal/tests/**,parm/**,metplus/parm/**,internal/scripts/**,manage_externals/**,docs/**,metplus/produtil/**,ush/**,metplus/wrappers/cyclone_plotter_wrapper.py sonar.python.coverage.reportPaths=coverage.xml sonar.sourceEncoding=UTF-8 diff --git a/internal/tests/pytests/component_versions/test_component_versions.py b/internal/tests/pytests/component_versions/test_component_versions.py new file mode 100644 index 0000000000..1d3b0b342a --- /dev/null +++ b/internal/tests/pytests/component_versions/test_component_versions.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import pytest + +from metplus import component_versions + +@pytest.mark.parametrize( + 'component, version, expected_result', [ + ('met', '11.1.1', '5.1'), + ('MET', '11.1.1', '5.1'), + ('met', '11.1', '5.1'), + ('met', '11.1.Z', '5.1'), + ('METcalcpy', '3.0.0', '6.0'), + ('metcalcpy', 'main_v3.0', '6.0'), + ('metcalcpy', 'v3.0.0', '6.0'), + ('metcalcpy', 'v3.0.0-beta3', '6.0'), + ('metcalcpy', 'v3.0.0-rc1', '6.0'), + ('METplus', '6.0-latest', '6.0'), + ('METplus', '3.0-latest', None), + ] +) +@pytest.mark.util +def test_get_coordinated_version(component, version, expected_result): + assert component_versions.get_coordinated_version(component, version) == expected_result + + +@pytest.mark.parametrize( + 'input_component, input_version, output_component, output_format, expected_result', [ + # get MET version for Docker dtcenter/metplus + ('metplus', '5.1.0', 'met', '{X}.{Y}.{Z}{N}', '11.1.1'), + ('metplus', '5.1.0-beta3', 'met', '{X}.{Y}.{Z}{N}', '11.1.1-beta3'), + ('metplus', '5.1.0-rc1', 'met', '{X}.{Y}.{Z}{N}', '11.1.1-rc1'), + ('metplus', '5.1-latest', 'met', '{X}.{Y}{N}', '11.1-latest'), + ('metplus', '5.1.0-beta3-dev', 'met', '{X}.{Y}.{Z}{N}', 'develop'), + # get METplus Analysis versions for Docker dtcenter/metplus-analysis + ('METplus', '5.1.0', 'metplotpy', 'v{X}.{Y}.{Z}{N}', 'v2.1.0'), + ('metplus', '5.1.0-beta3', 'METplotpy', 'v{X}.{Y}.{Z}{N}', 'v2.1.0-beta3'), + ('metplus', '5.1.0-dev', 'METplotpy', 'v{X}.{Y}.{Z}{N}', 'develop'), + ('metplus', '5.1.0-rc1', 'metplotpy', 'v{X}.{Y}.{Z}{N}', 'v2.1.0-rc1'), + ('metplus', '5.1.0-beta3-dev', 'metplotpy', 'v{X}.{Y}.{Z}{N}', 'develop'), + # get METplus main branch to trigger workflow from other repos, e.g. MET + ('MET', 'main_v11.1', 'METplus', 'main_v{X}.{Y}', 'main_v5.1'), + ('MET', 'main_v11.1-ref', 'METplus', 'main_v{X}.{Y}', 'main_v5.1'), + # get latest bugfix version from main branch or X.Y version + ('MET', 'main_v11.1', 'MET', '{X}.{Y}.{Z}{N}', '11.1.1'), + ('MET', '11.1.Z', 'MET', '{X}.{Y}.{Z}{N}', '11.1.1'), + ] +) +@pytest.mark.util +def test_get_component_version(input_component, input_version, output_component, output_format, expected_result): + assert component_versions.get_component_version(input_component, input_version, output_component, output_format) == expected_result + + +@pytest.mark.parametrize( + 'input_version, get_dev, expected_result', [ + ('5.1.0', True, 'v5.1.0'), + ('5.1.0', False, 'v5.1.0'), + ('5.1.0-beta3', True, 'v5.1.0-beta3'), + ('5.1.0-beta3', False, 'develop'), + ('5.1.0-rc1', True, 'v5.1.0-rc1'), + ('5.1.0-rc1', False, 'develop'), + ] +) +@pytest.mark.util +def test_get_component_version_get_dev(input_version, get_dev, expected_result): + assert component_versions.get_component_version('METplus', input_version, 'METplus', get_dev=get_dev) == expected_result diff --git a/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py b/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py index a0291936ac..17afa36267 100644 --- a/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py +++ b/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py @@ -377,7 +377,7 @@ def test_pcp_combine_derive(metplus_config, get_test_data_dir, config_overrides, stat_list = 'sum,min,max,range,mean,stdev,vld_count' fcst_name = 'APCP' fcst_level = 'A03' - fcst_fmt = f'-field \'name="{fcst_name}"; level="{fcst_level}";\'' + fcst_fmt = f'\'name="{fcst_name}"; level="{fcst_level}";\'' config = metplus_config test_data_dir = get_test_data_dir() @@ -427,13 +427,13 @@ def test_pcp_combine_derive(metplus_config, get_test_data_dir, config_overrides, verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" out_dir = wrapper.c_dict.get('FCST_OUTPUT_DIR') expected_cmds = [(f"{app_path} -derive {stat_list} " - f"{fcst_input_dir}/2005080700/24.tm00_G212 " - f"{fcst_input_dir}/2005080700/21.tm00_G212 " - f"{fcst_input_dir}/2005080700/18.tm00_G212 " - f"{fcst_input_dir}/2005080700/15.tm00_G212 " - f"{fcst_input_dir}/2005080700/12.tm00_G212 " - f"{fcst_input_dir}/2005080700/09.tm00_G212 " - f"{fcst_fmt} {extra_fields}" + f"{fcst_input_dir}/2005080700/24.tm00_G212 {fcst_fmt} " + f"{fcst_input_dir}/2005080700/21.tm00_G212 {fcst_fmt} " + f"{fcst_input_dir}/2005080700/18.tm00_G212 {fcst_fmt} " + f"{fcst_input_dir}/2005080700/15.tm00_G212 {fcst_fmt} " + f"{fcst_input_dir}/2005080700/12.tm00_G212 {fcst_fmt} " + f"{fcst_input_dir}/2005080700/09.tm00_G212 {fcst_fmt} " + f"{extra_fields}" f"{out_dir}/2005080700_f24_A18.nc {verbosity}"), ] diff --git a/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py b/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py index e69a324418..6ba6280a80 100644 --- a/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py +++ b/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py @@ -271,19 +271,19 @@ def test_plot_point_obs(metplus_config, config_overrides, env_var_values): out_dir = wrapper.c_dict.get('OUTPUT_DIR') expected_cmds = [ (f"{app_path} {verbosity} " - f'"{input_dir}/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc" ' + f'{input_dir}/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc ' f"{out_dir}/nam_and_ndas.20120409.t12z.prepbufr_CONFIG.ps"), (f"{app_path} {verbosity} " - f'"{input_dir}/pb2nc/ndas.20120410.t00z.prepbufr.tm00.nc" ' + f'{input_dir}/pb2nc/ndas.20120410.t00z.prepbufr.tm00.nc ' f"{out_dir}/nam_and_ndas.20120410.t00z.prepbufr_CONFIG.ps"), ] # add -point_obs argument if template has 2 items if ('PLOT_POINT_OBS_INPUT_TEMPLATE' in config_overrides and len(config_overrides['PLOT_POINT_OBS_INPUT_TEMPLATE'].split(',')) > 1): - common_str = f' -point_obs "{input_dir}/ascii2nc/trmm_' - expected_cmds[0] += f'{common_str}2012040912_3hr.nc"' - expected_cmds[1] += f'{common_str}2012041000_3hr.nc"' + common_str = f' -point_obs {input_dir}/ascii2nc/trmm_' + expected_cmds[0] += f'{common_str}2012040912_3hr.nc' + expected_cmds[1] += f'{common_str}2012041000_3hr.nc' # add -plot_grid argument if provided if 'PLOT_POINT_OBS_GRID_INPUT_TEMPLATE' in config_overrides: diff --git a/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py index 9dc06f9f88..6483aa4826 100644 --- a/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py +++ b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py @@ -173,6 +173,12 @@ def set_minimum_config_settings(config): 'MODEL1': '{custom}', 'MODEL_LIST': '{custom}'}, {'METPLUS_MODEL': 'model = ["CUSTOM_MODEL"];'}), + # 17 - fcst_lev + ({'FCST_LEVEL_LIST': 'R5'}, + {'METPLUS_FCST_LEVEL': 'fcst_lev = ["R5"];'}), + # 17 - obs_lev + ({'OBS_LEVEL_LIST': 'R7'}, + {'METPLUS_OBS_LEVEL': 'obs_lev = ["R7"];'}), ] ) diff --git a/internal/tests/use_cases/all_use_cases.txt b/internal/tests/use_cases/all_use_cases.txt index ef58c7ff60..1d6443d998 100644 --- a/internal/tests/use_cases/all_use_cases.txt +++ b/internal/tests/use_cases/all_use_cases.txt @@ -88,6 +88,9 @@ Category: data_assimilation 0::StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface::model_applications/data_assimilation/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface.conf 1::StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed::model_applications/data_assimilation/StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed.conf:: py_embed +Category: fire +0::GridStat_fcstWRF_obsMMA_fire_perimeter::model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter.conf::py_embed + Category: land_surface 0::PointStat_fcstCESM_obsFLUXNET2015_TCI:: model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf:: metplotpy_env, py_embed diff --git a/metplus/VERSION b/metplus/VERSION index 4ba6860f7e..c50c8b94c7 100644 --- a/metplus/VERSION +++ b/metplus/VERSION @@ -1 +1 @@ -6.0.0-beta6-dev +6.0.0-rc1-dev \ No newline at end of file diff --git a/metplus/component_versions.py b/metplus/component_versions.py new file mode 100755 index 0000000000..be8d2c406f --- /dev/null +++ b/metplus/component_versions.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 + +# Dictionary to track version numbers for each METplus component +# The key of each entry is the coordinated release version, e.g. 6.0 +# The value is another dictionary where the key is the METplus component name +# (in lower-case) and the value is a string that contains the latest +# X.Y.Z version of that component or None if no version is available. +# Add entries for a new coordinated release at the top of the dictionary. +# The versions should be updated when a bugfix release is created. +# METexpress does not include a release for the beta versions, so the value for +# METexpress should be set to None until the official coordinated release +# has been created. + +import sys + +VERSION_LOOKUP = { + '6.0': { + 'metplus': '6.0.0', + 'met': '12.0.0', + 'metplotpy': '3.0.0', + 'metcalcpy': '3.0.0', + 'metdataio': '3.0.0', + 'metviewer': '6.0.0', + 'metexpress': None, + }, + '5.1': { + 'metplus': '5.1.0', + 'met': '11.1.1', + 'metplotpy': '2.1.0', + 'metcalcpy': '2.1.0', + 'metdataio': '2.1.0', + 'metviewer': '5.1.0', + 'metexpress': '5.3.3', + }, +} + +# assumes the 2nd highest version in the table is the latest +# the highest version in the table is in development +LATEST_OFFICIAL_RELEASE = sorted(VERSION_LOOKUP.keys(), reverse=True)[1] + +DEFAULT_OUTPUT_FORMAT = "v{X}.{Y}.{Z}{N}" + +def get_component_version(input_component, input_version, output_component, + output_format=DEFAULT_OUTPUT_FORMAT, get_dev=True): + """!Get the version of a requested METplus component given another METplus + component and its version. Parses out X.Y version numbers of input version + to find desired version. Optionally specific format of output content. + If input version ends with "-dev", then return "develop". + + @param input_component name of METplus component to use to find version, + e.g. MET, METplus, or METplotpy (case-insensitive). + @param input_version version of input_component to search. + @param output_component name of METplus component to obtain version number + @param output_format (optional) format to use to output version number. + {X}, {Y}, and {Z} will be replaced with x, y, and z version numbers from + X.Y.Z. {N} will be replaced with development version if found in the + input version, e.g. "-beta3" or "-rc1" + @param get_dev (optional) if True, get corresponding -beta or -rc version. + If False, return "develop" if input is beta or rc. + @returns string of requested version number, or "develop" if input version + ends with "-dev", or None if version number could not be determined. + """ + if ('-dev' in input_version or + (not get_dev and any(ext in input_version for ext in ['-beta', '-rc']))): + return 'develop' + coord_version = get_coordinated_version(input_component, input_version) + versions = VERSION_LOOKUP.get(coord_version) + if versions is None: + return None + output_version = versions.get(output_component.lower()) + if output_version is None: + return None + x, y, z = output_version.split('.') + dev_version = input_version.split('-')[1:] + dev_version = '' if not dev_version else f"-{dev_version[0]}" + return output_format.format(X=x, Y=y, Z=z, N=dev_version) + + +def get_coordinated_version(component, version): + """!Get coordinated release version number based on the X.Y version number + of a given METplus component. + + @param component name of METplus component to search (case-insensitive) + @param version number of version to search for. Can be formatted with main_v + prefix and development release info, e.g. main_vX.Y or X.Y.Z-beta3. + @returns string of coordinated release version number X.Y or None. + """ + # remove main_v or v prefix, remove content after dash + search_version = version.removeprefix('main_').lstrip('v').split('-')[0] + # get X.Y only + search_version = '.'.join(search_version.split('.')[:2]) + # look for component version that begins with search version + for coord_version, versions in VERSION_LOOKUP.items(): + if versions.get(component.lower()).startswith(search_version): + return coord_version + return None + +def main(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('-i', '--input_component', + default='metplus', + help='Name of METplus component to use to find version.' + ' Default is METplus.') + parser.add_argument('-v', '--input_version', + default=LATEST_OFFICIAL_RELEASE, + help='version of input_component to search.' + ' Default is latest official release') + parser.add_argument('-o', '--output_component', required=True, + help='name of METplus component to obtain version') + parser.add_argument('-f', '--output_format', + default=DEFAULT_OUTPUT_FORMAT, + help='format to use to output version number.' + '{X}, {Y}, and {Z} will be replaced with x, y, and' + ' z version numbers from X.Y.Z. {N} will be ' + 'replaced with development version if found in the' + ' input version, e.g. "-beta3" or "-rc1".' + ' Default is v{X}.{Y}.{Z}{N}') + parser.add_argument('--get_dev_version', action=argparse.BooleanOptionalAction, + default=True, + help='If True, get corresponding -beta or -rc version. ' + 'If False, return develop if development version.') + args = parser.parse_args() + return get_component_version(args.input_component, args.input_version, + args.output_component, args.output_format, + args.get_dev_version) + + +if __name__ == "__main__": + version = main() + if not version: + sys.exit(1) + + print(version) diff --git a/metplus/wrappers/pcp_combine_wrapper.py b/metplus/wrappers/pcp_combine_wrapper.py index e00b3153cf..e916813d3d 100755 --- a/metplus/wrappers/pcp_combine_wrapper.py +++ b/metplus/wrappers/pcp_combine_wrapper.py @@ -531,8 +531,8 @@ def setup_derive_method(self, time_info, lookback, data_src): files_found = [] for input_file in input_files: - # exclude field info and set it with -field self.args.append(input_file) + self.args.append(field_info) files_found.append((input_file, field_info)) else: @@ -540,7 +540,7 @@ def setup_derive_method(self, time_info, lookback, data_src): files_found = self.get_accumulation(time_info, lookback, data_src, - field_info_after_file=False) + field_info_after_file=True) if not files_found: self.missing_input_count += 1 msg = ( @@ -553,9 +553,6 @@ def setup_derive_method(self, time_info, lookback, data_src): self.log_error(msg) return None - # set -field name and level from first file field info - self.args.append(f'-field {files_found[0][1]}') - self._handle_input_thresh_argument(data_src) return files_found @@ -652,7 +649,8 @@ def get_accumulation(self, time_info, accum, data_src, accum_relative = get_relativedelta(accum, 'S') # using 1 hour for now smallest_input_accum = min( - [lev['amount'] for lev in self.c_dict['ACCUM_DICT_LIST']] + [ti_get_seconds_from_relativedelta(lev['amount'], search_time - accum_relative) + for lev in self.c_dict['ACCUM_DICT_LIST']] ) if smallest_input_accum == 9999999: smallest_input_accum = 3600 @@ -746,8 +744,9 @@ def _add_file_and_field_info_to_args(self, search_file, field_info, def _find_file_for_accum(self, accum_dict, total_accum, time_info, search_time, data_src, custom): - if (accum_dict['amount'] > total_accum and - accum_dict['template'] is None): + # get number of seconds from relativedelta accum amount using search time + accum_seconds = ti_get_seconds_from_relativedelta(accum_dict['amount'], search_time) + if accum_seconds > total_accum and accum_dict['template'] is None: return None, None, None self.c_dict['SUPPRESS_WARNINGS'] = True @@ -774,11 +773,12 @@ def _find_file_for_accum(self, accum_dict, total_accum, time_info, "than remaining accumulation.") return None, None, None else: - accum_amount = accum_dict['amount'] + accum_amount = accum_seconds search_time_info = { 'valid': search_time, 'lead': lead, + 'custom': custom, } field_info = self.get_field_string( time_info=search_time_info, @@ -808,7 +808,8 @@ def get_lowest_fcst_file(self, valid_time, data_src, custom): self.c_dict[data_src+'_MAX_FORECAST'], 'H' ) smallest_input_accum = min( - [lev['amount'] for lev in self.c_dict['ACCUM_DICT_LIST']] + [ti_get_seconds_from_relativedelta(lev['amount'], valid_time) + for lev in self.c_dict['ACCUM_DICT_LIST']] ) # if smallest input accumulation is greater than an hour, search hourly @@ -862,6 +863,9 @@ def find_input_file(self, init_time, valid_time, search_accum, data_src, custom): lead = 0 in_template = self.c_dict[data_src+'_INPUT_TEMPLATE'] + # use rel_time as offset to compute seconds from + # relativedelta search_accum for time_info level -- can be init or valid + rel_time = valid_time if ('{lead?' in in_template or ('{init?' in in_template and '{valid?' in in_template)): @@ -872,15 +876,17 @@ def find_input_file(self, init_time, valid_time, search_accum, data_src, # ti_calculate cannot currently handle both init and valid lead = (valid_time - init_time).total_seconds() input_dict = {'init': init_time, 'lead': lead} + rel_time = init_time else: if self.c_dict[f'{data_src}_CONSTANT_INIT']: input_dict = {'init': init_time} + rel_time = init_time else: input_dict = {'valid': valid_time} time_info = ti_calculate(input_dict) time_info['custom'] = custom - time_info['level'] = int(search_accum) + time_info['level'] = ti_get_seconds_from_relativedelta(search_accum, rel_time) input_path = self.find_data(time_info, data_type=data_src, return_list=True, mandatory=False) if input_path: @@ -1005,8 +1011,8 @@ def _build_input_accum_list(self, data_src, time_info): template = accum accum = '9999999S' - # convert accum amount to seconds from time string - amount = get_seconds_from_string(accum, 'H', time_info['valid']) + # convert accum amount to relativedelta from time string + amount = get_relativedelta(accum, default_unit='H') accum_dict_list.append({'amount': amount, 'name': name, diff --git a/metplus/wrappers/plot_point_obs_wrapper.py b/metplus/wrappers/plot_point_obs_wrapper.py index 97f62a2880..e5222264b2 100755 --- a/metplus/wrappers/plot_point_obs_wrapper.py +++ b/metplus/wrappers/plot_point_obs_wrapper.py @@ -180,7 +180,7 @@ def create_c_dict(self): def get_command(self): return (f"{self.app_path} -v {self.c_dict['VERBOSITY']}" - f' "{self.infiles[0]}" {self.get_output_path()}' + f' {self.infiles[0]} {self.get_output_path()}' f" {' '.join(self.args)}") def find_input_files(self, time_info): @@ -222,7 +222,7 @@ def set_command_line_arguments(self, time_info): """ # if more than 1 input file was found, add them with -point_obs for infile in self.infiles[1:]: - self.args.append(f'-point_obs "{infile}"') + self.args.append(f'-point_obs {infile}') if self.c_dict.get('GRID_INPUT_PATH'): grid_file = do_string_sub(self.c_dict['GRID_INPUT_PATH'], diff --git a/metplus/wrappers/stat_analysis_wrapper.py b/metplus/wrappers/stat_analysis_wrapper.py index 4f7525fcff..60ba1dea8b 100755 --- a/metplus/wrappers/stat_analysis_wrapper.py +++ b/metplus/wrappers/stat_analysis_wrapper.py @@ -342,7 +342,8 @@ def _run_stat_analysis_job(self, runtime_settings): value = f'"{value}"' else: value = f'[{value}]' - value = f'{item.lower()} = {value};' + item = item.lower().replace('_level', '_lev') + value = f'{item} = {value};' self.env_var_dict[key] = value # send environment variables to logger diff --git a/parm/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter.conf b/parm/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter.conf new file mode 100644 index 0000000000..3fc777c747 --- /dev/null +++ b/parm/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter.conf @@ -0,0 +1,103 @@ +[config] + +### +# Processes to run +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#process-list +### + +PROCESS_LIST = UserScript, GenVxMask, GridStat + + +### +# Time Info +# LOOP_BY options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +# LEAD_SEQ is the list of forecast leads to process +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#timing-control +### + +USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_FOR_EACH + +LOOP_BY = INIT +INIT_TIME_FMT = %Y%m%d%H +INIT_BEG = 2018060116 +INIT_END = 2018060116 +INIT_INCREMENT = 1d + +LEAD_SEQ = 4H, 23H, 32H + + +### +# File I/O +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info +### + +SCRIPT_DIR = {PARM_BASE}/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter + +USER_SCRIPT_INPUT_DIR = {INPUT_BASE}/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter/mma +USER_SCRIPT_OUTPUT_DIR = {OUTPUT_BASE}/poly + + +GEN_VX_MASK_INPUT_DIR = +GEN_VX_MASK_INPUT_TEMPLATE = "lambert 472 472 37.402645 -107.88144 -107.808 0.02754 6371.229 37.461 N" + +GEN_VX_MASK_INPUT_MASK_DIR = +GEN_VX_MASK_INPUT_MASK_TEMPLATE = {USER_SCRIPT_OUTPUT_DIR}/fire_perim_{valid?fmt=%Y%m%d_%H}.poly + +GEN_VX_MASK_OUTPUT_DIR = +GEN_VX_MASK_OUTPUT_TEMPLATE = {OUTPUT_BASE}/mask/fire_perim_{valid?fmt=%Y%m%d_%H}_mask.nc + +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE}/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter/wrf +FCST_GRID_STAT_INPUT_TEMPLATE = PYTHON_NUMPY + +OBS_GRID_STAT_INPUT_TEMPLATE = {GEN_VX_MASK_OUTPUT_TEMPLATE} + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/grid_stat +GRID_STAT_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d%H} + + +### +# UserScript Settings +# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#userscript +### + +USER_SCRIPT_COMMAND = python3 {SCRIPT_DIR}/find_and_read_fire_perim_poly.py {USER_SCRIPT_INPUT_DIR} {valid?fmt=%Y%m%d%H} {USER_SCRIPT_OUTPUT_DIR} + + +### +# GenVxMask Settings +# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#genvxmask +### + +GEN_VX_MASK_OPTIONS = -type poly + + +### +# GridStat Settings (optional) +# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#gridstat +### + +#LOG_GRID_STAT_VERBOSITY = 4 + +MODEL = WRF_FIRE +OBTYPE = FIRE_PERIM_KML + +FCST_VAR1_NAME = {SCRIPT_DIR}/read_wrfout_fire.py {FCST_GRID_STAT_INPUT_DIR}/{init?fmt=%Y%m%d_%H} {valid?fmt=%Y%m%d_%H%M%S} +FCST_VAR1_LEVELS = L0 +FCST_VAR1_THRESH = >0.5 + +OBS_VAR1_NAME = FIRE_PERIM +OBS_VAR1_LEVELS = "*,*" +OBS_VAR1_THRESH = ==1 +OBS_VAR1_OPTIONS = set_attr_valid = "{valid?fmt=%Y%m%d_%H%M%S}" + + +GRID_STAT_REGRID_TO_GRID = FCST + +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT + +GRID_STAT_NC_PAIRS_FLAG_LATLON = TRUE diff --git a/parm/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter/find_and_read_fire_perim_poly.py b/parm/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter/find_and_read_fire_perim_poly.py new file mode 100644 index 0000000000..6c348908a8 --- /dev/null +++ b/parm/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter/find_and_read_fire_perim_poly.py @@ -0,0 +1,76 @@ +import sys +import os +from datetime import datetime, timedelta +from glob import glob +import xml.etree.ElementTree as ET + +KML_TEMPLATE = 'N328SF_%m%d%Y%H*.kml' +VALID_FORMAT = '%Y%m%d%H' + +if len(sys.argv) != 4: + print("ERROR: Must supply input directory, valid time (YYYYMMDDHH), and output directory to script") + sys.exit(1) + +# read input directory +input_dir = sys.argv[1] + +# parse valid time +try: + valid_time = datetime.strptime(sys.argv[2], VALID_FORMAT) +except ValueError: + print(f"ERROR: Invalid format for valid time: {sys.argv[2]} (Should match {VALID_FORMAT})") + sys.exit(1) + +# get previous hour valid time +valid_one_hour_ago = valid_time - timedelta(hours=1) + +# build output path +output_dir = sys.argv[3] +output_file = f"fire_perim_{valid_time.strftime('%Y%m%d_%H')}.poly" +output_path = os.path.join(output_dir, output_file) + +# create output directory if it does not exist +os.makedirs(output_dir, exist_ok=True) + +# find input file +input_file = None +input_glob = os.path.join(input_dir, valid_time.strftime(KML_TEMPLATE)) +found_files = glob(input_glob) + +# if no files were found, search one hour ago +if not found_files: + input_glob = os.path.join(input_dir, valid_one_hour_ago.strftime(KML_TEMPLATE)) + found_files = glob(input_glob) + + if not found_files: + print(f"ERROR: Could not find any files for {valid_time} in {input_dir}") + sys.exit(1) + + # if multiple files are found for previous hour, use last file + if len(found_files) > 1: + print(f"WARNING: Found multiple files: {found_files}") + print("Processing the LAST file") + input_file = found_files[-1] + +elif len(found_files) > 1: + print(f"WARNING: Found multiple files: {found_files}") + print("Processing the FIRST file") + +if not input_file: + input_file = found_files[0] + +print(f"Parsing file: {input_file}") + +tree = ET.parse(input_file) +root = tree.getroot() + +search_path = './/{*}coordinates' +coordinates = root.find(search_path).text.split() + +with open(output_path, 'w') as file_handle: + file_handle.write('FIRE_PERIM\n') + for coord in coordinates: + lon, lat, elev = coord.split(',') + file_handle.write(f'{lat} {lon}\n') + +print(f"Wrote output to {output_path}") diff --git a/parm/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter/read_wrfout_fire.py b/parm/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter/read_wrfout_fire.py new file mode 100644 index 0000000000..1505e563a3 --- /dev/null +++ b/parm/use_cases/model_applications/fire/GridStat_fcstWRF_obsMMA_fire_perimeter/read_wrfout_fire.py @@ -0,0 +1,91 @@ +import sys +import os +from glob import glob +from datetime import datetime + +import xarray as xr + +VAR_NAME = 'FIRE_AREA' +LONG_NAME = 'Fire Area' +FILE_DATE_FORMAT = '%Y-%m-%d_%H:%M:%S' +MET_DATE_FORMAT ='%Y%m%d_%H%M%S' +VALID_FORMAT = '%Y%m%d_%H%M%S' +WRF_FIRE_TEMPLATE = "wrfout_fire_d02_%Y-%m-%d_%H %M %S" +EARTH_RADIUS = 6371.229 + +if len(sys.argv) != 3: + print("ERROR: Must supply input directory and valid time (YYYYMMDDHH) to script") + sys.exit(1) + +# read input directory +input_dir = sys.argv[1] + +# parse valid time +try: + valid_time = datetime.strptime(sys.argv[2], VALID_FORMAT) +except ValueError: + print(f"ERROR: Invalid format for valid time: {sys.argv[2]} (Should match {VALID_FORMAT})") + sys.exit(1) + +# find input file +input_glob = os.path.join(input_dir, valid_time.strftime(WRF_FIRE_TEMPLATE)) +found_files = glob(input_glob) +if not found_files: + print(f"ERROR: Could not find any files for {valid_time} in {input_dir}") + sys.exit(1) + +input_path = found_files[0] + +ds = xr.open_dataset(input_path, decode_times=False) + +valid_dt = datetime.strptime(ds['Times'][0].values.tobytes().decode(), + FILE_DATE_FORMAT) +init_dt = datetime.strptime(ds.attrs['START_DATE'], FILE_DATE_FORMAT) +lead_td = valid_dt - init_dt +lead_hours = lead_td.days * 24 + (lead_td.seconds//3600) +lead_hms = (f"{str(lead_hours).zfill(2)}" + f"{str((lead_td.seconds//60)%60).zfill(2)}00") + +nx = ds.dims['west_east_subgrid'] +ny = ds.dims['south_north_subgrid'] + +d_km = ds.attrs['DX'] * ds.dims['west_east'] / nx / 1000 + +lat_ll = float(ds['FXLAT'][0][0][0]) +lon_ll = float(ds['FXLONG'][0][0][0]) + +met_data = ds[VAR_NAME][0] +met_data = met_data[::-1] +met_data.attrs['valid'] = valid_dt.strftime(MET_DATE_FORMAT) +met_data.attrs['init'] = init_dt.strftime(MET_DATE_FORMAT) +met_data.attrs['lead'] = lead_hms +met_data.attrs['accum'] = '000000' +met_data.attrs['name'] = VAR_NAME +met_data.attrs['long_name'] = LONG_NAME +met_data.attrs['level'] = 'Z0' +met_data.attrs['units'] = '%' +met_data.attrs['grid'] = { + 'type': ds.attrs['MAP_PROJ_CHAR'], + 'hemisphere': 'N' if float(ds.attrs['POLE_LAT']) > 0 else 'S', + 'name': VAR_NAME, + 'nx': nx, + 'ny': ny, + 'lat_pin': lat_ll, + 'lon_pin': lon_ll, + 'x_pin': 0.0, + 'y_pin': 0.0, + 'lon_orient': float(ds.attrs['CEN_LON']), + 'd_km': d_km, + 'r_km': EARTH_RADIUS, + 'scale_lat_1': float(ds.attrs['TRUELAT1']), + 'scale_lat_2': float(ds.attrs['TRUELAT2']), +} + +print(met_data) +for key,value in met_data.attrs.items(): + if key == 'grid': + print(f"{key}:") + for key2,value2 in value.items(): + print(f" {key2}: {value2}") + else: + print(f"{key}: {value}") diff --git a/ush/run_metplus.py b/ush/run_metplus.py index 6f329867ac..b7a99274ad 100755 --- a/ush/run_metplus.py +++ b/ush/run_metplus.py @@ -45,12 +45,6 @@ def main(): if not config: return False - # warn if calling master_metplus.py - if basename(__file__) == 'master_metplus.py': - msg = ("master_metplus.py has been renamed to run_metplus.py. " - "This script name will be removed in a future version.") - config.logger.warning(msg) - total_errors = run_metplus(config) return post_run_cleanup(config, 'METplus', total_errors)