Feature #2682 Component version lookup (#2733)

* per #2714, deprecate master_metplus.py

* exclude metplus/scripts from SonarQube scan to prevent incorrect duplicate code error since script lives in ush

* added Dockerfile to build image with MET/METplus and METplus Analysis Python packages

* start of script to get versions of METplus components

* per #2682, created METplus component version lookup table and functions to get formatted version of requested components

* add descriptions for tests

* add update information and made script callable

* made script executable

* add workflow to trigger on release and create dtcenter/metplus and dtcenter/metplus-analysis Docker images

* add scripts used by new workflow

* removed build hook scripts since we will be building images via GHA

* change logic to get MET version using component version script instead of build hook file that just adds 6 to major version

* add missing shebang

* make script callable directly and make output component a required input

* pass LATEST_TAG to push script

* ensure only 1st line of version file is read to avoid newline

* turn on use case to test

* turn off use case

.github/jobs/docker_build_metplus_images.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# assumes SOURCE_BRANCH is set before calling script
+
+source "${GITHUB_WORKSPACE}"/.github/jobs/bash_functions.sh
+
+dockerhub_repo=dtcenter/metplus
+dockerhub_repo_analysis=dtcenter/metplus-analysis
+
+# check if tag is official or bugfix release -- no -betaN or -rcN suffix
+is_official=1
+if [[ ! "${SOURCE_BRANCH}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+  is_official=0
+fi
+
+# remove v prefix
+metplus_version=${SOURCE_BRANCH:1}
+
+# Get MET tag and adjust MET Docker repo if develop
+met_tag=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o MET -f "{X}.{Y}-latest")
+echo "$met_tag"
+
+MET_DOCKER_REPO=met
+if [ "$met_tag" == "develop" ]; then
+    MET_DOCKER_REPO=met-dev
+fi
+
+# get METplus Analysis tool versions
+METDATAIO_VERSION=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o METdataio)
+METCALCPY_VERSION=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o METcalcpy)
+METPLOTPY_VERSION=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o METplotpy)
+
+# Build metplus image
+METPLUS_IMAGE_NAME=${dockerhub_repo}:${metplus_version}
+if ! time_command docker build -t "$METPLUS_IMAGE_NAME" \
+       --build-arg SOURCE_VERSION="$SOURCE_BRANCH" \
+       --build-arg MET_TAG="$met_tag" \
+       --build-arg MET_DOCKER_REPO="$MET_DOCKER_REPO" \
+       -f "${GITHUB_WORKSPACE}"/internal/scripts/docker/Dockerfile \
+       "${GITHUB_WORKSPACE}"; then
+    exit 1
+fi
+
+# Build metplus-analysis image
+METPLUS_A_IMAGE_NAME=${dockerhub_repo_analysis}:${metplus_version}
+if ! time_command docker build -t "$METPLUS_A_IMAGE_NAME" \
+       --build-arg METPLUS_BASE_TAG="${metplus_version}" \
+       --build-arg METDATAIO_VERSION="${METDATAIO_VERSION}" \
+       --build-arg METCALCPY_VERSION="${METCALCPY_VERSION}" \
+       --build-arg METPLOTPY_VERSION="${METPLOTPY_VERSION}" \
+       -f "${GITHUB_WORKSPACE}"/internal/scripts/docker/Dockerfile.metplus-analysis \
+       "${GITHUB_WORKSPACE}"; then
+    exit 1
+fi
+
+# if official release, create X.Y-latest tag as well
+if [ "${is_official}" == 0 ]; then
+    LATEST_TAG=$(echo "$metplus_version" | cut -f1,2 -d'.')-latest
+    docker tag "${METPLUS_IMAGE_NAME}" "${dockerhub_repo}:${LATEST_TAG}"
+    docker tag "${METPLUS_A_IMAGE_NAME}" "${dockerhub_repo_analysis}:${LATEST_TAG}"
+    echo LATEST_TAG="${LATEST_TAG}" >> "$GITHUB_OUTPUT"
+fi

.github/jobs/docker_push_metplus_images.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# assumes SOURCE_BRANCH is set before calling script
+# assumes latest_tag will be set if pushing an official or bugfix release
+
+source "${GITHUB_WORKSPACE}"/.github/jobs/bash_functions.sh
+
+# get names of images to push
+
+dockerhub_repo=dtcenter/metplus
+dockerhub_repo_analysis=dtcenter/metplus-analysis
+
+# remove v prefix
+metplus_version=${SOURCE_BRANCH:1}
+
+METPLUS_IMAGE_NAME=${dockerhub_repo}:${metplus_version}
+METPLUS_A_IMAGE_NAME=${dockerhub_repo_analysis}:${metplus_version}
+
+# skip docker push if credentials are not set
+if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PASSWORD+x} ]; then
+    echo "DockerHub credentials not set. Skipping docker push"
+    exit 0
+fi
+
+echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
+
+# push images
+
+if ! time_command docker push "${METPLUS_IMAGE_NAME}"; then
+  exit 1
+fi
+
+if ! time_command docker push "${METPLUS_A_IMAGE_NAME}"; then
+  exit 1
+fi
+
+# only push X.Y-latest tag if official or bugfix release
+# shellcheck disable=SC2154
+if [ "${LATEST_TAG}" != "" ]; then
+    if ! time_command docker push "${dockerhub_repo}:${LATEST_TAG}"; then
+      exit 1
+    fi
+    if ! time_command docker push "${dockerhub_repo_analysis}:${LATEST_TAG}"; then
+      exit 1
+    fi
+fi

.github/jobs/docker_setup.sh

@@ -31,7 +31,8 @@ echo "TIMING: docker pull ${DOCKERHUB_TAG} took `printf '%02d' $(($duration / 60
 # set DOCKERFILE_PATH that is used by docker hook script get_met_version
 export DOCKERFILE_PATH=${GITHUB_WORKSPACE}/internal/scripts/docker/Dockerfile
 
-MET_TAG=`${GITHUB_WORKSPACE}/internal/scripts/docker/hooks/get_met_version`
+metplus_version=$(head -n 1 "${GITHUB_WORKSPACE}/metplus/VERSION")
+MET_TAG=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o MET -f "{X}.{Y}-latest")
 
 MET_DOCKER_REPO=met-dev
 if [ "${MET_TAG}" != "develop" ]; then

.github/workflows/release-docker-images.yml

@@ -0,0 +1,34 @@
+name: Create Docker images for release
+
+on:
+  release:
+    types:
+      - published
+
+jobs:
+  build_and_push:
+    name: Build and Push Images
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Get and check tag name
+        id: get_tag_name
+        run: |
+          SOURCE_BRANCH=${GITHUB_REF#refs/tags/}
+          if [[ ! "${SOURCE_BRANCH}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+            echo "ERROR: Tag name (${SOURCE_BRANCH}) does not start with vX.Y.Z format"
+            exit 1
+          fi
+          echo SOURCE_BRANCH=${SOURCE_BRANCH} >> $GITHUB_OUTPUT
+      - name: Build metplus and metplus-analysis images
+        id: build_images
+        run: .github/jobs/docker_build_metplus_images.sh
+        env:
+          SOURCE_BRANCH: ${{ steps.get_tag_name.outputs.SOURCE_BRANCH }}
+      - name: Push metplus and metplus-analysis images
+        run: .github/jobs/docker_push_metplus_images.sh
+        env:
+          SOURCE_BRANCH: ${{ steps.get_tag_name.outputs.SOURCE_BRANCH }}
+          LATEST_TAG: ${{ steps.build_images.outputs.LATEST_TAG }}
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

internal/scripts/docker/Dockerfile.metplus-analysis

@@ -0,0 +1,16 @@
+ARG METPLUS_BASE_REPO=metplus
+ARG METPLUS_BASE_TAG
+
+FROM dtcenter/${METPLUS_BASE_REPO}:${METPLUS_BASE_TAG}
+LABEL org.opencontainers.image.authors="mccabe@ucar.edu"
+
+ARG METDATAIO_VERSION
+ARG METCALCPY_VERSION
+ARG METPLOTPY_VERSION
+
+RUN git clone --branch "$METDATAIO_VERSION" https://github.com/dtcenter/METdataio \
+    && pip install -r METdataio/requirements.txt && pip install METdataio/. \
+    && git clone --branch "$METCALCPY_VERSION" https://github.com/dtcenter/METcalcpy \
+    && pip install -r METcalcpy/requirements.txt && pip install METcalcpy/. \
+    && git clone --branch "$METPLOTPY_VERSION" https://github.com/dtcenter/METplotpy \
+    && pip install -r METplotpy/requirements.txt && pip install METplotpy/.

internal/scripts/sonarqube/sonar-project.properties

@@ -4,6 +4,7 @@ sonar.projectName=METplus
 sonar.projectVersion=SONAR_PROJECT_VERSION
 sonar.branch.name=SONAR_BRANCH_NAME
 sonar.sources=docs,internal,manage_externals,metplus,parm,ush
+sonar.exclusions=metplus/scripts/**
 sonar.coverage.exclusions=internal/tests/**,parm/**,metplus/parm/**,internal/scripts/**,manage_externals/**,docs/**,metplus/produtil/**,ush/**,metplus/wrappers/cyclone_plotter_wrapper.py
 sonar.python.coverage.reportPaths=coverage.xml
 sonar.sourceEncoding=UTF-8

internal/tests/pytests/component_versions/test_component_versions.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+import pytest
+
+from metplus import component_versions
+
+@pytest.mark.parametrize(
+    'component, version, expected_result', [
+        ('met', '11.1.1', '5.1'),
+        ('MET', '11.1.1', '5.1'),
+        ('met', '11.1', '5.1'),
+        ('met', '11.1.Z', '5.1'),
+        ('METcalcpy', '3.0.0', '6.0'),
+        ('metcalcpy', 'main_v3.0', '6.0'),
+        ('metcalcpy', 'v3.0.0', '6.0'),
+        ('metcalcpy', 'v3.0.0-beta3', '6.0'),
+        ('metcalcpy', 'v3.0.0-rc1', '6.0'),
+        ('METplus', '6.0-latest', '6.0'),
+        ('METplus', '3.0-latest', None),
+    ]
+)
+@pytest.mark.util
+def test_get_coordinated_version(component, version, expected_result):
+    assert component_versions.get_coordinated_version(component, version) == expected_result
+
+
+@pytest.mark.parametrize(
+    'input_component, input_version, output_component, output_format, expected_result', [
+        # get MET version for Docker dtcenter/metplus
+        ('metplus', '5.1.0', 'met', '{X}.{Y}.{Z}{N}', '11.1.1'),
+        ('metplus', '5.1.0-beta3', 'met', '{X}.{Y}.{Z}{N}', '11.1.1-beta3'),
+        ('metplus', '5.1.0-rc1', 'met', '{X}.{Y}.{Z}{N}', '11.1.1-rc1'),
+        ('metplus', '5.1-latest', 'met', '{X}.{Y}{N}', '11.1-latest'),
+        ('metplus', '5.1.0-beta3-dev', 'met', '{X}.{Y}.{Z}{N}', 'develop'),
+        # get METplus Analysis versions for Docker dtcenter/metplus-analysis
+        ('METplus', '5.1.0', 'metplotpy', 'v{X}.{Y}.{Z}{N}', 'v2.1.0'),
+        ('metplus', '5.1.0-beta3', 'METplotpy', 'v{X}.{Y}.{Z}{N}', 'v2.1.0-beta3'),
+        ('metplus', '5.1.0-dev', 'METplotpy', 'v{X}.{Y}.{Z}{N}', 'develop'),
+        ('metplus', '5.1.0-rc1', 'metplotpy', 'v{X}.{Y}.{Z}{N}', 'v2.1.0-rc1'),
+        ('metplus', '5.1.0-beta3-dev', 'metplotpy', 'v{X}.{Y}.{Z}{N}', 'develop'),
+        # get METplus main branch to trigger workflow from other repos, e.g. MET
+        ('MET', 'main_v11.1', 'METplus', 'main_v{X}.{Y}', 'main_v5.1'),
+        ('MET', 'main_v11.1-ref', 'METplus', 'main_v{X}.{Y}', 'main_v5.1'),
+        # get latest bugfix version from main branch or X.Y version
+        ('MET', 'main_v11.1', 'MET', '{X}.{Y}.{Z}{N}', '11.1.1'),
+        ('MET', '11.1.Z', 'MET', '{X}.{Y}.{Z}{N}', '11.1.1'),
+    ]
+)
+@pytest.mark.util
+def test_get_component_version(input_component, input_version, output_component, output_format, expected_result):
+    assert component_versions.get_component_version(input_component, input_version, output_component, output_format) == expected_result

metplus/component_versions.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+
+# Dictionary to track version numbers for each METplus component
+# The key of each entry is the coordinated release version, e.g. 6.0
+# The value is another dictionary where the key is the METplus component name
+#   (in lower-case) and the value is a string that contains the latest
+#   X.Y.Z version of that component or None if no version is available.
+# Add entries for a new coordinated release at the top of the dictionary.
+# The versions should be updated when a bugfix release is created.
+# METexpress does not include a release for the beta versions, so the value for
+#   METexpress should be set to None until the official coordinated release
+#   has been created.
+
+import sys
+
+VERSION_LOOKUP = {
+    '6.0': {
+        'metplus': '6.0.0',
+        'met': '12.0.0',
+        'metplotpy': '3.0.0',
+        'metcalcpy': '3.0.0',
+        'metdataio': '3.0.0',
+        'metviewer': '6.0.0',
+        'metexpress': None,
+    },
+    '5.1': {
+        'metplus': '5.1.0',
+        'met': '11.1.1',
+        'metplotpy': '2.1.0',
+        'metcalcpy': '2.1.0',
+        'metdataio': '2.1.0',
+        'metviewer': '5.1.0',
+        'metexpress': '5.3.3',
+    },
+}
+
+DEFAULT_OUTPUT_FORMAT = "v{X}.{Y}.{Z}{N}"
+
+def get_component_version(input_component, input_version, output_component,
+                          output_format=DEFAULT_OUTPUT_FORMAT):
+    """!Get the version of a requested METplus component given another METplus
+    component and its version. Parses out X.Y version numbers of input version
+    to find desired version. Optionally specific format of output content.
+    If input version ends with "-dev", then return "develop".
+
+    @param input_component name of METplus component to use to find version,
+    e.g. MET, METplus, or METplotpy (case-insensitive).
+    @param input_version version of input_component to search.
+    @param output_component name of METplus component to obtain version number
+    @param output_format (optional) format to use to output version number.
+     {X}, {Y}, and {Z} will be replaced with x, y, and z version numbers from
+     X.Y.Z. {N} will be replaced with development version if found in the
+     input version, e.g. "-beta3" or "-rc1"
+     @returns string of requested version number, or "develop" if input version
+     ends with "-dev", or None if version number could not be determined.
+    """
+    if input_version.endswith('-dev'):
+        return 'develop'
+    coord_version = get_coordinated_version(input_component, input_version)
+    versions = VERSION_LOOKUP.get(coord_version)
+    if versions is None:
+        return None
+    output_version = versions.get(output_component.lower())
+    if output_version is None:
+        return None
+    x, y, z = output_version.split('.')
+    dev_version = input_version.split('-')[1:]
+    dev_version = '' if not dev_version else f"-{dev_version[0]}"
+    return output_format.format(X=x, Y=y, Z=z, N=dev_version)
+
+
+def get_coordinated_version(component, version):
+    """!Get coordinated release version number based on the X.Y version number
+    of a given METplus component.
+
+    @param component name of METplus component to search (case-insensitive)
+    @param version number of version to search for. Can be formatted with main_v
+    prefix and development release info, e.g. main_vX.Y or X.Y.Z-beta3.
+    @returns string of coordinated release version number X.Y or None.
+    """
+    # remove main_v or v prefix, remove content after dash
+    search_version = version.removeprefix('main_').lstrip('v').split('-')[0]
+    # get X.Y only
+    search_version = '.'.join(search_version.split('.')[:2])
+    # look for component version that begins with search version
+    for coord_version, versions in VERSION_LOOKUP.items():
+        if versions.get(component.lower()).startswith(search_version):
+            return coord_version
+    return None
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-i', '--input_component',
+                        default='metplus',
+                        help='Name of METplus component to use to find version')
+    parser.add_argument('-v', '--input_version',
+                        default=next(iter(VERSION_LOOKUP)),
+                        help='version of input_component to search')
+    parser.add_argument('-o', '--output_component', required=True,
+                        help='name of METplus component to obtain version')
+    parser.add_argument('-f', '--output_format',
+                        default=DEFAULT_OUTPUT_FORMAT,
+                        help='format to use to output version number.'
+                             '{X}, {Y}, and {Z} will be replaced with x, y, and'
+                             ' z version numbers from X.Y.Z. {N} will be '
+                             'replaced with development version if found in the'
+                             'input version, e.g. "-beta3" or "-rc1"')
+    args = parser.parse_args()
+    return get_component_version(args.input_component, args.input_version,
+                                 args.output_component, args.output_format)
+
+
+if __name__ == "__main__":
+    version = main()
+    if not version:
+        sys.exit(1)
+
+    print(version)