From 7bd9d45f1ab77923bd7ff6023b841b546f742a51 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 08:09:25 +0000 Subject: [PATCH] Chore: Update DevOps tooling from central repository --- .github/workflows/bootstrap.yaml | 119 +++++++++++++++++++++++---- .github/workflows/builds.yaml | 7 +- .github/workflows/dependencies.yaml | 26 ++++-- .github/workflows/documentation.yaml | 2 +- .github/workflows/linting.yaml | 56 +++++++++++++ .github/workflows/release.yaml | 6 +- .github/workflows/security.yaml | 12 +-- .github/workflows/test-release.yaml | 4 +- .github/workflows/testing.yaml | 5 +- .pre-commit-config.yaml | 8 +- .prettierignore | 1 + scripts/template-to-repo.sh | 11 +-- 12 files changed, 205 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/linting.yaml diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml index 4adbdf0..a1ebb30 100644 --- a/.github/workflows/bootstrap.yaml +++ b/.github/workflows/bootstrap.yaml @@ -31,33 +31,122 @@ jobs: path: ".devops" - name: "Update repository workflows and create PR" + id: update-repository env: GH_TOKEN: ${{ github.token }} + # yamllint disable rule:line-length run: | - # Remove update-devops-tooling branch if it exists - git branch -d update-devops-tooling || true - git push origin --delete update-devops-tooling || true - git config user.name "github-actions[bot]" - git config user.email \ - "41898282+github-actions[bot]@users.noreply.github.com" - git checkout -b "update-devops-tooling" + ### SHELL CODE START ### + + REPO_DIR=$(git rev-parse --show-toplevel) + # Ensure working from top-level of GIT repository + CURRENT_DIR=$(pwd) + if [ "$REPO_DIR" != "$CURRENT_DIR" ]; then + echo "Changing directory to: $REPO_DIR" + if ! (cd "$REPO_DIR"); then + echo "Error: unable to change directory"; exit 1 + fi + fi + + # Define a function to allow selective opt-out of devops tooling + OPT_OUT=".devops-exclusions" + perform_operation() { + ELEMENT="$1" + if [ ! -f "$OPT_OUT" ]; then + # Opt-out file does not exist; all operations will be performed + return 1 + else + if grep -Fxq "$ELEMENT" "$OPT_OUT" + then + # Element is excluded from processing + return 0 + else + # Element should be processed + return 1 + fi + fi + } + + echo "Removing remote branch if it exists: update-devops-tooling" + git push origin --delete update-devops-tooling || : + STRING=$(dd if=/dev/urandom bs=1k count=1 2>/dev/null | tr -dc 'a-zA-Z0-9' | head -c 10) + git checkout -b "update-$STRING" + + # Configure GIT + TEST=$(git config -l) + if [ -n "$TEST" ]; then + git config user.name "github-actions[bot]" + git config user.email \ + "41898282+github-actions[bot]@users.noreply.github.com" + fi + FOLDERS=".github .github/workflows scripts" - FILES=".pre-commit-config.yaml .prettierignore .gitignore" for FOLDER in ${FOLDERS}; do + # Check to see if operation should be skipped + if (perform_operation "$FOLDER"); then + echo "Opted out of DevOps folder: $FOLDER" + continue + else # If necessary, create target folder if [ ! -d "$FOLDER" ]; then - mkdir "$FOLDER" + echo "Creating target folder: $FOLDER" + mkdir "$FOLDER" fi # Update folder contents + echo "Updating folder contents: $FOLDER" cp -a .devops/"$FOLDER"/. "$FOLDER" + fi done + # Copy specified files into repository root + FILES=".pre-commit-config.yaml .prettierignore .gitignore" for FILE in ${FILES}; do + if (perform_operation "$FILE"); then + echo "Opted out of DevOps file: $FILE" + else + echo "Copying file: $FILE" cp .devops/"$FILE" "$FILE" + fi done - git add . - git commit -m "Chore: Update DevOps tooling from central repository" - git push --set-upstream origin update-devops-tooling - gh pr create --title \ - "Chore: Pull DevOps tooling from upstream repository" \ - --body 'This process automated by a GitHub workflow: bootstrap.yaml' + + # If no changes required, do not throw an error + if [ -z "$(git status --porcelain)" ]; then + echo "No updates/changes to commit"; exit 0 + else + # Set a flag for use by the next action/step + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + if [ -n "$GITHUB_TOKEN" ]; then + git add . + if ! (git commit -as -S -m "Chore: Update DevOps tooling from central repository [skip-ci]" \ + -m "This commit created by automation/scripting" --no-verify); then + echo "Commit failed; aborting"; exit 1 + else + git push --set-upstream origin update-devops-tooling + # ToDo: need to verify if we are running in a GHA + gh pr create --title \ + "Chore: Pull DevOps tooling from upstream repository" \ + --body 'Automated by a GitHub workflow: bootstrap.yaml' + fi + else + echo "Script running in GitHub Actions workflow; proceeding to next step" + fi + ### SHELL CODE END ### + + - name: Create Pull Request + if: steps.update-repository.outputs.changed == 'true' + uses: peter-evans/create-pull-request@v5 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + token: ${{ github.token }} + commit-message: "Chore: Update DevOps tooling from central repository [skip-ci]" + signoff: "true" + branch: update-devops-tooling + delete-branch: true + title: "Chore: Update DevOps tooling from central repository [skip-ci]" + body: | + Update repository with content from upstream: os-climate/devops-toolkit + labels: | + automated pr + draft: false diff --git a/.github/workflows/builds.yaml b/.github/workflows/builds.yaml index 94cd68c..8de5ef0 100644 --- a/.github/workflows/builds.yaml +++ b/.github/workflows/builds.yaml @@ -1,5 +1,5 @@ --- -name: "🧪 Test builds (matrix)" +name: "🧪 Test builds (Matrix)" # yamllint disable-line rule:truthy on: @@ -12,14 +12,15 @@ on: jobs: pre-release: - # Don't run if pull request is NOT merged - # if: github.event.pull_request.merged == true runs-on: "ubuntu-latest" continue-on-error: true + # Don't run when pull request is merged + if: github.event.pull_request.merged == false strategy: fail-fast: false matrix: python-version: ["3.9", "3.10", "3.11"] + steps: - name: "Populate environment variables" id: setenv diff --git a/.github/workflows/dependencies.yaml b/.github/workflows/dependencies.yaml index e6d12ba..8edd981 100644 --- a/.github/workflows/dependencies.yaml +++ b/.github/workflows/dependencies.yaml @@ -1,15 +1,15 @@ --- -name: "⛔️ Update dependencies" +name: "♻️ Update dependencies" # yamllint disable-line rule:truthy on: workflow_dispatch: schedule: - - cron: "0 8 * * FRI" + - cron: "0 8 1 * *" jobs: update-dependencies: - name: "Update Python modules" + name: "Update dependencies" runs-on: ubuntu-latest permissions: # IMPORTANT: mandatory to raise the PR @@ -22,15 +22,27 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v4 + - name: "Checkout repository" + uses: actions/checkout@v4 - - name: Update dependencies - uses: ModeSevenIndustrialSolutions/update-deps-action@v1 + - name: "Set up Python" + uses: actions/setup-python@v5 + + - name: "Update Python dependencies" + uses: pdm-project/update-deps-action@v1 with: sign-off-commit: "true" token: ${{ secrets.GH_TOKEN }} - commit-message: "Chore: Update dependencies and pdm.lock" + commit-message: "Chore: Update dependencies and pdm.lock [skip ci]" pr-title: "Update Python module dependencies" update-strategy: eager # Whether to install PDM plugins before update install-plugins: "false" + + - name: "Export dependencies" + run: | + pdm export --without-hashes -o requirements.txt + + # Ideally, we should export requirements.txt then amend the earlier PR + # update-deps-action could be modified to export PR number as as output + # Or we add the option to export the requirements.txt in that action diff --git a/.github/workflows/documentation.yaml b/.github/workflows/documentation.yaml index d3badc0..3796a60 100644 --- a/.github/workflows/documentation.yaml +++ b/.github/workflows/documentation.yaml @@ -1,5 +1,5 @@ --- -name: "🗒️ Build documentation" +name: "🗒️ Build documentation (Matrix)" # yamllint disable-line rule:truthy on: diff --git a/.github/workflows/linting.yaml b/.github/workflows/linting.yaml new file mode 100644 index 0000000..db12e80 --- /dev/null +++ b/.github/workflows/linting.yaml @@ -0,0 +1,56 @@ +--- +name: "⛔️ Standalone linting checks" + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: + pull_request: + types: [opened, reopened, edited, synchronize] + branches: + - "*" + - "!update-devops-tooling" + +jobs: + linting: + + name: "Unsupported by pre-commit.ci" + runs-on: "ubuntu-latest" + # Don't run when pull request is merged + if: github.event.pull_request.merged == false + + steps: + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: "Checkout repository" + uses: actions/checkout@v4 + + - name: "Install linting tools" + run: | + pip install --upgrade pip + pip install pre-commit mypy + + - name: "Run mypy using pre-commit" + run: pre-commit run mypy -a + + # yamllint disable rule:line-length + # yamllint disable rule:comments-indentation + # yamllint disable rule:comments + + # Provided below as an example, in case needed in future + # - name: "Install dependencies" + # run: | + # SOURCE=".pre-commit-config.yaml" + # echo "Install Python dependencies from: $SOURCE" + # echo "With: pip install $PKGS" + # PKGS=$(yq '.repos[] | select (.repo == "https://github.com/pre-commit/mirrors-mypy")' .pre-commit-config.yaml | \ + # grep additional_dependencies | \ + # awk -F: '{print $2}' | \ + # sed "s/\[//g" | \ + # sed "s/\]//g" | \ + # sed "s/,//g" | \ + # sed 's/"//g') + # pip install $PKGS diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d9c0ede..7b99e21 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -76,7 +76,7 @@ jobs: contents: write steps: - name: "⬇ Download build artefacts" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ github.ref_name }} path: dist/ @@ -111,7 +111,7 @@ jobs: id-token: write steps: - name: "⬇ Download build artefacts" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ github.ref_name }} path: dist/ @@ -145,7 +145,7 @@ jobs: id-token: write steps: - name: "⬇ Download build artefacts" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ github.ref_name }} path: dist/ diff --git a/.github/workflows/security.yaml b/.github/workflows/security.yaml index a24ce21..9d34824 100644 --- a/.github/workflows/security.yaml +++ b/.github/workflows/security.yaml @@ -4,7 +4,7 @@ # For more information see: # https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: "⛔️ Security auditing" +name: "⛔️ Security auditing (Matrix)" # yamllint disable-line rule:truthy on: @@ -19,10 +19,13 @@ jobs: build: name: "Audit Python dependencies" runs-on: ubuntu-latest + # Don't run when pull request is merged + if: github.event.pull_request.merged == false strategy: fail-fast: false matrix: python-version: ["3.9", "3.10", "3.11"] + steps: - name: "Checkout repository" uses: actions/checkout@v4 @@ -48,10 +51,3 @@ jobs: - name: "Run: pip-audit" uses: pypa/gh-action-pip-audit@v1.0.8 - with: - ignore-vulns: | - PYSEC-2023-163 - -# Name | Version | ID | -# --- | --- | --- | --- | --- -# numexpr | 2.8.7 | PYSEC-2023-163 | diff --git a/.github/workflows/test-release.yaml b/.github/workflows/test-release.yaml index e2f9bdf..ad2a83b 100644 --- a/.github/workflows/test-release.yaml +++ b/.github/workflows/test-release.yaml @@ -86,7 +86,7 @@ jobs: contents: write steps: - name: "⬇ Download build artefacts" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: Development path: dist/ @@ -132,7 +132,7 @@ jobs: id-token: write steps: - name: "⬇ Download build artefacts" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: Development path: dist/ diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index 899529e..bb80cec 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -1,5 +1,5 @@ --- -name: "🧪 Unit tests" +name: "🧪 Unit tests (Matrix)" # yamllint disable-line rule:truthy on: @@ -14,10 +14,13 @@ jobs: build: name: "Run unit tests" runs-on: ubuntu-latest + # Don't run when pull request is merged + if: github.event.pull_request.merged == false strategy: fail-fast: false matrix: python-version: ["3.9", "3.10", "3.11"] + steps: - name: "Checkout repository" uses: actions/checkout@v4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 349dca5..2d63b0e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,6 +51,7 @@ repos: - id: mixed-line-ending args: ["--fix=lf"] - id: name-tests-test + args: ["--pytest-test-first"] # Do not allow direct push to main/master branches - id: no-commit-to-branch # - id: pretty-format-json @@ -63,7 +64,7 @@ repos: hooks: - id: prettier args: - ['--ignore-unknown', '--no-error-on-unmatched-pattern', '!chart/**'] + ['--ignore-unknown'] # Lint: Markdown - repo: https://github.com/igorshubovych/markdownlint-cli @@ -97,7 +98,7 @@ repos: rev: 2.1.1 hooks: - id: bashate - args: ["--ignore=E006"] + args: ["--ignore=E006,E011"] - repo: https://github.com/shellcheck-py/shellcheck-py rev: v0.9.0.6 @@ -121,6 +122,7 @@ repos: rev: 5.13.2 hooks: - id: isort + args: ["--profile", "black"] - repo: https://github.com/adrienverge/yamllint.git rev: v1.33.0 @@ -159,4 +161,4 @@ repos: - id: mypy verbose: true args: [--show-error-codes] - additional_dependencies: ["types-requests"] + additional_dependencies: ["pytest", "types-requests"] diff --git a/.prettierignore b/.prettierignore index a932ba9..6491e42 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,4 @@ **/.pre-commit-config.yaml **/*.yaml **/*.yml +**/.git/** diff --git a/scripts/template-to-repo.sh b/scripts/template-to-repo.sh index bc4a2ab..8fe5759 100755 --- a/scripts/template-to-repo.sh +++ b/scripts/template-to-repo.sh @@ -5,7 +5,7 @@ THIS_SCRIPT=$(basename "$0") echo "This script: $SELF" -TEMPLATE_NAME=osc-data-extraction +TEMPLATE_NAME=osc-python-template ALT_TEMPLATE_NAME="${TEMPLATE_NAME//-/_}" ### Shared functions @@ -13,7 +13,7 @@ ALT_TEMPLATE_NAME="${TEMPLATE_NAME//-/_}" # Renames files/folders containing template name rename_object() { if [ $# -ne 1 ]; then - echo "Function requires an argument: rename_object [filesystem object]"; exit 1 + echo "Function requires an argumeent: rename_object [filesystem object]"; exit 1 else FS_OBJECT="$1" fi @@ -62,13 +62,6 @@ file_content_substitution() { return fi - if [ "$BASE_FILENAME" != "pyproject.toml" ]; then - return - else - echo "Processing: pyproject.toml" - set -x - fi - COUNT=0 if (grep "$TEMPLATE_NAME" "$FILENAME" > /dev/null 2>&1); then MATCHES=$(grep -c "$TEMPLATE_NAME" "$FILENAME")