Run all checks #259
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run all checks | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| ref: | |
| description: 'The git ref to build the package for' | |
| required: false | |
| default: '' | |
| type: string | |
| use_lkg: | |
| description: 'Whether to use the last known good versions of dependencies' | |
| required: false | |
| default: True | |
| type: boolean | |
| # nightly | |
| schedule: | |
| - cron: '0 0 * * *' | |
| # Only run once per PR, canceling any previous runs | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| # Precompute the ref if the workflow was triggered by a workflow dispatch rather than copying this logic repeatedly | |
| env: | |
| ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || null }} | |
| # we want to use the LKG if that is explicitly requested, or if we're in a PR, but not a nightly run | |
| # the final `|| ''` is because env vars are always converted to strings and the string 'false' is truthy (!!) | |
| # (see https://github.com/orgs/community/discussions/25645) | |
| use_lkg: ${{ (github.event_name == 'workflow_dispatch' && inputs.use_lkg) || github.event_name == 'pull_request' || ''}} | |
| jobs: | |
| eval: | |
| name: Evaluate changes | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| name: Checkout repository | |
| with: | |
| ref: ${{ env.ref }} | |
| fetch-depth: 2 | |
| # We want to enforce the following rules for PRs: | |
| # * if all modifications are to README.md | |
| # no testing is needed | |
| # * if there are modifications to docs/* or to any code | |
| # then docs need to be built to verify consistency | |
| # * if there are modifications to notebooks/* or to any code | |
| # then notebooks need to be run to verify consistency | |
| # * for any code changes (or changes to metadata files) | |
| # linting and testing should be run | |
| # For a PR build, HEAD will be the merge commit, and we want to diff against the base branch, | |
| # which will be the first parent: HEAD^ | |
| # (For non-PR changes, we will always perform all CI tasks) | |
| # Note that GitHub Actions provides path filters, but they operate at the workflow level, not the job level | |
| - run: | | |
| if ($env:GITHUB_EVENT_NAME -eq 'pull_request') { | |
| $editedFiles = git diff HEAD^ --name-only | |
| $editedFiles # echo edited files to enable easier debugging | |
| $codeChanges = $false | |
| $docChanges = $false | |
| $nbChanges = $false | |
| $changeType = "none" | |
| foreach ($file in $editedFiles) { | |
| switch -Wildcard ($file) { | |
| "README.md" { Continue } | |
| ".gitignore" { Continue } | |
| "econml/_version.py" { Continue } | |
| "prototypes/*" { Continue } | |
| "images/*" { Continue } | |
| "doc/*" { $docChanges = $true; Continue } | |
| "notebooks/*" { $nbChanges = $true; Continue } | |
| default { $codeChanges = $true; Continue } | |
| } | |
| } | |
| } | |
| echo "buildDocs=$(($env:GITHUB_EVENT_NAME -ne 'pull_request') -or ($docChanges -or $codeChanges))" >> $env:GITHUB_OUTPUT | |
| echo "buildNbs=$(($env:GITHUB_EVENT_NAME -ne 'pull_request') -or ($nbChanges -or $codeChanges))" >> $env:GITHUB_OUTPUT | |
| echo "testCode=$(($env:GITHUB_EVENT_NAME -ne 'pull_request') -or $codeChanges)" >> $env:GITHUB_OUTPUT | |
| shell: pwsh | |
| name: Determine type of code change | |
| id: eval | |
| outputs: | |
| buildDocs: ${{ steps.eval.outputs.buildDocs }} | |
| buildNbs: ${{ steps.eval.outputs.buildNbs }} | |
| testCode: ${{ steps.eval.outputs.testCode }} | |
| lint: | |
| name: Lint code | |
| needs: [eval] | |
| if: ${{ needs.eval.outputs.testCode == 'True' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| name: Checkout repository | |
| with: | |
| ref: ${{ env.ref }} | |
| - uses: actions/setup-python@v4 | |
| name: Setup Python | |
| with: | |
| python-version: '3.9' | |
| - run: python -m pip install --upgrade pip && pip install --upgrade setuptools | |
| name: Ensure latest pip and setuptools | |
| - run: 'pip install pycodestyle && pycodestyle econml' | |
| notebooks: | |
| name: Run notebooks | |
| needs: [eval] | |
| if: ${{ needs.eval.outputs.buildNbs == 'True' }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| kind: [except-customer-scenarios, customer-scenarios] | |
| include: | |
| - kind: "except-customer-scenarios" | |
| extras: "[tf,plt]" | |
| pattern: "(?!CustomerScenarios)" | |
| install_graphviz: true | |
| version: '3.8' # no supported version of tensorflow for 3.9 | |
| - kind: "customer-scenarios" | |
| extras: "[plt,dowhy]" | |
| pattern: "CustomerScenarios" | |
| version: '3.9' | |
| install_graphviz: false | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v3 | |
| name: Checkout repository | |
| with: | |
| ref: ${{ env.ref }} | |
| - uses: actions/setup-python@v4 | |
| name: Setup Python | |
| with: | |
| python-version: ${{ matrix.version }} | |
| - run: python -m pip install --upgrade pip && pip install --upgrade setuptools | |
| name: Ensure latest pip and setuptools | |
| - run: sudo apt-get -yq install graphviz | |
| name: Install graphviz | |
| if: ${{ matrix.install_graphviz }} | |
| # Add verbose flag to pip installation if in debug mode | |
| - run: pip install -e .${{ matrix.extras }} ${{ fromJSON('["","-v"]')[runner.debug] }} ${{ env.use_lkg && '-r lkg-notebook.txt' }} | |
| name: Install econml | |
| # Install notebook requirements (if not already done as part of lkg) | |
| - run: pip install jupyter jupyter-client nbconvert nbformat seaborn xgboost tqdm | |
| name: Install notebook requirements | |
| if: ${{ !env.use_lkg }} | |
| - run: pip freeze --exclude-editable > notebooks-${{ matrix.version }}-${{ matrix.kind }}-requirements.txt | |
| name: Save installed packages | |
| - uses: actions/upload-artifact@v3 | |
| name: Upload installed packages | |
| with: | |
| name: requirements | |
| path: notebooks-${{ matrix.version }}-${{ matrix.kind }}-requirements.txt | |
| - run: pip install pytest pytest-runner coverage | |
| name: Install pytest | |
| - run: python setup.py pytest | |
| name: Run notebook tests | |
| id: run_tests | |
| env: | |
| PYTEST_ADDOPTS: '-m "notebook"' | |
| NOTEBOOK_DIR_PATTERN: ${{ matrix.pattern }} | |
| COVERAGE_PROCESS_START: 'setup.cfg' | |
| - run: mv .coverage .coverage.${{ matrix.kind }} | |
| # Run whether or not the tests passed, but only if they ran at all | |
| if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome) | |
| name: Make coverage filename unique | |
| - uses: actions/upload-artifact@v3 | |
| name: Upload coverage report | |
| if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome) | |
| with: | |
| name: coverage | |
| path: .coverage.${{ matrix.kind }} | |
| tests: | |
| name: "Run tests" | |
| needs: [eval] | |
| if: ${{ needs.eval.outputs.testCode == 'True' }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| python-version: ['3.7', '3.8', '3.9', '3.10'] | |
| kind: [serial, other, dml, main, treatment] | |
| exclude: | |
| # Serial tests fail randomly on mac sometimes, so we don't run them there | |
| - os: macos-latest | |
| kind: serial | |
| # Python 3.7 is broken on the mac runner image, see https://github.com/actions/runner-images/issues/7764 | |
| - os: macos-latest | |
| python-version: '3.7' | |
| # Assign the correct package and testing options for each kind of test | |
| include: | |
| - kind: serial | |
| opts: '-m "serial" -n 1' | |
| extras: "[tf,plt]" | |
| - kind: other | |
| opts: '-m "cate_api" -n auto' | |
| extras: "[tf,plt]" | |
| - kind: dml | |
| opts: '-m "dml"' | |
| extras: "[tf,plt]" | |
| - kind: main | |
| opts: '-m "not (notebook or automl or dml or serial or cate_api or treatment_featurization)" -n 2' | |
| extras: "[tf,plt,dowhy]" | |
| - kind: treatment | |
| opts: '-m "treatment_featurization" -n auto' | |
| extras: "[tf,plt]" | |
| fail-fast: false | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| name: Checkout repository | |
| with: | |
| ref: ${{ env.ref }} | |
| - uses: actions/setup-python@v4 | |
| name: Setup Python | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - run: python -m pip install --upgrade pip && pip install --upgrade setuptools | |
| name: Ensure latest pip and setuptools | |
| # Add verbose flag to pip installation if in debug mode | |
| - run: pip install -e .${{ matrix.extras }} ${{ fromJSON('["","-v"]')[runner.debug] }} ${{ env.use_lkg && '-r lkg.txt' }} | |
| name: Install econml | |
| - run: pip freeze --exclude-editable > tests-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.kind }}-requirements.txt | |
| name: Save installed packages | |
| - uses: actions/upload-artifact@v3 | |
| name: Upload installed packages | |
| with: | |
| name: requirements | |
| path: tests-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.kind }}-requirements.txt | |
| - run: pip install pytest pytest-runner coverage | |
| name: Install pytest | |
| - run: python setup.py pytest | |
| name: Run tests | |
| id: run_tests | |
| env: | |
| PYTEST_ADDOPTS: ${{ matrix.opts }} | |
| COVERAGE_PROCESS_START: 'setup.cfg' | |
| - run: mv .coverage .coverage.${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.kind }} | |
| # Run whether or not the tests passed, but only if they ran at all | |
| if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome) | |
| name: Make coverage filename unique | |
| - uses: actions/upload-artifact@v3 | |
| name: Upload coverage report | |
| if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome) | |
| with: | |
| name: coverage | |
| path: .coverage.${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.kind }} | |
| coverage-report: | |
| name: "Coverage report" | |
| needs: [tests, notebooks] | |
| if: success() || failure() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| name: Checkout repository | |
| with: | |
| ref: ${{ env.ref }} | |
| - uses: actions/download-artifact@v3 | |
| name: Get coverage reports | |
| with: | |
| name: coverage | |
| path: coverage | |
| - uses: actions/setup-python@v4 | |
| name: Setup Python | |
| with: | |
| python-version: '3.8' | |
| - run: pip install coverage | |
| name: Install coverage | |
| - run: coverage combine coverage/ | |
| name: Combine coverage reports | |
| - run: coverage report -m --format=markdown > $GITHUB_STEP_SUMMARY | |
| name: Generate coverage report | |
| - run: coverage html | |
| name: Generate coverage html --fail-under=86 | |
| - uses: actions/upload-artifact@v3 | |
| name: Upload coverage report | |
| with: | |
| name: coverage | |
| path: htmlcov | |
| build: | |
| name: Build package | |
| needs: [eval] | |
| if: ${{ needs.eval.outputs.testCode == 'True' }} | |
| uses: ./.github/workflows/publish-package.yml | |
| with: | |
| publish: false | |
| repository: testpypi | |
| # don't have access to env context here for some reason | |
| ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || null }} | |
| # can't use env context here so need to duplicate expression, but these are true boolean values so don't need extra string logic | |
| use_lkg: ${{ (github.event_name == 'workflow_dispatch' && inputs.use_lkg) || github.event_name == 'pull_request' }} | |
| docs: | |
| name: Build documentation | |
| needs: [eval] | |
| if: ${{ needs.eval.outputs.buildDocs == 'True' }} | |
| uses: ./.github/workflows/publish-documentation.yml | |
| with: | |
| publish: false | |
| environment: test | |
| # don't have access to env context here for some reason | |
| ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || null }} | |
| # can't use env context here so need to duplicate expression, but these are true boolean values so don't need extra string logic | |
| use_lkg: ${{ (github.event_name == 'workflow_dispatch' && inputs.use_lkg) || github.event_name == 'pull_request' }} | |
| verify: | |
| name: Verify CI checks | |
| needs: [lint, notebooks, tests, build, docs] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: exit 1 | |
| name: At least one check failed or was cancelled | |
| if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} | |
| - run: exit 0 | |
| name: All checks passed | |
| if: ${{ !(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }} |