Run all checks #987
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run all checks | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| ref: | |
| description: 'The git ref to build the package for' | |
| required: false | |
| default: '' | |
| type: string | |
| use_lkg: | |
| description: 'Whether to use the last known good versions of dependencies' | |
| required: false | |
| default: True | |
| type: boolean | |
| # nightly | |
| schedule: | |
| - cron: '0 0 * * *' | |
| # Only run once per PR, canceling any previous runs | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| # Precompute the ref if the workflow was triggered by a workflow dispatch rather than copying this logic repeatedly | |
| env: | |
| ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || null }} | |
| # we want to use the LKG if that is explicitly requested, or if we're in a PR, but not a nightly run | |
| # the final `|| ''` is because env vars are always converted to strings and the string 'false' is truthy (!!) | |
| # (see https://github.com/orgs/community/discussions/25645) | |
| use_lkg: ${{ (github.event_name == 'workflow_dispatch' && inputs.use_lkg) || github.event_name == 'pull_request' || ''}} | |
| jobs: | |
| eval: | |
| name: Evaluate changes | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.ref }} | |
| fetch-depth: 2 | |
| # We want to enforce the following rules for PRs: | |
| # * if all modifications are to README.md | |
| # no testing is needed | |
| # * if there are modifications to docs/* or to any code | |
| # then docs need to be built to verify consistency | |
| # * if there are modifications to notebooks/* or to any code | |
| # then notebooks need to be run to verify consistency | |
| # * for any code changes (or changes to metadata files) | |
| # linting and testing should be run | |
| # For a PR build, HEAD will be the merge commit, and we want to diff against the base branch, | |
| # which will be the first parent: HEAD^ | |
| # (For non-PR changes, we will always perform all CI tasks) | |
| # Note that GitHub Actions provides path filters, but they operate at the workflow level, not the job level | |
| - name: Determine type of code change | |
| run: | | |
| if ($env:GITHUB_EVENT_NAME -eq 'pull_request') { | |
| $editedFiles = git diff HEAD^ --name-only | |
| $editedFiles # echo edited files to enable easier debugging | |
| $codeChanges = $false | |
| $docChanges = $false | |
| $nbChanges = $false | |
| $changeType = "none" | |
| foreach ($file in $editedFiles) { | |
| switch -Wildcard ($file) { | |
| "README.md" { Continue } | |
| ".gitignore" { Continue } | |
| "econml/_version.py" { Continue } | |
| "prototypes/*" { Continue } | |
| "images/*" { Continue } | |
| "doc/*" { $docChanges = $true; Continue } | |
| "notebooks/*" { $nbChanges = $true; Continue } | |
| default { $codeChanges = $true; Continue } | |
| } | |
| } | |
| } | |
| echo "buildDocs=$(($env:GITHUB_EVENT_NAME -ne 'pull_request') -or ($docChanges -or $codeChanges))" >> $env:GITHUB_OUTPUT | |
| echo "buildNbs=$(($env:GITHUB_EVENT_NAME -ne 'pull_request') -or ($nbChanges -or $codeChanges))" >> $env:GITHUB_OUTPUT | |
| echo "testCode=$(($env:GITHUB_EVENT_NAME -ne 'pull_request') -or $codeChanges)" >> $env:GITHUB_OUTPUT | |
| shell: pwsh | |
| id: eval | |
| outputs: | |
| buildDocs: ${{ steps.eval.outputs.buildDocs }} | |
| buildNbs: ${{ steps.eval.outputs.buildNbs }} | |
| testCode: ${{ steps.eval.outputs.testCode }} | |
| lint: | |
| name: Lint code | |
| needs: [eval] | |
| if: ${{ needs.eval.outputs.testCode == 'True' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.ref }} | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: Ensure latest pip and setuptools | |
| run: python -m pip install --upgrade pip && pip install --upgrade setuptools | |
| - name: Run Ruff | |
| run: 'pip install ruff && ruff check' | |
| notebooks: | |
| name: Run notebooks | |
| needs: [eval] | |
| if: ${{ needs.eval.outputs.buildNbs == 'True' }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| kind: [except-customer-scenarios, customer-scenarios] | |
| include: | |
| - kind: "except-customer-scenarios" | |
| extras: "[tf,plt,ray]" | |
| pattern: "(?!CustomerScenarios)" | |
| install_graphviz: true | |
| version: '3.8' # no supported version of tensorflow for 3.9 | |
| - kind: "customer-scenarios" | |
| extras: "[plt,dowhy]" | |
| pattern: "CustomerScenarios" | |
| version: '3.9' | |
| install_graphviz: false | |
| fail-fast: false | |
| env: | |
| id_string: ${{ matrix.kind }}-${{ matrix.version }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.ref }} | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.version }} | |
| - name: Install uv | |
| # check if we're running on windows | |
| run: ${{ runner.os == 'Windows' && 'irm https://astral.sh/uv/install.ps1 | iex' || 'curl -LsSf https://astral.sh/uv/install.sh | sh' }} | |
| - name: Install graphviz | |
| run: sudo apt-get -yq install graphviz | |
| if: ${{ matrix.install_graphviz }} | |
| # Add verbose flag to pip installation if in debug mode | |
| - name: Install econml | |
| run: uv pip install --system -e .${{ matrix.extras }} ${{ fromJSON('["","-v"]')[runner.debug] }} ${{ env.use_lkg && '-r lkg-notebook.txt' }} | |
| # Install notebook requirements (if not already done as part of lkg) | |
| - name: Install notebook requirements | |
| run: uv pip install --system jupyter jupyter-client nbconvert nbformat seaborn xgboost tqdm | |
| if: ${{ !env.use_lkg }} | |
| - name: Save installed packages | |
| run: pip freeze --exclude-editable > notebooks-${{ env.id_string }}-requirements.txt | |
| - name: Upload installed packages | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: requirements-${{ env.id_string }} | |
| path: notebooks-${{ env.id_string }}-requirements.txt | |
| - name: Install pytest | |
| run: uv pip install --system pytest pytest-xdist pytest-cov coverage[toml] | |
| - name: Run notebook tests | |
| run: python -m pytest | |
| id: run_tests | |
| env: | |
| PYTEST_ADDOPTS: '-m "notebook"' | |
| NOTEBOOK_DIR_PATTERN: ${{ matrix.pattern }} | |
| COVERAGE_PROCESS_START: 'pyproject.toml' | |
| - name: Make coverage filename unique | |
| run: mv .coverage .coverage.${{ env.id_string }} | |
| # Run whether or not the tests passed, but only if they ran at all | |
| if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome) | |
| - name: Upload coverage report | |
| uses: actions/upload-artifact@v4 | |
| if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome) | |
| with: | |
| name: coverage-${{ env.id_string }} | |
| path: .coverage.${{ env.id_string }} | |
| # need to include hidden files since path starts with . | |
| include-hidden-files: true | |
| - name: Make test XML filename unique | |
| run: mv junit/test-results.xml ${{ env.id_string }}-test-results.xml | |
| # Run whether or not the tests passed, but only if they ran at all | |
| if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome) | |
| - name: Upload test XML files | |
| uses: actions/upload-artifact@v4 | |
| if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome) | |
| with: | |
| name: tests-${{ env.id_string }} | |
| path: ${{ env.id_string }}-test-results.xml | |
| - name: Upload notebook outputs | |
| uses: actions/upload-artifact@v4 | |
| if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome) | |
| with: | |
| name: notebooks-${{ env.id_string }} | |
| path: notebooks/output/ | |
| tests: | |
| name: "Run tests" | |
| needs: [eval] | |
| if: ${{ needs.eval.outputs.testCode == 'True' }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] | |
| kind: [serial, other, dml, main, treatment, ray] | |
| exclude: | |
| # Serial tests fail randomly on mac sometimes, so we don't run them there | |
| - os: macos-latest | |
| kind: serial | |
| # Mac ARM doesn't support tensorflow versions compatible with python 3.8 | |
| - os: macos-latest | |
| python-version: '3.8' | |
| # Ray tests run out of memory on Windows | |
| - os: windows-latest | |
| kind: ray | |
| # Ray doesn't currently support Python 3.12 | |
| - python-version: '3.12' | |
| kind: ray | |
| # Assign the correct package and testing options for each kind of test | |
| include: | |
| - kind: serial | |
| opts: '-m "serial and not ray" -n 1' | |
| extras: "" | |
| - kind: other | |
| opts: '-m "cate_api and not ray" -n auto' | |
| extras: "[tf,plt]" | |
| - kind: dml | |
| opts: '-m "dml and not ray"' | |
| extras: "[tf,plt]" | |
| - kind: main | |
| opts: '-m "not (notebook or automl or dml or serial or cate_api or treatment_featurization or ray)" -n 2' | |
| extras: "[tf,plt,dowhy]" | |
| - kind: treatment | |
| opts: '-m "treatment_featurization and not ray" -n auto' | |
| extras: "[tf,plt]" | |
| - kind: ray | |
| opts: '-m "ray"' | |
| extras: "[ray]" | |
| fail-fast: false | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| id_string: ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.kind }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.ref }} | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install libomp | |
| # lightgbm needs libomp on mac | |
| run: brew install libomp | |
| if: matrix.os == 'macos-latest' | |
| - name: Install uv | |
| # check if we're running on windows | |
| run: ${{ runner.os == 'Windows' && 'irm https://astral.sh/uv/install.ps1 | iex' || 'curl -LsSf https://astral.sh/uv/install.sh | sh' }} | |
| # Add verbose flag to pip installation if in debug mode | |
| - name: Install econml | |
| run: uv pip install --system -e .${{ matrix.extras }} ${{ fromJSON('["","-v"]')[runner.debug] }} ${{ env.use_lkg && '-r lkg.txt' }} | |
| - name: Save installed packages | |
| run: pip freeze --exclude-editable > tests-${{ env.id_string }}-requirements.txt | |
| - name: Upload installed packages | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: requirements-${{ env.id_string }} | |
| path: tests-${{ env.id_string }}-requirements.txt | |
| - name: Install pytest | |
| run: uv pip install --system pytest pytest-xdist pytest-cov coverage[toml] | |
| - name: Run tests | |
| run: python -m pytest | |
| id: run_tests | |
| env: | |
| PYTEST_ADDOPTS: ${{ matrix.opts }} | |
| COVERAGE_PROCESS_START: 'pyproject.toml' | |
| - name: Make coverage filename unique | |
| run: mv .coverage .coverage.${{ env.id_string }} | |
| # Run whether or not the tests passed, but only if they ran at all | |
| if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome) | |
| - name: Upload coverage report | |
| uses: actions/upload-artifact@v4 | |
| if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome) | |
| with: | |
| name: coverage-${{ env.id_string }} | |
| path: .coverage.${{ env.id_string }} | |
| # need to include hidden files since path starts with . | |
| include-hidden-files: true | |
| - name: Make test XML filename unique | |
| run: mv junit/test-results.xml ${{ env.id_string }}-test-results.xml | |
| # Run whether or not the tests passed, but only if they ran at all | |
| if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome) | |
| - name: Upload test XML files | |
| uses: actions/upload-artifact@v4 | |
| if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome) | |
| with: | |
| name: tests-${{ env.id_string }} | |
| path: ${{ env.id_string }}-test-results.xml | |
| store-reqs-per-env: | |
| name: Store requirements for LKG updates | |
| if: (success() || failure()) && (github.event_name == 'workflow_dispatch' && !inputs.use_lkg || github.event_name == 'schedule') | |
| strategy: | |
| matrix: | |
| kind: [tests] | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] | |
| extras: ["[tf,plt,dowhy,ray]"] | |
| include: | |
| # explicitly add the two notebook extras | |
| - kind: notebooks | |
| os: ubuntu-latest | |
| python-version: '3.8' | |
| extras: "[tf,plt,ray]" | |
| - kind: notebooks | |
| os: ubuntu-latest | |
| python-version: '3.9' | |
| extras: "[plt,dowhy]" | |
| exclude: | |
| # Mac ARM doesn't support tensorflow versions compatible with python 3.8 | |
| - os: macos-latest | |
| python-version: '3.8' | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.ref }} | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| # check if we're running on windows | |
| run: ${{ runner.os == 'Windows' && 'irm https://astral.sh/uv/install.ps1 | iex' || 'curl -LsSf https://astral.sh/uv/install.sh | sh' }} | |
| - name: Install econml | |
| # add all extras used by any of the individual tests, which is a superset of what's actually used anywhere | |
| run: uv pip install --system -e .${{ matrix.extras }} | |
| - name: Install notebook requirements | |
| run: uv pip install --system jupyter jupyter-client nbconvert nbformat seaborn xgboost tqdm | |
| if: matrix.kind == 'notebooks' | |
| - name: Save installed packages | |
| run: pip freeze --exclude-editable > lkg-${{ matrix.kind }}-${{ matrix.os }}-${{ matrix.python-version }}.txt | |
| - name: Upload installed packages | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: lkg-${{ matrix.kind }}-${{ matrix.os }}-${{ matrix.python-version }} | |
| path: lkg-${{ matrix.kind }}-${{ matrix.os }}-${{ matrix.python-version }}.txt | |
| coverage-report: | |
| name: "Coverage report" | |
| needs: [tests, notebooks] | |
| if: success() || failure() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.ref }} | |
| - name: Get coverage reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: coverage-* | |
| path: coverage | |
| merge-multiple: true | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.8' | |
| - name: Install coverage | |
| run: pip install coverage[toml] | |
| - name: Combine coverage reports | |
| run: coverage combine coverage/ | |
| - name: Generate coverage report | |
| run: coverage report -m --format=markdown > $GITHUB_STEP_SUMMARY | |
| - name: Generate coverage html --fail-under=86 | |
| run: coverage html | |
| - name: Upload coverage report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-report | |
| path: htmlcov | |
| merge-artifacts: | |
| name: "Merge artifacts" | |
| needs: [coverage-report, tests, notebooks, store-reqs-per-env] | |
| if: success() || failure() | |
| strategy: | |
| matrix: | |
| artifact: [requirements, tests, coverage, notebooks, lkg-tests, lkg-notebooks] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: "Merge artifacts" | |
| uses: actions/upload-artifact/merge@v4 | |
| with: | |
| name: ${{ matrix.artifact }} | |
| pattern: "${{ matrix.artifact }}-*" | |
| delete-merged: true | |
| # if we are re-running a job in a subsequent attempt, some of the other artifacts may not exist in this attempt (e.g. notebooks, if only non-notebook tests failed) | |
| # Unlike with plain upload-artifact, there's no way to ignore the situation where no files are found when using the v4 merge action | |
| # (see https://github.com/actions/upload-artifact/issues/520), so just continue on error isntead | |
| continue-on-error: true | |
| generate-lkg: | |
| name: "Generate updated last known good files" | |
| needs: [merge-artifacts] | |
| if: (success() || failure()) && (github.event_name == 'workflow_dispatch' && !inputs.use_lkg || github.event_name == 'schedule') | |
| strategy: | |
| matrix: | |
| kind: [tests, notebooks] | |
| include: | |
| - kind: tests | |
| output-name: lkg | |
| - kind: notebooks | |
| output-name: lkg-notebook | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.ref }} | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| - name: Download files | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: lkg-${{ matrix.kind }} | |
| path: requirements | |
| - name: "Generate last known good versions" | |
| run: python .github/workflows/generate_lkg.py requirements "lkg-${{ matrix.kind }}-(?P<os>[a-z]+)-(latest|[0-9.]+)-(?P<pyversion>[0-9.]+).txt" ${{ matrix.output-name }}.txt | |
| - name: "Upload last known good versions" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.output-name }} | |
| path: ${{ matrix.output-name }}.txt | |
| merge-lkg: | |
| name: "Merge last known good versions" | |
| needs: [generate-lkg] | |
| if: success() || failure() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: "Merge last known good versions" | |
| uses: actions/upload-artifact/merge@v4 | |
| with: | |
| # can't use just "lkg" for the name since that artifact already exists and merge won't overwrite it even when delete-merged is true | |
| name: lkg-files | |
| pattern: "lkg*" | |
| delete-merged: true | |
| build: | |
| name: Build package | |
| needs: [eval] | |
| if: ${{ needs.eval.outputs.testCode == 'True' }} | |
| uses: ./.github/workflows/publish-package.yml | |
| with: | |
| publish: false | |
| environment: test | |
| # don't have access to env context here for some reason | |
| ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || null }} | |
| # can't use env context here so need to duplicate expression, but these are true boolean values so don't need extra string logic | |
| use_lkg: ${{ (github.event_name == 'workflow_dispatch' && inputs.use_lkg) || github.event_name == 'pull_request' }} | |
| docs: | |
| name: Build documentation | |
| needs: [eval] | |
| if: ${{ needs.eval.outputs.buildDocs == 'True' }} | |
| uses: ./.github/workflows/publish-documentation.yml | |
| with: | |
| publish: false | |
| environment: test | |
| # don't have access to env context here for some reason | |
| ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || null }} | |
| # can't use env context here so need to duplicate expression, but these are true boolean values so don't need extra string logic | |
| use_lkg: ${{ (github.event_name == 'workflow_dispatch' && inputs.use_lkg) || github.event_name == 'pull_request' }} | |
| verify: | |
| name: Verify CI checks | |
| needs: [lint, notebooks, tests, build, docs] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: At least one check failed or was cancelled | |
| run: exit 1 | |
| if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} | |
| - name: All checks passed | |
| run: exit 0 | |
| if: ${{ !(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }} |