Skip to content
Run all checks

Improve discreteness handling, allow binary outcomes #280

Sign in to view logs

Improve discreteness handling, allow binary outcomes

Improve discreteness handling, allow binary outcomes #280

Workflow file for this run

.github/workflows/ci.yml at 058c3e8
name: Run all checks
on:
pull_request:
branches:
- main
workflow_dispatch:
inputs:
ref:
description: 'The git ref to build the package for'
required: false
default: ''
type: string
use_lkg:
description: 'Whether to use the last known good versions of dependencies'
required: false
default: True
type: boolean
# nightly
schedule:
- cron: '0 0 * * *'
# Only run once per PR, canceling any previous runs
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
# Precompute the ref if the workflow was triggered by a workflow dispatch rather than copying this logic repeatedly
env:
ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || null }}
# we want to use the LKG if that is explicitly requested, or if we're in a PR, but not a nightly run
# the final `|| ''` is because env vars are always converted to strings and the string 'false' is truthy (!!)
# (see https://github.com/orgs/community/discussions/25645)
use_lkg: ${{ (github.event_name == 'workflow_dispatch' && inputs.use_lkg) || github.event_name == 'pull_request' || ''}}
jobs:
eval:
name: Evaluate changes
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
name: Checkout repository
with:
ref: ${{ env.ref }}
fetch-depth: 2
# We want to enforce the following rules for PRs:
# * if all modifications are to README.md
# no testing is needed
# * if there are modifications to docs/* or to any code
# then docs need to be built to verify consistency
# * if there are modifications to notebooks/* or to any code
# then notebooks need to be run to verify consistency
# * for any code changes (or changes to metadata files)
# linting and testing should be run
# For a PR build, HEAD will be the merge commit, and we want to diff against the base branch,
# which will be the first parent: HEAD^
# (For non-PR changes, we will always perform all CI tasks)
# Note that GitHub Actions provides path filters, but they operate at the workflow level, not the job level
- run: |
if ($env:GITHUB_EVENT_NAME -eq 'pull_request') {
$editedFiles = git diff HEAD^ --name-only
$editedFiles # echo edited files to enable easier debugging
$codeChanges = $false
$docChanges = $false
$nbChanges = $false
$changeType = "none"
foreach ($file in $editedFiles) {
switch -Wildcard ($file) {
"README.md" { Continue }
".gitignore" { Continue }
"econml/_version.py" { Continue }
"prototypes/*" { Continue }
"images/*" { Continue }
"doc/*" { $docChanges = $true; Continue }
"notebooks/*" { $nbChanges = $true; Continue }
default { $codeChanges = $true; Continue }
}
}
}
echo "buildDocs=$(($env:GITHUB_EVENT_NAME -ne 'pull_request') -or ($docChanges -or $codeChanges))" >> $env:GITHUB_OUTPUT
echo "buildNbs=$(($env:GITHUB_EVENT_NAME -ne 'pull_request') -or ($nbChanges -or $codeChanges))" >> $env:GITHUB_OUTPUT
echo "testCode=$(($env:GITHUB_EVENT_NAME -ne 'pull_request') -or $codeChanges)" >> $env:GITHUB_OUTPUT
shell: pwsh
name: Determine type of code change
id: eval
outputs:
buildDocs: ${{ steps.eval.outputs.buildDocs }}
buildNbs: ${{ steps.eval.outputs.buildNbs }}
testCode: ${{ steps.eval.outputs.testCode }}
lint:
name: Lint code
needs: [eval]
if: ${{ needs.eval.outputs.testCode == 'True' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
name: Checkout repository
with:
ref: ${{ env.ref }}
- uses: actions/setup-python@v4
name: Setup Python
with:
python-version: '3.9'
- run: python -m pip install --upgrade pip && pip install --upgrade setuptools
name: Ensure latest pip and setuptools
- run: 'pip install pycodestyle && pycodestyle econml'
notebooks:
name: Run notebooks
needs: [eval]
if: ${{ needs.eval.outputs.buildNbs == 'True' }}
runs-on: ubuntu-latest
strategy:
matrix:
kind: [except-customer-scenarios, customer-scenarios]
include:
- kind: "except-customer-scenarios"
extras: "[tf,plt]"
pattern: "(?!CustomerScenarios)"
install_graphviz: true
version: '3.8' # no supported version of tensorflow for 3.9
- kind: "customer-scenarios"
extras: "[plt,dowhy]"
pattern: "CustomerScenarios"
version: '3.9'
install_graphviz: false
fail-fast: false
steps:
- uses: actions/checkout@v3
name: Checkout repository
with:
ref: ${{ env.ref }}
- uses: actions/setup-python@v4
name: Setup Python
with:
python-version: ${{ matrix.version }}
- run: python -m pip install --upgrade pip && pip install --upgrade setuptools
name: Ensure latest pip and setuptools
- run: sudo apt-get -yq install graphviz
name: Install graphviz
if: ${{ matrix.install_graphviz }}
# Add verbose flag to pip installation if in debug mode
- run: pip install -e .${{ matrix.extras }} ${{ fromJSON('["","-v"]')[runner.debug] }} ${{ env.use_lkg && '-r lkg-notebook.txt' }}
name: Install econml
# Install notebook requirements (if not already done as part of lkg)
- run: pip install jupyter jupyter-client nbconvert nbformat seaborn xgboost tqdm
name: Install notebook requirements
if: ${{ !env.use_lkg }}
- run: pip freeze --exclude-editable > notebooks-${{ matrix.version }}-${{ matrix.kind }}-requirements.txt
name: Save installed packages
- uses: actions/upload-artifact@v3
name: Upload installed packages
with:
name: requirements
path: notebooks-${{ matrix.version }}-${{ matrix.kind }}-requirements.txt
- run: pip install pytest pytest-runner coverage
name: Install pytest
- run: python setup.py pytest
name: Run notebook tests
id: run_tests
env:
PYTEST_ADDOPTS: '-m "notebook"'
NOTEBOOK_DIR_PATTERN: ${{ matrix.pattern }}
COVERAGE_PROCESS_START: 'setup.cfg'
- run: mv .coverage .coverage.${{ matrix.kind }}
# Run whether or not the tests passed, but only if they ran at all
if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome)
name: Make coverage filename unique
- uses: actions/upload-artifact@v3
name: Upload coverage report
if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome)
with:
name: coverage
path: .coverage.${{ matrix.kind }}
tests:
name: "Run tests"
needs: [eval]
if: ${{ needs.eval.outputs.testCode == 'True' }}
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
python-version: ['3.7', '3.8', '3.9', '3.10']
kind: [serial, other, dml, main, treatment]
exclude:
# Serial tests fail randomly on mac sometimes, so we don't run them there
- os: macos-latest
kind: serial
# Python 3.7 is broken on the mac runner image, see https://github.com/actions/runner-images/issues/7764
- os: macos-latest
python-version: '3.7'
# Assign the correct package and testing options for each kind of test
include:
- kind: serial
opts: '-m "serial" -n 1'
extras: "[tf,plt]"
- kind: other
opts: '-m "cate_api" -n auto'
extras: "[tf,plt]"
- kind: dml
opts: '-m "dml"'
extras: "[tf,plt]"
- kind: main
opts: '-m "not (notebook or automl or dml or serial or cate_api or treatment_featurization)" -n 2'
extras: "[tf,plt,dowhy]"
- kind: treatment
opts: '-m "treatment_featurization" -n auto'
extras: "[tf,plt]"
fail-fast: false
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
name: Checkout repository
with:
ref: ${{ env.ref }}
- uses: actions/setup-python@v4
name: Setup Python
with:
python-version: ${{ matrix.python-version }}
- run: python -m pip install --upgrade pip && pip install --upgrade setuptools
name: Ensure latest pip and setuptools
# Add verbose flag to pip installation if in debug mode
- run: pip install -e .${{ matrix.extras }} ${{ fromJSON('["","-v"]')[runner.debug] }} ${{ env.use_lkg && '-r lkg.txt' }}
name: Install econml
- run: pip freeze --exclude-editable > tests-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.kind }}-requirements.txt
name: Save installed packages
- uses: actions/upload-artifact@v3
name: Upload installed packages
with:
name: requirements
path: tests-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.kind }}-requirements.txt
- run: pip install pytest pytest-runner coverage
name: Install pytest
- run: python setup.py pytest
name: Run tests
id: run_tests
env:
PYTEST_ADDOPTS: ${{ matrix.opts }}
COVERAGE_PROCESS_START: 'setup.cfg'
- run: mv .coverage .coverage.${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.kind }}
# Run whether or not the tests passed, but only if they ran at all
if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome)
name: Make coverage filename unique
- uses: actions/upload-artifact@v3
name: Upload coverage report
if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome)
with:
name: coverage
path: .coverage.${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.kind }}
coverage-report:
name: "Coverage report"
needs: [tests, notebooks]
if: success() || failure()
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
name: Checkout repository
with:
ref: ${{ env.ref }}
- uses: actions/download-artifact@v3
name: Get coverage reports
with:
name: coverage
path: coverage
- uses: actions/setup-python@v4
name: Setup Python
with:
python-version: '3.8'
- run: pip install coverage
name: Install coverage
- run: coverage combine coverage/
name: Combine coverage reports
- run: coverage report -m --format=markdown > $GITHUB_STEP_SUMMARY
name: Generate coverage report
- run: coverage html
name: Generate coverage html --fail-under=86
- uses: actions/upload-artifact@v3
name: Upload coverage report
with:
name: coverage
path: htmlcov
build:
name: Build package
needs: [eval]
if: ${{ needs.eval.outputs.testCode == 'True' }}
uses: ./.github/workflows/publish-package.yml
with:
publish: false
repository: testpypi
# don't have access to env context here for some reason
ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || null }}
# can't use env context here so need to duplicate expression, but these are true boolean values so don't need extra string logic
use_lkg: ${{ (github.event_name == 'workflow_dispatch' && inputs.use_lkg) || github.event_name == 'pull_request' }}
docs:
name: Build documentation
needs: [eval]
if: ${{ needs.eval.outputs.buildDocs == 'True' }}
uses: ./.github/workflows/publish-documentation.yml
with:
publish: false
environment: test
# don't have access to env context here for some reason
ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || null }}
# can't use env context here so need to duplicate expression, but these are true boolean values so don't need extra string logic
use_lkg: ${{ (github.event_name == 'workflow_dispatch' && inputs.use_lkg) || github.event_name == 'pull_request' }}
verify:
name: Verify CI checks
needs: [lint, notebooks, tests, build, docs]
if: always()
runs-on: ubuntu-latest
steps:
- run: exit 1
name: At least one check failed or was cancelled
if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
- run: exit 0
name: All checks passed
if: ${{ !(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}