From 30b0f89421c6c4d39738b0caa8b0cb0112ac9bf2 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 13:32:11 -0400 Subject: [PATCH] def files, bash scipts for containers, testing yaml --- .github/workflows/test-container-dl-calc.yaml | 67 +++++++++++++++++++ container/beethoven_dl_calc.def | 6 +- container/build_dl_calc.sh | 8 +++ container/run_container_dl_calc.sh | 23 +++++++ container/run_dl_calc_local_tests.sh | 23 +++++++ tests/testthat/test_calculate.R | 13 ++++ tests/testthat/test_download.R | 13 ++++ 7 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/test-container-dl-calc.yaml create mode 100644 container/build_dl_calc.sh create mode 100644 container/run_container_dl_calc.sh create mode 100644 container/run_dl_calc_local_tests.sh create mode 100644 tests/testthat/test_calculate.R create mode 100644 tests/testthat/test_download.R diff --git a/.github/workflows/test-container-dl-calc.yaml b/.github/workflows/test-container-dl-calc.yaml new file mode 100644 index 0000000..2ae5810 --- /dev/null +++ b/.github/workflows/test-container-dl-calc.yaml @@ -0,0 +1,67 @@ +name: CI tests for download and calculate with Apptainer + +on: + push: + branches: + - [main, dev] + pull_request: + branches: + - [main, dev] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Install Apptainer dependencies + run: | + sudo apt-get update && sudo apt-get install -y \ + libseccomp-dev \ + squashfs-tools \ + cryptsetup \ + wget + + - name: Install Apptainer + run: | + wget https://github.com/apptainer/apptainer/releases/download/v1.1.0/apptainer-1.1.0.tar.gz + tar -xvf apptainer-1.1.0.tar.gz + cd apptainer-1.1.0 + ./mconfig && make -C builddir && sudo make -C builddir install + + - name: Restore .sif file from cache + id: cache-sif + uses: actions/cache@v3 + with: + path: beethoven_dl_calc.sif + key: sif-cache-${{ hashFiles('../../containers/beethoven_dl_calc.def') }} + restore-keys: sif-cache- + + - name: Build the Apptainer container (if cache miss) + if: steps.cache-sif.outputs.cache-hit != 'true' + run: | + apptainer build my_container.sif path/to/Singularity.def + + - name: Cache the .sif file + if: steps.cache-sif.outputs.cache-hit != 'true' + uses: actions/cache@v3 + with: + path: beethoven_dl_calc.sif + key: sif-cache-${{ hashFiles('../../containers/beethoven_dl_calc.def') }} + + - name: Create run_dl_calc_tests.R dynamically + run: | + echo 'testthat::test_file("/mnt/tests/testthat/test_download.R")' > tests/testthat/run_dl_calc_tests.R + echo 'testthat::test_file("/mnt/tests/testthat/test_calculate.R")' >> tests/testthat/run_dl_calc_tests.R + + + - name: Run tests inside the container + run: | + apptainer exec \ + --bind $PWD/inst:/pipeline \ + --bind $PWD/input:/input \ + --bind $PWD/_targets:/opt/_targets \ + --bind $PWD:/mnt \ + Rscript mnt/tests/testhat/run_dl_calc_tests.R diff --git a/container/beethoven_dl_calc.def b/container/beethoven_dl_calc.def index 09beae1..5b5227d 100644 --- a/container/beethoven_dl_calc.def +++ b/container/beethoven_dl_calc.def @@ -1,10 +1,6 @@ BootStrap: docker From: rocker/geospatial:latest -%files - /ddn/gs1/tools/set/R432/lib64/R/library /R_packages - - %post # Update package list apt-get update @@ -27,7 +23,7 @@ From: rocker/geospatial:latest mkdir /input mkdir /opt/_targets - Rscript -e "install.packages(c('pak','targets','tarchetypes','crew','crew.cluster'))" + Rscript -e "install.packages(c('pak','targets','tarchetypes','crew','crew.cluster','testthat'))" Rscript -e "pak::pkg_install('NIEHS/amadeus')" Rscript -e "remotes::install_github('NIEHS/beethoven')" diff --git a/container/build_dl_calc.sh b/container/build_dl_calc.sh new file mode 100644 index 0000000..b66b1a0 --- /dev/null +++ b/container/build_dl_calc.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# usage: build_apptainer_image.sh [full file path] +# where full file path ends with .sif, with full directory path to save the image +# after the image is built, group write/execution privileges are given + +# Recommended to run this script interactively via `sh build_dl_calc.sh` +apptainer build --fakeroot beethoven_dl_calc.sif beethoven_dl_calc.def \ No newline at end of file diff --git a/container/run_container_dl_calc.sh b/container/run_container_dl_calc.sh new file mode 100644 index 0000000..49e7eb0 --- /dev/null +++ b/container/run_container_dl_calc.sh @@ -0,0 +1,23 @@ +#!/bin/bash +#SBATCH --job-name=beethoven_001 +#SBATCH --partition=geo +#SBATCH --mem=128G +#SBATCH --cpus-per-task=4 +#SBATCH --ntasks=16 +#SBATCH --output=slurm_messages/slurm-%j.out +#SBATCH --error=slurm_messages/slurm-%j.err +#SBATCH --mail-user=kyle.messier@nih.gov +#SBATCH --mail-type=ALL + + + + +apptainer exec \ + --bind $PWD/inst:/pipeline \ + --bind $PWD/input:/input \ + --bind $PWD/_targets:/opt/_targets \ + --bind $PWD:/mnt \ + beethoven_dl_calc.sif \ + Rscript /mnt/run.R + + diff --git a/container/run_dl_calc_local_tests.sh b/container/run_dl_calc_local_tests.sh new file mode 100644 index 0000000..9404e8a --- /dev/null +++ b/container/run_dl_calc_local_tests.sh @@ -0,0 +1,23 @@ +#!/bin/bash +#SBATCH --job-name=dl_calc_tests +#SBATCH --partition=geo +#SBATCH --mem=128G +#SBATCH --cpus-per-task=4 +#SBATCH --ntasks=16 +#SBATCH --output=slurm_messages/slurm-%j.out +#SBATCH --error=slurm_messages/slurm-%j.err +#SBATCH --mail-user=kyle.messier@nih.gov +#SBATCH --mail-type=ALL + + + + +apptainer exec \ + --bind $PWD/../inst:/pipeline \ + --bind $PWD/../input:/input \ + --bind $PWD/../_targets:/opt/_targets \ + --bind $PWD/..:/mnt \ + beethoven_dl_calc.sif \ + Rscript /mnt/tests/testthat/test_download.R + + diff --git a/tests/testthat/test_calculate.R b/tests/testthat/test_calculate.R new file mode 100644 index 0000000..94a4871 --- /dev/null +++ b/tests/testthat/test_calculate.R @@ -0,0 +1,13 @@ +################################################################################ +##### unit and integration tests for covariate calculations +##### Currently arbitrary placeholders +##### + +################################################################################ +##### +testthat::test_that("calculate targets work", { + + a <- 1 + testthat::expect_equal(a, 1) + +}) diff --git a/tests/testthat/test_download.R b/tests/testthat/test_download.R new file mode 100644 index 0000000..24cebc1 --- /dev/null +++ b/tests/testthat/test_download.R @@ -0,0 +1,13 @@ +################################################################################ +##### unit and integration tests for data download +##### Currently arbitrary placeholders +##### + +################################################################################ +##### loadargs +testthat::test_that("download targets work", { + + a <- 1 + testthat::expect_equal(a, 1) + +})