From e3f9539051790898f4f4b6b5ede28c7aa6f6e42f Mon Sep 17 00:00:00 2001 From: mitchellmanware Date: Tue, 24 Sep 2024 10:05:41 -0400 Subject: [PATCH 01/20] export convert_cv_index_rset; skip_on_ci; expect_warning --- NAMESPACE | 1 + R/base_learner.R | 1 + tests/testthat/test-calculate.R | 1 + tests/testthat/test-injection.R | 3 +- tests/testthat/test-meta.R | 2 +- tools/targets-old/targets_configuration.csv | 74 +-------------------- tools/targets-old/targets_lookup.csv | 4 +- 7 files changed, 12 insertions(+), 74 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 5733c210..d82ed22b 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -9,6 +9,7 @@ export(calc_geos_strict) export(calc_gmted_direct) export(calc_narr2) export(calculate) +export(convert_cv_index_rset) export(feature_raw_download) export(fit_base_learner) export(fit_base_tune) diff --git a/R/base_learner.R b/R/base_learner.R index 7f5de0a4..a4f0fa18 100644 --- a/R/base_learner.R +++ b/R/base_learner.R @@ -506,6 +506,7 @@ assign_learner_cv <- #' training-test data.frames and a column of labels. #' @author Insang Song #' @importFrom rsample make_splits manual_rset +#' @export convert_cv_index_rset <- function( cvindex, diff --git a/tests/testthat/test-calculate.R b/tests/testthat/test-calculate.R index 91e6e62e..4b6e49ee 100644 --- a/tests/testthat/test-calculate.R +++ b/tests/testthat/test-calculate.R @@ -5,6 +5,7 @@ ################################################################################ ##### par_narr testthat::test_that("par_narr (weasd + omega)", { + testthat::skip_on_ci() withr::local_package("rlang") # sample location diff --git a/tests/testthat/test-injection.R b/tests/testthat/test-injection.R index aa4403e9..dfa6e682 100644 --- a/tests/testthat/test-injection.R +++ b/tests/testthat/test-injection.R @@ -123,7 +123,7 @@ testthat::test_that("set_target_years", { ################################################################################ ##### inject_calc -testthat::test_that("inject_calc (hms + hms)", { +testthat::test_that("inject_calc (hms + nei)", { withr::local_package("rlang") @@ -188,6 +188,7 @@ testthat::test_that("inject_calc (hms + hms)", { crs = "EPSG:4326" ) + testthat::skip_on_ci() # expect no errors (nei) testthat::expect_no_error( nei_injected <- inject_calculate( diff --git a/tests/testthat/test-meta.R b/tests/testthat/test-meta.R index 056ad75d..4c96426c 100644 --- a/tests/testthat/test-meta.R +++ b/tests/testthat/test-meta.R @@ -257,7 +257,7 @@ testthat::test_that("predict_meta_learner", { ) # expect no error for meta fit - testthat::expect_no_error( + testthat::expect_warning( meta4 <- fit_meta_learner( data = dt_meta, p_col_sel = 0.5, diff --git a/tools/targets-old/targets_configuration.csv b/tools/targets-old/targets_configuration.csv index c3a3fe04..00b1dd77 100644 --- a/tools/targets-old/targets_configuration.csv +++ b/tools/targets-old/targets_configuration.csv @@ -1,71 +1,3 @@ -"index","last_updated","command","name_targets_short","value","class","name_targets_long","rclass","role_suffix","stage","source","spacetime","remarks" -0,"2024/3/15","paste0","root_custom","../../../../group/set/Projects/NRT-AP-Model/","path","character_feature_userroot","character","feature_userroot","","","","" -1,"2024/3/15","getwd","root_absolute","missing","path","character_feature_absoluteroot","character","feature_absoluteroot","","","","" -2,"2024/3/15","paste0","root_relative",".","path","character_feature_relativeroot","character","feature_relativeroot","","","","" -3,"2024/3/15","file.path","dir_input","input","path","character_feature_raw","character","feature","raw","","","" -4,"2024/3/15","file.path","dir_output","output","path","character_feature_base_fit","character","feature_base","fit","","","" -5,"2024/3/15","file.path","dir_input_aqs","input/aqs","path","character_feature_raw_aqs","character","feature","raw","aqs","","" -6,"2024/3/15","file.path","dir_input_nei","input/nei","path","character_feature_raw_nei","character","feature","raw","nei","","" -7,"2024/3/15","file.path","dir_input_narr","input/narr","path","character_feature_raw_narr","character","feature","raw","narr","","" -8,"2024/3/15","read.csv","file_narr_variables","inst/targets/narr_variables.csv","path","character_feature_lookup_raw_narr","character","feature_lookup","raw","narr","","" -9,"2024/3/15","file.path","dir_input_modis","input/modis","path","character_feature_raw_modis","character","feature","raw","modis","","" -10,"2024/3/15","file.path","dir_input_nlcd","input/nlcd/raw","path","character_feature_raw_nlcd","character","feature","raw","nlcd","","" -11,"2024/3/15","file.path","dir_input_ecoregions","input/ecoregions","path","character_feature_raw_ecoregions","character","feature","raw","ecoregions","","" -12,"2024/3/15","file.path","dir_input_koppen","input/koppen_geiger","path","character_feature_raw_koppengeiger","character","feature","raw","koppengeiger","","" -13,"2024/3/15","file.path","dir_input_gmted","input/gmted","path","character_feature_raw_gmted","character","feature","raw","gmted","","" -14,"2024/3/15","file.path","dir_input_sedac_population","input/sedac_population","path","character_feature_raw_population","character","feature","raw","population","","" -15,"2024/3/15","file.path","dir_input_sedac_groads","input/sedac_groads","path","character_feature_raw_groads","character","feature","raw","groads","","" -16,"2024/3/15","file.path","dir_input_hms","input/HMS_Smoke","path","character_feature_raw_hms","character","feature","raw","hms","","" -17,"2024/3/15","file.path","dir_input_tri","input/tri","path","character_feature_raw_tri","character","feature","raw","tri","","" -18,"2024/3/15","file.path","dir_input_geoscf","input/geos","path","character_feature_raw_geoscf","character","feature","raw","geoscf","","" -19,"2024/3/15","file.path","dir_input_modis_mod11","input/modis/raw/61/MOD11A1","path","character_feature_dir_raw_modis_mod11","character","feature_dir","raw","modis_mod11","","" -20,"2024/3/15","file.path","dir_input_modis_mod13","input/modis/raw/61/MOD13A2","path","character_feature_dir_raw_modis_mod13","character","feature_dir","raw","modis_mod13","","" -21,"2024/3/15","file.path","dir_input_modis_mcd19","input/modis/raw/61/MCD19A2","path","character_feature_dir_raw_modis_mcd19","character","feature_dir","raw","modis_mcd19","","" -22,"2024/3/15","file.path","dir_input_modis_mod09","input/modis/raw/61/MOD09GA","path","character_feature_dir_raw_modis_mod09","character","feature_dir","raw","modis_mod09","","" -23,"2024/3/15","file.path","dir_input_modis_mod06","input/modis/raw/61/MOD06_L2","path","character_feature_dir_raw_modis_mod06","character","feature_dir","raw","modis_mod06","","" -24,"2024/3/15","file.path","dir_input_viirs","input/modis/raw/5000/VNP46A2","path","character_feature_dir_raw_viirs","character","feature_dir","raw","viirs","","" -30,"2024/3/15","paste0","fileext_aqs","*.csv$","extension","character_feature_fileext_calc_aqs","character","feature_fileext","calc","aqs","","" -31,"2024/3/15","paste0","fileext_nei","*.csv$","extension","character_feature_fileext_calc_nei","character","feature_fileext","calc","nei","","" -32,"2024/3/15","paste0","fileext_narr","*.nc$","extension","character_feature_fileext_calc_narr","character","feature_fileext","calc","narr","","" -34,"2024/3/15","paste0","fileext_modis","*.hdf$","extension","character_feature_fileext_calc_modis","character","feature_fileext","calc","modis","","" -35,"2024/3/15","paste0","fileext_viirs","*.h5$","extension","character_feature_fileext_calc_viirs","character","feature_fileext","calc","viirs","","" -36,"2024/3/15","paste0","fileext_nlcd","*.img$","extension","character_feature_fileext_calc_nlcd","character","feature_fileext","calc","nlcd","","" -37,"2024/3/15","paste0","fileext_ecoregions","*.shp$","extension","character_feature_fileext_calc_ecoregions","character","feature_fileext","calc","ecoregions","","" -38,"2024/3/15","paste0","fileext_koppen","*.tif$","extension","character_feature_fileext_calc_koppengeiger","character","feature_fileext","calc","koppengeiger","","" -39,"2024/3/15","paste0","fileext_gmted","*.nc$","extension","character_feature_fileext_calc_gmted","character","feature_fileext","calc","gmted","","" -40,"2024/3/15","paste0","fileext_sedac_population","*.shp$","extension","character_feature_fileext_calc_population","character","feature_fileext","calc","population","","" -41,"2024/3/15","paste0","fileext_sedac_groads","*.tif$","extension","character_feature_fileext_calc_groads","character","feature_fileext","calc","groads","","" -42,"2024/3/15","paste0","fileext_hms","*.shp$","extension","character_feature_fileext_calc_hms","character","feature_fileext","calc","hms","","" -43,"2024/3/15","paste0","fileext_tri","*.nc$","extension","character_feature_fileext_calc_tri","character","feature_fileext","calc","tri","","" -44,"2024/3/15","paste0","fileext_geoscf","*.nc$","extension","character_feature_fileext_calc_geoscf","character","feature_fileext","calc","geoscf","","" -45,"2024/3/15","paste0","file_list_mod11","mod11_static_list.txt","status","character_feature_list_calc_modis_mod11","character","feature_list","calc","modis_mod11","","" -46,"2024/3/15","paste0","file_list_mod13","mod13_static_list.txt","status","character_feature_list_calc_modis_mod13","character","feature_list","calc","modis_mod13","","" -47,"2024/3/15","paste0","file_list_mcd19","mcd19_static_list.txt","status","character_feature_list_calc_modis_mcd19","character","feature_list","calc","modis_mcd19","","" -48,"2024/3/15","paste0","file_list_mod06","mod06_static_list.txt","status","character_feature_list_calc_modis_mod06","character","feature_list","calc","modis_mod06","","" -49,"2024/3/15","paste0","file_list_mod09","mod09_static_list.txt","status","character_feature_list_calc_modis_mod09","character","feature_list","calc","modis_mod09","","" -50,"2024/3/15","paste0","file_list_tri","tri_static_list.txt","status","character_feature_list_calc_tri","character","feature_list","calc","tri","","" -67,"2024/3/15","paste0","char_siteid","site_id","identifier","character_feature_pointid_calc","character","feature_pointid","calc","","","" -68,"2024/3/15","paste0","char_timeid","time","identifier","character_feature_timeid_calc","character","feature_timeid","calc","","","" -69,"2024/3/15","paste0","file_feat_dummies","feature_dummies.rds","status","datatable_feature_timedummy_calc","datatable","feature_timedummy","calc","","","" -70,"2024/3/15","paste0","file_aqs_pm","aqs_cleaned.rds","status","datatable_feature_pm25_calc_aqs","datatable","feature_pm25","calc","aqs","","" -71,"2024/3/15","paste0","name_dep","pm25","label","character_feature_pm25name_process","character","feature_pm25name","process","","","" -72,"2024/3/15","readLines","file_name_indep","list_covariate_names.txt","label","","","","","","","deprecated?" -73,"2024/3/15","paste0","file_grid_prediction","prediction_grid.rds","status","datatable_prediction_locs_calc","datatable","prediction_locs","calc","","grid","" -74,"2024/3/15","as.integer","nthreads","16","setting","integer_feature_nthreads_calc","integer","feature_nthreads","calc","","","" -91,"2024/3/15","file.path","dir_input_nei2017","input/nei/nei_onroad_byregions_2017","path","character_feature_dir_calc_nei_2017","character","feature_dir","calc","nei","2017","" -92,"2024/3/15","file.path","dir_input_nei2020","input/nei/nei_onroad_byregions_2020","path","character_feature_dir_calc_nei_2020","character","feature_dir","calc","nei","2020","" -96,"2024/3/15","paste0","file_input_sedac_groads","groads-v1-americas-gdb/gROADS-v1-americas.gdb","path","character_feature_dir_calc_groads","character","feature_dir","calc","groads","","" -97,"2024/3/15","paste0","file_input_sedac_population","gpw_v4_population_density_adjusted_to_2015_unwpp_country_totals_rev11_2020_30_sec.tif","path","character_feature_dir_calc_population","character","feature_dir","calc","population","","" -93,"2024/4/1","as.Date","date_start","2020-01-01","domain","character_feature_calcdate_start","character","feature","calc","","date_start","" -94,"2024/4/1","as.Date","date_end","2020-12-31","domain","character_feature_calcdate_end","character","feature","calc","","date_end","" -95,"2024/3/15","strsplit","extent","-126|-62|22|52","domain","character_feature_calcextent","character","feature","calc","","extent","" -98,"2024/3/29","strsplit","nei_year_sequence","2017|2017|2020|2020|2020","domain","character_feature_calc_nei_years","character","feature","calc","nei","years","" -99,"2024/3/29","strsplit","nlcd_year_sequence","2019|2019|2019|2021|2021","domain","character_feature_calc_nlcd_years","character","feature","calc","nlcd","years","" -101,"2024/3/29","strsplit","sedac_population_year","2020|2020|2020|2020|2020","domain","character_feature_calc_population_years","character","feature","calc","population","years","" -1001,"2024/3/29","paste0","nei_year_sequence_test","2020","domain","character_feature_calc_test_nei_years","character","feature","calc_test","nei","years","" -1002,"2024/3/29","paste0","nlcd_year_sequence_test","2019","domain","character_feature_calc_test","character","feature","calc_test","","","" -1003,"2024/3/29","as.integer","sedac_population_year_test","2020","domain","character_feature_calc_test_population_year","character","feature","calc_test","","","" -1004,"2024/3/29","strsplit","tri_year_sequence_test","2020|2020","domain","character_feature_calc_test_tri_year","character","feature","calc_test","","","" -100,"2024/3/29","as.integer","nthreads_calc","20","setting","","","","","","","deprecated" -102,"2024/3/29","strsplit","tri_year_sequence","2018|2019|2020|2021|2022","domain","character_feature_calc_tri","character","feature","calc","tri","","" -103,"2024/3/29","paste0","slurm_user_email","songi2@nih.gov","setting","character_config_calc_useremail","character","config","calc","useremail","","" +version https://git-lfs.github.com/spec/v1 +oid sha256:4340251ee65e4b1de71596e095ada2c7210d7a21b523e09852a87d0a8bb5fa8d +size 10898 diff --git a/tools/targets-old/targets_lookup.csv b/tools/targets-old/targets_lookup.csv index a9fadc82..1cfcdb8f 100644 --- a/tools/targets-old/targets_lookup.csv +++ b/tools/targets-old/targets_lookup.csv @@ -1 +1,3 @@ -"target","description" +version https://git-lfs.github.com/spec/v1 +oid sha256:4dfd0967ceb612d6feacc355f2b17395a141f60112654202b5ccae6e4d656002 +size 23 From d922834bff800b10492c05a6500c876fc69db6bd Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Mon, 30 Sep 2024 16:19:49 -0400 Subject: [PATCH 02/20] rebasing + gitignore for tokens --- .gitignore | 6 +++++- tests/testdata/nei/onroadnc17.csv | 3 +++ tests/testdata/test_nc_output.nc | 3 +++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 tests/testdata/nei/onroadnc17.csv create mode 100644 tests/testdata/test_nc_output.nc diff --git a/.gitignore b/.gitignore index a9617054..593e14a5 100644 --- a/.gitignore +++ b/.gitignore @@ -106,4 +106,8 @@ slurm_error.log /doc/ # interactive targets -targets_start.Rout \ No newline at end of file +targets_start.Rout + +.netrc +.urs_cookies +beethoven_branching_notes.txt \ No newline at end of file diff --git a/tests/testdata/nei/onroadnc17.csv b/tests/testdata/nei/onroadnc17.csv new file mode 100644 index 00000000..137e06ab --- /dev/null +++ b/tests/testdata/nei/onroadnc17.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad7d3bc7921efcc3cbecd7eae2eecaa22dbc7f1fc523c841a9bda004192ad631 +size 4880179 diff --git a/tests/testdata/test_nc_output.nc b/tests/testdata/test_nc_output.nc new file mode 100644 index 00000000..4bdf0f32 --- /dev/null +++ b/tests/testdata/test_nc_output.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6a73251e3268cd4030109cb8f861737388ce10e64357852dc8f3ceb316279d8 +size 19804 From 819dcd2f65af3795ed28f20cb8ac459126e0f9c8 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Mon, 30 Sep 2024 16:29:52 -0400 Subject: [PATCH 03/20] remove old test data --- tests/testdata/nei/onroadnc17.csv | 3 --- tests/testdata/test_nc_output.nc | 3 --- 2 files changed, 6 deletions(-) delete mode 100644 tests/testdata/nei/onroadnc17.csv delete mode 100644 tests/testdata/test_nc_output.nc diff --git a/tests/testdata/nei/onroadnc17.csv b/tests/testdata/nei/onroadnc17.csv deleted file mode 100644 index 137e06ab..00000000 --- a/tests/testdata/nei/onroadnc17.csv +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ad7d3bc7921efcc3cbecd7eae2eecaa22dbc7f1fc523c841a9bda004192ad631 -size 4880179 diff --git a/tests/testdata/test_nc_output.nc b/tests/testdata/test_nc_output.nc deleted file mode 100644 index 4bdf0f32..00000000 --- a/tests/testdata/test_nc_output.nc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d6a73251e3268cd4030109cb8f861737388ce10e64357852dc8f3ceb316279d8 -size 19804 From fc275d216ee2d0331cc296c915b7eeaf86fe6722 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 09:27:15 -0400 Subject: [PATCH 04/20] set up container, def, and archive old bash scripts --- .../init_targets_storage.sh | 0 .../run_interactive.sh | 0 run_slurm.sh => archive/run_slurm.sh | 0 setup_hook.sh => archive/setup_hook.sh | 0 container/beethoven_dl_calc.def | 46 +++++++++++++++++++ 5 files changed, 46 insertions(+) rename init_targets_storage.sh => archive/init_targets_storage.sh (100%) rename run_interactive.sh => archive/run_interactive.sh (100%) rename run_slurm.sh => archive/run_slurm.sh (100%) rename setup_hook.sh => archive/setup_hook.sh (100%) create mode 100644 container/beethoven_dl_calc.def diff --git a/init_targets_storage.sh b/archive/init_targets_storage.sh similarity index 100% rename from init_targets_storage.sh rename to archive/init_targets_storage.sh diff --git a/run_interactive.sh b/archive/run_interactive.sh similarity index 100% rename from run_interactive.sh rename to archive/run_interactive.sh diff --git a/run_slurm.sh b/archive/run_slurm.sh similarity index 100% rename from run_slurm.sh rename to archive/run_slurm.sh diff --git a/setup_hook.sh b/archive/setup_hook.sh similarity index 100% rename from setup_hook.sh rename to archive/setup_hook.sh diff --git a/container/beethoven_dl_calc.def b/container/beethoven_dl_calc.def new file mode 100644 index 00000000..09beae18 --- /dev/null +++ b/container/beethoven_dl_calc.def @@ -0,0 +1,46 @@ +BootStrap: docker +From: rocker/geospatial:latest + +%files + /ddn/gs1/tools/set/R432/lib64/R/library /R_packages + + +%post + # Update package list + apt-get update + + # Install locales and generate the necessary locale + apt-get install -y locales + locale-gen en_US.UTF-8 + + # Install fonts for Unicode support + apt-get install -y fonts-dejavu fonts-liberation fonts-noto fonts-unifont + + # Set locale for the environment + echo "LANG=en_US.UTF-8" >> /etc/default/locale + echo "LC_ALL=en_US.UTF-8" >> /etc/default/locale + export LANG=en_US.UTF-8 + export LC_ALL=en_US.UTF-8 + + + mkdir /pipeline + mkdir /input + mkdir /opt/_targets + + Rscript -e "install.packages(c('pak','targets','tarchetypes','crew','crew.cluster'))" + Rscript -e "pak::pkg_install('NIEHS/amadeus')" + Rscript -e "remotes::install_github('NIEHS/beethoven')" + + + + +%environment + # Set locale for the container environment + export LANG=en_US.UTF-8 + export LC_ALL=en_US.UTF-8 + export TERM=xterm-256color + +%runscript + +%labels + basic geospatial with targets and crew plus unicode text so the target progress prints nicely \ No newline at end of file From bad244e022f6b108614b42e67ae91a0fd6eee8a4 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 09:53:44 -0400 Subject: [PATCH 05/20] add dev to branches triggering GH actions --- .github/workflows/check-standard.yaml | 4 ++-- .github/workflows/lint.yaml | 4 ++-- .github/workflows/pkgdown.yaml | 4 ++-- .github/workflows/test-coverage.yaml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/check-standard.yaml b/.github/workflows/check-standard.yaml index dec2daf8..308a0d5b 100644 --- a/.github/workflows/check-standard.yaml +++ b/.github/workflows/check-standard.yaml @@ -2,9 +2,9 @@ # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help on: push: - branches: [main, master] + branches: [main, master, dev] pull_request: - branches: [main, master] + branches: [main, master, dev] name: R-CMD-check diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index f60d0479..da54e138 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -2,9 +2,9 @@ # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help on: push: - branches: [main, master] + branches: [main, master, dev] pull_request: - branches: [main, master] + branches: [main, master, dev] name: lint diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index 184ae644..f57508e7 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -2,9 +2,9 @@ # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help on: push: - branches: [main, master] + branches: [main, master, dev] pull_request: - branches: [main, master] + branches: [main, master, dev] release: types: [published] workflow_dispatch: diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index 220709d7..51f2bbb2 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -3,9 +3,9 @@ # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help on: push: - branches: [main, master] + branches: [main, master, dev] pull_request: - branches: [main, master] + branches: [main, master, dev] name: test-coverage-local From 30b0f89421c6c4d39738b0caa8b0cb0112ac9bf2 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 13:32:11 -0400 Subject: [PATCH 06/20] def files, bash scipts for containers, testing yaml --- .github/workflows/test-container-dl-calc.yaml | 67 +++++++++++++++++++ container/beethoven_dl_calc.def | 6 +- container/build_dl_calc.sh | 8 +++ container/run_container_dl_calc.sh | 23 +++++++ container/run_dl_calc_local_tests.sh | 23 +++++++ tests/testthat/test_calculate.R | 13 ++++ tests/testthat/test_download.R | 13 ++++ 7 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/test-container-dl-calc.yaml create mode 100644 container/build_dl_calc.sh create mode 100644 container/run_container_dl_calc.sh create mode 100644 container/run_dl_calc_local_tests.sh create mode 100644 tests/testthat/test_calculate.R create mode 100644 tests/testthat/test_download.R diff --git a/.github/workflows/test-container-dl-calc.yaml b/.github/workflows/test-container-dl-calc.yaml new file mode 100644 index 00000000..2ae5810a --- /dev/null +++ b/.github/workflows/test-container-dl-calc.yaml @@ -0,0 +1,67 @@ +name: CI tests for download and calculate with Apptainer + +on: + push: + branches: + - [main, dev] + pull_request: + branches: + - [main, dev] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Install Apptainer dependencies + run: | + sudo apt-get update && sudo apt-get install -y \ + libseccomp-dev \ + squashfs-tools \ + cryptsetup \ + wget + + - name: Install Apptainer + run: | + wget https://github.com/apptainer/apptainer/releases/download/v1.1.0/apptainer-1.1.0.tar.gz + tar -xvf apptainer-1.1.0.tar.gz + cd apptainer-1.1.0 + ./mconfig && make -C builddir && sudo make -C builddir install + + - name: Restore .sif file from cache + id: cache-sif + uses: actions/cache@v3 + with: + path: beethoven_dl_calc.sif + key: sif-cache-${{ hashFiles('../../containers/beethoven_dl_calc.def') }} + restore-keys: sif-cache- + + - name: Build the Apptainer container (if cache miss) + if: steps.cache-sif.outputs.cache-hit != 'true' + run: | + apptainer build my_container.sif path/to/Singularity.def + + - name: Cache the .sif file + if: steps.cache-sif.outputs.cache-hit != 'true' + uses: actions/cache@v3 + with: + path: beethoven_dl_calc.sif + key: sif-cache-${{ hashFiles('../../containers/beethoven_dl_calc.def') }} + + - name: Create run_dl_calc_tests.R dynamically + run: | + echo 'testthat::test_file("/mnt/tests/testthat/test_download.R")' > tests/testthat/run_dl_calc_tests.R + echo 'testthat::test_file("/mnt/tests/testthat/test_calculate.R")' >> tests/testthat/run_dl_calc_tests.R + + + - name: Run tests inside the container + run: | + apptainer exec \ + --bind $PWD/inst:/pipeline \ + --bind $PWD/input:/input \ + --bind $PWD/_targets:/opt/_targets \ + --bind $PWD:/mnt \ + Rscript mnt/tests/testhat/run_dl_calc_tests.R diff --git a/container/beethoven_dl_calc.def b/container/beethoven_dl_calc.def index 09beae18..5b5227d0 100644 --- a/container/beethoven_dl_calc.def +++ b/container/beethoven_dl_calc.def @@ -1,10 +1,6 @@ BootStrap: docker From: rocker/geospatial:latest -%files - /ddn/gs1/tools/set/R432/lib64/R/library /R_packages - - %post # Update package list apt-get update @@ -27,7 +23,7 @@ From: rocker/geospatial:latest mkdir /input mkdir /opt/_targets - Rscript -e "install.packages(c('pak','targets','tarchetypes','crew','crew.cluster'))" + Rscript -e "install.packages(c('pak','targets','tarchetypes','crew','crew.cluster','testthat'))" Rscript -e "pak::pkg_install('NIEHS/amadeus')" Rscript -e "remotes::install_github('NIEHS/beethoven')" diff --git a/container/build_dl_calc.sh b/container/build_dl_calc.sh new file mode 100644 index 00000000..b66b1a0b --- /dev/null +++ b/container/build_dl_calc.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# usage: build_apptainer_image.sh [full file path] +# where full file path ends with .sif, with full directory path to save the image +# after the image is built, group write/execution privileges are given + +# Recommended to run this script interactively via `sh build_dl_calc.sh` +apptainer build --fakeroot beethoven_dl_calc.sif beethoven_dl_calc.def \ No newline at end of file diff --git a/container/run_container_dl_calc.sh b/container/run_container_dl_calc.sh new file mode 100644 index 00000000..49e7eb04 --- /dev/null +++ b/container/run_container_dl_calc.sh @@ -0,0 +1,23 @@ +#!/bin/bash +#SBATCH --job-name=beethoven_001 +#SBATCH --partition=geo +#SBATCH --mem=128G +#SBATCH --cpus-per-task=4 +#SBATCH --ntasks=16 +#SBATCH --output=slurm_messages/slurm-%j.out +#SBATCH --error=slurm_messages/slurm-%j.err +#SBATCH --mail-user=kyle.messier@nih.gov +#SBATCH --mail-type=ALL + + + + +apptainer exec \ + --bind $PWD/inst:/pipeline \ + --bind $PWD/input:/input \ + --bind $PWD/_targets:/opt/_targets \ + --bind $PWD:/mnt \ + beethoven_dl_calc.sif \ + Rscript /mnt/run.R + + diff --git a/container/run_dl_calc_local_tests.sh b/container/run_dl_calc_local_tests.sh new file mode 100644 index 00000000..9404e8ac --- /dev/null +++ b/container/run_dl_calc_local_tests.sh @@ -0,0 +1,23 @@ +#!/bin/bash +#SBATCH --job-name=dl_calc_tests +#SBATCH --partition=geo +#SBATCH --mem=128G +#SBATCH --cpus-per-task=4 +#SBATCH --ntasks=16 +#SBATCH --output=slurm_messages/slurm-%j.out +#SBATCH --error=slurm_messages/slurm-%j.err +#SBATCH --mail-user=kyle.messier@nih.gov +#SBATCH --mail-type=ALL + + + + +apptainer exec \ + --bind $PWD/../inst:/pipeline \ + --bind $PWD/../input:/input \ + --bind $PWD/../_targets:/opt/_targets \ + --bind $PWD/..:/mnt \ + beethoven_dl_calc.sif \ + Rscript /mnt/tests/testthat/test_download.R + + diff --git a/tests/testthat/test_calculate.R b/tests/testthat/test_calculate.R new file mode 100644 index 00000000..94a48719 --- /dev/null +++ b/tests/testthat/test_calculate.R @@ -0,0 +1,13 @@ +################################################################################ +##### unit and integration tests for covariate calculations +##### Currently arbitrary placeholders +##### + +################################################################################ +##### +testthat::test_that("calculate targets work", { + + a <- 1 + testthat::expect_equal(a, 1) + +}) diff --git a/tests/testthat/test_download.R b/tests/testthat/test_download.R new file mode 100644 index 00000000..24cebc16 --- /dev/null +++ b/tests/testthat/test_download.R @@ -0,0 +1,13 @@ +################################################################################ +##### unit and integration tests for data download +##### Currently arbitrary placeholders +##### + +################################################################################ +##### loadargs +testthat::test_that("download targets work", { + + a <- 1 + testthat::expect_equal(a, 1) + +}) From 4c42f5aea4bafdec861cda500fb80cbb89db5e4f Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 13:36:55 -0400 Subject: [PATCH 07/20] fixing [on] typo --- .github/workflows/test-container-dl-calc.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-container-dl-calc.yaml b/.github/workflows/test-container-dl-calc.yaml index 2ae5810a..124717c1 100644 --- a/.github/workflows/test-container-dl-calc.yaml +++ b/.github/workflows/test-container-dl-calc.yaml @@ -2,11 +2,9 @@ name: CI tests for download and calculate with Apptainer on: push: - branches: - - [main, dev] + branches: [main, master, dev] pull_request: - branches: - - [main, dev] + branches: [main, master, dev] jobs: build: From 969565bf2a57637b9c83833b84a7e1a7086bc92d Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 13:58:07 -0400 Subject: [PATCH 08/20] GH action path correction --- .github/workflows/test-container-dl-calc.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-container-dl-calc.yaml b/.github/workflows/test-container-dl-calc.yaml index 124717c1..ff35c3be 100644 --- a/.github/workflows/test-container-dl-calc.yaml +++ b/.github/workflows/test-container-dl-calc.yaml @@ -34,7 +34,7 @@ jobs: uses: actions/cache@v3 with: path: beethoven_dl_calc.sif - key: sif-cache-${{ hashFiles('../../containers/beethoven_dl_calc.def') }} + key: sif-cache-${{ hashFiles('containers/beethoven_dl_calc.def') }} restore-keys: sif-cache- - name: Build the Apptainer container (if cache miss) @@ -47,7 +47,7 @@ jobs: uses: actions/cache@v3 with: path: beethoven_dl_calc.sif - key: sif-cache-${{ hashFiles('../../containers/beethoven_dl_calc.def') }} + key: sif-cache-${{ hashFiles('containers/beethoven_dl_calc.def') }} - name: Create run_dl_calc_tests.R dynamically run: | From 66c65aaf6c16dea6f41aca1e253271d030c866f7 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 13:58:46 -0400 Subject: [PATCH 09/20] name change --- .github/workflows/test-container-dl-calc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-container-dl-calc.yaml b/.github/workflows/test-container-dl-calc.yaml index ff35c3be..6ec92b5d 100644 --- a/.github/workflows/test-container-dl-calc.yaml +++ b/.github/workflows/test-container-dl-calc.yaml @@ -1,4 +1,4 @@ -name: CI tests for download and calculate with Apptainer +name: test coverage for download and calculate via Apptainer on: push: From 27d64e3fdfe46d38a655cd26e7853ac3664d363d Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 14:05:10 -0400 Subject: [PATCH 10/20] def file was wrong name --- .github/workflows/test-container-dl-calc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-container-dl-calc.yaml b/.github/workflows/test-container-dl-calc.yaml index 6ec92b5d..789bf450 100644 --- a/.github/workflows/test-container-dl-calc.yaml +++ b/.github/workflows/test-container-dl-calc.yaml @@ -40,7 +40,7 @@ jobs: - name: Build the Apptainer container (if cache miss) if: steps.cache-sif.outputs.cache-hit != 'true' run: | - apptainer build my_container.sif path/to/Singularity.def + apptainer build beethoven_dl_calc.sif beethoven_dl_calc.def - name: Cache the .sif file if: steps.cache-sif.outputs.cache-hit != 'true' From f4db89526354e1e29ebf3743e63499f05c98c3e2 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 14:09:54 -0400 Subject: [PATCH 11/20] rel path update for container build --- .github/workflows/test-container-dl-calc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-container-dl-calc.yaml b/.github/workflows/test-container-dl-calc.yaml index 789bf450..7259685b 100644 --- a/.github/workflows/test-container-dl-calc.yaml +++ b/.github/workflows/test-container-dl-calc.yaml @@ -40,7 +40,7 @@ jobs: - name: Build the Apptainer container (if cache miss) if: steps.cache-sif.outputs.cache-hit != 'true' run: | - apptainer build beethoven_dl_calc.sif beethoven_dl_calc.def + apptainer build container/beethoven_dl_calc.sif beethoven_dl_calc.def - name: Cache the .sif file if: steps.cache-sif.outputs.cache-hit != 'true' From 89a2f954a733dd25f31c97067294c1eb43ba4816 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 14:17:42 -0400 Subject: [PATCH 12/20] path fix, fakeroot --- .github/workflows/test-container-dl-calc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-container-dl-calc.yaml b/.github/workflows/test-container-dl-calc.yaml index 7259685b..473f50b9 100644 --- a/.github/workflows/test-container-dl-calc.yaml +++ b/.github/workflows/test-container-dl-calc.yaml @@ -40,7 +40,7 @@ jobs: - name: Build the Apptainer container (if cache miss) if: steps.cache-sif.outputs.cache-hit != 'true' run: | - apptainer build container/beethoven_dl_calc.sif beethoven_dl_calc.def + apptainer build --fakeroot beethoven_dl_calc.sif container/beethoven_dl_calc.def - name: Cache the .sif file if: steps.cache-sif.outputs.cache-hit != 'true' From 8af11cdd18894ae8cc08b9127c897454d9218ff5 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 14:30:49 -0400 Subject: [PATCH 13/20] path change for test functions --- .github/workflows/test-container-dl-calc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-container-dl-calc.yaml b/.github/workflows/test-container-dl-calc.yaml index 473f50b9..aca2f279 100644 --- a/.github/workflows/test-container-dl-calc.yaml +++ b/.github/workflows/test-container-dl-calc.yaml @@ -62,4 +62,4 @@ jobs: --bind $PWD/input:/input \ --bind $PWD/_targets:/opt/_targets \ --bind $PWD:/mnt \ - Rscript mnt/tests/testhat/run_dl_calc_tests.R + Rscript tests/testhat/run_dl_calc_tests.R From 7afc578fee4a0a0b7e218a437cdd4583f63ea4e3 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 14:43:10 -0400 Subject: [PATCH 14/20] cache fixing attempt --- .github/workflows/test-container-dl-calc.yaml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-container-dl-calc.yaml b/.github/workflows/test-container-dl-calc.yaml index aca2f279..78966be6 100644 --- a/.github/workflows/test-container-dl-calc.yaml +++ b/.github/workflows/test-container-dl-calc.yaml @@ -34,8 +34,9 @@ jobs: uses: actions/cache@v3 with: path: beethoven_dl_calc.sif - key: sif-cache-${{ hashFiles('containers/beethoven_dl_calc.def') }} - restore-keys: sif-cache- + key: sif-cache-${{ runner.os }}-${{ hashFiles('container/beethoven_dl_calc.def') }} + restore-keys: | + sif-cache-${{ runner.os }}- - name: Build the Apptainer container (if cache miss) if: steps.cache-sif.outputs.cache-hit != 'true' @@ -47,7 +48,7 @@ jobs: uses: actions/cache@v3 with: path: beethoven_dl_calc.sif - key: sif-cache-${{ hashFiles('containers/beethoven_dl_calc.def') }} + key: sif-cache-${{ hashFiles('container/beethoven_dl_calc.def') }} - name: Create run_dl_calc_tests.R dynamically run: | @@ -56,10 +57,5 @@ jobs: - name: Run tests inside the container - run: | - apptainer exec \ - --bind $PWD/inst:/pipeline \ - --bind $PWD/input:/input \ - --bind $PWD/_targets:/opt/_targets \ - --bind $PWD:/mnt \ - Rscript tests/testhat/run_dl_calc_tests.R + run: | + apptainer exec container/beethoven_dl_calc.sif Rscript tests/testhat/run_dl_calc_tests.R From ab3dfb79cda5ecf9cdbbe48b8441ddb083e525f2 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 14:57:30 -0400 Subject: [PATCH 15/20] path fix attempts --- .github/workflows/test-container-dl-calc.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-container-dl-calc.yaml b/.github/workflows/test-container-dl-calc.yaml index 78966be6..97dae8b9 100644 --- a/.github/workflows/test-container-dl-calc.yaml +++ b/.github/workflows/test-container-dl-calc.yaml @@ -48,14 +48,14 @@ jobs: uses: actions/cache@v3 with: path: beethoven_dl_calc.sif - key: sif-cache-${{ hashFiles('container/beethoven_dl_calc.def') }} + key: sif-cache-${{ hashFiles('beethoven_dl_calc.def') }} - name: Create run_dl_calc_tests.R dynamically run: | - echo 'testthat::test_file("/mnt/tests/testthat/test_download.R")' > tests/testthat/run_dl_calc_tests.R - echo 'testthat::test_file("/mnt/tests/testthat/test_calculate.R")' >> tests/testthat/run_dl_calc_tests.R + echo 'testthat::test_file("tests/testthat/test_download.R")' > tests/testthat/run_dl_calc_tests.R + echo 'testthat::test_file("tests/testthat/test_calculate.R")' >> tests/testthat/run_dl_calc_tests.R - name: Run tests inside the container run: | - apptainer exec container/beethoven_dl_calc.sif Rscript tests/testhat/run_dl_calc_tests.R + apptainer exec beethoven_dl_calc.sif Rscript tests/testhat/run_dl_calc_tests.R From 775c98d57bc3f776eea5394b1febf360bc1283a6 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 15:16:05 -0400 Subject: [PATCH 16/20] change order of yaml names to include new R test file in mnt --- .github/workflows/test-container-dl-calc.yaml | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-container-dl-calc.yaml b/.github/workflows/test-container-dl-calc.yaml index 97dae8b9..1cf4b681 100644 --- a/.github/workflows/test-container-dl-calc.yaml +++ b/.github/workflows/test-container-dl-calc.yaml @@ -14,6 +14,11 @@ jobs: - name: Checkout code uses: actions/checkout@v2 + - name: Create run_dl_calc_tests.R dynamically + run: | + echo 'testthat::test_file("tests/testthat/test_download.R")' > tests/testthat/run_dl_calc_tests.R + echo 'testthat::test_file("tests/testthat/test_calculate.R")' >> tests/testthat/run_dl_calc_tests.R + - name: Install Apptainer dependencies run: | sudo apt-get update && sudo apt-get install -y \ @@ -34,7 +39,7 @@ jobs: uses: actions/cache@v3 with: path: beethoven_dl_calc.sif - key: sif-cache-${{ runner.os }}-${{ hashFiles('container/beethoven_dl_calc.def') }} + key: sif-cache-${{ runner.os }}-${{ hashFiles('beethoven_dl_calc.def') }} restore-keys: | sif-cache-${{ runner.os }}- @@ -50,12 +55,14 @@ jobs: path: beethoven_dl_calc.sif key: sif-cache-${{ hashFiles('beethoven_dl_calc.def') }} - - name: Create run_dl_calc_tests.R dynamically - run: | - echo 'testthat::test_file("tests/testthat/test_download.R")' > tests/testthat/run_dl_calc_tests.R - echo 'testthat::test_file("tests/testthat/test_calculate.R")' >> tests/testthat/run_dl_calc_tests.R - - name: Run tests inside the container - run: | - apptainer exec beethoven_dl_calc.sif Rscript tests/testhat/run_dl_calc_tests.R + - name: Run R tests + run: | + apptainer exec \ + --bind $PWD/inst:/pipeline \ + --bind $PWD/input:/input \ + --bind $PWD/_targets:/opt/_targets \ + --bind $PWD:/mnt \ + beethoven_dl_calc.sif \ + Rscript /mnt/tests/testhat/run_dl_calc_tests.R From 691aa16cabc8abd3ec00e51827753cc115b1b7e6 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 15:33:30 -0400 Subject: [PATCH 17/20] remove _targets/ path from mount --- .github/workflows/test-container-dl-calc.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-container-dl-calc.yaml b/.github/workflows/test-container-dl-calc.yaml index 1cf4b681..28e91e51 100644 --- a/.github/workflows/test-container-dl-calc.yaml +++ b/.github/workflows/test-container-dl-calc.yaml @@ -53,7 +53,7 @@ jobs: uses: actions/cache@v3 with: path: beethoven_dl_calc.sif - key: sif-cache-${{ hashFiles('beethoven_dl_calc.def') }} + key: sif-cache-${{ runner.os }}-${{ hashFiles('beethoven_dl_calc.def') }} @@ -62,7 +62,6 @@ jobs: apptainer exec \ --bind $PWD/inst:/pipeline \ --bind $PWD/input:/input \ - --bind $PWD/_targets:/opt/_targets \ --bind $PWD:/mnt \ beethoven_dl_calc.sif \ Rscript /mnt/tests/testhat/run_dl_calc_tests.R From ddee94842eb02bbba51d4a673193f456c2149212 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 15:47:01 -0400 Subject: [PATCH 18/20] cache updating and check sections --- .github/workflows/test-container-dl-calc.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-container-dl-calc.yaml b/.github/workflows/test-container-dl-calc.yaml index 28e91e51..4ce7e76b 100644 --- a/.github/workflows/test-container-dl-calc.yaml +++ b/.github/workflows/test-container-dl-calc.yaml @@ -1,4 +1,4 @@ -name: test coverage for download and calculate via Apptainer +name: Test Coverage for Download and Calculate via Apptainer on: push: @@ -16,6 +16,7 @@ jobs: - name: Create run_dl_calc_tests.R dynamically run: | + mkdir -p tests/testthat # Ensure the directory exists echo 'testthat::test_file("tests/testthat/test_download.R")' > tests/testthat/run_dl_calc_tests.R echo 'testthat::test_file("tests/testthat/test_calculate.R")' >> tests/testthat/run_dl_calc_tests.R @@ -39,7 +40,7 @@ jobs: uses: actions/cache@v3 with: path: beethoven_dl_calc.sif - key: sif-cache-${{ runner.os }}-${{ hashFiles('beethoven_dl_calc.def') }} + key: sif-cache-${{ runner.os }}-${{ hashFiles('container/beethoven_dl_calc.def') }} restore-keys: | sif-cache-${{ runner.os }}- @@ -53,9 +54,14 @@ jobs: uses: actions/cache@v3 with: path: beethoven_dl_calc.sif - key: sif-cache-${{ runner.os }}-${{ hashFiles('beethoven_dl_calc.def') }} - + key: sif-cache-${{ runner.os }}-${{ hashFiles('container/beethoven_dl_calc.def') }} + - name: Check if .sif file exists + run: | + if [ ! -f beethoven_dl_calc.sif ]; then + echo "Error: .sif file not found!" + exit 1 + fi - name: Run R tests run: | @@ -64,4 +70,4 @@ jobs: --bind $PWD/input:/input \ --bind $PWD:/mnt \ beethoven_dl_calc.sif \ - Rscript /mnt/tests/testhat/run_dl_calc_tests.R + Rscript /mnt/tests/testthat/run_dl_calc_tests.R From ee9da0bd3bc39b23c3d0eb95651927ca6fb6ace0 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 16:03:49 -0400 Subject: [PATCH 19/20] test cache of sif --- tests/testthat/test_download.R | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/testthat/test_download.R b/tests/testthat/test_download.R index 24cebc16..fc706b8d 100644 --- a/tests/testthat/test_download.R +++ b/tests/testthat/test_download.R @@ -10,4 +10,7 @@ testthat::test_that("download targets work", { a <- 1 testthat::expect_equal(a, 1) + b <- 2 + testthat::expect_equal(b, 2) + }) From 51a67334e20857faf31d266d0ba80b1d5e581623 Mon Sep 17 00:00:00 2001 From: kyle-messier Date: Tue, 1 Oct 2024 23:39:06 -0400 Subject: [PATCH 20/20] _targets.R, test and run bash scripts --- _targets.R | 209 ++++++++++++++------------- container/run_container_dl_calc.sh | 10 +- container/run_dl_calc_local_tests.sh | 4 +- 3 files changed, 115 insertions(+), 108 deletions(-) diff --git a/_targets.R b/_targets.R index e90e0308..1cc6de42 100755 --- a/_targets.R +++ b/_targets.R @@ -1,123 +1,80 @@ library(targets) library(tarchetypes) -library(future) -library(future.batchtools) library(dplyr) -library( - beethoven, - lib.loc = "/ddn/gs1/home/manwareme/R/x86_64-pc-linux-gnu-library/4.3" -) -library(tidymodels) -library(bonsai) -# library( -# torch, -# lib.loc = "/ddn/gs1/biotools/R/lib64/R/library" -# ) +library(crew) +library(future) +library(beethoven) +library(amadeus) -Sys.setenv("LD_LIBRARY_PATH" = paste("/ddn/gs1/biotools/R/lib64/R/customlib", Sys.getenv("LD_LIBRARY_PATH"), sep = ":")) -# replacing yaml file. +# targets store location corresponds to _targets/ in the root of the project tar_config_set( - store = "/ddn/gs1/home/manwareme/beethoven/beethoven_targets" + store = "/opt/_targets" +) + +# crew contollers +# For now, one is set, but we can explore the use of multiple controllers +# Can also explore making the workers input for bash script or Rscript +geo_controller <- crew_controller_local( + name = "geo_controller", + workers = 16L, + launch_max = 8L, + seconds_idle = 120 ) -# maximum future exportable object size is set 50GB -# TODO: the maximum size error did not appear until recently -# and suddenly appeared. Need to investigate the cause. -# Should be removed after the investigation. -# options(future.globals.maxSize = 50 * 2^30) -options(future.globals.maxSize = 60 * 1024^3) # 60 GiB -generate_list_download <- FALSE +# Setting up the NASA Earthdata token inside the container +# This needs to be tested +if (!nzchar(Sys.getenv("NASA_EARTHDATA_TOKEN"))){ + tar_source("/mnt/NASA_token_setup.R") + file.exists(".netrc") + file.exists(".urs_cookies") + file.exists(".dodsrc") +} + arglist_download <- set_args_download( char_period = c("2018-01-01", "2022-12-31"), - char_input_dir = "input", - nasa_earth_data_token = NULL,#Sys.getenv("NASA_EARTHDATA_TOKEN"), - mod06_filelist = "inst/targets/mod06_links_2018_2022.csv", - export = generate_list_download, - path_export = "inst/targets/download_spec.qs" + char_input_dir = "/input", + nasa_earth_data_token = Sys.getenv("NASA_EARTHDATA_TOKEN"), + mod06_filelist = "/pipeline/targets/mod06_links_2018_2022.csv", + export = TRUE, + path_export = "/pipeline/targets/download_spec.qs" ) -generate_list_calc <- FALSE -arglist_common <- - set_args_calc( - char_siteid = "site_id", - char_timeid = "time", - char_period = c("2018-01-01", "2022-12-31"), - num_extent = c(-126, -62, 22, 52), - char_user_email = paste0(Sys.getenv("USER"), "@nih.gov"), - export = generate_list_calc, - path_export = "inst/targets/calc_spec.qs", - char_input_dir = "/ddn/gs1/group/set/Projects/NRT-AP-Model/input" - ) -tar_source("inst/targets/targets_initialize.R") -tar_source("inst/targets/targets_download.R") -tar_source("inst/targets/targets_calculate_fit.R") -tar_source("inst/targets/targets_calculate_predict.R") -tar_source("inst/targets/targets_baselearner.R") -tar_source("inst/targets/targets_metalearner.R") -tar_source("inst/targets/targets_predict.R") -# bypass option -Sys.setenv("BTV_DOWNLOAD_PASS" = "TRUE") -# -# bind custom built GDAL -# Users should export the right path to the GDAL library -# by export LD_LIBRARY_PATH=.... command. +### NOTE: It is important to source the scipts after the global variables are defined from the set_args functions + #tar_source("/pipeline/targets/targets_aqs.R") + tar_source("/pipeline/targets/targets_download.R") -# arglist_common is generated above -plan( - list( - tweak( - future.batchtools::batchtools_slurm, - template = "inst/targets/template_slurm.tmpl", - resources = - list( - memory = 8, - log.file = "slurm_run.log", - ncpus = 1, partition = "geo", ntasks = 1, - email = arglist_common$char_user_email, - error.file = "slurm_error.log" - ) - ), - multicore - ) -) +# Toy test files - note we will not have functions defined like this directly in +# the _targets.R file +my_fun_a <- function(n) { + rnorm(n) +} + +my_fun_b <- function(x) { + x^2 +} -# # invalidate any nodes older than 180 days: force running the pipeline -# tar_invalidate(any_of(tar_older(Sys.time() - as.difftime(180, units = "days")))) -# # nullify download target if bypass option is set -if (Sys.getenv("BTV_DOWNLOAD_PASS") == "TRUE") { - target_download <- NULL -} -# targets options -# For GPU support, users should be aware of setting environment -# variables and GPU versions of the packages. -# TODO: check if the controller and resources setting are required tar_option_set( - packages = c( - "beethoven", "amadeus", "chopin", "targets", "tarchetypes", - "data.table", "sf", "terra", "exactextractr", - #"crew", "crew.cluster", - "tigris", "dplyr", - "future.batchtools", "qs", "collapse", "bonsai", - "tidymodels", "tune", "rsample", "torch", "brulee", - "glmnet", "xgboost", - "future", "future.apply", "future.callr", "callr", - "stars", "rlang", "parallelly" - ), - library = c("/ddn/gs1/group/set/isong-archive/r-libs"), - repository = "local", + packages = + c( "amadeus", "targets", "tarchetypes", + "data.table", "sf", "terra", "exactextractr", + "dplyr", "qs", "callr", "stars", "rlang"), + controller = crew_controller_group(geo_controller), + resources = tar_resources( + crew = tar_resources_crew(controller = "geo_controller") + ), error = "abridge", memory = "transient", format = "qs", @@ -127,15 +84,65 @@ tar_option_set( seed = 202401L ) -# should run tar_make_future() + list( + tar_target(name = A, command = my_fun_a(100)), + tar_target(name = B, command = my_fun_b(A), pattern = A), + tar_target(name = save_input, command = saveRDS(B, "/input/input.rds")), + tar_target( # Test download data with amadeus + download_test, + amadeus::download_narr( + variables = c("weasd", "omega"), + year = c(2023, 2023), + directory_to_save = "/input/narr_monolevel", + acknowledgement = TRUE, + download = TRUE, + remove_command = TRUE + ) + ), + target_download + ) + + +# Style below that uses sources scripts for targets by pipeline step +# Note that variables created in _targets.R are in the same local +# environment as the sourced scripts + +# list( +# target_init, +# target_download + # target_calculate_fit, + # target_baselearner#, + # target_metalearner, + # target_calculate_predict, + # target_predict, + # # documents and summary statistics + # targets::tar_target( + # summary_urban_rural, + # summary_prediction( + # grid_filled, + # level = "point", + # contrast = "urbanrural")) + # , + # targets::tar_target( + # summary_state, + # summary_prediction( + # grid_filled, + # level = "point", + # contrast = "state" + # ) + # ) +# ) + +# targets::tar_visnetwork(targets_only = TRUE) +# END OF FILE -list( - target_init, - target_download, - target_calculate_fit, - target_baselearner, - target_metalearner, - target_calculate_predict#, +# list( +# target_init, +# target_download, +# target_calculate_fit, +# target_baselearner, +# target_metalearner, +# target_calculate_predict#, # target_predict, # # documents and summary statistics # targets::tar_target( diff --git a/container/run_container_dl_calc.sh b/container/run_container_dl_calc.sh index 49e7eb04..5d0909bb 100644 --- a/container/run_container_dl_calc.sh +++ b/container/run_container_dl_calc.sh @@ -1,17 +1,17 @@ #!/bin/bash -#SBATCH --job-name=beethoven_001 +#SBATCH --job-name=download_calc #SBATCH --partition=geo #SBATCH --mem=128G #SBATCH --cpus-per-task=4 #SBATCH --ntasks=16 -#SBATCH --output=slurm_messages/slurm-%j.out -#SBATCH --error=slurm_messages/slurm-%j.err +#SBATCH --output=../slurm_messages/slurm-%j.out +#SBATCH --error=../slurm_messages/slurm-%j.err #SBATCH --mail-user=kyle.messier@nih.gov #SBATCH --mail-type=ALL - - +# Run the container +# .sif file sites in "root/container", thus we need to go up one level with bind mounts apptainer exec \ --bind $PWD/inst:/pipeline \ --bind $PWD/input:/input \ diff --git a/container/run_dl_calc_local_tests.sh b/container/run_dl_calc_local_tests.sh index 9404e8ac..72c81a20 100644 --- a/container/run_dl_calc_local_tests.sh +++ b/container/run_dl_calc_local_tests.sh @@ -4,8 +4,8 @@ #SBATCH --mem=128G #SBATCH --cpus-per-task=4 #SBATCH --ntasks=16 -#SBATCH --output=slurm_messages/slurm-%j.out -#SBATCH --error=slurm_messages/slurm-%j.err +#SBATCH --output=../slurm_messages/slurm-%j.out +#SBATCH --error=../slurm_messages/slurm-%j.err #SBATCH --mail-user=kyle.messier@nih.gov #SBATCH --mail-type=ALL