debug in progress (20241115)

.github/workflows/check-standard.yaml

@@ -2,9 +2,9 @@
 # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
 on:
   push:
-    branches: [main, master]
+    branches: [main, master, dev]
   pull_request:
-    branches: [main, master]
+    branches: [main, master, dev]
 
 name: R-CMD-check
 

.github/workflows/lint.yaml

@@ -2,9 +2,9 @@
 # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
 on:
   push:
-    branches: [main, master]
+    branches: [main, master, dev]
   pull_request:
-    branches: [main, master]
+    branches: [main, master, dev]
 
 name: lint
 

.github/workflows/pkgdown.yaml

@@ -2,9 +2,9 @@
 # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
 on:
   push:
-    branches: [main, master]
+    branches: [main, master, dev]
   pull_request:
-    branches: [main, master]
+    branches: [main, master, dev]
   release:
     types: [published]
   workflow_dispatch:

.github/workflows/test-container-dl-calc.yaml

@@ -0,0 +1,73 @@
+name: Test Coverage for Download and Calculate via Apptainer
+
+on:
+  push:
+    branches: [main, master, dev]
+  pull_request:
+    branches: [main, master, dev]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Create run_dl_calc_tests.R dynamically
+        run: |
+          mkdir -p tests/testthat  # Ensure the directory exists
+          echo 'testthat::test_file("tests/testthat/test_download.R")' > tests/testthat/run_dl_calc_tests.R
+          echo 'testthat::test_file("tests/testthat/test_calculate.R")' >> tests/testthat/run_dl_calc_tests.R
+
+      - name: Install Apptainer dependencies
+        run: |
+          sudo apt-get update && sudo apt-get install -y \
+          libseccomp-dev \
+          squashfs-tools \
+          cryptsetup \
+          wget
+
+      - name: Install Apptainer
+        run: |
+          wget https://github.com/apptainer/apptainer/releases/download/v1.1.0/apptainer-1.1.0.tar.gz
+          tar -xvf apptainer-1.1.0.tar.gz
+          cd apptainer-1.1.0
+          ./mconfig && make -C builddir && sudo make -C builddir install
+
+      - name: Restore .sif file from cache
+        id: cache-sif
+        uses: actions/cache@v3
+        with:
+          path: beethoven_dl_calc.sif
+          key: sif-cache-${{ runner.os }}-${{ hashFiles('container/beethoven_dl_calc.def') }}
+          restore-keys: |
+            sif-cache-${{ runner.os }}-
+
+      - name: Build the Apptainer container (if cache miss)
+        if: steps.cache-sif.outputs.cache-hit != 'true'
+        run: |
+          apptainer build --force --fakeroot beethoven_dl_calc.sif container/beethoven_dl_calc.def
+
+      - name: Cache the .sif file
+        if: steps.cache-sif.outputs.cache-hit != 'true'
+        uses: actions/cache@v3
+        with:
+          path: beethoven_dl_calc.sif
+          key: sif-cache-${{ runner.os }}-${{ hashFiles('container/beethoven_dl_calc.def') }}
+
+      - name: Check if .sif file exists
+        run: |
+          if [ ! -f beethoven_dl_calc.sif ]; then
+            echo "Error: .sif file not found!"
+            exit 1
+          fi
+
+      - name: Run R tests
+        run: |
+          apptainer exec \
+            --bind $PWD/inst:/pipeline \
+            --bind $PWD/input:/input \
+            --bind $PWD:/mnt \
+            beethoven_dl_calc.sif \
+            Rscript /mnt/tests/testthat/run_dl_calc_tests.R

.github/workflows/test-coverage.yaml

@@ -3,9 +3,9 @@
 # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
 on:
   push:
-    branches: [main, master]
+    branches: [main, master, dev]
   pull_request:
-    branches: [main, master]
+    branches: [main, master, dev]
 
 name: test-coverage-local
 

.gitignore

@@ -95,6 +95,7 @@ _targets
 # future batchtools outputs
 .future
 slurm_error.log
+slurm/
 
 # SLURM messages and logs
 **/*.err
@@ -110,4 +111,10 @@ targets_start.Rout
 
 .netrc
 .urs_cookies
-beethoven_branching_notes.txt
+beethoven_branching_notes.txt
+
+# NASA Earthdata login credentials
+inst/extdata/nasa_token.txt
+
+# flexible library for beethoven ad-hoc update
+lib-flex

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: beethoven
 Title: Building an Extensible, rEproducible, Test-driven, Harmonized, Open-source, Versioned, ENsemble model for air quality
-Version: 0.4.1
+Version: 0.4.2
 Authors@R: c(
     person("Kyle", "Messier", , "kyle.messier@nih.gov", role = c("aut", "cre"), comment = c(ORCID = "0000-0001-9508-9623")),
     person("Insang", "Song", role = c("aut", "ctb"), comment = c(ORCID = "0000-0001-8732-3256")),

NAMESPACE

@@ -15,6 +15,7 @@ export(fit_base_learner)
 export(fit_base_tune)
 export(fit_meta_learner)
 export(fl_dates)
+export(fl_dates_flatten)
 export(generate_cv_index_sp)
 export(generate_cv_index_spt)
 export(generate_cv_index_ts)
@@ -38,6 +39,7 @@ export(pred_colname)
 export(predict_meta_learner)
 export(process_geos_bulk)
 export(process_narr2)
+export(query_modis_files)
 export(read_locs)
 export(read_paths)
 export(reduce_list)
@@ -51,10 +53,10 @@ export(switch_generate_cv_rset)
 export(switch_model)
 export(unmarshal_function)
 export(vis_spt_rset)
-importFrom(amadeus,calc_covariates)
 importFrom(amadeus,calc_prepare_locs)
-importFrom(amadeus,calc_temporal_dummies)
 importFrom(amadeus,calc_worker)
+importFrom(amadeus,calculate_covariates)
+importFrom(amadeus,calculate_temporal_dummies)
 importFrom(amadeus,check_for_null_parameters)
 importFrom(amadeus,download_data)
 importFrom(amadeus,download_sanitize_path)

R/calc_postprocessing.R

@@ -286,7 +286,7 @@ post_calc_df_year_expand <- function(
 #' @note This version assumes the time_id contains Date-like strings.
 #' @return data.frame
 #' @importFrom data.table merge.data.table
-#' @importFrom amadeus calc_temporal_dummies
+#' @importFrom amadeus calculate_temporal_dummies
 #' @export
 post_calc_merge_all <-
   function(
@@ -314,7 +314,7 @@ post_calc_merge_all <-
     # need POSIXt class for amadeus function
     locs_merged[[time_id]] <- as.POSIXct(locs_merged[[time_id]])
     locs_merged <-
-      amadeus::calc_temporal_dummies(
+      amadeus::calculate_temporal_dummies(
         locs = locs_merged,
         locs_id = locs_id,
         year = target_years

R/calculate.R

@@ -40,7 +40,8 @@ calc_geos_strict <-
         paths <- list.files(
           path,
           pattern = "GEOS-CF.v01.rpl",
-          full.names = TRUE
+          full.names = TRUE,
+          recursive = TRUE
         )
         paths <- paths[grep(
           ".nc4",
@@ -203,13 +204,18 @@ calc_geos_strict <-
       return(rast_ext)
 
     }
-    future::plan(future::multicore, workers = 10)
+    # future::plan(future::multicore, workers = 10)
+    # rast_summary <-
+    #   future.apply::future_lapply(
+    #     future_inserted,
+    #     function(fs) summary_byvar(fs = fs)
+    #   )
+    # future::plan(future::sequential)
     rast_summary <-
-      future.apply::future_lapply(
+      lapply(
         future_inserted,
         function(fs) summary_byvar(fs = fs)
       )
-    future::plan(future::sequential)
     rast_summary <- data.table::rbindlist(rast_summary)
 
     return(rast_summary)
@@ -519,10 +525,11 @@ par_narr <- function(domain, path, date, locs, nthreads = 24L) {
   if (!dir.exists(path)) {
     stop("The specified path does not exist.")
   }
-  future::plan(future::multicore, workers = nthreads)
+  # future::plan(future::multicore, workers = nthreads)
 
   res <-
-    future.apply::future_lapply(
+    # future.apply::future_lapply(
+    lapply(
       domain,
       function(x) {
         from <- process_narr2(
@@ -538,7 +545,32 @@ par_narr <- function(domain, path, date, locs, nthreads = 24L) {
       },
       future.seed = TRUE
     )
-  future::plan(future::sequential)
+  # future::plan(future::sequential)
   return(res)
 
 }
+
+#' Identify MODIS files
+#' @description
+#' This function identifies the relevant MODIS file paths based on
+#' path, list of julian dates, and index. Designed to help set arguments
+#' for the `inject_modis_par` function.
+#' @keywords Calculation
+#' @param path A character vector specifying the path to the MODIS data.
+#' @param list A list of julian dates.
+#' @param index An integer specifying the index of the julian date to use.
+#' @return A character vector of MODIS file paths.
+#' @export
+query_modis_files <- function(path, list, index) {
+  grep_files <- list.files(
+    path,
+    full.names = TRUE,
+    recursive = TRUE
+  ) |> grep(
+    pattern = paste0(
+      "A", list[[index]], collapse = "|"
+    ),
+    value = TRUE
+  )
+  return(grep_files)
+}

R/helpers.R

@@ -0,0 +1,34 @@
+# Helper functions for checking SLURM jobs and nodes
+
+# nocov start
+job <- function(job_id) {
+  system(
+    paste0("sacct -j ", job_id, " --format=JobID,Elapsed,TotalCPU,MaxRSS")
+  )
+}
+
+kb_to_gb <- function(kb) {
+  gb <- kb / (1024^2)
+  return(gb)
+}
+
+geo <- function() {
+  system("srun --partition=geo --cpus-per-task=1 --pty top")
+}
+
+node <- function(node = "gn040815") {
+  system(paste0("scontrol show node ", node))
+}
+
+queue <- function() {
+  system("squeue -u $USER")
+}
+
+cancel <- function() {
+  system("scancel -u $USER")
+}
+
+batch <- function(file = "run.sh") {
+  system(paste0("sbatch ", file))
+}
+# nocov end