diff --git a/NAMESPACE b/NAMESPACE
index 8aa69a3b..4fb063ae 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -1,12 +1,5 @@
# Generated by roxygen2: do not edit by hand
-export(base_learner_cv_fit)
-export(base_learner_cv_outcome)
-export(base_learner_fit)
-export(base_learner_fit_cnn)
-export(base_learner_fit_ranger)
-export(base_learner_fit_xgboost)
-export(base_learner_prep)
export(build_pipeline)
export(calc_covariates)
export(calc_ecoregion)
@@ -18,55 +11,15 @@ export(calc_temporal_dummies)
export(calc_tri)
export(check_crs_is_valid)
export(check_data_completeness)
-export(check_for_null_parameters)
export(check_input_raster_in_extent)
export(check_means_are_valid)
export(check_output_locs_are_valid)
-export(check_temporal_range)
-export(check_url_status)
-export(check_urls)
export(check_variances_are_valid)
export(convert_stdt)
export(convert_stdt_sftime)
export(convert_stdt_spatrastdataset)
export(convert_stdt_spatvect)
export(convert_stobj_to_stdt)
-export(download_aqs_data)
-export(download_data)
-export(download_ecoregion_data)
-export(download_geos_cf_data)
-export(download_gmted_data)
-export(download_koppen_geiger_data)
-export(download_merra2_data)
-export(download_modis_data)
-export(download_narr_monolevel_data)
-export(download_narr_p_levels_data)
-export(download_nei_data)
-export(download_nlcd_data)
-export(download_noaa_hms_smoke_data)
-export(download_permit)
-export(download_remove_command)
-export(download_remove_zips)
-export(download_run)
-export(download_sanitize_path)
-export(download_sedac_groads_data)
-export(download_sedac_population_data)
-export(download_setup_dir)
-export(download_sink)
-export(download_tri_data)
-export(download_unzip)
-export(extract_urls)
-export(generate_block_sp_index)
-export(generate_cv_index)
-export(generate_cv_index_lblo)
-export(generate_cv_index_lblto)
-export(generate_cv_index_lbto)
-export(generate_cv_index_lolo)
-export(generate_cv_index_lolto)
-export(generate_cv_index_loto)
-export(generate_cv_index_random)
-export(generate_date_sequence)
-export(generate_spt_index)
export(generic_base_learner)
export(generic_meta_learner)
export(is_stdt)
@@ -79,17 +32,13 @@ export(modis_prefilter_sds)
export(modis_preprocess_vnp46)
export(modis_warp_stars)
export(modis_worker)
-export(read_commands)
-export(test_download_functions)
import(BART)
-import(rvest)
import(spData)
importClassesFrom(terra,SpatRaster)
importClassesFrom(terra,SpatRasterDataset)
importClassesFrom(terra,SpatVector)
importFrom(data.table,.SD)
importFrom(data.table,as.data.table)
-importFrom(data.table,copy)
importFrom(data.table,fread)
importFrom(data.table,melt)
importFrom(data.table,merge.data.table)
@@ -104,11 +53,8 @@ importFrom(foreach,"%dopar%")
importFrom(foreach,foreach)
importFrom(future,cluster)
importFrom(future,plan)
-importFrom(httr,GET)
-importFrom(httr,HEAD)
importFrom(methods,is)
importFrom(parallelly,availableWorkers)
-importFrom(ranger,ranger)
importFrom(sf,st_as_sf)
importFrom(sf,st_as_sfc)
importFrom(sf,st_bbox)
@@ -118,17 +64,14 @@ importFrom(sf,st_covered_by)
importFrom(sf,st_crs)
importFrom(sf,st_drop_geometry)
importFrom(sf,st_geometry)
-importFrom(sf,st_join)
importFrom(sf,st_sf)
importFrom(sf,st_transform)
importFrom(sf,st_union)
importFrom(sf,st_within)
importFrom(stars,read_stars)
importFrom(stars,st_warp)
-importFrom(stats,kmeans)
importFrom(stats,predict)
importFrom(stats,reshape)
-importFrom(stringr,str_split_i)
importFrom(terra,buffer)
importFrom(terra,coltab)
importFrom(terra,crop)
@@ -146,14 +89,6 @@ importFrom(terra,same.crs)
importFrom(terra,sds)
importFrom(terra,snap)
importFrom(terra,tapp)
-importFrom(terra,time)
importFrom(terra,vect)
-importFrom(testthat,expect_true)
-importFrom(torch,torch_reshape)
-importFrom(torch,torch_tensor)
importFrom(utils,data)
-importFrom(utils,download.file)
-importFrom(utils,head)
importFrom(utils,read.csv)
-importFrom(utils,tail)
-importFrom(xgboost,xgb.train)
diff --git a/R/base_learner.R b/R/base_learner.R
deleted file mode 100644
index b7f91b20..00000000
--- a/R/base_learner.R
+++ /dev/null
@@ -1,298 +0,0 @@
-### Base learner fit
-## Potential improvement: integration to tidymodels
-## TODO: validation set in training set
-
-#' Data preparation for base learners
-#' @param learner character(1). One of `"cnn"`, `"randomforest"`, `"xgboost"`
-#' @param data stdt. see \code{\link{convert_stobj_to_stdt}}
-#' @param dependent_name Name of dependent variable. Default is `"pm2.5"`
-#' @param independent_name character. Names of independent variables.
-#' @returns A list of two matrices (except for cnn) or
-#' multidimensional arrays (cnn) depending on learners
-#' @author Insang Song
-#' @importFrom data.table as.data.table
-#' @importFrom torch torch_tensor
-#' @importFrom torch torch_reshape
-#' @export
-base_learner_prep <- function(
- learner = c("cnn", "randomforest", "xgboost"),
- data,
- dependent_name = "pm2.5",
- independent_name
-) {
- learner <- match.arg(learner)
- # read data.table from stdt
- data <- data.frame(data[[1]])
-
- if (learner == "cnn") {
- # nocov start
- ## data sorting: stdt is supposed to be sorted already
- # dimensions
- dim_s <- dim(unique(data[, 1:2]))[1]
- dim_t <- dim(unique(data[, 3]))[1]
- dim_p <- length(independent_name)
-
- ymat <-
- torch::torch_tensor(
- torch::torch_reshape(
- matrix(data[, dependent_name], ncol = 1L),
- list(dim_s, dim_t, dim_p)
- )
- )
- xmat <-
- torch::torch_tensor(
- torch::torch_reshape(
- data[, independent_name],
- list(dim_s, dim_t, dim_p)
- )
- )
- # nocov end
- } else {
- ymat <- matrix(data[, dependent_name], ncol = 1L)
- xmat <- as.matrix(data[, independent_name])
- }
-
- res <- list(
- ymat = ymat,
- xmat = xmat
- )
- return(res)
-}
-
-
-#' Split training-test data for base learner fitting
-#' @param learner character(1). Currently one of `"randomforest"`, `"xgboost"`,
-#' and `"cnn"`
-#' @param ymat data.frame or matrix. Dependent variable.
-#' @param xmat data.frame or matrix. Independent variables.
-#' @param cv_index integer. Index per cross-validation method.
-#' See \code{?generate_cv_index} for details.
-#' @param fun base_learner_fit_*
-#' @param ... Arguments passed to the argument \code{fun}
-#' @returns List of 4 with xtrain, ytrain, xtest, and ytest.
-#' @author Insang Song
-#' @export
-base_learner_cv_fit <- function(
- learner,
- ymat,
- xmat,
- cv_index,
- fun,
- ...
-) {
-
- cviter <- sort(unique(cv_index))
- cvlist <- vector("list", length = length(cviter))
-
- for (iter in cviter) {
- # row
- xtrain <- xmat[cviter != iter, ]
- xtest <- xmat[cviter == iter, ]
- ytrain <- ymat[cviter != iter, ]
- ytest <- ymat[cviter == iter, ]
-
- train_fitted <-
- fun(ymat = ytrain,
- xmat = xtrain,
- ...)
- test_fitted <-
- switch(learner,
- randomforest = predict(train_fitted, data.frame(cbind(ytest, xtest))),
- xgboost = predict(train_fitted, xtest),
- cnn = stop("cnn prediction is not yet implemented.\n")
- )
- cvlist[[iter]] <-
- list(trained = train_fitted,
- tested = test_fitted)
- }
-
- return(cvlist)
-
-}
-
-
-#' Fit base learner
-#' @param data stdt. See \code{\link{convert_stobj_to_stdt}}
-#' @param learner character(1). Currently one of `"randomforest"`, `"xgboost"`,
-#' and `"cnn"`
-#' @param dependent_name character(1). Name of the dependent variable.
-#' @param independent_name character(1). Names of independent variables.
-#' @param cv_mode character(1). One of
-#' \code{"lolo"} (leave-one-location-out),
-#' \code{"loto"} (leave-one-time-out),
-#' \code{"lolto"} (leave-one-location-time-out),
-#' \code{"lblo"} (leave-block-location-out),
-#' \code{"lbto"} (leave-block-time-out),
-#' \code{"lblto"} (leave-block-location-time-out), and
-#' \code{"random"} (full random selection)
-#' @param cv_fold integer(1). Number of folds for cross-validation.
-#' @param sp_fold integer(1). Number of subfolds for spatial blocks.
-#' @param t_fold integer(1). Number of subfolds for temporal blocks.
-#' @param blocks integer(2)/sf/SpatVector object.
-#' @param block_id character(1). The unique identifier of each block.
-#' @param return_full_object logical(1).
-#' \code{TRUE} will return the object whose class is dependent on
-#' the package in use, whereas \code{FALSE} will only return the prediction
-#' with spatial/temporal information. Full object is potentially helpful for
-#' debugging.
-#' @param ... Arguments passed to \code{base_learner_fit_*}.
-#' @seealso
-#' \link{base_learner_cv_fit}
-#' \link{base_learner_fit_xgboost}
-#' \link{base_learner_fit_ranger}
-#' \link{base_learner_fit_cnn}
-#' \link[xgboost]{xgboost}, \link[ranger]{ranger}
-#' @returns List of length \code{cv_fold}, \code{sp_fold * t_fold}, or
-#' \code{prod(blocks)} (when \code{blocks} is a numeric object of length 2) /
-#' \code{nrow(blocks)} (\code{blocks} is sf or SpatVector).
-#' @author Insang Song
-#' @description This function fits a selected base learner, which is supposed
-#' to be one of random forest, XGBoost, and convolutional neural network,
-#' using one of spatiotemporal cross-validation approaches.
-#' @export
-base_learner_fit <- function(
- data,
- learner = c("randomforest", "xgboost", "cnn"),
- dependent_name = "pm2.5",
- independent_name = NULL,
- cv_mode = c("lolo", "loto", "lolto", "random", "lblo", "lbto", "lblto"),
- cv_fold = 5L,
- sp_fold = NULL,
- t_fold = NULL,
- blocks = NULL,
- block_id = NULL,
- return_full_object = FALSE,
- ...) {
- if (is.null(independent_name)) {
- stop("independent_name cannot be null.\n")
- }
- if (!is_stdt(data)) {
- stop("data should be stdt. See ?convert_stobj_to_stdt.\n")
- }
-
- learner <- tolower(learner)
- learner <- match.arg(learner)
- cv_mode <- match.arg(cv_mode)
-
- cv_index <-
- generate_cv_index(covars = data,
- cv_mode = cv_mode,
- cv_fold = cv_fold,
- sp_fold = sp_fold,
- t_fold = t_fold,
- blocks = blocks,
- block_id = block_id)
-
- data_prep <-
- base_learner_prep(
- learner = learner,
- data = data,
- dependent_name = dependent_name,
- independent_name = independent_name
- )
-
- # switch and assign actual function to run
- run_foo <-
- switch(learner,
- randomforest = base_learner_fit_ranger,
- xgboost = base_learner_fit_xgboost,
- cnn = base_learner_fit_cnn)
- # cv fit and test
- cv_res <-
- base_learner_cv_fit(
- learner = learner,
- ymat = data_prep$ymat,
- xmat = data_prep$xmat,
- cv_index = cv_index,
- fun = run_foo,
- ...
- )
- return(cv_res)
-}
-
-#' Fit random forests with ranger
-#' @param ymat data.frame or matrix. Dependent variable.
-#' @param xmat data.frame or matrix. Independent variables.
-#' @param ... Arguments passed to [`ranger::ranger`]
-#' @return ranger object.
-#' @author Insang Song
-#' @importFrom ranger ranger
-#' @export
-base_learner_fit_ranger <- function(
- ymat,
- xmat,
- ...
-) {
- ranger::ranger(y = ymat, x = xmat, ...)
-}
-
-
-# nocov start
-#' Fit convolutional neural networks with neural network library
-#' @param ymat [`torch::torch_tensor`]. Dependent variable.
-#' @param xmat [`torch::torch_tensor`]. Independent variables.
-#' @param ... Arguments passed to fitting function
-#' @return torch-compatible object
-#' @author Insang Song
-#' @description This function uses torch as a backend.
-#' Torch currently supports CPU, CUDA, and Metal (Apple Silicon graphics),
-#' thus users can leverage available computational assets
-#' to accelerate the model fitting.
-#' @export
-base_learner_fit_cnn <- function(
- ymat,
- xmat,
- ...) {
- # check if MPS (for Apple Silicon) / CUDA is available ...
- # fit model (spt-array and 3d-pooling)
- return(NULL)
-}
-# nocov end
-
-#' Fit XGBoost model
-#' @param ymat data.frame or matrix. Dependent variable.
-#' @param xmat data.frame or matrix. Independent variables.
-#' @param ... Arguments passed to [`xgboost::xgboost`]
-#' @return xgboost object
-#' @author Insang Song
-#' @importFrom xgboost xgb.train
-#' @export
-base_learner_fit_xgboost <- function(
- ymat,
- xmat,
- ...) {
- xgboost::xgboost(data = xmat, label = ymat, ...)
-}
-
-
-#' Return cvfold outcome values
-#' @param data stdt. See \code{\link{convert_stobj_to_stdt}}
-#' @param cv_index integer. Index per cross-validation method.
-#' See \code{\link{generate_cv_index}}
-#' @param dependent_name character(1). Name of the dependent variable.
-#' @author Insang Song
-#' @returns List length of \code{max(cv_index)}
-#' with numeric vectors when `length(dependent_name) == 1` or
-#' data.frames `length(dependent_name) > 1`
-#' @export
-base_learner_cv_outcome <-
- function(
- data,
- cv_index,
- dependent_name
- ) {
- ys <- data.frame(data$stdt)
- #ys <- ys[, ]
- unique_cvi <- unique(cv_index)
- len_cvi <- length(unique_cvi)
-
- # initialize list
- cvindex_l <- vector("list", length = len_cvi)
-
- # using for loop to save memory
- for (idx in seq_len(len_cvi)) {
- # dependent variable
- cvindex_l[[idx]] <- ys[cv_index == unique_cvi[idx], dependent_name]
- }
- return(cvindex_l)
- }
diff --git a/R/calculate_covariates.R b/R/calculate_covariates.R
index dff95648..d3ef2774 100644
--- a/R/calculate_covariates.R
+++ b/R/calculate_covariates.R
@@ -234,7 +234,7 @@ calc_nlcd_ratio <- function(path,
us_main <- sf::st_union(get("us_states")) |>
terra::vect() |>
terra::project(y = terra::crs(sites))
- sites <- data_vect_b <- sites |>
+ data_vect_b <- sites |>
terra::intersect(x = us_main)
if (!terra::same.crs(data_vect_b, nlcd)) {
data_vect_b <- terra::project(data_vect_b, terra::crs(nlcd))
diff --git a/R/check_temporal_range.R b/R/check_temporal_range.R
deleted file mode 100644
index ad485af0..00000000
--- a/R/check_temporal_range.R
+++ /dev/null
@@ -1,45 +0,0 @@
-# Function to test temporal range of various data sets throughout pipeline
-# Date created: 2023-11-03
-# Mitchell Manware
-
-#' Check if input data is within user defined temporal range
-#'
-#' @param data file path to data
-#' @param start_range beginning of temporal range
-#' @param end_range end of temporal range
-#' @param data_type Point, Polygon, or Raster data
-#' @param timezone character.
-#' Time zone in three uppercase letters (e.g., UTC, EST, etc.)
-#' @param time_column character. Name of the column storing
-#' time information.
-#' @return A logical vector reporting whether time observations in data are
-#' within defined temporal range
-#' @importFrom terra rast
-#' @importFrom terra vect
-#' @importFrom terra time
-#' @author Mitchell Manware
-#' @export
-check_temporal_range <- function(
- data = NULL,
- start_range = "2018-01-01",
- end_range = "2022-12-31",
- data_type = NULL,
- timezone = "UTC",
- time_column = "Date") {
- #### 1. create sequence of dates
- start_date <- as.POSIXlt(start_range, tz = timezone)
- end_date <- as.POSIXlt(end_range, tz = timezone)
- range <- seq(start_date, end_date, 86400)
- #### 2. import data and define dates vector
- if (data_type %in% c("Point", "Polygon")) {
- data <- terra::vect(data)
- dates <- as.POSIXlt(unlist(data[[time_column]]), tz = timezone)
- } else if (data_type == "Raster") {
- data <- terra::rast(data)
- dates <- as.POSIXlt(time(data), tz = timezone)
- }
- #### 3. check dates are within range
- checked <- data.frame(dates %in% range)
- #### 4. return logical vector
- return(checked)
-}
diff --git a/R/cross_validation.R b/R/cross_validation.R
deleted file mode 100644
index c8b1ba8a..00000000
--- a/R/cross_validation.R
+++ /dev/null
@@ -1,390 +0,0 @@
-#' Generate spatio-temporal cross-validation index
-#' @param covars stdt. See \code{\link{convert_stobj_to_stdt}}
-#' for details.
-#' @param cv_mode character(1). One of `"lolo"` (leave-one-location-out),
-#' `"loto"` (leave-one-time-out),
-#' `"lolto"` (leave-one-location-time-out),
-#' `"lblo"` (leave-block-location-out),
-#' `"lbto"` (leave-block-time-out),
-#' `"lblto"` (leave-block-location-time-out)
-#' `"random"` (full random selection)
-#' @param cv_fold integer(1). Number of folds for cross-validation.
-#' @param sp_fold integer(1). Number of subfolds for spatial blocks.
-#' @param t_fold integer(1). Number of subfolds for temporal blocks.
-#' @param blocks integer(2)/sf/SpatVector object.
-#' @param block_id character(1). The unique identifier of each block.
-#' @details \code{blocks} is NULL as default
-#' (i.e., no block structure is considered); then "lb*" cv_mode
-#' is not working.
-#' \code{blocks} should be one of integer vector (length 2),
-#' sf, sftime, or SpatVector object.
-#' Please be advised that we cannot provide error messages
-#' related to the \code{blocks} type invalidity
-#' due to the cyclometric complexity.
-#' If any valid object is assigned here,
-#' cv_fold will not make sense and be ignored.
-#' Definition of unit grid (horizontal, vertical) size,
-#' arbitrary shape of blocks (sf/SpatVector case)
-#' where users can consider common spatial hierarchies such as
-#' states, counties, hydrological unit code areas, etc.
-#' \code{lblto} mode will accept \code{sp_fold} and \code{t_fold}.
-#' The maximum of results in this case is \code{sp_fold * t_fold}.
-#' @returns A numeric vector with length of the number of input rows
-#' @author Insang Song
-#' @export
-generate_cv_index <- function(
- covars,
- cv_mode = c("lolo", "loto", "lolto", "random", "lblo", "lbto", "lblto"),
- cv_fold = 5L,
- sp_fold = NULL,
- t_fold = NULL,
- blocks = NULL,
- block_id = NULL) {
- # type check
- if (!any("stdt" %in% class(covars))) {
- stop("Only stdt object is acceptable. Please consider
- using convert_stobj_to_stdt.\n")
- }
- cv_mode <- match.arg(cv_mode)
-
- # no block check
- # sensible cv_mode and cv_fold
- if (startsWith(cv_mode, "lb") && is.null(cv_fold)) {
- stop("Inputs for blocks argument are invalid.
- Please revisit the help page of this function
- for the details of proper setting of blocks.\n")
- }
-
- if (cv_mode != "lblto") {
- if ((!is.null(sp_fold) || !is.null(t_fold))) {
- stop("sp_fold and t_fold values are only applicable to
- cv_mode == 'lblto'\n")
- }
- }
-
- index_cv <- switch(cv_mode,
- lolo = generate_cv_index_lolo(covars),
- loto = generate_cv_index_loto(covars),
- lolto = generate_cv_index_lolto(covars),
- lblo = generate_cv_index_lblo(covars, cv_fold, blocks, block_id),
- lbto = generate_cv_index_lbto(covars, cv_fold),
- lblto = generate_cv_index_lblto(covars,
- sp_fold = sp_fold, t_fold = t_fold,
- blocks = blocks, block_id = block_id
- ),
- random = generate_cv_index_random(covars, cv_fold)
- )
-
- return(index_cv)
-}
-
-
-#' Generate unique spatiotemporal identifier from stdt object
-#' @param covars stdt. See [convert_stobj_to_stdt]
-#' @param mode One of `"spatial"` or `"spatiotemporal"`
-#' @description It generates unique spatiotemporal identifier in the
-#' input stdt object. Regardless of mode values
-#' (should be one of `"spatial"` or `"spatiotemporal"`),
-#' the index column will be named "sp_index".
-#' @returns stdt with a new index named "sp_index"
-#' @author Insang Song
-#' @export
-generate_spt_index <- function(
- covars,
- mode = c("spatial", "spatiotemporal")) {
- mode <- match.arg(mode)
- # generate unique sp_index
- covar_dt <- covars$stdt
-
- if ("sp_index" %in% colnames(covar_dt)) {
- return(covars)
- }
- covar_dt[["sp_index"]] <-
- paste0(covar_dt[["lon"]], "_", covar_dt[["lat"]])
-
- if (mode == "spatiotemporal") {
- covar_dt[["sp_index"]] <-
- paste0(covar_dt[["sp_index"]], "_", covar_dt[["time"]])
- }
-
- covars$stdt <- covar_dt
- return(covars)
-}
-
-# nolint start
-#' Generate blocked spatial index
-#' @param covars stdt.
-#' @param cv_fold integer(1). Number of folds for cross-validation.
-#' @param blocks numeric(2)/sf/SpatVector configuration of blocks.
-#' @param block_id character(1). The unique identifier of each block.
-#' @details "Block" in this function refers to a group of contiguous spatial/temporal entities.
-#' Regardless of `mode` value (should be one of `"spatial"` or `"spatiotemporal"`),
-#' the index column will be named "sp_index". \code{block_id} should be set with a proper field name
-#' when blocks is a sf or SpatVector object. When \code{cv_fold} is an integer, then the coordinates stored in \code{stdt} object are clustered with k-means. In this case, the result object will include attributes (accessible with \code{attr} function) about
-#' "kmeans_centers" (coordinates of centers) and "kmeans_sizes" (size of each cluster)
-#' @return stdt
-#' @author Insang Song
-#' @importFrom methods is
-#' @importFrom stats kmeans
-#' @importFrom sf st_join
-#' @importFrom terra intersect
-#' @importFrom data.table .SD
-#' @importFrom data.table copy
-#' @export
-# nolint end
-generate_block_sp_index <- function(
- covars,
- cv_fold = NULL,
- blocks = NULL,
- block_id = NULL) {
- detected_class <- class(blocks)[1]
-
- if (!is.null(cv_fold)) {
- # deterministic k-fold... may be improved later
- coords <- unique(covars$stdt[, .SD, .SDcols = seq(1, 2)])
- coordsm <- data.table::copy(coords)
- coordsm <- as.matrix(coordsm)
-
- km_index <- stats::kmeans(
- x = coordsm,
- centers = cv_fold,
- iter.max = 20L
- )
-
- coords$sp_index <- km_index$cluster
- covars$stdt <- data.table::copy(
- data.table::merge.data.table(
- covars$stdt,
- coords,
- by = c("lon", "lat")
- )
- )
-
- attr(covars, "kmeans_centers") <- km_index$centers
- attr(covars, "kmeans_sizes") <- km_index$size
- }
-
- if (inherits(blocks, "sf") ||
- inherits(blocks, "sftime") ||
- inherits(blocks, "SpatVector")) {
- if (is.null(block_id)) {
- stop("block_id must be set for this type of argument blocks.\n")
- }
- if (any(duplicated(unlist(blocks[[block_id]])))) {
- stop("block_id has duplicates.
- Please make sure 'blocks' argument consists of unique identifiers.\n")
- }
-
- # spatial join
- fun_stjoin <- switch(detected_class,
- sf = sf::st_join,
- sftime = sf::st_join,
- SpatVector = terra::intersect
- )
-
- covars_recov <- convert_stdt(covars, class_to = detected_class)
- covars_recov_id <- fun_stjoin(covars_recov, blocks[, block_id])
- covars$stdt[["sp_index"]] <- unlist(covars_recov_id[[block_id]])
- }
-
- if (is.numeric(blocks)) {
- step_lon <- blocks[1]
- step_lat <- blocks[2]
-
- covars_recov_id <- data.table::copy(covars$stdt)
-
- vlon <- unlist(covars_recov_id[["lon"]])
- vlat <- unlist(covars_recov_id[["lat"]])
- vlon_cuts <- seq(min(vlon), max(vlon), step_lon)
- vlat_cuts <- seq(min(vlat), max(vlat), step_lat)
-
- x_range <-
- cut(vlon, vlon_cuts, include.lowest = TRUE)
- y_range <-
- cut(vlat, vlat_cuts, include.lowest = TRUE)
- xy_range <- paste(
- as.character(x_range),
- as.character(y_range),
- sep = "|"
- )
- xy_range_num <- as.numeric(factor(xy_range))
- covars$stdt[["sp_index"]] <- xy_range_num
- }
-
- return(covars)
-}
-
-#' Generate spatio-temporal cross-validation index (leave-one-time-out)
-#' @param covars stdt. See \code{\link{convert_stobj_to_stdt}} for details.
-#' @author Insang Song
-#' @return An integer vector.
-#' @export
-generate_cv_index_loto <-
- function(
- covars
- ) {
- origin_ts <- covars$stdt$time
- sorted_ts <- sort(unique(origin_ts))
- cv_index <- as.numeric(factor(origin_ts, levels = sorted_ts))
- return(cv_index)
- }
-
-#' Generate spatio-temporal cross-validation index (leave-one-location-out)
-#' @param covars stdt. See \code{\link{convert_stobj_to_stdt}} for details.
-#' @author Insang Song
-#' @return An integer vector.
-#' @export
-generate_cv_index_lolo <-
- function(
- covars
- ) {
- covars_sp_index <- generate_spt_index(covars, mode = "spatial")
- sp_index_origin <- unlist(covars_sp_index$stdt[["sp_index"]])
- sp_index_unique <- sort(unique(sp_index_origin))
- cv_index <- as.numeric(factor(sp_index_origin, levels = sp_index_unique))
- return(cv_index)
- }
-
-
-#' Generate spatio-temporal cross-validation index leave-one-location-time-out)
-#' @param covars stdt. See \code{\link{convert_stobj_to_stdt}} for details.
-#' @author Insang Song
-#' @return An integer vector.
-#' @export
-generate_cv_index_lolto <-
- function(
- covars
- ) {
- rows <- nrow(covars$stdt)
- cv_index <- seq(1, rows)
- return(cv_index)
- }
-
-# nolint start
-#' Generate spatio-temporal cross-validation index (leave-block-location-out)
-#' @param covars stdt. See \code{\link{convert_stobj_to_stdt}} for details.
-#' @param cv_fold integer(1). Number of folds for cross-validation.
-#' @param blocks integer(2)/sf/SpatVector object.
-#' @param block_id character(1). The unique identifier of each block.
-#' @author Insang Song
-#' @return An integer vector.
-#' @export
-# nolint end
-generate_cv_index_lblo <-
- function(
- covars,
- cv_fold = NULL,
- blocks = NULL,
- block_id = NULL
- ) {
- if (is.null(cv_fold) && is.null(blocks)) {
- stop("Argument cv_fold cannot be NULL unless
- valid argument for blocks is entered.
- Please set a proper number.\n")
- }
- covars_sp_index <- generate_block_sp_index(
- covars,
- cv_fold = cv_fold, blocks, block_id
- )
-
- cv_index <- covars_sp_index$stdt$sp_index
-
- # if cv_index is character (when block vector is entered)
- # convert cv_index to factor
- if (is.character(cv_index)) {
- cv_index <- factor(cv_index)
- }
- return(cv_index)
- }
-
-# nolint start
-#' Generate spatio-temporal cross-validation index (leave-block-time-out)
-#' @param covars stdt. See \code{\link{convert_stobj_to_stdt}} for details.
-#' @param cv_fold integer(1). Number of folds for cross-validation.
-#' @author Insang Song
-#' @return An integer vector.
-#' @export
-# nolint end
-generate_cv_index_lbto <- function(
- covars,
- cv_fold = NULL) {
- if (is.null(cv_fold)) {
- stop("Argument cv_fold cannot be NULL. Please set a proper number.\n")
- }
-
- origin_ts <- covars$stdt$time
- origin_ts_min <- min(origin_ts)
- origin_ts_diff <- as.integer(origin_ts - origin_ts_min)
-
- # We assume that we use daily data...
- # Since POSIX* is based on seconds, we divide
- # 86400 (1 day) from diff
- if (startsWith(class(origin_ts_min)[1], "POSIX")) {
- origin_ts_diff <- origin_ts_diff / 86400
- }
- origin_ts_diff <- origin_ts_diff + 1
- sorted_ts <- sort(unique(origin_ts))
- length_ts <- length(sorted_ts)
-
- if (length_ts < cv_fold) {
- stop(sprintf("The total length of time series in the input is
- shorter than cv_fold.\n
- Length of the input time series: %d\n
- cv_fold: %d\n", length_ts, cv_fold))
- }
- unit_split <- ceiling(length_ts / cv_fold)
- cv_index <- ceiling(origin_ts_diff / unit_split)
- return(cv_index)
-}
-
-# nolint start
-#' Generate spatio-temporal cross-validation index (leave-block-location-time-out)
-#' @param covars stdt. See \code{\link{convert_stobj_to_stdt}} for details.
-#' @param sp_fold integer(1). Number of subfolds for spatial blocks.
-#' @param t_fold integer(1). Number of subfolds for temporal blocks.
-#' @param blocks integer(2)/sf/SpatVector object.
-#' @param block_id character(1). The unique identifier of each block.
-#' @author Insang Song
-#' @return An integer vector.
-#' @details The maximum of results is \code{sp_fold * t_fold}.
-#' @export
-# nolint end
-generate_cv_index_lblto <- function(
- covars,
- sp_fold,
- t_fold,
- blocks,
- block_id = NULL) {
- covars_sp_index <- generate_block_sp_index(
- covars,
- cv_fold = sp_fold, blocks, block_id
- )
- ts_index <- generate_cv_index_lbto(
- covars,
- cv_fold = t_fold
- )
- spt_index <- sprintf(
- "S%04d-T%04d",
- covars_sp_index$stdt$sp_index, ts_index
- )
- cv_index <- as.numeric(factor(spt_index))
- return(cv_index)
-}
-
-
-#' Generate spatio-temporal cross-validation index (random)
-#' @param covars stdt. See \code{\link{convert_stobj_to_stdt}} for details.
-#' @param cv_fold integer(1). Number of folds for cross-validation.
-#' @author Insang Song
-#' @return An integer vector with unique values of \code{seq(1, cv_fold)}
-#' @export
-generate_cv_index_random <- function(
- covars,
- cv_fold = NULL) {
- if (is.null(cv_fold)) {
- stop("Argument cv_fold cannot be NULL. Please set a proper number.\n")
- }
- rows <- nrow(covars$stdt)
- cv_index <- sample(seq(1, cv_fold), rows, replace = TRUE)
- return(cv_index)
-}
diff --git a/R/download.R b/R/download.R
deleted file mode 100644
index e2c34fe0..00000000
--- a/R/download.R
+++ /dev/null
@@ -1,2550 +0,0 @@
-# download.R
-#' Download raw data from sources
-#' @param dataset_name character(1). Dataset to download.
-#' @param directory_to_save character(1). Directory to save / unzip
-#' (if zip files are downloaded) data.
-#' @param data_download_acknowledgement logical(1). By setting \code{TRUE} the
-#' user acknowledge that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param ... Arguments passed to each download function.
-#' @note
-#' - All download function names are in \code{download_*_data} formats
-#' @author Insang Song
-#' @seealso
-#' For details of each download function per dataset,
-#' Please refer to:
-#' * \link{download_aqs_data}: "aqs", "AQS"
-#' * \link{download_ecoregion_data}: "ecoregion"
-#' * \link{download_geos_cf_data}: "geos"
-#' * \link{download_gmted_data}: "gmted", "GMTED"
-#' * \link{download_koppen_geiger_data}: "koppen", "koppengeiger"
-#' * \link{download_merra2_data}: "merra2", "merra", "MERRA", "MERRA2"
-#' * \link{download_narr_monolevel_data}: "narr_monolevel", "monolevel"
-#' * \link{download_narr_p_levels_data}: "narr_p_levels", "p_levels", "plevels"
-#' * \link{download_nlcd_data}: "nlcd", "NLCD"
-#' * \link{download_noaa_hms_smoke_data}: "noaa", "smoke", "hms"
-#' * \link{download_sedac_groads_data}: "sedac_groads", "groads"
-#' * \link{download_sedac_population_data}: "sedac_population", "population"
-#' * \link{download_modis_data}: "modis", "MODIS"
-#' * \link{download_tri_data}: "tri", "TRI"
-#' * \link{download_nei_data}: "nei", "NEI"
-#' @returns NULL
-#' @export
-download_data <-
- function(
- dataset_name = c("aqs", "ecoregion", "geos", "gmted", "koppen",
- "koppengeiger", "merra2", "merra", "narr_monolevel",
- "modis", "narr_p_levels", "nlcd", "noaa", "sedac_groads",
- "sedac_population", "groads", "population", "plevels",
- "p_levels", "monolevel", "hms", "smoke", "tri", "nei"),
- directory_to_save = NULL,
- data_download_acknowledgement = FALSE,
- ...
- ) {
-
- dataset_name <- tolower(dataset_name)
- dataset_name <- match.arg(dataset_name)
-
- # determine whether the data exist and deter proceeding?
- what_to_run <- switch(dataset_name,
- aqs = download_aqs_data,
- ecoregion = download_ecoregion_data,
- geos = download_geos_cf_data,
- gmted = download_gmted_data,
- koppen = download_koppen_geiger_data,
- koppengeiger = download_koppen_geiger_data,
- merra2 = download_merra2_data,
- merra = download_merra2_data,
- narr_monolevel = download_narr_monolevel_data,
- monolevel = download_narr_monolevel_data,
- narr_p_levels = download_narr_p_levels_data,
- p_levels = download_narr_p_levels_data,
- plevels = download_narr_p_levels_data,
- nlcd = download_nlcd_data,
- noaa = download_noaa_hms_smoke_data,
- smoke = download_noaa_hms_smoke_data,
- hms = download_noaa_hms_smoke_data,
- sedac_groads = download_sedac_groads_data,
- groads = download_sedac_groads_data,
- sedac_population = download_sedac_population_data,
- population = download_sedac_population_data,
- modis = download_modis_data,
- tri = download_tri_data,
- nei = download_nei_data
- )
-
- tryCatch(
- {
- what_to_run(
- directory_to_save = directory_to_save,
- data_download_acknowledgement = data_download_acknowledgement,
- ...
- )
- },
- error = function(e) {
- print(e)
- print(args(what_to_run))
- stop(paste0("Please refer to the argument list and ",
- "the error message above to rectify the error.\n"))
- }
- )
- }
-
-# nolint start
-#' Download daily data from AQS datamart
-#' @param parameter_code integer(1). length of 5.
-#' EPA pollutant parameter code. For details, please refer to
-#' [AQS parameter codes](https://aqs.epa.gov/aqsweb/documents/codetables/parameters.html)
-# nolint end
-#' @param year_start integer(1). length of 4.
-#' Start year for downloading data.
-#' @param year_end integer(1). length of 4.
-#' End year for downloading data.
-#' @param resolution_temporal character(1).
-#' Name of column containing POC values.
-#' Currently, no value other than `"daily"` works.
-#' @param directory_to_download character(1).
-#' Directory to download zip files from AQS data mart.
-#' @param directory_to_save character(1).
-#' Directory to decompress zip files.
-#' @param url_aqs_download character(1).
-#' URL to the AQS pre-generated datasets.
-#' @param data_download_acknowledgement logical(1). By setting \code{TRUE} the
-#' user acknowledge that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param unzip logical(1). Unzip zip files. Default \code{TRUE}.
-#' @param remove_zip logical(1). Remove zip file from directory_to_download.
-#' Default \code{FALSE}.
-#' @param download logical(1). \code{FALSE} will generate a *.txt file
-#' containing all download commands. By setting \code{TRUE} the function
-#' will download all of the requested data files.
-#' @param remove_command logical(1).
-#' Remove (\code{TRUE}) or keep (\code{FALSE})
-#' the text file containing download commands. Default is FALSE.
-#' @author Mariana Kassien, Insang Song, Mitchell Manware
-#' @returns NULL; Separate comma-separated value (CSV) files of
-#' monitors and the daily representative values
-#' will be stored in \code{directory_to_save}.
-#' @export
-download_aqs_data <-
- function(parameter_code = 88101,
- year_start = 2018,
- year_end = 2022,
- resolution_temporal = "daily",
- directory_to_download = "./input/aqs/",
- directory_to_save = "./input/aqs/",
- url_aqs_download = "https://aqs.epa.gov/aqsweb/airdata/",
- data_download_acknowledgement = FALSE,
- unzip = TRUE,
- remove_zip = FALSE,
- download = FALSE,
- remove_command = FALSE) {
- #### 1. check for data download acknowledgement
- download_permit(
- data_download_acknowledgement =
- data_download_acknowledgement
- )
- #### 2. check for null parameteres
- check_for_null_parameters(mget(ls()))
- #### 3. directory setup
- directory_to_download <- download_sanitize_path(directory_to_download)
- directory_to_save <- download_sanitize_path(directory_to_save)
- download_setup_dir(directory_to_download)
- download_setup_dir(directory_to_save)
- #### 4. define year sequence
- year_sequence <- seq(year_start, year_end, 1)
- #### 5. build URLs
- download_urls <- sprintf(
- paste(url_aqs_download,
- resolution_temporal,
- "_",
- parameter_code,
- "_%.0f.zip",
- sep = ""
- ),
- year_sequence
- )
- #### 6. check for valid URL
- if (!(check_url_status(download_urls[1]))) {
- stop(paste0(
- "Invalid year returns HTTP code 404. ",
- "Check `year_start` parameter.\n"
- ))
- }
- #### 5. build download file name
- download_names <- sprintf(
- paste(directory_to_download,
- "aqs_",
- resolution_temporal,
- "_",
- parameter_code,
- "_%.0f.zip",
- sep = ""
- ),
- year_sequence
- )
- #### 6. build download command
- download_commands <- paste0(
- "curl ",
- download_urls,
- " --output ",
- download_names,
- "\n"
- )
- #### 7. initiate "..._curl_commands.txt"
- commands_txt <- paste0(
- directory_to_download,
- "aqs_",
- parameter_code,
- "_",
- year_start, "_", year_end,
- "_",
- resolution_temporal,
- "_curl_commands.txt"
- )
- download_sink(commands_txt)
- #### 8. concatenate and print download commands to "..._curl_commands.txt"
- writeLines(download_commands)
- #### 9. finish "..._curl_commands.txt" file
- sink()
- #### 10. build system command
- system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
- )
- #### 11. download data
- if (!any(file.exists(download_names))) {
- download_run(
- download = download,
- system_command = system_command
- )
- }
- #### 12. unzip data
- for (n in seq_along(download_names)) {
- download_unzip(
- file_name = download_names[n],
- directory_to_unzip = directory_to_save,
- unzip = unzip
- )
- }
- #### 13. remove command file
- download_remove_command(
- commands_txt = commands_txt,
- remove = remove_command
- )
- #### 14. remove zip files
- for (d in seq_along(download_names)) {
- download_remove_zips(
- remove = remove_zip,
- download_name = download_names[d]
- )
- }
- }
-
-
-
-
-#' Download Ecoregion Shapefiles from EPA
-#' @description
-#' The \code{download_ecoregion_data()} function accesses and downloads
-#' Ecoregions level 3 data, where all pieces of information in the higher
-#' levels are included.
-#' @note In Linux systems as of December 2023, downloading Ecoregion data from
-#' EPA Data Commons will result in certificate errors. This is bypassed by
-#' manually identifying .crt file link in your browser by connecting to
-#' https://gaftp.epa.gov then clicking a lock icon in the address bar.
-#' (TO DEVELOPERS: see the comments in source code)
-#' Currently we bundle the pre-downloaded crt and its PEM (which is accepted
-#' in wget command) file in ./inst/extdata.
-#' @param directory_to_download character(1). Directory to download zip file
-#' of Ecoregion level 3 shapefiles
-#' @param directory_to_save character(1). Directory to decompress zip files.
-#' @param data_download_acknowledgement logical(1). By setting \code{TRUE} the
-#' user acknowledge that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param unzip logical(1). Unzip zip files. Default \code{TRUE}.
-#' @param remove_zip logical(1). Remove zip file from directory_to_download.
-#' Default \code{FALSE}.
-#' @param download logical(1). \code{FALSE} will generate a *.txt file
-#' containing all download commands. By setting \code{TRUE} the function
-#' will download all of the requested data files.
-#' @param remove_command logical(1).
-#' Remove (\code{TRUE}) or keep (\code{FALSE})
-#' the text file containing download commands.
-#' @param epa_certificate_path character(1). Path to the certificate file
-#' for EPA DataCommons. Default is
-#' 'extdata/cacert_gaftp_epa.pem' under the package installation path.
-#' @author Insang Song
-#' @returns NULL;
-#' @importFrom utils download.file
-#' @export
-download_ecoregion_data <- function(
- directory_to_download = "./input/ecoregions/",
- directory_to_save = "./input/ecoregions/",
- data_download_acknowledgement = FALSE,
- unzip = TRUE,
- remove_zip = FALSE,
- download = FALSE,
- remove_command = TRUE,
- epa_certificate_path =
- system.file("extdata/cacert_gaftp_epa.pem",
- package = "beethoven")
-) {
- #### 1. data download acknowledgement
- download_permit(data_download_acknowledgement = data_download_acknowledgement)
- #### 2. check for null parameters
- check_for_null_parameters(mget(ls()))
- #### 3. directory setup
- download_setup_dir(directory_to_save)
- download_setup_dir(directory_to_download)
- directory_to_download <- download_sanitize_path(directory_to_download)
- directory_to_save <- download_sanitize_path(directory_to_save)
- #### 4. Check the presence of file
- ## This part is hard-coded as the original file appears to
- ## be a misnomer. May need to be modified accordingly in the future.
- path_downloaded_file <- sprintf(
- "%sus_eco_l3_state_boundaries.shp",
- directory_to_save
- )
- if (file.exists(path_downloaded_file)) {
- message("Requested files exist in the target directory.\n")
- return(NULL)
- }
- #### 5. define download URL
- if (startsWith(Sys.info()["sysname"], "Linux")) {
- if (!file.exists(epa_certificate_path)) {
- message("URL should be identified in web browser
- Lock icon in the address bar at https://gaftp.epa.gov
- Click Show Certificate
- access Details then find URL with *.crt extension
- copy and replace the url below.\n"
- )
- download_crt_target <- gsub("pem", "crt", epa_certificate_path)
- certificate_url <-
- "http://cacerts.digicert.com/DigiCertGlobalG2TLSRSASHA2562020CA1-1.crt"
- utils::download.file(certificate_url, download_crt_target)
- system(paste(
- "openssl x509",
- "-inform DER",
- "-outform PEM",
- "-in",
- download_crt_target,
- "-out",
- epa_certificate_path
- ))
- }
- }
-
- download_url <- paste0(
- "https://gaftp.epa.gov/EPADataCommons/ORD/Ecoregions/us/",
- "us_eco_l3_state_boundaries.zip"
- )
- #### 6. build download file name
- download_name <- sprintf(
- "%sus_eco_l3_state_boundaries.zip",
- directory_to_download
- )
- #### 7. build download command
- download_command <-
- paste0(
- "wget --ca-certificate=",
- epa_certificate_path,
- " ",
- download_url,
- " -O ",
- download_name,
- "\n"
- )
- #### 8. initiate "..._curl_commands.txt" file
- commands_txt <- paste0(
- directory_to_download,
- "us_eco_l3_state_boundaries_",
- Sys.Date(),
- "_wget_command.txt"
- )
- #### 9. concatenate
- download_sink(commands_txt)
- #### 10. concatenate and print download commands to "..._wget_commands.txt"
- cat(download_command)
- #### 11. finish "...curl_commands.txt" file
- sink()
- #### 12. build system command
- system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
- )
- #### 13. download data
- download_run(
- download = download,
- system_command = system_command
- )
- #### 14. remove download command
- download_remove_command(commands_txt = commands_txt,
- remove = remove_command)
- if (download) {
- #### 15. unzip files
- download_unzip(
- file_name = download_name,
- directory_to_unzip = directory_to_save,
- unzip = unzip
- )
- #### 16. remove zip files
- download_remove_zips(
- remove = remove_zip,
- download_name = download_name
- )
- }
-}
-
-# nolint start
-#' Download atmospheric composition data from the NASA Global Earth Observing System (GEOS) model.
-#' @description
-#' The \code{download_goes_cf_data()} function accesses and downloads various
-#' atmospheric composition collections from the [NASA Global Earth Observing System (GEOS) model](https://gmao.gsfc.nasa.gov/GEOS_systems/).
-# nolint end
-#' @param date_start character(1). length of 10. Start date for downloading
-#' data. Format YYYY-MM-DD (ex. September 1, 2023 = `"2023-09-01"`).
-#' @param date_end character(1). length of 10. End date for downloading data.
-#' Format YYYY-MM-DD (ex. September 1, 2023 = `"2023-09-01"`).
-#' @param collection character(1). GEOS-CF data collection file name.
-#' @param directory_to_save character(1). Directory to save data.
-#' @param data_download_acknowledgement logical(1). By setting \code{TRUE} the
-#' user acknowledge that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param download logical(1). \code{FALSE} will generate a *.txt file
-#' containing all download commands. By setting \code{TRUE} the function
-#' will download all of the requested data files.
-#' @param remove_command logical(1).
-#' Remove (\code{TRUE}) or keep (\code{FALSE})
-#' the text file containing download commands.
-#' @author Mitchell Manware, Insang Song
-#' @return NULL;
-#' @export
-download_geos_cf_data <- function(
- date_start = "2023-09-01",
- date_end = "2023-09-01",
- collection =
- c(
- "aqc_tavg_1hr_g1440x721_v1", "chm_tavg_1hr_g1440x721_v1",
- "met_tavg_1hr_g1440x721_x1", "xgc_tavg_1hr_g1440x721_x1",
- "chm_inst_1hr_g1440x721_p23", "met_inst_1hr_g1440x721_p23"
- ),
- directory_to_save = "./input/data/geos_cf/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- remove_command = FALSE) {
- #### 1. check for data download acknowledgement
- download_permit(data_download_acknowledgement = data_download_acknowledgement)
- #### 2. check for null parameters
- check_for_null_parameters(mget(ls()))
- #### 3. directory setup
- download_setup_dir(directory_to_save)
- directory_to_save <- download_sanitize_path(directory_to_save)
- #### 4. match collection
- collection <- match.arg(collection)
- #### 5. define date sequence
- date_sequence <- generate_date_sequence(
- date_start,
- date_end,
- sub_hyphen = TRUE
- )
- #### 6. define time sequence
- collection_end <- substr(collection, nchar(collection), nchar(collection))
- if (collection_end == "1") {
- time_sequence <- seq(from = 30, to = 2330, by = 100)
- } else if (collection_end == "3") {
- time_sequence <- seq(from = 0, to = 2300, by = 100)
- }
- time_sequence <- sprintf("%04d", time_sequence)
- #### 7. define URL base
- base <- "https://portal.nccs.nasa.gov/datashare/gmao/geos-cf/v1/ana/"
- #### 8. initiate "..._wget_commands.txt" file
- commands_txt <- paste0(
- directory_to_save,
- collection,
- "_",
- date_start,
- "_",
- date_end,
- "_wget_commands.txt"
- )
-
- download_sink(commands_txt)
- #### 9. concatenate and print download commands to "..._wget_commands.txt"
- for (d in seq_along(date_sequence)) {
- date <- date_sequence[d]
- year <- substr(date, 1, 4)
- month <- substr(date, 5, 6)
- day <- substr(date, 7, 8)
- for (t in seq_along(time_sequence)) {
- download_url <- paste0(
- base,
- "Y",
- year,
- "/M",
- month,
- "/D",
- day,
- "/GEOS-CF.v01.rpl.",
- collection,
- ".",
- date,
- "_",
- time_sequence[t],
- "z.nc4"
- )
- if (t == 1) {
- if (!(check_url_status(download_url))) {
- sink()
- file.remove(commands_txt)
- stop(paste0(
- "Invalid date returns HTTP code 404. ",
- "Check `date_start` parameter.\n"
- ))
- }
- }
- download_folder <- paste0(
- directory_to_save,
- collection
- )
- download_command <- paste0(
- "wget ",
- download_url,
- " -P ",
- download_folder,
- "\n"
- )
- cat(download_command)
- }
- }
- #### 9. finish "..._wget_commands.txt" file
- sink()
- #### 10. build system command
- system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
- )
- #### 11. download data
- download_run(
- download = download,
- system_command = system_command
- )
- download_remove_command(
- commands_txt = commands_txt,
- remove = remove_command
- )
-}
-
-# nolint start
-#' Download global elevation data from the Global Multi-resolution Terrain Elevation Data (GMTED2010).
-#' @description
-#' The \code{download_gmted_data()} function acesses and downloads Global
-#' Multi-resolution Terrain Elevation Data (GMTED2010) from
-#' [U.S. Geological Survey and National Geospatial-Intelligence Agency](https://www.usgs.gov/coastal-changes-and-impacts/gmted2010).
-#' @param statistic character(1). Available statistics include `"Breakline Emphasis"`, `"Systematic Subsample"`, `"Median Statistic"`,
-#' `"Minimum Statistic"`, `"Mean Statistic"`, `"Maximum Statistic"`, and
-#' `"Standard Deviation Statistic"`.
-#' @param resolution character(1). Available resolutions include `"7.5 arc-seconds"`, `"15 arc-seconds"`, and `"30 arc-seconds"`.
-#' @param directory_to_download character(1). Directory to download zip files
-#' from Global Multi-resolution Terrain Elevation Data (GMTED2010).
-#' @param directory_to_save character(1). Directory to decompress zip files.
-#' @param data_download_acknowledgement logical(1). By setting \code{TRUE} the
-#' user acknowledge that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param unzip logical(1). Unzip zip files. Default is \code{TRUE}.
-#' @param remove_zip logical(1). Remove zip file from directory_to_download.
-#' Default is \code{FALSE}.
-#' @param download logical(1). \code{FALSE} will generate a *.txt file
-#' containing all download commands. By setting \code{TRUE} the function
-#' will download all of the requested data files.
-#' @param remove_command logical(1).
-#' Remove (\code{TRUE}) or keep (\code{FALSE})
-#' the text file containing download commands. Default is FALSE.
-#' @author Mitchell Manware, Insang Song
-# nolint end
-#' @return NULL;
-#' @export
-download_gmted_data <- function(
- statistic = c(
- "Breakline Emphasis", "Systematic Subsample",
- "Median Statistic", "Minimum Statistic",
- "Mean Statistic", "Maximum Statistic",
- "Standard Deviation Statistic"
- ),
- resolution = c("7.5 arc-seconds", "15 arc-seconds", "30 arc-seconds"),
- directory_to_download = "./input/gmted/",
- directory_to_save = "./input/gmted/",
- data_download_acknowledgement = FALSE,
- unzip = TRUE,
- remove_zip = FALSE,
- download = FALSE,
- remove_command = FALSE) {
- #### 1. check for data download acknowledgement
- download_permit(data_download_acknowledgement = data_download_acknowledgement)
- #### 2. check for null parameters
- check_for_null_parameters(mget(ls()))
- #### 3. directory setup
- download_setup_dir(directory_to_download)
- download_setup_dir(directory_to_save)
- directory_to_download <- download_sanitize_path(directory_to_download)
- directory_to_save <- download_sanitize_path(directory_to_save)
- #### 4. check for valid statistic
- statistic <- match.arg(statistic)
- #### 5. check for valid resolution
- resolution <- match.arg(resolution)
- #### 6. define URL base
- base <- paste0(
- "https://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo",
- "/downloads/GMTED/Grid_ZipFiles/"
- )
- #### 7. define URL statistic code
- statistics <- c(
- "Breakline Emphasis", "Systematic Subsample",
- "Median Statistic", "Minimum Statistic",
- "Mean Statistic", "Maximum Statistic",
- "Standard Deviation Statistic"
- )
- statistic_codes <- c("be", "ds", "md", "mi", "mn", "mx", "sd")
- statistic_codes <- cbind(statistics, statistic_codes)
- statistic_code <- subset(statistic_codes, statistics == statistic)[2]
- #### 8. define URL resolution code
- resolutions <- c("7.5 arc-seconds", "15 arc-seconds", "30 arc-seconds")
- resolution_codes <- c("75", "15", "30")
- resolution_codes <- cbind(resolutions, resolution_codes)
- resolution_code <- subset(resolution_codes, resolutions == resolution)[2]
- #### 9. build url
- download_url <- paste0(
- base,
- statistic_code,
- resolution_code,
- "_grd.zip"
- )
- #### 10. build download file name
- download_name <- paste0(
- directory_to_download,
- "gmted2010_",
- statistic_code,
- resolution_code,
- "_grd.zip"
- )
- #### 11. build download command
- download_command <- paste0(
- "curl -s -o ",
- download_name,
- " --url ",
- download_url,
- "\n"
- )
- #### 12. initiate "..._curl_commands.txt"
- commands_txt <- paste0(
- directory_to_download,
- "gmted_",
- gsub(" ", "", statistic),
- "_",
- gsub(" ", "", resolution),
- "_",
- Sys.Date(),
- "_curl_command.txt"
- )
- download_sink(commands_txt)
- #### 13. concatenate and print download command to "..._curl_commands.txt"
- cat(download_command)
- #### 14. finish "..._curl_commands.txt" file
- sink()
- #### 15. build system command
- system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
- )
- #### 16. download data
- download_run(
- download = download,
- system_command = system_command
- )
- #### 18. Remove command file
- download_remove_command(
- commands_txt = commands_txt,
- remove = remove_command
- )
- if (download) {
- #### 17. end if unzip == FALSE
- download_unzip(
- file_name = download_name,
- directory_to_unzip = directory_to_save,
- unzip = unzip
- )
- #### 19. remove zip files
- download_remove_zips(
- remove = remove_zip,
- download_name = download_name
- )
- }
-}
-
-# nolint start
-#' Download meteorological and atmospheric data from the Modern-Era Retrospective analysis for Research and Applications, Version 2 (MERRA-2) model.
-#' @description
-#' The \code{download_merra2_data()} function accesses and downloads various
-#' meteorological and atmospheric collections from the [Modern-Era Retrospective analysis for Research and Applications, Version 2 (MERRA-2)](https://gmao.gsfc.nasa.gov/reanalysis/MERRA-2/).
-#' @param date_start character(1). length of 10. Start date for downloading
-#' data. Format YYYY-MM-DD (ex. September 1, 2023 is `"2023-09-01"`).
-#' @param date_end character(1). length of 10. End date for downloading data.
-#' Format YYYY-MM-DD (ex. September 1, 2023 is `"2023-09-01"`).
-#' @param collection character(1). MERRA-2 data collection file name.
-#' @param directory_to_save character(1). Directory to save data.
-#' @param data_download_acknowledgement logical(1). By setting \code{TRUE} the
-#' user acknowledge that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param download logical(1). \code{FALSE} will generate a *.txt file
-#' containing all download commands. By setting \code{TRUE} the function
-#' will download all of the requested data files.
-#' @param remove_command logical(1).
-#' Remove (\code{TRUE}) or keep (\code{FALSE})
-#' the text file containing download commands.
-#' @author Mitchell Manware, Insang Song
-#' @return NULL;
-#' @export
-# nolint end
-download_merra2_data <- function(
- date_start = "2023-09-01",
- date_end = "2023-09-01",
- collection = c(
- "inst1_2d_asm_Nx", "inst1_2d_int_Nx", "inst1_2d_lfo_Nx",
- "inst3_3d_asm_Np", "inst3_3d_aer_Nv", "inst3_3d_asm_Nv",
- "inst3_3d_chm_Nv", "inst3_3d_gas_Nv", "inst3_2d_gas_Nx",
- "inst6_3d_ana_Np", "inst6_3d_ana_Nv", "statD_2d_slv_Nx",
- "tavg1_2d_adg_Nx", "tavg1_2d_aer_Nx", "tavg1_2d_chm_Nx",
- "tavg1_2d_csp_Nx", "tavg1_2d_flx_Nx", "tavg1_2d_int_Nx",
- "tavg1_2d_lfo_Nx", "tavg1_2d_lnd_Nx", "tavg1_2d_ocn_Nx",
- "tavg1_2d_rad_Nx", "tavg1_2d_slv_Nx", "tavg3_3d_mst_Ne",
- "tavg3_3d_trb_Ne", "tavg3_3d_nav_Ne", "tavg3_3d_cld_Np",
- "tavg3_3d_mst_Np", "tavg3_3d_rad_Np", "tavg3_3d_tdt_Np",
- "tavg3_3d_trb_Np", "tavg3_3d_udt_Np", "tavg3_3d_odt_Np",
- "tavg3_3d_qdt_Np", "tavg3_3d_asm_Nv", "tavg3_3d_cld_Nv",
- "tavg3_3d_mst_Nv", "tavg3_3d_rad_Nv", "tavg3_2d_glc_Nx"
- ),
- directory_to_save = "./input/merra2/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- remove_command = FALSE) {
- #### 1. check for data download acknowledgement
- download_permit(data_download_acknowledgement = data_download_acknowledgement)
- #### 2. directory setup
- download_setup_dir(directory_to_save)
- directory_to_save <- download_sanitize_path(directory_to_save)
- #### 3. check for null parameters
- check_for_null_parameters(mget(ls()))
- #### 4. check if collection is recognized
- identifiers <- c(
- "inst1_2d_asm_Nx M2I1NXASM 10.5067/3Z173KIE2TPD",
- "inst1_2d_int_Nx M2I1NXINT 10.5067/G0U6NGQ3BLE0",
- "inst1_2d_lfo_Nx M2I1NXLFO 10.5067/RCMZA6TL70BG",
- "inst3_3d_asm_Np M2I3NPASM 10.5067/QBZ6MG944HW0",
- "inst3_3d_aer_Nv M2I3NVAER 10.5067/LTVB4GPCOTK2",
- "inst3_3d_asm_Nv M2I3NVASM 10.5067/WWQSXQ8IVFW8",
- "inst3_3d_chm_Nv M2I3NVCHM 10.5067/HO9OVZWF3KW2",
- "inst3_3d_gas_Nv M2I3NVGAS 10.5067/96BUID8HGGX5",
- "inst3_2d_gas_Nx M2I3NXGAS 10.5067/HNGA0EWW0R09",
- "inst6_3d_ana_Np M2I6NPANA 10.5067/A7S6XP56VZWS",
- "inst6_3d_ana_Nv M2I6NVANA 10.5067/IUUF4WB9FT4W",
- "statD_2d_slv_Nx M2SDNXSLV 10.5067/9SC1VNTWGWV3",
- "tavg1_2d_adg_Nx M2T1NXADG 10.5067/HM00OHQBHKTP",
- "tavg1_2d_aer_Nx M2T1NXAER 10.5067/KLICLTZ8EM9D",
- "tavg1_2d_chm_Nx M2T1NXCHM 10.5067/3RQ5YS674DGQ",
- "tavg1_2d_csp_Nx M2T1NXCSP 10.5067/H0VVAD8F6MX5",
- "tavg1_2d_flx_Nx M2T1NXFLX 10.5067/7MCPBJ41Y0K6",
- "tavg1_2d_int_Nx M2T1NXINT 10.5067/Q5GVUVUIVGO7",
- "tavg1_2d_lfo_Nx M2T1NXLFO 10.5067/L0T5GEG1NYFA",
- "tavg1_2d_lnd_Nx M2T1NXLND 10.5067/RKPHT8KC1Y1T",
- "tavg1_2d_ocn_Nx M2T1NXOCN 10.5067/Y67YQ1L3ZZ4R",
- "tavg1_2d_rad_Nx M2T1NXRAD 10.5067/Q9QMY5PBNV1T",
- "tavg1_2d_slv_Nx M2T1NXSLV 10.5067/VJAFPLI1CSIV",
- "tavg3_3d_mst_Ne M2T3NEMST 10.5067/JRUZ3SJ3ZJ72",
- "tavg3_3d_trb_Ne M2T3NETRB 10.5067/4I7ZI35QRH8K",
- "tavg3_3d_nav_Ne M2T3NENAV 10.5067/N5WAKNS1UYQN",
- "tavg3_3d_cld_Np M2T3NPCLD 10.5067/TX10URJSKT53",
- "tavg3_3d_mst_Np M2T3NPMST 10.5067/0TUFO90Q2PMS",
- "tavg3_3d_rad_Np M2T3NPRAD 10.5067/3UGE8WQXZAOK",
- "tavg3_3d_tdt_Np M2T3NPTDT 10.5067/9NCR9DDDOPFI",
- "tavg3_3d_trb_Np M2T3NPTRB 10.5067/ZRRJPGWL8AVL",
- "tavg3_3d_udt_Np M2T3NPUDT 10.5067/CWV0G3PPPWFW",
- "tavg3_3d_odt_Np M2T3NPODT 10.5067/S0LYTK57786Z",
- "tavg3_3d_qdt_Np M2T3NPQDT 10.5067/A9KWADY78YHQ",
- "tavg3_3d_asm_Nv M2T3NVASM 10.5067/SUOQESM06LPK",
- "tavg3_3d_cld_Nv M2T3NVCLD 10.5067/F9353J0FAHIH",
- "tavg3_3d_mst_Nv M2T3NVMST 10.5067/ZXTJ28TQR1TR",
- "tavg3_3d_rad_Nv M2T3NVRAD 10.5067/7GFQKO1T43RW",
- "tavg3_2d_glc_Nx M2T3NXGLC 10.5067/9ETB4TT5J6US"
- )
- identifiers <- lapply(identifiers, strsplit, split = " ")
- identifiers <- lapply(identifiers, function(x) matrix(x[[1]], nrow = 1))
- identifiers <- do.call(rbind, identifiers)
- identifiers_df <- as.data.frame(identifiers)
- colnames(identifiers_df) <- c("collection_id", "estd_name", "DOI")
- if (!(collection %in% identifiers_df$collection_id)) {
- print(identifiers_df)
- stop(paste0("Requested collection is not recognized.\n
- Please refer to the table above to find a proper collection.\n"))
- }
- #### 5. define date sequence
- date_sequence <- generate_date_sequence(
- date_start,
- date_end,
- sub_hyphen = TRUE
- )
- #### 6. define year + month sequence
- yearmonth_sequence <- unique(substr(date_sequence, 1, 6))
- #### 7. define ESDT name and DOI
- identifiers_df_requested <- subset(identifiers_df,
- subset =
- identifiers_df$collection_id ==
- collection
- )
- esdt_name <- identifiers_df_requested[, 2]
- cat(paste0(
- "Collection: ",
- collection,
- " | ESDT Name: ",
- esdt_name,
- " | DOI: ",
- identifiers_df_requested[, 3],
- "\n"
- ))
- #### 8. define URL base
- #### NOTE: sorted and defined manually according to
- #### https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/ \&
- #### https://goldsmr5.gesdisc.eosdis.nasa.gov/data/MERRA2/
- esdt_name_4 <- c(
- "M2I1NXASM", "M2I1NXINT", "M2I1NXLFO", "M2I3NXGAS",
- "M2SDNXSLV", "M2T1NXADG", "M2T1NXAER", "M2T1NXCHM",
- "M2T1NXCSP", "M2T1NXFLX", "M2T1NXINT", "M2T1NXLFO",
- "M2T1NXLND", "M2T1NXOCN", "M2T1NXRAD", "M2T1NXSLV",
- "M2T3NXGLC"
- )
- esdt_name_5 <- c(
- "M2I3NPASM", "M2I3NVAER", "M2I3NVASM", "M2I3NVCHM",
- "M2I3NVGAS", "M2I6NPANA", "M2I6NVANA", "M2T3NEMST",
- "M2T3NENAV", "M2T3NETRB", "M2T3NPCLD", "M2T3NPMST",
- "M2T3NPODT", "M2T3NPQDT", "M2T3NPRAD", "M2T3NPTDT",
- "M2T3NPTRB", "M2T3NPUDT", "M2T3NVASM", "M2T3NVCLD",
- "M2T3NVMST", "M2T3NVRAD"
- )
- if (esdt_name %in% esdt_name_4) {
- base <- "https://goldsmr4.gesdisc.eosdis.nasa.gov/data/MERRA2/"
- } else if (esdt_name %in% esdt_name_5) {
- base <- "https://goldsmr5.gesdisc.eosdis.nasa.gov/data/MERRA2/"
- }
- #### 9. identify download URLs
- list_urls <- NULL
- for (y in seq_along(yearmonth_sequence)) {
- year <- substr(yearmonth_sequence[y], 1, 4)
- month <- substr(yearmonth_sequence[y], 5, 6)
- if (y == 1) {
- base_url <- paste0(
- base,
- esdt_name,
- ".5.12.4/",
- year,
- "/",
- month,
- "/"
- )
- if (!(check_url_status(base_url))) {
- stop(paste0(
- "Invalid date returns HTTP code 404. ",
- "Check `date_start` parameter.\n"
- ))
- }
- }
- list_urls_month <- system(
- paste0(
- "wget -q -nH -nd ",
- "\"",
- base,
- esdt_name,
- ".5.12.4/",
- year,
- "/",
- month,
- "/\"",
- " -O - | grep .nc4 | awk -F'\"' ",
- "'{print $4}'"
- ),
- intern = TRUE
- )
- list_urls <- c(list_urls, list_urls_month)
- }
- #### 10. match list_urls to date sequence
- list_urls_date_sequence <- list_urls[substr(list_urls, 28, 35) %in%
- date_sequence]
- #### 11. separate data and metadata
- list_urls_data <- list_urls_date_sequence[grep("*.xml",
- list_urls_date_sequence,
- invert = TRUE
- )]
- list_urls_metadata <- list_urls_date_sequence[grep("*.xml",
- list_urls_date_sequence,
- invert = FALSE
- )]
- #### 12. initiate "..._wget_commands.txt" file
- commands_txt <- paste0(
- directory_to_save,
- collection,
- "_",
- date_start,
- "_",
- date_end,
- "_wget_commands.txt"
- )
- download_sink(commands_txt)
- #### 13. concatenate and print download commands to "..._wget_commands.txt"
- for (l in seq_along(date_sequence)) {
- year <- as.character(substr(date_sequence[l], 1, 4))
- month <- as.character(substr(date_sequence[l], 5, 6))
- download_url <- paste0(
- base,
- esdt_name,
- ".5.12.4/",
- year,
- "/",
- month,
- "/",
- list_urls_data[l]
- )
- download_folder <- paste0(
- directory_to_save,
- collection
- )
- download_command <- paste0(
- "wget ",
- download_url,
- " -P ",
- download_folder,
- "\n"
- )
- cat(download_command)
- download_url_metadata <- paste0(
- base,
- esdt_name,
- ".5.12.4/",
- year,
- "/",
- month,
- "/",
- list_urls_metadata[l]
- )
- download_folder_metadata <- paste0(
- directory_to_save,
- collection,
- "/metadata/"
- )
- download_command_metadata <- paste0(
- "wget ",
- download_url_metadata,
- " -P ",
- download_folder_metadata,
- "\n"
- )
- cat(download_command_metadata)
- }
- #### 14. finish "..._wget_commands.txt"
- sink()
- #### 15. build system command
- system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
- )
- #### 16. download data
- download_run(
- download = download,
- system_command = system_command
- )
- #### 17. Remove command file
- download_remove_command(
- commands_txt = commands_txt,
- remove = remove_command
- )
-}
-
-# nolint start
-#' download_narr_monolevel_data: download monolevel meteorological data from NOAA NCEP North American Regional Reanalysis (NARR) model.
-#' @description
-#' The \code{download_narr_monolevel_data} function accesses and downloads
-#' monolevel meteorological data from [NOAA NCEP North American Regional Reanalysis (NARR)](https://psl.noaa.gov/data/gridded/data.narr.html).
-# nolint end
-#' @param year_start integer(1). length of 4. Start of year range for
-#' downloading data.
-#' @param year_end integer(1). length of 4. End of year range for downloading
-#' data.
-#' @param variables character. Variable(s) name acronym.
-#' @param directory_to_save character(1). Directory(s) to save downloaded data
-#' files.
-#' @param data_download_acknowledgement logical(1). By setting \code{TRUE} the
-#' user acknowledge that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param download logical(1). \code{FALSE} will generate a *.txt file
-#' containing all download commands. By setting \code{TRUE} the function
-#' will download all of the requested data files.
-#' @param remove_command logical(1).
-#' Remove (\code{TRUE}) or keep (\code{FALSE})
-#' the text file containing download commands.
-#' @author Mitchell Manware, Insang Song
-#' @return NULL;
-#' @export
-download_narr_monolevel_data <- function(
- year_start = 2022,
- year_end = 2022,
- variables = NULL,
- directory_to_save = "./input/narr/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- remove_command = FALSE) {
- #### 1. check for data download acknowledgement
- download_permit(data_download_acknowledgement = data_download_acknowledgement)
- #### 2. check for null parameters
- check_for_null_parameters(mget(ls()))
- #### 3. directory setup
- download_setup_dir(directory_to_save)
- directory_to_save <- download_sanitize_path(directory_to_save)
- #### 4. define years sequence
- if (any(nchar(year_start) != 4, nchar(year_end) != 4)) {
- stop("year_start and year_end should be 4-digit integers.\n")
- }
- years <- seq(year_start, year_end, 1)
- #### 5. define variables
- variables_list <- as.vector(variables)
- #### 6. define URL base
- base <- "https://downloads.psl.noaa.gov/Datasets/NARR/Dailies/monolevel/"
- #### 7. initiate "..._curl_commands.txt"
- commands_txt <- paste0(
- directory_to_save,
- "narr_monolevel_",
- year_start, "_", year_end,
- "_curl_commands.txt"
- )
- download_sink(commands_txt)
- #### 8. concatenate and print download commands to "..._curl_commands.txt"
- for (v in seq_along(variables_list)) {
- variable <- variables_list[v]
- folder <- paste0(directory_to_save, variable, "/")
- if (!(file.exists(folder))) {
- dir.create(folder)
- }
- for (y in seq_along(years)) {
- year <- years[y]
- url <- paste0(
- base,
- variable,
- ".",
- year,
- ".nc"
- )
- if (y == 1) {
- if (!(check_url_status(url))) {
- sink()
- file.remove(commands_txt)
- stop(paste0(
- "Invalid year returns HTTP code 404. ",
- "Check `year_start` parameter.\n"
- ))
- }
- }
- destfile <- paste0(
- directory_to_save,
- variable,
- "/",
- variable,
- ".",
- year,
- ".nc"
- )
- command <- paste0(
- "curl -s -o ",
- destfile,
- " --url ",
- url,
- "\n"
- )
- cat(command)
- }
- }
- #### 9. finish "..._curl_commands.txt"
- sink()
- #### 10. build system command
- system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
- )
- #### 11. download data
- download_run(
- download = download,
- system_command = system_command
- )
- #### 12. remove command text file
- download_remove_command(
- commands_txt = commands_txt,
- remove = remove_command
- )
-}
-
-# nolint start
-#' Download pressure level meteorological data from NOAA NCEP North American Regional Reanalysis (NARR) model.
-# nolint end
-#' @description
-#' The \code{download_narr_p_levels_data} function accesses and downloads
-#' pressure level meteorological data from [NOAA NCEP North American Regional
-#' Reanalysis (NARR)](https://psl.noaa.gov/data/gridded/data.narr.html).
-#' @param year_start integer(1). length of 4. Start of year range for
-#' downloading data.
-#' @param year_end integer(1). length of 4. End of year range for downloading
-#' data.
-#' @param variables character(1). Variable(s) name acronym.
-#' @param directory_to_save character(1). Directory(s) to save downloaded data
-#' files.
-#' @param data_download_acknowledgement logical(1). By setting \code{TRUE} the
-#' user acknowledge that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param download logical(1). \code{FALSE} will generate a *.txt file
-#' containing all download commands. By setting \code{TRUE} the function
-#' will download all of the requested data files.
-#' @param remove_command logical(1).
-#' Remove (\code{TRUE}) or keep (\code{FALSE})
-#' the text file containing download commands.
-#' @author Mitchell Manware, Insang Song
-#' @return NULL;
-#' @export
-download_narr_p_levels_data <- function(
- year_start = 2022,
- year_end = 2022,
- variables = NULL,
- directory_to_save = "./input/narr/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- remove_command = FALSE) {
- #### 1. check for data download acknowledgement
- download_permit(data_download_acknowledgement = data_download_acknowledgement)
- #### 2. check for null parameters
- check_for_null_parameters(mget(ls()))
- #### 3. directory setup
- download_setup_dir(directory_to_save)
- directory_to_save <- download_sanitize_path(directory_to_save)
- #### 4. define years sequence
- years <- seq(year_start, year_end, 1)
- #### 5. define months sequence
- months <- sprintf("%02d", seq(1, 12, by = 1))
-
- #### 6. define variables
- variables_list <- as.vector(variables)
- #### 7. define URL base
- base <- "https://downloads.psl.noaa.gov//Datasets/NARR/Dailies/pressure/"
- #### 8. initiate "..._curl_commands.txt"
- commands_txt <- paste0(
- directory_to_save,
- "narr_p_levels_",
- year_start,
- "_",
- year_end,
- "_curl_commands.txt"
- )
- download_sink(commands_txt)
- #### 9. concatenate download commands to "..._curl_commands.txt"
- for (v in seq_along(variables_list)) {
- variable <- variables_list[v]
- folder <- paste0(directory_to_save, variable, "/")
- if (!(file.exists(folder))) {
- dir.create(folder)
- }
- for (y in seq_along(years)) {
- year <- years[y]
- for (m in seq_along(months)) {
- month <- months[m]
- url <- paste0(
- base,
- variable,
- ".",
- year,
- month,
- ".nc"
- )
- if (m == 1) {
- if (!(check_url_status(url))) {
- sink()
- file.remove(commands_txt)
- stop(paste0(
- "Invalid year returns HTTP code 404. ",
- "Check `year_start` parameter.\n"
- ))
- }
- }
- destfile <- paste0(
- directory_to_save,
- variable,
- "/",
- variable,
- ".",
- year,
- month,
- ".nc"
- )
- command <- paste0(
- "curl -s -o ",
- destfile,
- " --url ",
- url,
- "\n"
- )
- cat(command)
- }
- }
- }
- #### 10. finish "..._curl_commands.txt"
- sink()
- #### 11. build system command
- system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
- )
- #### 12. download data
- download_run(
- download = download,
- system_command = system_command
- )
- #### 13. Remove command file
- download_remove_command(
- commands_txt = commands_txt,
- remove = remove_command
- )
-}
-
-
-# nolint start
-#' Download land cover data from the National Land Cover Database Science Research Products.
-# nolint end
-#' @description
-#' The \code{download_nlcd_data()} function accesses and downloads
-#' land cover data from the
-#' [NLCD Science Research Products](https://www.mrlc.gov/data) data base.
-#' @param year integer(1). Available years for Coterminous United States
-#' include `2001`, `2004`, `2006`, `2008`, `2011`, `2013`, `2016`,
-#' `2019`, and `2021`.
-#' Available years for Alaska include `2001`, `2011`, and `2016`.
-#' @param collection character(1). `"Coterminous United States"` or `"Alaska"`.
-#' @param directory_to_download character(1). Directory to download zip files
-#' from National Land Cover Database Science Research Products.
-#' @param directory_to_save character(1). Directory to decompress zip files.
-#' @param data_download_acknowledgement logical(1). By setting \code{TRUE} the
-#' user acknowledge that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param unzip logical(1). Unzip zip files. Default is \code{TRUE}.
-#' @param remove_zip logical(1). Remove zip files from directory_to_download.
-#' Default is \code{FALSE}.
-#' @param download logical(1). \code{FALSE} will generate a *.txt file
-#' containing all download commands. By setting \code{TRUE} the function
-#' will download all of the requested data files.
-#' @param remove_command logical(1).
-#' Remove (\code{TRUE}) or keep (\code{FALSE})
-#' the text file containing download commands.
-#' @author Mitchell Manware, Insang Song
-#' @returns NULL;
-#' @export
-download_nlcd_data <- function(
- year = 2021,
- collection = "Coterminous United States",
- directory_to_download = "./input/nlcd/",
- directory_to_save = "./input/nlcd/",
- data_download_acknowledgement = FALSE,
- unzip = TRUE,
- remove_zip = FALSE,
- download = FALSE,
- remove_command = FALSE) {
- #### 1. check for data download acknowledgement
- download_permit(data_download_acknowledgement = data_download_acknowledgement)
- #### 2. check for null parameters
- check_for_null_parameters(mget(ls()))
- #### 3. directory setup
- download_setup_dir(directory_to_download)
- download_setup_dir(directory_to_save)
- directory_to_download <- download_sanitize_path(directory_to_download)
- directory_to_save <- download_sanitize_path(directory_to_save)
- #### 4. check for valid years
- valid_years <- c(2001, 2004, 2006, 2008, 2011, 2013, 2016, 2019, 2021)
- if (!(year %in% valid_years)) {
- stop(paste0("Requested year is not recognized.\n"))
- }
- #### 5. define URL base
- base <- "https://s3-us-west-2.amazonaws.com/mrlc/"
- #### 6. define collection code
- if (collection == "Coterminous United States") {
- collection_code <- paste0(
- "nlcd_",
- as.character(year),
- "_land_cover_l48_"
- )
- } else if (collection == "Alaska") {
- collection_code <- paste0(
- "NLCD_",
- as.character(year),
- "_Land_Cover_AK_"
- )
- }
- #### 7. define release date
- #### NOTE: release dates identified by inspecting URLs on from
- #### https://www.mrlc.gov/data?f%5B0%5D=category%3ALand%20Cover
- if (year == 2021 && collection == "Coterminous United States") {
- release_date <- "20230630"
- } else if (!(year == 2021) && collection == "Coterminous United States") {
- release_date <- "20210604"
- } else if (collection == "Alaska") {
- release_date <- "20200724"
- }
- #### 8. build URL
- download_url <- paste0(
- base,
- collection_code,
- release_date,
- ".zip"
- )
- #### 9. build download file name
- download_name <- paste0(
- directory_to_download,
- tolower(collection_code),
- release_date,
- ".zip"
- )
- #### 10. build system command
- download_command <- paste0(
- "curl -o ",
- download_name,
- " --url ",
- download_url,
- "\n"
- )
- #### 11. initiate "..._curl_command.txt"
- commands_txt <- paste0(
- directory_to_download,
- tolower(collection_code),
- Sys.Date(),
- "_curl_command.txt"
- )
- download_sink(commands_txt)
- #### 12. concatenate and print download command to "..._curl_commands.txt"
- cat(download_command)
- #### 13. finish "..._curl_command.txt"
- sink()
- #### 14. build system command
- system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
- )
- #### 15. download data
- download_run(
- download = download,
- system_command = system_command
- )
- #### 16. end if unzip == FALSE
- download_unzip(
- file_name = download_name,
- directory_to_unzip = directory_to_save,
- unzip = unzip
- )
- #### 17. remove zip files
- download_remove_zips(
- remove = remove_zip,
- download_name = download_name
- )
- #### 18. remove command text
- download_remove_command(
- commands_txt = commands_txt,
- remove = remove_command
- )
-}
-
-# nolint start
-#' Download Global Roads Open Access Data Set (gROADS), v1 (1980-2010) data from NASA Socioeconomic Data and Applications Center (SEDAC).
-#' @description
-#' The \code{download_sedac_groads_data()} function accesses and downloads
-#' roads data from the National Aeronautics and Space
-#' Administration's (NASA) [Global Roads Open Access Data Set](https://sedac.ciesin.columbia.edu/data/set/groads-global-roads-open-access-v1/data-download).
-#' @param data_format character(1). Data can be downloaded as `"Shapefile"` or
-#' `"Geodatabase"`. (Only `"Geodatabase"` available for `"Global"` region).
-#' @param data_region character(1). Data can be downloaded for `"Global"`,
-#' `"Africa"`, `"Asia"`, `"Europe"`, `"Americas"`, `"Oceania East"`, and `"Oceania West"`.
-#' @param directory_to_download character(1). Directory to download zip files
-#' from NASA Global Roads Open Access Data Set.
-#' @param directory_to_save character(1). Directory to decompress zip files.
-#' @param data_download_acknowledgement logical(1). By setting \code{TRUE} the
-#' user acknowledge that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param unzip logical(1). Unzip zip files. Default is \code{TRUE}.
-#' @param remove_zip logical(1). Remove zip files from directory_to_download.
-#' Default is \code{FALSE}.
-#' @param download logical(1). \code{FALSE} will generate a *.txt file
-#' containing all download commands. By setting \code{TRUE} the function
-#' will download all of the requested data files.
-#' @param remove_command logical(1).
-#' Remove (\code{TRUE}) or keep (\code{FALSE})
-#' the text file containing download commands.
-#' @author Mitchell Manware, Insang Song
-#' @return NULL;
-#' @export
-download_sedac_groads_data <- function(
- data_format = c("Shapefile", "Geodatabase"),
- data_region = c("Americas", "Global", "Africa", "Asia", "Europe", "Oceania East", "Oceania West"),
- directory_to_download = "./input/sedac_groads/",
- directory_to_save = "./input/sedac_groads/",
- data_download_acknowledgement = FALSE,
- unzip = TRUE,
- remove_zip = FALSE,
- download = FALSE,
- remove_command = FALSE) {
- # nolint end
- #### 1. check for data download acknowledgement
- download_permit(data_download_acknowledgement = data_download_acknowledgement)
- #### 2. check for null parameters
- check_for_null_parameters(mget(ls()))
- #### 3. directory setup
- download_setup_dir(directory_to_download)
- download_setup_dir(directory_to_save)
- directory_to_download <- download_sanitize_path(directory_to_download)
- directory_to_save <- download_sanitize_path(directory_to_save)
- #### 4. check if region is valid
- data_format <- match.arg(data_format)
- data_region <- match.arg(data_region)
- #### 5. define URL base
- base <- paste0(
- "https://sedac.ciesin.columbia.edu/downloads/data/groads/",
- "groads-global-roads-open-access-v1/",
- "groads-v1-"
- )
- #### 6. define data format
- if (data_format == "Shapefile" && data_region == "Global") {
- message("Geodatabase format utilized for 'Global' dataset.\n")
- format <- "gdb"
- } else if (data_format == "Shapefile" && !(data_region == "Global")) {
- format <- "shp"
- } else if (data_format == "Geodatabase") {
- format <- "gdb"
- }
- #### 7. coerce region to lower case
- region <- tolower(data_region)
- #### 8. build download URL
- download_url <- paste0(
- base,
- gsub(" ", "-", region),
- "-",
- format,
- ".zip"
- )
- #### 9. build download file name
- download_name <- paste0(
- directory_to_download,
- "groads_v1_",
- gsub(" ", "-", region),
- "_",
- format,
- ".zip"
- )
- #### 10. build system command
- download_command <- paste0(
- "curl -n -c ~/.urs_cookies -b ~/.urs_cookies -LJ",
- " -o ",
- download_name,
- " --url ",
- download_url,
- "\n"
- )
- #### 11. initiate "..._curl_commands.txt"
- commands_txt <- paste0(
- directory_to_download,
- "sedac_groads_",
- gsub(" ", "_", region),
- "_",
- Sys.Date(),
- "_curl_command.txt"
- )
- download_sink(commands_txt)
- #### 12. concatenate and print download command to "..._curl_commands.txt"
- cat(download_command)
- #### 13. finish "..._curl_commands.txt" file
- sink()
- #### 14. build system command
- system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
- )
- #### 15. download data
- download_run(
- download = download,
- system_command = system_command
- )
- #### 16. end if unzip == FALSE
- download_unzip(
- file_name = download_name,
- directory_to_unzip = directory_to_save,
- unzip = unzip
- )
- #### 17. Remove command file
- download_remove_command(
- commands_txt = commands_txt,
- remove = remove_command
- )
- #### 18. remove zip files
- download_remove_zips(
- remove = remove_zip,
- download_name = download_name
- )
-}
-
-# nolint start
-#' Download UN WPP-Adjusted population density data from NASA Socioeconomic Data and Applications Center (SEDAC)
-#' @description
-#' The \code{download_sedac_population_data()} function accesses and downloads
-#' population density data from the National Aeronatuics and Space
-#' Administration's (NASA) [UN WPP-Adjusted Population Density, v4.11]( https://sedac.ciesin.columbia.edu/data/set/gpw-v4-population-density-adjusted-to-2015-unwpp-country-totals-rev11).
-#' @param year character(1). Available years are `2000`, `2005`, `2010`, `2015`, and
-#' `2020`, or `"all"` for all years.
-#' @param data_format character(1). Individual year data can be downloaded as
-#' `"ASCII"` or `"GeoTIFF"`. "all" years is downloaded as `"netCDF"`.
-#' @param data_resolution character(1). Available resolutions are 30 second
-#' (approx. 1 km), 2.5 minute (approx. 5 km), 15 minute (approx. 30 km),
-#' 30 minute (approx. 55 km), and 60 minute (approx. 110 km).
-#' @param directory_to_download character(1). Directory to download zip files
-#' from NASA UN WPP-Adjusted Population Density, v4.11.
-#' @param directory_to_save character(1). Directory to decompress zip files.
-#' @param data_download_acknowledgement logical(1). By setting \code{TRUE} the
-#' user acknowledge that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param download logical(1). \code{FALSE} will generate a *.txt file
-#' containing all download commands. By setting \code{TRUE} the function
-#' will download all of the requested data files.
-#' @param unzip logical(1). Unzip zip files. Default is \code{TRUE}.
-#' @param remove_zip logical(1). Remove zip files from directory_to_download.
-#' Default is \code{FALSE}.
-#' @param remove_command logical(1).
-#' Remove (\code{TRUE}) or keep (\code{FALSE})
-#' the text file containing download commands.
-#' @author Mitchell Manware, Insang Song
-# nolint end
-#' @return NULL;
-#' @export
-download_sedac_population_data <- function(
- year = "2020",
- data_format = c("GeoTIFF", "ASCII", "netCDF"),
- data_resolution = "60 minute",
- directory_to_download = "./input/sedac_population/",
- directory_to_save = "./input/sedac_population/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- unzip = TRUE,
- remove_zip = FALSE,
- remove_command = FALSE
-) {
- #### 1. check for data download acknowledgement
- download_permit(data_download_acknowledgement = data_download_acknowledgement)
- #### 2. check for null parameters
- check_for_null_parameters(mget(ls()))
- #### 3. directory setup
- download_setup_dir(directory_to_download)
- download_setup_dir(directory_to_save)
- directory_to_download <- download_sanitize_path(directory_to_download)
- directory_to_save <- download_sanitize_path(directory_to_save)
- #### 4. define URL base
- base <- paste0("https://sedac.ciesin.columbia.edu/downloads/data/gpw-v4/")
- #### 5. define year
- year <- ifelse(year == "all", "totpop", as.character(year))
- #### 6. define data resolution
- resolution_namecodes <- cbind(
- c(
- "60 minute", "30 second", "2.5 minute",
- "15 minute", "30 minute"
- ),
- c(
- "1_deg", "30_sec", "2pt5_min",
- "15_min", "30_min"
- )
- )
- resolution <-
- resolution_namecodes[resolution_namecodes[, 1] == data_resolution][2]
- #### 7. 30 second resolution not available for all years
- if (year == "totpop" && resolution == "30_sec") {
- resolution <- "2pt5_min"
- cat(paste0(
- "30 second resolution not available for all years. Returning",
- " highest (2.5 minute) resolution.\n"
- ))
- }
-
- data_format <- match.arg(data_format)
- #### 8. define data format
- if (data_format == "GeoTIFF") {
- if (year != "totpop") {
- format <- "tif"
- } else {
- format <- "nc"
- cat(paste0(
- "Data for all years is only available in netCDF format. ",
- "Data will be downloaded as netCDF.\n"
- ))
- }
- }
- if (data_format == "ASCII") {
- if (year != "totpop") {
- format <- "asc"
- } else {
- format <- "nc"
- cat(paste0(
- "Data for all years is only available in netCDF format. ",
- "Data will be downloaded as netCDF.\n"
- ))
- }
- }
- if (data_format == "netCDF") {
- format <- "nc"
- }
- #### 9. build download URL
- download_url <- paste0(
- base,
- "gpw-v4-population-density-adjusted-to-2015-unwpp-",
- "country-totals-rev11/",
- "gpw-v4-population-density-adjusted-to-2015-unwpp-",
- "country-totals-rev11_",
- year,
- "_",
- resolution,
- "_",
- format,
- ".zip"
- )
- #### 10. build download file name
- download_name <- paste0(
- directory_to_download,
- "gpw_v4_population_density_adjusted_to_2015_unwpp_",
- "country_totals_rev11_",
- year,
- "_",
- resolution,
- "_",
- format,
- ".zip"
- )
- #### 11. build system command
- download_command <- paste0(
- "curl -n -c ~/.urs_cookies -b ~/.urs_cookies -LJ",
- " -o ",
- download_name,
- " --url ",
- download_url,
- "\n"
- )
- #### 12. initiate "..._curl_command.txt"
- commands_txt <- paste0(
- directory_to_download,
- "sedac_population_",
- year,
- "_",
- resolution,
- "_",
- Sys.Date(),
- "_curl_commands.txt"
- )
- download_sink(commands_txt)
- #### 13. concatenate and print download command to "..._curl_commands.txt"
- cat(download_command)
- #### 14. finish "..._curl_commands.txt" file
- sink()
- #### 15. build system command
- system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
- )
- #### 16. download data
- download_run(
- download = download,
- system_command = system_command
- )
- #### 17. end if unzip == FALSE
- download_unzip(
- file_name = download_name,
- directory_to_unzip = directory_to_save,
- unzip = unzip
- )
- #### 18. Remove command file
- download_remove_command(
- commands_txt = commands_txt,
- remove = remove_command
- )
- #### 19. remove zip files
- download_remove_zips(
- remove = remove_zip,
- download_name = download_name
- )
-}
-
-# nolint start
-#' Download daily wildfire smoke plume data from NOAA Hazard Mapping System Fire and Smoke Product
-#' @description
-#' The \code{download_noaa_hms_smoke_data()} function accesses and downloads
-#' wildfire smoke plume coverage data from
-#' the National Oceanic and Atmospheric Administration's (NOAA)
-#' [Hazard Mapping System Fire and Smoke Product](https://www.ospo.noaa.gov/Products/land/hms.html#0).
-# nolint end
-#' @param date_start character(1). length of 10. Start date for downloading
-#' data. Format YYYY-MM-DD (ex. September 1, 2023 is `"2023-09-01"`).
-#' @param date_end character(1). length of 10. End date for downloading data.
-#' Format YYYY-MM-DD (ex. September 10, 2023 is `"2023-09-10"`).
-#' @param data_format character(1). "Shapefile" or "KML".
-#' @param directory_to_download character(1). Directory to download zip files
-#' from NOAA Hazard Mapping System Fire and Smoke Product. (Ignored if
-#' \code{data_format = "KML"}.)
-#' @param directory_to_save character(1). Directory to save unzipped shapefiles
-#' and KML files.
-#' @param data_download_acknowledgement logical(1).
-#' By setting \code{TRUE} the
-#' user acknowledge that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param download logical(1). \code{FALSE} will generate a *.txt file
-#' containing all download commands. By setting \code{TRUE} the function
-#' will download all of the requested data files.
-#' @param unzip logical(1). Unzip zip files. Default is \code{TRUE}. (Ignored
-#' if \code{data_format = "KML"}.)
-#' @param remove_zip logical(1). Remove zip files from
-#' directory_to_download. Default is \code{FALSE}.
-#' (Ignored if \code{data_format = "KML"}.)
-#' @param remove_command logical(1).
-#' Remove (\code{TRUE}) or keep (\code{FALSE})
-#' the text file containing download commands.
-#' @importFrom utils head
-#' @importFrom utils tail
-#' @author Mitchell Manware, Insang Song
-#' @return NULL;
-#' @export
-download_noaa_hms_smoke_data <- function(
- date_start = "2023-09-01",
- date_end = "2023-09-01",
- data_format = "Shapefile",
- directory_to_download = "./input/noaa_hms/",
- directory_to_save = "./input/noaa_hms/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- remove_command = FALSE,
- unzip = TRUE,
- remove_zip = FALSE) {
- #### 1. check for data download acknowledgement
- download_permit(data_download_acknowledgement = data_download_acknowledgement)
- #### 2. check for null parameters
- check_for_null_parameters(mget(ls()))
- #### 3. directory setup
- download_setup_dir(directory_to_download)
- download_setup_dir(directory_to_save)
- directory_to_download <- download_sanitize_path(directory_to_download)
- directory_to_save <- download_sanitize_path(directory_to_save)
- #### 4. check for unzip == FALSE && remove_zip == TRUE
- if (unzip == FALSE && remove_zip == TRUE) {
- stop(paste0(
- "Arguments unzip = FALSE and remove_zip = TRUE are not ",
- "acceptable together. Please change one.\n"
- ))
- }
- #### 5. define date sequence
- date_sequence <- generate_date_sequence(
- date_start,
- date_end,
- sub_hyphen = TRUE
- )
- #### 6. define URL base
- base <- "https://satepsanone.nesdis.noaa.gov/pub/FIRE/web/HMS/Smoke_Polygons/"
- #### 7. initiate "..._curl_commands.txt"
- commands_txt <- paste0(
- directory_to_download,
- "hms_smoke_",
- utils::head(date_sequence, n = 1),
- "_",
- utils::tail(date_sequence, n = 1),
- "_curl_commands.txt"
- )
- download_sink(commands_txt)
- #### 8. concatenate and print download commands to "..._curl_commands.txt"
- download_names <- NULL
- for (f in seq_along(date_sequence)) {
- year <- substr(date_sequence[f], 1, 4)
- month <- substr(date_sequence[f], 5, 6)
- if (data_format == "Shapefile") {
- suffix <- ".zip"
- directory_to_cat <- directory_to_download
- } else if (data_format == "KML") {
- suffix <- ".kml"
- directory_to_cat <- directory_to_save
- }
- url <- paste0(
- base,
- data_format,
- "/",
- year,
- "/",
- month,
- "/hms_smoke",
- date_sequence[f],
- suffix
- )
- if (f == 1) {
- if (!(check_url_status(url))) {
- sink()
- file.remove(commands_txt)
- stop(paste0(
- "Invalid date returns HTTP code 404. ",
- "Check `date_start` parameter.\n"
- ))
- }
- }
- destfile <- paste0(
- directory_to_cat,
- "hms_smoke_",
- data_format,
- "_",
- date_sequence[f],
- suffix
- )
- download_names <- c(download_names, destfile)
- command <- paste0(
- "curl -s -o ",
- destfile,
- " --url ",
- url,
- "\n"
- )
- cat(command)
- }
- #### 9. finish "..._curl_commands.txt"
- sink()
- #### 10. build system command
- system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
- )
- #### 11. download data
- download_run(
- download = download,
- system_command = system_command
- )
- #### 12. remove command file
- download_remove_command(
- remove = remove_command,
- commands_txt = commands_txt
- )
- #### 13. end if data_format == "KML"
- if (data_format == "KML") {
- return(cat(paste0("KML files cannot be unzipped.\n")))
- }
- #### 14. unzip downloaded zip files
- for (d in seq_along(download_names)) {
- download_unzip(
- file_name = download_names[d],
- directory_to_unzip = directory_to_save,
- unzip = unzip
- )
- }
- #### 15. remove zip files
- download_remove_zips(
- remove = remove_zip,
- download_name = download_names
- )
-}
-
-
-# nolint start
-#' Download climate classification data from the present and future Köppen-Geiger climate classification maps.
-#' @description
-#' The \code{download_koppen_geiger_data()} function accesses and downloads
-#' climate classification data from the Present and future
-#' Köppen-Geiger climate classification maps at
-#' 1-km resolution ([link for article](https://www.nature.com/articles/sdata2018214); [link for data](https://figshare.com/articles/dataset/Present_and_future_K_ppen-Geiger_climate_classification_maps_at_1-km_resolution/6396959/2)).
-# nolint end
-#' @param time_period character(1). Available times are `"Present"` (1980-2016)
-#' and `"Future"` (2071-2100). ("Future" classifications are based on scenario
-#' RCP8.5).
-#' @param data_resolution character(1). Available resolutions are `"0.0083"`
-#' degrees (approx. 1 km), `"0.083"` degrees (approx. 10 km), and
-#' `"0.5"` degrees (approx. 50 km).
-#' @param directory_to_download character(1). Directory to download zip files
-#' from Present and future Köppen-Geiger climate classification maps at 1-km
-#' resolution.
-#' @param directory_to_save character(1). Directory to decompress zip files.
-#' @param data_download_acknowledgement logical(1). By setting \code{TRUE} the
-#' user acknowledge that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param unzip logical(1). Unzip zip files. Default is \code{TRUE}.
-#' @param remove_zip logical(1). Remove zip files from directory_to_download.
-#' Default is \code{FALSE}.
-#' @param download logical(1). \code{FALSE} will generate a *.txt file
-#' containing all download commands. By setting \code{TRUE} the function
-#' will download all of the requested data files.
-#' @param remove_command logical(1).
-#' Remove (\code{TRUE}) or keep (\code{FALSE})
-#' the text file containing download commands.
-#' @author Mitchell Manware, Insang Song
-#' @return NULL;
-#' @export
-download_koppen_geiger_data <- function(
- time_period = c("Present", "Future"),
- data_resolution = c("0.0083", "0.083", "0.5"),
- directory_to_download = "./input/koppen_geiger/",
- directory_to_save = "./input/koppen_geiger/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- unzip = TRUE,
- remove_zip = FALSE,
- remove_command = FALSE) {
- #### 1. check for data download acknowledgement
- download_permit(data_download_acknowledgement = data_download_acknowledgement)
- #### 2. check for null parameters
- check_for_null_parameters(mget(ls()))
- #### 3. directory setup
- download_setup_dir(directory_to_download)
- download_setup_dir(directory_to_save)
- directory_to_download <- download_sanitize_path(directory_to_download)
- directory_to_save <- download_sanitize_path(directory_to_save)
- #### 4. check for data resolution
- data_resolution <- match.arg(data_resolution)
- #### 5. check for valid time period
- time_period <- match.arg(time_period)
- #### 6. define time period
- period <- tolower(time_period)
- #### 7. define data resolution
- data_resolution <- gsub("\\.", "p", data_resolution)
- #### 8 define download URL
- download_url <- "https://figshare.com/ndownloader/files/12407516"
- #### 9 build download file name
- download_name <- paste0(
- directory_to_download,
- "koppen_geiger_",
- period,
- "_",
- data_resolution,
- ".zip"
- )
- #### 10. build download command
- download_command <- paste0(
- "wget ",
- download_url,
- " -O ",
- download_name,
- "\n"
- )
- #### 11. initiate "..._wget_commands.txt"
- commands_txt <- paste0(
- directory_to_download,
- "koppen_geiger_",
- time_period,
- "_",
- data_resolution,
- "_",
- Sys.Date(),
- "_wget_command.txt"
- )
- download_sink(commands_txt)
- #### 12. concatenate and print download command to "..._wget_commands.txt"
- cat(download_command)
- #### 13. finish "..._wget_commands.txt" file
- sink()
- #### 14. build system command
- system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
- )
- #### 15. download data
- download_run(
- download = download,
- system_command = system_command
- )
-
- if (unzip) {
- #### 17. remove unwanted files
- unwanted_names <- list.files(
- path = directory_to_save,
- pattern = "Beck_KG",
- full.names = TRUE
- )
- unwanted_names <- as.vector(c(
- unwanted_names,
- paste0(
- directory_to_save,
- "KoppenGeiger.m"
- )
- ))
- tif <- paste0(
- directory_to_save,
- "/Beck_KG_V1_",
- period,
- "_",
- data_resolution,
- ".tif"
- )
- unwanted_names <- unwanted_names[grep(
- pattern = tif,
- unwanted_names,
- invert = TRUE
- )]
- file.remove(unwanted_names)
- }
-
- #### 18. Remove command file
- download_remove_command(
- commands_txt = commands_txt,
- remove = remove_command
- )
- if (download) {
- #### 16. end if unzip == FALSE
- download_unzip(
- file_name = download_name,
- directory_to_unzip = directory_to_save,
- unzip = unzip
- )
- #### 19. remove zip files
- download_remove_zips(
- remove = remove_zip,
- download_name = download_name
- )
- }
-}
-
-
-# nolint start
-#' Download MODIS product files
-# nolint end
-#' @description Need maintenance for the directory path change
-#' in NASA EOSDIS. This function first retrieves the all hdf download links
-#' on a certain day, then only selects the relevant tiles from the retrieved
-#' links. Download is only done at the queried horizontal-vertical tile number
-#' combinations. An exception is MOD06_L2 product, which is produced
-#' every five minutes every day.
-#' @note \code{date_start} and \code{date_end} should be in the same year.
-#' Directory structure looks like
-#' input/modis/raw/\{version\}/\{product\}/\{year\}/\{day_of_year\}
-#' Please note that \code{date_start} and \code{date_end} are
-#' ignored if \code{product == 'MOD06_L2'}.
-#' @param date_start character(1). length of 10. Start date for downloading
-#' data. Format YYYY-MM-DD (ex. September 1, 2023 is `"2023-09-01"`).
-#' @param date_end character(1). length of 10. End date for downloading data.
-#' Format YYYY-MM-DD (ex. September 1, 2023 is `"2023-09-01"`).
-#' @param product character(1).
-#' One of `c("MOD09GA", "MOD11A1", "MOD06_L2", "MCD19A2", "MOD13A2", "VNP46A2")`
-#' @param version character(1). Default is `"61"`, meaning v061.
-#' @param horizontal_tiles integer(2). Horizontal tile numbers
-#' `c({start}, {end})`. Default is `c(7, 13)`.
-#' @param vertical_tiles integer(2). Vertical tile numbers
-#' `c({start}, {end})`. Default is `c(3, 6)`.
-#' @param nasa_earth_data_token character(1).
-#' Token for downloading data from NASA. Should be set before
-#' trying running the function.
-#' @param directory_to_save character(1). Directory to save data.
-#' @param data_download_acknowledgement logical(1). By setting \code{TRUE} the
-#' user acknowledge that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param mod06_links character(1). CSV file path to MOD06_L2 download links
-#' from NASA LPDAAC. Default is `NULL`.
-#' @param download logical(1). Download data or only save wget commands.
-#' @param remove_command logical(1). Remove (\code{TRUE}) or keep (\code{FALSE})
-#' the text file containing download commands.
-#' @author Mitchell Manware, Insang Song
-#' @import rvest
-#' @returns NULL;
-#' @export
-download_modis_data <- function(
- date_start = "2023-09-01",
- date_end = "2023-09-01",
- product = c(
- "MOD09GA", "MOD11A1", "MOD06_L2",
- "MCD19A2", "MOD13A2", "VNP46A2"
- ),
- version = "61",
- horizontal_tiles = c(7, 13),
- vertical_tiles = c(3, 6),
- nasa_earth_data_token = NULL,
- directory_to_save = "./input/modis/raw/",
- data_download_acknowledgement = FALSE,
- mod06_links = NULL,
- download = TRUE,
- remove_command = FALSE) {
- #### 1. check for data download acknowledgement
- download_permit(data_download_acknowledgement = data_download_acknowledgement)
- #### 2. directory setup
- download_setup_dir(directory_to_save)
- directory_to_save <- download_sanitize_path(directory_to_save)
-
- #### 3. check for NASA earth data token
- if (is.null(nasa_earth_data_token)) {
- stop("Please provide NASA EarthData Login token.\n")
- }
- #### 4. check for product
- product <- match.arg(product)
-
- if (substr(date_start, 1, 4) != substr(date_end, 1, 4)) {
- if (product != "MOD06_L2") {
- stop("date_start and date_end should be in the same year.\n")
- }
- }
-
- #### 5. check for version
- if (is.null(version)) {
- stop("Please select a data version.\n")
- }
-
- #### 6. check for valid horizontal tiles
- if (!all(horizontal_tiles %in% seq(0, 35))) {
- stop("Horizontal tiles are not in the proper range [0, 35].\n")
- }
- if (!all(vertical_tiles %in% seq(0, 17))) {
- stop("Vertical tiles are not in the proper range [0, 17].\n")
- }
-
- #### 7. define horizontal and vertical tiles
- tiles_horizontal <-
- seq(
- horizontal_tiles[1],
- horizontal_tiles[2]
- )
- tiles_horizontal <-
- sprintf("h%02d", tiles_horizontal)
-
- tiles_vertical <-
- seq(
- vertical_tiles[1],
- vertical_tiles[2]
- )
- tiles_vertical <-
- sprintf("v%02d", tiles_vertical)
-
- #### 8. define requested tiles
- tiles_df <- expand.grid(
- h = tiles_horizontal,
- v = tiles_vertical
- )
- tiles_requested <-
- paste0(tiles_df$h, tiles_df$v)
-
- #### 9. Reuse ladsweb home url
- ladsurl <- "https://ladsweb.modaps.eosdis.nasa.gov/"
- version <- ifelse(startsWith(product, "VNP"), "5000", version)
-
- #### 10. MOD06_L2 manual input
- if (product == "MOD06_L2") {
- mod06l2_url1 <-
- "https://ladsweb.modaps.eosdis.nasa.gov/"
- mod06l2_url2 <-
- "search/order/4/MOD06_L2--61/"
- mod06l2_url3 <-
- "%s..%s/DNB/-130,52,-60,20"
- mod06l2_url_template <-
- paste0(mod06l2_url1, mod06l2_url2, mod06l2_url3)
- mod06l2_full <-
- sprintf(mod06l2_url_template, date_start, date_end)
-
- if (is.null(mod06_links)) {
- stop(paste(
- "Please provide a CSV file path to MOD06_L2 download links.
- You may download it from the link:\n", mod06l2_full,
- "\nTime length up to one month is recommended.\n"
- ))
- }
-
- #### 10-1. Parse urls in csv
- file_url <- read.csv(mod06_links)
- file_url <- unlist(file_url[, 2])
- download_url <-
- paste0(
- substr(ladsurl, 1, nchar(ladsurl) - 1),
- file_url
- )
-
- #### 10-2. Parse dates from csv
- file_dates <-
- regmatches(
- file_url,
- regexpr("[2][0-2][0-9]{2,2}[0-3][0-9]{2,2}", file_url)
- )
- file_dates <- as.integer(file_dates)
- date_start <- as.Date(as.character(min(file_dates)), format = "%Y%j")
- date_end <- as.Date(as.character(max(file_dates)), format = "%Y%j")
-
- #### 10-3. initiate "..._wget_commands.txt" file
- commands_txt <- paste0(
- directory_to_save,
- product,
- "_",
- date_start,
- "_",
- date_end,
- "_wget_commands.txt"
- )
-
- #### 10-4. write download_command
- download_command <- paste0(
- "wget -e robots=off -m -np -R .html,.tmp ",
- "-nH --cut-dirs=3 \"",
- download_url,
- "\" --header \"Authorization: Bearer ",
- nasa_earth_data_token,
- "\" -P ",
- directory_to_save,
- "\n"
- )
-
- # avoid any possible errors by removing existing command files
- download_sink(commands_txt)
- cat(download_command)
- sink()
-
- system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
- )
- download_run(
- download = download,
- system_command = system_command
- )
-
- message("Requests were processed.\n")
-
- download_remove_command(
- commands_txt = commands_txt,
- remove = remove_command
- )
- return(NULL)
- }
-
-
- #### 11. define date sequence
- date_sequence <- generate_date_sequence(
- date_start,
- date_end,
- sub_hyphen = FALSE
- )
- # In a certain year, list all available dates
- year <- as.character(substr(date_start, 1, 4))
- filedir_year_url <-
- paste0(
- ladsurl,
- "archive/allData/",
- version,
- "/",
- product,
- "/",
- year
- )
-
- list_available_d <-
- rvest::read_html(filedir_year_url) |>
- rvest::html_elements("tr") |>
- rvest::html_attr("data-name")
- # no conditional assignment at this moment.
-
- # remove NAs
- # 12. Queried year's available days
- date_sequence <- list_available_d[!is.na(list_available_d)]
- date_sequence_i <- as.integer(date_sequence)
- # Queried dates to integer range
- date_start_i <- as.integer(strftime(date_start, "%j"))
- date_end_i <- as.integer(strftime(date_end, "%j"))
- date_range_julian <- seq(date_start_i, date_end_i)
- date_sequence_in <- (date_sequence_i %in% date_range_julian)
-
- message(sprintf(
- "%d / %d days of data available in the queried dates.\n",
- sum(date_sequence_in), length(date_range_julian)
- ))
- date_sequence <- date_sequence[date_sequence_in]
-
-
- #### 13. initiate "..._wget_commands.txt" file
- commands_txt <- paste0(
- directory_to_save,
- product,
- "_",
- date_start,
- "_",
- date_end,
- "_wget_commands.txt"
- )
-
- # avoid any possible errors by removing existing command files
- download_sink(commands_txt)
- #### 14. append download commands to text file
- for (d in seq_along(date_sequence)) {
- day <- date_sequence[d]
- filedir_url <-
- paste0(
- filedir_year_url,
- "/",
- day
- )
-
- filelist <-
- rvest::read_html(filedir_url) |>
- rvest::html_elements("tr") |>
- rvest::html_attr("data-path")
-
- filelist_sub <-
- grep(
- paste0("(", paste(tiles_requested, collapse = "|"), ")"),
- filelist,
- value = TRUE
- )
- download_url <- sprintf("%s%s", ladsurl, filelist_sub)
- # Main wget run
- download_command <- paste0(
- "wget -e robots=off -m -np -R .html,.tmp ",
- "-nH --cut-dirs=3 \"",
- download_url,
- "\" --header \"Authorization: Bearer ",
- nasa_earth_data_token,
- "\" -P ",
- directory_to_save,
- "\n"
- )
- #### 15. concatenate and print download commands to "..._wget_commands.txt"
- cat(download_command)
- }
-
- #### 16. finish "..._wget_commands.txt"
- sink(file = NULL)
-
- #### 17.
- system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
- )
- download_run(
- download = download,
- system_command = system_command
- )
-
- message("Requests were processed.\n")
-
- download_remove_command(commands_txt = commands_txt,
- remove = remove_command)
-
-}
-
-
-#' Download data from EPA toxic release inventory
-#' @param year_start integer(1). length of 4. Start year for downloading data.
-#' @param year_end integer(1). length of 4. End year for downloading data.
-#' @param directory_to_save character(1). Directory to download files.
-#' @param data_download_acknowledgement logical(1). By setting \code{TRUE} the
-#' user acknowledge that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param download logical(1). \code{FALSE} will generate a *.txt file
-#' containing all download commands. By setting \code{TRUE} the function
-#' will download all of the requested data files.
-#' @param remove_command logical(1). Remove (\code{TRUE}) or keep (\code{FALSE})
-#' the text file containing download commands.
-#' @author Mariana Kassien, Insang Song
-#' @returns NULL; Yearly comma-separated value (CSV) raw files for each year
-#' @export
-download_tri_data <- function(
- year_start = 2018L,
- year_end = 2022L,
- directory_to_save = "./input/tri/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- remove_command = FALSE
-) {
- #### 1. check for data download acknowledgement
- download_permit(data_download_acknowledgement = data_download_acknowledgement)
- #### 2. directory setup
- download_setup_dir(directory_to_save)
- directory_to_save <- download_sanitize_path(directory_to_save)
-
- #### 3. define measurement data paths
- url_download <-
- "https://data.epa.gov/efservice/downloads/tri/mv_tri_basic_download/"
- year_sequence <- seq(year_start, year_end, 1)
- download_urls <- sprintf(
- paste(url_download, "%.0f", "_US/csv", sep = ""),
- year_sequence
- )
- download_names <-
- sprintf(paste0(directory_to_save,
- "tri_raw_%.0f.csv"),
- year_sequence)
-
- #### 4. build download command
- download_commands <- paste0("curl -L ",
- download_urls,
- " --output ",
- download_names,
- "\n")
- #### 5. initiate "..._curl_commands.txt"
- commands_txt <- paste0(
- directory_to_save,
- "TRI_",
- year_start, "_", year_end,
- "_",
- Sys.Date(),
- "_curl_commands.txt"
- )
- download_sink(commands_txt)
- #### 6. concatenate and print download commands to "..._curl_commands.txt"
- writeLines(download_commands)
- #### 7. finish "..._curl_commands.txt" file
- sink()
- #### 8. build system command
- system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
- )
- #### 9. download data
- download_run(download = download,
- system_command = system_command)
- message("Requests were processed.\n")
- #### 10. remove download commands
- download_remove_command(commands_txt = commands_txt,
- remove = remove_command)
-
-}
-
-
-# nolint start
-#' Download data from EPA National Emission Inventory aggregated on-road emission data
-# nolint end
-#' @param directory_to_save character(1). Directory to download files.
-#' @param data_download_acknowledgement logical(1). By setting \code{TRUE} the
-#' user acknowledge that the data downloaded using this function may be very
-#' large and use lots of machine storage and memory.
-#' @param download logical(1). \code{FALSE} will generate a *.txt file
-#' containing all download commands. By setting \code{TRUE} the function
-#' will download all of the requested data files.
-#' @param remove_command logical(1).
-#' Remove (\code{TRUE}) or keep (\code{FALSE})
-#' the text file containing download commands.
-#' @param year_target Available years of NEI data.
-#' Default is \code{c(2017L, 2020L)}.
-#' @param unzip logical(1). Unzip the downloaded zip files.
-#' Default is \code{FALSE}.
-#' @param epa_certificate_path character(1). Path to the certificate file
-#' for EPA DataCommons. Default is
-#' 'extdata/cacert_gaftp_epa.pem' under the package installation path.
-#' @author Ranadeep Daw, Insang Song
-#' @returns NULL; Two comma-separated value (CSV) raw files for 2017 and 2020
-#' @export
-download_nei_data <- function(
- directory_to_save = "./input/nei/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- remove_command = FALSE,
- year_target = c(2017L, 2020L),
- unzip = FALSE,
- epa_certificate_path =
- system.file("extdata/cacert_gaftp_epa.pem",
- package = "beethoven")
-) {
- #### 1. check for data download acknowledgement
- download_permit(data_download_acknowledgement = data_download_acknowledgement)
- #### 2. directory setup
- download_setup_dir(directory_to_save)
- directory_to_save <- download_sanitize_path(directory_to_save)
-
- #### 5. define download URL
- if (startsWith(Sys.info()["sysname"], "Linux")) {
- if (!file.exists(epa_certificate_path)) {
- message("URL should be identified in web browser
- Lock icon in the address bar at https://gaftp.epa.gov
- Click Show Certificate
- access Details then find URL with *.crt extension
- copy and replace the url below.\n"
- )
- download_crt_target <- gsub("pem", "crt", epa_certificate_path)
- certificate_url <-
- "http://cacerts.digicert.com/DigiCertGlobalG2TLSRSASHA2562020CA1-1.crt"
- utils::download.file(certificate_url, download_crt_target)
- system(paste("openssl x509",
- "-inform DER",
- "-outform PEM",
- "-in",
- download_crt_target,
- "-out",
- epa_certificate_path))
- }
- }
-
- #### 3. define measurement data paths
- url_download_base <- "https://gaftp.epa.gov/air/nei/%d/data_summaries/"
- url_download_remain <-
- c("2017v1/2017neiApr_onroad_byregions.zip",
- "2020nei_onroad_byregion.zip")
- download_urls <-
- paste0(
- sprintf(url_download_base, year_target),
- url_download_remain
- )
- download_names_file <-
- c("2017neiApr_onroad_byregions.zip",
- "2020nei_onroad_byregion.zip")
- download_names <- paste0(directory_to_save, download_names_file)
-
- #### 4. build download command
- download_commands <-
- paste0("wget --ca-certificate=",
- epa_certificate_path,
- " ",
- download_urls,
- " -O ",
- download_names,
- "\n")
-
- #### 5. initiate "..._curl_commands.txt"
- commands_txt <- paste0(
- directory_to_save,
- "NEI_AADT_",
- paste(year_target, collapse = "-"),
- "_",
- Sys.Date(),
- "_wget_commands.txt"
- )
- download_sink(commands_txt)
- #### 6. concatenate and print download commands to "..._curl_commands.txt"
- writeLines(download_commands)
- #### 7. finish "..._curl_commands.txt" file
- sink()
- #### 8. build system command
- system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
- )
- #### 9. download data
- download_run(download = download,
- system_command = system_command)
-
- #### 10. unzip data
- # note that this part does not utilize download_unzip
- # as duplicate file names are across multiple zip files
- if (download) {
- if (unzip) {
- dir_unzip <- sub(".zip", "", download_names)
- for (fn in seq_along(dir_unzip)) {
- utils::unzip(zipfile = download_names[fn], exdir = dir_unzip[fn])
- }
- }
- }
- message("Requests were processed.\n")
- #### 10. remove download commands
- download_remove_command(commands_txt = commands_txt,
- remove = remove_command)
-
-}
diff --git a/R/download_support.R b/R/download_support.R
deleted file mode 100644
index cd6ab494..00000000
--- a/R/download_support.R
+++ /dev/null
@@ -1,200 +0,0 @@
-################################################################################
-# Date created: 2023-12-06
-# Packages required: None
-################################################################################
-
-
-#' Check if input directory exists
-#' @param directory character(1) directory path
-#' @description If directory does not exist, the directory
-#' will be created.
-#' @returns NULL
-#' @export
-download_setup_dir <-
- function(directory) {
- if (!dir.exists(directory)) {
- dir.create(directory, recursive = TRUE)
- }
- }
-
-
-#' Sanitize path to end with a forward slash
-#' @param directory character(1). Path
-#' @returns character ending with a forward slash.
-#' @export
-download_sanitize_path <-
- function(directory) {
- #### 1. directory setup
- chars_dir <- nchar(directory)
- if (substr(
- directory,
- chars_dir,
- chars_dir
- ) != "/") {
- directory <-
- paste(directory,
- "/",
- sep = ""
- )
- }
- return(directory)
- }
-
-
-#' Check for data download acknowledgement
-#' @param data_download_acknowledgement logical(1). Whether to
-#' start downloading
-#' @returns NULL
-#' @export
-download_permit <-
- function(data_download_acknowledgement) {
- if (!data_download_acknowledgement) {
- stop(paste0(
- "Data download acknowledgement is set to FALSE. ",
- "Please acknowledge that the data downloaded using this ",
- "function may be very large and use lots of machine storage ",
- "and memory.\n"
- ))
- }
- }
-
-
-#' download_run: execute or skip \code{system_command}
-#' in data download function.
-#'
-#' @description
-#' Execute or skip the commands listed in the ...wget/curl_commands.txt file
-#' produced by one of the data download functions.
-#' @param download logical(1). Execute (\code{TRUE}) or
-#' skip (\code{FALSE}) download.
-#' @param system_command character(1). Linux command to execute downloads.
-#' Inherited from data download function.
-#' @returns NULL
-#' @export
-download_run <- function(
- download = FALSE,
- system_command = NULL) {
- if (download == TRUE) {
- cat(paste0("Downloading requested files...\n"))
- system(command = system_command)
- cat(paste0("Requested files have been downloaded.\n"))
- } else {
- cat(paste0("Skipping data download.\n"))
- return(NULL)
- }
-}
-
-
-#' Remove or keep wget command file
-#'
-#' @param commands_txt character(1). Path of download commands
-#' @param remove logical(1). Remove (\code{TRUE}) or
-#' keep (\code{FALSE}) commands
-#' @returns NULL
-#' @export
-download_remove_command <-
- function(commands_txt = NULL,
- remove = FALSE) {
- if (remove) {
- file.remove(commands_txt)
- }
- }
-
-
-#' Start sink download commands into a text file
-#' @param command_txt character(1). file path to export commands.
-#' @returns NULL
-#' @export
-download_sink <-
- function(command_txt) {
- if (file.exists(command_txt)) {
- file.remove(command_txt)
- }
- sink(file = command_txt, append = FALSE)
- }
-
-
-#' Unzip downloaded data
-#' @param file_name character(1). Full zip file path
-#' @param directory_to_unzip character(1). Directory to unzip
-#' data
-#' @param unzip logical(1). Unzip (\code{TRUE}) or not.
-#' @returns NULL
-#' @export
-download_unzip <-
- function(file_name,
- directory_to_unzip,
- unzip = TRUE) {
- if (!unzip) {
- cat(paste0("Downloaded files will not be unzipped.\n"))
- return(NULL)
- }
-
- cat(paste0("Unzipping files...\n"))
- unzip(file_name,
- exdir = directory_to_unzip
- )
- cat(paste0(
- "Files unzipped and saved in ",
- directory_to_unzip,
- ".\n"
- ))
- }
-
-
-#' Remove downloaded zip files
-#' @param remove logical(1). Confirm removal. Default is FALSE.
-#' @param download_name character. Full zip file path
-#' @returns NULL
-#' @export
-download_remove_zips <-
- function(remove = FALSE,
- download_name) {
- #### remove zip files
- if (remove) {
- cat(paste0("Removing download files...\n"))
- file.remove(download_name)
- cat(paste0("Download files removed.\n"))
- }
- }
-
-
-#' Check for null arguments
-#' @param parameters parameters passed to function (called by
-#' \code{mget(ls())}.)
-#' @returns NULL
-#' @export
-check_for_null_parameters <-
- function(
- parameters) {
- parameters_status <- any(unlist(lapply(parameters, is.null)))
- if (parameters_status) {
- stop(paste0("One or more parameters are NULL\n"))
- }
- }
-
-#' Generate sequence of dates based on `date_start` and `date_end`.
-#' @param date_start character(1). Beginning of date sequence.
-#' @param date_end character(1). End of date sequence.
-#' @param sub_hyphen logical(1). Substitute hyphen in dates. If `TRUE`, returns
-#' date sequence as "YYYYMMDD". If `FALSE`, returns date sequence as
-#' "YYYY-MM-DD".
-#' @returns vector
-#' @export
-generate_date_sequence <-
- function(
- date_start,
- date_end,
- sub_hyphen = TRUE) {
- dates_original <- seq(
- as.Date(date_start, format = "%Y-%m-%d"),
- as.Date(date_end, format = "%Y-%m-%d"),
- "day"
- )
- if (sub_hyphen == TRUE) {
- dates_sub_hyphen <- gsub("-", "", as.character(dates_original))
- return(dates_sub_hyphen)
- } else {
- return(dates_original)
- }
- }
diff --git a/R/download_test_support.R b/R/download_test_support.R
deleted file mode 100644
index 2115c165..00000000
--- a/R/download_test_support.R
+++ /dev/null
@@ -1,117 +0,0 @@
-###############################################################################
-# Functions used to implement and simplify unit tests on data download
-# functions
-# Date created: 2023-11-30
-# Date modified: 2024-01-21
-###############################################################################
-
-#' Check if sample of download URLs have HTTP Status 200
-#' @param url Download URL to be checked.
-#' @param method httr method to obtain URL (`"HEAD"`` or `"GET"`)
-#' @author Insang Song; Mitchell Manware
-#' @importFrom httr HEAD
-#' @importFrom httr GET
-#' @return logical object
-#' @export
-check_url_status <- function(
- url,
- method = c("HEAD", "GET")
-) {
- method <- match.arg(method)
- http_status_ok <- 200
- if (method == "HEAD") {
- hd <- httr::HEAD(url)
- } else if (method == "GET") {
- hd <- httr::GET(url)
- }
-
- status <- hd$status_code
- Sys.sleep(0.5)
- return(status == http_status_ok)
-}
-
-#' Read commands and convert to character vector
-#' @param commands_path file path with wget/curl commands
-#' @return character vector containing download commands
-#' @importFrom utils read.csv
-#' @export
-read_commands <- function(
- commands_path = commands_path) {
- commands <- utils::read.csv(commands_path, header = FALSE)
- commands <- commands[seq_len(nrow(commands)), ]
- return(commands)
-}
-
-#' Extract URLs from download commands
-#' @param commands character vector containing download commands
-#' @param position URL position in the vector
-#' @importFrom stringr str_split_i
-#' @return character vector containing download URLs
-#' @export
-extract_urls <- function(
- commands = commands,
- position = NULL) {
- if (is.null(position)) {
- cat(paste0("URL position in command is not defined.\n"))
- return(NULL)
- }
- url_list <- NULL
- for (c in seq_along(commands)) {
- url <- stringr::str_split_i(commands[c], " ", position)
- url_list <- c(url_list, url)
- }
- return(url_list)
-}
-
-#' Sample download URLs and apply `check_url_status` function
-#' @param urls character vector of URLs
-#' @param size number of observations to be sampled from \code{urls}
-#' @param method httr method to obtain URL (`"HEAD"` or `"GET"`)
-#' @return logical vector for URL status = 200
-#' @export
-check_urls <- function(
- urls = urls,
- size = NULL,
- method = c("HEAD", "GET")) {
- if (is.null(size)) {
- cat(paste0("URL sample size is not defined.\n"))
- return(NULL)
- }
- if (length(urls) < size) {
- size <- length(urls)
- }
- method <- match.arg(method)
-
- url_sample <- sample(urls, size, replace = FALSE)
- url_status <- sapply(url_sample,
- check_url_status,
- method = method
- )
- return(url_status)
-}
-
-#' Apply download function-specific unit tests
-#' @param directory_to_save directory to test saving
-#' @param directory_to_download directory to test download
-#' @param commands_path file path with download commands
-#' @param url_status logical vector for URL status = 200
-#' @importFrom testthat expect_true
-#' @return NULL
-#' @export
-test_download_functions <- function(
- directory_to_download = NULL,
- directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status) {
- # test that directory_to_download exists
- # skip test if directory_to_download is default (NULL)
- if (!(is.null(directory_to_download))) {
- testthat::expect_true(dir.exists(directory_to_download))
- }
- # test that directory_to_save exists
- testthat::expect_true(dir.exists(directory_to_save))
- # test that commands_path exists
- testthat::expect_true(file.exists(commands_path))
- # test that sample of download URLs all have HTTP status 200
- testthat::expect_true(all(url_status))
-}
diff --git a/R/meta_learner.R b/R/meta_learner.R
index 24f96813..57742abd 100644
--- a/R/meta_learner.R
+++ b/R/meta_learner.R
@@ -5,7 +5,6 @@
#' named.
#' @param kfolds integer, index of k-folds for cross-validation. This should be
#' produced with regards to spatial and/or temporal considerations
-#' To make cross-validation indices, see [generate_cv_index]
#' @param y dependent variable
#' @param ... Passed arguments to \link[BART]{wbart}
#' @return meta_fit_obj object of meta learner
diff --git a/input/Rinput/processing_functions/torch_tensor_manipulation.R b/input/Rinput/processing_functions/torch_tensor_manipulation.R
index bf0ea379..861cde79 100644
--- a/input/Rinput/processing_functions/torch_tensor_manipulation.R
+++ b/input/Rinput/processing_functions/torch_tensor_manipulation.R
@@ -155,5 +155,4 @@ luz::setup(model, loss= luz::luz_metric_rmse(), optimizer = torch::optim_adam(mo
luz::evaluate()
luz::fit(model, vart)
-torch::nnf_avg_pool2d(inputs = )
diff --git a/inst/extdata/downloaded_files_metadata.csv b/inst/extdata/downloaded_files_metadata.csv
deleted file mode 100644
index 97b01201..00000000
--- a/inst/extdata/downloaded_files_metadata.csv
+++ /dev/null
@@ -1,48 +0,0 @@
-name,year,folder,filename
-nlcd,2019,./input/nlcd/raw/,nlcd_2019_land_cover_l48_20210604.tif
-nlcd,2021,./input/nlcd/raw/,nlcd_2021_land_cover_l48_20230630.tif
-hms_smoke,,./input/HMS_Smoke/data/,hms_smokeyyyymmdd.shp
-koppen_geiger,,./input/koppen_geiger/raw/,Beck_KG_V1_present_0p0083.tif
-ecoregions,,./input/ecoregions/raw/,us_eco_l3_state_boundaries.shp
-nighttime_lights,,./input/modis/raw/5000/VNP46A2/yyyy/yday/,
-land_aod,,./input/modis/raw/61/MCD19A2/yyyy/yday/,
-surf_refl,,./input/modis/raw/61/MOD09GA/yyyy/yday/,
-lst_emi,,./input/modis/raw/61/MOD11A1/yyyy/yday/,
-vi,,./input/modis/raw/61/MOD13A2/yyyy/yday/,
-chm_tavg,,./input/geos/chm_tavg_1hr_g1440x721_v1/,GEOS-CF.v01.rpl.chm_tavg_1hr_g1440x721_v1.yyyymmdd_HH30z.nc4
-aqc_tavg,,./input/geos/aqc_tavg_1hr_g1440x721_v1/,GEOS-CF.v01.rpl.aqc_tavg_1hr_g1440x721_v1.yyyymmdd_HH30z.nc4
-aqs,,./input/aqs/,daily_88101_yyyy.csv
-air_sfc,,./input/narr/air.sfc/,air.sfc.yyyy.nc
-albedo,,./input/narr/albedo/,albedo.yyyy.nc
-apcp,,./input/narr/apcp/,apcp.yyyy.nc
-dswrf,,./input/narr/dswrf/,dswrf.yyyy.nc
-evap,,./input/narr/evap/,evap.yyyy.nc
-hcdc,,./input/narr/hcdc/,hcdc.yyyy.nc
-hpbl,,./input/narr/hpbl/,hpbl.yyyy.nc
-lcdc,,./input/narr/lcdc/,lcdc.yyyy.nc
-lhtfl,,./input/narr/lhtfl/,lhtfl.yyyy.nc
-mcdc,,./input/narr/mcdc/,mcdc.yyyy.nc
-omega,,./input/narr/omega/,omega.yyyymm.nc
-pr_wtr,,./input/narr/pr_wtr/,pr_wtr.yyyy.nc
-prate,,./input/narr/prate/,prate.yyyy.nc
-pres_sfc,,./input/narr/pres.sfc/,pres.sfc.yyyy.nc
-shtfl,,./input/narr/shtfl/,shtfl.yyyy.nc
-shum,,./input/narr/shum/,shum.yyyymm.nc
-snowc,,./input/narr/snowc/,snowc.yyyy.nc
-soilm,,./input/narr/soilm/,soilm.yyyy.nc
-tcdc,,./input/narr/tcdc/,tcdc.yyyy.nc
-ulwrf_sfc,,./input/narr/ulwrf.sfc/,ulwrf.sfc.yyyy.nc
-uwnd_10m,,./input/narr/uwnd.10m/,uwnd.10m.yyyy.nc
-vis,,./input/narr/vis/,vis.yyyy.nc
-vwnd_10m,,./input/narr/vwnd.10m/,vwnd.10m.yyyy.nc
-weasd,,./input/narr/weasd/,weasd.yyyy.nc
-tri,,./input/tri/,tri_raw_yyyy.csv
-nei_onroad_123,,./input/nei/yyyynei_onroad_byregion/,onroad123.csv
-nei_onroad_4,,./input/nei/yyyynei_onroad_byregion/,onroad_4.csv
-nei_onroad_5,,./input/nei/yyyynei_onroad_byregion/,onroad_5.csv
-nei_onroad_67,,./input/nei/yyyynei_onroad_byregion/,onroad_67.csv
-nei_onroad_8910,,./input/nei/yyyynei_onroad_byregion/,onroad_8910.csv
-modis,,./input/modis/raw/,
-air_2m,,./input/NCEP-NCAR-Reanalysis-1/,air.2m.gauss.2022.nc
-ncep_ncar,,./input/NCEP-NCAR-Reanalysis-1/,tdc.eatm.gauss.2022.nc
-gmted,,,
\ No newline at end of file
diff --git a/man/base_learner_cv_fit.Rd b/man/base_learner_cv_fit.Rd
deleted file mode 100644
index df88a6d8..00000000
--- a/man/base_learner_cv_fit.Rd
+++ /dev/null
@@ -1,32 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/base_learner.R
-\name{base_learner_cv_fit}
-\alias{base_learner_cv_fit}
-\title{Split training-test data for base learner fitting}
-\usage{
-base_learner_cv_fit(learner, ymat, xmat, cv_index, fun, ...)
-}
-\arguments{
-\item{learner}{character(1). Currently one of \code{"randomforest"}, \code{"xgboost"},
-and \code{"cnn"}}
-
-\item{ymat}{data.frame or matrix. Dependent variable.}
-
-\item{xmat}{data.frame or matrix. Independent variables.}
-
-\item{cv_index}{integer. Index per cross-validation method.
-See \code{?generate_cv_index} for details.}
-
-\item{fun}{base_learner_fit_*}
-
-\item{...}{Arguments passed to the argument \code{fun}}
-}
-\value{
-List of 4 with xtrain, ytrain, xtest, and ytest.
-}
-\description{
-Split training-test data for base learner fitting
-}
-\author{
-Insang Song
-}
diff --git a/man/base_learner_cv_outcome.Rd b/man/base_learner_cv_outcome.Rd
deleted file mode 100644
index 0cf01696..00000000
--- a/man/base_learner_cv_outcome.Rd
+++ /dev/null
@@ -1,27 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/base_learner.R
-\name{base_learner_cv_outcome}
-\alias{base_learner_cv_outcome}
-\title{Return cvfold outcome values}
-\usage{
-base_learner_cv_outcome(data, cv_index, dependent_name)
-}
-\arguments{
-\item{data}{stdt. See \code{\link{convert_stobj_to_stdt}}}
-
-\item{cv_index}{integer. Index per cross-validation method.
-See \code{\link{generate_cv_index}}}
-
-\item{dependent_name}{character(1). Name of the dependent variable.}
-}
-\value{
-List length of \code{max(cv_index)}
-with numeric vectors when \code{length(dependent_name) == 1} or
-data.frames \code{length(dependent_name) > 1}
-}
-\description{
-Return cvfold outcome values
-}
-\author{
-Insang Song
-}
diff --git a/man/base_learner_fit.Rd b/man/base_learner_fit.Rd
deleted file mode 100644
index 042a7d8f..00000000
--- a/man/base_learner_fit.Rd
+++ /dev/null
@@ -1,78 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/base_learner.R
-\name{base_learner_fit}
-\alias{base_learner_fit}
-\title{Fit base learner}
-\usage{
-base_learner_fit(
- data,
- learner = c("randomforest", "xgboost", "cnn"),
- dependent_name = "pm2.5",
- independent_name = NULL,
- cv_mode = c("lolo", "loto", "lolto", "random", "lblo", "lbto", "lblto"),
- cv_fold = 5L,
- sp_fold = NULL,
- t_fold = NULL,
- blocks = NULL,
- block_id = NULL,
- return_full_object = FALSE,
- ...
-)
-}
-\arguments{
-\item{data}{stdt. See \code{\link{convert_stobj_to_stdt}}}
-
-\item{learner}{character(1). Currently one of \code{"randomforest"}, \code{"xgboost"},
-and \code{"cnn"}}
-
-\item{dependent_name}{character(1). Name of the dependent variable.}
-
-\item{independent_name}{character(1). Names of independent variables.}
-
-\item{cv_mode}{character(1). One of
-\code{"lolo"} (leave-one-location-out),
-\code{"loto"} (leave-one-time-out),
-\code{"lolto"} (leave-one-location-time-out),
-\code{"lblo"} (leave-block-location-out),
-\code{"lbto"} (leave-block-time-out),
-\code{"lblto"} (leave-block-location-time-out), and
-\code{"random"} (full random selection)}
-
-\item{cv_fold}{integer(1). Number of folds for cross-validation.}
-
-\item{sp_fold}{integer(1). Number of subfolds for spatial blocks.}
-
-\item{t_fold}{integer(1). Number of subfolds for temporal blocks.}
-
-\item{blocks}{integer(2)/sf/SpatVector object.}
-
-\item{block_id}{character(1). The unique identifier of each block.}
-
-\item{return_full_object}{logical(1).
-\code{TRUE} will return the object whose class is dependent on
-the package in use, whereas \code{FALSE} will only return the prediction
-with spatial/temporal information. Full object is potentially helpful for
-debugging.}
-
-\item{...}{Arguments passed to \code{base_learner_fit_*}.}
-}
-\value{
-List of length \code{cv_fold}, \code{sp_fold * t_fold}, or
-\code{prod(blocks)} (when \code{blocks} is a numeric object of length 2) /
-\code{nrow(blocks)} (\code{blocks} is sf or SpatVector).
-}
-\description{
-This function fits a selected base learner, which is supposed
-to be one of random forest, XGBoost, and convolutional neural network,
-using one of spatiotemporal cross-validation approaches.
-}
-\seealso{
-\link{base_learner_cv_fit}
-\link{base_learner_fit_xgboost}
-\link{base_learner_fit_ranger}
-\link{base_learner_fit_cnn}
-\link[xgboost]{xgboost}, \link[ranger]{ranger}
-}
-\author{
-Insang Song
-}
diff --git a/man/base_learner_fit_cnn.Rd b/man/base_learner_fit_cnn.Rd
deleted file mode 100644
index 8f4dd54e..00000000
--- a/man/base_learner_fit_cnn.Rd
+++ /dev/null
@@ -1,27 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/base_learner.R
-\name{base_learner_fit_cnn}
-\alias{base_learner_fit_cnn}
-\title{Fit convolutional neural networks with neural network library}
-\usage{
-base_learner_fit_cnn(ymat, xmat, ...)
-}
-\arguments{
-\item{ymat}{\code{\link[torch:torch_tensor]{torch::torch_tensor}}. Dependent variable.}
-
-\item{xmat}{\code{\link[torch:torch_tensor]{torch::torch_tensor}}. Independent variables.}
-
-\item{...}{Arguments passed to fitting function}
-}
-\value{
-torch-compatible object
-}
-\description{
-This function uses torch as a backend.
-Torch currently supports CPU, CUDA, and Metal (Apple Silicon graphics),
-thus users can leverage available computational assets
-to accelerate the model fitting.
-}
-\author{
-Insang Song
-}
diff --git a/man/base_learner_fit_ranger.Rd b/man/base_learner_fit_ranger.Rd
deleted file mode 100644
index ee1c40ea..00000000
--- a/man/base_learner_fit_ranger.Rd
+++ /dev/null
@@ -1,24 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/base_learner.R
-\name{base_learner_fit_ranger}
-\alias{base_learner_fit_ranger}
-\title{Fit random forests with ranger}
-\usage{
-base_learner_fit_ranger(ymat, xmat, ...)
-}
-\arguments{
-\item{ymat}{data.frame or matrix. Dependent variable.}
-
-\item{xmat}{data.frame or matrix. Independent variables.}
-
-\item{...}{Arguments passed to \code{\link[ranger:ranger]{ranger::ranger}}}
-}
-\value{
-ranger object.
-}
-\description{
-Fit random forests with ranger
-}
-\author{
-Insang Song
-}
diff --git a/man/base_learner_fit_xgboost.Rd b/man/base_learner_fit_xgboost.Rd
deleted file mode 100644
index 0f1745c7..00000000
--- a/man/base_learner_fit_xgboost.Rd
+++ /dev/null
@@ -1,24 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/base_learner.R
-\name{base_learner_fit_xgboost}
-\alias{base_learner_fit_xgboost}
-\title{Fit XGBoost model}
-\usage{
-base_learner_fit_xgboost(ymat, xmat, ...)
-}
-\arguments{
-\item{ymat}{data.frame or matrix. Dependent variable.}
-
-\item{xmat}{data.frame or matrix. Independent variables.}
-
-\item{...}{Arguments passed to \code{\link[xgboost:xgb.train]{xgboost::xgboost}}}
-}
-\value{
-xgboost object
-}
-\description{
-Fit XGBoost model
-}
-\author{
-Insang Song
-}
diff --git a/man/base_learner_prep.Rd b/man/base_learner_prep.Rd
deleted file mode 100644
index 39bd0022..00000000
--- a/man/base_learner_prep.Rd
+++ /dev/null
@@ -1,32 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/base_learner.R
-\name{base_learner_prep}
-\alias{base_learner_prep}
-\title{Data preparation for base learners}
-\usage{
-base_learner_prep(
- learner = c("cnn", "randomforest", "xgboost"),
- data,
- dependent_name = "pm2.5",
- independent_name
-)
-}
-\arguments{
-\item{learner}{character(1). One of \code{"cnn"}, \code{"randomforest"}, \code{"xgboost"}}
-
-\item{data}{stdt. see \code{\link{convert_stobj_to_stdt}}}
-
-\item{dependent_name}{Name of dependent variable. Default is \code{"pm2.5"}}
-
-\item{independent_name}{character. Names of independent variables.}
-}
-\value{
-A list of two matrices (except for cnn) or
-multidimensional arrays (cnn) depending on learners
-}
-\description{
-Data preparation for base learners
-}
-\author{
-Insang Song
-}
diff --git a/man/check_for_null_parameters.Rd b/man/check_for_null_parameters.Rd
deleted file mode 100644
index eb934ca5..00000000
--- a/man/check_for_null_parameters.Rd
+++ /dev/null
@@ -1,15 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download_support.R
-\name{check_for_null_parameters}
-\alias{check_for_null_parameters}
-\title{Check for null arguments}
-\usage{
-check_for_null_parameters(parameters)
-}
-\arguments{
-\item{parameters}{parameters passed to function (called by
-\code{mget(ls())}.)}
-}
-\description{
-Check for null arguments
-}
diff --git a/man/check_temporal_range.Rd b/man/check_temporal_range.Rd
deleted file mode 100644
index f9f665f3..00000000
--- a/man/check_temporal_range.Rd
+++ /dev/null
@@ -1,40 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/check_temporal_range.R
-\name{check_temporal_range}
-\alias{check_temporal_range}
-\title{Check if input data is within user defined temporal range}
-\usage{
-check_temporal_range(
- data = NULL,
- start_range = "2018-01-01",
- end_range = "2022-12-31",
- data_type = NULL,
- timezone = "UTC",
- time_column = "Date"
-)
-}
-\arguments{
-\item{data}{file path to data}
-
-\item{start_range}{beginning of temporal range}
-
-\item{end_range}{end of temporal range}
-
-\item{data_type}{Point, Polygon, or Raster data}
-
-\item{timezone}{character.
-Time zone in three uppercase letters (e.g., UTC, EST, etc.)}
-
-\item{time_column}{character. Name of the column storing
-time information.}
-}
-\value{
-A logical vector reporting whether time observations in data are
-within defined temporal range
-}
-\description{
-Check if input data is within user defined temporal range
-}
-\author{
-Mitchell Manware
-}
diff --git a/man/check_url_status.Rd b/man/check_url_status.Rd
deleted file mode 100644
index 01e054c2..00000000
--- a/man/check_url_status.Rd
+++ /dev/null
@@ -1,22 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download_test_support.R
-\name{check_url_status}
-\alias{check_url_status}
-\title{Check if sample of download URLs have HTTP Status 200}
-\usage{
-check_url_status(url, method = c("HEAD", "GET"))
-}
-\arguments{
-\item{url}{Download URL to be checked.}
-
-\item{method}{httr method to obtain URL (\verb{"HEAD"`` or }"GET"`)}
-}
-\value{
-logical object
-}
-\description{
-Check if sample of download URLs have HTTP Status 200
-}
-\author{
-Insang Song; Mitchell Manware
-}
diff --git a/man/check_urls.Rd b/man/check_urls.Rd
deleted file mode 100644
index 3540f2ac..00000000
--- a/man/check_urls.Rd
+++ /dev/null
@@ -1,21 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download_test_support.R
-\name{check_urls}
-\alias{check_urls}
-\title{Sample download URLs and apply \code{check_url_status} function}
-\usage{
-check_urls(urls = urls, size = NULL, method = c("HEAD", "GET"))
-}
-\arguments{
-\item{urls}{character vector of URLs}
-
-\item{size}{number of observations to be sampled from \code{urls}}
-
-\item{method}{httr method to obtain URL (\code{"HEAD"} or \code{"GET"})}
-}
-\value{
-logical vector for URL status = 200
-}
-\description{
-Sample download URLs and apply \code{check_url_status} function
-}
diff --git a/man/download_aqs_data.Rd b/man/download_aqs_data.Rd
deleted file mode 100644
index 78a32bd9..00000000
--- a/man/download_aqs_data.Rd
+++ /dev/null
@@ -1,73 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download.R
-\name{download_aqs_data}
-\alias{download_aqs_data}
-\title{Download daily data from AQS datamart}
-\usage{
-download_aqs_data(
- parameter_code = 88101,
- year_start = 2018,
- year_end = 2022,
- resolution_temporal = "daily",
- directory_to_download = "./input/aqs/",
- directory_to_save = "./input/aqs/",
- url_aqs_download = "https://aqs.epa.gov/aqsweb/airdata/",
- data_download_acknowledgement = FALSE,
- unzip = TRUE,
- remove_zip = FALSE,
- download = FALSE,
- remove_command = FALSE
-)
-}
-\arguments{
-\item{parameter_code}{integer(1). length of 5.
-EPA pollutant parameter code. For details, please refer to
-\href{https://aqs.epa.gov/aqsweb/documents/codetables/parameters.html}{AQS parameter codes}}
-
-\item{year_start}{integer(1). length of 4.
-Start year for downloading data.}
-
-\item{year_end}{integer(1). length of 4.
-End year for downloading data.}
-
-\item{resolution_temporal}{character(1).
-Name of column containing POC values.
-Currently, no value other than \code{"daily"} works.}
-
-\item{directory_to_download}{character(1).
-Directory to download zip files from AQS data mart.}
-
-\item{directory_to_save}{character(1).
-Directory to decompress zip files.}
-
-\item{url_aqs_download}{character(1).
-URL to the AQS pre-generated datasets.}
-
-\item{data_download_acknowledgement}{logical(1). By setting \code{TRUE} the
-user acknowledge that the data downloaded using this function may be very
-large and use lots of machine storage and memory.}
-
-\item{unzip}{logical(1). Unzip zip files. Default \code{TRUE}.}
-
-\item{remove_zip}{logical(1). Remove zip file from directory_to_download.
-Default \code{FALSE}.}
-
-\item{download}{logical(1). \code{FALSE} will generate a *.txt file
-containing all download commands. By setting \code{TRUE} the function
-will download all of the requested data files.}
-
-\item{remove_command}{logical(1).
-Remove (\code{TRUE}) or keep (\code{FALSE})
-the text file containing download commands. Default is FALSE.}
-}
-\value{
-NULL; Separate comma-separated value (CSV) files of
-monitors and the daily representative values
-will be stored in \code{directory_to_save}.
-}
-\description{
-Download daily data from AQS datamart
-}
-\author{
-Mariana Kassien, Insang Song, Mitchell Manware
-}
diff --git a/man/download_data.Rd b/man/download_data.Rd
deleted file mode 100644
index 23adadde..00000000
--- a/man/download_data.Rd
+++ /dev/null
@@ -1,60 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download.R
-\name{download_data}
-\alias{download_data}
-\title{Download raw data from sources}
-\usage{
-download_data(
- dataset_name = c("aqs", "ecoregion", "geos", "gmted", "koppen", "koppengeiger",
- "merra2", "merra", "narr_monolevel", "modis", "narr_p_levels", "nlcd", "noaa",
- "sedac_groads", "sedac_population", "groads", "population", "plevels", "p_levels",
- "monolevel", "hms", "smoke", "tri", "nei"),
- directory_to_save = NULL,
- data_download_acknowledgement = FALSE,
- ...
-)
-}
-\arguments{
-\item{dataset_name}{character(1). Dataset to download.}
-
-\item{directory_to_save}{character(1). Directory to save / unzip
-(if zip files are downloaded) data.}
-
-\item{data_download_acknowledgement}{logical(1). By setting \code{TRUE} the
-user acknowledge that the data downloaded using this function may be very
-large and use lots of machine storage and memory.}
-
-\item{...}{Arguments passed to each download function.}
-}
-\description{
-Download raw data from sources
-}
-\note{
-\itemize{
-\item All download function names are in \code{download_*_data} formats
-}
-}
-\seealso{
-For details of each download function per dataset,
-Please refer to:
-\itemize{
-\item \link{download_aqs_data}: "aqs", "AQS"
-\item \link{download_ecoregion_data}: "ecoregion"
-\item \link{download_geos_cf_data}: "geos"
-\item \link{download_gmted_data}: "gmted", "GMTED"
-\item \link{download_koppen_geiger_data}: "koppen", "koppengeiger"
-\item \link{download_merra2_data}: "merra2", "merra", "MERRA", "MERRA2"
-\item \link{download_narr_monolevel_data}: "narr_monolevel", "monolevel"
-\item \link{download_narr_p_levels_data}: "narr_p_levels", "p_levels", "plevels"
-\item \link{download_nlcd_data}: "nlcd", "NLCD"
-\item \link{download_noaa_hms_smoke_data}: "noaa", "smoke", "hms"
-\item \link{download_sedac_groads_data}: "sedac_groads", "groads"
-\item \link{download_sedac_population_data}: "sedac_population", "population"
-\item \link{download_modis_data}: "modis", "MODIS"
-\item \link{download_tri_data}: "tri", "TRI"
-\item \link{download_nei_data}: "nei", "NEI"
-}
-}
-\author{
-Insang Song
-}
diff --git a/man/download_ecoregion_data.Rd b/man/download_ecoregion_data.Rd
deleted file mode 100644
index fbacbd5b..00000000
--- a/man/download_ecoregion_data.Rd
+++ /dev/null
@@ -1,65 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download.R
-\name{download_ecoregion_data}
-\alias{download_ecoregion_data}
-\title{Download Ecoregion Shapefiles from EPA}
-\usage{
-download_ecoregion_data(
- directory_to_download = "./input/ecoregions/",
- directory_to_save = "./input/ecoregions/",
- data_download_acknowledgement = FALSE,
- unzip = TRUE,
- remove_zip = FALSE,
- download = FALSE,
- remove_command = TRUE,
- epa_certificate_path = system.file("extdata/cacert_gaftp_epa.pem", package =
- "beethoven")
-)
-}
-\arguments{
-\item{directory_to_download}{character(1). Directory to download zip file
-of Ecoregion level 3 shapefiles}
-
-\item{directory_to_save}{character(1). Directory to decompress zip files.}
-
-\item{data_download_acknowledgement}{logical(1). By setting \code{TRUE} the
-user acknowledge that the data downloaded using this function may be very
-large and use lots of machine storage and memory.}
-
-\item{unzip}{logical(1). Unzip zip files. Default \code{TRUE}.}
-
-\item{remove_zip}{logical(1). Remove zip file from directory_to_download.
-Default \code{FALSE}.}
-
-\item{download}{logical(1). \code{FALSE} will generate a *.txt file
-containing all download commands. By setting \code{TRUE} the function
-will download all of the requested data files.}
-
-\item{remove_command}{logical(1).
-Remove (\code{TRUE}) or keep (\code{FALSE})
-the text file containing download commands.}
-
-\item{epa_certificate_path}{character(1). Path to the certificate file
-for EPA DataCommons. Default is
-'extdata/cacert_gaftp_epa.pem' under the package installation path.}
-}
-\value{
-NULL;
-}
-\description{
-The \code{download_ecoregion_data()} function accesses and downloads
-Ecoregions level 3 data, where all pieces of information in the higher
-levels are included.
-}
-\note{
-In Linux systems as of December 2023, downloading Ecoregion data from
-EPA Data Commons will result in certificate errors. This is bypassed by
-manually identifying .crt file link in your browser by connecting to
-https://gaftp.epa.gov then clicking a lock icon in the address bar.
-(TO DEVELOPERS: see the comments in source code)
-Currently we bundle the pre-downloaded crt and its PEM (which is accepted
-in wget command) file in ./inst/extdata.
-}
-\author{
-Insang Song
-}
diff --git a/man/download_geos_cf_data.Rd b/man/download_geos_cf_data.Rd
deleted file mode 100644
index 41313cfd..00000000
--- a/man/download_geos_cf_data.Rd
+++ /dev/null
@@ -1,51 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download.R
-\name{download_geos_cf_data}
-\alias{download_geos_cf_data}
-\title{Download atmospheric composition data from the NASA Global Earth Observing System (GEOS) model.}
-\usage{
-download_geos_cf_data(
- date_start = "2023-09-01",
- date_end = "2023-09-01",
- collection = c("aqc_tavg_1hr_g1440x721_v1", "chm_tavg_1hr_g1440x721_v1",
- "met_tavg_1hr_g1440x721_x1", "xgc_tavg_1hr_g1440x721_x1",
- "chm_inst_1hr_g1440x721_p23", "met_inst_1hr_g1440x721_p23"),
- directory_to_save = "./input/data/geos_cf/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- remove_command = FALSE
-)
-}
-\arguments{
-\item{date_start}{character(1). length of 10. Start date for downloading
-data. Format YYYY-MM-DD (ex. September 1, 2023 = \code{"2023-09-01"}).}
-
-\item{date_end}{character(1). length of 10. End date for downloading data.
-Format YYYY-MM-DD (ex. September 1, 2023 = \code{"2023-09-01"}).}
-
-\item{collection}{character(1). GEOS-CF data collection file name.}
-
-\item{directory_to_save}{character(1). Directory to save data.}
-
-\item{data_download_acknowledgement}{logical(1). By setting \code{TRUE} the
-user acknowledge that the data downloaded using this function may be very
-large and use lots of machine storage and memory.}
-
-\item{download}{logical(1). \code{FALSE} will generate a *.txt file
-containing all download commands. By setting \code{TRUE} the function
-will download all of the requested data files.}
-
-\item{remove_command}{logical(1).
-Remove (\code{TRUE}) or keep (\code{FALSE})
-the text file containing download commands.}
-}
-\value{
-NULL;
-}
-\description{
-The \code{download_goes_cf_data()} function accesses and downloads various
-atmospheric composition collections from the \href{https://gmao.gsfc.nasa.gov/GEOS_systems/}{NASA Global Earth Observing System (GEOS) model}.
-}
-\author{
-Mitchell Manware, Insang Song
-}
diff --git a/man/download_gmted_data.Rd b/man/download_gmted_data.Rd
deleted file mode 100644
index 5cb60bc4..00000000
--- a/man/download_gmted_data.Rd
+++ /dev/null
@@ -1,60 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download.R
-\name{download_gmted_data}
-\alias{download_gmted_data}
-\title{Download global elevation data from the Global Multi-resolution Terrain Elevation Data (GMTED2010).}
-\usage{
-download_gmted_data(
- statistic = c("Breakline Emphasis", "Systematic Subsample", "Median Statistic",
- "Minimum Statistic", "Mean Statistic", "Maximum Statistic",
- "Standard Deviation Statistic"),
- resolution = c("7.5 arc-seconds", "15 arc-seconds", "30 arc-seconds"),
- directory_to_download = "./input/gmted/",
- directory_to_save = "./input/gmted/",
- data_download_acknowledgement = FALSE,
- unzip = TRUE,
- remove_zip = FALSE,
- download = FALSE,
- remove_command = FALSE
-)
-}
-\arguments{
-\item{statistic}{character(1). Available statistics include \code{"Breakline Emphasis"}, \code{"Systematic Subsample"}, \code{"Median Statistic"},
-\code{"Minimum Statistic"}, \code{"Mean Statistic"}, \code{"Maximum Statistic"}, and
-\code{"Standard Deviation Statistic"}.}
-
-\item{resolution}{character(1). Available resolutions include \code{"7.5 arc-seconds"}, \code{"15 arc-seconds"}, and \code{"30 arc-seconds"}.}
-
-\item{directory_to_download}{character(1). Directory to download zip files
-from Global Multi-resolution Terrain Elevation Data (GMTED2010).}
-
-\item{directory_to_save}{character(1). Directory to decompress zip files.}
-
-\item{data_download_acknowledgement}{logical(1). By setting \code{TRUE} the
-user acknowledge that the data downloaded using this function may be very
-large and use lots of machine storage and memory.}
-
-\item{unzip}{logical(1). Unzip zip files. Default is \code{TRUE}.}
-
-\item{remove_zip}{logical(1). Remove zip file from directory_to_download.
-Default is \code{FALSE}.}
-
-\item{download}{logical(1). \code{FALSE} will generate a *.txt file
-containing all download commands. By setting \code{TRUE} the function
-will download all of the requested data files.}
-
-\item{remove_command}{logical(1).
-Remove (\code{TRUE}) or keep (\code{FALSE})
-the text file containing download commands. Default is FALSE.}
-}
-\value{
-NULL;
-}
-\description{
-The \code{download_gmted_data()} function acesses and downloads Global
-Multi-resolution Terrain Elevation Data (GMTED2010) from
-\href{https://www.usgs.gov/coastal-changes-and-impacts/gmted2010}{U.S. Geological Survey and National Geospatial-Intelligence Agency}.
-}
-\author{
-Mitchell Manware, Insang Song
-}
diff --git a/man/download_koppen_geiger_data.Rd b/man/download_koppen_geiger_data.Rd
deleted file mode 100644
index c145c2f2..00000000
--- a/man/download_koppen_geiger_data.Rd
+++ /dev/null
@@ -1,62 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download.R
-\name{download_koppen_geiger_data}
-\alias{download_koppen_geiger_data}
-\title{Download climate classification data from the present and future Köppen-Geiger climate classification maps.}
-\usage{
-download_koppen_geiger_data(
- time_period = c("Present", "Future"),
- data_resolution = c("0.0083", "0.083", "0.5"),
- directory_to_download = "./input/koppen_geiger/",
- directory_to_save = "./input/koppen_geiger/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- unzip = TRUE,
- remove_zip = FALSE,
- remove_command = FALSE
-)
-}
-\arguments{
-\item{time_period}{character(1). Available times are \code{"Present"} (1980-2016)
-and \code{"Future"} (2071-2100). ("Future" classifications are based on scenario
-RCP8.5).}
-
-\item{data_resolution}{character(1). Available resolutions are \code{"0.0083"}
-degrees (approx. 1 km), \code{"0.083"} degrees (approx. 10 km), and
-\code{"0.5"} degrees (approx. 50 km).}
-
-\item{directory_to_download}{character(1). Directory to download zip files
-from Present and future Köppen-Geiger climate classification maps at 1-km
-resolution.}
-
-\item{directory_to_save}{character(1). Directory to decompress zip files.}
-
-\item{data_download_acknowledgement}{logical(1). By setting \code{TRUE} the
-user acknowledge that the data downloaded using this function may be very
-large and use lots of machine storage and memory.}
-
-\item{download}{logical(1). \code{FALSE} will generate a *.txt file
-containing all download commands. By setting \code{TRUE} the function
-will download all of the requested data files.}
-
-\item{unzip}{logical(1). Unzip zip files. Default is \code{TRUE}.}
-
-\item{remove_zip}{logical(1). Remove zip files from directory_to_download.
-Default is \code{FALSE}.}
-
-\item{remove_command}{logical(1).
-Remove (\code{TRUE}) or keep (\code{FALSE})
-the text file containing download commands.}
-}
-\value{
-NULL;
-}
-\description{
-The \code{download_koppen_geiger_data()} function accesses and downloads
-climate classification data from the Present and future
-Köppen-Geiger climate classification maps at
-1-km resolution (\href{https://www.nature.com/articles/sdata2018214}{link for article}; \href{https://figshare.com/articles/dataset/Present_and_future_K_ppen-Geiger_climate_classification_maps_at_1-km_resolution/6396959/2}{link for data}).
-}
-\author{
-Mitchell Manware, Insang Song
-}
diff --git a/man/download_merra2_data.Rd b/man/download_merra2_data.Rd
deleted file mode 100644
index e028b3df..00000000
--- a/man/download_merra2_data.Rd
+++ /dev/null
@@ -1,59 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download.R
-\name{download_merra2_data}
-\alias{download_merra2_data}
-\title{Download meteorological and atmospheric data from the Modern-Era Retrospective analysis for Research and Applications, Version 2 (MERRA-2) model.}
-\usage{
-download_merra2_data(
- date_start = "2023-09-01",
- date_end = "2023-09-01",
- collection = c("inst1_2d_asm_Nx", "inst1_2d_int_Nx", "inst1_2d_lfo_Nx",
- "inst3_3d_asm_Np", "inst3_3d_aer_Nv", "inst3_3d_asm_Nv", "inst3_3d_chm_Nv",
- "inst3_3d_gas_Nv", "inst3_2d_gas_Nx", "inst6_3d_ana_Np", "inst6_3d_ana_Nv",
- "statD_2d_slv_Nx", "tavg1_2d_adg_Nx", "tavg1_2d_aer_Nx", "tavg1_2d_chm_Nx",
- "tavg1_2d_csp_Nx", "tavg1_2d_flx_Nx", "tavg1_2d_int_Nx", "tavg1_2d_lfo_Nx",
- "tavg1_2d_lnd_Nx", "tavg1_2d_ocn_Nx", "tavg1_2d_rad_Nx", "tavg1_2d_slv_Nx",
- "tavg3_3d_mst_Ne", "tavg3_3d_trb_Ne", "tavg3_3d_nav_Ne", "tavg3_3d_cld_Np",
-
- "tavg3_3d_mst_Np", "tavg3_3d_rad_Np", "tavg3_3d_tdt_Np", "tavg3_3d_trb_Np",
- "tavg3_3d_udt_Np", "tavg3_3d_odt_Np", "tavg3_3d_qdt_Np", "tavg3_3d_asm_Nv",
- "tavg3_3d_cld_Nv", "tavg3_3d_mst_Nv", "tavg3_3d_rad_Nv", "tavg3_2d_glc_Nx"),
- directory_to_save = "./input/merra2/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- remove_command = FALSE
-)
-}
-\arguments{
-\item{date_start}{character(1). length of 10. Start date for downloading
-data. Format YYYY-MM-DD (ex. September 1, 2023 is \code{"2023-09-01"}).}
-
-\item{date_end}{character(1). length of 10. End date for downloading data.
-Format YYYY-MM-DD (ex. September 1, 2023 is \code{"2023-09-01"}).}
-
-\item{collection}{character(1). MERRA-2 data collection file name.}
-
-\item{directory_to_save}{character(1). Directory to save data.}
-
-\item{data_download_acknowledgement}{logical(1). By setting \code{TRUE} the
-user acknowledge that the data downloaded using this function may be very
-large and use lots of machine storage and memory.}
-
-\item{download}{logical(1). \code{FALSE} will generate a *.txt file
-containing all download commands. By setting \code{TRUE} the function
-will download all of the requested data files.}
-
-\item{remove_command}{logical(1).
-Remove (\code{TRUE}) or keep (\code{FALSE})
-the text file containing download commands.}
-}
-\value{
-NULL;
-}
-\description{
-The \code{download_merra2_data()} function accesses and downloads various
-meteorological and atmospheric collections from the \href{https://gmao.gsfc.nasa.gov/reanalysis/MERRA-2/}{Modern-Era Retrospective analysis for Research and Applications, Version 2 (MERRA-2)}.
-}
-\author{
-Mitchell Manware, Insang Song
-}
diff --git a/man/download_modis_data.Rd b/man/download_modis_data.Rd
deleted file mode 100644
index 4cf6368e..00000000
--- a/man/download_modis_data.Rd
+++ /dev/null
@@ -1,78 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download.R
-\name{download_modis_data}
-\alias{download_modis_data}
-\title{Download MODIS product files}
-\usage{
-download_modis_data(
- date_start = "2023-09-01",
- date_end = "2023-09-01",
- product = c("MOD09GA", "MOD11A1", "MOD06_L2", "MCD19A2", "MOD13A2", "VNP46A2"),
- version = "61",
- horizontal_tiles = c(7, 13),
- vertical_tiles = c(3, 6),
- nasa_earth_data_token = NULL,
- directory_to_save = "./input/modis/raw/",
- data_download_acknowledgement = FALSE,
- mod06_links = NULL,
- download = TRUE,
- remove_command = FALSE
-)
-}
-\arguments{
-\item{date_start}{character(1). length of 10. Start date for downloading
-data. Format YYYY-MM-DD (ex. September 1, 2023 is \code{"2023-09-01"}).}
-
-\item{date_end}{character(1). length of 10. End date for downloading data.
-Format YYYY-MM-DD (ex. September 1, 2023 is \code{"2023-09-01"}).}
-
-\item{product}{character(1).
-One of \code{c("MOD09GA", "MOD11A1", "MOD06_L2", "MCD19A2", "MOD13A2", "VNP46A2")}}
-
-\item{version}{character(1). Default is \code{"61"}, meaning v061.}
-
-\item{horizontal_tiles}{integer(2). Horizontal tile numbers
-\code{c({start}, {end})}. Default is \code{c(7, 13)}.}
-
-\item{vertical_tiles}{integer(2). Vertical tile numbers
-\code{c({start}, {end})}. Default is \code{c(3, 6)}.}
-
-\item{nasa_earth_data_token}{character(1).
-Token for downloading data from NASA. Should be set before
-trying running the function.}
-
-\item{directory_to_save}{character(1). Directory to save data.}
-
-\item{data_download_acknowledgement}{logical(1). By setting \code{TRUE} the
-user acknowledge that the data downloaded using this function may be very
-large and use lots of machine storage and memory.}
-
-\item{mod06_links}{character(1). CSV file path to MOD06_L2 download links
-from NASA LPDAAC. Default is \code{NULL}.}
-
-\item{download}{logical(1). Download data or only save wget commands.}
-
-\item{remove_command}{logical(1). Remove (\code{TRUE}) or keep (\code{FALSE})
-the text file containing download commands.}
-}
-\value{
-NULL;
-}
-\description{
-Need maintenance for the directory path change
-in NASA EOSDIS. This function first retrieves the all hdf download links
-on a certain day, then only selects the relevant tiles from the retrieved
-links. Download is only done at the queried horizontal-vertical tile number
-combinations. An exception is MOD06_L2 product, which is produced
-every five minutes every day.
-}
-\note{
-\code{date_start} and \code{date_end} should be in the same year.
-Directory structure looks like
-input/modis/raw/\{version\}/\{product\}/\{year\}/\{day_of_year\}
-Please note that \code{date_start} and \code{date_end} are
-ignored if \code{product == 'MOD06_L2'}.
-}
-\author{
-Mitchell Manware, Insang Song
-}
diff --git a/man/download_narr_monolevel_data.Rd b/man/download_narr_monolevel_data.Rd
deleted file mode 100644
index ed7a2545..00000000
--- a/man/download_narr_monolevel_data.Rd
+++ /dev/null
@@ -1,50 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download.R
-\name{download_narr_monolevel_data}
-\alias{download_narr_monolevel_data}
-\title{download_narr_monolevel_data: download monolevel meteorological data from NOAA NCEP North American Regional Reanalysis (NARR) model.}
-\usage{
-download_narr_monolevel_data(
- year_start = 2022,
- year_end = 2022,
- variables = NULL,
- directory_to_save = "./input/narr/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- remove_command = FALSE
-)
-}
-\arguments{
-\item{year_start}{integer(1). length of 4. Start of year range for
-downloading data.}
-
-\item{year_end}{integer(1). length of 4. End of year range for downloading
-data.}
-
-\item{variables}{character. Variable(s) name acronym.}
-
-\item{directory_to_save}{character(1). Directory(s) to save downloaded data
-files.}
-
-\item{data_download_acknowledgement}{logical(1). By setting \code{TRUE} the
-user acknowledge that the data downloaded using this function may be very
-large and use lots of machine storage and memory.}
-
-\item{download}{logical(1). \code{FALSE} will generate a *.txt file
-containing all download commands. By setting \code{TRUE} the function
-will download all of the requested data files.}
-
-\item{remove_command}{logical(1).
-Remove (\code{TRUE}) or keep (\code{FALSE})
-the text file containing download commands.}
-}
-\value{
-NULL;
-}
-\description{
-The \code{download_narr_monolevel_data} function accesses and downloads
-monolevel meteorological data from \href{https://psl.noaa.gov/data/gridded/data.narr.html}{NOAA NCEP North American Regional Reanalysis (NARR)}.
-}
-\author{
-Mitchell Manware, Insang Song
-}
diff --git a/man/download_narr_p_levels_data.Rd b/man/download_narr_p_levels_data.Rd
deleted file mode 100644
index f96f2893..00000000
--- a/man/download_narr_p_levels_data.Rd
+++ /dev/null
@@ -1,50 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download.R
-\name{download_narr_p_levels_data}
-\alias{download_narr_p_levels_data}
-\title{Download pressure level meteorological data from NOAA NCEP North American Regional Reanalysis (NARR) model.}
-\usage{
-download_narr_p_levels_data(
- year_start = 2022,
- year_end = 2022,
- variables = NULL,
- directory_to_save = "./input/narr/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- remove_command = FALSE
-)
-}
-\arguments{
-\item{year_start}{integer(1). length of 4. Start of year range for
-downloading data.}
-
-\item{year_end}{integer(1). length of 4. End of year range for downloading
-data.}
-
-\item{variables}{character(1). Variable(s) name acronym.}
-
-\item{directory_to_save}{character(1). Directory(s) to save downloaded data
-files.}
-
-\item{data_download_acknowledgement}{logical(1). By setting \code{TRUE} the
-user acknowledge that the data downloaded using this function may be very
-large and use lots of machine storage and memory.}
-
-\item{download}{logical(1). \code{FALSE} will generate a *.txt file
-containing all download commands. By setting \code{TRUE} the function
-will download all of the requested data files.}
-
-\item{remove_command}{logical(1).
-Remove (\code{TRUE}) or keep (\code{FALSE})
-the text file containing download commands.}
-}
-\value{
-NULL;
-}
-\description{
-The \code{download_narr_p_levels_data} function accesses and downloads
-pressure level meteorological data from \href{https://psl.noaa.gov/data/gridded/data.narr.html}{NOAA NCEP North American Regional Reanalysis (NARR)}.
-}
-\author{
-Mitchell Manware, Insang Song
-}
diff --git a/man/download_nei_data.Rd b/man/download_nei_data.Rd
deleted file mode 100644
index b1d02482..00000000
--- a/man/download_nei_data.Rd
+++ /dev/null
@@ -1,51 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download.R
-\name{download_nei_data}
-\alias{download_nei_data}
-\title{Download data from EPA National Emission Inventory aggregated on-road emission data}
-\usage{
-download_nei_data(
- directory_to_save = "./input/nei/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- remove_command = FALSE,
- year_target = c(2017L, 2020L),
- unzip = FALSE,
- epa_certificate_path = system.file("extdata/cacert_gaftp_epa.pem", package =
- "beethoven")
-)
-}
-\arguments{
-\item{directory_to_save}{character(1). Directory to download files.}
-
-\item{data_download_acknowledgement}{logical(1). By setting \code{TRUE} the
-user acknowledge that the data downloaded using this function may be very
-large and use lots of machine storage and memory.}
-
-\item{download}{logical(1). \code{FALSE} will generate a *.txt file
-containing all download commands. By setting \code{TRUE} the function
-will download all of the requested data files.}
-
-\item{remove_command}{logical(1).
-Remove (\code{TRUE}) or keep (\code{FALSE})
-the text file containing download commands.}
-
-\item{year_target}{Available years of NEI data.
-Default is \code{c(2017L, 2020L)}.}
-
-\item{unzip}{logical(1). Unzip the downloaded zip files.
-Default is \code{FALSE}.}
-
-\item{epa_certificate_path}{character(1). Path to the certificate file
-for EPA DataCommons. Default is
-'extdata/cacert_gaftp_epa.pem' under the package installation path.}
-}
-\value{
-NULL; Two comma-separated value (CSV) raw files for 2017 and 2020
-}
-\description{
-Download data from EPA National Emission Inventory aggregated on-road emission data
-}
-\author{
-Ranadeep Daw, Insang Song
-}
diff --git a/man/download_nlcd_data.Rd b/man/download_nlcd_data.Rd
deleted file mode 100644
index 7714d025..00000000
--- a/man/download_nlcd_data.Rd
+++ /dev/null
@@ -1,59 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download.R
-\name{download_nlcd_data}
-\alias{download_nlcd_data}
-\title{Download land cover data from the National Land Cover Database Science Research Products.}
-\usage{
-download_nlcd_data(
- year = 2021,
- collection = "Coterminous United States",
- directory_to_download = "./input/nlcd/",
- directory_to_save = "./input/nlcd/",
- data_download_acknowledgement = FALSE,
- unzip = TRUE,
- remove_zip = FALSE,
- download = FALSE,
- remove_command = FALSE
-)
-}
-\arguments{
-\item{year}{integer(1). Available years for Coterminous United States
-include \code{2001}, \code{2004}, \code{2006}, \code{2008}, \code{2011}, \code{2013}, \code{2016},
-\code{2019}, and \code{2021}.
-Available years for Alaska include \code{2001}, \code{2011}, and \code{2016}.}
-
-\item{collection}{character(1). \code{"Coterminous United States"} or \code{"Alaska"}.}
-
-\item{directory_to_download}{character(1). Directory to download zip files
-from National Land Cover Database Science Research Products.}
-
-\item{directory_to_save}{character(1). Directory to decompress zip files.}
-
-\item{data_download_acknowledgement}{logical(1). By setting \code{TRUE} the
-user acknowledge that the data downloaded using this function may be very
-large and use lots of machine storage and memory.}
-
-\item{unzip}{logical(1). Unzip zip files. Default is \code{TRUE}.}
-
-\item{remove_zip}{logical(1). Remove zip files from directory_to_download.
-Default is \code{FALSE}.}
-
-\item{download}{logical(1). \code{FALSE} will generate a *.txt file
-containing all download commands. By setting \code{TRUE} the function
-will download all of the requested data files.}
-
-\item{remove_command}{logical(1).
-Remove (\code{TRUE}) or keep (\code{FALSE})
-the text file containing download commands.}
-}
-\value{
-NULL;
-}
-\description{
-The \code{download_nlcd_data()} function accesses and downloads
-land cover data from the
-\href{https://www.mrlc.gov/data}{NLCD Science Research Products} data base.
-}
-\author{
-Mitchell Manware, Insang Song
-}
diff --git a/man/download_noaa_hms_smoke_data.Rd b/man/download_noaa_hms_smoke_data.Rd
deleted file mode 100644
index 673f3daf..00000000
--- a/man/download_noaa_hms_smoke_data.Rd
+++ /dev/null
@@ -1,67 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download.R
-\name{download_noaa_hms_smoke_data}
-\alias{download_noaa_hms_smoke_data}
-\title{Download daily wildfire smoke plume data from NOAA Hazard Mapping System Fire and Smoke Product}
-\usage{
-download_noaa_hms_smoke_data(
- date_start = "2023-09-01",
- date_end = "2023-09-01",
- data_format = "Shapefile",
- directory_to_download = "./input/noaa_hms/",
- directory_to_save = "./input/noaa_hms/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- remove_command = FALSE,
- unzip = TRUE,
- remove_zip = FALSE
-)
-}
-\arguments{
-\item{date_start}{character(1). length of 10. Start date for downloading
-data. Format YYYY-MM-DD (ex. September 1, 2023 is \code{"2023-09-01"}).}
-
-\item{date_end}{character(1). length of 10. End date for downloading data.
-Format YYYY-MM-DD (ex. September 10, 2023 is \code{"2023-09-10"}).}
-
-\item{data_format}{character(1). "Shapefile" or "KML".}
-
-\item{directory_to_download}{character(1). Directory to download zip files
-from NOAA Hazard Mapping System Fire and Smoke Product. (Ignored if
-\code{data_format = "KML"}.)}
-
-\item{directory_to_save}{character(1). Directory to save unzipped shapefiles
-and KML files.}
-
-\item{data_download_acknowledgement}{logical(1).
-By setting \code{TRUE} the
-user acknowledge that the data downloaded using this function may be very
-large and use lots of machine storage and memory.}
-
-\item{download}{logical(1). \code{FALSE} will generate a *.txt file
-containing all download commands. By setting \code{TRUE} the function
-will download all of the requested data files.}
-
-\item{remove_command}{logical(1).
-Remove (\code{TRUE}) or keep (\code{FALSE})
-the text file containing download commands.}
-
-\item{unzip}{logical(1). Unzip zip files. Default is \code{TRUE}. (Ignored
-if \code{data_format = "KML"}.)}
-
-\item{remove_zip}{logical(1). Remove zip files from
-directory_to_download. Default is \code{FALSE}.
-(Ignored if \code{data_format = "KML"}.)}
-}
-\value{
-NULL;
-}
-\description{
-The \code{download_noaa_hms_smoke_data()} function accesses and downloads
-wildfire smoke plume coverage data from
-the National Oceanic and Atmospheric Administration's (NOAA)
-\href{https://www.ospo.noaa.gov/Products/land/hms.html#0}{Hazard Mapping System Fire and Smoke Product}.
-}
-\author{
-Mitchell Manware, Insang Song
-}
diff --git a/man/download_permit.Rd b/man/download_permit.Rd
deleted file mode 100644
index d8edffd0..00000000
--- a/man/download_permit.Rd
+++ /dev/null
@@ -1,15 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download_support.R
-\name{download_permit}
-\alias{download_permit}
-\title{Check for data download acknowledgement}
-\usage{
-download_permit(data_download_acknowledgement)
-}
-\arguments{
-\item{data_download_acknowledgement}{logical(1). Whether to
-start downloading}
-}
-\description{
-Check for data download acknowledgement
-}
diff --git a/man/download_remove_command.Rd b/man/download_remove_command.Rd
deleted file mode 100644
index 2eeb4121..00000000
--- a/man/download_remove_command.Rd
+++ /dev/null
@@ -1,17 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download_support.R
-\name{download_remove_command}
-\alias{download_remove_command}
-\title{Remove or keep wget command file}
-\usage{
-download_remove_command(commands_txt = NULL, remove = FALSE)
-}
-\arguments{
-\item{commands_txt}{character(1). Path of download commands}
-
-\item{remove}{logical(1). Remove (\code{TRUE}) or
-keep (\code{FALSE}) commands}
-}
-\description{
-Remove or keep wget command file
-}
diff --git a/man/download_remove_zips.Rd b/man/download_remove_zips.Rd
deleted file mode 100644
index a8937cd5..00000000
--- a/man/download_remove_zips.Rd
+++ /dev/null
@@ -1,16 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download_support.R
-\name{download_remove_zips}
-\alias{download_remove_zips}
-\title{Remove downloaded zip files}
-\usage{
-download_remove_zips(remove = FALSE, download_name)
-}
-\arguments{
-\item{remove}{logical(1). Confirm removal. Default is FALSE.}
-
-\item{download_name}{character. Full zip file path}
-}
-\description{
-Remove downloaded zip files
-}
diff --git a/man/download_run.Rd b/man/download_run.Rd
deleted file mode 100644
index cf26c431..00000000
--- a/man/download_run.Rd
+++ /dev/null
@@ -1,20 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download_support.R
-\name{download_run}
-\alias{download_run}
-\title{download_run: execute or skip \code{system_command}
-in data download function.}
-\usage{
-download_run(download = FALSE, system_command = NULL)
-}
-\arguments{
-\item{download}{logical(1). Execute (\code{TRUE}) or
-skip (\code{FALSE}) download.}
-
-\item{system_command}{character(1). Linux command to execute downloads.
-Inherited from data download function.}
-}
-\description{
-Execute or skip the commands listed in the ...wget/curl_commands.txt file
-produced by one of the data download functions.
-}
diff --git a/man/download_sanitize_path.Rd b/man/download_sanitize_path.Rd
deleted file mode 100644
index 251d2365..00000000
--- a/man/download_sanitize_path.Rd
+++ /dev/null
@@ -1,17 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download_support.R
-\name{download_sanitize_path}
-\alias{download_sanitize_path}
-\title{Sanitize path to end with a forward slash}
-\usage{
-download_sanitize_path(directory)
-}
-\arguments{
-\item{directory}{character(1). Path}
-}
-\value{
-character ending with a forward slash.
-}
-\description{
-Sanitize path to end with a forward slash
-}
diff --git a/man/download_sedac_groads_data.Rd b/man/download_sedac_groads_data.Rd
deleted file mode 100644
index b8553f92..00000000
--- a/man/download_sedac_groads_data.Rd
+++ /dev/null
@@ -1,59 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download.R
-\name{download_sedac_groads_data}
-\alias{download_sedac_groads_data}
-\title{Download Global Roads Open Access Data Set (gROADS), v1 (1980-2010) data from NASA Socioeconomic Data and Applications Center (SEDAC).}
-\usage{
-download_sedac_groads_data(
- data_format = c("Shapefile", "Geodatabase"),
- data_region = c("Americas", "Global", "Africa", "Asia", "Europe", "Oceania East",
- "Oceania West"),
- directory_to_download = "./input/sedac_groads/",
- directory_to_save = "./input/sedac_groads/",
- data_download_acknowledgement = FALSE,
- unzip = TRUE,
- remove_zip = FALSE,
- download = FALSE,
- remove_command = FALSE
-)
-}
-\arguments{
-\item{data_format}{character(1). Data can be downloaded as \code{"Shapefile"} or
-\code{"Geodatabase"}. (Only \code{"Geodatabase"} available for \code{"Global"} region).}
-
-\item{data_region}{character(1). Data can be downloaded for \code{"Global"},
-\code{"Africa"}, \code{"Asia"}, \code{"Europe"}, \code{"Americas"}, \code{"Oceania East"}, and \code{"Oceania West"}.}
-
-\item{directory_to_download}{character(1). Directory to download zip files
-from NASA Global Roads Open Access Data Set.}
-
-\item{directory_to_save}{character(1). Directory to decompress zip files.}
-
-\item{data_download_acknowledgement}{logical(1). By setting \code{TRUE} the
-user acknowledge that the data downloaded using this function may be very
-large and use lots of machine storage and memory.}
-
-\item{unzip}{logical(1). Unzip zip files. Default is \code{TRUE}.}
-
-\item{remove_zip}{logical(1). Remove zip files from directory_to_download.
-Default is \code{FALSE}.}
-
-\item{download}{logical(1). \code{FALSE} will generate a *.txt file
-containing all download commands. By setting \code{TRUE} the function
-will download all of the requested data files.}
-
-\item{remove_command}{logical(1).
-Remove (\code{TRUE}) or keep (\code{FALSE})
-the text file containing download commands.}
-}
-\value{
-NULL;
-}
-\description{
-The \code{download_sedac_groads_data()} function accesses and downloads
-roads data from the National Aeronautics and Space
-Administration's (NASA) \href{https://sedac.ciesin.columbia.edu/data/set/groads-global-roads-open-access-v1/data-download}{Global Roads Open Access Data Set}.
-}
-\author{
-Mitchell Manware, Insang Song
-}
diff --git a/man/download_sedac_population_data.Rd b/man/download_sedac_population_data.Rd
deleted file mode 100644
index fd1dcb74..00000000
--- a/man/download_sedac_population_data.Rd
+++ /dev/null
@@ -1,63 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download.R
-\name{download_sedac_population_data}
-\alias{download_sedac_population_data}
-\title{Download UN WPP-Adjusted population density data from NASA Socioeconomic Data and Applications Center (SEDAC)}
-\usage{
-download_sedac_population_data(
- year = "2020",
- data_format = c("GeoTIFF", "ASCII", "netCDF"),
- data_resolution = "60 minute",
- directory_to_download = "./input/sedac_population/",
- directory_to_save = "./input/sedac_population/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- unzip = TRUE,
- remove_zip = FALSE,
- remove_command = FALSE
-)
-}
-\arguments{
-\item{year}{character(1). Available years are \code{2000}, \code{2005}, \code{2010}, \code{2015}, and
-\code{2020}, or \code{"all"} for all years.}
-
-\item{data_format}{character(1). Individual year data can be downloaded as
-\code{"ASCII"} or \code{"GeoTIFF"}. "all" years is downloaded as \code{"netCDF"}.}
-
-\item{data_resolution}{character(1). Available resolutions are 30 second
-(approx. 1 km), 2.5 minute (approx. 5 km), 15 minute (approx. 30 km),
-30 minute (approx. 55 km), and 60 minute (approx. 110 km).}
-
-\item{directory_to_download}{character(1). Directory to download zip files
-from NASA UN WPP-Adjusted Population Density, v4.11.}
-
-\item{directory_to_save}{character(1). Directory to decompress zip files.}
-
-\item{data_download_acknowledgement}{logical(1). By setting \code{TRUE} the
-user acknowledge that the data downloaded using this function may be very
-large and use lots of machine storage and memory.}
-
-\item{download}{logical(1). \code{FALSE} will generate a *.txt file
-containing all download commands. By setting \code{TRUE} the function
-will download all of the requested data files.}
-
-\item{unzip}{logical(1). Unzip zip files. Default is \code{TRUE}.}
-
-\item{remove_zip}{logical(1). Remove zip files from directory_to_download.
-Default is \code{FALSE}.}
-
-\item{remove_command}{logical(1).
-Remove (\code{TRUE}) or keep (\code{FALSE})
-the text file containing download commands.}
-}
-\value{
-NULL;
-}
-\description{
-The \code{download_sedac_population_data()} function accesses and downloads
-population density data from the National Aeronatuics and Space
-Administration's (NASA) \href{https://sedac.ciesin.columbia.edu/data/set/gpw-v4-population-density-adjusted-to-2015-unwpp-country-totals-rev11}{UN WPP-Adjusted Population Density, v4.11}.
-}
-\author{
-Mitchell Manware, Insang Song
-}
diff --git a/man/download_setup_dir.Rd b/man/download_setup_dir.Rd
deleted file mode 100644
index a6f7b35e..00000000
--- a/man/download_setup_dir.Rd
+++ /dev/null
@@ -1,15 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download_support.R
-\name{download_setup_dir}
-\alias{download_setup_dir}
-\title{Check if input directory exists}
-\usage{
-download_setup_dir(directory)
-}
-\arguments{
-\item{directory}{character(1) directory path}
-}
-\description{
-If directory does not exist, the directory
-will be created.
-}
diff --git a/man/download_sink.Rd b/man/download_sink.Rd
deleted file mode 100644
index a40b0ed8..00000000
--- a/man/download_sink.Rd
+++ /dev/null
@@ -1,14 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download_support.R
-\name{download_sink}
-\alias{download_sink}
-\title{Start sink download commands into a text file}
-\usage{
-download_sink(command_txt)
-}
-\arguments{
-\item{command_txt}{character(1). file path to export commands.}
-}
-\description{
-Start sink download commands into a text file
-}
diff --git a/man/download_tri_data.Rd b/man/download_tri_data.Rd
deleted file mode 100644
index 5a613aa4..00000000
--- a/man/download_tri_data.Rd
+++ /dev/null
@@ -1,42 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download.R
-\name{download_tri_data}
-\alias{download_tri_data}
-\title{Download data from EPA toxic release inventory}
-\usage{
-download_tri_data(
- year_start = 2018L,
- year_end = 2022L,
- directory_to_save = "./input/tri/",
- data_download_acknowledgement = FALSE,
- download = FALSE,
- remove_command = FALSE
-)
-}
-\arguments{
-\item{year_start}{integer(1). length of 4. Start year for downloading data.}
-
-\item{year_end}{integer(1). length of 4. End year for downloading data.}
-
-\item{directory_to_save}{character(1). Directory to download files.}
-
-\item{data_download_acknowledgement}{logical(1). By setting \code{TRUE} the
-user acknowledge that the data downloaded using this function may be very
-large and use lots of machine storage and memory.}
-
-\item{download}{logical(1). \code{FALSE} will generate a *.txt file
-containing all download commands. By setting \code{TRUE} the function
-will download all of the requested data files.}
-
-\item{remove_command}{logical(1). Remove (\code{TRUE}) or keep (\code{FALSE})
-the text file containing download commands.}
-}
-\value{
-NULL; Yearly comma-separated value (CSV) raw files for each year
-}
-\description{
-Download data from EPA toxic release inventory
-}
-\author{
-Mariana Kassien, Insang Song
-}
diff --git a/man/download_unzip.Rd b/man/download_unzip.Rd
deleted file mode 100644
index 9ced54ee..00000000
--- a/man/download_unzip.Rd
+++ /dev/null
@@ -1,19 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download_support.R
-\name{download_unzip}
-\alias{download_unzip}
-\title{Unzip downloaded data}
-\usage{
-download_unzip(file_name, directory_to_unzip, unzip = TRUE)
-}
-\arguments{
-\item{file_name}{character(1). Full zip file path}
-
-\item{directory_to_unzip}{character(1). Directory to unzip
-data}
-
-\item{unzip}{logical(1). Unzip (\code{TRUE}) or not.}
-}
-\description{
-Unzip downloaded data
-}
diff --git a/man/extract_urls.Rd b/man/extract_urls.Rd
deleted file mode 100644
index 6310a43a..00000000
--- a/man/extract_urls.Rd
+++ /dev/null
@@ -1,19 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download_test_support.R
-\name{extract_urls}
-\alias{extract_urls}
-\title{Extract URLs from download commands}
-\usage{
-extract_urls(commands = commands, position = NULL)
-}
-\arguments{
-\item{commands}{character vector containing download commands}
-
-\item{position}{URL position in the vector}
-}
-\value{
-character vector containing download URLs
-}
-\description{
-Extract URLs from download commands
-}
diff --git a/man/generate_block_sp_index.Rd b/man/generate_block_sp_index.Rd
deleted file mode 100644
index 2397e036..00000000
--- a/man/generate_block_sp_index.Rd
+++ /dev/null
@@ -1,33 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/cross_validation.R
-\name{generate_block_sp_index}
-\alias{generate_block_sp_index}
-\title{Generate blocked spatial index}
-\usage{
-generate_block_sp_index(covars, cv_fold = NULL, blocks = NULL, block_id = NULL)
-}
-\arguments{
-\item{covars}{stdt.}
-
-\item{cv_fold}{integer(1). Number of folds for cross-validation.}
-
-\item{blocks}{numeric(2)/sf/SpatVector configuration of blocks.}
-
-\item{block_id}{character(1). The unique identifier of each block.}
-}
-\value{
-stdt
-}
-\description{
-Generate blocked spatial index
-}
-\details{
-"Block" in this function refers to a group of contiguous spatial/temporal entities.
-Regardless of \code{mode} value (should be one of \code{"spatial"} or \code{"spatiotemporal"}),
-the index column will be named "sp_index". \code{block_id} should be set with a proper field name
-when blocks is a sf or SpatVector object. When \code{cv_fold} is an integer, then the coordinates stored in \code{stdt} object are clustered with k-means. In this case, the result object will include attributes (accessible with \code{attr} function) about
-"kmeans_centers" (coordinates of centers) and "kmeans_sizes" (size of each cluster)
-}
-\author{
-Insang Song
-}
diff --git a/man/generate_cv_index.Rd b/man/generate_cv_index.Rd
deleted file mode 100644
index 862250b2..00000000
--- a/man/generate_cv_index.Rd
+++ /dev/null
@@ -1,65 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/cross_validation.R
-\name{generate_cv_index}
-\alias{generate_cv_index}
-\title{Generate spatio-temporal cross-validation index}
-\usage{
-generate_cv_index(
- covars,
- cv_mode = c("lolo", "loto", "lolto", "random", "lblo", "lbto", "lblto"),
- cv_fold = 5L,
- sp_fold = NULL,
- t_fold = NULL,
- blocks = NULL,
- block_id = NULL
-)
-}
-\arguments{
-\item{covars}{stdt. See \code{\link{convert_stobj_to_stdt}}
-for details.}
-
-\item{cv_mode}{character(1). One of \code{"lolo"} (leave-one-location-out),
-\code{"loto"} (leave-one-time-out),
-\code{"lolto"} (leave-one-location-time-out),
-\code{"lblo"} (leave-block-location-out),
-\code{"lbto"} (leave-block-time-out),
-\code{"lblto"} (leave-block-location-time-out)
-\code{"random"} (full random selection)}
-
-\item{cv_fold}{integer(1). Number of folds for cross-validation.}
-
-\item{sp_fold}{integer(1). Number of subfolds for spatial blocks.}
-
-\item{t_fold}{integer(1). Number of subfolds for temporal blocks.}
-
-\item{blocks}{integer(2)/sf/SpatVector object.}
-
-\item{block_id}{character(1). The unique identifier of each block.}
-}
-\value{
-A numeric vector with length of the number of input rows
-}
-\description{
-Generate spatio-temporal cross-validation index
-}
-\details{
-\code{blocks} is NULL as default
-(i.e., no block structure is considered); then "lb*" cv_mode
-is not working.
-\code{blocks} should be one of integer vector (length 2),
-sf, sftime, or SpatVector object.
-Please be advised that we cannot provide error messages
-related to the \code{blocks} type invalidity
-due to the cyclometric complexity.
-If any valid object is assigned here,
-cv_fold will not make sense and be ignored.
-Definition of unit grid (horizontal, vertical) size,
-arbitrary shape of blocks (sf/SpatVector case)
-where users can consider common spatial hierarchies such as
-states, counties, hydrological unit code areas, etc.
-\code{lblto} mode will accept \code{sp_fold} and \code{t_fold}.
-The maximum of results in this case is \code{sp_fold * t_fold}.
-}
-\author{
-Insang Song
-}
diff --git a/man/generate_cv_index_lblo.Rd b/man/generate_cv_index_lblo.Rd
deleted file mode 100644
index 4cc1b194..00000000
--- a/man/generate_cv_index_lblo.Rd
+++ /dev/null
@@ -1,26 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/cross_validation.R
-\name{generate_cv_index_lblo}
-\alias{generate_cv_index_lblo}
-\title{Generate spatio-temporal cross-validation index (leave-block-location-out)}
-\usage{
-generate_cv_index_lblo(covars, cv_fold = NULL, blocks = NULL, block_id = NULL)
-}
-\arguments{
-\item{covars}{stdt. See \code{\link{convert_stobj_to_stdt}} for details.}
-
-\item{cv_fold}{integer(1). Number of folds for cross-validation.}
-
-\item{blocks}{integer(2)/sf/SpatVector object.}
-
-\item{block_id}{character(1). The unique identifier of each block.}
-}
-\value{
-An integer vector.
-}
-\description{
-Generate spatio-temporal cross-validation index (leave-block-location-out)
-}
-\author{
-Insang Song
-}
diff --git a/man/generate_cv_index_lblto.Rd b/man/generate_cv_index_lblto.Rd
deleted file mode 100644
index 087b5501..00000000
--- a/man/generate_cv_index_lblto.Rd
+++ /dev/null
@@ -1,31 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/cross_validation.R
-\name{generate_cv_index_lblto}
-\alias{generate_cv_index_lblto}
-\title{Generate spatio-temporal cross-validation index (leave-block-location-time-out)}
-\usage{
-generate_cv_index_lblto(covars, sp_fold, t_fold, blocks, block_id = NULL)
-}
-\arguments{
-\item{covars}{stdt. See \code{\link{convert_stobj_to_stdt}} for details.}
-
-\item{sp_fold}{integer(1). Number of subfolds for spatial blocks.}
-
-\item{t_fold}{integer(1). Number of subfolds for temporal blocks.}
-
-\item{blocks}{integer(2)/sf/SpatVector object.}
-
-\item{block_id}{character(1). The unique identifier of each block.}
-}
-\value{
-An integer vector.
-}
-\description{
-Generate spatio-temporal cross-validation index (leave-block-location-time-out)
-}
-\details{
-The maximum of results is \code{sp_fold * t_fold}.
-}
-\author{
-Insang Song
-}
diff --git a/man/generate_cv_index_lbto.Rd b/man/generate_cv_index_lbto.Rd
deleted file mode 100644
index 8274a24e..00000000
--- a/man/generate_cv_index_lbto.Rd
+++ /dev/null
@@ -1,22 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/cross_validation.R
-\name{generate_cv_index_lbto}
-\alias{generate_cv_index_lbto}
-\title{Generate spatio-temporal cross-validation index (leave-block-time-out)}
-\usage{
-generate_cv_index_lbto(covars, cv_fold = NULL)
-}
-\arguments{
-\item{covars}{stdt. See \code{\link{convert_stobj_to_stdt}} for details.}
-
-\item{cv_fold}{integer(1). Number of folds for cross-validation.}
-}
-\value{
-An integer vector.
-}
-\description{
-Generate spatio-temporal cross-validation index (leave-block-time-out)
-}
-\author{
-Insang Song
-}
diff --git a/man/generate_cv_index_lolo.Rd b/man/generate_cv_index_lolo.Rd
deleted file mode 100644
index 2c61f4d6..00000000
--- a/man/generate_cv_index_lolo.Rd
+++ /dev/null
@@ -1,20 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/cross_validation.R
-\name{generate_cv_index_lolo}
-\alias{generate_cv_index_lolo}
-\title{Generate spatio-temporal cross-validation index (leave-one-location-out)}
-\usage{
-generate_cv_index_lolo(covars)
-}
-\arguments{
-\item{covars}{stdt. See \code{\link{convert_stobj_to_stdt}} for details.}
-}
-\value{
-An integer vector.
-}
-\description{
-Generate spatio-temporal cross-validation index (leave-one-location-out)
-}
-\author{
-Insang Song
-}
diff --git a/man/generate_cv_index_lolto.Rd b/man/generate_cv_index_lolto.Rd
deleted file mode 100644
index c83fd1e4..00000000
--- a/man/generate_cv_index_lolto.Rd
+++ /dev/null
@@ -1,20 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/cross_validation.R
-\name{generate_cv_index_lolto}
-\alias{generate_cv_index_lolto}
-\title{Generate spatio-temporal cross-validation index leave-one-location-time-out)}
-\usage{
-generate_cv_index_lolto(covars)
-}
-\arguments{
-\item{covars}{stdt. See \code{\link{convert_stobj_to_stdt}} for details.}
-}
-\value{
-An integer vector.
-}
-\description{
-Generate spatio-temporal cross-validation index leave-one-location-time-out)
-}
-\author{
-Insang Song
-}
diff --git a/man/generate_cv_index_loto.Rd b/man/generate_cv_index_loto.Rd
deleted file mode 100644
index f8a538d1..00000000
--- a/man/generate_cv_index_loto.Rd
+++ /dev/null
@@ -1,20 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/cross_validation.R
-\name{generate_cv_index_loto}
-\alias{generate_cv_index_loto}
-\title{Generate spatio-temporal cross-validation index (leave-one-time-out)}
-\usage{
-generate_cv_index_loto(covars)
-}
-\arguments{
-\item{covars}{stdt. See \code{\link{convert_stobj_to_stdt}} for details.}
-}
-\value{
-An integer vector.
-}
-\description{
-Generate spatio-temporal cross-validation index (leave-one-time-out)
-}
-\author{
-Insang Song
-}
diff --git a/man/generate_cv_index_random.Rd b/man/generate_cv_index_random.Rd
deleted file mode 100644
index 1ad28b60..00000000
--- a/man/generate_cv_index_random.Rd
+++ /dev/null
@@ -1,22 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/cross_validation.R
-\name{generate_cv_index_random}
-\alias{generate_cv_index_random}
-\title{Generate spatio-temporal cross-validation index (random)}
-\usage{
-generate_cv_index_random(covars, cv_fold = NULL)
-}
-\arguments{
-\item{covars}{stdt. See \code{\link{convert_stobj_to_stdt}} for details.}
-
-\item{cv_fold}{integer(1). Number of folds for cross-validation.}
-}
-\value{
-An integer vector with unique values of \code{seq(1, cv_fold)}
-}
-\description{
-Generate spatio-temporal cross-validation index (random)
-}
-\author{
-Insang Song
-}
diff --git a/man/generate_date_sequence.Rd b/man/generate_date_sequence.Rd
deleted file mode 100644
index 3fd60139..00000000
--- a/man/generate_date_sequence.Rd
+++ /dev/null
@@ -1,23 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download_support.R
-\name{generate_date_sequence}
-\alias{generate_date_sequence}
-\title{Generate sequence of dates based on \code{date_start} and \code{date_end}.}
-\usage{
-generate_date_sequence(date_start, date_end, sub_hyphen = TRUE)
-}
-\arguments{
-\item{date_start}{character(1). Beginning of date sequence.}
-
-\item{date_end}{character(1). End of date sequence.}
-
-\item{sub_hyphen}{logical(1). Substitute hyphen in dates. If \code{TRUE}, returns
-date sequence as "YYYYMMDD". If \code{FALSE}, returns date sequence as
-"YYYY-MM-DD".}
-}
-\value{
-vector
-}
-\description{
-Generate sequence of dates based on \code{date_start} and \code{date_end}.
-}
diff --git a/man/generate_spt_index.Rd b/man/generate_spt_index.Rd
deleted file mode 100644
index 5c56db83..00000000
--- a/man/generate_spt_index.Rd
+++ /dev/null
@@ -1,25 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/cross_validation.R
-\name{generate_spt_index}
-\alias{generate_spt_index}
-\title{Generate unique spatiotemporal identifier from stdt object}
-\usage{
-generate_spt_index(covars, mode = c("spatial", "spatiotemporal"))
-}
-\arguments{
-\item{covars}{stdt. See \link{convert_stobj_to_stdt}}
-
-\item{mode}{One of \code{"spatial"} or \code{"spatiotemporal"}}
-}
-\value{
-stdt with a new index named "sp_index"
-}
-\description{
-It generates unique spatiotemporal identifier in the
-input stdt object. Regardless of mode values
-(should be one of \code{"spatial"} or \code{"spatiotemporal"}),
-the index column will be named "sp_index".
-}
-\author{
-Insang Song
-}
diff --git a/man/meta_learner_fit.Rd b/man/meta_learner_fit.Rd
index 6da4291c..0e33a6ea 100644
--- a/man/meta_learner_fit.Rd
+++ b/man/meta_learner_fit.Rd
@@ -14,8 +14,7 @@ named.
}}
\item{kfolds}{integer, index of k-folds for cross-validation. This should be
-produced with regards to spatial and/or temporal considerations
-To make cross-validation indices, see \link{generate_cv_index}}
+produced with regards to spatial and/or temporal considerations}
\item{y}{dependent variable}
diff --git a/man/read_commands.Rd b/man/read_commands.Rd
deleted file mode 100644
index 52dca047..00000000
--- a/man/read_commands.Rd
+++ /dev/null
@@ -1,17 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download_test_support.R
-\name{read_commands}
-\alias{read_commands}
-\title{Read commands and convert to character vector}
-\usage{
-read_commands(commands_path = commands_path)
-}
-\arguments{
-\item{commands_path}{file path with wget/curl commands}
-}
-\value{
-character vector containing download commands
-}
-\description{
-Read commands and convert to character vector
-}
diff --git a/man/test_download_functions.Rd b/man/test_download_functions.Rd
deleted file mode 100644
index 602c9ff6..00000000
--- a/man/test_download_functions.Rd
+++ /dev/null
@@ -1,25 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/download_test_support.R
-\name{test_download_functions}
-\alias{test_download_functions}
-\title{Apply download function-specific unit tests}
-\usage{
-test_download_functions(
- directory_to_download = NULL,
- directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status
-)
-}
-\arguments{
-\item{directory_to_download}{directory to test download}
-
-\item{directory_to_save}{directory to test saving}
-
-\item{commands_path}{file path with download commands}
-
-\item{url_status}{logical vector for URL status = 200}
-}
-\description{
-Apply download function-specific unit tests
-}
diff --git a/tests/testdata/create_testdata.R b/tests/testdata/create_testdata.R
deleted file mode 100644
index 772fa178..00000000
--- a/tests/testdata/create_testdata.R
+++ /dev/null
@@ -1,413 +0,0 @@
-#' Spatial subset on the testdata area:
-#' Durham, Wake, and Orange counties in North Carolina, US
-#' @author Eva Marques
-#' @param sp SpatRaster or SpatVector from terra package
-#' @return an object similar to sp but cropped on the testdata area
-sp_subset <- function(sp) {
- nc_file <- system.file("shape/nc.shp", package = "sf")
- nc <- terra::vect(nc_file)
- ext <- nc[nc$NAME %in% c("Durham", "Wake", "Orange")]
- ext_proj <- terra::project(ext, terra::crs(sp))
- if (class(sp)[[1]] == "SpatRaster") {
- crop_sp <- terra::crop(sp, ext_proj, mask = TRUE, snap = "out")
- } else {
- crop_sp <- terra::crop(sp, ext_proj)
- }
- return(crop_sp)
-}
-
-#' Creates the testdata directory if it does not already exists.
-#' @author Eva Marques
-#' @param dir_path a character path to the directory.
-create_dir <- function(dir_path) {
- if (!file.exists(dir_path)) {
- dir.create(dir_path)
- }
-}
-
-
-#' The data paths are stored in a .csv file but they sometimes contain encoded
-#' dates with characters 'yday', yyyy', 'mm', 'dd'. This function replace these
-#' character strings by the good value contained in date.
-#' @author Eva Marques
-#' @param string a character (eg: filepath)
-#' @param date a Date
-#' @return the same string with encoded characters replaced by date info.
-replace_dateinfo <- function(string, date) {
- output <- gsub("yyyy", lubridate::year(date), string) |>
- gsub(
- pattern = "yday",
- replacement = stringr::str_pad(lubridate::yday(date),
- 3,
- pad = "0"
- )
- ) |>
- gsub(
- pattern = "mm",
- replacement = stringr::str_pad(lubridate::month(date),
- 2,
- pad = "0"
- )
- ) |>
- gsub(
- pattern = "dd",
- replacement = stringr::str_pad(lubridate::day(date),
- 2,
- pad = "0"
- )
- )
- return(output)
-}
-
-#' Create raw testdata files for National Land Cover Dataset
-#' (for both years 2019, 2021)
-#' @author Eva Marques
-#' @param dict_path character path to the .csv file storing all data paths
-#' @param testdata_path character path to the folder where testdata should
-#' be stored
-testdata_nlcd <- function(
- dict_path = "../../inst/extdata/downloaded_files_metadata.csv",
- testdata_path = "raw/nlcd/") {
- name <- folder <- filename <- NULL
- create_dir(testdata_path)
- dict <- data.table::fread(dict_path, sep = ",")
- fpath <- paste0(
- dict[name == "nlcd", folder],
- dict[name == "nlcd", filename]
- )
- r <- terra::rast(fpath)
- r_samp <- sp_subset(r)
- testdata_file <- paste0(
- testdata_path,
- dict[name == "nlcd", filename]
- )
- terra::writeRaster(r_samp, testdata_file, overwrite = TRUE)
-}
-
-#' Create raw testdata files for ecoregions
-#' @author Eva Marques
-#' @param dict_path character path to the .csv file storing all data paths
-#' @param testdata_path character path to the folder where testdata should
-#' be stored
-testdata_ecoregions <- function(
- dict_path = "../../inst/extdata/downloaded_files_metadata.csv",
- testdata_path = "raw/ecoregions/") {
- name <- folder <- filename <- NULL
- create_dir(testdata_path)
- dict <- data.table::fread(dict_path, sep = ",")
- fpath <- paste0(
- dict[name == "ecoregions", folder],
- dict[name == "ecoregions", filename]
- )
- r <- terra::vect(fpath)
- r_samp <- sp_subset(r)
- testdata_file <- paste0(
- testdata_path,
- dict[name == "ecoregions", filename]
- )
- terra::writeVector(r_samp, testdata_file, overwrite = TRUE)
-}
-
-
-#' Create raw testdata files for Koppen-Geiger climate zone binary variables
-#' @author Eva Marques
-#' @param dict_path character path to the .csv file storing all data paths
-#' @param testdata_path character path to the folder where testdata should
-#' be stored
-testdata_kg <- function(
- dict_path = "../../inst/extdata/downloaded_files_metadata.csv",
- testdata_path = "raw/koppen_geiger/") {
- name <- folder <- filename <- NULL
- create_dir(testdata_path)
- dict <- data.table::fread(dict_path, sep = ",")
- fpath <- paste0(
- dict[name == "koppen_geiger", folder],
- dict[name == "koppen_geiger", filename]
- )
- r <- terra::rast(fpath)
- r_samp <- sp_subset(r)
- testdata_file <- paste0(
- testdata_path,
- dict[name == "koppen_geiger", filename]
- )
- terra::writeRaster(r_samp, testdata_file, overwrite = TRUE)
-}
-
-#' Create raw testdata file for NARR variables stored as monthly rasters
-#' @author Eva Marques
-#' @param dict_path character path to the .csv file storing all data paths
-#' @param start_date Date of when testdata starts
-#' @param end_date Date of when testdata ends
-#' @param testdata_path character path to the folder where testdata should
-#' be stored
-#' @param var character of the variable name
-testdata_narr <- function(
- dict_path = "../../inst/extdata/downloaded_files_metadata.csv",
- start_date = as.Date("2022-01-01"),
- end_date = as.Date("2022-01-02"),
- testdata_path = "raw/narr/",
- var) {
- name <- folder <- filename <- NULL
- create_dir(testdata_path)
- dict <- data.table::fread(dict_path, sep = ",")
- # files are stored per year
- # we consider that start_date and end_date have the same year
- fpath <- paste0(
- dict[name == var, folder],
- dict[name == var, filename]
- ) |>
- replace_dateinfo(start_date)
- period <- seq(lubridate::yday(start_date), lubridate::yday(end_date), 1)
- r_var <- terra::rast(fpath)[[period]]
- r_var_samp <- sp_subset(r_var)
- # same here
- new_fpath <- paste0(
- testdata_path,
- dict[name == var, filename]
- ) |>
- replace_dateinfo(start_date)
- terra::writeCDF(r_var_samp,
- new_fpath,
- varname = names(r_var_samp),
- unit = terra::units(r_var_samp),
- split = TRUE,
- overwrite = TRUE
- )
- cat("✓ ", new_fpath, "\n")
-}
-
-#' Create raw testdata file for NARR variables stored as monthly rasters
-#' with pressure levels (omega and shum variables)
-#' @author Eva Marques
-#' @param dict_path character path to the .csv file storing all data paths
-#' @param start_date Date of when testdata starts
-#' @param end_date Date of when testdata ends
-#' @param testdata_path character path to the folder where testdata should
-#' be stored
-#' @param var character of the variable name ("omega" or "shum")
-testdata_narr_lev <- function(
- dict_path = "../../inst/extdata/downloaded_files_metadata.csv",
- start_date = as.Date("2022-01-01"),
- end_date = as.Date("2022-01-02"),
- testdata_path = "raw/narr/",
- var) {
- name <- folder <- filename <- NULL
- create_dir(testdata_path)
- dict <- data.table::fread(dict_path, sep = ",")
- # files are stored per month
- # we consider that start_date and end_date have the same month
- fpath <- paste0(
- dict[name == var, folder],
- dict[name == var, filename]
- ) |>
- replace_dateinfo(start_date)
- period <- seq(start_date, end_date, 1)
- r <- terra::rast(fpath)
- r_var <- r[[terra::time(r) %in% period]]
- r_var_samp <- sp_subset(r_var)
- new_fpath <- paste0(
- testdata_path,
- dict[name == var, filename]
- ) |>
- replace_dateinfo(start_date)
- terra::writeCDF(r_var_samp,
- new_fpath,
- varname = names(r_var_samp),
- unit = terra::units(r_var_samp),
- split = TRUE,
- overwrite = TRUE
- )
- cat("✓ ", new_fpath, "\n")
-}
-
-#' Create raw testdata file for GEOS-CF variables
-#' Note1: raw downloaded files have .nc4 extension, which leads to a warning
-#' when testdata .nc4 files are created.
-#' Note2: an error can occur when creating ncfile. Relaunching the code solves
-#' the problem for now...
-#' @author Eva Marques, Mitchell Manware, Insang Song
-#' @param dict_path character path to the .csv file storing all data paths
-#' @param start_date Date of when testdata starts
-#' @param end_date Date of when testdata ends
-#' @param testdata_path character path to the folder where testdata should
-#' be stored
-#' @param collection character of the data collection ("aqc" or "chm")
-testdata_geos <- function(
- dict_path = "../../inst/extdata/downloaded_files_metadata.csv",
- start_date = as.Date("2022-01-01"),
- end_date = as.Date("2022-01-02"),
- testdata_path,
- collection) {
- name <- folder <- filename <- NULL
- create_dir(testdata_path)
- period <- seq(from = start_date, to = end_date, by = "1 day")
- dict <- data.table::fread(dict_path, sep = ",")
- for (p in period) {
- p <- as.Date(p)
- for (h in seq(0, 23, by = 1)) {
- new_fpath <- paste0(
- testdata_path,
- dict[name == collection, filename]
- ) |>
- replace_dateinfo(p) |>
- gsub(pattern = "HH", replacement = stringr::str_pad(h, 2, pad = "0"))
- if (!file.exists(new_fpath)) {
- fpath <- paste0(
- dict[name == collection, folder],
- dict[name == collection, filename]
- ) |>
- replace_dateinfo(p) |>
- gsub(pattern = "HH", replacement = stringr::str_pad(h, 2, pad = "0"))
- r <- terra::rast(fpath)
- terra::crs(r) <- "EPSG:4326"
- r_samp <- sp_subset(r)
- terra::writeCDF(r_samp,
- new_fpath,
- varname = names(r_samp),
- split = TRUE,
- unit = terra::units(r_samp),
- overwrite = TRUE
- )
- cat("\n✓ ", new_fpath)
- }
- }
- }
-}
-
-#' Create raw testdata files for TRI
-#' @author Eva Marques
-#' @param dict_path character path to the .csv file storing all data paths
-#' @param year numeric giving the year
-#' @param testdata_path character path to the folder where testdata should
-#' be stored
-testdata_tri <- function(
- dict_path = "../../inst/extdata/downloaded_files_metadata.csv",
- year = 2022,
- testdata_path = "raw/tri/") {
- name <- folder <- filename <- NULL
- create_dir(testdata_path)
- dict <- data.table::fread(dict_path, sep = ",")
- tri_path <- paste0(
- dict[name == "tri", folder],
- dict[name == "tri", filename]
- ) |>
- replace_dateinfo(date = as.Date(paste0(year, "-01-01")))
- tri_samp <- data.table::fread(tri_path) |>
- subset(get("1. YEAR") == year &
- get("7. COUNTY") %in% c("DURHAM", "WAKE", "ORANGE"))
- new_fpath <- paste0(
- testdata_path,
- dict[name == "tri", filename]
- ) |>
- replace_dateinfo(date = as.Date(paste0(year, "-01-01")))
- data.table::fwrite(
- tri_samp,
- new_fpath
- )
-}
-
-#' Create raw testdata files for AQS
-#' (start_date and end_date are supposed to be from the same year)
-#' @author Eva Marques
-#' @param dict_path character path to the .csv file storing all data paths
-#' @param testdata_path character path to the folder where testdata should
-#' be stored
-testdata_aqs <- function(
- dict_path = "../../inst/extdata/downloaded_files_metadata.csv",
- start_date = as.Date("2022-01-01"),
- end_date = as.Date("2022-01-02"),
- testdata_path = "raw/aqs/") {
- name <- folder <- filename <- NULL
- create_dir(testdata_path)
- dict <- data.table::fread(dict_path, sep = ",")
- fpath <- paste0(
- dict[name == "aqs", folder],
- dict[name == "aqs", filename]
- ) |>
- replace_dateinfo(date = start_date)
- aqs <- data.table::fread(fpath) |>
- subset(get("Date Local") >= start_date & get("Date Local") <= end_date)
- aqs_vect <- terra::vect(aqs,
- geom = c("Longitude", "Latitude"),
- crs = "EPSG:4326"
- )
- aqs_samp <- sp_subset(aqs_vect) |>
- data.table::as.data.table()
- new_fpath <- paste0(
- testdata_path,
- dict[name == "aqs", filename]
- ) |>
- replace_dateinfo(start_date)
- data.table::fwrite(
- aqs_samp,
- new_fpath
- )
-}
-
-
-#' Create raw testdata files for NEI
-#' @author Eva Marques
-#' @param dict_path character path to the .csv file storing all data paths
-#' @param testdata_path character path to the folder where testdata should
-#' be stored
-testdata_nei <- function(
- dict_path = "../../inst/extdata/downloaded_files_metadata.csv",
- testdata_path = "raw/NEI/") {
- name <- folder <- filename <- NULL
- create_dir(testdata_path)
- dict <- data.table::fread(dict_path, sep = ",")
- onroads <- c("4", "5", "67", "123", "8910")
- for (onr in onroads) {
- nei_path <- paste0(
- dict[name == paste0("nei_onroad_", onr), folder],
- dict[name == paste0("nei_onroad_", onr), filename]
- ) |>
- replace_dateinfo(date = "2020-01-01")
- nei_samp <- data.table::fread(nei_path) |>
- subset(get("county") %in% c("Durham", "Wake", "Orange"))
-
- new_fpath <- paste0(
- testdata_path,
- dict[name == paste0("nei_onroad_", onr), filename]
- ) |>
- replace_dateinfo("2020-01-01")
- data.table::fwrite(
- nei_samp,
- new_fpath
- )
- }
-}
-
-
-
-#' Create raw testdata files for HMS smoke shapefiles
-#' Important note: the extracted dates are not the same than the rest of
-#' testdata in favor of more interesting smoke polygons above
-#' Wake-Durham-Orange area.
-#' @author Eva Marques
-#' @param dict_path character path to the .csv file storing all data paths
-#' @param testdata_path character path to the folder where testdata should
-#' be stored
-testdata_hms_smoke <- function(
- dict_path = "../../inst/extdata/downloaded_files_metadata.csv",
- testdata_path = "raw/hms_smoke/") {
- name <- folder <- filename <- NULL
- create_dir(testdata_path)
- dict <- data.table::fread(dict_path, sep = ",")
- period <- c("2022-06-18", "2022-06-21")
- for (p in period) {
- fpath <- paste0(
- dict[name == "hms_smoke", folder],
- dict[name == "hms_smoke", filename]
- ) |>
- replace_dateinfo(date = as.Date(p))
- r <- terra::vect(fpath)
- r_samp <- sp_subset(r)
- new_fpath <- paste0(
- testdata_path,
- dict[name == "hms_smoke", filename]
- ) |>
- replace_dateinfo(as.Date(p))
- terra::writeVector(r_samp, new_fpath, overwrite = TRUE)
- }
-}
diff --git a/tests/testdata/raw/nlcd/nlcd_2019_land_cover_l48_20210604.tif b/tests/testdata/raw/nlcd/nlcd_2019_land_cover_l48_20210604.tif
deleted file mode 100644
index c102296b..00000000
Binary files a/tests/testdata/raw/nlcd/nlcd_2019_land_cover_l48_20210604.tif and /dev/null differ
diff --git a/tests/testdata/raw/nlcd/nlcd_2019_land_cover_l48_20210604.tif.aux.xml b/tests/testdata/raw/nlcd/nlcd_2019_land_cover_l48_20210604.tif.aux.xml
deleted file mode 100644
index 85b33d64..00000000
--- a/tests/testdata/raw/nlcd/nlcd_2019_land_cover_l48_20210604.tif.aux.xml
+++ /dev/null
@@ -1,2346 +0,0 @@
-
-
- NLCD Land Cover Class
-
-
- value
- 0
- 0
-
-
- NLCD Land Cover Class
- 2
- 0
-
-
- Histogram
- 1
- 0
-
-
- Red
- 0
- 0
-
-
- Green
- 0
- 0
-
-
- Blue
- 0
- 0
-
-
- Opacity
- 0
- 0
-
-
- 0
- Unclassified
- 7853863229
- 0
- 0
- 0
- 0
-
-
- 1
-
- 0
- 0
- 0
- 0
- 255
-
-
- 2
-
- 0
- 0
- 0
- 0
- 255
-
-
- 3
-
- 0
- 0
- 0
- 0
- 255
-
-
- 4
-
- 0
- 0
- 0
- 0
- 255
-
-
- 5
-
- 0
- 0
- 0
- 0
- 255
-
-
- 6
-
- 0
- 0
- 0
- 0
- 255
-
-
- 7
-
- 0
- 0
- 0
- 0
- 255
-
-
- 8
-
- 0
- 0
- 0
- 0
- 255
-
-
- 9
-
- 0
- 0
- 0
- 0
- 255
-
-
- 10
-
- 0
- 0
- 0
- 0
- 255
-
-
- 11
- Open Water
- 472399232
- 70
- 107
- 159
- 255
-
-
- 12
- Perennial Snow/Ice
- 962418
- 209
- 222
- 248
- 255
-
-
- 13
-
- 0
- 0
- 0
- 0
- 255
-
-
- 14
-
- 0
- 0
- 0
- 0
- 255
-
-
- 15
-
- 0
- 0
- 0
- 0
- 255
-
-
- 16
-
- 0
- 0
- 0
- 0
- 255
-
-
- 17
-
- 0
- 0
- 0
- 0
- 255
-
-
- 18
-
- 0
- 0
- 0
- 0
- 255
-
-
- 19
-
- 0
- 0
- 0
- 0
- 255
-
-
- 20
-
- 0
- 0
- 0
- 0
- 255
-
-
- 21
- Developed, Open Space
- 240566180
- 222
- 197
- 197
- 255
-
-
- 22
- Developed, Low Intensity
- 153288747
- 217
- 146
- 130
- 255
-
-
- 23
- Developed, Medium Intensity
- 92578072
- 235
- 0
- 0
- 255
-
-
- 24
- Developed, High Intensity
- 33121466
- 171
- 0
- 0
- 255
-
-
- 25
-
- 0
- 0
- 0
- 0
- 255
-
-
- 26
-
- 0
- 0
- 0
- 0
- 255
-
-
- 27
-
- 0
- 0
- 0
- 0
- 255
-
-
- 28
-
- 0
- 0
- 0
- 0
- 255
-
-
- 29
-
- 0
- 0
- 0
- 0
- 255
-
-
- 30
-
- 0
- 0
- 0
- 0
- 255
-
-
- 31
- Barren Land
- 87406005
- 179
- 172
- 159
- 255
-
-
- 32
-
- 0
- 0
- 0
- 0
- 255
-
-
- 33
-
- 0
- 0
- 0
- 0
- 255
-
-
- 34
-
- 0
- 0
- 0
- 0
- 255
-
-
- 35
-
- 0
- 0
- 0
- 0
- 255
-
-
- 36
-
- 0
- 0
- 0
- 0
- 255
-
-
- 37
-
- 0
- 0
- 0
- 0
- 255
-
-
- 38
-
- 0
- 0
- 0
- 0
- 255
-
-
- 39
-
- 0
- 0
- 0
- 0
- 255
-
-
- 40
-
- 0
- 0
- 0
- 0
- 255
-
-
- 41
- Deciduous Forest
- 833976610
- 104
- 171
- 95
- 255
-
-
- 42
- Evergreen Forest
- 1033039764
- 28
- 95
- 44
- 255
-
-
- 43
- Mixed Forest
- 305029988
- 181
- 197
- 143
- 255
-
-
- 44
-
- 0
- 0
- 0
- 0
- 255
-
-
- 45
-
- 0
- 0
- 0
- 0
- 255
-
-
- 46
-
- 0
- 0
- 0
- 0
- 255
-
-
- 47
-
- 0
- 0
- 0
- 0
- 255
-
-
- 48
-
- 0
- 0
- 0
- 0
- 255
-
-
- 49
-
- 0
- 0
- 0
- 0
- 255
-
-
- 50
-
- 0
- 0
- 0
- 0
- 255
-
-
- 51
-
- 0
- 0
- 0
- 0
- 255
-
-
- 52
- Shrub/Scrub
- 1961779404
- 204
- 184
- 121
- 255
-
-
- 53
-
- 0
- 0
- 0
- 0
- 255
-
-
- 54
-
- 0
- 0
- 0
- 0
- 255
-
-
- 55
-
- 0
- 0
- 0
- 0
- 255
-
-
- 56
-
- 0
- 0
- 0
- 0
- 255
-
-
- 57
-
- 0
- 0
- 0
- 0
- 255
-
-
- 58
-
- 0
- 0
- 0
- 0
- 255
-
-
- 59
-
- 0
- 0
- 0
- 0
- 255
-
-
- 60
-
- 0
- 0
- 0
- 0
- 255
-
-
- 61
-
- 0
- 0
- 0
- 0
- 255
-
-
- 62
-
- 0
- 0
- 0
- 0
- 255
-
-
- 63
-
- 0
- 0
- 0
- 0
- 255
-
-
- 64
-
- 0
- 0
- 0
- 0
- 255
-
-
- 65
-
- 0
- 0
- 0
- 0
- 255
-
-
- 66
-
- 0
- 0
- 0
- 0
- 255
-
-
- 67
-
- 0
- 0
- 0
- 0
- 255
-
-
- 68
-
- 0
- 0
- 0
- 0
- 255
-
-
- 69
-
- 0
- 0
- 0
- 0
- 255
-
-
- 70
-
- 0
- 0
- 0
- 0
- 255
-
-
- 71
- Herbaceous
- 1198000354
- 223
- 223
- 194
- 255
-
-
- 72
-
- 0
- 0
- 0
- 0
- 255
-
-
- 73
-
- 0
- 0
- 0
- 0
- 255
-
-
- 74
-
- 0
- 0
- 0
- 0
- 255
-
-
- 75
-
- 0
- 0
- 0
- 0
- 255
-
-
- 76
-
- 0
- 0
- 0
- 0
- 255
-
-
- 77
-
- 0
- 0
- 0
- 0
- 255
-
-
- 78
-
- 0
- 0
- 0
- 0
- 255
-
-
- 79
-
- 0
- 0
- 0
- 0
- 255
-
-
- 80
-
- 0
- 0
- 0
- 0
- 255
-
-
- 81
- Hay/Pasture
- 560647664
- 220
- 217
- 57
- 255
-
-
- 82
- Cultivated Crops
- 1464715609
- 171
- 108
- 40
- 255
-
-
- 83
-
- 0
- 0
- 0
- 0
- 255
-
-
- 84
-
- 0
- 0
- 0
- 0
- 255
-
-
- 85
-
- 0
- 0
- 0
- 0
- 255
-
-
- 86
-
- 0
- 0
- 0
- 0
- 255
-
-
- 87
-
- 0
- 0
- 0
- 0
- 255
-
-
- 88
-
- 0
- 0
- 0
- 0
- 255
-
-
- 89
-
- 0
- 0
- 0
- 0
- 255
-
-
- 90
- Woody Wetlands
- 403631293
- 184
- 217
- 235
- 255
-
-
- 91
-
- 0
- 0
- 0
- 0
- 255
-
-
- 92
-
- 0
- 0
- 0
- 0
- 255
-
-
- 93
-
- 0
- 0
- 0
- 0
- 255
-
-
- 94
-
- 0
- 0
- 0
- 0
- 255
-
-
- 95
- Emergent Herbaceous Wetlands
- 137098525
- 108
- 159
- 184
- 255
-
-
- 96
-
- 0
- 96
- 96
- 96
- 255
-
-
- 97
-
- 0
- 97
- 97
- 97
- 255
-
-
- 98
-
- 0
- 98
- 98
- 98
- 255
-
-
- 99
-
- 0
- 99
- 99
- 99
- 255
-
-
- 100
-
- 0
- 100
- 100
- 100
- 255
-
-
- 101
-
- 0
- 101
- 101
- 101
- 255
-
-
- 102
-
- 0
- 102
- 102
- 102
- 255
-
-
- 103
-
- 0
- 103
- 103
- 103
- 255
-
-
- 104
-
- 0
- 104
- 104
- 104
- 255
-
-
- 105
-
- 0
- 105
- 105
- 105
- 255
-
-
- 106
-
- 0
- 106
- 106
- 106
- 255
-
-
- 107
-
- 0
- 107
- 107
- 107
- 255
-
-
- 108
-
- 0
- 108
- 108
- 108
- 255
-
-
- 109
-
- 0
- 109
- 109
- 109
- 255
-
-
- 110
-
- 0
- 110
- 110
- 110
- 255
-
-
- 111
-
- 0
- 111
- 111
- 111
- 255
-
-
- 112
-
- 0
- 112
- 112
- 112
- 255
-
-
- 113
-
- 0
- 113
- 113
- 113
- 255
-
-
- 114
-
- 0
- 114
- 114
- 114
- 255
-
-
- 115
-
- 0
- 115
- 115
- 115
- 255
-
-
- 116
-
- 0
- 116
- 116
- 116
- 255
-
-
- 117
-
- 0
- 117
- 117
- 117
- 255
-
-
- 118
-
- 0
- 118
- 118
- 118
- 255
-
-
- 119
-
- 0
- 119
- 119
- 119
- 255
-
-
- 120
-
- 0
- 120
- 120
- 120
- 255
-
-
- 121
-
- 0
- 121
- 121
- 121
- 255
-
-
- 122
-
- 0
- 122
- 122
- 122
- 255
-
-
- 123
-
- 0
- 123
- 123
- 123
- 255
-
-
- 124
-
- 0
- 124
- 124
- 124
- 255
-
-
- 125
-
- 0
- 125
- 125
- 125
- 255
-
-
- 126
-
- 0
- 126
- 126
- 126
- 255
-
-
- 127
-
- 0
- 127
- 127
- 127
- 255
-
-
- 128
-
- 0
- 128
- 128
- 128
- 255
-
-
- 129
-
- 0
- 129
- 129
- 129
- 255
-
-
- 130
-
- 0
- 130
- 130
- 130
- 255
-
-
- 131
-
- 0
- 131
- 131
- 131
- 255
-
-
- 132
-
- 0
- 132
- 132
- 132
- 255
-
-
- 133
-
- 0
- 133
- 133
- 133
- 255
-
-
- 134
-
- 0
- 134
- 134
- 134
- 255
-
-
- 135
-
- 0
- 135
- 135
- 135
- 255
-
-
- 136
-
- 0
- 136
- 136
- 136
- 255
-
-
- 137
-
- 0
- 137
- 137
- 137
- 255
-
-
- 138
-
- 0
- 138
- 138
- 138
- 255
-
-
- 139
-
- 0
- 139
- 139
- 139
- 255
-
-
- 140
-
- 0
- 140
- 140
- 140
- 255
-
-
- 141
-
- 0
- 141
- 141
- 141
- 255
-
-
- 142
-
- 0
- 142
- 142
- 142
- 255
-
-
- 143
-
- 0
- 143
- 143
- 143
- 255
-
-
- 144
-
- 0
- 144
- 144
- 144
- 255
-
-
- 145
-
- 0
- 145
- 145
- 145
- 255
-
-
- 146
-
- 0
- 146
- 146
- 146
- 255
-
-
- 147
-
- 0
- 147
- 147
- 147
- 255
-
-
- 148
-
- 0
- 148
- 148
- 148
- 255
-
-
- 149
-
- 0
- 149
- 149
- 149
- 255
-
-
- 150
-
- 0
- 150
- 150
- 150
- 255
-
-
- 151
-
- 0
- 151
- 151
- 151
- 255
-
-
- 152
-
- 0
- 152
- 152
- 152
- 255
-
-
- 153
-
- 0
- 153
- 153
- 153
- 255
-
-
- 154
-
- 0
- 154
- 154
- 154
- 255
-
-
- 155
-
- 0
- 155
- 155
- 155
- 255
-
-
- 156
-
- 0
- 156
- 156
- 156
- 255
-
-
- 157
-
- 0
- 157
- 157
- 157
- 255
-
-
- 158
-
- 0
- 158
- 158
- 158
- 255
-
-
- 159
-
- 0
- 159
- 159
- 159
- 255
-
-
- 160
-
- 0
- 160
- 160
- 160
- 255
-
-
- 161
-
- 0
- 161
- 161
- 161
- 255
-
-
- 162
-
- 0
- 162
- 162
- 162
- 255
-
-
- 163
-
- 0
- 163
- 163
- 163
- 255
-
-
- 164
-
- 0
- 164
- 164
- 164
- 255
-
-
- 165
-
- 0
- 165
- 165
- 165
- 255
-
-
- 166
-
- 0
- 166
- 166
- 166
- 255
-
-
- 167
-
- 0
- 167
- 167
- 167
- 255
-
-
- 168
-
- 0
- 168
- 168
- 168
- 255
-
-
- 169
-
- 0
- 169
- 169
- 169
- 255
-
-
- 170
-
- 0
- 170
- 170
- 170
- 255
-
-
- 171
-
- 0
- 171
- 171
- 171
- 255
-
-
- 172
-
- 0
- 172
- 172
- 172
- 255
-
-
- 173
-
- 0
- 173
- 173
- 173
- 255
-
-
- 174
-
- 0
- 174
- 174
- 174
- 255
-
-
- 175
-
- 0
- 175
- 175
- 175
- 255
-
-
- 176
-
- 0
- 176
- 176
- 176
- 255
-
-
- 177
-
- 0
- 177
- 177
- 177
- 255
-
-
- 178
-
- 0
- 178
- 178
- 178
- 255
-
-
- 179
-
- 0
- 179
- 179
- 179
- 255
-
-
- 180
-
- 0
- 180
- 180
- 180
- 255
-
-
- 181
-
- 0
- 181
- 181
- 181
- 255
-
-
- 182
-
- 0
- 182
- 182
- 182
- 255
-
-
- 183
-
- 0
- 183
- 183
- 183
- 255
-
-
- 184
-
- 0
- 184
- 184
- 184
- 255
-
-
- 185
-
- 0
- 185
- 185
- 185
- 255
-
-
- 186
-
- 0
- 186
- 186
- 186
- 255
-
-
- 187
-
- 0
- 187
- 187
- 187
- 255
-
-
- 188
-
- 0
- 188
- 188
- 188
- 255
-
-
- 189
-
- 0
- 189
- 189
- 189
- 255
-
-
- 190
-
- 0
- 190
- 190
- 190
- 255
-
-
- 191
-
- 0
- 191
- 191
- 191
- 255
-
-
- 192
-
- 0
- 192
- 192
- 192
- 255
-
-
- 193
-
- 0
- 193
- 193
- 193
- 255
-
-
- 194
-
- 0
- 194
- 194
- 194
- 255
-
-
- 195
-
- 0
- 195
- 195
- 195
- 255
-
-
- 196
-
- 0
- 196
- 196
- 196
- 255
-
-
- 197
-
- 0
- 197
- 197
- 197
- 255
-
-
- 198
-
- 0
- 198
- 198
- 198
- 255
-
-
- 199
-
- 0
- 199
- 199
- 199
- 255
-
-
- 200
-
- 0
- 200
- 200
- 200
- 255
-
-
- 201
-
- 0
- 201
- 201
- 201
- 255
-
-
- 202
-
- 0
- 202
- 202
- 202
- 255
-
-
- 203
-
- 0
- 203
- 203
- 203
- 255
-
-
- 204
-
- 0
- 204
- 204
- 204
- 255
-
-
- 205
-
- 0
- 205
- 205
- 205
- 255
-
-
- 206
-
- 0
- 206
- 206
- 206
- 255
-
-
- 207
-
- 0
- 207
- 207
- 207
- 255
-
-
- 208
-
- 0
- 208
- 208
- 208
- 255
-
-
- 209
-
- 0
- 209
- 209
- 209
- 255
-
-
- 210
-
- 0
- 210
- 210
- 210
- 255
-
-
- 211
-
- 0
- 211
- 211
- 211
- 255
-
-
- 212
-
- 0
- 212
- 212
- 212
- 255
-
-
- 213
-
- 0
- 213
- 213
- 213
- 255
-
-
- 214
-
- 0
- 214
- 214
- 214
- 255
-
-
- 215
-
- 0
- 215
- 215
- 215
- 255
-
-
- 216
-
- 0
- 216
- 216
- 216
- 255
-
-
- 217
-
- 0
- 217
- 217
- 217
- 255
-
-
- 218
-
- 0
- 218
- 218
- 218
- 255
-
-
- 219
-
- 0
- 219
- 219
- 219
- 255
-
-
- 220
-
- 0
- 220
- 220
- 220
- 255
-
-
- 221
-
- 0
- 221
- 221
- 221
- 255
-
-
- 222
-
- 0
- 222
- 222
- 222
- 255
-
-
- 223
-
- 0
- 223
- 223
- 223
- 255
-
-
- 224
-
- 0
- 224
- 224
- 224
- 255
-
-
- 225
-
- 0
- 225
- 225
- 225
- 255
-
-
- 226
-
- 0
- 226
- 226
- 226
- 255
-
-
- 227
-
- 0
- 227
- 227
- 227
- 255
-
-
- 228
-
- 0
- 228
- 228
- 228
- 255
-
-
- 229
-
- 0
- 229
- 229
- 229
- 255
-
-
- 230
-
- 0
- 230
- 230
- 230
- 255
-
-
- 231
-
- 0
- 231
- 231
- 231
- 255
-
-
- 232
-
- 0
- 232
- 232
- 232
- 255
-
-
- 233
-
- 0
- 233
- 233
- 233
- 255
-
-
- 234
-
- 0
- 234
- 234
- 234
- 255
-
-
- 235
-
- 0
- 235
- 235
- 235
- 255
-
-
- 236
-
- 0
- 236
- 236
- 236
- 255
-
-
- 237
-
- 0
- 237
- 237
- 237
- 255
-
-
- 238
-
- 0
- 238
- 238
- 238
- 255
-
-
- 239
-
- 0
- 239
- 239
- 239
- 255
-
-
- 240
-
- 0
- 240
- 240
- 240
- 255
-
-
- 241
-
- 0
- 241
- 241
- 241
- 255
-
-
- 242
-
- 0
- 242
- 242
- 242
- 255
-
-
- 243
-
- 0
- 243
- 243
- 243
- 255
-
-
- 244
-
- 0
- 244
- 244
- 244
- 255
-
-
- 245
-
- 0
- 245
- 245
- 245
- 255
-
-
- 246
-
- 0
- 246
- 246
- 246
- 255
-
-
- 247
-
- 0
- 247
- 247
- 247
- 255
-
-
- 248
-
- 0
- 248
- 248
- 248
- 255
-
-
- 249
-
- 0
- 249
- 249
- 249
- 255
-
-
- 250
-
- 0
- 250
- 250
- 250
- 255
-
-
- 251
-
- 0
- 251
- 251
- 251
- 255
-
-
- 252
-
- 0
- 252
- 252
- 252
- 255
-
-
- 253
-
- 0
- 253
- 253
- 253
- 255
-
-
- 254
-
- 0
- 254
- 254
- 254
- 255
-
-
- 255
-
- 0
- 255
- 255
- 255
- 255
-
-
-
-
diff --git a/tests/testdata/raw/nlcd/nlcd_2021_land_cover_l48_20230630.tif b/tests/testdata/raw/nlcd/nlcd_2021_land_cover_l48_20230630.tif
deleted file mode 100644
index 43a5d038..00000000
Binary files a/tests/testdata/raw/nlcd/nlcd_2021_land_cover_l48_20230630.tif and /dev/null differ
diff --git a/tests/testdata/raw/nlcd/nlcd_2021_land_cover_l48_20230630.tif.aux.xml b/tests/testdata/raw/nlcd/nlcd_2021_land_cover_l48_20230630.tif.aux.xml
deleted file mode 100644
index b25aa006..00000000
--- a/tests/testdata/raw/nlcd/nlcd_2021_land_cover_l48_20230630.tif.aux.xml
+++ /dev/null
@@ -1,2346 +0,0 @@
-
-
- NLCD Land Cover Class
-
-
- value
- 0
- 0
-
-
- NLCD Land Cover Class
- 2
- 0
-
-
- Histogram
- 1
- 0
-
-
- Red
- 0
- 0
-
-
- Green
- 0
- 0
-
-
- Blue
- 0
- 0
-
-
- Opacity
- 0
- 0
-
-
- 0
- Unclassified
- 7853863229
- 0
- 0
- 0
- 0
-
-
- 1
-
- 0
- 0
- 0
- 0
- 255
-
-
- 2
-
- 0
- 0
- 0
- 0
- 255
-
-
- 3
-
- 0
- 0
- 0
- 0
- 255
-
-
- 4
-
- 0
- 0
- 0
- 0
- 255
-
-
- 5
-
- 0
- 0
- 0
- 0
- 255
-
-
- 6
-
- 0
- 0
- 0
- 0
- 255
-
-
- 7
-
- 0
- 0
- 0
- 0
- 255
-
-
- 8
-
- 0
- 0
- 0
- 0
- 255
-
-
- 9
-
- 0
- 0
- 0
- 0
- 255
-
-
- 10
-
- 0
- 0
- 0
- 0
- 255
-
-
- 11
- Open Water
- 466021535
- 70
- 107
- 159
- 255
-
-
- 12
- Perennial Snow/Ice
- 959819
- 209
- 222
- 248
- 255
-
-
- 13
-
- 0
- 0
- 0
- 0
- 255
-
-
- 14
-
- 0
- 0
- 0
- 0
- 255
-
-
- 15
-
- 0
- 0
- 0
- 0
- 255
-
-
- 16
-
- 0
- 0
- 0
- 0
- 255
-
-
- 17
-
- 0
- 0
- 0
- 0
- 255
-
-
- 18
-
- 0
- 0
- 0
- 0
- 255
-
-
- 19
-
- 0
- 0
- 0
- 0
- 255
-
-
- 20
-
- 0
- 0
- 0
- 0
- 255
-
-
- 21
- Developed, Open Space
- 240377730
- 222
- 197
- 197
- 255
-
-
- 22
- Developed, Low Intensity
- 154252969
- 217
- 146
- 130
- 255
-
-
- 23
- Developed, Medium Intensity
- 94385151
- 235
- 0
- 0
- 255
-
-
- 24
- Developed, High Intensity
- 33594478
- 171
- 0
- 0
- 255
-
-
- 25
-
- 0
- 0
- 0
- 0
- 255
-
-
- 26
-
- 0
- 0
- 0
- 0
- 255
-
-
- 27
-
- 0
- 0
- 0
- 0
- 255
-
-
- 28
-
- 0
- 0
- 0
- 0
- 255
-
-
- 29
-
- 0
- 0
- 0
- 0
- 255
-
-
- 30
-
- 0
- 0
- 0
- 0
- 255
-
-
- 31
- Barren Land
- 88976651
- 179
- 172
- 159
- 255
-
-
- 32
-
- 0
- 0
- 0
- 0
- 255
-
-
- 33
-
- 0
- 0
- 0
- 0
- 255
-
-
- 34
-
- 0
- 0
- 0
- 0
- 255
-
-
- 35
-
- 0
- 0
- 0
- 0
- 255
-
-
- 36
-
- 0
- 0
- 0
- 0
- 255
-
-
- 37
-
- 0
- 0
- 0
- 0
- 255
-
-
- 38
-
- 0
- 0
- 0
- 0
- 255
-
-
- 39
-
- 0
- 0
- 0
- 0
- 255
-
-
- 40
-
- 0
- 0
- 0
- 0
- 255
-
-
- 41
- Deciduous Forest
- 832916302
- 104
- 171
- 95
- 255
-
-
- 42
- Evergreen Forest
- 1027738936
- 28
- 95
- 44
- 255
-
-
- 43
- Mixed Forest
- 304954315
- 181
- 197
- 143
- 255
-
-
- 44
-
- 0
- 0
- 0
- 0
- 255
-
-
- 45
-
- 0
- 0
- 0
- 0
- 255
-
-
- 46
-
- 0
- 0
- 0
- 0
- 255
-
-
- 47
-
- 0
- 0
- 0
- 0
- 255
-
-
- 48
-
- 0
- 0
- 0
- 0
- 255
-
-
- 49
-
- 0
- 0
- 0
- 0
- 255
-
-
- 50
-
- 0
- 0
- 0
- 0
- 255
-
-
- 51
-
- 0
- 0
- 0
- 0
- 255
-
-
- 52
- Shrub/Scrub
- 1953262413
- 204
- 184
- 121
- 255
-
-
- 53
-
- 0
- 0
- 0
- 0
- 255
-
-
- 54
-
- 0
- 0
- 0
- 0
- 255
-
-
- 55
-
- 0
- 0
- 0
- 0
- 255
-
-
- 56
-
- 0
- 0
- 0
- 0
- 255
-
-
- 57
-
- 0
- 0
- 0
- 0
- 255
-
-
- 58
-
- 0
- 0
- 0
- 0
- 255
-
-
- 59
-
- 0
- 0
- 0
- 0
- 255
-
-
- 60
-
- 0
- 0
- 0
- 0
- 255
-
-
- 61
-
- 0
- 0
- 0
- 0
- 255
-
-
- 62
-
- 0
- 0
- 0
- 0
- 255
-
-
- 63
-
- 0
- 0
- 0
- 0
- 255
-
-
- 64
-
- 0
- 0
- 0
- 0
- 255
-
-
- 65
-
- 0
- 0
- 0
- 0
- 255
-
-
- 66
-
- 0
- 0
- 0
- 0
- 255
-
-
- 67
-
- 0
- 0
- 0
- 0
- 255
-
-
- 68
-
- 0
- 0
- 0
- 0
- 255
-
-
- 69
-
- 0
- 0
- 0
- 0
- 255
-
-
- 70
-
- 0
- 0
- 0
- 0
- 255
-
-
- 71
- Herbaceous
- 1223488270
- 223
- 223
- 194
- 255
-
-
- 72
-
- 0
- 0
- 0
- 0
- 255
-
-
- 73
-
- 0
- 0
- 0
- 0
- 255
-
-
- 74
-
- 0
- 0
- 0
- 0
- 255
-
-
- 75
-
- 0
- 0
- 0
- 0
- 255
-
-
- 76
-
- 0
- 0
- 0
- 0
- 255
-
-
- 77
-
- 0
- 0
- 0
- 0
- 255
-
-
- 78
-
- 0
- 0
- 0
- 0
- 255
-
-
- 79
-
- 0
- 0
- 0
- 0
- 255
-
-
- 80
-
- 0
- 0
- 0
- 0
- 255
-
-
- 81
- Hay/Pasture
- 558088764
- 220
- 217
- 57
- 255
-
-
- 82
- Cultivated Crops
- 1453539695
- 171
- 108
- 40
- 255
-
-
- 83
-
- 0
- 0
- 0
- 0
- 255
-
-
- 84
-
- 0
- 0
- 0
- 0
- 255
-
-
- 85
-
- 0
- 0
- 0
- 0
- 255
-
-
- 86
-
- 0
- 0
- 0
- 0
- 255
-
-
- 87
-
- 0
- 0
- 0
- 0
- 255
-
-
- 88
-
- 0
- 0
- 0
- 0
- 255
-
-
- 89
-
- 0
- 0
- 0
- 0
- 255
-
-
- 90
- Woody Wetlands
- 403110859
- 184
- 217
- 235
- 255
-
-
- 91
-
- 0
- 0
- 0
- 0
- 255
-
-
- 92
-
- 0
- 0
- 0
- 0
- 255
-
-
- 93
-
- 0
- 0
- 0
- 0
- 255
-
-
- 94
-
- 0
- 0
- 0
- 0
- 255
-
-
- 95
- Emergent Herbaceous Wetlands
- 142573444
- 108
- 159
- 184
- 255
-
-
- 96
-
- 0
- 96
- 96
- 96
- 255
-
-
- 97
-
- 0
- 97
- 97
- 97
- 255
-
-
- 98
-
- 0
- 98
- 98
- 98
- 255
-
-
- 99
-
- 0
- 99
- 99
- 99
- 255
-
-
- 100
-
- 0
- 100
- 100
- 100
- 255
-
-
- 101
-
- 0
- 101
- 101
- 101
- 255
-
-
- 102
-
- 0
- 102
- 102
- 102
- 255
-
-
- 103
-
- 0
- 103
- 103
- 103
- 255
-
-
- 104
-
- 0
- 104
- 104
- 104
- 255
-
-
- 105
-
- 0
- 105
- 105
- 105
- 255
-
-
- 106
-
- 0
- 106
- 106
- 106
- 255
-
-
- 107
-
- 0
- 107
- 107
- 107
- 255
-
-
- 108
-
- 0
- 108
- 108
- 108
- 255
-
-
- 109
-
- 0
- 109
- 109
- 109
- 255
-
-
- 110
-
- 0
- 110
- 110
- 110
- 255
-
-
- 111
-
- 0
- 111
- 111
- 111
- 255
-
-
- 112
-
- 0
- 112
- 112
- 112
- 255
-
-
- 113
-
- 0
- 113
- 113
- 113
- 255
-
-
- 114
-
- 0
- 114
- 114
- 114
- 255
-
-
- 115
-
- 0
- 115
- 115
- 115
- 255
-
-
- 116
-
- 0
- 116
- 116
- 116
- 255
-
-
- 117
-
- 0
- 117
- 117
- 117
- 255
-
-
- 118
-
- 0
- 118
- 118
- 118
- 255
-
-
- 119
-
- 0
- 119
- 119
- 119
- 255
-
-
- 120
-
- 0
- 120
- 120
- 120
- 255
-
-
- 121
-
- 0
- 121
- 121
- 121
- 255
-
-
- 122
-
- 0
- 122
- 122
- 122
- 255
-
-
- 123
-
- 0
- 123
- 123
- 123
- 255
-
-
- 124
-
- 0
- 124
- 124
- 124
- 255
-
-
- 125
-
- 0
- 125
- 125
- 125
- 255
-
-
- 126
-
- 0
- 126
- 126
- 126
- 255
-
-
- 127
-
- 0
- 127
- 127
- 127
- 255
-
-
- 128
-
- 0
- 128
- 128
- 128
- 255
-
-
- 129
-
- 0
- 129
- 129
- 129
- 255
-
-
- 130
-
- 0
- 130
- 130
- 130
- 255
-
-
- 131
-
- 0
- 131
- 131
- 131
- 255
-
-
- 132
-
- 0
- 132
- 132
- 132
- 255
-
-
- 133
-
- 0
- 133
- 133
- 133
- 255
-
-
- 134
-
- 0
- 134
- 134
- 134
- 255
-
-
- 135
-
- 0
- 135
- 135
- 135
- 255
-
-
- 136
-
- 0
- 136
- 136
- 136
- 255
-
-
- 137
-
- 0
- 137
- 137
- 137
- 255
-
-
- 138
-
- 0
- 138
- 138
- 138
- 255
-
-
- 139
-
- 0
- 139
- 139
- 139
- 255
-
-
- 140
-
- 0
- 140
- 140
- 140
- 255
-
-
- 141
-
- 0
- 141
- 141
- 141
- 255
-
-
- 142
-
- 0
- 142
- 142
- 142
- 255
-
-
- 143
-
- 0
- 143
- 143
- 143
- 255
-
-
- 144
-
- 0
- 144
- 144
- 144
- 255
-
-
- 145
-
- 0
- 145
- 145
- 145
- 255
-
-
- 146
-
- 0
- 146
- 146
- 146
- 255
-
-
- 147
-
- 0
- 147
- 147
- 147
- 255
-
-
- 148
-
- 0
- 148
- 148
- 148
- 255
-
-
- 149
-
- 0
- 149
- 149
- 149
- 255
-
-
- 150
-
- 0
- 150
- 150
- 150
- 255
-
-
- 151
-
- 0
- 151
- 151
- 151
- 255
-
-
- 152
-
- 0
- 152
- 152
- 152
- 255
-
-
- 153
-
- 0
- 153
- 153
- 153
- 255
-
-
- 154
-
- 0
- 154
- 154
- 154
- 255
-
-
- 155
-
- 0
- 155
- 155
- 155
- 255
-
-
- 156
-
- 0
- 156
- 156
- 156
- 255
-
-
- 157
-
- 0
- 157
- 157
- 157
- 255
-
-
- 158
-
- 0
- 158
- 158
- 158
- 255
-
-
- 159
-
- 0
- 159
- 159
- 159
- 255
-
-
- 160
-
- 0
- 160
- 160
- 160
- 255
-
-
- 161
-
- 0
- 161
- 161
- 161
- 255
-
-
- 162
-
- 0
- 162
- 162
- 162
- 255
-
-
- 163
-
- 0
- 163
- 163
- 163
- 255
-
-
- 164
-
- 0
- 164
- 164
- 164
- 255
-
-
- 165
-
- 0
- 165
- 165
- 165
- 255
-
-
- 166
-
- 0
- 166
- 166
- 166
- 255
-
-
- 167
-
- 0
- 167
- 167
- 167
- 255
-
-
- 168
-
- 0
- 168
- 168
- 168
- 255
-
-
- 169
-
- 0
- 169
- 169
- 169
- 255
-
-
- 170
-
- 0
- 170
- 170
- 170
- 255
-
-
- 171
-
- 0
- 171
- 171
- 171
- 255
-
-
- 172
-
- 0
- 172
- 172
- 172
- 255
-
-
- 173
-
- 0
- 173
- 173
- 173
- 255
-
-
- 174
-
- 0
- 174
- 174
- 174
- 255
-
-
- 175
-
- 0
- 175
- 175
- 175
- 255
-
-
- 176
-
- 0
- 176
- 176
- 176
- 255
-
-
- 177
-
- 0
- 177
- 177
- 177
- 255
-
-
- 178
-
- 0
- 178
- 178
- 178
- 255
-
-
- 179
-
- 0
- 179
- 179
- 179
- 255
-
-
- 180
-
- 0
- 180
- 180
- 180
- 255
-
-
- 181
-
- 0
- 181
- 181
- 181
- 255
-
-
- 182
-
- 0
- 182
- 182
- 182
- 255
-
-
- 183
-
- 0
- 183
- 183
- 183
- 255
-
-
- 184
-
- 0
- 184
- 184
- 184
- 255
-
-
- 185
-
- 0
- 185
- 185
- 185
- 255
-
-
- 186
-
- 0
- 186
- 186
- 186
- 255
-
-
- 187
-
- 0
- 187
- 187
- 187
- 255
-
-
- 188
-
- 0
- 188
- 188
- 188
- 255
-
-
- 189
-
- 0
- 189
- 189
- 189
- 255
-
-
- 190
-
- 0
- 190
- 190
- 190
- 255
-
-
- 191
-
- 0
- 191
- 191
- 191
- 255
-
-
- 192
-
- 0
- 192
- 192
- 192
- 255
-
-
- 193
-
- 0
- 193
- 193
- 193
- 255
-
-
- 194
-
- 0
- 194
- 194
- 194
- 255
-
-
- 195
-
- 0
- 195
- 195
- 195
- 255
-
-
- 196
-
- 0
- 196
- 196
- 196
- 255
-
-
- 197
-
- 0
- 197
- 197
- 197
- 255
-
-
- 198
-
- 0
- 198
- 198
- 198
- 255
-
-
- 199
-
- 0
- 199
- 199
- 199
- 255
-
-
- 200
-
- 0
- 200
- 200
- 200
- 255
-
-
- 201
-
- 0
- 201
- 201
- 201
- 255
-
-
- 202
-
- 0
- 202
- 202
- 202
- 255
-
-
- 203
-
- 0
- 203
- 203
- 203
- 255
-
-
- 204
-
- 0
- 204
- 204
- 204
- 255
-
-
- 205
-
- 0
- 205
- 205
- 205
- 255
-
-
- 206
-
- 0
- 206
- 206
- 206
- 255
-
-
- 207
-
- 0
- 207
- 207
- 207
- 255
-
-
- 208
-
- 0
- 208
- 208
- 208
- 255
-
-
- 209
-
- 0
- 209
- 209
- 209
- 255
-
-
- 210
-
- 0
- 210
- 210
- 210
- 255
-
-
- 211
-
- 0
- 211
- 211
- 211
- 255
-
-
- 212
-
- 0
- 212
- 212
- 212
- 255
-
-
- 213
-
- 0
- 213
- 213
- 213
- 255
-
-
- 214
-
- 0
- 214
- 214
- 214
- 255
-
-
- 215
-
- 0
- 215
- 215
- 215
- 255
-
-
- 216
-
- 0
- 216
- 216
- 216
- 255
-
-
- 217
-
- 0
- 217
- 217
- 217
- 255
-
-
- 218
-
- 0
- 218
- 218
- 218
- 255
-
-
- 219
-
- 0
- 219
- 219
- 219
- 255
-
-
- 220
-
- 0
- 220
- 220
- 220
- 255
-
-
- 221
-
- 0
- 221
- 221
- 221
- 255
-
-
- 222
-
- 0
- 222
- 222
- 222
- 255
-
-
- 223
-
- 0
- 223
- 223
- 223
- 255
-
-
- 224
-
- 0
- 224
- 224
- 224
- 255
-
-
- 225
-
- 0
- 225
- 225
- 225
- 255
-
-
- 226
-
- 0
- 226
- 226
- 226
- 255
-
-
- 227
-
- 0
- 227
- 227
- 227
- 255
-
-
- 228
-
- 0
- 228
- 228
- 228
- 255
-
-
- 229
-
- 0
- 229
- 229
- 229
- 255
-
-
- 230
-
- 0
- 230
- 230
- 230
- 255
-
-
- 231
-
- 0
- 231
- 231
- 231
- 255
-
-
- 232
-
- 0
- 232
- 232
- 232
- 255
-
-
- 233
-
- 0
- 233
- 233
- 233
- 255
-
-
- 234
-
- 0
- 234
- 234
- 234
- 255
-
-
- 235
-
- 0
- 235
- 235
- 235
- 255
-
-
- 236
-
- 0
- 236
- 236
- 236
- 255
-
-
- 237
-
- 0
- 237
- 237
- 237
- 255
-
-
- 238
-
- 0
- 238
- 238
- 238
- 255
-
-
- 239
-
- 0
- 239
- 239
- 239
- 255
-
-
- 240
-
- 0
- 240
- 240
- 240
- 255
-
-
- 241
-
- 0
- 241
- 241
- 241
- 255
-
-
- 242
-
- 0
- 242
- 242
- 242
- 255
-
-
- 243
-
- 0
- 243
- 243
- 243
- 255
-
-
- 244
-
- 0
- 244
- 244
- 244
- 255
-
-
- 245
-
- 0
- 245
- 245
- 245
- 255
-
-
- 246
-
- 0
- 246
- 246
- 246
- 255
-
-
- 247
-
- 0
- 247
- 247
- 247
- 255
-
-
- 248
-
- 0
- 248
- 248
- 248
- 255
-
-
- 249
-
- 0
- 249
- 249
- 249
- 255
-
-
- 250
-
- 0
- 250
- 250
- 250
- 255
-
-
- 251
-
- 0
- 251
- 251
- 251
- 255
-
-
- 252
-
- 0
- 252
- 252
- 252
- 255
-
-
- 253
-
- 0
- 253
- 253
- 253
- 255
-
-
- 254
-
- 0
- 254
- 254
- 254
- 255
-
-
- 255
-
- 0
- 255
- 255
- 255
- 255
-
-
-
-
diff --git a/tests/testdata/testdata_generation.Rmd b/tests/testdata/testdata_generation.Rmd
deleted file mode 100644
index 0424d799..00000000
--- a/tests/testdata/testdata_generation.Rmd
+++ /dev/null
@@ -1,282 +0,0 @@
----
-title: "Create testdata on a small area"
-output: html_document
-date: "2024-01-02"
----
-
-# Testdata area and period
-
-### Small area
-
-```{r}
-nc_file <- system.file("shape/nc.shp", package = "sf")
-nc <- terra::vect(nc_file)
-sp_samp <- nc[nc$NAME %in% c("Durham", "Wake", "Orange")]
-terra::plot(sp_samp)
-```
-
-### 2022-01-01 to 2022-01-02
-
-```{r}
-start_date <- as.Date("2022-01-01")
-end_date <- as.Date("2022-01-02")
-```
-
-### Load covariate file names
-
-```{r}
-dict_path <- "../../inst/extdata/downloaded_files_metadata.csv"
-dict <- data.table::fread(dict_path, sep = ",")
-```
-
-### File path name
-
-```{r}
-string <- "/folder/yyyy/yday/varname_yyyymmdd.nc"
-date <- as.Date("2022-08-01")
-replace_dateinfo(string, date)
-```
-```{r}
-raw_path <- "/Volumes/set/Projects/NRT-AP-Model/tests/testdata/raw/"
-```
-
-# Extract sampled area from each downloaded data
-
-## NLCD
-
-Extraction
-
-```{r, eval = FALSE}
-testdata_path <- paste0(raw_path, "nlcd/")
-testdata_nlcd(testdata_path = testdata_path)
-```
-
-Example of output
-
-```{r}
-nlcd_sub_2019 <- terra::rast(paste0(
- "raw/nlcd/",
- "nlcd_2019_land_cover_l48_20210604.tif"
-))
-nlcd_sub_2021 <- terra::rast(paste0(
- "raw/nlcd/",
- "nlcd_2021_land_cover_l48_20230630.tif"
-))
-terra::plot(nlcd_sub_2019)
-terra::plot(nlcd_sub_2021)
-```
-
-## Ecoregions
-
-Extraction
-
-```{r, eval = FALSE}
-testdata_path <- paste0(raw_path, "ecoregions/")
-testdata_ecoregions(testdata_path = testdata_path)
-```
-
-Example of output
-
-```{r}
-ecor_sub <- terra::vect(paste0(
- "raw/",
- "ecoregions/us_eco_l3_state_boundaries.shp"
-))
-terra::plot(ecor_sub)
-```
-
-## Koppen-Geiger Climate classification
-
-Extraction
-
-```{r, eval = FALSE}
-testdata_path <- paste0(raw_path, "koppen_geiger/")
-testdata_kg(testdata_path = testdata_path)
-```
-
-Example of output
-
-```{r}
-kg_sub <- terra::rast(paste0(
- "raw/koppen_geiger/",
- "Beck_KG_V1_present_0p0083.tif"
-))
-terra::plot(kp_sub)
-```
-
-## NARR reanalysis
-
-#### Variables stored as monthly rasters
-
-Extraction
-
-```{r, eval = FALSE}
-narr_names <- c(
- "air_sfc", "albedo", "apcp", "dswrf", "evap", "hcdc", "hpbl",
- "lcdc", "lhtfl", "mcdc", "pr_wtr", "prate", "pres_sfc", "shtfl",
- "snowc", "soilm", "tcdc", "ulwrf_sfc", "uwnd_10m", "vis",
- "vwnd_10m", "weasd"
-)
-testdata_path <- paste0(raw_path, "narr/")
-for (n in narr_names) {
- testdata_narr(var = n, testdata_path = testdata_path)
-}
-```
-
-Example of output
-
-```{r}
-albedo_sub <- terra::rast("raw/narr/albedo.2022.nc")
-terra::plot(albedo_sub)
-```
-
-#### Variables stored as monthly * atm levels rasters
-
-Extraction
-
-```{r, eval = FALSE}
-narr_lev_names <- c("omega", "shum")
-testdata_path <- paste0(raw_path, "narr/")
-for (n in narr_lev_names) {
- testdata_narr_lev(var = n, testdata_path = testdata_path)
-}
-```
-
-Example of output
-
-```{r}
-omega_sub <- terra::rast("raw/narr/omega.202201.nc")
-terra::plot(omega_sub)
-```
-
-## GEOS CF
-
-### AQC
-
-Extraction
-
-```{r, eval = FALSE}
-aqc_path <- paste0(raw_path, "geos/aqc_tavg_1hr_g1440x721_v1/")
-testdata_geos(testdata_path = aqc_path, collection = "aqc_tavg")
-```
-
-New file is not exactly the same than raw file but contains all covariates and units:
-
-```{r}
-r_path <- paste0(
- aqc_path,
- "GEOS-CF.v01.rpl.aqc_tavg_1hr_g1440x721_v1.20220101_0030z.nc4"
-)
-aqc_sub <- terra::rast(r_path)
-aqc_sub
-terra::plot(aqc_sub)
-```
-
-### CHM
-
-Extraction
-
-```{r, eval = FALSE}
-chm_path <- paste0(raw_path, "geos/chm_tavg_1hr_g1440x721_v1/")
-testdata_geos(testdata_path = chm_path, collection = "chm_tavg")
-```
-
-Example extracted
-
-```{r}
-r_path <- paste0(
- chm_path,
- "GEOS-CF.v01.rpl.chm_tavg_1hr_g1440x721_v1.20220101_0030z.nc4"
-)
-chm_sub <- terra::rast(r_path)
-chm_sub
-terra::plot(chm_sub[[1:16]])
-terra::plot(chm_sub[[17:32]])
-terra::plot(chm_sub[[33:48]])
-terra::plot(chm_sub[[48:52]])
-```
-
-## Toxic Release Inventory (TRI)
-
-Extraction
-
-```{r, eval = FALSE}
-testdata_path <- paste0(raw_path, "tri/")
-testdata_tri(dict_path = "../../inst/extdata/downloaded_files_metadata.csv",
- year = 2022,
- testdata_path = testdata_path)
-```
-
-Example of output
-
-```{r}
-tri_samp <- data.table::fread("raw/tri/tri_raw_2022.csv")
-head(tri_samp)
-```
-
-
-## Air Quality System (AQS)
-
-Extraction
-
-```{r, eval = FALSE}
-testdata_path <- paste0(raw_path, "aqs/")
-testdata_aqs(testdata_path = testdata_path)
-```
-
-Example of output
-
-```{r}
-aqs_samp <- data.table::fread("raw/aqs/daily_88101_2022.csv")
-head(aqs_samp)
-```
-
-## National Emissions Inventory (NEI)
-
-Extraction
-
-```{r, eval = FALSE}
-testdata_path <- paste0(raw_path, "NEI/")
-testdata_nei(testdata_path = testdata_path)
-```
-
-```{r}
-nei_samp <- data.table::fread("raw/NEI/onroad_8910.csv")
-head(nei_samp)
-```
-
-
-## HMS Smoke
-
-Extraction
-
-```{r, eval = FALSE}
-testdata_path <- paste0(raw_path, "hms_smoke/")
-testdata_hms_smoke(testdata_path = testdata_path)
-```
-
-Example of output
-
-```{r}
-smoke_samp <- terra::vect(paste0(
- "raw/hms_smoke/",
- "hms_smoke20220618.shp"
-))
-terra::plot(x = smoke_samp, y = "Density")
-terra::plot(sp_samp, lty = 2, lwd = 4, add = TRUE)
-```
-
-Comparison to the map on the entire US
-
-```{r}
-if (requireNamespace("spData", quietly = TRUE)) {
- library(spData)
- data(us_states)
-}
-hms_jun18 <- terra::vect(paste0(
- "/Volumes/set/Projects/NRT-AP-Model/input/",
- "HMS_Smoke/data/hms_smoke20220618.shp"
-))
-terra::plot(terra::project(terra::vect(us_states), terra::crs(hmsday)))
-terra::plot(x = hmsday, y = "Density", alpha = .5, add = TRUE)
-```
diff --git a/tests/testthat/test-base_learner.R b/tests/testthat/test-base_learner.R
deleted file mode 100644
index abfb9081..00000000
--- a/tests/testthat/test-base_learner.R
+++ /dev/null
@@ -1,262 +0,0 @@
-# Tests for base_learner functions
-
-testthat::test_that("base_learner_fit works", {
- withr::local_package("terra")
- withr::local_package("dplyr")
- withr::local_package("data.table")
- withr::local_package("ranger")
- withr::local_package("xgboost")
-
- nsp <- 50L
- nt <- 60L
- np <- 10L
-
- # random data with 50 (spatial) * 60 (temporal) * 4 (covariates)
- sphere <-
- terra::ext(c(xmin = 0, xmax = 100, ymin = 0, ymax = 80))
- pnts <- terra::spatSample(sphere, nsp, lonlat = FALSE, as.points = TRUE)
- pst <- split(seq(1, nt), seq(1, nt))
- pst <- lapply(pst,
- function(x) {
- pnts$pid <- seq(1, nsp)
- return(pnts)})
- pst <- Reduce(rbind, pst)
- pst$time <- rep(as.Date("2024-01-19") + seq(0, nt - 1), each = nsp)
- pst$pm2.5 <- rgamma(nsp * nt, 32, 1.6)
-
- pstx <- rgamma(nsp * nt * np, 1, 0.002)
- pstx <- matrix(pstx, nrow = nsp * nt, ncol = np)
- pstx <- as.data.frame(pstx)
- cns_covar <- sprintf("X%d", seq(1, np))
- colnames(pstx) <- cns_covar
- pst <- cbind(pst, pstx)
-
- pstdt <- convert_stobj_to_stdt(pst)
-
- colindx <-
- grep(
- paste0("(", paste(sprintf("X%d", seq(1, np)), collapse = "|"), ")"),
- colnames(pstdt$stdt))
-
- testthat::expect_no_error(
- test_fit_rf <- base_learner_fit(pstdt,
- "randomforest",
- independent_name = cns_covar,
- cv_mode = "lblo", blocks = c(25L, 20L),
- num.trees = 100L)
- )
- testthat::expect_no_error(
- test_fit_xg <- base_learner_fit(pstdt,
- "xgboost",
- independent_name = cns_covar,
- cv_mode = "lblo", blocks = c(25L, 20L),
- nrounds = 50L)
- )
- testthat::expect_true(is.list(test_fit_rf))
- testthat::expect_true(is.list(test_fit_xg))
-})
-
-
-testthat::test_that("base learner data cv fit: ranger", {
- withr::local_package("terra")
- withr::local_package("ranger")
- withr::local_package("dplyr")
- withr::local_package("data.table")
-
- nsp <- 50L
- nt <- 60L
- np <- 10L
-
- # random data with 50 (spatial)*60 (temporal)*4 (covariates)
- sphere <-
- terra::ext(c(xmin = 0, xmax = 100, ymin = 0, ymax = 80))
- pnts <- terra::spatSample(sphere, nsp, lonlat = FALSE, as.points = TRUE)
- pst <- split(seq(1, nt), seq(1, nt))
- pst <- lapply(pst,
- function(x) {
- pnts$pid <- seq(1, nsp)
- return(pnts)})
- pst <- Reduce(rbind, pst)
- pst$time <- rep(as.Date("2024-01-19") + seq(0, nt - 1), each = nsp)
- pst$pm2.5 <- rgamma(nsp * nt, 32, 1.6)
-
- pstx <- rgamma(nsp * nt * np, 1, 0.002)
- pstx <- matrix(pstx, nrow = nsp * nt, ncol = np)
- pstx <- as.data.frame(pstx)
- colnames(pstx) <- sprintf("X%d", seq(1, np))
- pst <- cbind(pst, pstx)
-
- pstdt <- convert_stobj_to_stdt(pst)
-
- colindx <-
- grep(
- paste0("(", paste(sprintf("X%d", seq(1, np)), collapse = "|"), ")"),
- colnames(pstdt$stdt))
-
- res_datap <- base_learner_prep(
- learner = "xgboost",
- data = pstdt,
- dependent_name = "pm2.5",
- independent_name = sprintf("X%d", seq(1, np))
- )
- testthat::expect_no_error(
- res_check_fit_rf <-
- base_learner_fit_ranger(res_datap$ymat, res_datap$xmat, num.trees = 50L)
- )
- testthat::expect_no_error(
- res_check_fit_xg <-
- base_learner_fit_xgboost(res_datap$ymat, res_datap$xmat, nrounds = 10L)
- )
-
-})
-
-testthat::test_that("base learner cv fit: rf and xgboost", {
- withr::local_package("terra")
- withr::local_package("xgboost")
- withr::local_package("ranger")
- withr::local_package("dplyr")
- withr::local_package("data.table")
-
- nsp <- 50L
- nt <- 60L
- np <- 10L
-
- # random data with 50 (spatial)*60 (temporal)*4 (covariates)
- sphere <-
- terra::ext(c(xmin = 0, xmax = 100, ymin = 0, ymax = 80))
- pnts <- terra::spatSample(sphere, nsp, lonlat = FALSE, as.points = TRUE)
- pst <- split(seq(1, nt), seq(1, nt))
- pst <- lapply(pst,
- function(x) {
- pnts$pid <- seq(1, nsp)
- return(pnts)})
- pst <- Reduce(rbind, pst)
- pst$time <- rep(as.Date("2024-01-19") + seq(0, nt - 1), each = nsp)
- pst$pm2.5 <- rgamma(nsp * nt, 32, 1.6)
-
- pstx <- rgamma(nsp * nt * np, 1, 0.002)
- pstx <- matrix(pstx, nrow = nsp * nt, ncol = np)
- pstx <- as.data.frame(pstx)
- colnames(pstx) <- sprintf("X%d", seq(1, np))
- pst <- cbind(pst, pstx)
-
- testthat::expect_no_error(
- pstdt <- convert_stobj_to_stdt(pst)
- )
- colindx <-
- grep(
- paste0("(", paste(sprintf("X%d", seq(1, np)), collapse = "|"), ")"),
- colnames(pstdt$stdt))
-
- res_datap <- base_learner_prep(
- learner = "xgboost",
- data = pstdt,
- dependent_name = "pm2.5",
- independent_name = sprintf("X%d", seq(1, np))
- )
-
- testthat::expect_no_error(
- res_check_fit_xg <-
- base_learner_fit_xgboost(res_datap$ymat, res_datap$xmat, nrounds = 30L)
- )
-
- testthat::expect_no_error(
- res_check_fit_ranger_lblo <-
- base_learner_fit(
- data = pstdt,
- learner = "randomforest",
- dependent_name = "pm2.5",
- independent_name = colnames(pstdt$stdt)[-1:-5],
- cv_mode = "lblo",
- blocks = c(20, 20),
- num.trees = 100L
- )
- )
-
- testthat::expect_no_error(
- res_xgb_cv <- base_learner_fit(
- data = pstdt,
- learner = "xgboost",
- dependent_name = "pm2.5",
- independent_name = colnames(pstdt$stdt)[-1:-5],
- cv_mode = "lblo",
- blocks = c(25, 25),
- nrounds = 30L
- )
- )
-
-})
-
-testthat::test_that("base_learner_cv_outcome works", {
- withr::local_package("sf")
- withr::local_package("terra")
- withr::local_package("data.table")
- withr::local_options(list(sf_use_s2 = FALSE))
-
- nsp <- 50L
- nt <- 60L
- np <- 10L
-
- # random data with 50 (spatial)*60 (temporal)*4 (covariates)
- sphere <-
- terra::ext(c(xmin = 0, xmax = 100, ymin = 0, ymax = 80))
- pnts <- terra::spatSample(sphere, nsp, lonlat = FALSE, as.points = TRUE)
- pst <- split(seq(1, nt), seq(1, nt))
- pst <- lapply(pst,
- function(x) {
- pnts$pid <- seq(1, nsp)
- return(pnts)})
- pst <- Reduce(rbind, pst)
- pst$time <- rep(as.Date("2024-01-19") + seq(0, nt - 1), each = nsp)
- pst$pm2.5 <- rgamma(nsp * nt, 32, 1.6)
- pst$no2 <- rgamma(nsp * nt, 32, 1.6)
-
- pstx <- rgamma(nsp * nt * np, 1, 0.002)
- pstx <- matrix(pstx, nrow = nsp * nt, ncol = np)
- pstx <- as.data.frame(pstx)
- colnames(pstx) <- sprintf("X%d", seq(1, np))
- pst <- cbind(pst, pstx)
-
- pstdt <- convert_stobj_to_stdt(pst)
- colindx <-
- grep(
- paste0("(", paste(sprintf("X%d", seq(1, np)), collapse = "|"), ")"),
- colnames(pstdt$stdt))
-
- # generate cv index
- stcv <-
- generate_cv_index(
- covars = pstdt,
- cv_mode = "lblo",
- blocks = c(30, 30)
- )
-
- ## univariate case
- testthat::expect_no_error(
- stdt_cvo1 <-
- base_learner_cv_outcome(
- data = pstdt,
- cv_index = stcv,
- dependent_name = c("pm2.5")
- )
- )
-
- testthat::expect_true(class(stdt_cvo1)[1] == "list")
- testthat::expect_true(all(sapply(stdt_cvo1, is.numeric)))
-
- ## multivariate case
- testthat::expect_no_error(
- stdt_cvo <-
- base_learner_cv_outcome(
- data = pstdt,
- cv_index = stcv,
- dependent_name = c("pm2.5", "no2")
- )
- )
-
- testthat::expect_true(class(stdt_cvo)[1] == "list")
- testthat::expect_true(all(sapply(stdt_cvo, is.data.frame)))
-})
-
-
-### ------ test end ------
diff --git a/tests/testthat/test-calculate_covariates.R b/tests/testthat/test-calculate_covariates.R
index 6f85288c..59f6b8b2 100644
--- a/tests/testthat/test-calculate_covariates.R
+++ b/tests/testthat/test-calculate_covariates.R
@@ -282,16 +282,18 @@ testthat::test_that("Check extract_nlcd_ratio works", {
withr::local_package("exactextractr")
withr::local_package("spData")
- point_wake <- cbind(lon = -78.72, lat = 35.71, dem = 20)
- point_durham <- cbind(lon = -78.86, lat = 36.06, dem = 15)
+ point_us1 <- cbind(lon = -114.7, lat = 38.9, dem = 40)
+ point_us2 <- cbind(lon = -114, lat = 39, dem = 15)
+ point_ak <- cbind(lon = -155.997, lat = 69.3884, dem = 100) # alaska
point_fr <- cbind(lon = 2.957, lat = 43.976, dem = 15) # france
- eg_data <- rbind(point_wake, point_durham, point_fr) |>
+ eg_data <- rbind(point_us1, point_us2, point_ak, point_fr) |>
as.data.frame() |>
terra::vect(crs = "EPSG:4326")
path_testdata <-
testthat::test_path(
- "../testdata/raw/nlcd/"
+ "..",
+ "testdata"
)
# CHECK INPUT (error message)
# -- buf_radius is numeric
@@ -315,7 +317,7 @@ testthat::test_that("Check extract_nlcd_ratio works", {
path = path_testdata),
"year is not a numeric."
)
- # -- year has not likely value
+ # -- year has likely value
testthat::expect_error(
calc_nlcd_ratio(sites = eg_data,
year = 20192,
@@ -369,16 +371,16 @@ testthat::test_that("Check extract_nlcd_ratio works", {
testthat::expect_equal(class(output)[1], "SpatVector")
# -- crs is the same than input
testthat::expect_true(terra::same.crs(eg_data, output))
- # -- out-of-mainland-US points removed (France)
+ # -- out-of-mainland-US points removed (France and Alaska)
testthat::expect_equal(nrow(output), 2)
# -- initial names are still in the output SpatVector
testthat::expect_true(all(names(eg_data) %in% names(output)))
# -- check the value of some of the points in the US
testthat::expect_equal(
- output$LDU_EFO_0_03000_2021[1], 0.1266191, tolerance = 1e-7
+ output$LDU_EFO_0_03000_2021[1], 0.7940682, tolerance = 1e-7
)
testthat::expect_equal(
- output$LDU_SHB_0_03000_2021[2], 0.01309295, tolerance = 1e-7
+ output$LDU_SHB_0_03000_2021[2], 0.9987249, tolerance = 1e-7
)
# -- class fraction rows should sum to 1
testthat::expect_equal(
diff --git a/tests/testthat/test-cross_validation.R b/tests/testthat/test-cross_validation.R
deleted file mode 100644
index 84881210..00000000
--- a/tests/testthat/test-cross_validation.R
+++ /dev/null
@@ -1,260 +0,0 @@
-#' @author Insang Song
-#' @title space-time cross-validation function tests
-
-testthat::test_that("leave-ones run without errors", {
- withr::local_package("sf")
- withr::local_package("sftime")
- withr::local_package("terra")
- withr::local_package("data.table")
- withr::local_package("dplyr")
- withr::local_options(list(sf_use_s2 = FALSE))
-
- set.seed(202311)
- nco <- sf::st_read("../testdata/test_nc_output.nc") |>
- unique()
- nco_s <- nco |>
- dplyr::sample_n(10)
- ncost <-
- split(x = seq(1, 10), f = seq(1, 10)) |>
- lapply(function(x) as.POSIXct("2022-11-30") + (x * 8.64e4)) |>
- lapply(function(x) mutate(nco_s, time = x)) |>
- Reduce(rbind, x = _)
- # to sftime
- ncost <-
- sftime::st_as_sftime(
- ncost,
- time_column_name = "time")
-
- expect_error(generate_cv_index(ncost, "lolo"))
- expect_error(generate_cv_index(ncost, "lolo", sp_fold = 3L))
-
- # convert to stdt
- ncostdt <- convert_stobj_to_stdt(ncost)
-
- ncostdte <- ncostdt
- ncostdte$stdt$sp_index <- seq(1, nrow(ncostdte$stdt))
- expect_no_error(sppre <- generate_spt_index(ncostdte, "spatial"))
- expect_no_error(spprest <- generate_spt_index(ncostdt, "spatiotemporal"))
-
- # spatial and temporal unique values
- slength <- nrow(nco_s)
- tlength <- length(unique(ncost$time))
- rlength <- nrow(ncost)
-
- # indices
- index_lolo <- generate_cv_index(ncostdt, "lolo")
- index_loto <- generate_cv_index(ncostdt, "loto")
- index_lolto <- generate_cv_index(ncostdt, "lolto")
-
- ncodto <- data.table::copy(ncostdt$stdt)
- ncodto$lolo <- index_lolo
- ncodto$loto <- index_loto
- ncodto$lolto <- index_lolto
-
- # type check
- testthat::expect_true(is.numeric(index_lolo))
- testthat::expect_true(is.numeric(index_loto))
- testthat::expect_true(is.numeric(index_lolto))
-
- testthat::expect_equal(max(index_lolo), slength)
- testthat::expect_equal(max(index_loto), tlength)
- testthat::expect_equal(max(index_lolto), rlength)
-
-}
-)
-
-
-testthat::test_that("leave-block-outs work as expected", {
- withr::local_package("sf")
- withr::local_package("sftime")
- withr::local_package("terra")
- withr::local_package("data.table")
- withr::local_package("dplyr")
- withr::local_options(list(sf_use_s2 = FALSE))
-
- set.seed(202311)
- nco <-
- sf::st_read(
- testthat::test_path("..", "testdata/test_nc_output.nc")
- ) |>
- unique()
- sf::st_crs(nco) <- "OGC:WGS84"
-
- # data preparation for larger stdt than the previous example
- ncost <-
- split(x = seq(1, 10), f = seq(1, 10)) |>
- lapply(function(x) as.POSIXct("2022-11-30") + (x * 8.64e4)) |>
- lapply(function(x) mutate(nco, time = x)) |>
- Reduce(rbind, x = _)
-
- ncost <-
- sftime::st_as_sftime(
- ncost,
- time_column_name = "time")
-
- # to stdt
- ncostdt <- convert_stobj_to_stdt(ncost)
-
- ncv_fold <- 5L
- ncv_sp <- 3L
- ncv_t <- 5L
-
- testthat::expect_no_error(
- index_lblo <- generate_cv_index(ncostdt, "lblo", cv_fold = ncv_fold)
- )
- testthat::expect_no_error(
- ret_spblock <- generate_block_sp_index(ncostdt, cv_fold = ncv_fold)
- )
- testthat::expect_no_error(
- index_lbto <- generate_cv_index(ncostdt, "lbto", cv_fold = ncv_fold)
- )
- testthat::expect_no_error(
- index_lblto <-
- generate_cv_index(
- ncostdt,
- "lblto",
- sp_fold = ncv_sp,
- t_fold = ncv_t)
- )
-
- # max value test
- testthat::expect_equal(max(index_lblo), ncv_fold)
- testthat::expect_equal(max(index_lbto), ncv_fold)
- testthat::expect_equal(max(index_lblto), ncv_sp * ncv_t)
-
- # raw coordinate based lblo should have attributes
- testthat::expect_identical(class(ret_spblock), c("list", "stdt"))
- testthat::expect_true(!is.null(attr(ret_spblock, "kmeans_centers")))
- testthat::expect_true(!is.null(attr(ret_spblock, "kmeans_sizes")))
-
- # external block data is present
- eco4 <-
- readRDS(
- testthat::test_path("..", "testdata/ecoregion_lv4.rds")
- )
- suppressWarnings(sf::st_crs(eco4) <- "OGC:WGS84")
- eco4d <- eco4 |>
- dplyr::group_by(US_L3CODE, US_L3NAME) |>
- dplyr::summarize(nsubregions = n()) |>
- ungroup()
- neco4 <- nrow(eco4d)
-
-
- ncodt <- data.table::copy(ncostdt$stdt)
- # when sf input is entered
- index_lblo_sf <-
- generate_cv_index(
- ncostdt, "lblo",
- blocks = eco4d,
- block_id = "US_L3NAME")
- # error case: blocks are sf, but no block_id
- testthat::expect_error(
- generate_cv_index(
- ncostdt, "lblo",
- blocks = eco4d,
- block_id = NULL)
- )
-
- # when SpatVector input is entered
- index_lblo_tr <-
- generate_cv_index(
- ncostdt, "lblo",
- blocks = terra::vect(eco4d),
- block_id = "US_L3NAME")
-
- # no block_id
- expect_error(
- generate_cv_index(
- ncostdt, "lblo",
- blocks = terra::vect(eco4d),
- block_id = "USS")
- )
-
-
- # If returned index is character, convert to integer via factor
- if (any(is.factor(index_lblo_sf), is.factor(index_lblo_tr))) {
- index_lblo_sfn <- as.numeric(index_lblo_sf)
- index_lblo_trn <- as.numeric(index_lblo_tr)
- }
-
- # worked as expected with the external blocks reference
- testthat::expect_true(is.numeric(index_lblo_sfn))
- testthat::expect_true(is.numeric(index_lblo_trn))
- testthat::expect_equal(max(index_lblo_sfn), neco4)
- testthat::expect_equal(max(index_lblo_trn), neco4)
-
- # using sf or SpatVector should return the same result
- testthat::expect_true(all.equal(index_lblo_sf, index_lblo_tr))
-
- # intentional duplicates in block name
- eco4e <- eco4d
- eco4e$US_L3NAME[2] <- eco4e$US_L3NAME[1]
- testthat::expect_error(
- generate_cv_index(
- ncostdt, "lblo",
- blocks = terra::vect(eco4e),
- block_id = "US_L3NAME")
- )
-
- # numeric block case
- testthat::expect_no_error(
- generate_cv_index(
- ncostdt, "lblo",
- blocks = c(0.5, 0.5)
- )
- )
-
- # lblo error case
- testthat::expect_error(
- generate_cv_index(ncostdt, "lblo", cv_fold = NULL, blocks = NULL)
- )
-
- # lbto error cases
- testthat::expect_error(
- generate_cv_index(ncostdt, "lbto", NULL)
- )
- testthat::expect_error(
- generate_cv_index(ncostdt, "lbto", 20L)
- )
-}
-)
-
-
-testthat::test_that("random cross-validation abides", {
- withr::local_package("sf")
- withr::local_package("sftime")
- withr::local_package("terra")
- withr::local_package("data.table")
- withr::local_package("dplyr")
- withr::local_options(list(sf_use_s2 = FALSE))
-
- nco <- sf::st_read("../testdata/test_nc_output.nc") |>
- unique()
-
- # data preparation for larger stdt than the previous example
- ncost <-
- split(x = seq(1, 10), f = seq(1, 10)) |>
- lapply(function(x) as.POSIXct("2022-11-30") + (x * 8.64e4)) |>
- lapply(function(x) mutate(nco, time = x)) |>
- Reduce(rbind, x = _)
-
- ncost <-
- sftime::st_as_sftime(
- ncost,
- time_column_name = "time")
-
- # to stdt
- ncostdt <- convert_stobj_to_stdt(ncost)
- ncv_fold <- 10L
- testthat::expect_no_error(
- index_random <- generate_cv_index(ncostdt, "random", cv_fold = ncv_fold)
- )
-
- testthat::expect_true(is.numeric(index_random))
- testthat::expect_equal(max(index_random), ncv_fold)
-
- # error case
- testthat::expect_error(
- generate_cv_index(ncostdt, "random", NULL)
- )
-})
diff --git a/tests/testthat/test-download_functions.R b/tests/testthat/test-download_functions.R
deleted file mode 100644
index 6ba42fa2..00000000
--- a/tests/testthat/test-download_functions.R
+++ /dev/null
@@ -1,1122 +0,0 @@
-#' @author Mitchell Manware
-#' @description Unit test for for checking data download functions.
-testthat::test_that("Error when data_download_acknowledgement = FALSE", {
- download_datasets <- c("aqs", "ecoregion", "geos", "gmted", "koppen",
- "koppengeiger", "merra2", "merra", "narr_monolevel",
- "narr_p_levels", "nlcd", "noaa", "sedac_groads",
- "sedac_population", "groads", "population", "plevels",
- "p_levels", "monolevel", "hms", "smoke")
- for (d in seq_along(download_datasets)) {
- expect_error(
- download_data(dataset_name = download_datasets[d],
- data_download_acknowledgement = FALSE),
- paste0("Please refer to the argument list and ",
- "the error message above to rectify the error.\n")
- )
- }
-})
-
-testthat::test_that("Error when one parameter is NULL.", {
- download_datasets <- c("aqs", "ecoregion", "geos", "gmted", "koppen",
- "koppengeiger", "merra2", "merra", "narr_monolevel",
- "narr_p_levels", "nlcd", "noaa", "sedac_groads",
- "sedac_population", "groads", "population", "plevels",
- "p_levels", "monolevel", "hms", "smoke")
- for (d in seq_along(download_datasets)) {
- expect_error(
- download_data(dataset_name = download_datasets[d],
- data_download_acknowledgement = TRUE,
- directory_to_save = NULL),
- paste0("Please refer to the argument list and ",
- "the error message above to rectify the error.\n")
- )
- }
-})
-
-testthat::test_that("Errors when temporal ranges invalid.", {
- expect_error(
- download_geos_cf_data(
- date_start = "1900-01-01",
- collection = "aqc_tavg_1hr_g1440x721_v1",
- data_download_acknowledgement = TRUE,
- directory_to_save = "../testdata"
- )
- )
- expect_error(
- download_aqs_data(
- year_start = 1900,
- data_download_acknowledgement = TRUE,
- directory_to_save = "../testdata"
- )
- )
- expect_error(
- download_narr_monolevel_data(
- year_start = 1900,
- collection = "air.sfc",
- data_download_acknowledgement = TRUE,
- directory_to_save = "../testdata"
- )
- )
- expect_error(
- download_narr_p_levels_data(
- year_start = 1900,
- collection = "omega",
- data_download_acknowledgement = TRUE,
- directory_to_save = "../testdata"
- )
- )
- expect_error(
- download_merra2_data(
- date_start = "1900-01-01",
- collection = "inst1_2d_asm_Nx",
- directory_to_save = "../testdata",
- data_download_acknowledgement = TRUE
- )
- )
- expect_error(
- download_noaa_hms_smoke_data(
- date_start = "1900-01-01",
- directory_to_save = "../testdata",
- directory_to_download = "../testdata",
- data_download_acknowledgement = TRUE
- )
- )
-})
-
-testthat::test_that("EPA AQS download URLs have HTTP status 200.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- year_start <- 2018
- year_end <- 2022
- resolution_temporal <- "daily"
- parameter_code <- 88101
- directory_to_download <- testthat::test_path("..", "testdata/")
- directory_to_save <- testthat::test_path("..", "testdata/")
- # run download function
- download_data(dataset_name = "aqs",
- year_start = year_start,
- year_end = year_end,
- directory_to_save = directory_to_save,
- directory_to_download = directory_to_download,
- data_download_acknowledgement = TRUE,
- unzip = FALSE,
- remove_zip = FALSE,
- download = FALSE,
- remove_command = FALSE)
- # define file path with commands
- commands_path <-
- paste0(
- directory_to_download,
- "aqs_",
- parameter_code,
- "_",
- year_start, "_", year_end,
- "_",
- resolution_temporal,
- "_curl_commands.txt")
- # import commands
- commands <- read_commands(commands_path = commands_path)
- # extract urls
- urls <- extract_urls(commands = commands, position = 2)
- # check HTTP URL status
- url_status <- check_urls(urls = urls, size = length(urls), method = "HEAD")
- # implement unit tets
- test_download_functions(directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
-})
-
-
-testthat::test_that("Ecoregion download URLs have HTTP status 200.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- directory_to_download <- testthat::test_path("..", "testdata/")
- directory_to_save <- testthat::test_path("..", "testdata/")
- certificate <- system.file("extdata/cacert_gaftp_epa.pem",
- package = "beethoven")
- # run download function
- download_data(dataset_name = "ecoregion",
- directory_to_save = directory_to_save,
- directory_to_download = directory_to_download,
- data_download_acknowledgement = TRUE,
- unzip = FALSE,
- remove_zip = FALSE,
- download = FALSE,
- remove_command = FALSE,
- epa_certificate_path = certificate)
- # define file path with commands
- commands_path <- paste0(
- directory_to_download,
- "us_eco_l3_state_boundaries_",
- Sys.Date(),
- "_wget_command.txt"
- )
- # import commands
- commands <- read_commands(commands_path = commands_path)
- # extract urls
- urls <- extract_urls(commands = commands, position = 3)
- # check HTTP URL status
- url_status <-
- httr::HEAD(urls, config = httr::config(cainfo = certificate))
- url_status <- url_status$status_code
- # implement unit tets
- test_download_functions(directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
-})
-
-testthat::test_that("GEOS-CF download URLs have HTTP status 200.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- date_start <- "2019-09-09"
- date_end <- "2019-09-21"
- collections <- c("aqc_tavg_1hr_g1440x721_v1",
- "chm_inst_1hr_g1440x721_p23")
- directory_to_save <- testthat::test_path("..", "testdata/")
- for (c in seq_along(collections)) {
- # run download function
- download_data(dataset_name = "geos",
- date_start = date_start,
- date_end = date_end,
- collection = collections[c],
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE)
- # define file path with commands
- commands_path <- paste0(directory_to_save,
- collections[c],
- "_",
- date_start,
- "_",
- date_end,
- "_wget_commands.txt")
- # import commands
- commands <- read_commands(commands_path = commands_path)
- # extract urls
- urls <- extract_urls(commands = commands, position = 2)
- # check HTTP URL status
- url_status <- check_urls(urls = urls, size = 20L, method = "HEAD")
- # implement unit tests
- test_download_functions(directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
- }
-})
-
-testthat::test_that("GMTED download URLs have HTTP status 200.", {
- withr::local_package("httr")
- # function parameters
- statistics <- c("Breakline Emphasis",
- # "Systematic Subsample",
- # "Median Statistic", "Minimum Statistic",
- # "Mean Statistic", "Maximum Statistic",
- "Standard Deviation Statistic")
- resolution <- "7.5 arc-seconds"
- directory_to_download <- testthat::test_path("..", "testdata/")
- directory_to_save <- testthat::test_path("..", "testdata/")
- for (s in seq_along(statistics)) {
- # run download function
- download_data(dataset_name = "gmted",
- statistic = statistics[s],
- resolution = resolution,
- directory_to_download = directory_to_download,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- unzip = FALSE,
- remove_zip = FALSE,
- download = FALSE)
- # define file path with commands
- commands_path <- paste0(directory_to_download,
- "gmted_",
- gsub(" ", "", statistics[s]),
- "_",
- gsub(" ", "", resolution),
- "_",
- Sys.Date(),
- "_curl_command.txt")
- # import commands
- commands <- read_commands(commands_path = commands_path)
- # extract urls
- urls <- extract_urls(commands = commands, position = 6)
- # check HTTP URL status
- url_status <- check_urls(urls = urls, size = 1L, method = "HEAD")
- # implement unit tests
- test_download_functions(directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
- }
-})
-
-testthat::test_that("MERRA2 download URLs have HTTP status 200.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- date_start <- "2022-02-14"
- date_end <- "2022-03-08"
- collections <- c("inst1_2d_asm_Nx", "inst3_3d_asm_Np")
- directory_to_save <- testthat::test_path("..", "testdata/")
- for (c in seq_along(collections)) {
- # run download function
- download_data(dataset_name = "merra2",
- date_start = date_start,
- date_end = date_end,
- collection = collections[c],
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE)
- # define path with commands
- commands_path <- paste0(directory_to_save,
- collections[c],
- "_",
- date_start,
- "_",
- date_end,
- "_wget_commands.txt")
- # import commands
- commands <- read_commands(commands_path = commands_path)
- # extract urls
- urls <- extract_urls(commands = commands, position = 2)
- # check HTTP URL status
- url_status <- check_urls(urls = urls, size = 30L, method = "HEAD")
- # implement unit tests
- test_download_functions(directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
- }
-})
-
-testthat::test_that("MERRA2 returns message with unrecognized collection.", {
- # function parameters
- collections <- "uNrEcOgNiZeD"
- directory_to_save <- testthat::test_path("..", "testdata/")
- testthat::expect_error(
- download_data(
- dataset_name = "merra",
- collection = collections,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE
- )
- )
-})
-
-testthat::test_that("NARR monolevel download URLs have HTTP status 200.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- year_start <- 2018
- year_end <- 2018
- variables <- c("weasd", "air.2m")
- directory_to_save <- testthat::test_path("..", "testdata/")
- # run download function
- download_data(dataset_name = "narr_monolevel",
- year_start = year_start,
- year_end = year_end,
- variables = variables,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE)
- # define path with commands
- commands_path <- paste0(directory_to_save,
- "narr_monolevel_",
- year_start, "_", year_end,
- "_curl_commands.txt")
- # import commands
- commands <- read_commands(commands_path = commands_path)
- # extract urls
- urls <- extract_urls(commands = commands, position = 6)
- # check HTTP URL status
- url_status <- check_urls(urls = urls, size = 5L, method = "HEAD")
- # implement unit tests
- test_download_functions(directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
-})
-
-testthat::test_that("NARR p-levels download URLs have HTTP status 200.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- year_start <- 2020
- year_end <- 2021
- variables <- c("shum", "omega")
- directory_to_save <- testthat::test_path("..", "testdata/")
- # run download function
- download_data(dataset_name = "narr_p_levels",
- year_start = year_start,
- year_end = year_end,
- variables = variables,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE)
- # define file path with commands
- commands_path <- paste0(directory_to_save,
- "narr_p_levels_",
- year_start, "_", year_end,
- "_curl_commands.txt")
- # import commands
- commands <- read_commands(commands_path = commands_path)
- # extract urls
- urls <- extract_urls(commands = commands, position = 6)
- # check HTTP URL status
- url_status <- check_urls(urls = urls, size = 20L, method = "HEAD")
- # implement unit tests
- test_download_functions(directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
-})
-
-testthat::test_that("NOAA HMS Smoke download URLs have HTTP status 200.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- date_start <- "2022-08-12"
- date_end <- "2022-09-21"
- directory_to_download <- testthat::test_path("..", "testdata/")
- directory_to_save <- testthat::test_path("..", "testdata/")
- # run download function
- download_data(dataset_name = "smoke",
- date_start = date_start,
- date_end = date_end,
- directory_to_download = directory_to_download,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- remove_command = FALSE,
- unzip = FALSE,
- remove_zip = FALSE)
- # define file path with commands
- commands_path <- paste0(directory_to_download,
- "hms_smoke_",
- gsub("-", "", date_start),
- "_",
- gsub("-", "", date_end),
- "_curl_commands.txt")
- # import commands
- commands <- read_commands(commands_path = commands_path)
- # extract urls
- urls <- extract_urls(commands = commands, position = 6)
- # check HTTP URL status
- url_status <- check_urls(urls = urls, size = 30L, method = "HEAD")
- # implement unit tests
- test_download_functions(directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
-})
-
-testthat::test_that("NLCD download URLs have HTTP status 200.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- years <- c(2021, 2019, 2016)
- collections <- c(rep("Coterminous United States", 2), "Alaska")
- collection_codes <- c(rep("l48", 2), "ak")
- directory_to_download <- testthat::test_path("..", "testdata/")
- directory_to_save <- testthat::test_path("..", "testdata/")
- # run download function
- for (y in seq_along(years)) {
- download_data(dataset_name = "nlcd",
- year = years[y],
- collection = collections[y],
- directory_to_download = directory_to_download,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- remove_command = FALSE,
- unzip = FALSE,
- remove_zip = FALSE)
- # define file path with commands
- commands_path <- paste0(directory_to_download,
- "nlcd_",
- years[y],
- "_land_cover_",
- collection_codes[y],
- "_",
- Sys.Date(),
- "_curl_command.txt")
- # import commands
- commands <- read_commands(commands_path = commands_path)
- # extract urls
- urls <- extract_urls(commands = commands, position = 5)
- # check HTTP URL status
- url_status <- check_urls(urls = urls, size = 1L, method = "HEAD")
- # implement unit tests
- test_download_functions(directory_to_download = directory_to_download,
- directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
- }
- testthat::expect_error(
- download_data(dataset_name = "nlcd",
- year = 2000,
- collection = "Coterminous United States",
- directory_to_download = directory_to_download,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- remove_command = TRUE,
- unzip = FALSE,
- remove_zip = FALSE)
- )
-
-})
-
-testthat::test_that("SEDAC groads download URLs have HTTP status 200.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- data_regions <- c("Americas", "Global")
- data_formats <- c("Geodatabase", "Shapefile")
- directory_to_download <- testthat::test_path("..", "testdata/")
- directory_to_save <- testthat::test_path("..", "testdata/")
- # run download function
- for (r in seq_along(data_regions)) {
- data_region <- data_regions[r]
- for (f in seq_along(data_formats)) {
- download_data(dataset_name = "sedac_groads",
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- data_format = data_formats[f],
- data_region = data_region,
- directory_to_download = directory_to_download,
- download = FALSE,
- unzip = FALSE,
- remove_zip = FALSE,
- remove_command = FALSE)
- # define file path with commands
- commands_path <- paste0(directory_to_download,
- "sedac_groads_",
- gsub(" ", "_", tolower(data_region)),
- "_",
- Sys.Date(),
- "_curl_command.txt")
- # import commands
- commands <- read_commands(commands_path = commands_path)
- # extract urls
- urls <- extract_urls(commands = commands, position = 11)
- # check HTTP URL status
- url_status <- check_urls(urls = urls, size = 1L, method = "GET")
- # implement unit tests
- test_download_functions(directory_to_download = directory_to_download,
- directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
- }
- }
-
- testthat::expect_message(
- download_data(dataset_name = "sedac_groads",
- data_format = "Shapefile",
- data_region = "Global",
- directory_to_download = directory_to_download,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- unzip = FALSE,
- remove_zip = FALSE,
- remove_command = TRUE)
- )
-})
-
-testthat::test_that("SEDAC population download URLs have HTTP status 200.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- years <- c("2020")
- data_formats <- c("GeoTIFF")
- data_resolutions <- cbind(c("30 second"),
- c("30_sec"))
- directory_to_download <- testthat::test_path("..", "testdata/")
- directory_to_save <- testthat::test_path("..", "testdata/")
- for (f in seq_along(data_formats)) {
- data_format <- data_formats[f]
- for (y in seq_along(years)) {
- year <- years[y]
- for (r in seq_len(nrow(data_resolutions))) {
- # run download function
- download_data(dataset_name = "sedac_population",
- year = year,
- data_format = data_format,
- data_resolution = data_resolutions[r, 1],
- directory_to_download = directory_to_download,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- unzip = FALSE,
- remove_zip = FALSE,
- remove_command = FALSE)
- # define file path with commands
- if (year == "all") {
- year <- "totpop"
- } else {
- year <- year
- }
- if (year == "totpop" && data_resolutions[r, 2] == "30_sec") {
- resolution <- "2pt5_min"
- } else {
- resolution <- data_resolutions[r, 2]
- }
- commands_path <- paste0(directory_to_download,
- "sedac_population_",
- year,
- "_",
- resolution,
- "_",
- Sys.Date(),
- "_curl_commands.txt")
- # import commands
- commands <- read_commands(commands_path = commands_path)
- # extract urls
- urls <- extract_urls(commands = commands, position = 11)
- # check HTTP URL status
- url_status <- check_urls(urls = urls, size = 1L, method = "GET")
- # implement unit tests
- test_download_functions(directory_to_download = directory_to_download,
- directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
- }
- }
- }
-})
-
-testthat::test_that("SEDAC population data types are coerced.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- year <- c("totpop")
- data_formats <- c("GeoTIFF", "ASCII", "netCDF")
- data_resolutions <- c("30 second", "2pt5_min")
- directory_to_download <- "../testdata/"
- directory_to_save <- "../testdata/"
- for (f in seq_along(data_formats)) {
- download_data(dataset_name = "sedac_population",
- year = year,
- data_format = data_formats[f],
- data_resolution = data_resolutions[1],
- directory_to_download = directory_to_download,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- unzip = FALSE,
- remove_zip = FALSE,
- remove_command = FALSE)
- commands_path <- paste0(directory_to_download,
- "sedac_population_",
- year,
- "_",
- data_resolutions[2],
- "_",
- Sys.Date(),
- "_curl_commands.txt")
- # import commands
- commands <- read_commands(commands_path = commands_path)
- # extract urls
- urls <- extract_urls(commands = commands, position = 11)
- # check HTTP URL status
- url_status <- check_urls(urls = urls, size = 1L, method = "GET")
- # implement unit tests
- test_download_functions(directory_to_download = directory_to_download,
- directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
- }
-})
-
-testthat::test_that("Koppen Geiger download URLs have HTTP status 200.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- time_periods <- c("Present", "Future")
- data_resolutions <- c("0.0083")
- directory_to_download <- testthat::test_path("..", "testdata/")
- directory_to_save <- testthat::test_path("..", "testdata/")
- # run download function
- for (p in seq_along(time_periods)) {
- time_period <- time_periods[p]
- for (d in seq_along(data_resolutions)) {
- download_data(dataset_name = "koppen",
- time_period = time_period,
- data_resolution = data_resolutions[d],
- directory_to_download = directory_to_download,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- unzip = FALSE,
- remove_zip = FALSE,
- download = FALSE,
- remove_command = FALSE)
- # define file path with commands
- commands_path <- paste0(directory_to_download,
- "koppen_geiger_",
- time_period,
- "_",
- gsub("\\.",
- "p",
- data_resolutions[d]),
- "_",
- Sys.Date(),
- "_wget_command.txt")
- # import commands
- commands <- read_commands(commands_path = commands_path)
- # extract urls
- urls <- extract_urls(commands = commands, position = 2)
- # check HTTP URL status
- url_status <- check_urls(urls = urls, size = 1L, method = "GET")
- # implement unit tests
- test_download_functions(directory_to_download = directory_to_download,
- directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
- }
- }
-})
-
-testthat::test_that("MODIS-MOD09GA download URLs have HTTP status 200.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- years <- 2020
- product <- "MOD09GA"
- version <- "61"
- horizontal_tiles <- c(12, 13)
- vertical_tiles <- c(5, 6)
- nasa_earth_data_token <- "tOkEnPlAcEhOlDeR"
- directory_to_save <- testthat::test_path("..", "testdata/")
- for (y in seq_along(years)) {
- date_start <- paste0(years[y], "-06-20")
- date_end <- paste0(years[y], "-06-24")
- # run download function
- download_data(dataset_name = "modis",
- date_start = date_start,
- date_end = date_end,
- product = product,
- version = version,
- horizontal_tiles = horizontal_tiles,
- vertical_tiles = vertical_tiles,
- nasa_earth_data_token = nasa_earth_data_token,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- remove_command = FALSE)
- # define file path with commands
- commands_path <- paste0(
- directory_to_save,
- product,
- "_",
- date_start,
- "_",
- date_end,
- "_wget_commands.txt"
- )
- # import commands
- commands <- read_commands(commands_path = commands_path)[, 2]
- # extract urls
- urls <- extract_urls(commands = commands, position = 4)
- # check HTTP URL status
- url_status <- check_urls(urls = urls, size = 10L, method = "HEAD")
- # implement unit tests
- test_download_functions(directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
- }
-})
-
-
-testthat::test_that("MODIS-MOD06L2 download URLs have HTTP status 200.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- product <- "MOD06_L2"
- version <- "61"
- date_start <- "2019-02-18"
- date_end <- "2019-02-18"
- nasa_earth_data_token <- "tOkEnPlAcEhOlDeR"
- horizontal_tiles <- c(8, 10)
- vertical_tiles <- c(4, 5)
- directory_to_save <- testthat::test_path("..", "testdata/")
-
- testthat::expect_error(
- kax <- download_data(dataset_name = "modis",
- date_start = date_start,
- date_end = date_end,
- product = product,
- version = version,
- horizontal_tiles = horizontal_tiles,
- vertical_tiles = vertical_tiles,
- nasa_earth_data_token = nasa_earth_data_token,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- mod06_links = NULL,
- remove_command = FALSE)
- )
- # link check
- tdir <- tempdir()
- faux_urls <-
- rbind(
- c(4387858920,
- "/archive/allData/61/MOD06_L2/2019/049/MOD06_L2.A2019049.0720.061.2019049194350.hdf",
- 28267915)
- )
-
- faux_urls <- data.frame(faux_urls)
- mod06_scenes <- paste0(tdir, "/mod06_example.csv")
- write.csv(faux_urls, mod06_scenes, row.names = FALSE)
-
- download_data(dataset_name = "modis",
- date_start = date_start,
- date_end = date_end,
- product = product,
- version = version,
- horizontal_tiles = horizontal_tiles,
- vertical_tiles = vertical_tiles,
- nasa_earth_data_token = nasa_earth_data_token,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- mod06_links = mod06_scenes,
- remove_command = FALSE)
-
- # define file path with commands
- commands_path <- paste0(
- directory_to_save,
- product,
- "_",
- date_start,
- "_",
- date_end,
- "_wget_commands.txt"
- )
- # import commands
- commands <- read_commands(commands_path = commands_path)[, 2]
- # extract urls
- urls <- extract_urls(commands = commands, position = 4)
- # check HTTP URL status
- url_status <- check_urls(urls = urls, size = 1L, method = "HEAD")
- # implement unit tests
- test_download_functions(directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
-})
-
-
-testthat::test_that("MODIS download error cases.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- years <- 2020
- product <- "MOD09GA"
- version <- "61"
- horizontal_tiles <- c(12, 13)
- vertical_tiles <- c(5, 6)
- nasa_earth_data_token <- "tOkEnPlAcEhOlDeR"
- directory_to_save <- testthat::test_path("..", "testdata/")
- date_start <- paste0(years, "-06-25")
- date_end <- paste0(years, "-06-30")
-
- # no token
- testthat::expect_no_error(
- download_data(dataset_name = "modis",
- date_start = date_start,
- date_end = date_end,
- product = product,
- version = version,
- horizontal_tiles = horizontal_tiles,
- vertical_tiles = vertical_tiles,
- nasa_earth_data_token = nasa_earth_data_token,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- remove_command = FALSE)
- )
-
- # no token
- testthat::expect_error(
- download_data(dataset_name = "modis",
- date_start = date_start,
- date_end = date_end,
- product = product,
- version = version,
- horizontal_tiles = horizontal_tiles,
- vertical_tiles = vertical_tiles,
- nasa_earth_data_token = NULL,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- remove_command = FALSE)
- )
-
- # year difference between date_start and date_end
- testthat::expect_error(
- download_data(dataset_name = "modis",
- date_start = date_start,
- date_end = "2024-03-28",
- product = "MOD11A1",
- version = version,
- horizontal_tiles = horizontal_tiles,
- vertical_tiles = vertical_tiles,
- nasa_earth_data_token = NULL,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- remove_command = FALSE)
- )
-
- # null version
- testthat::expect_error(
- download_data(dataset_name = "modis",
- date_start = date_start,
- date_end = date_end,
- product = product,
- version = NULL,
- horizontal_tiles = horizontal_tiles,
- vertical_tiles = vertical_tiles,
- nasa_earth_data_token = nasa_earth_data_token,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- remove_command = FALSE)
- )
-
- # invalid tile range (horizontal)
- testthat::expect_error(
- download_data(dataset_name = "modis",
- date_start = date_start,
- date_end = date_end,
- product = product,
- version = NULL,
- horizontal_tiles = c(-13, -3),
- vertical_tiles = vertical_tiles,
- nasa_earth_data_token = NULL,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- remove_command = FALSE)
- )
-
- # invalid tile range (horizontal)
- testthat::expect_error(
- download_data(dataset_name = "modis",
- date_start = date_start,
- date_end = date_end,
- product = product,
- version = NULL,
- horizontal_tiles = horizontal_tiles,
- vertical_tiles = c(100, 102),
- nasa_earth_data_token = NULL,
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- remove_command = FALSE)
- )
-
-
- # define file path with commands
- commands_path <- paste0(
- directory_to_save,
- product,
- "_",
- date_start,
- "_",
- date_end,
- "_wget_commands.txt"
- )
- # import commands
- commands <- read_commands(commands_path = commands_path)[, 2]
- # extract urls
- urls <- extract_urls(commands = commands, position = 4)
- # check HTTP URL status
- url_status <- check_urls(urls = urls, size = 10L, method = "HEAD")
- # implement unit tests
- test_download_functions(directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
-})
-
-
-
-testthat::test_that("EPA TRI download URLs have HTTP status 200.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- directory_to_save <- testthat::test_path("..", "testdata/")
- # run download function
- download_data(dataset_name = "tri",
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- remove_command = FALSE)
- year_start <- 2018L
- year_end <- 2022L
-
- # define file path with commands
- commands_path <- paste0(
- directory_to_save,
- "TRI_",
- year_start, "_", year_end,
- "_",
- Sys.Date(),
- "_curl_commands.txt"
- )
-
- # import commands
- commands <- read_commands(commands_path = commands_path)
- # extract urls
- urls <- extract_urls(commands = commands, position = 3)
- # check HTTP URL status
- url_status <- check_urls(urls = urls, size = 1L, method = "GET")
- # implement unit tests
- test_download_functions(directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
-})
-
-
-testthat::test_that("EPA NEI (AADT) download URLs have HTTP status 200.", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- directory_to_save <- testthat::test_path("..", "testdata/")
- certificate <- system.file("extdata/cacert_gaftp_epa.pem",
- package = "beethoven")
- # run download function
- year_target <- c(2017L, 2020L)
- download_data(dataset_name = "nei",
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- year_target = year_target,
- remove_command = FALSE,
- epa_certificate_path = certificate
- )
- # define file path with commands
- commands_path <- paste0(
- directory_to_save,
- "NEI_AADT_",
- paste(year_target, collapse = "-"),
- "_",
- Sys.Date(),
- "_wget_commands.txt"
- )
-
- # import commands
- commands <- read_commands(commands_path = commands_path)
- # extract urls
- urls <- extract_urls(commands = commands, position = 3)
- # check HTTP URL status
- url_status <-
- httr::HEAD(urls[1], config = httr::config(cainfo = certificate))
- url_status <- url_status$status_code
- # implement unit tests
- test_download_functions(directory_to_save = directory_to_save,
- commands_path = commands_path,
- url_status = url_status)
- # remove file with commands after test
- file.remove(commands_path)
-})
-
-testthat::test_that("Test error cases in EPA gaftp sources 1", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- tdir <- tempdir()
- directory_to_save <- testthat::test_path("..", "testdata/")
- certificate <- file.path(tdir, "cacert_gaftp_epa.pem")
- # run download function
- year_target <- c(2017L)
- testthat::expect_message(
- download_data(dataset_name = "nei",
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- year_target = year_target,
- remove_command = FALSE,
- epa_certificate_path = certificate
- )
- )
- # define file path with commands
- commands_path <- paste0(
- directory_to_save,
- "NEI_AADT_",
- paste(year_target, collapse = "-"),
- "_",
- Sys.Date(),
- "_wget_commands.txt"
- )
- # remove file with commands after test
- testthat::expect_true(file.exists(commands_path))
- file.remove(commands_path)
- file.remove(certificate)
- file.remove(sub("pem", "crt", certificate))
-})
-
-testthat::test_that("Test error cases in EPA gaftp sources 2", {
- withr::local_package("httr")
- withr::local_package("stringr")
- # function parameters
- tdir <- tempdir(check = TRUE)
- directory_to_save <- testthat::test_path("..", "testdata/")
- certificate <- file.path(tdir, "cacert_gaftp_epa.pem")
- # run download function
-
- testthat::expect_message(
- download_data(dataset_name = "ecoregion",
- directory_to_save = directory_to_save,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- remove_command = FALSE,
- directory_to_download = directory_to_save,
- epa_certificate_path = certificate
- )
- )
-
- # define file path with commands
- commands_path <- paste0(
- directory_to_save,
- "us_eco_l3_state_boundaries_",
- Sys.Date(),
- "_wget_command.txt"
- )
-
- # remove file with commands after test
- testthat::expect_true(file.exists(commands_path))
- file.remove(commands_path)
- file.remove(certificate)
- file.remove(sub("pem", "crt", certificate))
-})
diff --git a/tests/testthat/test-temporal_range.R b/tests/testthat/test-temporal_range.R
deleted file mode 100644
index 1685d716..00000000
--- a/tests/testthat/test-temporal_range.R
+++ /dev/null
@@ -1,28 +0,0 @@
-#' @author Mitchell Manware
-#' @description unit test for checking covariate, base learner and meta learner
-#' temporal range
-#'
-testthat::test_that("covariates are within temporal range.", {
- withr::local_package("terra")
- # use NCEP-NCAR Reanalysis 1 data as example
- paths <- list.files("../testdata/NCEP-NCAR-Reanalysis-1/",
- full.names = TRUE)
- iswithin <- NULL
- for (p in seq_along(paths)) {
- iswithin <- check_temporal_range(data = paths[p],
- data_type = "Raster")
- iswithin <- rbind(iswithin, iswithin)
- }
- # expect all elements equal true
- expect_equal(all(iswithin), TRUE)
-})
-
-testthat::test_that("test_nc_output.nc is within temporal range.", {
- withr::local_package("terra")
- # assumes that test_nc_output.nc is point data
- path <- "../testdata/test_nc_output.nc"
- iswithin <- check_temporal_range(data = path,
- data_type = "Point")
- # expect all elements equal true
- expect_equal(all(iswithin), TRUE)
-})
diff --git a/vignettes/cross_validation_types.Rmd b/vignettes/cross_validation_types.Rmd
deleted file mode 100644
index 51978d62..00000000
--- a/vignettes/cross_validation_types.Rmd
+++ /dev/null
@@ -1,198 +0,0 @@
----
-title: "Spatiotemporal cross validation types"
-output: rmarkdown::html_vignette
-vignette: >
- %\VignetteIndexEntry{Spatiotemporal cross validation types}
- %\VignetteEngine{knitr::rmarkdown}
- %\VignetteEncoding{UTF-8}
-date: "2024-01-22"
-author: "Insang Song"
----
-
-```{r init, echo = FALSE, include = FALSE}
-knitr::opts_chunk$set(
- warning = FALSE,
- message = FALSE,
- fig.width = 6,
- fig.height = 3.9,
- res = "retina"
-)
-pkgs <- c("data.table", "sf", "terra", "sftime", "dplyr", "ggplot2")
-invisible(sapply(pkgs, library, character.only = TRUE, quietly = TRUE))
-options(sf_use_s2 = FALSE)
-
-source("../R/manipulate_spacetime_data.R")
-source("../R/cross_validation.R")
-```
-
-```{r data-conversion}
-set.seed(202311)
-nco <- sf::st_read("../tests/testdata/test_nc_output.nc") |>
- unique()
-nco_s <- nco |>
- dplyr::sample_n(10)
-ncost <-
- split(x = seq(1, 10), f = seq(1, 10)) |>
- lapply(function(x) as.POSIXct("2022-11-30") + (x * 8.64e4)) |>
- lapply(function(x) mutate(nco_s, time = x)) |>
- Reduce(rbind, x = _)
-dim(ncost)
-
-ncost <-
- sftime::st_as_sftime(ncost,
- time_column_name = "time"
- )
-
-# to stdt
-ncostdt <- convert_stobj_to_stdt(ncost)
-
-# random-ordered sftime -> stdt: is ordering working well?
-ncost_dis <- ncost[sample(1:100, 100), ]
-ncostdtd <- convert_stobj_to_stdt(ncost_dis)
-
-all.equal(ncostdt$stdt, ncostdtd$stdt)
-```
-
-
-```{r lo-family}
-index_lolo <- generate_cv_index(ncostdt, "lolo")
-index_loto <- generate_cv_index(ncostdt, "loto")
-index_lolto <- generate_cv_index(ncostdt, "lolto")
-
-ncodto <- data.table::copy(ncostdtd$stdt)
-ncodto$lolo <- index_lolo
-ncodto$loto <- index_loto
-ncodto$lolto <- index_lolto
-
-ncodtol <- ncodto |>
- data.table::melt(
- id.vars = c("lon", "lat", "time"),
- measure.vars = c("lolo", "loto", "lolto")
- )
-ncodtol$value <- factor(ncodtol$value)
-
-ggplot(
- data = ncodtol[variable != "lolto", ],
- mapping = aes(x = lon, y = lat, color = value)
-) +
- geom_point() +
- labs(title = "Leave-one-location-out and Leave-one-time-out") +
- facet_grid(variable ~ time)
-
-ggplot(
- data = ncodtol[variable == "lolto", ],
- mapping = aes(x = lon, y = lat, color = value)
-) +
- geom_point() +
- labs(title = "Leave-one-location-time-out") +
- facet_wrap(~time)
-```
-
-
-```{r lb-family}
-# data preparation for larger stdt than the previous example
-ncost <-
- split(x = seq(1, 10), f = seq(1, 10)) |>
- lapply(function(x) as.POSIXct("2022-11-30") + (x * 8.64e4)) |>
- lapply(function(x) mutate(nco, time = x)) |>
- Reduce(rbind, x = _)
-dim(ncost)
-
-ncost <-
- sftime::st_as_sftime(
- ncost,
- time_column_name = "time"
- )
-
-# to stdt
-ncostdt <- convert_stobj_to_stdt(ncost)
-
-index_lblo <- generate_cv_index(ncostdt, "lblo", cv_fold = 5L)
-index_lbto <- generate_cv_index(ncostdt, "lbto", cv_fold = 5L)
-index_lblto <- generate_cv_index(ncostdt, "lblto", sp_fold = 3, t_fold = 5)
-```
-
-```{r lb-family-plot}
-ncodt <- data.table::copy(ncostdt$stdt)
-ncodt$lblo <- index_lblo
-ncodt$lbto <- index_lbto
-ncodt$lblto <- index_lblto
-
-ncodtl <- ncodt |>
- data.table::melt(
- id.vars = c("lon", "lat", "time"),
- measure.vars = c("lblo", "lbto", "lblto")
- )
-ncodtl$value <- factor(ncodtl$value)
-
-ggplot(
- data = ncodtl[variable != "lblto", ],
- mapping = aes(x = lon, y = lat, color = value)
-) +
- geom_point() +
- labs(title = "Leave-block-location-out and\nLeave-block-time-out") +
- facet_grid(variable ~ time)
-
-ggplot(
- data = ncodtl[variable == "lblto", ],
- mapping = aes(x = lon, y = lat, color = value)
-) +
- geom_point() +
- labs(title = "Leave-block-location-time-out") +
- facet_wrap(~time)
-```
-
-```{r lb-external}
-eco4 <- readRDS("../tests/testdata/ecoregion_lv4.rds")
-eco4d <- eco4 |>
- group_by(US_L3CODE, US_L3NAME) |>
- dplyr::summarize(nsubregions = n()) |>
- ungroup()
-
-
-ncodt <- data.table::copy(ncostdt$stdt)
-index_lblo_sf <-
- generate_cv_index(
- ncostdt,
- "lblo",
- blocks = eco4d,
- block_id = "US_L3NAME"
- )
-ncodt$lblo_bl <- factor(index_lblo_sf)
-
-index_lblo_tr <-
- generate_cv_index(
- ncostdt,
- "lblo",
- blocks = terra::vect(eco4d),
- block_id = "US_L3NAME"
- )
-ncodt$lblo_blt <- factor(index_lblo_tr)
-
-all.equal(index_lblo_sf, index_lblo_tr)
-
-ggplot(
- data = ncodt,
- mapping = aes(x = lon, y = lat, color = lblo_bl)
-) +
- geom_point() +
- labs(title = "Leave-block-location-out (Ecoregion level 3)") +
- facet_wrap(~time)
-
-# TODO: irregular temporal blocks
-```
-
-
-
-```{r random-family}
-index_random <- generate_cv_index(ncostdt, "random", cv_fold = 10L)
-ncodt$random <- factor(index_random)
-
-ggplot(
- data = ncodt,
- mapping = aes(x = lon, y = lat, color = random)
-) +
- geom_point() +
- labs(title = "Random") +
- facet_wrap(~time)
-```
diff --git a/vignettes/download_functions.Rmd b/vignettes/download_functions.Rmd
deleted file mode 100644
index 274bc25b..00000000
--- a/vignettes/download_functions.Rmd
+++ /dev/null
@@ -1,1062 +0,0 @@
----
-title: "download_data() and NASA EarthData Account"
-output: rmarkdown::html_vignette
-vignette: >
- %\VignetteIndexEntry{download_data() and NASA EarthData Account}
- %\VignetteEngine{knitr::rmarkdown}
- %\VignetteEncoding{UTF-8}
-date: "2024-01-18"
-author: "Mitchell Manware"
----
-
-```{r, echo = FALSE, warning = FALSE, message = FALSE}
-# packages
-library(knitr)
-library(testthat)
-# source functions
-sapply(
- list.files(
- "../R/",
- pattern = "download",
- full.names = TRUE
- ),
- source
-)
-```
-
-## Motivation
-
-The `data_download()` function was developed to improve researchers' access to publicly available geospatial data sets. Although the data are already available online, using a web browser to manually download hundreds or thousands of data files is slow, arduous, and not (efficiently) repeatable. Additionally, as users may not be familiar with creating download recipes in Bash (Unix shell), `data_download()` allows researchers to download data directly with R, a common coding language in the field of environmental health research. Finally, function-izing data downloads is useful for repeated code or automated analysis pipelines.
-
-## data_download()
-
-`data_download()` is capable of accessing and downloading geospatial data sets, collections, and variables from a variety of sources. This wrapper function calls on source-specific data download functions, each utilizing a unique combination of input parameters, host URL, naming convention, and data formats.
-
-```{r, echo = FALSE}
-functions <- c(
- "download_aqs_data", "download_ecoregion_data",
- "download_geos_cf_data", "download_gmted_data",
- "download_koppen_geiger_data", "download_merra2_data",
- "download_narr_monolevel_data", "download_narr_p_levels_data",
- "download_nlcd_data", "download_noaa_hms_smoke_data",
- "download_sedac_groads_data", "download_sedac_population_data",
- "download_modis_data"
-)
-source <- c(
- "US EPA Air Data Pre-Generated Data Files",
- "US EPA Ecoregions",
- "NASA Goddard Earth Observing System Composition Forecasting (GEOS-CF)",
- "USGS Global Multi-resolution Terrain Elevation Data (GMTED2010)",
- "Köppen-Geiger Climate Classification (Beck et al., 2018)",
- paste0(
- "NASA Modern-Era Retrospective analysis for Research and ",
- "Applications, Version 2 (MERRA-2)"
- ),
- "NOAA NCEP North American Regional Reanalysis (NARR) (monolevel variables)",
- paste0(
- "NOAA NCEP North American Regional Reanalysis (NARR) (pressure ",
- "levels variables)"
- ),
- "MRLC Consortium National Land Cover Database (NLCD)",
- "NOAA Hazard Mapping System Fire and Smoke Product",
- "NASA SEDAC Global Roads Open Access Data Set",
- "NASA SEDAC UN WPP-Adjusted Population Density",
- "NASA Moderate Resolution Imaging Spectroradiometer (MODIS)"
-)
-
-link <- c(
- "https://aqs.epa.gov/aqsweb/airdata/download_files.html",
- "https://www.epa.gov/eco-research/ecoregions",
- "https://gmao.gsfc.nasa.gov/GEOS_systems/",
- "https://www.usgs.gov/coastal-changes-and-impacts/gmted2010",
- "https://www.nature.com/articles/sdata2018214",
- "https://gmao.gsfc.nasa.gov/reanalysis/MERRA-2/",
- "https://psl.noaa.gov/data/gridded/data.narr.html",
- "https://psl.noaa.gov/data/gridded/data.narr.html",
- "https://www.mrlc.gov/data",
- "https://www.ospo.noaa.gov/Products/land/hms.html#0",
- paste0(
- "https://sedac.ciesin.columbia.edu/data/set/groads-",
- "global-roads-open-access-v1/data-download"
- ),
- paste0(
- "https://sedac.ciesin.columbia.edu/data/set/gpw-v4-",
- "population-density-adjusted-to-2015-unwpp-country-",
- "totals-rev11"
- ),
- "https://modis.gsfc.nasa.gov/data/"
-)
-source <- paste0(
- "[",
- source,
- "](",
- link,
- ")"
-)
-
-functions_sources <- data.frame(functions, source)
-colnames(functions_sources) <- c("Function", "Source")
-kable(functions_sources,
- caption =
- "Source-Specific Download Functions and Sources"
-)
-```
-
-It is important to note that `data_download()` calls a source-specific function based on the `dataset_name =` parameter. Using the source-specific function directly will return the exact same data (**if the parameters are the same**), but `data_download()` may be beneficial if using a `for` loop to download data from various sources. For example, `download_data(dataset_name = "hms", ...)` will return the same data as `download_noaa_hms_smoke(...)` assuming that `...` indicates the same parameters.
-
-### Parameters
-
-User-defined parameters differ based on the data source. Required parameters for each source can be checked with `names(formals())`.
-
-```{r}
-names(formals(download_noaa_hms_smoke_data))
-names(formals(download_narr_monolevel_data))
-```
-
-The two functions have different required parameters because `download_noaa_hms_smoke_data()` uses a daily temporal resolution while `download_narr_monolevel_data()` uses yearly, but they share some common, standard parameters.
-
-#### Standard parameters
-
-Four parameters are included in all of the data download functions.
-
-```{r, echo = FALSE}
-parameter <- c(
- "directory_to_save",
- "data_download_acknowledgement",
- "download", "remove_command"
-)
-type <- c("Character", "Logical", "Logical", "Logical")
-description <- c(
- paste0(
- "There must be a directory to save downloaded ",
- "data. Default = './input/DATASET_NAME/'."
- ),
- paste0(
- "User must acknowledge that downloading geospatial ",
- "data can be very lage and may use lots of machine ",
- "storage and memory."
- ),
- paste0(
- "Run or skip the data download. Utilized primarily ",
- "for unit tests (see [Unit Tests])."
- ),
- paste0(
- "Remove or retain the text file containing the ",
- "generated download commands. Utilized primarily ",
- "for unit tests (see [Unit Tests] and [4. Initiate ",
- "\"...commands.txt\"])."
- )
-)
-parameter_descriptions <- data.frame(parameter, type, description)
-colnames(parameter_descriptions) <- c("Parameter", "Type", "Description")
-kable(parameter_descriptions)
-```
-
-Additionally, the `dataset_name =` parameter must be specified when using `data_download()`, but is assumed when using a source-specific download function.
-
-### Function Structure
-
-Although each source-specific download function is unique, they all follow the same general structure. The following chunks of code have been **adopted** from `download_noaa_hms_smoke_data()` to demonstrate the functions' structure.
-
-[1. Clean Parameters]
-
-[2. Generate Download URLs]
-
-[3. Generate download file names]
-
-[4. Initiate "...commands.txt"]
-
-[5. Concatenate download commands]
-
-[6. Finalize "...commands.txt"]
-
-[7. Run commands in "...commands.txt"]
-
-[8. Zip files (if applicable)](#zip-files-if-applicable)
-
-#### 1. Clean parameters
-
-Cleaning the user-defined parameters is highly dependent on the parameters themselves and the desired URL to be created. A common parameter cleaning step is creating a date-time sequence based on a given temporal range and required format, in this case `YYYYMMDD`.
-
-```{r}
-# user defined parameters
-date_start <- "2023-12-28"
-date_end <- "2024-01-02"
-```
-
-```{r}
-date_sequence <- seq(
- as.Date(date_start, format = "%Y-%m-%d"),
- as.Date(date_end, format = "%Y-%m-%d"),
- "day"
-)
-date_sequence <- gsub("-", "", as.character(date_sequence))
-date_sequence
-```
-
-#### 2. Generate download URLs
-
-The URL base and pattern are identified by manually inspecting the download link on the source-specific web page. `download_noaa_hms_smoke_data()` utilizes the year, month, date, and data format to generate the download url.
-
-```{r}
-# user defined parameters
-data_format <- "Shapefile"
-suffix <- ".zip"
-```
-
-```{r}
-urls <- NULL
-for (d in seq_along(date_sequence)) {
- year <- substr(date_sequence[d], 1, 4)
- month <- substr(date_sequence[d], 5, 6)
- base <- "https://satepsanone.nesdis.noaa.gov/pub/FIRE/web/HMS/Smoke_Polygons/"
- url <- paste0(
- base,
- data_format,
- "/",
- year,
- "/",
- month,
- "/hms_smoke",
- date_sequence[d],
- suffix
- )
- urls <- c(urls, url)
-}
-urls
-```
-
-A download URL is created for each date in `date_sequence` based on the fixed pattern.
-
-#### 3. Generate download file names
-
-The generation of download file names also follows a fixed pattern, typically a combination of the user-defined download directory, data set name, spatiotemporal characteristic, data type, and, if applicable, specific variable name. Unlike the download URLs, the download file names can be defined in any way by the writer of the function, but using the previously defined characteristics is useful for identification.
-
-```{r}
-# user defined parameters
-directory_to_download <- "./data/"
-```
-
-```{r}
-download_file_names <- NULL
-for (d in seq_along(date_sequence)) {
- download_file_name <- paste0(
- directory_to_download,
- "hms_smoke_",
- data_format,
- "_",
- date_sequence[d],
- suffix
- )
- download_file_names <- c(download_file_names, download_file_name)
-}
-download_file_names
-```
-
-A download URL is created for each date in `date_sequence` based on the fixed pattern.
-
-#### 4. Initiate "...commands.txt"
-
-An important aspect of the data download function is its `sink()...cat()...sink()` structure. Rather than using the `utils::download.file()` function, a text file is created to store all of the download commands generated from the URLs and file names.
-
-This structure is utilized for several reasons:
-
-- Consistent structure for all the source-specific download functions.
-
-- The `download.file()` function cannot accept vectors of URLs and destination files for downloading. An additional `for` loop to download data will increase function complexity and may reduce performance.
-
-- Writing commands in Bash (Unix shell) script allows for specific arguments and flags.
-
-- Storing the download URLs without immediately running the download allows for unit testing and URL checking (more on this in [Unit Tests]).
-
-The text file containing the download commands is named based on the data set, temporal range, and data transfer method.
-
-```{r}
-commands_txt <- paste0(
- directory_to_download,
- "hms_smoke_",
- head(date_sequence, n = 1),
- "_",
- tail(date_sequence, n = 1),
- "_curl_commands.txt"
-)
-```
-
-Create and sink the text file.
-
-```{r, eval = FALSE}
-sink(commands_txt)
-```
-
-#### 5. Concatenate download commands
-
-The Linux-based download commands are written according to the data transfer method, download URL, download file name, and additional arguments. Which additional arguments are included, and their order, depend on the data transfer method and URL type.
-
-For more information on `curl` and `wget`, the two data transfer methods utilized by the data download functions, see [curl.1 the man page](https://curl.se/docs/manpage.html) and [GNU Wget 1.21.1-dirty Manual](https://www.gnu.org/software/wget/manual/wget.html) (latest version as of January 8, 2024).
-
-The `cat()` function will store each of the download commands written in the `for` loop to the previously sunk commands text file (`commands_txt`).
-
-```{r, eval = FALSE}
-for (d in seq_along(date_sequence)) {
- download_comamnd <- paste0(
- "curl -s -o ",
- download_file_names[d],
- " --url ",
- urls[d],
- "\n"
- )
- cat(download_comamnd)
-}
-```
-
-#### 6. Finalize "...commands.txt"
-
-After the download commands have been concatenated to the commands text file, a second `sink()` command is run to finalize the file and stop the appending of R output.
-
-```{r, eval = FALSE}
-sink()
-```
-
-#### 7. Run commands in "...commands.txt"
-
-A "system command" must be created to run all of the download commands stored in the commands text file. In bash script, `.` indicates to run all of the commands within a given script. In this case, we will run all of the commands within the commands text file.
-
-```{r}
-system_command <- paste0(
- ". ",
- commands_txt,
- "\n"
-)
-system_command
-```
-
-Running the `system_command` deploys a "helper function", `download_run()`, a function created to reduce repeated code across the source-specific download functions. The function takes two parameters, `system_command =`, which indicates the command to be run, and `download =`, a user-defined logical parameter.
-
-```{r}
-download_run <- function(
- download = FALSE,
- system_command = NULL) {
- if (download == TRUE) {
- cat(paste0("Downloading requested files...\n"))
- system(command = system_command)
- cat(paste0("Requested files have been downloaded.\n"))
- } else {
- cat(paste0("Skipping data download.\n"))
- return(NULL)
- }
-}
-```
-
-The data download is initiated by running `download_run()` with the system command identified and `download = TRUE`.
-
-```{r, eval = FALSE}
-download_run(
- download = TRUE,
- system_command = system_command
-)
-```
-
-Checking the download directory shows that all of the requested files have been downloaded.
-
-```{r, eval = FALSE}
-list.files(path = directory_to_download)
-```
-
-```{r, echo = FALSE}
-paste0("hms_smoke_Shapefile_", date_sequence, ".zip")
-```
-
-
-#### 8. Zip files (if applicable) {#zip-files-if-applicable}
-
-All of the source-specific data download functions follow this general pattern, but those functions which download zip files require additional steps to inflate and remove the downloaded zip files, if desired. Each of these two steps are run by helper functions, and they are run by the user-defined `unzip = ` and `remove_zip = ` parameters in `data_download()`.
-
-`download_unzip()` inflates zip files if `unzip = TRUE`, and skips inflation if `unzip = FALSE`.
-
-```{r}
-download_unzip <-
- function(file_name,
- directory_to_unzip,
- unzip = TRUE) {
- if (!unzip) {
- cat(paste0("Downloaded files will not be unzipped.\n"))
- return(NULL)
- }
- cat(paste0("Unzipping files...\n"))
- unzip(file_name,
- exdir = directory_to_unzip
- )
- cat(paste0(
- "Files unzipped and saved in ",
- directory_to_unzip,
- ".\n"
- ))
- }
-```
-
-`download_remove_zips()` removes the downloaded zip files if `remove = TRUE`, and skips removal if `remove = FALSE`.
-
-```{r}
-download_remove_zips <-
- function(remove = FALSE,
- download_name) {
- if (remove) {
- cat(paste0("Removing download files...\n"))
- file.remove(download_name)
- cat(paste0("Download files removed.\n"))
- }
- }
-```
-
-For this demonstration we will unzip (inflate) the downloaded zip files but we will not delete them.
-
-```{r, eval = FALSE}
-for (f in seq_along(download_file_names)) {
- download_unzip(
- file_name = download_file_names[f],
- directory_to_unzip = directory_to_download,
- unzip = TRUE
- )
-}
-download_remove_zips(
- download_name = download_file_names,
- remove = FALSE
-)
-```
-
-```{r, echo = FALSE}
-for (f in seq_along(download_file_names)) {
- cat(paste0("Unzipping files...\n"))
- cat(paste0(
- "Files unzipped and saved in ",
- directory_to_download,
- ".\n"
- ))
-}
-```
-
-
-Listing the files again shows that the contents of the zip files have been inflated and the zip files have been retained.
-
-```{r, eval = FALSE}
-list.files(path = directory_to_download)
-```
-
-```{r, echo = FALSE}
-zips <- paste0("hms_smoke_Shapefile_", date_sequence, ".zip")
-for (s in seq_along(date_sequence)) {
- shps <- c(
- paste0("hms_smoke", date_sequence[s], ".dbf"),
- paste0("hms_smoke", date_sequence[s], ".prj"),
- paste0("hms_smoke", date_sequence[s], ".shp"),
- paste0("hms_smoke", date_sequence[s], ".shx")
- )
- zips <- c(zips, shps)
-}
-zips
-```
-
-The download function was structured successfully.
-
-## Unit Tests
-
-The previous outline successfully cleaned parameters, generated URLs, and downloaded data, but how can we be sure that it will continue to work with different temporal ranges and data types? To this end, unit tests have been implemented to ensure that each data download function runs properly and that URLs produced by [2. Generate download URLs] are valid and accessible. Like the download functions, the unit tests rely on "helper" functions to reduce repeated code across the tests.
-
-### Helper functions
-
-`read_commands()` imports the commands text file and converts the data frame to a vector.
-
-```{r}
-read_commands <- function(
- commands_path = commands_path) {
- commands <- utils::read.csv(commands_path, header = FALSE)
- commands <- commands[seq_len(nrow(commands)), ]
- return(commands)
-}
-```
-
-`extract_urls()` extracts each download URL from the vector of commands. The `position =` of the URL within the download command is determined in [5. Concatenate download commands].
-
-```{r}
-# function to extract URLs from vector
-extract_urls <- function(
- commands = commands,
- position = NULL) {
- if (is.null(position)) {
- cat(paste0("URL position in command is not defined.\n"))
- return(NULL)
- }
- url_list <- NULL
- for (c in seq_along(commands)) {
- url <- stringr::str_split_i(commands[c], " ", position)
- url_list <- c(url_list, url)
- }
- return(url_list)
-}
-```
-
-`check_url_status()` is the most important of the download test "helper" functions. This function utilizes `httr::HEAD()` and `httr::GET()` to check the HTTP response status of a given URL. The desired HTTP response status is 200, which means the URL is valid and accessible. `check_url_status()` returns a logical value to indicate whether the URL returns HTTP status 200 (`TRUE`) or not (`FALSE`). For more information on HTTP status', see [HTTP response status codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status).
-
-```{r}
-check_url_status <- function(
- url,
- method = "HEAD") {
- http_status_ok <- 200
- if (method == "HEAD") {
- hd <- httr::HEAD(url)
- } else if (method == "GET") {
- hd <- httr::GET(url)
- }
- status <- hd$status_code
- return(status == http_status_ok)
-}
-```
-
-`check_urls()` applies `check_url_status()` to a random sample of URLs extracted by `extract_urls()`. The sample size will vary based on the data set and spatio-temporal parameters being tested. The function returns a logical vector containing the output from `check_url_status()`.
-
-```{r}
-check_urls <- function(
- urls = urls,
- size = NULL,
- method = "HEAD") {
- if (is.null(size)) {
- cat(paste0("URL sample size is not defined.\n"))
- return(NULL)
- }
- if (length(urls) < size) {
- size <- length(urls)
- }
- url_sample <- sample(urls, size, replace = FALSE)
- url_status <- sapply(url_sample,
- check_url_status,
- method = method
- )
- return(url_status)
-}
-```
-
-### testthat
-
-To demonstrate a test in action, test the URLs generated by `download_data()` for the NOAA HMS Smoke data set.
-
-For more information see [testthat](https://testthat.r-lib.org/).
-
-```{r, eval = FALSE}
-library(testthat)
-testthat::test_that(
- "Valid dates return HTTP response status = 200.",
- {
- # parameters
- test_start <- "2023-12-28"
- test_end <- "2024-01-02"
- test_directory <- "./data"
- # download
- download_data(
- dataset_name = "noaa",
- date_start = test_start,
- date_end = test_end,
- data_format = "Shapefile",
- directory_to_download = test_directory,
- directory_to_save = test_directory,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- remove_command = FALSE,
- unzip = FALSE,
- remove_zip = FALSE
- )
- commands_path <- paste0(
- test_directory,
- "hms_smoke_",
- gsub("-", "", test_start),
- "_",
- gsub("-", "", test_end),
- "_curl_commands.txt"
- )
- # helpers
- commands <- read_commands(commands_path = commands_path)
- urls <- extract_urls(commands = commands, position = 6)
- url_status <- check_urls(urls = urls, size = 6, method = "HEAD")
- # test for true
- expect_true(all(url_status))
- }
-)
-```
-
-```{r, echo = FALSE}
-library(testthat)
-testthat::test_that(
- "Valid dates return HTTP response status = 200.",
- {
- # parameters
- test_start <- "2023-12-28"
- test_end <- "2024-01-02"
- test_directory <- "../inst/extdata/"
- # download
- download_data(
- dataset_name = "noaa",
- date_start = test_start,
- date_end = test_end,
- data_format = "Shapefile",
- directory_to_download = test_directory,
- directory_to_save = test_directory,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- remove_command = FALSE,
- unzip = FALSE,
- remove_zip = FALSE
- )
- commands_path <- paste0(
- test_directory,
- "hms_smoke_",
- gsub("-", "", test_start),
- "_",
- gsub("-", "", test_end),
- "_curl_commands.txt"
- )
- # helpers
- commands <- read_commands(commands_path = commands_path)
- urls <- extract_urls(commands = commands, position = 6)
- url_status <- check_urls(urls = urls, size = 6, method = "HEAD")
- # test for true
- expect_true(all(url_status))
- # remove after test
- file.remove(commands_path)
- }
-)
-```
-
-
-Although the `testthat::test_that(...)` chunk contains 32 lines of code, the unit test is performed by `expect_true(all(url_status))`. In words, this line is expecting (`expect_true()`) that all (`all()`) of the sampled URLs return HTTP response status 200 (`url_status`). Since this expectation was met, the test passed!
-
-For an alternate example, we can use a start and end date that are known to not have data. As the URLs associated with these dates do not exist, we expect the function will fail. This test utilizes `expect_error()` because the `data_download()` wrapper function returns an error message if the underlying source-specific download function returns an error.
-
-```{r}
-testthat::test_that(
- "Invalid dates cause function to fail.",
- {
- # parameters
- test_start <- "1800-01-01"
- test_end <- "1800-01-02"
- test_directory <- "../inst/extdata/"
- # test for error
- testthat::expect_error(
- download_data(
- dataset_name = "noaa",
- date_start = test_start,
- date_end = test_end,
- data_format = "Shapefile",
- directory_to_download = test_directory,
- directory_to_save = test_directory,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- remove_command = FALSE,
- unzip = FALSE,
- remove_zip = FALSE
- )
- )
- }
-)
-```
-
-
-This test utilizes `expect_error()` because the `data_download()` wrapper function returns an error message if the underlying source-specific download function returns an error. If we directly used the `download_noaa_hms_smoke` function, we would expect and receive an error.
-
-```{r}
-testthat::test_that(
- "Invalid dates cause function to fail.",
- {
- # parameters
- test_start <- "1800-01-01"
- test_end <- "1800-01-02"
- test_directory <- "../inst/extdata/"
- # test for error
- testthat::expect_error(
- download_noaa_hms_smoke_data(
- date_start = test_start,
- date_end = test_end,
- data_format = "Shapefile",
- directory_to_download = test_directory,
- directory_to_save = test_directory,
- data_download_acknowledgement = TRUE,
- download = FALSE,
- remove_command = FALSE,
- unzip = FALSE,
- remove_zip = FALSE
- )
- )
- }
-)
-```
-
-
-As expected, the test passes because the NOAA HMS Smoke data set does not contain data for January 1-2, 1800.
-
-These unit tests are just two of many implemented on `download_data()` and the accompanying source-specific download functions, but they demonstrate how unit testing helps build stable code.
-
-## Download Example
-
-With the function structure outlined and the unit tests in place, we can now perform a data download. To begin, check the parameters required by the source-specific data download function.
-
-```{r}
-names(formals(download_noaa_hms_smoke_data))
-```
-
-Define the parameters.
-
-```{r}
-date_start <- "2023-12-28"
-date_end <- "2024-01-02"
-data_format <- "Shapefile"
-data_directory <- "./download_example/"
-data_download_acknowledgement <- TRUE
-download <- TRUE # run data download
-remove_command <- TRUE # delete "...commands.txt" file
-unzip <- TRUE # inflate (unzip) downloaded zip files
-remove_zip <- FALSE # retain downloaded zip files
-```
-
-Download the data.
-
-```{r, eval = FALSE}
-download_data(
- dataset_name = "hms",
- date_start = date_start,
- date_end = date_end,
- directory_to_download = data_directory,
- directory_to_save = data_directory,
- data_download_acknowledgement = data_download_acknowledgement,
- download = download,
- remove_command = remove_command,
- unzip = unzip,
- remove_zip = remove_zip
-)
-```
-
-```{r, echo = FALSE}
-to_cat <-
- paste0(
- "Downloading requested files...\n",
- "Requested files have been downloaded.\n"
- )
-cat(to_cat)
-for (f in seq_along(download_file_names)) {
- cat(paste0("Unzipping files...\n"))
- cat(paste0(
- "Files unzipped and saved in ",
- data_directory,
- ".\n"
- ))
-}
-```
-
-Checking the directory shows that all of our desired data has been downloaded and inflated, and the original zip files have been retained.
-
-```{r, eval = FALSE}
-list.files(data_directory)
-```
-
-```{r, echo = FALSE}
-zips <- paste0("hms_smoke_Shapefile_", date_sequence, ".zip")
-for (s in seq_along(date_sequence)) {
- shps <- c(
- paste0("hms_smoke", date_sequence[s], ".dbf"),
- paste0("hms_smoke", date_sequence[s], ".prj"),
- paste0("hms_smoke", date_sequence[s], ".shp"),
- paste0("hms_smoke", date_sequence[s], ".shx")
- )
- zips <- c(zips, shps)
-}
-zips
-```
-
-## NASA EarthData Account
-
-As mentioned in [Motivation], `data_download()` provides access to **publicly available** geospatial data. Although publicly available, some of the NASA data sources require a NASA EarthData Account.
-
-For example, the UN WPP-Adjusted population density data from NASA Socioeconomic Data and Applications Center (SEDAC) requires an EarthData account. Without an EarthData Account and the prerequisite files prepared, the data download functions will return an error.
-
-```{r, eval = FALSE}
-download_data(
- dataset_name = "sedac_population",
- year = "2020",
- data_format = "GeoTIFF",
- data_resolution = "60 minute",
- directory_to_download = "./sedac_population/",
- directory_to_save = "./sedac_population",
- data_download_acknowledgement = TRUE,
- download = TRUE,
- unzip = TRUE,
- remove_zip = FALSE,
- remove_command = TRUE
-)
-```
-
-```{r, echo = FALSE}
-to_cat <-
- paste0(
- "Downloading requested files...\n",
- "Requested files have been downloaded.\n",
- "Unzipping files...\n\n",
- "Warning in unzip(file_name, exdir = directory_to_unzip): ",
- "error 1 in extracting from zip file\n\n",
- "Files unzipped and saved in ./sedac_population/.\n\n"
- )
-cat(to_cat)
-```
-
-The downloaded file cannot be unzipped because the data was not downloaded properly.
-
-### Register or log in
-
-Visit [https://urs.earthdata.nasa.gov/](http://urs.earthdata.nasa.gov) to register for or log into a NASA EarthData account.
-
-
-
-Account registration provides access to NASA's Earth Observing System Data and Information System (EOSDIS) and its twelve Distributed Active Archive Centers (DAAC), including:
-
-- Alaska Satellite Facility (ASF) DAAC
-- Atmospheric Science Data Center (ASDC)
-- Crustal Dynamics Data Information System (CDDIS)
-- Global Hydrometeorology Resource Center (GHRC)
-- Goddard Earth Sciences Data and Information Services Center (GES DISC)
-- Land Processes DAAC (LP DAAC)
-- Level 1 and Atmosphere Archive and Distribution System (LAADS) DAAC
-- National Snow and Ice Data Center (NSIDC) DAAC
-- Oak Ridge National Laboratory (ORNL) DAAC
-- Ocean Biology DAAC (OB.DAAC)
-- Physical Oceanography DAAC (PO.DAAC)
-- Socioeconomic Data and Applications Center (SEDAC)
-
-See for more information.
-
-### Approved applications
-
-After creating an account, navigate to "My Profile"(https://urs.earthdata.nasa.gov/profile), and then to "Applications \> Authorized Apps". This "Authorized Apps" page specifies which NASA EarthData applications can use your login credentials. For this example, ensure that authorization is enabled for "SEDAC Website", "SEDAC Website (Alpha)", and "SEDAC Website (Beta)".
-
-
-
-### Prerequisite files
-
-With a NASA EarthData Account and the required applications authorized to use the credentials, it is time to create the prerequisite files.
-
-**Note** As the steps to produce the prerequisite files is very specific, much of the code used is adopted from [How to Generate Earthdata Prerequisite Files](https://disc.gsfc.nasa.gov/information/howto?title=How%20to%20Generate%20Earthdata%20Prerequisite%20Files) on NASA GES DISC's "How-To's" webpage ([link](https://disc.gsfc.nasa.gov/information/howto)).
-
-#### `.netrc`
-
-The following commands create the `.netrc` file, which contains your NASA EarthData Account credentials.
-
-First, set your working directory to the home directory. Setting a working directory differs between Mac/Linux and Windows machines.
-
-```{r, eval = FALSE}
-if (.Platform$OS.type == "unix") {
- setwd("~/")
-} else if (.Platform$OS.type == "windows") {
- setwd("C:/")
-}
-```
-
-Create a file named `.netrc` with `file.create()`.
-
-```{r, eval = FALSE}
-file.create(".netrc")
-```
-
-Open a connection to `.netrc` with `sink()`. Write the line `machine urs...` replacing `YOUR_USERNAME` and `YOUR_PASSWORD` with your NASA EarthData username and password, respectively. After writing the line, close the connection with `sink()` again.
-
-```{r, eval = FALSE}
-sink(".netrc")
-writeLines(
- "machine urs.earthdata.nasa.gov login YOUR_USERNAME password YOUR_PASSWORD"
-)
-sink()
-```
-
-Edit the settings so only you, the owner of the file, can read and write `.netrc`.
-
-```{r, eval = FALSE}
-system("chmod 0600 .netrc")
-```
-
-After, check to ensure the file was created properly.
-
-```{r, eval = FALSE}
-file.exists(".netrc")
-```
-
-```{r, echo = FALSE}
-TRUE
-```
-
-```{r, eval = FALSE}
-readLines(".netrc")
-```
-
-```{r, echo = FALSE}
-paste0(
- "machine urs.earthdata.nasa.gov login YOUR_USERNAME password YOUR_PASSWORD"
-)
-```
-
-
-#### `.urs_cookies`
-
-The following commands create the `.urs_cookies` file.
-
-First, set your working directory to the home directory.
-
-```{r, eval = FALSE}
-if (.Platform$OS.type == "unix") {
- setwd("~/")
-} else if (.Platform$OS.type == "windows") {
- setwd("C:/")
-}
-```
-
-Create a file named `.netrc` with `file.create()`.
-
-```{r, eval = FALSE}
-file.create(".urs_cookies")
-```
-
-After, check to ensure the file was created properly.
-
-```{r, eval = FALSE}
-file.exists(".urs_cookies")
-```
-
-```{r, echo = FALSE}
-TRUE
-```
-
-#### `.dodsrc`
-
-The following commands create the `.urs_cookies` file.
-
-First, set your working directory to the home directory.
-
-```{r, eval = FALSE}
-if (.Platform$OS.type == "unix") {
- setwd("~/")
-} else if (.Platform$OS.type == "windows") {
- setwd("C:/")
-}
-```
-
-Create a file named ".dodsrc" with `file.create()`
-
-```{r, eval = FALSE}
-file.create(".dodsrc")
-```
-
-Open a connection to `.dodsrc` with `sink()`. Write the lines beginning with `HTTP.`, replacing `YOUR_USERNAME` and `YOUR_PASSWORD` with your NASA EarthData username and password, respectively. After writing the line, close the connection with `sink()` again.
-
-```{r, eval = FALSE}
-sink(".dodsrc")
-writeLines(
- paste0(
- "HTTP.NETRC=YOUR_HOME_DIRECTORY/.netrc\n",
- "HTTP.COOKIE.JAR=YOUR_HOME_DIRECTORY/.urs_cookies"
- )
-)
-sink()
-```
-
-After, check to ensure the file was created properly.
-
-```{r, eval = FALSE}
-file.exists(".dodsrc")
-```
-
-```{r, echo = FALSE}
-TRUE
-```
-
-```{r, eval = FALSE}
-readLines(".dodsrc")
-```
-
-```{r, echo = FALSE}
-paste0(
- c(
- "HTTP.NETRC=YOUR_HOME_DIRECTORY/.netrc",
- "HTTP.COOKIE.JAR=YOUR_HOME_DIRECTORY/.urs_cookies"
- )
-)
-```
-
-If working on a **Windows** machine, copy the `.dodsrc` file to the project working directory. Replace `YOUR_WORKING_DIRECTORY` with the absolute path to the project working directory.
-
-```{r}
-if (.Platform$OS.type == "windows") {
- file.copy(
- "C:/.dodsrc",
- "YOUR_WORKING_DIRECTORY/.dodsrc"
- )
-}
-```
-
-It is important to ensure that these commands, as well as your username, password, and home directory, are typed without error, as a single problem with any of these files will result in a failed download.
-
-If the files have been created correctly, the UN WPP-Adjusted population density data from NASA Socioeconomic Data and Applications Center (SEDAC) will be downloaded and unzipped without returning an error.
-
-```{r, eval = FALSE}
-download_data(
- dataset_name = "sedac_population",
- year = "2020",
- data_format = "GeoTIFF",
- data_resolution = "60 minute",
- directory_to_download = "./sedac_population/",
- directory_to_save = "./sedac_population",
- data_download_acknowledgement = TRUE,
- download = TRUE,
- unzip = TRUE,
- remove_zip = FALSE,
- remove_command = TRUE
-)
-```
-
-```{r, echo = FALSE}
-to_cat <-
- paste0(
- "Downloading requested files...\n",
- "Requested files have been downloaded.\n",
- "Unzipping files...\n",
- "Files unzipped and saved in ./sedac_population/.\n"
- )
-cat(to_cat)
-```
-
-```{r, eval = FALSE}
-list.files("./sedac_population")
-```
-
-```{r, echo = FALSE}
-sedac_files <- c(
- paste0(
- "gpw_v4_population_density_adjusted_to_2015_unwpp_country_totals_",
- "rev11_2020_1_deg_tif_readme.txt"
- ),
- paste0(
- "gpw_v4_population_density_adjusted_to_2015_unwpp_country_totals_",
- "rev11_2020_1_deg_tif.zip"
- ),
- paste0(
- "gpw_v4_population_density_adjusted_to_2015_unwpp_country_totals_",
- "rev11_2020_1_deg.tif"
- )
-)
-sedac_files
-```
-
-## References
-
-- EOSDIS Distributed Active Archive Centers (DAAC). *National Aeronautics and Space Administration (NASA)*. Date accessed: January 3, 2024. [https://www.earthdata.nasa.gov/eosdis/daacs]().
-- How to Generate Earthdata Prerequisite Files. *National Aeronautics and Space Administration (NASA)*. Date accessed: January 3, 2024. [https://disc.gsfc.nasa.gov/information/howto?title=How%20to%20Generate%20Earthdata%20Prerequisite%20Files]().
-
-## Code Example
-
-The following is the entire R code used to create `download_noaa_hms_smoke_data()`.
-
-```{r}
-download_noaa_hms_smoke_data
-```