From 7f6618dd4ca12e3f0a285211b8c5fd01bf8dde1b Mon Sep 17 00:00:00 2001 From: jgabry Date: Wed, 2 Mar 2022 11:04:02 -0700 Subject: [PATCH 1/5] fix for CRAN and update NEWS --- DESCRIPTION | 6 +++--- NEWS.md | 6 +++++- R/loo-package.R | 2 +- man/loo-package.Rd | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index d0cd9d35..783b63ae 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Package: loo Type: Package Title: Efficient Leave-One-Out Cross-Validation and WAIC for Bayesian Models -Version: 2.4.1.9000 -Date: 2020-12-07 +Version: 2.5.0 +Date: 2022-03-02 Authors@R: c(person("Aki", "Vehtari", email = "Aki.Vehtari@aalto.fi", role = c("aut")), person("Jonah", "Gabry", email = "jsg2201@columbia.edu", role = c("cre", "aut")), person("Mans", "Magnusson", role = c("aut")), @@ -51,5 +51,5 @@ Suggests: VignetteBuilder: knitr Encoding: UTF-8 SystemRequirements: pandoc (>= 1.12.3), pandoc-citeproc -RoxygenNote: 7.1.1 +RoxygenNote: 7.1.2 Roxygen: list(markdown = TRUE) diff --git a/NEWS.md b/NEWS.md index 5500b86e..e6ea449f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,6 @@ -# Items for next release go here +# loo 2.5.0 + +### Improvements * Speed improvement from simplifying the normalization when fitting the generalized Pareto distribution. (#187, @sethaxen) @@ -8,6 +10,8 @@ updated when using moment matching. (#167, @fweber144) * Switch unit tests from Travis to GitHub Actions. (#164) +* Added parallel likelihood computation to speedup `loo_subsample()` when using posterior approximations. (#171, @kdubovikov) + ### Bug fixes * Fixed a bug causing the normalizing constant of the PSIS (log) weights not diff --git a/R/loo-package.R b/R/loo-package.R index df45fa71..a1947b03 100644 --- a/R/loo-package.R +++ b/R/loo-package.R @@ -8,7 +8,7 @@ #' #' @description #' \if{html}{ -#' \figure{stanlogo.png}{options: width="50px" alt="mc-stan.org"} +#' \figure{stanlogo.png}{options: width="50" alt="mc-stan.org"} #' } #' *Stan Development Team* #' diff --git a/man/loo-package.Rd b/man/loo-package.Rd index 5e30dd31..d17a5607 100644 --- a/man/loo-package.Rd +++ b/man/loo-package.Rd @@ -6,7 +6,7 @@ \title{Efficient LOO-CV and WAIC for Bayesian models} \description{ \if{html}{ - \figure{stanlogo.png}{options: width="50px" alt="mc-stan.org"} + \figure{stanlogo.png}{options: width="50" alt="mc-stan.org"} } \emph{Stan Development Team} From e58450def14e044bfdc75a68a37068a01ea8faaf Mon Sep 17 00:00:00 2001 From: jgabry Date: Wed, 2 Mar 2022 11:33:20 -0700 Subject: [PATCH 2/5] Update references closes #196 --- man-roxygen/moment-matching-references.R | 7 +++---- man/loo_moment_match.Rd | 6 +++--- man/loo_moment_match_split.Rd | 6 +++--- vignettes/loo2-lfo.Rmd | 3 ++- vignettes/loo2-moment-matching.Rmd | 6 +++--- vignettes/loo2-non-factorized.Rmd | 3 ++- 6 files changed, 16 insertions(+), 15 deletions(-) diff --git a/man-roxygen/moment-matching-references.R b/man-roxygen/moment-matching-references.R index b929d348..c7cf0471 100644 --- a/man-roxygen/moment-matching-references.R +++ b/man-roxygen/moment-matching-references.R @@ -1,5 +1,4 @@ #' @references -#' Paananen, T., Piironen, J., Buerkner, P.-C., Vehtari, A. (2020). -#' Implicitly Adaptive Importance Sampling. -#' [preprint arXiv:1906.08850](https://arxiv.org/abs/1906.08850) -#' +#' Paananen, T., Piironen, J., Buerkner, P.-C., Vehtari, A. (2021). +#' Implicitly adaptive importance sampling. *Statistics and Computing*, 31, 16. +#' doi:10.1007/s11222-020-09982-2. arXiv preprint arXiv:1906.08850. diff --git a/man/loo_moment_match.Rd b/man/loo_moment_match.Rd index d76ceffa..09b28485 100644 --- a/man/loo_moment_match.Rd +++ b/man/loo_moment_match.Rd @@ -107,9 +107,9 @@ and \code{log_lik_i_upars}. # See the vignette for loo_moment_match() } \references{ -Paananen, T., Piironen, J., Buerkner, P.-C., Vehtari, A. (2020). -Implicitly Adaptive Importance Sampling. -\href{https://arxiv.org/abs/1906.08850}{preprint arXiv:1906.08850} +Paananen, T., Piironen, J., Buerkner, P.-C., Vehtari, A. (2021). +Implicitly adaptive importance sampling. \emph{Statistics and Computing}, 31, 16. +doi:10.1007/s11222-020-09982-2. arXiv preprint arXiv:1906.08850. } \seealso{ \code{\link[=loo]{loo()}}, \code{\link[=loo_moment_match_split]{loo_moment_match_split()}} diff --git a/man/loo_moment_match_split.Rd b/man/loo_moment_match_split.Rd index b7ecbc78..db69ad6f 100644 --- a/man/loo_moment_match_split.Rd +++ b/man/loo_moment_match_split.Rd @@ -88,9 +88,9 @@ Takes in the moment matching total transformation, transforms only half of the draws, and computes a single elpd using multiple importance sampling. } \references{ -Paananen, T., Piironen, J., Buerkner, P.-C., Vehtari, A. (2020). -Implicitly Adaptive Importance Sampling. -\href{https://arxiv.org/abs/1906.08850}{preprint arXiv:1906.08850} +Paananen, T., Piironen, J., Buerkner, P.-C., Vehtari, A. (2021). +Implicitly adaptive importance sampling. \emph{Statistics and Computing}, 31, 16. +doi:10.1007/s11222-020-09982-2. arXiv preprint arXiv:1906.08850. } \seealso{ \code{\link[=loo]{loo()}}, \code{\link[=loo_moment_match]{loo_moment_match()}} diff --git a/vignettes/loo2-lfo.Rmd b/vignettes/loo2-lfo.Rmd index 99abeb7e..905558e7 100644 --- a/vignettes/loo2-lfo.Rmd +++ b/vignettes/loo2-lfo.Rmd @@ -635,7 +635,8 @@ many times. For more details on approximate LFO-CV, we refer to Bürkner et al. ## References -Bürkner P. C., Gabry J., & Vehtari A. (2020). Approximate leave-future-out cross-validation for Bayesian time series models. *Journal of Statistical Computation and Simulation*. \doi:10.1080/00949655.2020.1783262. [Online](https://www.tandfonline.com/doi/full/10.1080/00949655.2020.1783262). [arXiv preprint](https://arxiv.org/abs/1902.06281). +Bürkner P. C., Gabry J., & Vehtari A. (2020). Approximate leave-future-out cross-validation for time series models. *Journal of Statistical Computation and Simulation*, 90(14):2499-2523. \doi:/10.1080/00949655.2020.1783262. +[Online](https://www.tandfonline.com/doi/full/10.1080/00949655.2020.1783262). [arXiv preprint](https://arxiv.org/abs/1902.06281). Vehtari A., Gelman A., & Gabry J. (2017). Practical Bayesian model evaluation using leave-one-out cross-validation and WAIC. *Statistics and Computing*, 27(5), 1413--1432. \doi:10.1007/s11222-016-9696-4. [Online](https://link.springer.com/article/10.1007/s11222-016-9696-4). [arXiv preprint arXiv:1507.04544](https://arxiv.org/abs/1507.04544). diff --git a/vignettes/loo2-moment-matching.Rmd b/vignettes/loo2-moment-matching.Rmd index 1b976478..40564efb 100644 --- a/vignettes/loo2-moment-matching.Rmd +++ b/vignettes/loo2-moment-matching.Rmd @@ -311,9 +311,9 @@ Gelman, A., and Hill, J. (2007). *Data Analysis Using Regression and Multilevel Stan Development Team (2020) _RStan: the R interface to Stan, Version 2.21.1_ https://mc-stan.org -Paananen, T., Piironen, J., Buerkner, P.-C., Vehtari, A. (2020). -Implicitly Adaptive Importance Sampling. -[arXiv preprint arXiv:1906.08850](https://arxiv.org/abs/1906.08850). +Paananen, T., Piironen, J., Buerkner, P.-C., Vehtari, A. (2021). +Implicitly adaptive importance sampling. _Statistics and Computing_, 31, 16. +\doi:10.1007/s11222-020-09982-2. arXiv preprint arXiv:1906.08850. Vehtari, A., Gelman, A., and Gabry, J. (2017). Practical Bayesian model evaluation using leave-one-out cross-validation and WAIC. _Statistics and Computing_. 27(5), 1413--1432. \doi:10.1007/s11222-016-9696-4. Links: [published](https://link.springer.com/article/10.1007/s11222-016-9696-4) | [arXiv preprint](https://arxiv.org/abs/1507.04544). diff --git a/vignettes/loo2-non-factorized.Rmd b/vignettes/loo2-non-factorized.Rmd index cf50bb38..a48b0256 100644 --- a/vignettes/loo2-non-factorized.Rmd +++ b/vignettes/loo2-non-factorized.Rmd @@ -708,7 +708,8 @@ can be expressed in terms of a multivariate normal likelihood. Anselin L. (1988). *Spatial econometrics: methods and models*. Dordrecht: Kluwer Academic. -Bürkner P. C., Gabry J., & Vehtari A. (2020). Efficient leave-one-out cross-validation for Bayesian non-factorized normal and Student-t models. [ArXiv preprint](https://arxiv.org/abs/1810.10559). +Bürkner P. C., Gabry J., & Vehtari A. (2020). Efficient leave-one-out cross-validation for Bayesian non-factorized normal and Student-t models. *Computational Statistics*, \doi:10.1007/s00180-020-01045-4. +[ArXiv preprint](https://arxiv.org/abs/1810.10559). Sundararajan S. & Keerthi S. S. (2001). Predictive approaches for choosing hyperparameters in Gaussian processes. *Neural Computation*, 13(5), 1103--1118. From 409de413f56a453d6d4e5a096efceb8388b2d39b Mon Sep 17 00:00:00 2001 From: jgabry Date: Wed, 9 Mar 2022 15:48:59 -0700 Subject: [PATCH 3/5] add links to FAQ --- NEWS.md | 10 ++++++---- R/loo-glossary.R | 7 ++++++- R/loo.R | 4 +++- R/loo_compare.R | 3 +++ R/psis.R | 4 ++++ man/loo-glossary.Rd | 4 ++++ man/loo.Rd | 2 ++ man/loo_compare.Rd | 6 ++++++ man/psis.Rd | 4 ++++ 9 files changed, 38 insertions(+), 6 deletions(-) diff --git a/NEWS.md b/NEWS.md index e6ea449f..f799eb86 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,21 +2,23 @@ ### Improvements +* New [Frequently Asked Questions page](https://mc-stan.org/loo/articles/online-only/faq.html) on the package website. (#143) + * Speed improvement from simplifying the normalization when fitting the generalized Pareto distribution. (#187, @sethaxen) -* Fixed bug where attribute storing normalizing constant of PSIS weights wasn't -updated when using moment matching. (#167, @fweber144) +* Added parallel likelihood computation to speedup `loo_subsample()` when using posterior approximations. (#171, @kdubovikov) * Switch unit tests from Travis to GitHub Actions. (#164) -* Added parallel likelihood computation to speedup `loo_subsample()` when using posterior approximations. (#171, @kdubovikov) - ### Bug fixes * Fixed a bug causing the normalizing constant of the PSIS (log) weights not to get updated when performing moment matching with `save_psis = TRUE` (#166, @fweber144). +* Fixed bug where the attribute storing normalizing constant of PSIS weights +wasn't updated when using moment matching. (#167, @fweber144) + # loo 2.4.1 * Fixed issue reported by CRAN where one of the vignettes errored on an M1 Mac diff --git a/R/loo-glossary.R b/R/loo-glossary.R index 603ac465..ab1ba813 100644 --- a/R/loo-glossary.R +++ b/R/loo-glossary.R @@ -5,7 +5,12 @@ #' @template loo-and-psis-references #' @template bayesvis-reference #' -#' @description Note: VGG2017 refers to Vehtari, Gelman, and Gabry (2017). See +#' @description +#' The pages provides definitions to key terms. Also see the +#' [FAQ page](https://mc-stan.org/loo/articles/online-only/faq.html) on +#' the __loo__ website for answers to frequently asked questions. +#' +#' Note: VGG2017 refers to Vehtari, Gelman, and Gabry (2017). See #' **References**, below. #' #' @section ELPD and `elpd_loo`: diff --git a/R/loo.R b/R/loo.R index 57f2a166..28632437 100644 --- a/R/loo.R +++ b/R/loo.R @@ -87,8 +87,10 @@ #' } #' #' @seealso -#' * The **loo** package [vignettes](https://mc-stan.org/loo/articles/index.html) +#' * The __loo__ package [vignettes](https://mc-stan.org/loo/articles/index.html) #' for demonstrations. +#' * The [FAQ page](https://mc-stan.org/loo/articles/online-only/faq.html) on +#' the __loo__ website for answers to frequently asked questions. #' * [psis()] for the underlying Pareto Smoothed Importance Sampling (PSIS) #' procedure used in the LOO-CV approximation. #' * [pareto-k-diagnostic] for convenience functions for looking at diagnostics. diff --git a/R/loo_compare.R b/R/loo_compare.R index 2ec6639e..ac059bbd 100644 --- a/R/loo_compare.R +++ b/R/loo_compare.R @@ -42,6 +42,9 @@ #' distribution, a practice derived for Gaussian linear models or #' asymptotically, and which only applies to nested models in any case. #' +#' @seealso +#' * The [FAQ page](https://mc-stan.org/loo/articles/online-only/faq.html) on +#' the __loo__ website for answers to frequently asked questions. #' @template loo-and-psis-references #' #' @examples diff --git a/R/psis.R b/R/psis.R index e82a1ef9..274c5bcb 100644 --- a/R/psis.R +++ b/R/psis.R @@ -68,6 +68,10 @@ #' @seealso #' * [loo()] for approximate LOO-CV using PSIS. #' * [pareto-k-diagnostic] for PSIS diagnostics. +#' * The __loo__ package [vignettes](https://mc-stan.org/loo/articles/index.html) +#' for demonstrations. +#' * The [FAQ page](https://mc-stan.org/loo/articles/online-only/faq.html) on +#' the __loo__ website for answers to frequently asked questions. #' #' @template loo-and-psis-references #' diff --git a/man/loo-glossary.Rd b/man/loo-glossary.Rd index 65897784..041eb11e 100644 --- a/man/loo-glossary.Rd +++ b/man/loo-glossary.Rd @@ -4,6 +4,10 @@ \alias{loo-glossary} \title{LOO package glossary} \description{ +The pages provides definitions to key terms. Also see the +\href{https://mc-stan.org/loo/articles/online-only/faq.html}{FAQ page} on +the \strong{loo} website for answers to frequently asked questions. + Note: VGG2017 refers to Vehtari, Gelman, and Gabry (2017). See \strong{References}, below. } diff --git a/man/loo.Rd b/man/loo.Rd index 8114104a..25811df9 100644 --- a/man/loo.Rd +++ b/man/loo.Rd @@ -326,6 +326,8 @@ Pareto smoothed importance sampling. \itemize{ \item The \strong{loo} package \href{https://mc-stan.org/loo/articles/index.html}{vignettes} for demonstrations. +\item The \href{https://mc-stan.org/loo/articles/online-only/faq.html}{FAQ page} on +the \strong{loo} website for answers to frequently asked questions. \item \code{\link[=psis]{psis()}} for the underlying Pareto Smoothed Importance Sampling (PSIS) procedure used in the LOO-CV approximation. \item \link{pareto-k-diagnostic} for convenience functions for looking at diagnostics. diff --git a/man/loo_compare.Rd b/man/loo_compare.Rd index f27977d9..4a699bd6 100644 --- a/man/loo_compare.Rd +++ b/man/loo_compare.Rd @@ -102,3 +102,9 @@ Vehtari, A., Simpson, D., Gelman, A., Yao, Y., and Gabry, J. (2019). Pareto smoothed importance sampling. \href{https://arxiv.org/abs/1507.02646}{preprint arXiv:1507.02646} } +\seealso{ +\itemize{ +\item The \href{https://mc-stan.org/loo/articles/online-only/faq.html}{FAQ page} on +the \strong{loo} website for answers to frequently asked questions. +} +} diff --git a/man/psis.Rd b/man/psis.Rd index 39c692b1..8a7bb005 100644 --- a/man/psis.Rd +++ b/man/psis.Rd @@ -155,5 +155,9 @@ Pareto smoothed importance sampling. \itemize{ \item \code{\link[=loo]{loo()}} for approximate LOO-CV using PSIS. \item \link{pareto-k-diagnostic} for PSIS diagnostics. +\item The \strong{loo} package \href{https://mc-stan.org/loo/articles/index.html}{vignettes} +for demonstrations. +\item The \href{https://mc-stan.org/loo/articles/online-only/faq.html}{FAQ page} on +the \strong{loo} website for answers to frequently asked questions. } } From fb3893f98b797c17b14366ac22eb3f73882cbe09 Mon Sep 17 00:00:00 2001 From: jgabry Date: Wed, 9 Mar 2022 15:51:05 -0700 Subject: [PATCH 4/5] more links to FAQ --- R/diagnostics.R | 5 ++++- man/pareto-k-diagnostic.Rd | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/R/diagnostics.R b/R/diagnostics.R index 66458b9a..29a9a1f1 100644 --- a/R/diagnostics.R +++ b/R/diagnostics.R @@ -80,7 +80,10 @@ #' **over-optimistic when the estimate of \eqn{k} is greater than 0.7**. #' } #' -#' @seealso [psis()] for the implementation of the PSIS algorithm. +#' @seealso +#' * [psis()] for the implementation of the PSIS algorithm. +#' * The [FAQ page](https://mc-stan.org/loo/articles/online-only/faq.html) on +#' the __loo__ website for answers to frequently asked questions. #' #' @template loo-and-psis-references #' diff --git a/man/pareto-k-diagnostic.Rd b/man/pareto-k-diagnostic.Rd index e51e396d..c268ffcd 100644 --- a/man/pareto-k-diagnostic.Rd +++ b/man/pareto-k-diagnostic.Rd @@ -174,5 +174,9 @@ Pareto smoothed importance sampling. \href{https://arxiv.org/abs/1507.02646}{preprint arXiv:1507.02646} } \seealso{ -\code{\link[=psis]{psis()}} for the implementation of the PSIS algorithm. +\itemize{ +\item \code{\link[=psis]{psis()}} for the implementation of the PSIS algorithm. +\item The \href{https://mc-stan.org/loo/articles/online-only/faq.html}{FAQ page} on +the \strong{loo} website for answers to frequently asked questions. +} } From 79ef4cb636bc9e20f170ef003a7c765ec0a91fb9 Mon Sep 17 00:00:00 2001 From: jgabry Date: Thu, 10 Mar 2022 18:39:12 -0700 Subject: [PATCH 5/5] minor doc edits --- R/loo.R | 37 ++++++++++++++-------------- R/loo_compare.R | 40 +++++++++++++++---------------- man/loo.Rd | 29 +++++++++++----------- man/loo_compare.Rd | 40 +++++++++++++++---------------- man/loo_subsample.Rd | 5 ++-- man/parallel_psis_list.Rd | 11 +++++---- man/psis_approximate_posterior.Rd | 6 ++--- man/update.psis_loo_ss.Rd | 5 ++-- 8 files changed, 88 insertions(+), 85 deletions(-) diff --git a/R/loo.R b/R/loo.R index 28632437..fab18e62 100644 --- a/R/loo.R +++ b/R/loo.R @@ -17,16 +17,16 @@ #' with MCMC. If MCMC draws are used and `r_eff` is not provided then #' the reported PSIS effective sample sizes and Monte Carlo error estimates #' will be over-optimistic. If the posterior draws are independent then -#' `r_eff=1` and can be omitted. See the [relative_eff()] -#' helper functions for computing `r_eff`. -#' +#' `r_eff=1` and can be omitted. The warning message thrown when `r_eff` is +#' not specified can be disabled by setting `r_eff` to `NA`. See the +#' [relative_eff()] helper functions for computing `r_eff`. #' @param save_psis Should the `"psis"` object created internally by `loo()` be #' saved in the returned object? The `loo()` function calls [psis()] #' internally but by default discards the (potentially large) `"psis"` object #' after using it to compute the LOO-CV summaries. Setting `save_psis=TRUE` -#' will add a `psis_object` component to the list returned by `loo`. Currently -#' this is only needed if you plan to use the [E_loo()] function to compute -#' weighted expectations after running `loo`. +#' will add a `psis_object` component to the list returned by `loo`. +#' Currently this is only needed if you plan to use the [E_loo()] function to +#' compute weighted expectations after running `loo`. #' @template cores #' @template is_method #' @@ -47,32 +47,31 @@ #' `c("psis_loo", "loo")` and components: #' \describe{ #' \item{`estimates`}{ -#' A matrix with two columns (`Estimate`, `SE`) and three rows -#' (`elpd_loo`, `p_loo`, `looic`). This -#' contains point estimates and standard errors of the expected log pointwise -#' predictive density (`elpd_loo`), the effective number of parameters -#' (`p_loo`) and the LOO information criterion `looic` (which is -#' just `-2 * elpd_loo`, i.e., converted to deviance scale). +#' A matrix with two columns (`Estimate`, `SE`) and three rows (`elpd_loo`, +#' `p_loo`, `looic`). This contains point estimates and standard errors of the +#' expected log pointwise predictive density ([`elpd_loo`][loo-glossary]), the +#' effective number of parameters ([`p_loo`][loo-glossary]) and the LOO +#' information criterion `looic` (which is just `-2 * elpd_loo`, i.e., +#' converted to deviance scale). #' } #' #' \item{`pointwise`}{ #' A matrix with five columns (and number of rows equal to the number of #' observations) containing the pointwise contributions of the measures #' (`elpd_loo`, `mcse_elpd_loo`, `p_loo`, `looic`, `influence_pareto_k`). -#' in addition to the three measures in \code{estimates}, we also report -#' pointwise values of the Monte Carlo standard error of \code{elpd_loo} -#' (\code{mcse_elpd_loo}), and statistics describing the influence of -#' each observation on the posterior distribution (\code{influence_pareto_k}). +#' in addition to the three measures in `estimates`, we also report +#' pointwise values of the Monte Carlo standard error of [`elpd_loo`][loo-glossary] +#' ([`mcse_elpd_loo`][loo-glossary]), and statistics describing the influence of +#' each observation on the posterior distribution (`influence_pareto_k`). #' These are the estimates of the shape parameter \eqn{k} of the #' generalized Pareto fit to the importance ratios for each leave-one-out -#' distribution. See the [pareto-k-diagnostic] page for details. +#' distribution (see the [pareto-k-diagnostic] page for details). #' } #' -#' #' \item{`diagnostics`}{ #' A named list containing two vectors: #' * `pareto_k`: Importance sampling reliability diagnostics. By default, -#' these are equal to the \code{influence_pareto_k} in \code{pointwise}. +#' these are equal to the `influence_pareto_k` in `pointwise`. #' Some algorithms can improve importance sampling reliability and #' modify these diagnostics. See the [pareto-k-diagnostic] page for details. #' * `n_eff`: PSIS effective sample size estimates. diff --git a/R/loo_compare.R b/R/loo_compare.R index ac059bbd..bdab9d85 100644 --- a/R/loo_compare.R +++ b/R/loo_compare.R @@ -17,28 +17,28 @@ #' #' @details #' When comparing two fitted models, we can estimate the difference in their -#' expected predictive accuracy by the difference in `elpd_loo` or -#' `elpd_waic` (or multiplied by \eqn{-2}, if desired, to be on the -#' deviance scale). +#' expected predictive accuracy by the difference in +#' [`elpd_loo`][loo-glossary] or `elpd_waic` (or multiplied by \eqn{-2}, if +#' desired, to be on the deviance scale). #' -#' When using `loo_compare()`, the returned matrix will have one row per -#' model and several columns of estimates. The values in the `elpd_diff` -#' and `se_diff` columns of the returned matrix are computed by making -#' pairwise comparisons between each model and the model with the largest ELPD -#' (the model in the first row). For this reason the `elpd_diff` column -#' will always have the value `0` in the first row (i.e., the difference -#' between the preferred model and itself) and negative values in subsequent -#' rows for the remaining models. +#' When using `loo_compare()`, the returned matrix will have one row per model +#' and several columns of estimates. The values in the +#' [`elpd_diff`][loo-glossary] and [`se_diff`][loo-glossary] columns of the +#' returned matrix are computed by making pairwise comparisons between each +#' model and the model with the largest ELPD (the model in the first row). For +#' this reason the `elpd_diff` column will always have the value `0` in the +#' first row (i.e., the difference between the preferred model and itself) and +#' negative values in subsequent rows for the remaining models. #' -#' To compute the standard error of the difference in ELPD --- which should -#' not be expected to equal the difference of the standard errors --- we use a -#' paired estimate to take advantage of the fact that the same set of \eqn{N} -#' data points was used to fit both models. These calculations should be most -#' useful when \eqn{N} is large, because then non-normality of the -#' distribution is not such an issue when estimating the uncertainty in these -#' sums. These standard errors, for all their flaws, should give a better -#' sense of uncertainty than what is obtained using the current standard -#' approach of comparing differences of deviances to a Chi-squared +#' To compute the standard error of the difference in [ELPD][loo-glossary] --- +#' which should not be expected to equal the difference of the standard errors +#' --- we use a paired estimate to take advantage of the fact that the same +#' set of \eqn{N} data points was used to fit both models. These calculations +#' should be most useful when \eqn{N} is large, because then non-normality of +#' the distribution is not such an issue when estimating the uncertainty in +#' these sums. These standard errors, for all their flaws, should give a +#' better sense of uncertainty than what is obtained using the current +#' standard approach of comparing differences of deviances to a Chi-squared #' distribution, a practice derived for Gaussian linear models or #' asymptotically, and which only applies to nested models in any case. #' diff --git a/man/loo.Rd b/man/loo.Rd index 25811df9..0eaee55e 100644 --- a/man/loo.Rd +++ b/man/loo.Rd @@ -67,16 +67,17 @@ self-normalizing importance sampling when using posterior draws obtained with MCMC. If MCMC draws are used and \code{r_eff} is not provided then the reported PSIS effective sample sizes and Monte Carlo error estimates will be over-optimistic. If the posterior draws are independent then -\code{r_eff=1} and can be omitted. See the \code{\link[=relative_eff]{relative_eff()}} -helper functions for computing \code{r_eff}.} +\code{r_eff=1} and can be omitted. The warning message thrown when \code{r_eff} is +not specified can be disabled by setting \code{r_eff} to \code{NA}. See the +\code{\link[=relative_eff]{relative_eff()}} helper functions for computing \code{r_eff}.} \item{save_psis}{Should the \code{"psis"} object created internally by \code{loo()} be saved in the returned object? The \code{loo()} function calls \code{\link[=psis]{psis()}} internally but by default discards the (potentially large) \code{"psis"} object after using it to compute the LOO-CV summaries. Setting \code{save_psis=TRUE} -will add a \code{psis_object} component to the list returned by \code{loo}. Currently -this is only needed if you plan to use the \code{\link[=E_loo]{E_loo()}} function to compute -weighted expectations after running \code{loo}.} +will add a \code{psis_object} component to the list returned by \code{loo}. +Currently this is only needed if you plan to use the \code{\link[=E_loo]{E_loo()}} function to +compute weighted expectations after running \code{loo}.} \item{cores}{The number of cores to use for parallelization. This defaults to the option \code{mc.cores} which can be set for an entire R session by @@ -117,12 +118,12 @@ The \code{loo()} methods return a named list with class \code{c("psis_loo", "loo")} and components: \describe{ \item{\code{estimates}}{ -A matrix with two columns (\code{Estimate}, \code{SE}) and three rows -(\code{elpd_loo}, \code{p_loo}, \code{looic}). This -contains point estimates and standard errors of the expected log pointwise -predictive density (\code{elpd_loo}), the effective number of parameters -(\code{p_loo}) and the LOO information criterion \code{looic} (which is -just \code{-2 * elpd_loo}, i.e., converted to deviance scale). +A matrix with two columns (\code{Estimate}, \code{SE}) and three rows (\code{elpd_loo}, +\code{p_loo}, \code{looic}). This contains point estimates and standard errors of the +expected log pointwise predictive density (\code{\link[=loo-glossary]{elpd_loo}}), the +effective number of parameters (\code{\link[=loo-glossary]{p_loo}}) and the LOO +information criterion \code{looic} (which is just \code{-2 * elpd_loo}, i.e., +converted to deviance scale). } \item{\code{pointwise}}{ @@ -130,12 +131,12 @@ A matrix with five columns (and number of rows equal to the number of observations) containing the pointwise contributions of the measures (\code{elpd_loo}, \code{mcse_elpd_loo}, \code{p_loo}, \code{looic}, \code{influence_pareto_k}). in addition to the three measures in \code{estimates}, we also report -pointwise values of the Monte Carlo standard error of \code{elpd_loo} -(\code{mcse_elpd_loo}), and statistics describing the influence of +pointwise values of the Monte Carlo standard error of \code{\link[=loo-glossary]{elpd_loo}} +(\code{\link[=loo-glossary]{mcse_elpd_loo}}), and statistics describing the influence of each observation on the posterior distribution (\code{influence_pareto_k}). These are the estimates of the shape parameter \eqn{k} of the generalized Pareto fit to the importance ratios for each leave-one-out -distribution. See the \link{pareto-k-diagnostic} page for details. +distribution (see the \link{pareto-k-diagnostic} page for details). } \item{\code{diagnostics}}{ diff --git a/man/loo_compare.Rd b/man/loo_compare.Rd index 4a699bd6..c35c2056 100644 --- a/man/loo_compare.Rd +++ b/man/loo_compare.Rd @@ -42,28 +42,28 @@ By default the print method shows only the most important information. Use } \details{ When comparing two fitted models, we can estimate the difference in their -expected predictive accuracy by the difference in \code{elpd_loo} or -\code{elpd_waic} (or multiplied by \eqn{-2}, if desired, to be on the -deviance scale). +expected predictive accuracy by the difference in +\code{\link[=loo-glossary]{elpd_loo}} or \code{elpd_waic} (or multiplied by \eqn{-2}, if +desired, to be on the deviance scale). -When using \code{loo_compare()}, the returned matrix will have one row per -model and several columns of estimates. The values in the \code{elpd_diff} -and \code{se_diff} columns of the returned matrix are computed by making -pairwise comparisons between each model and the model with the largest ELPD -(the model in the first row). For this reason the \code{elpd_diff} column -will always have the value \code{0} in the first row (i.e., the difference -between the preferred model and itself) and negative values in subsequent -rows for the remaining models. +When using \code{loo_compare()}, the returned matrix will have one row per model +and several columns of estimates. The values in the +\code{\link[=loo-glossary]{elpd_diff}} and \code{\link[=loo-glossary]{se_diff}} columns of the +returned matrix are computed by making pairwise comparisons between each +model and the model with the largest ELPD (the model in the first row). For +this reason the \code{elpd_diff} column will always have the value \code{0} in the +first row (i.e., the difference between the preferred model and itself) and +negative values in subsequent rows for the remaining models. -To compute the standard error of the difference in ELPD --- which should -not be expected to equal the difference of the standard errors --- we use a -paired estimate to take advantage of the fact that the same set of \eqn{N} -data points was used to fit both models. These calculations should be most -useful when \eqn{N} is large, because then non-normality of the -distribution is not such an issue when estimating the uncertainty in these -sums. These standard errors, for all their flaws, should give a better -sense of uncertainty than what is obtained using the current standard -approach of comparing differences of deviances to a Chi-squared +To compute the standard error of the difference in \link[=loo-glossary]{ELPD} --- +which should not be expected to equal the difference of the standard errors +--- we use a paired estimate to take advantage of the fact that the same +set of \eqn{N} data points was used to fit both models. These calculations +should be most useful when \eqn{N} is large, because then non-normality of +the distribution is not such an issue when estimating the uncertainty in +these sums. These standard errors, for all their flaws, should give a +better sense of uncertainty than what is obtained using the current +standard approach of comparing differences of deviances to a Chi-squared distribution, a practice derived for Gaussian linear models or asymptotically, and which only applies to nested models in any case. } diff --git a/man/loo_subsample.Rd b/man/loo_subsample.Rd index 858809e9..55b204b3 100644 --- a/man/loo_subsample.Rd +++ b/man/loo_subsample.Rd @@ -60,8 +60,9 @@ self-normalizing importance sampling when using posterior draws obtained with MCMC. If MCMC draws are used and \code{r_eff} is not provided then the reported PSIS effective sample sizes and Monte Carlo error estimates will be over-optimistic. If the posterior draws are independent then -\code{r_eff=1} and can be omitted. See the \code{\link[=relative_eff]{relative_eff()}} -helper functions for computing \code{r_eff}.} +\code{r_eff=1} and can be omitted. The warning message thrown when \code{r_eff} is +not specified can be disabled by setting \code{r_eff} to \code{NA}. See the +\code{\link[=relative_eff]{relative_eff()}} helper functions for computing \code{r_eff}.} \item{save_psis}{Should the \code{"psis"} object created internally by \code{loo_subsample()} be saved in the returned object? See \code{\link[=loo]{loo()}} for details.} diff --git a/man/parallel_psis_list.Rd b/man/parallel_psis_list.Rd index 5669964f..d98e7315 100644 --- a/man/parallel_psis_list.Rd +++ b/man/parallel_psis_list.Rd @@ -54,16 +54,17 @@ self-normalizing importance sampling when using posterior draws obtained with MCMC. If MCMC draws are used and \code{r_eff} is not provided then the reported PSIS effective sample sizes and Monte Carlo error estimates will be over-optimistic. If the posterior draws are independent then -\code{r_eff=1} and can be omitted. See the \code{\link[=relative_eff]{relative_eff()}} -helper functions for computing \code{r_eff}.} +\code{r_eff=1} and can be omitted. The warning message thrown when \code{r_eff} is +not specified can be disabled by setting \code{r_eff} to \code{NA}. See the +\code{\link[=relative_eff]{relative_eff()}} helper functions for computing \code{r_eff}.} \item{save_psis}{Should the \code{"psis"} object created internally by \code{loo()} be saved in the returned object? The \code{loo()} function calls \code{\link[=psis]{psis()}} internally but by default discards the (potentially large) \code{"psis"} object after using it to compute the LOO-CV summaries. Setting \code{save_psis=TRUE} -will add a \code{psis_object} component to the list returned by \code{loo}. Currently -this is only needed if you plan to use the \code{\link[=E_loo]{E_loo()}} function to compute -weighted expectations after running \code{loo}.} +will add a \code{psis_object} component to the list returned by \code{loo}. +Currently this is only needed if you plan to use the \code{\link[=E_loo]{E_loo()}} function to +compute weighted expectations after running \code{loo}.} \item{cores}{The number of cores to use for parallelization. This defaults to the option \code{mc.cores} which can be set for an entire R session by diff --git a/man/psis_approximate_posterior.Rd b/man/psis_approximate_posterior.Rd index 504c5181..47101f38 100644 --- a/man/psis_approximate_posterior.Rd +++ b/man/psis_approximate_posterior.Rd @@ -44,9 +44,9 @@ setting \code{mc.cores} interactively or in a script is fine). saved in the returned object? The \code{loo()} function calls \code{\link[=psis]{psis()}} internally but by default discards the (potentially large) \code{"psis"} object after using it to compute the LOO-CV summaries. Setting \code{save_psis=TRUE} -will add a \code{psis_object} component to the list returned by \code{loo}. Currently -this is only needed if you plan to use the \code{\link[=E_loo]{E_loo()}} function to compute -weighted expectations after running \code{loo}.} +will add a \code{psis_object} component to the list returned by \code{loo}. +Currently this is only needed if you plan to use the \code{\link[=E_loo]{E_loo()}} function to +compute weighted expectations after running \code{loo}.} \item{...}{For the \code{loo.function()} method and the \code{loo_i()} function, these are the data, posterior draws, and other arguments to pass diff --git a/man/update.psis_loo_ss.Rd b/man/update.psis_loo_ss.Rd index 281cabac..8c400693 100644 --- a/man/update.psis_loo_ss.Rd +++ b/man/update.psis_loo_ss.Rd @@ -55,8 +55,9 @@ self-normalizing importance sampling when using posterior draws obtained with MCMC. If MCMC draws are used and \code{r_eff} is not provided then the reported PSIS effective sample sizes and Monte Carlo error estimates will be over-optimistic. If the posterior draws are independent then -\code{r_eff=1} and can be omitted. See the \code{\link[=relative_eff]{relative_eff()}} -helper functions for computing \code{r_eff}.} +\code{r_eff=1} and can be omitted. The warning message thrown when \code{r_eff} is +not specified can be disabled by setting \code{r_eff} to \code{NA}. See the +\code{\link[=relative_eff]{relative_eff()}} helper functions for computing \code{r_eff}.} \item{cores}{The number of cores to use for parallelization. This defaults to the option \code{mc.cores} which can be set for an entire R session by