% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/sim.R
\name{sim.master}
\alias{sim.master}
\title{Master function for running simulations.}
\usage{
sim.master(
  n,
  p,
  conformal.pred.funs,
  n0 = n,
  in.sample = FALSE,
  nrep = 20,
  seed = NULL,
  verbose = FALSE,
  file = NULL,
  file.rep = 5,
  x.dist = c("normal", "binom", "sn", "mix"),
  cor = c("none", "pair", "auto", "rand"),
  rho = 0.5,
  k = 5,
  standardize = TRUE,
  mean.fun = c("linear", "additive", "rotated"),
  m = 4,
  sparsity = c("strong", "weak"),
  s = round(log(p)),
  weak.decay = 0.5,
  mu.seed = NULL,
  error.dist = c("normal", "unif", "t", "sn"),
  sigma = 1,
  bval = 3,
  snr = NULL,
  sigma.type = c("const", "var"),
  df = 3,
  alpha = 5,
  omitted.vars = FALSE
)
}
\arguments{
\item{n, p}{Number of observations and features, respectively.}

\item{conformal.pred.funs}{A list containing conformal inference functions
to use. Each function here must take three arguments: x, y, x0 --- and no
more. These will typically be defined with a wrapper around, e.g.,
\code{\link{conformal.pred}} or \code{\link{conformal.pred.split}}. See
examples below.}

\item{n0}{The number of points at which to make predictions. Default is n.}

\item{in.sample}{Should the original x points be used for predictions (within
each repetition)? If TRUE, then the n0 argument (above) is ignored. Default
is FALSE.}

\item{nrep}{Number of repetitions over which to average results. Default is 
20.}

\item{seed}{Seed to be set for the overall random number generation, i.e.,
set before repetitions are begun (for reproducibility of the simulation
results). Default is NULL, which effectively sets no seed.}

\item{verbose}{Should intermediate progress be printed out? Default is FALSE.}

\item{file, file.rep}{Name of a file to which simulation results will be saved
(using saveRDS), and a number of repetitions after which intermediate
results will be saved. Setting file to NULL is interpreted to mean that no
simulations results should be saved; setting file.rep to 0 is interpreted
to mean that simulations results should be saved at the very end, i.e., no
intermediate saving. Defaults are NULL and 5, respectively.}

\item{x.dist, cor, rho, k, standardize, mean.fun, m, sparsity, s, weak.decay, }{error.dist,sigma,bval,snr,sigma.type,df,alpha,omitted.vars Arguments to 
pass to \code{\link{sim.xy}}, see the latter's help file for details.}
}
\value{
A list with the following components.
  \itemize{
  \item ave.cov, ave.len, ave.err: lists of length m, where m denotes the
   number of conformal methods evaluated (i.e., the length of the list
   conformal.pred.funs). Each element of ave.cov corresponds to a conformal
   method considered, and is a vector whose length is equal to the number of
   tuning parameters internal to this method, containing the empirical
   coverages of conformal intervals, averaged across the repetitions. The
   lists ave.len and ave.err are analogous, except they contain the average
   lengths of conformal intervals, and average test errors, respectively.
  \item ave.opt, ave.tim: lists of length m, analogous to ave.cov, ave.len,
   and ave.err (see above), but where ave.opt contains the average (relative)
   optimism of the methods, a unitless measure of model complexity defined
   by (test error - training error) / (test error); and ave.tim contains
   the average runtimes (in seconds) of the methods.
  \item sd.cov, sd.len, sd.err, sd.opt, sd.tim: same as above, but containing
    deviations, rather than averages, across the reptitions.
  \item cov, len, err, opt, tim: lists of length m, reporting the full set
    of metrics across repetitions (rather than averages or standard
    deviations).
  \item lev: a matrix of dimension nrep x n0, reporting the leverage scores
    of the test points, across the repetitions.
  \item ave.best.err, ave.best.len: matrices of dimension 5 x m, providing a
   summary of the average coverage, length, error, optimism, and time
   metrics (see description above), but at only one particular tuning
   parameter value for each method considered --- this is the tuning parameter
   value that either gives the best average error, or best average length.
  \item sd.best.err, sd.best.len: matrices of dimension 5 x m,
   providing a summary of the standard deviation of coverage, length, error,
   optimism, and time metrics for each method's best tuning parameter value
   (analogous to the construction of ave.best.err and ave.best.len).
  }
}
\description{
Run a set of simulations with the specified configuration.
}
\examples{
## Linear regression: test out conformal intervals and parametric intervals
## across a variety of settings

\dontrun{  
# Set some overall simulation parameters
n = 100; p = 10 # Numbers of observations and features
s = 10 # Number of truly relevant features
n0 = 100 # Number of points at which to make predictions
nrep = 50 # Number of repetitions for a given setting
sigma = 1 # Marginal error standard deviation
alpha = 0.1 # Miscoverage level

# Define conformal inference functions: these are basically just wrappers
# around a particular instatiation of conformal.pred or conformal.pred.split,
# i.e., particular training and prediction functions
my.lm.funs = lm.funs()
my.conf.fun = function(x, y, x0) {
  conformal.pred(x,y,x0,alpha=alpha,verb="\t\t",
                train.fun=my.lm.funs$train,
                predict.fun=my.lm.funs$predict)
}
my.split.fun = function(x, y, x0) {
  conformal.pred.split(x,y,x0,alpha=alpha,
                      train.fun=my.lm.funs$train,
                      predict.fun=my.lm.funs$predict)
}

# Hack together our own "conformal" inference function, really, just one that
# returns the parametric intervals
my.param.fun = function(x, y, x0) {
  lm.obj = lm(y~x)
  mat = predict(lm.obj,list(x=x0),interval="predict",level=1-alpha)
  # Return proper outputs in proper formatting
  return(list(pred=mat[,"fit",drop=F],lo=mat[,"lwr",drop=F],
              up=mat[,"upr",drop=F],fit=matrix(lm.obj$fit,ncol=1)))
}

# Now put together a list with our three conformal inference functions
conformal.pred.funs = list(my.conf.fun, my.split.fun, my.param.fun)
names(conformal.pred.funs) = c("Conformal","Split conformal","Parametric")

# Set some path, where we can save the rds files containing sim results
path = "."

# Simulation setting A: classical setup --- linear (fully supported) mean
# function; normal, homoskedastic errors; and normal, uncorrelated features
cat("========================================\n")
cat("                SETTING A               \n")
cat("========================================\n")
simA = sim.master(n, p, conformal.pred.funs, n0=n0, nrep=nrep, verb=TRUE,
  file=paste0(path,"simA.rds"), x.dist="normal", cor="none",
  mean.fun="linear", s=s, error.dist="normal", sigma=sigma, snr=snr,
  sigma.type="const")

# Simulation setting B: now, nonlinear mean, and t2 errors
cat("========================================\n")
cat("                SETTING B               \n")
cat("========================================\n")
simB = sim.master(n, p, conformal.pred.funs, n0=n0, nrep=nrep, verb=TRUE,
  file=paste0(path,"simB.rds"), x.dist="normal", cor="none",
  mean.fun="additive", m=4, s=s, error.dist="t", df=2, sigma=sigma, snr=snr,
  sigma.type="const")

# Simulation setting C: now, nonlinear mean, heteroskedastic t2 errors, and
# auto-correlated mix of x variables
cat("========================================\n")
cat("                SETTING C               \n")
cat("========================================\n")
simC = sim.master(n, p, conformal.pred.funs, n0=n0, nrep=nrep, verb=TRUE,
  file=paste0(path,"simC.rds"), x.dist="mix", cor="auto", k=5,
  mean.fun="linear", s=s, error.dist="t", df=2, sigma=sigma, snr=snr,
  sigma.type="var")
}
}
\references{
"Distribution-Free Predictive Inference for Regression" by Lei,
  G'Sell, Rinaldo, Tibshirani, Wasserman (2018).
}
\seealso{
\code{\link{sim.xy}}
}
