\section*{Checklist}

\begin{enumerate}

\item For all authors\ldots
\begin{enumerate}
  \item Do the main claims made in the abstract and introduction accurately reflect the paper's contributions and scope?
    \answerYes{}
  \item Did you describe the limitations of your work?
    \answerYes{See \cref{sec:discussion}.}
  \item Did you discuss any potential negative societal impacts of your work?
    \answerYes{See \cref{sec:broader_impact}.}
  \item Have you read the ethics review guidelines and ensured that your paper conforms to them?
    \answerYes{}
\end{enumerate}

\item If you are including theoretical results...
\begin{enumerate}
  \item Did you state the full set of assumptions of all theoretical results?
    \answerYes{Assumptions are listed in the theorem/lemma statements, as well as in \cref{sec:assumptions}.}
	\item Did you include complete proofs of all theoretical results?
    \answerYes{See \cref{sec:proof_dgp_gp,sec:proof_tails,sec:dgp_second_moments}.}
\end{enumerate}

\item If you ran experiments...
\begin{enumerate}
  \item Did you include the code, data, and instructions needed to reproduce the main experimental results (either in the supplemental material or as a URL)?
    \answerYes{Code is in supplementary.}
  \item Did you specify all the training details (e.g., data splits, hyperparameters, how they were chosen)?
    \answerYes{See \cref{sec:experimental_details}.}
	\item Did you report error bars (e.g., with respect to the random seed after running experiments multiple times)?
    \answerYes{%
      Our results for optimized networks are run 3-5 times, and indicate as such in the results figures.
      Due to compute resource constraints we only have one HMC run for each experiment in \cref{sec:experiments_bayesian}.
    }
	\item Did you include the total amount of compute and the type of resources used (e.g., type of GPUs, internal cluster, or cloud provider)?
    \answerYes{See \cref{sec:experimental_details}.}
\end{enumerate}

\item If you are using existing assets (e.g., code, data, models) or curating/releasing new assets...
\begin{enumerate}
  \item If your work uses existing assets, did you cite the creators?
    \answerYes{
      In \cref{sec:experiments_bayesian,sec:experiments_optimized} we supply citations for all datasets.
      In \cref{sec:experimental_details} we supply citations for all software packages.
    }
  \item Did you mention the license of the assets?
    \answerNo{
      To the best of our knowledge, none of the datasets we use have licenses.
      We mention that all software packages we use are open source.
    }
  \item Did you include any new assets either in the supplemental material or as a URL?
    \answerNo{}
  \item Did you discuss whether and how consent was obtained from people whose data you're using/curating?
    \answerNA{}
  \item Did you discuss whether the data you are using/curating contains personally identifiable information or offensive content?
    \answerNo{
      We use commonly-available public datasets that,
      to the best of our knowledge, do not pose any privacy or content concerns.
    }
\end{enumerate}

\item If you used crowdsourcing or conducted research with human subjects...
\begin{enumerate}
  \item Did you include the full text of instructions given to participants and screenshots, if applicable?
    \answerNA{}
  \item Did you describe any potential participant risks, with links to Institutional Review Board (IRB) approvals, if applicable?
    \answerNA{}
  \item Did you include the estimated hourly wage paid to participants and the total amount spent on participant compensation?
    \answerNA{}
\end{enumerate}

\end{enumerate}

\section*{Old work below}

\paragraph{Additional observations for 2-layer additive Deep GP.}
The additive and generative Deep GP represent two different classes of behavior in their infinite-width limit.
In the additive case, the first two moments of the prior remain constant with $H$.
However, the distribution of $h_\text{add}(\cdot)$ itself changes as $H$ increases: from a Deep GP to a (shallow) GP.
The general Deep GP exhibits different behavior.
Its second moment changes with $H$ while its tails also become more Gaussian.

\paragraph{Additional observations for general Deep GP.}
In the general case, how width affects the second moment depends on the choice of $k_g(\cdot, \cdot)$.
For example, if $k_g(\cdot, \cdot)$ is a (non-additive) RBF kernel, width results in faster decaying covariance functions.
To decouple covariance and width, we can make use of the following construction:
\begin{observation}
  Let $g(\ensuremath{\mathbf{f}}(\cdot))$ be a 2-layer zero-mean Deep GP of width-$H$.
  There exists a width-$1$ Deep GP $g'(f'(\cdot))$ with an equivalent second moment.
\end{observation}
\begin{proof}
  Let $k_g(\cdot, \cdot): \ensuremath{\mathbb{R}}^{H \times H} \to \ensuremath{\mathbb{R}}$ be the covariance function of $g(\cdot)$.
  The covariance of $g(\ensuremath{\mathbf{f}}(\cdot))$ is given by:
  %
  \begin{align*}
    \Ev{g(\ensuremath{\mathbf{f}}(\ensuremath{\mathbf{x}})) g(\ensuremath{\mathbf{f}}(\ensuremath{\mathbf{x}}'))}
    &= \int k_g(\ensuremath{\mathbf{f}}(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}(\ensuremath{\mathbf{x}}')) \: p(\ensuremath{\mathbf{f}}(\ensuremath{\mathbf{x}})) \: p(\ensuremath{\mathbf{f}}(\ensuremath{\mathbf{x}}')) \intd \ensuremath{\mathbf{f}}(\ensuremath{\mathbf{x}}) \intd \ensuremath{\mathbf{f}}(\ensuremath{\mathbf{x}}').
    \\
    &= \int k_g(\ensuremath{\mathbf{f}}(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}(\ensuremath{\mathbf{x}}')) \: \prod_{i=1}^H \left[ p(f_i(\ensuremath{\mathbf{x}})) \: p(f_i(\ensuremath{\mathbf{x}}')) \right] \intd f_1(\ensuremath{\mathbf{x}}) \ldots \intd f_H(\ensuremath{\mathbf{x}}').
  \end{align*}
  %
  Let $k_g'(\cdot, \cdot)$ be defined as:
  %
  \begin{align*}
    k_g'(f(\ensuremath{\mathbf{x}}), f(\ensuremath{\mathbf{x}}'))
    &= \int k_g(\ensuremath{\mathbf{f}}(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}(\ensuremath{\mathbf{x}}')) \: \prod_{i=1}^{H-1} \left[ p(f_i(\ensuremath{\mathbf{x}})) \: p(f_i(\ensuremath{\mathbf{x}}')) \right] \intd f_1(\ensuremath{\mathbf{x}}) \ldots \intd f_{H-1}(\ensuremath{\mathbf{x}}').
  \end{align*}
  %
  In other words, $k_g'(f(\ensuremath{\mathbf{x}}), f(\ensuremath{\mathbf{x}}))$ is the covariance of $g(\ensuremath{\mathbf{f}}(\cdot))$ without $f_H(\ensuremath{\mathbf{x}}), f_H(\ensuremath{\mathbf{x}}')$ marginalized out.
  A simple manipulation of kernel construction rules will show that $k_g'(\cdot, \cdot)$ is a valid kernel.
  If we set $k_g'$ as the covariance function of $g'$, then $g'(f'(\cdot))$ will have the same second moment as $g(\ensuremath{\mathbf{f}}(\cdot))$.
\end{proof}
It is worth noting that the tails of this width-$1$ construction will be more Gaussian than that of $g(\ensuremath{\mathbf{f}}(\ensuremath{\mathbf{x}}))$.
(This can be proved with a simple application of Jensen's inequality.)

\paragraph{Additional observations for stationary Deep GP.}
\begin{observation}
  Let $h(\cdot) = g(\ensuremath{\mathbf{f}}(\cdot))$ be a 2-layer zero-mean Deep GP.
  If $k_g$ is a stationary kernel---or more generally, if $k_g(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}') = o^2$ for all $\ensuremath{\mathbf{z}} \in \ensuremath{\mathbb{R}}^H$---then the marginal distributions $p\left( h(\ensuremath{\mathbf{x}}) \right)$ are Gaussian.
\end{observation}
\begin{proof}
  The Gaussian distribution satisfies Carleman's condition, and therefore is uniquely determined by its moments.
  Consequentially, our proof strategy is showing that the moments of $h(\ensuremath{\mathbf{x}})$ match that of a Gaussian.
  For any $R \in \mathcal{N}$, we have that:
  %
  \begin{align*}
    \Evover{h(\cdot)}{ h(\ensuremath{\mathbf{x}})^R }
    &=
    \Evover{\ensuremath{\mathbf{f}}(\cdot)}{ \Evover{g(\cdot) \mid \ensuremath{\mathbf{f}}(\cdot) }{ g( \ensuremath{\mathbf{f}}(\ensuremath{\mathbf{x}}) )^R } }.
  \end{align*}
  %
  Since $g(\cdot) \mid \ensuremath{\mathbf{f}}(\cdot)$ is a zero-Gaussian process, the inner expectation is the $R^\text{th}$ moment of a zero-mean Gaussian.
  For odd $R$, we thus have $\Evover{h(\cdot)}{ h(\ensuremath{\mathbf{x}})^R } = \Evover{\ensuremath{\mathbf{f}}(\cdot)}{0} = 0$.
  For even $R$, we have
  $$
    \Evover{h(\cdot)}{h(\ensuremath{\mathbf{x}})^R } = \Evover{\ensuremath{\mathbf{f}}(\cdot)}{ (R-1)!! \: k( \ensuremath{\mathbf{f}}(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}(\ensuremath{\mathbf{x}}))^{R / 2} }
    = \Evover{\ensuremath{\mathbf{f}}(\cdot)}{ (R-1)!! \: o^R } = (R-1)!! \: o^R.
  $$
  Thus the moments of $h(\ensuremath{\mathbf{x}})$ match that of a Gaussian with zero mean and variance $o^2$.
\end{proof}

\section{Deep Gaussian Processes Collapse to Shallow Gaussian Processes}
\label{sec:dgp}
Deep GP \citep{damianou2013deep,damianou2015deep,bui2016deep,salimbeni2017doubly} are hierarchical models where layers $\ensuremath{\mathbf{f}}_1(\cdot) \ldots f_L(\cdot)$ are (vector-valued) GP:
\begin{equation}
  \text{DGP}(\ensuremath{\mathbf{x}}) = f_{L} \circ \ldots \circ \ensuremath{\mathbf{f}}_{1} \left( \ensuremath{\mathbf{x}} \right),
  \quad
  \ensuremath{\mathbf{f}}_{i}(\cdot) = [f_{i}^{(1)}( \cdot ), \ldots, f_{i}^{(H_i)}( \cdot ) ],
  \quad
  f_{i}^{(j)}( \cdot ) \stackrel{\text{i.i.d}}{\sim} \GP{ 0 }{ k_i(\cdot, \cdot) }.
  \label{eqn:dgp_def}
\end{equation}
$H_i$ is the width of the $i^\text{th}$ GP, and the output dimensions of each $\ensuremath{\mathbf{f}}_i(\cdot)$ are independent.
By using GP as the primary hierarchical building blocks, Deep GP are generally nonparametric and, assuming the GP layers use universal kernels \citep{micchelli2006universal}, have infinite representational capacity (see \cref{sec:dgp_capacity}).

\paragraph{Deep GP versus GP.}
Deep GP seek to offer more expressivity: conventional single-layer GP---though also nonparametric---are inherently limited by the choice of the prior covariance function \cite{bui2016deep,salimbeni2017doubly}.
For example, a GP with a RBF covariance is not suitable for data with discontinuities or sharp changes.
However, stacking two RBF GP together---$f_2(\ensuremath{\mathbf{f}}_1(\cdot))$---can overcome this limitation, since $\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})$ can encode a warping of $\ensuremath{\mathbf{x}}$ that ``smoothes'' the input data for $f_2(\cdot)$ (as we will show in \cref{fig:adaptive_vs_gaussian_rbf}).
Empirically, Deep GP have been shown to offer much more accurate predictive posteriors than standard GP \citep[e.g.][]{damianou2013deep,damianou2015deep,cutajar2017random,salimbeni2017doubly,havasi2018inference,blomqvist2019deep,dutordoir2020bayesian}.

\paragraph{Deep GP versus neural networks.}
(Bayesian) feed-forward neural networks are a strict subclass of Deep GP, albeit a degenerate one \cite{aitchison2020bigger,louizos2016structured,ober2020global}.
The first neural network layer is a GP with a linear kernel, while subsequent layers are GP with the kernel
$k(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}') = \beta^2 + \frac{1}{H_{i-1}} \sum_{i=1}^{H_{i-1}} \sigma( z_i ) \: \sigma( z_i )$.
A neural network, unlike other Deep GP, does not have infinite capacity.
Put loosely, a single neural network hidden unit corresponds to a single basis,
while in general a single Deep GP unit corresponds to a potentially-infinite basis.
See \citep{agrawal2020wide,aitchison2020bigger,cutajar2017random,duvenaud2014avoiding,dutordoir2021deep,ober2020global} and \cref{sec:dgp_degenerate} for more discussion on this connection.
The critical takeaway is that all of our Deep GP results apply to neural networks as well.

\subsection{Wide Deep GP are Gaussian Processes}
Having established a model where width does not effect capacity,
we now establish what remaining effects width has.
Empirical evidence suggests that the choice of width impacts Deep GP predictions \citep{bui2016deep,havasi2018inference}.
In practice it is common to make Deep GP as wide as comparably-sized neural networks;
\citet{salimbeni2017doubly} for example train Deep GP with $\geq30$ units per layer.

\emph{Surprisingly, here we prove that---in the limit of infinite width---Deep GP collapse to single-layer Gaussian processes.}
Our proof relies on the conditional covariance analysis of the previous section.
If the GP layers have non-pathological covariance functions\footnote{
  Any textbook kernel (isotropic kernels, dot product kernels, etc.)
  or any covariance function with a Fourier-Steiljes representation is ``non-pathological;''
  see \cref{sec:assumptions} for formal assumptions.
}---the Deep GP conditional covariance becomes almost surely constant with width (see \cref{lemma:dgp_cond_covar}, \cref{sec:proof_dgp_gp}).
Combining this with \cref{lemma:dgp_cf}:
\begin{theorem}
  \vspace{0.5em}
  Let $f_{L} \circ \ldots \circ \ensuremath{\mathbf{f}}_{1} \left( \ensuremath{\mathbf{x}} \right)$ be a zero-mean Deep GP (Eq.~\ref{eqn:dgp_def}),
  where each layer satisfies \cref{ass:kernels_harmonizable,ass:kernels_dimensionality} (non-pathological prior covariances that scale with dimensionality---see \cref{sec:assumptions}).
  Then $\lim_{H_L \to \infty} \cdots \lim_{H_1 \to \infty} f_{L} \circ \ldots \circ \ensuremath{\mathbf{f}}_{1} \left( \ensuremath{\mathbf{x}} \right)$
  converges in distribution to a (single-layer) GP.
  \label{thm:dgp_gp}
\end{theorem}
(See \cref{sec:proof_dgp_gp} for proof.)
The implications of \cref{thm:dgp_gp} are paradoxical and unsettling.
Deep GP are motivated as a more powerful model than standard GP.
However, as we make the model wider, we arrive back where we started---a Gaussian process (although one with a different prior covariance).

A neural network gains representational power in its GP limit, transitioning from a finite-basis model to a nonparametric model.
The Deep GP limit on the other hand has no additional representational power, since Deep GP are already universal approximators at any width.
(Indeed this fact motivates their use as a control.)
The only difference between finite and infinite width Deep GP is the prior distribution itself:
transitioning from non-Gaussian to Gaussian with increasing width.
In the next section, we investigate how this transition affects model performance.

\section{Discussion}
\label{sec:discussion}

This paper shows that, across typical neural networks, Deep GP, and Bayesian neural networks,
\emph{width is fundamentally detrimental after sufficient capacity is reached}.

Even with that result, we can ask when large width might still be beneficial?
Our results assume that layers are conditionally Gaussian, and different priors may have different effects.
Researchers have touted benefits of width, both in terms of a more favorable optimization landscape \citep{nguyen2017loss,li2018visualizing,soltanolkotabi2018theoretical,arora2019fine,du2019gradient} and better generalization through the double descent phenomenon \citep{belkin2019reconciling,nakkiran2020deep,cao2020generalization}.
Moreover, there are (non-Bayesian) infinite-width constructions
that avoid the limiting kernel behavior studied in this work \citep[e.g.][]{chizat2018global,mei2018mean,golikov2020towards,yang2020feature}.
We emphasize that our results do not conflict with these prior works, but rather reflect a different perspective.
The models we study correspond to a Gaussian prior on parameters.
Relaxing this correspondence, for example by changing the weight decay parameter, may lessen the consequences of width that we observe.
Nevertheless, our results suggest that the inductive bias of width is harmful, even if these undesirable effects can be avoided via careful construction.

Finally, it is worth considering when one might still choose a conventional shallow GP over a deep model.
An often-touted benefit of Gaussian processes is the ability to encode prior domain knowledge via the choice of covariance function.
In \cref{sec:dgp_second_moments}, we prove that certain prior covariances cannot be expressed by adaptable hierarchical models.
For example, a Deep GP that is composed of stationary GP layers cannot model anti-correlations a priori (\cref{thm:no_neg_covar}, \cref{sec:dgp_second_moments}),
whereas (single-layer) stationary GP can have positive and negative prior covariances.
Nevertheless, Deep GP are capable of modeling many common covariance functions, including the RBF, Mat\'ern, and rational quadratic kernels.
In \cref{sec:dgp_second_moments} we demonstrate a 2-layer Deep GP construction of any width that is capable of producing prior covariances that match  most isotropic kernels (\cref{thm:radial}, \cref{sec:dgp_second_moments}).
In other words, a Deep GP can match the first and second moments of most GP, while also offering an adaptable posterior.

\section{Experiments}
\subsection{Regression with Deep GP and Bayesian Neural Networks}
\label{sec:experiments_bayesian}

To isolate the effects of width and depth, each experiment compares Deep GP/Bayesian neural networks that share the same first and second prior moments,
and the Deep GP models use GP layers with universal kernels.
To remove any potential side effects from approximate inference methods,
we sample Deep GP/neural network posteriors using NUTS \citep{hoffman2014no} and do not use any stochastic inducing point \citep{salimbeni2017doubly,havasi2018inference} or finite basis \citep{cutajar2017random} approximations.
This inference is costly and scales cubically with $N$; therefore, we subsample all training datasets to $N \leq 1000$.
See \cref{sec:experimental_details} for experimental details.

\begin{figure}[t!]
  \centering
  \includegraphics[width=\linewidth]{figures/dgp_width.pdf}
  \caption{
    {\bf Top:} Test set log likelihood (LL) of 2-layer Deep GP (and neural networks) regression as a function of width (higher is better).

    Numbers are shifted so that $0$ corresponds to the limiting GP log likelihood.
    Narrow models achieve the best log likelihood, and performance degrades with width.
    {\bf Bottom:} Fit of the posterior kernel $k(\ensuremath{\mathbf{f}}_1(\cdot), \ensuremath{\mathbf{f}}_1(\cdot))$ on the training data, as measured by Gaussian log marginal likelihood (higher is better).
    $0$ corresponds to the limiting GP log marginal likelihood.
    Fit becomes increasingly worse with width.
  }
  \label{fig:dgp_width}
\end{figure}

\paragraph{Effect of width.}
We compare 2-layer Deep GP of various width on 6 regression datasets from the UCI dataset repository \citep{asuncion2007uci}
(see \cref{sec:additional_results} for 3-layer results).
The first GP layers use a RBF kernel for the prior covariance,
while the second layers use a sum of one-dimensional RBF covariance functions.
We additionally compare against the limiting (single-layer) GP with the same prior covariance ({\bf Lim. GP}).
For each dataset, we choose hyperparameters that maximize the Lim. GP log marginal likelihood.
In \cref{fig:dgp_width} (top row) we see a near-monotonic performance degradation as width increases.
The width-2 optimum may represent the ``sweet spot'' for Deep GP width,
but it may instead be a side-effect of inference difficulties for width-1 models
(see \cref{sec:additional_results} for a control experiment).
Regardless, as our theory predicts, \emph{width is detrimental to Deep GP predictive performance}.

We repeat the experiment for 2-layer neural networks
(and 3-layer models in \cref{sec:additional_results}),
where here the Lim. GP corresponds to the arc-cosine kernel \citep{cho2009kernel,lee2017deep}.
\cref{fig:dgp_width} indicates an optimal width with regards to test set log likelihood, usually between $8-16$ hidden units.
We expect this optimum exists (and differs from the Deep GP optimum) because narrow models have too few basis functions for these datasets.
Thus, after sufficient capacity, \emph{width is harmful to Bayesian neural networks}.

\paragraph{Adaptable versus non-adaptable RKHS.}
One way to measure the ``fit'' of a kernel $k(\cdot,\cdot)$ on a regression training dataset $\ensuremath{\boldmatrix{X}}, \ensuremath{\mathbf{y}}$ is the Gaussian log marginal likelihood
$\log \normaldist{ \ensuremath{\mathbf{y}}; \ensuremath{\mathbf{0}} }{\ensuremath{\boldmatrix{K}}(\ensuremath{\boldmatrix{X}}, \ensuremath{\boldmatrix{X}}) + \sigma^2 \ensuremath{\boldmatrix{I}}}$, where $\sigma^2$ is an observational noise parameter \citep[e.g.][]{rasmussen2006gaussian}.
To demonstrate how Deep GP/neural network posteriors correspond to adaptable RKHS mixtures,
the bottom row of \cref{fig:dgp_width} plots the ``kernel fit'' of $k_2(\ensuremath{\mathbf{f}}_1(\cdot), \ensuremath{\mathbf{f}}_1(\cdot))$ for posterior samples of $\ensuremath{\mathbf{f}}_1(\cdot)$ (see Eq.~\ref{eqn:dgp_posterior_mean}).
A higher fit corresponds to a model that is better adapted to the dataset $\ensuremath{\boldmatrix{X}}, \ensuremath{\mathbf{y}}$.
We see that narrower Deep GP almost universally achieve better kernel fit than wider Deep GP,
which converge to the same fit as the limiting GP.
(Standard deviations, depicted by shaded regions, are generally imperceptible.)
Bayesian neural networks achieve best ``kernel fit'' at $8-16$ hidden units, and then converge to the limiting Deep GP with further width.

\paragraph{Effect of depth, controlling for covariance.}
\cref{tab:dgp_depth} displays Deep GP test set log likelihood as a function of depth.
Again, we isolate the tail effects of depth by ensuring that all models share the same first and second moments.
We construct a GP and a 2-layer Deep GP that match the moments of a width-2 3-layer Deep GP with RBF covariances,
and we use hyperparameters that maximize the limiting GP marginal likelihood for each dataset.
Note that computing the limiting covariance of $\geq 3$ layer models involves intractable integrals that we approximate with quadrature (see \cref{sec:limiting_kernels}).
Our findings confirm that---in this controlled setting---depth unlike width improves test set performance.

\subsection{Standard (Optimized, Non-Bayesian) Neural Networks}
\label{sec:experiments_optimized}
Here our goal is to represent ``real world'' models without a Bayesian treatment,
and thus we do not control the prior covariance across models.
However, we scale the outputs of layer-$\ell$ by $1/\sqrt{H_\ell}$ to match the GP-limiting neural net construction in \cref{eqn:2layer_nn}.
We choose the $L_2$ regularization parameter so that it corresponds to a parameter prior of $\normaldist{0}{20}$.\footnote{
  The $L_2$ parameter equivalent to this prior decreases with $N$, and---due to the $\sqrt{1/{H_\ell}}$ scaling---effectively increases with width.
  The $\normaldist{0}{20}$ prior is equivalent to a weight decay of $10^{-4}$ for $N=50000$ and $H_\ell=100$.
}
All models are trained without data augmentation.
\begin{table}[t!]
  \caption{%
    Test set log likelihood (LL) of Deep GP regression as a function of depth (higher is better).
    $\text{Depth}=1$ refers to the limiting GP.
    For each dataset, the models are constructed to have the same first and second moments.
    Unlike width, deeper models generally have better performance.
  }
  \label{tab:dgp_depth}
  \centering
  \resizebox{\linewidth}{!}{%
    \input tables/uci_depth_ll.tex
  }
\end{table}
\begin{figure}
  \centering \makebox[\textwidth][c]{%
    \includegraphics[width=0.56\linewidth]{figures/mnist_acc.pdf}
    \hfil
    \includegraphics[width=0.415\linewidth]{figures/cifar10_acc.pdf}
  }
  \caption{%
    Effect of width on standard (non-Bayesian) neural networks.
    Shaded regions depict standard error.
    {\bf Left:} 3-layer feed-forward networks trained on various subsets of MNIST.
    Test set accuracy is maximized when width $\approx 100$,
    after which width becomes decremental.
    {\bf Right:} LeNet and wide ResNets trained on CIFAR-10.
    The best LeNet has $\geq 16000$ hidden units, likely because this model is too shallow for sufficient capacity.
    For deeper ResNet models, accuracy is best when width $\leq 500$.
  }
  \label{fig:mnist_cifar_acc}
\end{figure}
\cref{fig:mnist_cifar_acc} depicts test set accuracy for increasingly wide models trained on MNIST \citep{lecun1998gradient} (left) and CIFAR-10 \citep{krizhevsky2009learning} (right).
On MNIST we examine two sequences of 3-layer (non-convolutional) networks:
one where the width of both hidden layers increases simultaneously ($H_1 = H_2$),
and one where all but the final layer have fixed width ($H_1 = 256$).
We test these sequences on variously-sized subsets of the training data, and observe a maximum test set accuracy for width $\approx 100$ networks.
On CIFAR-10 we examine 5-layer LeNet models \citep{lecun1998gradient} and $8$- and $14$-layer wide ResNet models \citep{he2016deep,zagoruyko2016wide}.
For LeNet, we only vary the width of the fully connected layers, and we observe varied performance behavior with width, possibly indicating LeNet is too shallow for sufficient capacity on this dataset.
Indeed, deeper ResNets---where we increase width of all layers simultaneously---%
achieve best accuracy when width $\leq 500$.
These results reiterate that even in typical networks, once sufficient capacity is reached, \emph{large width adversely affects model performance}.

\section{A Covariance Perspective on Infinite Width}
\subsection{A Covariance Perspective on Gaussian Process Limiting Behavior}
\label{sec:kernelized_nn}

To decouple the effects of increasing width and capacity, we first prove a new result about GP limits for a more general class of models, including Deep GP as well as typical neural networks. This result forms a necessary foundation for the subsequent theorems that are one main contribution of this work.
To begin, note that the proof technique introduced by \citet{neal1995bayesian} and extended by others \citep{lee2017deep,matthews2018gaussian,garriga2019deep,novak2019bayesian,yang2019tensor,hron2020infinite}
relies on the multivariate central limit theorem, which requires a model with additive structure.
Deep GP do not generally decompose in an additive manner, so we establish a more general proof technique.
For simplicity, we first present it in the context of neural networks, and then extend it to a more general class of models.

Consider the 2-layer neural network $f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}))$, with
$\ensuremath{\mathbf{f}}_1: \ensuremath{\mathbb{R}}^D \to \ensuremath{\mathbb{R}}^{H_1}$ and $f_2 : \ensuremath{\mathbb{R}}^{H_1} \to \ensuremath{\mathbb{R}}$:
\begin{equation}
  \ensuremath{\mathbf{f}}_1(\cdot) = \ensuremath{\boldmatrix{W}}_1^\top (\cdot) + \beta \ensuremath{\mathbf{b}}_1,
  \qquad
  f_2(\cdot) = {\textstyle \frac{1}{\sqrt{H_1}}} \ensuremath{\mathbf{w}}_2^\top \ensuremath{\boldsymbol{\sigma}}(\cdot) + \beta b_2.
  \label{eqn:2layer_nn}
\end{equation}
$\ensuremath{\boldsymbol{\sigma}}(\cdot)$ is an elementwise nonlinearity,
$\beta$ is a positive constant,
and $\ensuremath{\boldmatrix{W}}_1$, $\ensuremath{\mathbf{b}}_1$, $\ensuremath{\mathbf{w}}_2$, and $b_2$ are i.i.d. Normal.
With randomly initialized parameters, $f_2(\ensuremath{\mathbf{f}}_1(\cdot)): \ensuremath{\mathbb{R}}^D \to \ensuremath{\mathbb{R}}$ is a prior distribution over functions,
and this distribution converges to a GP in the infinite width limit \citep{neal1995bayesian}.

\begin{lemma}\label{lemma:nn_cf}
  \vspace{0.5em}
  The neural network defined in \cref{eqn:2layer_nn} is a Gaussian process if and only if---%
  for any finite set of inputs $\ensuremath{\boldmatrix{X}} = [\ensuremath{\mathbf{x}}_1, \ldots, \ensuremath{\mathbf{x}}_N]$---%
  the conditional prior covariance $\Evover{\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{X}}, \ensuremath{\boldmatrix{W}}_1, \ensuremath{\mathbf{b}}_1}{ \ensuremath{\mathbf{f}}_2 \ensuremath{\mathbf{f}}_2^\top }$
  is almost surely equal to the marginal prior covariance $\Evover{\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{X}}}{ \ensuremath{\mathbf{f}}_2 \ensuremath{\mathbf{f}}_2^\top }$,
  where $\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{X}} \triangleq [f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_1)), \ldots, f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_N))]$.
\end{lemma}
\begin{proof}
  By definition, $f_2(\ensuremath{\mathbf{f}}_1(\cdot))$ is a GP if and only if
  $\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{X}}$ is multivariate Gaussian for any $\ensuremath{\boldmatrix{X}}$.
  From \cref{eqn:2layer_nn}, we have
  $
  p(\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{X}}, \ensuremath{\boldmatrix{W}}_1, \ensuremath{\mathbf{b}}_1) = \normaldist{\ensuremath{\mathbf{0}}}{\ensuremath{\boldmatrix{K}}_{\ensuremath{\boldmatrix{W}}_1, \ensuremath{\mathbf{b}}_1}(\ensuremath{\boldmatrix{X}},\ensuremath{\boldmatrix{X}})},
  $
  where $[\ensuremath{\boldmatrix{K}}_{\ensuremath{\boldmatrix{W}}_1, \ensuremath{\mathbf{b}}_1}(\ensuremath{\boldmatrix{X}},\ensuremath{\boldmatrix{X}})]_{ij} = \beta^2 + \frac{1}{H_1} \ensuremath{\boldsymbol{\sigma}}(\ensuremath{\boldmatrix{W}}_1^\top \ensuremath{\mathbf{x}}_i + \beta \ensuremath{\mathbf{b}}_1)^\top \ensuremath{\boldsymbol{\sigma}}(\ensuremath{\boldmatrix{W}}_1^\top \ensuremath{\mathbf{x}}_j + \beta \ensuremath{\mathbf{b}}_1)^\top$ ($\ensuremath{\boldmatrix{K}}_{\ensuremath{\boldmatrix{W}}_1, \ensuremath{\mathbf{b}}_1}(\ensuremath{\boldmatrix{X}}, \ensuremath{\boldmatrix{X}})$ is the appropriate kernel Gram matrix).
  Using Jensen's inequality, we have a lower bound on the characteristic function of $\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{X}}$:

  %
  \begin{align}
    \Evover{\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{X}}}{ \exp \left( i \ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_2 \right) }
    &=
    \Evover{\ensuremath{\boldmatrix{W}}_1, \ensuremath{\mathbf{b}}_1}{
      \Evover{\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{X}}, \ensuremath{\boldmatrix{W}}_1, \ensuremath{\mathbf{b}}_1}{ \exp \left( i \ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_2 \right) }
    }
    \tag{law of total expectation}
    \\
    &=
    \mathbb{E}_{\ensuremath{\boldmatrix{W}}_1, \ensuremath{\mathbf{b}}_1} \biggl[
      \exp \left( -\tfrac 1 2 \ensuremath{\mathbf{t}}^\top \ensuremath{\boldmatrix{K}}_{\ensuremath{\boldmatrix{W}}_1, \ensuremath{\mathbf{b}}_1}(\ensuremath{\boldmatrix{X}},\ensuremath{\boldmatrix{X}}) \ensuremath{\mathbf{t}} \right)
    \biggr]
    \tag{char. func. of a Gaussian}
    \\
    &\geq \exp \left(
      -\tfrac 1 2 \ensuremath{\mathbf{t}}^\top \Evover{ \ensuremath{\boldmatrix{W}}_1, \ensuremath{\mathbf{b}}_1 }{ \ensuremath{\boldmatrix{K}}_{\ensuremath{\boldmatrix{W}}_1, \ensuremath{\mathbf{b}}_1}(\ensuremath{\boldmatrix{X}},\ensuremath{\boldmatrix{X}})
      } \ensuremath{\mathbf{t}}
    \right).
    \tag{convexity of $\exp$}
  \end{align}
  %
  This lower bound happens to be the characteristic function
  of $\normaldist{\ensuremath{\mathbf{0}}}{\Evover{\ensuremath{\boldmatrix{W}}_1, \ensuremath{\mathbf{b}}_1}{ \ensuremath{\boldmatrix{K}}_{\ensuremath{\boldmatrix{W}}_1, \ensuremath{\mathbf{b}}_1}(\ensuremath{\boldmatrix{X}}, \ensuremath{\boldmatrix{X}}) }}.$
  Since $\exp$ is strictly convex, the characteristic function of $\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{X}}$ equals the Gaussian lower bound $\forall \ensuremath{\mathbf{t}}$
  if and only if $p(\ensuremath{\boldmatrix{K}}_{\ensuremath{\boldmatrix{W}}_1, \ensuremath{\mathbf{b}}_1}(\ensuremath{\boldmatrix{X}},\ensuremath{\boldmatrix{X}}) \mid \ensuremath{\boldmatrix{W}}_1, \ensuremath{\mathbf{b}}_1) = \Evover{\ensuremath{\mathbf{f}} \mid \ensuremath{\boldmatrix{X}}, \ensuremath{\boldmatrix{W}}_1, \ensuremath{\mathbf{b}}_1}{\ensuremath{\mathbf{f}}_2 \ensuremath{\mathbf{f}}_2^\top}$ is a constant with probability 1.
\end{proof}
Seeing that $\frac{1}{H_1} \ensuremath{\boldsymbol{\sigma}}(\ensuremath{\boldmatrix{W}}_1^\top \ensuremath{\mathbf{x}}_i + \beta \ensuremath{\mathbf{b}}_1)^\top \ensuremath{\boldsymbol{\sigma}}(\ensuremath{\boldmatrix{W}}_1^\top \ensuremath{\mathbf{x}}_j + \beta \ensuremath{\mathbf{b}}_1)^\top$
becomes a.s. constant as $H_1 \to \infty$,
\cref{lemma:nn_cf} re-establishes the result of \citet{neal1995bayesian} (see \cref{sec:additive_dgp_proof}).
Critically, unlike \citeauthor{neal1995bayesian}'s proof, \cref{lemma:nn_cf} neither relies on the central limit theorem nor requires $f_2(\ensuremath{\mathbf{f}}_1(\cdot))$ to be a neural network;
it holds if $p(\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_1), \ldots, \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_N))$ is Gaussian.
Therefore, we can generalize it to a larger class of models:
\begin{lemma}\label{lemma:dgp_cf}
  \vspace{0.5em}
  Let $f_2(\ensuremath{\mathbf{f}}_1(\cdot)) : \ensuremath{\mathbb{R}}^D \to \ensuremath{\mathbb{R}}$ be a hierarchical model where $f_2(\cdot) : \ensuremath{\mathbb{R}}^{H_1} \to \ensuremath{\mathbb{R}}$ is a GP and $\ensuremath{\mathbf{f}}_1(\cdot) : \ensuremath{\mathbb{R}}^D \to \ensuremath{\mathbb{R}}^{H_1}$ is a random vector-valued function
  (including a multilayer hierarchical model).

  Then $f_2(\ensuremath{\mathbf{f}}_1(\cdot))$ is a GP if and only if
  $\Evover{\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{X}}, \ensuremath{\mathbf{f}}_1(\cdot)}{ \ensuremath{\mathbf{f}}_2 \ensuremath{\mathbf{f}}_2^\top } = \Evover{\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{X}}}{ \ensuremath{\mathbf{f}}_2 \ensuremath{\mathbf{f}}_2^\top }$
  a.s. for all $\ensuremath{\boldmatrix{X}} = [\ensuremath{\mathbf{x}}_1, \ldots, \ensuremath{\mathbf{x}}_N]$.
\end{lemma}

The covariance perspective from \cref{lemma:nn_cf,lemma:dgp_cf} is revealing about GP limits.
As $\Evover{\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{X}}, \ensuremath{\mathbf{f}}_1(\cdot)}{ \ensuremath{\mathbf{f}}_2 \ensuremath{\mathbf{f}}_2^\top }$ converges to $\Evover{\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{X}}}{ \ensuremath{\mathbf{f}}_2 \ensuremath{\mathbf{f}}_2^\top }$
the model output becomes less and less dependent on $\ensuremath{\mathbf{f}}_1(\cdot)$.
In other words, $f_2(\ensuremath{\mathbf{f}}_1(\cdot))$ \emph{loses its hierarchical nature}.
We reiterate that \cref{lemma:dgp_cf} has no requirements about $f_2(\ensuremath{\mathbf{f}}_1(\cdot))$ transitioning from a finite to infinite basis,
nor %
does it require $f_2(\ensuremath{\mathbf{f}}_1(\cdot))$ to have additive structure.
We demonstrate its generality in the next section with surprising---and troubling---implications.

\section{Introduction}
\label{sec:intro}

Research has shown that deeper neural networks tend to be more expressive and efficient than wider networks under a variety of metrics \citep[e.g.][]{montufar2014number,romero2014fitnets,telgarsky2016benefits,poole2016exponential,raghu2017expressive,nguyen2017loss,lu2017expressive,chen2020towards}.
Nevertheless, there is resurgent interest in wide models due in part to empirical successes \citep[e.g.][]{zagoruyko2016wide} and theoretical analyses of limiting behavior.
When randomly initialized to create a distribution over functions, neural networks converge to Gaussian processes ({\bf GP}) as width increases.
This result, first proved for 2-layer networks \citep{neal1995bayesian}, has been extended to deeper networks \citep{lee2017deep,matthews2018gaussian}, convolutional networks \citep{garriga2019deep,novak2019bayesian}, and other architectures \cite{yang2019tensor,hron2020infinite}.
A similar limit exists for gradient-trained networks, which behave increasingly like kernel machines under the neural tangent kernel \citep[e.g.][]{jacot2018neural,du2019gradient,allen2019learning,arora2019exact,lee2019wide,yang2020tensor}.

While these limits simplify analyses, there is something unsettling about reducing neural networks to kernel methods.
\citet[][p. 161]{neal1995bayesian} describes the GP limit as ``disappointing,'' noting that
``infinite networks do not have hidden units that represent `hidden features'\ldots
often seen [as the] interesting aspect of neural network learning.''
Recent work indeed shows that
learned hierarchical features can be exponentially more efficient than the fixed shallow representations of kernels
\citep[e.g.][]{arora2019exact,allen2019can,allen2020backward,bai2020beyond,bengio2005curse,chen2020towards,ghorbani2019limitations,ghorbani2020neural,li2020learning,yehudai2019power}.
At the same time, wider networks can more accurately model complex functions \citep{goodfellow2016deep}.
Thus, wide limits appear to confound opposing phenomenon: increased capacity makes them more expressive, yet the loss of hierarchical features seems to make them less expressive.
This may explain the mixed empirical performance of limiting models:
outperforming finite width models in some scenarios \citep[e.g.][]{arora2020harnessing,lee2020finite,garriga2019deep},
yet falling short on more complex tasks \citep[e.g.][]{arora2019exact,bai2020beyond,fort2020deep,shankar2020neural,lee2019wide}.

This paper aims to decouple these effects of large width.
Our goal is to understand the inductive biases of wide networks,
after a network has ``sufficient'' capacity for a given modeling task.  We ask:
  \emph{
    If we control for the effects of increased capacity, what---if any---value remains in wide networks?
  }

To achieve this control, we note that a typical neural network layer corresponds to a finite basis, where elementwise nonlinearities transform each hidden feature into a \emph{single basis function}.
In order to decouple width from capacity, one could generalize these layers so that each nonlinearity produces any number of basis functions;
if each hidden feature gives rise to an infinite and universal basis, then hidden layers would have infinite representational capacity \emph{regardless of width}.
This generalization is in fact a well-studied class of hierarchical models---Deep Gaussian Processes ({\bf Deep GP}) \citep{damianou2013deep,damianou2015deep,bui2016deep,cutajar2017random,salimbeni2017doubly,havasi2018inference,dutordoir2020bayesian,dutordoir2021gpflux}---%
where standard neural net layers are replaced with vector-valued Gaussian processes.
Indeed, typical neural networks are a degenerate Deep GP subclass \cite{aitchison2020bigger,agrawal2020wide,duvenaud2014avoiding,ober2020global}.

We therefore have a generalization of neural networks where capacity is controlled,
from which we can glean insights about conventional networks that have sufficient representational power for a given modeling task.
Surprisingly, despite using Gaussian processes as the primary hierarchical component, we prove that \emph{Deep GP converge to (single-layer) GP in their infinite width limit} (\cref{thm:dgp_gp}).
Troubling implications immediately ensue:
large width is strictly detrimental to Deep GP, as the limiting model collapses to a shallower version of itself.
We support this theorem with an analysis of neural network and Deep GP posteriors, which \emph{become less adaptable as width increases}.
Specifically, we show that the posterior mean corresponds to a mixture of functions drawn from data-dependent (and thus adaptive) reproducing kernel Hilbert spaces, formalizing the above claim from  \citet{neal1995bayesian}.
As width increases, this mixture collapses to the data-independent kernel of the limiting GP, implying that wider models have less feature learning.
Finally, we present a novel tail analysis which indicates that \emph{width and depth have opposite effects}:
depth accentuates non-Gaussianity, sharpening peaks and fattening tails, whereas width increases Gaussianity (\cref{thm:mean_concentration,thm:tails}).

Our theoretical results hold for Deep GP and conventional (parametric) neural networks alike.
Experiments confirm that---%
after a model achieves sufficient capacity---\emph{width becomes harmful to model fit and performance}.
For nonparametric Deep GP, a width of 1 or 2 often achieves the best performance and fit.
Neural networks naturally require more hidden units; nevertheless, performance degrades after a certain width.
On small datasets ($N \leq 1000$) with low dimensionality, we find that models with $\leq16$ hidden units achieve best test set performance.
On larger datasets like CIFAR10, this ``sweet spot'' occurs later (at $\approx 500$ hidden units for sufficiently deep models), yet performance degrades beyond this width.
Our findings strongly suggest that narrower models have better inductive biases,
and wide models perform well \emph{in spite of}---not because of---large width.

\subsection{Notation}

\section{Large Width Limits the Adaptability of Hierarchical Posteriors}
\label{sec:posterior}
Even with \cref{thm:dgp_gp} and its troubling suggestions, it is not immediately clear exactly what is lost in the infinite-width limit.
Here, we quantify specific differences in the predictive capabilities of narrow versus wide models.
In particular, we analyze Deep GP/neural network posterior distributions, rather than focusing on a single model trained through optimization.
We show that these posteriors correspond to a mixture of \emph{data-dependent adaptable bases};
however, as width increases this mixture collapses to the (data-independent) basis of the limiting GP.
This result formalizes the often vague notion of \emph{feature learning}, and demonstrates that it is indeed lost in kernel limits.

\begin{figure}
  \centering
  \includegraphics[width=0.95\linewidth]{figures/adaptive_vs_gaussian_rbf_posterior.pdf}\\
  \vspace{0.2em}
  \includegraphics[width=0.95\linewidth]{figures/adaptive_vs_gaussian_rbf_kernel.pdf}
  \caption{
    {\bf Top:} Posterior of 2-layer RBF Deep GP fit to a noisy step function.
    A width-1 Deep GP fits the discontinuity at $\ensuremath{\mathbf{x}} = 0$.
    As with increases, the Deep GP converges to a GP with a stationary covariance unable to fit the step.
    {\bf Bottom:} Average posterior covariance $\Evover{\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}') \mid \ensuremath{\mathbf{y}}}{ k_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) }$.
    The width-1 posterior covariance is non-stationary, with little covariance around $\ensuremath{\mathbf{x}} = 0$.
    As width increases, the posterior covariance becomes stationary (as seen by the kernel's constant diagonals).
  }
  \label{fig:adaptive_vs_gaussian_rbf}
\end{figure}

\paragraph{Hierarchical posteriors correspond to a data-adaptable bases.}
Consider the (finite-width) 2-layer Deep GP $f_2(\ensuremath{\mathbf{f}}_1(\cdot))$, where $k_1(\cdot, \cdot)$ and $k_2(\cdot, \cdot)$ are the covariance functions of $\ensuremath{\mathbf{f}}_1(\cdot)$ and $f_2(\cdot)$.
Given training data $\ensuremath{\boldmatrix{X}}, \ensuremath{\mathbf{y}}$, define $\ensuremath{\boldmatrix{F}}_1 \triangleq[\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_1), \ldots, \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_N)]$
and $\ensuremath{\mathbf{f}}_2 \triangleq [f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_1)), \ldots, f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_N))]$.
Let $\ensuremath{\mathbf{x}}^*$ be a test input, and let $\ensuremath{\mathbf{f}}_1^*$ and $f_2^*$ equal $\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}^*)$ and $f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}^*))$
(see \cref{fig:2layer_model} in \cref{sec:dgp_posterior_factorization} for a graphical model).

Crucially, $\ensuremath{\mathbf{f}}_2$ and $f^*_2$ only depend on $\ensuremath{\boldmatrix{F}}_1$ and $\ensuremath{\mathbf{f}}^*_1$ through the covariances $\ensuremath{\boldmatrix{K}}_2(\ensuremath{\boldmatrix{F}}_1, \ensuremath{\boldmatrix{F}}_1)$, $\ensuremath{\mathbf{k}}_2(\ensuremath{\boldmatrix{F}}_1, \ensuremath{\mathbf{f}}^*_1)$, and $k_2(\ensuremath{\mathbf{f}}^*_1, \ensuremath{\mathbf{f}}^*_1)$
(which we abbreviate as $\ensuremath{\boldmatrix{K}}_2$, $\ensuremath{\mathbf{k}}_2^*$, and $k_2^{**}$):
\begin{align}
  p(\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{K}}_2)
  \sim \normaldist{\ensuremath{\mathbf{0}}}{\ensuremath{\boldmatrix{K}}_2},
  \quad
  p(f^*_2 \mid k^{**}_2, \ensuremath{\mathbf{k}}^*_2, \ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{f}}_2)
  \sim \normaldist{\ensuremath{\mathbf{k}}_2^{*\top} \ensuremath{\boldmatrix{K}}_2^{-1} \ensuremath{\mathbf{f}}_2}{\:\: k_2^{**} - \ensuremath{\mathbf{k}}_2^{*\top} \ensuremath{\boldmatrix{K}}_2^{-1} \ensuremath{\mathbf{k}}_2^*},
  \nonumber

\end{align}
By D-separation \citep[e.g.][Ch. 8]{bishop2006pattern}, we can factorize the posterior distribution as:
\begin{align}
  p(f_2^*, \ensuremath{\mathbf{f}}_2, \ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{k}}_2^*, k_2^{**} \mid \ensuremath{\mathbf{y}}) \: = \:
  p(f_2^* \mid \ensuremath{\mathbf{f}}_2, \ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{k}}_2^*, k_2^{**}) \:
  p(\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{y}}) \:
  p(\ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{k}}_2^*, k_2^{**} \mid \ensuremath{\mathbf{y}}).
  \label{eqn:2layer_dgp_factorization}
\end{align}
See derivation in \cref{sec:dgp_posterior_factorization}.
Applying the factorization in \cref{eqn:2layer_dgp_factorization}, the posterior mean is:
\begin{align}
  \Evover{f_2^* \mid \ensuremath{\mathbf{y}}}{ f_2^* }

  &= \Evover{\ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{k}}_2^* \mid \ensuremath{\mathbf{y}}}{
    \Evover{ \ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{y}} }{
      \ensuremath{\mathbf{k}}_2^{*\top} \ensuremath{\boldmatrix{K}}_2^{-1} \ensuremath{\mathbf{f}}_2
    }
  }
  = \expectedvalue_{ \ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{k}}_2^* \mid \ensuremath{\mathbf{y}}} \Bigl[ \:
    \ensuremath{\mathbf{k}}_2^{*\top}
    \overbracket{
      \ensuremath{\boldmatrix{K}}_2^{-1} \Evover{ \ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{y}} }{ \ensuremath{\mathbf{f}}_2 }
    }^{ \ensuremath{\boldsymbol{\alpha}} }
  \: \Bigr]
  \label{eqn:dgp_posterior_mean_kernel} \\
  &=
  \Evover{\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}^*), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_1), \ldots, \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_N) \mid \ensuremath{\mathbf{y}}}{ \textstyle \sum_{i=1}^N \: \alpha_i \: k_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_i), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}^*))},
  \label{eqn:dgp_posterior_mean}
\end{align}
where the second line follows from $\ensuremath{\boldmatrix{K}}_2$ and $\ensuremath{\mathbf{k}}_2^*$ being deterministic given $\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}^*)$, $\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_1)$, $\ldots$, $\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_N)$.
The term inside the \cref{eqn:dgp_posterior_mean} expectation is a function from the reproducing kernel Hilbert space (RKHS) defined by $k_2(\ensuremath{\mathbf{f}}_1(\cdot), \ensuremath{\mathbf{f}}_1(\cdot))$.
We can thus interpret this expectation as an infinite mixture of functions from different Hilbert spaces.
Because the mixture distribution $p(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}^*), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_1), \ldots, \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_N) \mid \ensuremath{\mathbf{y}})$ depends on $\ensuremath{\mathbf{y}}$, \cref{eqn:dgp_posterior_mean} is an \emph{adaptive data-dependent mixture of RKHS}.

\paragraph{Adaptability is lost in the Gaussian process limit.}
Recall from \cref{lemma:dgp_cf} that
the conditional prior covariance becomes deterministic as $f_2(\ensuremath{\mathbf{f}}_1(\cdot))$ converges to a GP.
In other words, the prior and posterior distributions over $\ensuremath{\boldmatrix{K}}_2$ and $\ensuremath{\mathbf{k}}_2^*$ become atomic:
$ p(\ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{k}}_2^*) =  p(\ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{k}}_2^* \mid \ensuremath{\mathbf{y}}) = \delta \:[ \: \ensuremath{\boldmatrix{K}}_{\text{lim}}, \ensuremath{\mathbf{k}}_\text{lim}^* \: ], $
where $\ensuremath{\boldmatrix{K}}_{\text{lim}}$ and $\ensuremath{\mathbf{k}}_{\text{lim}}^*$ are shorthand for $\Ev{ \ensuremath{\mathbf{f}}_2 \ensuremath{\mathbf{f}}_2^\top }$ and $\Ev{ \ensuremath{\mathbf{f}}_2 f_2^*}$ respectively.
\cref{eqn:dgp_posterior_mean_kernel} thus collapses to:
\begin{align}
  \lim_{H_1 \to \infty} \: \Evover{f_2^* \mid \ensuremath{\mathbf{y}}}{ f_2^* }
  =
  \Evover{ \delta [ \ensuremath{\boldmatrix{K}}_\text{lim}, \ensuremath{\mathbf{k}}^*_\text{lim} ] }{ \:
    \ensuremath{\mathbf{k}}_2^{*\top} \ensuremath{\boldsymbol{\alpha}}
  \: }
  = {\textstyle \sum_{i=1}^N} \: \alpha_i \: k_{\text{lim}} ( \ensuremath{\mathbf{x}}_i, \ensuremath{\mathbf{x}}^* ),
  \label{eqn:limiting_dgp_posterior_mean}
\end{align}
which is no longer a mixture of functions from different RKHS.
It is instead a function from a single RKHS (that of the limiting GP prior).
In other words, while Deep GP (and neural networks) perform \emph{kernel learning (or feature learning)} to adapt to training data,
this ability is lost with large width.

\paragraph{Example.}
Consider a Deep GP with RBF covariances
$
  k_1( \ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}' ) = \exp \left( {- \Vert \ensuremath{\mathbf{x}} - \ensuremath{\mathbf{x}}' \Vert^2/(2 D) } \right)
$
and
$
  k_2( \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}') ) = \exp \left( - \Vert \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}) - \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}') \Vert^2 / (2 H_1) \right).
$
As we show in \cref{sec:limiting_kernels}, this Deep GP converges to a GP with
$
  k_{\text{lim}}(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}') = \exp ( \exp( -\Vert \ensuremath{\mathbf{x}} - \ensuremath{\mathbf{x}}' \Vert^2 / (2 D) ) - 1 ).
$
Note that this limiting covariance is \emph{stationary}
and is ill-equipped to model the data step in \cref{fig:adaptive_vs_gaussian_rbf}.
However, because $\ensuremath{\mathbf{f}}_1(\cdot)$ is nonlinear, $k_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}'))$
is \emph{nonstationary}.
\cref{fig:adaptive_vs_gaussian_rbf} (top left) shows that the width-1 Deep GP posterior accurately models both this data.
The posterior covariance $\Evover{\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}') \mid \ensuremath{\mathbf{y}}}{ k_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) }$ (bottom left) features long-range correlations near $\ensuremath{\mathbf{x}} = \pm1$ and short-range correlations near $\ensuremath{\mathbf{x}} = 0$.
As width increases, we lose this nonstationarity and the posterior becomes a worse fit.

\section{Setup}

\subsection{Related Work}
\label{sec:related}

\paragraph{Effects of width.}
Works have shown that, given finite parameters, deeper models are more expressive than wider models \citep{montufar2014number,telgarsky2016benefits,lu2017expressive,poole2016exponential,raghu2017expressive}.
Similarly to our work, \citet{aitchison2020bigger} shows that width potentially harms prior and posterior expressivity of deep linear models.
Nevertheless, wide models have been shown to have favorable optimization landscapes \citep{nguyen2017loss,li2018visualizing,soltanolkotabi2018theoretical,arora2019fine,du2019gradient}
and are resistant to overfitting via double descent \citep{belkin2019reconciling,nakkiran2020deep,cao2020generalization}.
Our work controls for these factors by examining nonparametric hierarchical models with exact Bayesian inference,
and thus does not disagree with these other works.
Infinite width limits have received renewed interest
in Bayesian \citep{neal1995bayesian,lee2019wide,matthews2018gaussian,garriga2019deep,novak2019bayesian,yang2019tensor,hron2020infinite}
and non-Bayesian \citep{jacot2018neural,du2019gradient,allen2019learning,arora2019exact,lee2019wide,yang2020tensor,chizat2018global,mei2018mean,golikov2020towards,yang2020feature} settings.
Most of these works show that neural networks converge to kernel methods,
though recent work suggests that  this limiting behavior can be
avoided with different parameterizations \citep[e.g.][]{chizat2018global,mei2018mean,golikov2020towards,yang2020feature}.
Similarly to \citet{lee2019wide}, our Deep GP limit analysis sequentially increases the width of each layer,
though we hypothesize a similar proof exists where the width of all layers increases simultaneously (akin to \citep{matthews2018gaussian}).

\paragraph{Deep GP}
are introduced by \citet{damianou2013deep}.
A large portion of Deep GP research has thus far focused on scalable approximate inference methods
\citep{bui2016deep,dai2016variational,wang2016sequential,cutajar2017random,salimbeni2017doubly,havasi2018inference,dutordoir2021gpflux,ober2020global}.
Though prior work has studied tail properties of neural networks \citep{vladimirova2019understanding,zavatoneveth2021exact} and Deep GP with RBF kernels \citep{lu2020interpretable},
our work is---to the best of our knowlege---the first general result for Deep GP tails.
\citet{duvenaud2014avoiding} and \citet{dunlop2018deep} investigate pathological behaviors that arise with depth,
while \citet{agrawal2020wide} note that ``bottlenecked'' Deep GP have better performance and correlations among predictive tasks.
Our work complements these analysis by characterizing the effects of width.

\paragraph{Connections between Deep GP and neural networks.}
Many researchers have noted connections between neural networks and Deep GP \citep[e.g.][]{gal2016dropout,louizos2016structured,cutajar2017random,dutordoir2021deep}.
\citet{duvenaud2014avoiding} suggest that infinitely-wide neural networks with intermediate bottleneck layers are nonparametric Deep GP.
\citet{agrawal2020wide} formalize this connection,
but note that not all Deep GP can be constructed from bottlenecked neural networks
(see \cref{sec:proof_dgp_gp}).
In contrast to these prior works, we avoid reducing Deep GP to neural networks,
and instead reduce neural networks to degenerate Deep GP.

\section{The Difference Between Width and Depth: A Tail Analysis}
\label{sec:tails}
Our work so far has troubling implications for large width.
On the other hand, empirical evidence has shown that depth improves Deep GP performance---as it does for neural nets \citep[e.g.][]{salimbeni2017doubly,havasi2018inference,ober2020global} (though pathologies can emerge \cite{dunlop2018deep,duvenaud2014avoiding}).
Through a novel tail analysis, we show that width makes Deep GP priors more Gaussian, while depth makes them less Gaussian.
In other words, \emph{width and depth have opposite effects on Deep GP tails}, results that again also apply to typical neural networks.

\paragraph{Deep GP/neural networks are sharply peaked and heavy tailed.}

The proof technique used in \cref{lemma:nn_cf}
can be used to similarly bound the moment generating function of Deep GP marginals:
\begin{equation}
  \Evover{\ensuremath{\mathbf{f}}_2}{ e^{\ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_2} } =
  \Evover{\ensuremath{\boldmatrix{F}}_1}{ \Evover{\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ e^{\ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_2} }}
  \geq
  \exp \left( \frac 1 2 \ensuremath{\mathbf{t}}^\top \Evover{\ensuremath{\boldmatrix{F}}_1}{ \ensuremath{\boldmatrix{K}}_2(\ensuremath{\boldmatrix{F}}_1, \ensuremath{\boldmatrix{F}}_1) } \ensuremath{\mathbf{t}} \right)
  =
  \Evover{ \ensuremath{\mathbf{g}} \sim \normaldist{\ensuremath{\mathbf{0}}}{\ensuremath{\boldmatrix{K}}_\text{lim}}}{ e^{\ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{g}}} },
  \label{eqn:mgf_jensen}
\end{equation}
where $\ensuremath{\boldmatrix{K}}_\text{lim} \! = \! \Evover{\ensuremath{\mathbf{f}}_2}{ \ensuremath{\mathbf{f}}_2 \ensuremath{\mathbf{f}}_2^\top } \! = \! \Evover{\ensuremath{\boldmatrix{F}}_1}{\ensuremath{\boldmatrix{K}}_2(\ensuremath{\boldmatrix{F}}_1, \! \ensuremath{\boldmatrix{F}}_1)}$.
Generalizing these bounds to deeper models, we have:
\begin{theorem}
  Let $f_L \circ \ldots \circ \ensuremath{\mathbf{f}}_1(\cdot)$ be a zero-mean Deep GP.

  Given a finite set of inputs $\ensuremath{\boldmatrix{X}} = [\ensuremath{\mathbf{x}}_1, \ldots, \ensuremath{\mathbf{x}}_N]$, define
  $\ensuremath{\mathbf{f}}_\ell = [(f_\ell \circ \ldots \circ \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_1)), \ldots, (f_\ell \circ \ldots \circ \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_N))]$ for $\ell \in [1, L]$,
  and define $\ensuremath{\boldmatrix{K}}_\text{lim} = \Evover{\ensuremath{\mathbf{f}}_L}{ \ensuremath{\mathbf{f}}_L \ensuremath{\mathbf{f}}_L^\top }$.
  If $\Evover{\ensuremath{\mathbf{f}}_\ell \mid \ensuremath{\mathbf{f}}_{\ell - 1}}{ \ensuremath{\mathbf{f}}_\ell \ensuremath{\mathbf{f}}_\ell^\top }$
  is not constant a.s. for any $\ell \in [2, L]$,
  then,
  $
  p( \ensuremath{\mathbf{f}}_L = \ensuremath{\mathbf{0}}) > \normaldist{\ensuremath{\mathbf{g}} = \ensuremath{\mathbf{0}}; \ensuremath{\mathbf{0}}}{ \ensuremath{\boldmatrix{K}}_\text{lim} }.
  $
  %
  \label{thm:mean_concentration}
\end{theorem}
\begin{theorem}
  \label{thm:tails}
  Let $\ensuremath{\mathbf{t}} \in \ensuremath{\mathbb{R}}^N$.
  Using the same setup, notation, and assumptions as \cref{thm:mean_concentration},
  the odd moments of $\ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_L$ are zero and the even moments larger than $2$ are super-Gaussian,
  i.e. $\Evover{\ensuremath{\mathbf{f}}_L}{ (\ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_L)^{r} } \geq \Evover{ \ensuremath{\mathbf{g}} \sim \normaldist{\ensuremath{\mathbf{0}}}{\ensuremath{\boldmatrix{K}}_\text{lim}} }{ (\ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{g}})^r }$
  for all even $r \geq 4$.
  Moreover, if $k_L(\cdot, \cdot)$ is bounded almost everywhere, the moment generating function $\Evover{\ensuremath{\mathbf{f}}_L}{ \exp( \ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_L) }$ exists and is similarly super-Gaussian.
\end{theorem}
(See \cref{sec:proof_tails} for proofs.)
\cref{thm:mean_concentration} states that Deep GP marginals are more sharply peaked than a moment-matched Gaussian,
while \cref{thm:tails} states that they are also more heavy tailed.

\paragraph{Increasing depth leads to sharper peaks and heavier tails.}
To understand how depth affects this tail behavior, we examine the Jensen gap in \cref{eqn:mgf_jensen}.
Consider a 3-layer Deep GP $f_3(\ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\cdot)))$.
If we extend \cref{eqn:mgf_jensen} to 3-layer models,
we see that the Jensen gap cascades:
\begin{equation}
  \underbracket{\Evover{\ensuremath{\boldmatrix{F}}_1}{\Evover{\ensuremath{\boldmatrix{F}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \exp \left( \frac 1 2 \ensuremath{\mathbf{t}}^\top \ensuremath{\boldmatrix{K}}_3 \ensuremath{\mathbf{t}} \right) }}}_{\text{MGF of 3-layer Deep GP marginal}}
  \geq
  \underbracket{\Evover{\ensuremath{\boldmatrix{F}}_1}{ \exp \left( \frac 1 2 \ensuremath{\mathbf{t}}^\top \!\! \Evover{\ensuremath{\boldmatrix{F}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \ensuremath{\boldmatrix{K}}_3 } \ensuremath{\mathbf{t}} \right)}}_{\text{MGF of 2-layer Deep GP marginal}}
  \geq
  \underbracket{\exp\left( \frac 1 2 \ensuremath{\mathbf{t}}^\top \!\! \Evover{\ensuremath{\boldmatrix{F}}_1}{\Evover{\ensuremath{\boldmatrix{F}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \ensuremath{\boldmatrix{K}}_3 }} \ensuremath{\mathbf{t}} \right)}_{\text{MGF of $\normaldist{\ensuremath{\mathbf{0}}}{\Evover{\ensuremath{\boldmatrix{F}}_1}{\Evover{\ensuremath{\boldmatrix{F}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \ensuremath{\boldmatrix{K}}_3 }}}$}},

  \nonumber
\end{equation}
where $\ensuremath{\boldmatrix{K}}_3$ is short for $\ensuremath{\boldmatrix{K}}_3( \ensuremath{\boldmatrix{F}}_2(\ensuremath{\boldmatrix{F}}_1(\ensuremath{\boldmatrix{X}})), \ensuremath{\boldmatrix{F}}_2(\ensuremath{\boldmatrix{F}}_1(\ensuremath{\boldmatrix{X}})) )$.
The middle term is the moment generating function of a 2-layer Deep GP marginal
(where the second layer has covariance $\Evover{\ensuremath{\mathbf{f}}_2(\cdot)}{k_3(\ensuremath{\mathbf{f}}_2(\cdot), \ensuremath{\mathbf{f}}_2(\cdot))}$).
The right-most term is the moment generating function of a (single-layer) Gaussian.
Generalizing this cascade, we see that deeper models are more heavy-tailed.
A similar analysis on the characteristic function shows that the peak at the prior mean also becomes sharper with depth (see \cref{sec:proof_tails}).

\begin{figure}[t!]
  \centering
  \includegraphics[width=\textwidth]{figures/density_depth_width.pdf}
  \caption{
    Marginal density $p(y_1, y_2 \mid \ensuremath{\mathbf{x}}_1, \ensuremath{\mathbf{x}}_2)$
    for zero-mean Deep GP of various depths and widths on the $N=2$ dataset $\ensuremath{\mathbf{x}}_1 = -0.5$, $\ensuremath{\mathbf{x}}_2 = 0.5$.
    All 2-layer models have the same second moments (covariance is that of the 3-layer width-1 RBF-RBF-RBF Deep GP).
    {\bf Left to right:} width increases, the marginal becomes increasingly Gaussian, tails become thinner, and the peak at $[y_1, y_2] = \ensuremath{\mathbf{0}}$ loses density.
    {\bf Top to bottom:} depth increases, tails become fatter, and the peak becomes sharper.
  }
  \label{fig:density_depth_width}
\end{figure}

Adding additional layers to a Deep GP will change the model's prior covariance,
and thus the effects of depth cannot solely be explained by a tail analysis \citep{duvenaud2014avoiding,dunlop2018deep}.
Nevertheless, if we control for this change in covariance, we indeed see that depth leads to heavier tails.
In \cref{fig:density_depth_width} we compare
2-layer and 3-layer Deep GP.
The 3-layer models use GP layers with additively-decomposing RBF covariances,
while the 2-layer models use layers constructed to match the 3-layer models' prior covariance
(see \cref{sec:experimental_details} for construction details).
The $N=2$ marginal densities for the 3-layer models (bottom row) are more stretched than the 2-layer densities (top row).
We further confirm these effects in \cref{sec:additional_results}.

\paragraph{Increasing width leads to flatter peaks and Gaussian tails.}
Conversely, consider what happens when we make the model wider.
We define the sequence of increasingly wide 2-layer Deep GP:
\begin{align}
  \left\{ \text{DGP}^{(m)}(\cdot) \triangleq \frac{1}{\sqrt{m}} \sum_{i=1}^m f^{(i)}_2(f^{(i)}_1(\cdot)) \right\},
  \qquad
  \begin{split}
    f_1^{(i)}(\cdot) \stackrel{\text{i.i.d}}{\sim} \GP{0}{k_1(\cdot, \cdot)},
    \\
    f_2^{(i)}(\cdot) \stackrel{\text{i.i.d}}{\sim} \GP{0}{k_2(\cdot, \cdot)}.
  \end{split}
  \label{eqn:additive_dgp}
\end{align}
$\text{DGP}^{(m)}(\cdot)$ is a width-$m$ Deep GP, where the second layer decomposes additively over the $m$ dimensions.
By linearity of expectation, each model in the sequence shares the same prior covariance:
$
  \mathbb{E}[\text{DGP}^{(1)}(\ensuremath{\mathbf{x}}) \: \text{DGP}^{(1)}(\ensuremath{\mathbf{x}}') ] =
  \mathbb{E}[\text{DGP}^{(2)}(\ensuremath{\mathbf{x}}) \: \text{DGP}^{(2)}(\ensuremath{\mathbf{x}}') ] =
  \ldots
  \triangleq k_\text{lim}(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}').

$
Though each model has the same marginal covariance,
the \emph{conditional} covariance $\Evover{\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \ensuremath{\mathbf{f}}_2 \ensuremath{\mathbf{f}}_2^\top } = \frac 1 m \sum_{i=1}^m \ensuremath{\boldmatrix{K}}_2 ( \ensuremath{\mathbf{f}}_1^{(i)}, \ensuremath{\mathbf{f}}_1^{(i)} )$
becomes increasingly concentrated around $\ensuremath{\boldmatrix{K}}_\text{lim}(\ensuremath{\boldmatrix{X}}, \ensuremath{\boldmatrix{X}})$ as $m$ increases.
This consequentially shrinks the Jensen gap in \cref{eqn:mgf_jensen},
and so the Deep GP marginals become increasingly Gaussian.
We again visualize this effect in \cref{fig:density_depth_width},
which depicts marginal densities from 2-layer and 3-layer Deep GP of various width
(see \cref{sec:experimental_details} for details).
Compared with the limiting GP (right), the width-1 densities (left) appear sharper near $[0, 0]$ and more stretched at the tails.
As width increases, the peaks and tails look increasingly Gaussian (see also \cref{fig:width_diff_depth_width} in \cref{sec:additional_results}).
In this sense, width has the opposite effect as depth---deeper marginals are less Gaussian, while wider marginals are more Gaussian.

\section{Broader Impact}
\label{sec:broader_impact}

This paper analyzes two existing classes of models: Deep GP and neural networks.
We believe that our findings will be of interest to researchers and machine learning practitioners,
offering useful guidance for Deep GP and neural network architectures.
Because we are neither introducing new algorithms nor introducing new use cases of existing algorithms, we do not foresee any major ethical impacts from this work.
However, we do note that this paper primarily focuses on how width affects performance metrics (e.g. accuracy, log likelihood, etc.)
and does not focus on other metrics that may be of interest to practitioners and society at large (e.g. interpretability, energy usage, fairness, etc.).

\section{Deep Gaussian Processes}

In this section we discuss various Deep GP facts presented throughout the main paper.

\subsection{Capacity of Deep GP}
\label{sec:dgp_capacity}

Here we formalize the claim that Deep GP have ``infinite capacity.''
Standard Gaussian processes are nonparametric, and---if the prior covariance is a universal kernel \citep{micchelli2006universal}---%
then any function (or an arbitrarily precise approximation thereof) is a draw from its prior.
A Deep GP composes multiple GP as different layers.
If all GP layers use universal kernels for covariance priors, then any (arbitrarily precise approximation of a) function $h(\cdot)$ is a draw from the Deep GP prior.
(Draw the identity function from the first $L-1$ GP layers, and then draw $h(\cdot)$ from the last GP layer.)
In this sense, Deep GP as well as standard GP can model any function to arbitrary precision and in this sense have infinite capacity.

\subsection{Connection Between Neural Networks and Deep GP}
\label{sec:dgp_degenerate}

Throughout this paper we note that feed-forward (Bayesian) neural networks are a degenerate subclass of Deep Gaussian processes.
We will now formalize this connection, which has also been noted in several previous works \citep[e.g.][]{aitchison2020bigger,ober2020global,louizos2016structured}.

To show that the neural network defined in \cref{eqn:2layer_nn} is a (degenerate) Deep GP, we must show that each of its layers corresponds to a (degenerate, vector-valued) Gaussian process.
Recall that a Gaussian process $f(\cdot) \sim \mathcal{GP}$ is a distribution over functions
where every finite marginal distribution $\ensuremath{\mathbf{f}} = [f(\ensuremath{\mathbf{x}}_1), \ldots, f(\ensuremath{\mathbf{x}}_N)]$ is multivariate Gaussian consistent with some covariance function.
The first layer of the \cref{eqn:2layer_nn} neural network is given by
\[
  f_1^{(i)}(\ensuremath{\mathbf{x}}) = \ensuremath{\mathbf{w}}_1^{(i)\top} (\ensuremath{\mathbf{x}}) + \beta b_1^{(i)},
\]
where the entries of $\ensuremath{\mathbf{w}}_1^{(i)}$ and $b_1^{(i)}$ are i.i.d. unit Normal.
Using standard Gaussian identities, we have that $\ensuremath{\mathbf{f}}_1^{(i)} = \normaldist{\ensuremath{\mathbf{0}}}{\beta + \ensuremath{\boldmatrix{X}} \ensuremath{\boldmatrix{X}}^\top}$, where $\ensuremath{\boldmatrix{X}} = [\ensuremath{\mathbf{x}}_1, \ldots, \ensuremath{\mathbf{x}}_N]$.
Thus, the first layer corresponds to a vector-valued Gaussian process with the (degenerate) linear prior covariance.
The second layer of the \cref{eqn:2layer_nn} neural network is given by
\[
  f_2(\ensuremath{\mathbf{z}}) = {\textstyle \frac{1}{\sqrt{H_1}}} \ensuremath{\mathbf{w}}_2^\top \ensuremath{\boldsymbol{\sigma}}(\ensuremath{\mathbf{z}}) + \beta b_2,
\]
where again the entries of $\ensuremath{\mathbf{w}}_2$ and $b_2$ are i.i.d. unit Normal.
We have that $\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{F}}_1 = \normaldist{\ensuremath{\mathbf{0}}}{\beta + \ensuremath{\boldsymbol{\sigma}}(\ensuremath{\boldmatrix{F}}_1) \ensuremath{\boldsymbol{\sigma}}(\ensuremath{\boldmatrix{F}}_1)^\top}$, where $\ensuremath{\boldmatrix{F}}_1 = [\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_1), \ldots, \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_N)]$
and $\ensuremath{\boldsymbol{\sigma}}(\ensuremath{\boldmatrix{F}}_1)$ corresponds to the elementwise nonlinearity $\ensuremath{\boldsymbol{\sigma}}(\cdot)$ applied to each entry of $\ensuremath{\boldmatrix{F}}_1$.
Thus, the second layer also corresponds to a Gaussian process with a degenerate prior covariance.

\paragraph{Neural networks versus Deep GP.}
While (Bayesian) neural networks meet the definition of a Deep GP,
their covariance functions only correspond to a finite basis and therefore they do not have the same properties as nonparametric Deep GP
(i.e. the ability to model any function to arbitrary precision).
In this sense, it is common to treat neural networks and Deep GP as two separate classes of models with different predictive properties.
However, we emphasize that the theoretical results in this paper make no assumptions about whether or not a Deep GP is nonparametric,
and therefore the behaviors that we analyze are inherent to both classes of models.
In this sense, it is useful for our purposes to group nonparametric Deep GP and neural networks into a single class of models.

\subsection{Factorization of Deep GP Posterior}
\label{sec:dgp_posterior_factorization}

\begin{figure}[t!]
  \centering
  \begin{tikzpicture}[node distance=1cm]

    \node[obs]                 (X)  {$\ensuremath{\boldmatrix{X}}$} ; %
    \node[obs,    below=of X]  (x*) {$\ensuremath{\mathbf{x}}^*$} ; %
    \node[latent, right=of X]  (F1)  {$\ensuremath{\boldmatrix{F}}_1$} ; %
    \node[latent, below=of F1]  (f*1) {$\ensuremath{\mathbf{f}}^*_1$} ; %
    \node[latent, dashed, right=of F1, yshift=0.1cm]   (KFF)  {$\ensuremath{\boldmatrix{K}}_2$} ; %
    \node[latent, dashed, right=of F1, yshift=-0.90cm]    (KF*)  {$\ensuremath{\mathbf{k}}_2^*$} ; %
    \node[latent, dashed, right=of f*1, yshift=-0.1cm] (K**)  {$k_2^{**}$} ; %
    \node[latent, right=of KFF, yshift=-0.2cm] (f2)   {$\ensuremath{\mathbf{f}}_2$} ; %
    \node[latent, below=of f2]  (f*2) {$f^*_2$} ; %
    \node[obs,    right=of f2]  (y)  {$\ensuremath{\mathbf{y}}$} ; %

    \edge {X} {F1} ; %
    \edge {x*, F1} {f*1} ; %
    \edge {F1} {KFF} ; %
    \edge {F1, f*1} {KF*} ; %
    \edge {f*1} {K**} ; %
    \edge {KFF} {f2} ; %
    \edge {KFF, KF*, K**, f2} {f*2} ; %
    \edge {f2} {y} ; %
  \end{tikzpicture}
  \caption{
    2-layer Deep GP.
    $\ensuremath{\boldmatrix{X}}, \ensuremath{\mathbf{y}}$ are the training data; $\ensuremath{\mathbf{x}}^*$ is some unobserved test input.
    $\ensuremath{\boldmatrix{F}}_1$ and $\ensuremath{\mathbf{f}}^*_1$ are the first layer outputs for the training inputs and test input, respectively.
    $\ensuremath{\mathbf{f}}_2$ and $f^*_2$ are the second layer outputs for the train/test inputs,
    which only depend on $\ensuremath{\boldmatrix{F}}_1$, $\ensuremath{\mathbf{f}}^*_1$ through the prior covariance matrices
    $\ensuremath{\boldmatrix{K}}_2 = \ensuremath{\boldmatrix{K}}_2(\ensuremath{\boldmatrix{F}}_1, \ensuremath{\boldmatrix{F}}_1)$,
    $\ensuremath{\mathbf{k}}_2^* = \ensuremath{\mathbf{k}}_2(\ensuremath{\boldmatrix{F}}_1, \ensuremath{\mathbf{f}}_1^*)$, and
    $k_2^{**} = k_2(\ensuremath{\mathbf{f}}_1^*, \ensuremath{\mathbf{f}}_1^*)$.

  }
  \label{fig:2layer_model}
\end{figure}

Here we supply additional details for the \cref{sec:posterior} derivation of the Deep GP posterior mean.
Consider a 2-layer zero-mean Deep-GP $f_2(\ensuremath{\mathbf{f}}_1(\cdot))$.
\cref{fig:2layer_model} depicts the relationships between these variables, using the same notation as in \cref{sec:posterior}.
Now consider the posterior distribution
\begin{equation*}
  p(f_2^*, \ensuremath{\mathbf{f}}_2, \ensuremath{\boldmatrix{F}}_1, \ensuremath{\mathbf{f}}_1 \mid \ensuremath{\mathbf{y}}) \: = \:
  p(f_2^* \mid \ensuremath{\mathbf{f}}_2, \ensuremath{\mathbf{f}}_1^*, \ensuremath{\boldmatrix{F}}_1, \ensuremath{\mathbf{y}}) \:
  p(\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{F}}_1, \ensuremath{\mathbf{y}}, \ensuremath{\mathbf{f}}_1^*) \:
  p(\ensuremath{\mathbf{f}}_1^*, \ensuremath{\boldmatrix{F}}_1 \mid \ensuremath{\mathbf{y}}),
\end{equation*}
where we have omitted the dependence on $\ensuremath{\boldmatrix{X}}$ and $\ensuremath{\mathbf{x}}^*$ for clarity.
Apply the rules of D-separation using \cref{fig:2layer_model},
we see that $f^*_2$ only depends on $\ensuremath{\mathbf{y}}$ through $\ensuremath{\mathbf{f}}_2$, and thus $f^*_2$ is conditionally independent from $\ensuremath{\mathbf{y}}$ given $\ensuremath{\mathbf{f}}_2$.
Furthermore, we see that $\ensuremath{\mathbf{f}}_2$ is only connected to $\ensuremath{\mathbf{f}}^*_1$ through $f^*_2$, and so $\ensuremath{\mathbf{f}}_2$ is conditionally independent from $\ensuremath{\mathbf{f}}^*_1$ if $f^*_2$ is marginalized out.
Thus, we can simplify the posterior factorization to:
\begin{equation}
  p(f_2^*, \ensuremath{\mathbf{f}}_2, \ensuremath{\boldmatrix{F}}_1, \ensuremath{\mathbf{f}}_1 \mid \ensuremath{\mathbf{y}}) \: = \:
  p(f_2^* \mid \ensuremath{\mathbf{f}}_2, \ensuremath{\mathbf{f}}_1^*, \ensuremath{\boldmatrix{F}}_1) \:
  p(\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{F}}_1, \ensuremath{\mathbf{y}}) \:
  p(\ensuremath{\mathbf{f}}_1^*, \ensuremath{\boldmatrix{F}}_1 \mid \ensuremath{\mathbf{y}}),
  \label{eqn:2layer_dgp_factorization_nokernel}
\end{equation}
Crucially, $\ensuremath{\mathbf{f}}_2$ and $f^*_2$ only depend on $\ensuremath{\boldmatrix{F}}_1$ and $\ensuremath{\mathbf{f}}^*_1$ through $\ensuremath{\boldmatrix{K}}_2$, $\ensuremath{\mathbf{k}}_2^*$, and $k_2^{**}$:
\begin{align}
  p(\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{K}}_2)
  \sim \normaldist{\ensuremath{\mathbf{0}}}{\ensuremath{\boldmatrix{K}}_2},
  \quad
  p(f^*_2 \mid k^{**}_2, \ensuremath{\mathbf{k}}^*_2, \ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{f}}_2)
  \sim \normaldist{\ensuremath{\mathbf{k}}_2^{*\top} \ensuremath{\boldmatrix{K}}_2^{-1} \ensuremath{\mathbf{f}}_2}{\:\: k_2^{**} - \ensuremath{\mathbf{k}}_2^{*\top} \ensuremath{\boldmatrix{K}}_2^{-1} \ensuremath{\mathbf{k}}_2^*},
  \nonumber

\end{align}
(If $f_2(\ensuremath{\mathbf{f}}_1(\cdot))$ is a neural network or any other degenerate Deep GP,
the $\ensuremath{\boldmatrix{K}}_2^{-1}$ term can be replaced with its pseudoinverse.)
This relationship is also depicted graphically in \cref{fig:2layer_model}.
$\ensuremath{\boldmatrix{K}}_2$, $\ensuremath{\mathbf{k}}_2^*$, and $k_2^{**}$ are deterministic given $\ensuremath{\boldmatrix{F}}_1$ and $\ensuremath{\mathbf{f}}_1^*$,
and we do not ultimately care about the values of $\ensuremath{\boldmatrix{F}}_1$ and $\ensuremath{\mathbf{f}}_1^*$ since they are intermediate latent variables.
Therefore, we can rewrite the factorization in \cref{eqn:2layer_dgp_factorization_nokernel} where we replace $\ensuremath{\boldmatrix{F}}_1$, $\ensuremath{\mathbf{f}}_1^*$
with $\ensuremath{\boldmatrix{K}}_2$, $\ensuremath{\mathbf{k}}_2^*$, and $k_2^{**}$:
\begin{align}
  p(f_2^*, \ensuremath{\mathbf{f}}_2, \ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{k}}_2^*, k_2^{**} \mid \ensuremath{\mathbf{y}}) \: = \:
  p(f_2^* \mid \ensuremath{\mathbf{f}}_2, \ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{k}}_2^*, k_2^{**}) \:
  p(\ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{y}}) \:
  p(\ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{k}}_2^*, k_2^{**} \mid \ensuremath{\mathbf{y}}).
  \label{eqn:2layer_dgp_factorization_supp}
\end{align}
Applying the factorization in \cref{eqn:2layer_dgp_factorization}, the posterior mean is:
\begin{align}
  \Evover{f_2^* \mid \ensuremath{\mathbf{y}}}{ f_2^* }
  &= \Evover{\ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{k}}_2^*, k_2^{**} \mid \ensuremath{\mathbf{y}}}{
    \Evover{ \ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{y}} }{
      \expectedvalue_{ f_2^* \mid \ensuremath{\mathbf{f}}_2, \ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{k}}_2^*, k_2^{**} } \Bigl[ \: f_2^* \: \Bigr]
    }
  }
  \nonumber
  \\
  &= \Evover{\ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{k}}_2^* \mid \ensuremath{\mathbf{y}}}{
    \Evover{ \ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{y}} }{
      \ensuremath{\mathbf{k}}_2^{*\top} \ensuremath{\boldmatrix{K}}_2^{-1} \ensuremath{\mathbf{f}}_2
    }
  }
  \label{eqn:posterior_mean_supp_step2}
  \\
  &= \expectedvalue_{ \ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{k}}_2^* \mid \ensuremath{\mathbf{y}}} \Bigl[ \:
    \ensuremath{\mathbf{k}}_2^{*\top}
    \overbracket{
      \ensuremath{\boldmatrix{K}}_2^{-1} \Evover{ \ensuremath{\mathbf{f}}_2 \mid \ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{y}} }{ \ensuremath{\mathbf{f}}_2 }
    }^{ \ensuremath{\boldsymbol{\alpha}} }
  \: \Bigr]
  \label{eqn:posterior_mean_supp_step3}
\end{align}
(Again, if $f_2(\ensuremath{\mathbf{f}}_1(\cdot))$ is a neural network,
the $\ensuremath{\boldmatrix{K}}_2^{-1}$ term in \cref{eqn:posterior_mean_supp_step2,eqn:posterior_mean_supp_step3} can be replaced with its pseudoinverse.)
Finally, since $\ensuremath{\boldmatrix{K}}_2, \ensuremath{\mathbf{k}}^*_2$ are deterministic transforms of $\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}^*)$, $\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_1)$, $\ldots$, $\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_N)$,
we can rewrite \cref{eqn:posterior_mean_supp_step3} as:
\begin{align*}
  \Evover{f_2^* \mid \ensuremath{\mathbf{y}}}{ f_2^* }
  &=
  \Evover{\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}^*), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_1), \ldots, \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_N) \mid \ensuremath{\mathbf{y}}}{ \textstyle \sum_{i=1}^N \: \alpha_i \: k_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_i), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}^*))},
\end{align*}
which completes the derivation of \cref{eqn:dgp_posterior_mean} in \cref{sec:posterior}.

\section{What Prior Covariance Functions can be Modeled by Deep Gaussian Processes?}
\label{sec:dgp_second_moments}
The functional properties of standard Gaussian processes are largely determined by the choice of prior covariance function \citep{rasmussen2006gaussian}.
Any positive definite function is a valid GP covariance, making it possible to encode many types of functional priors.
For Deep GP, it is reasonable to assume that its prior second moment also has significant influence on its inductive bias and functional properties.
To that end, it is of interest to determine what covariances can be modeled by Deep GP a priori.

We present two theoretical results in this section.
The first is a negative result (\cref{thm:no_neg_covar}),
which states that Deep GP with stationary GP layers can only model non-negative covariance functions a priori.
This is in contrast to standard GP, which are capable of expressing anti-correlations with stationary covariance priors.
The second is a positive result (\cref{thm:radial}),
which demonstrates a Deep GP construction capable of modeling most isotropic covariance priors.
We note that isotropic functions (e.g. RBF, Mat\'ern, rational quadratic, etc.) are some of the most common covariance priors.

\begin{theorem}
  Let $\text{DGP}(\cdot) = f_L \circ \cdots \circ \ensuremath{\mathbf{f}}_1(\cdot)$ be a $L$-layer zero-mean Deep GP where $f_L(\cdot)$ has a mean-square continuous stationary prior covariance.
  Then $\Ev{\text{DGP}(\ensuremath{\mathbf{x}}) \: \text{DGP}(\ensuremath{\mathbf{x}}')} > 0$ for all $\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}'$.
  \label{thm:no_neg_covar}
\end{theorem}
\begin{proof}[Proof of Theorem~\ref{thm:no_neg_covar}]
  Throughout the proof, we will use the shorthand $\ensuremath{\mathbf{f}}_{\ell} = \ensuremath{\mathbf{f}}_\ell \circ \ldots \circ \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})$
  and $\ensuremath{\mathbf{f}}_{\ell}' = \ensuremath{\mathbf{f}}_\ell \circ \ldots \circ \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')$.
  Because $k_L(\cdot, \cdot)$ is stationary, we can express $\Ev{ \text{DGP}(\ensuremath{\mathbf{x}}) \: \text{DGP}(\ensuremath{\mathbf{x}}') }$ as:
  %
  \begin{align*}
    \Ev{ \text{DGP}(\ensuremath{\mathbf{x}}) \: \text{DGP}(\ensuremath{\mathbf{x}}') }
    &= \int k_L(\ensuremath{\mathbf{f}}_{L-1} - \ensuremath{\mathbf{f}}_{L-1}') \intd p(\ensuremath{\mathbf{f}}_{L-1}, \ensuremath{\mathbf{f}}_{L-1}').
  \end{align*}
  %
  Moreover, by Bochner's theorem, we can express $k_2(\ensuremath{\mathbf{f}}_{L-1} - \ensuremath{\mathbf{f}}_{L-1}')$ as the Fourier transform of some positive finite measure $\mu(\ensuremath{\boldsymbol{\xi}})$:
  %
  \begin{align}
    \Ev{ \text{DGP}(\ensuremath{\mathbf{x}}) \: \text{DGP}(\ensuremath{\mathbf{x}}') }
    &= \int \left( \int \exp( i \: \ensuremath{\boldsymbol{\xi}}^\top (\ensuremath{\mathbf{f}}_{L-1} - \ensuremath{\mathbf{f}}_{L-1}') )
    \intd \mu(\ensuremath{\boldsymbol{\xi}}) \right) \intd p(\ensuremath{\mathbf{f}}_{L-1}, \ensuremath{\mathbf{f}}_{L-1}').
    \label{eqn:stationary_bochner}
  \end{align}
  %
  Note that $|\exp(i \: \cdot)|$ is bounded everywhere, and $\mu(\ensuremath{\boldsymbol{\xi}})$ and $p(\ensuremath{\mathbf{f}}_{L-1}, \ensuremath{\mathbf{f}}_{L-1}')$ are finite measures.
  Therefore we can switch the orders of integration in \cref{eqn:stationary_bochner}:
  %
  \begin{align}
    \Ev{ \text{DGP}(\ensuremath{\mathbf{x}}) \: \text{DGP}(\ensuremath{\mathbf{x}}') }
    &= \int \left( \int \exp( i \: \ensuremath{\boldsymbol{\xi}}^\top (\ensuremath{\mathbf{f}}_{L-1} - \ensuremath{\mathbf{f}}_{L-1}' ) \intd p(\ensuremath{\mathbf{f}}_{L-1}, \ensuremath{\mathbf{f}}_{L-1}') \right) \intd \mu(\ensuremath{\boldsymbol{\xi}}).
    \label{eqn:stationary_bochner_switched}
    \\
    &= \int \left(
      \int \exp \left( i \:
        \begin{bmatrix} \ensuremath{\boldsymbol{\xi}} \\ -\ensuremath{\boldsymbol{\xi}} \end{bmatrix}^\top
        \begin{bmatrix} \ensuremath{\mathbf{f}}_{L-1} \\ \ensuremath{\mathbf{f}}_{L-1}' \end{bmatrix}
      \right)
      \intd p \left( \begin{bmatrix} \ensuremath{\mathbf{f}}_{L-1} \\ \ensuremath{\mathbf{f}}_{L-1}' \end{bmatrix} \right)
    \right) \intd \mu(\ensuremath{\boldsymbol{\xi}}).
    \nonumber
  \end{align}
  %
  Applying the characteristic function lower bound for Deep GP marginals (see \cref{sec:proof_tails}, Eq.~\ref{eqn:cf_bound}):
  %
  \begin{align}
    \Ev{ \text{DGP}(\ensuremath{\mathbf{x}}) \: \text{DGP}(\ensuremath{\mathbf{x}}') }
    &\geq \int \exp \left(
      - \frac 1 2
      \begin{bmatrix} \ensuremath{\boldsymbol{\xi}} \\ -\ensuremath{\boldsymbol{\xi}} \end{bmatrix}^\top
      \Ev{
        \begin{bmatrix} \ensuremath{\mathbf{f}}_{L-1} \\ \ensuremath{\mathbf{f}}_{L-1}' \end{bmatrix}
        \begin{bmatrix} \ensuremath{\mathbf{f}}_{L-1} \\ \ensuremath{\mathbf{f}}_{L-1}' \end{bmatrix}^\top
      }
      \begin{bmatrix} \ensuremath{\boldsymbol{\xi}} \\ -\ensuremath{\boldsymbol{\xi}} \end{bmatrix}
    \right) \intd \mu(\ensuremath{\boldsymbol{\xi}}).
    \label{eqn:stationary_bound}
  \end{align}
  %
  The integrand in \cref{eqn:stationary_bound} is a real-valued exponential, and so it is strictly positive.
  Since $\mu(\ensuremath{\boldsymbol{\xi}})$ is a positive measure, we have that $\Ev{ \text{DGP}(\ensuremath{\mathbf{x}}) \: \text{DGP}(\ensuremath{\mathbf{x}}') } > 0$.
\end{proof}

\begin{theorem}
  Let $k_\text{lim}(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}') = \varphi(\Vert \ensuremath{\mathbf{x}} - \ensuremath{\mathbf{x}}' \Vert_2^2)$ be a mean-square continuous isotropic covariance function
  that is valid on $\ensuremath{\mathbb{R}}^D \times \ensuremath{\mathbb{R}}^D$ for all $D \in \mathbb N$.
  For any width $H_1 \in \mathbb N$, the exists a 2-layer Deep GP $f_2(\ensuremath{\mathbf{f}}_1(\cdot))$ with $\ensuremath{\mathbf{f}}_1(\cdot) : \ensuremath{\mathbb{R}}^D \to \ensuremath{\mathbb{R}}^{H_1}$
  and $f_2(\cdot) : \ensuremath{\mathbb{R}}^{H_1} \to \ensuremath{\mathbb{R}}$ where $\Ev{f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})) f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}'))} = k_\text{lim}(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}')$.
  \label{thm:radial}
\end{theorem}
\begin{proof}[Proof of Theorem~\ref{thm:radial}]
  A classic result from \citet[][Thm.~2]{schoenberg1938metric} is that, for any mean-square continuous isotropic covariance function $\varphi(\Vert \ensuremath{\mathbf{x}} - \ensuremath{\mathbf{x}}' \Vert_2^2)$
  that is valid on $\ensuremath{\mathbb{R}}^D \times \ensuremath{\mathbb{R}}^D$ for all $D \in \mathbb N$, there exists some positive finite measure $\mu(\beta)$
  such that
  %
  \begin{equation}
    \varphi(\Vert \ensuremath{\mathbf{x}} - \ensuremath{\mathbf{x}}' \Vert_2^2) = \int \exp \left( - \frac 1 2 \Vert \ensuremath{\mathbf{x}} - \ensuremath{\mathbf{x}}' \Vert_2^2 \: \beta \right) \intd \mu(\beta).
  \end{equation}
  %
  Let $\text{DGP}^{(1)}(\cdot) = f_2(f_1(\cdot))$ be a 2-layer zero-mean Deep GP with width $H_1 = 1$,
  and let $k_1(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}') = \ensuremath{\mathbf{x}}^\top \ensuremath{\mathbf{x}}'$ and $k_2(z, z') = \int \exp( i \: \beta (z - z') ) \intd \mu(\beta)$.
  (By Bochner's theorem, we know that $k_2(\cdot, \cdot)$ is a valid covariance function.)
  Define $\tau = f_1(\ensuremath{\mathbf{x}}) - f_1(\ensuremath{\mathbf{x}}')$.
  Since $f_1(\ensuremath{\mathbf{x}})$ and $f_1(\ensuremath{\mathbf{x}}')$ are jointly Gaussian:
  \[
    p \left( \begin{bmatrix} f_1(\ensuremath{\mathbf{x}}) \\ f_1(\ensuremath{\mathbf{x}}') \end{bmatrix} \right)
    =
    \normaldist{%
      \begin{bmatrix} 0 \\ 0 \end{bmatrix}
    }{%
      \begin{bmatrix} \ensuremath{\mathbf{x}}^\top \ensuremath{\mathbf{x}} & \ensuremath{\mathbf{x}}^\top \ensuremath{\mathbf{x}}' \\ \ensuremath{\mathbf{x}}^{\prime \top} \ensuremath{\mathbf{x}} & \ensuremath{\mathbf{x}}^{\prime \top} \ensuremath{\mathbf{x}}' \end{bmatrix}
    }
  \]
  we have that $p(\tau) = \normaldist{\ensuremath{\mathbf{0}}}{\Vert \ensuremath{\mathbf{x}} - \ensuremath{\mathbf{x}}' \Vert_2^2}$.
  Substituting $\tau$ into \cref{eqn:stationary_bochner_switched}, we have:
  %
  \begin{align}
    \Ev{ \text{DGP}^{(1)}(\ensuremath{\mathbf{x}}) \: \text{DGP}^{(1)}(\ensuremath{\mathbf{x}}') }
    &= \int \left( \int \exp( i \: \beta \tau ) \intd p(\tau) \right) \intd \mu(\beta).
    \nonumber \\
    &= \int \exp \left( - \frac 1 2 \Vert \ensuremath{\mathbf{x}} - \ensuremath{\mathbf{x}}' \Vert_2^2 \: \beta \right) \intd \mu(\beta).
  \end{align}
  %
  Thus we have a width-1 Deep GP with prior covariance $k_\text{lim}(\cdot, \cdot)$.
  We can extend this construction to 2-layer Deep GP of any width using the additive sequence defined in \cref{eqn:additive_dgp}.
\end{proof}

\section{Experimental Details}
\label{sec:experimental_details}

The experiments are implemented in PyTorch \citep{paszke2019pytorch}, supplemented by the Pyro \citep{bingham2019pyro} and GPyTorch \citep{gardner2018gpytorch} libraries -- all of which are open source.
We run them on a cluster with GTX1080 and GTX2080 GPU,
and we estimate that we use $~1000$ hours of GPU compute time.
The largest experiments (CIFAR10 LeNet and ResNet with maximum width) require $48 GB$ of GPU memory;
all other experiments only require $\leq 11 GB$ of memory.

\paragraph{Datasets.}
The datasets for the regression experiments are from the UCI repository \citep{asuncion2007uci}.
Unless otherwise stated, we split these datasets into $75\%$ training data,
$15\%$ test data, and $10\%$ validation data.
For larger datasets, we subsample the training dataset to a maximum of $N=1000$ data points.
All input features are normalized to be between $-1$ and $1$,
and the $\ensuremath{\mathbf{y}}$ values are z-scored to have $0$ mean and unit variance.

For the non-Bayesian neural network experiments, we use the MNIST \citep{lecun1998gradient} and CIFAR10 \cite{krizhevsky2009learning} datasets.
We z-score the inputs so that each channel has $0$ mean and unit variance.
We use the standard $10,\!000$ data point test sets, and subsample the remaining data for training.

\paragraph{Deep GP models.}
All Deep GP models use GP layers with zero prior mean.
We perform inference without making any scalable approximations,
though we do add a constant diagonal of $10^{-4}$ to all prior covariances for stability.
We perform inference using the NUTS sampler \citep{hoffman2014no} implemented in Pyro \citep{bingham2019pyro},
using $500$ warmup steps, drawing $500$ samples, a target acceptance probability of $0.8$, an initial learning rate of $0.1$,
and a maximum tree depth of $10$.
To improve inference, we infer the ``whitened'' latent variables $\ensuremath{\boldmatrix{L}}_1^{-1} \ensuremath{\boldmatrix{F}}_1$ and $\ensuremath{\boldmatrix{L}}_2^{-1} \ensuremath{\mathbf{f}}_2$,
where $\ensuremath{\boldmatrix{L}}_1$ and $\ensuremath{\boldmatrix{L}}_2$ are the Cholesky factors of $\ensuremath{\boldmatrix{K}}_1(\ensuremath{\boldmatrix{X}}, \ensuremath{\boldmatrix{X}})$ and $\ensuremath{\boldmatrix{K}}_2(\ensuremath{\boldmatrix{F}}_1, \ensuremath{\boldmatrix{F}}_1)$ respectively.
For all width $\geq 2$ Deep GP, we initialize the latent variables by running $1000$ steps of Adam \citep{kingma2014adam}
with learning rate $0.01$
on the maximum a posteriori Deep GP objective.
Because width-$1$ Deep GP inference is more challenging (see the control experiment in \cref{sec:additional_results}), we instead initialize the latent variables of these models from the mean of a doubly stochastic variational Deep GP \citep{salimbeni2017doubly},
where we use $300$ inducing points per layer, $10$ function samples, and a minibatch size of $128$.
We optimize the variational Deep GP with Adam for $2000$ iterations, using an initial learning rate of $0.01$, dropping it by a factor of $10$ after $50\%$ and $75\%$ of training.

For each Deep GP model, we use hyperparameters that maximize the log marginal likelihood of the corresponding limiting GP.
To find these hyperparameters, we perform $100$ iterations of Adam on the limiting GP using a learning rate of $0.1$, initializing all covariance hyperparameters to $1$ and initializing the likelihood observational noise to $0.2$.

\paragraph{Bayesian neural network models.}
We train the Bayesian neural networks in a very similar manner.
However, we perform $2000$ warmup steps and draw $1000$ samples using NUTS.
Again, we use the same hyperparameters as the optimized limiting GP.

\paragraph{Non-Bayesian neural network models.}
We construct the models to match \cref{eqn:2layer_nn}---i.e. we scale the inputs to each layer based on their dimensionality.\footnote{
  We do not perform this scaling on the ResNet models, as this scaling is undone by batch normalization.
}
We optimize these models using Adam (SGD for ResNet models) with a learning rate of $0.1$,
dropping the learning rate by a factor of $10$ after $50\%$ and $75\%$ of training.
The weight decay parameter is set to correspond to a per-parameter prior of $\normaldist{0}{20}$.
We train the models for $20000$ iterations (MNIST) and $40000$ iterations (CIFAR10) with a minibatch size of $256$.

\paragraph{Effect of depth experiments in \cref{sec:experiments_bayesian}.}
In these experiments, our goal is to investigate the effects of depth while controlling for the first and second moments of the Deep GP models.
To that end, we construct a 3-layer Deep GP, 2-layer Deep GP, and a single-layer GP all with zero mean and the same prior covariance.
The 3-layer Deep GP uses a RBF covariance in the first layer, and sums of 1-dimensional RBF kernels in the other two layers.
We set the widths to be $H_1=2$ and $H_2=8$.
The 2-layer Deep GP uses a RBF covariance in the first layer with a width of $H_1=2$, while the second layer uses the following covariance:
\begin{equation}
  o_3^2 \left( 1 + \frac{2 o_2^2 \left( 1 - \frac{1}{H_1} \sum_{i=1}^{H_1} \exp\left(
    -\left( f^{(i)}_1(\ensuremath{\mathbf{x}}) - f^{(i)}_1(\ensuremath{\mathbf{x}}') \right)^2/ (2 \ell_2) \right ) \right)}{\ell_3^2}
  \right)^{-1/2}.
\end{equation}
We empirically confirm that the 2-layer and 3-layer models have the same prior covariance.

For the single layer GP, we compute the covariance of the 3-layer model using the formula in \cref{eqn:3layer_covar}.
We approximate the marginal distribution $p(\ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})), \ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}'))$ using Gauss-Hermite quadrature with 11 nodes.
Again, we empirically confirm that the single-layer and 3-layer models have the same prior covariance.

\paragraph{Visualizing $N=2$ marginal densities in \cref{sec:tails}.}
The 3-layer Deep GP uses a RBF covariance in the first layer, and sums of 1-dimensional RBF kernels in the other two layers.
We vary both widths $H_1$ and $H_2$ simultaneously, and we set all hyperparameters to $1$.
The 2-layer Deep GP are designed to match the prior covariance of the width-1 3-layer model.
To that end, the first layer uses an RBF covariance, and the second layer uses the following covariance:
\begin{equation}
  \frac{1}{H_1} \sum_{i=1}^{H_1} \left( 1 + 2 \left( 1 - \sum_{i=1}^{H_1} \exp\left(
    \frac{-\left( f^{(i)}_1(\ensuremath{\mathbf{x}}) - f^{(i)}_1(\ensuremath{\mathbf{x}}') \right)^2}{ 2 } \right ) \right)
  \right)^{-1/2}.
\end{equation}
Again, we empirically verify that these models have the same  prior covariance.
We approximate the marginal densities at $400$ evenly spaced grid points on $\ensuremath{\mathbf{y}} \in [-3, 3] \times [3, 3]$
using Gauss-Hermite quadrature with 7 nodes.

\section{Derivation of Deep GP Covariances and Limiting GP Covariances}
\label{sec:limiting_kernels}

Here we derive the prior covariances of various Deep GP architectures,
as well as the covariances of their corresponding infinite width GP limits.

\subsection{RBF + Additive RBF.}
First, consider a two layer Deep GP $f_2(\ensuremath{\mathbf{f}}_1(\cdot))$
where the first layer uses a RBF prior covariance
and the second layer uses a sum of 1-dimensional RBF covariances:
\begin{align*}
  k_1( \ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}' ) &= o_1^2 \exp \left( \frac{- \Vert \ensuremath{\mathbf{x}} - \ensuremath{\mathbf{x}}' \Vert^2_2}{2 \ell_1^2} \right),
  \\
  k_2( \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}') ) &= \frac{o_2^2}{H_1} \sum_{i=1}^{H_1} \exp \left( - \frac{( f_1^{(i)}(\ensuremath{\mathbf{x}}) - f_1^{(i)}(\ensuremath{\mathbf{x}}') )^2}{ 2 \ell_2^2 } \right),
\end{align*}
where $H_1$ is the width of the Deep GP, and $o_1$, $\ell_1$, $o_2$, and $\ell_2$ are hyperparameters.
This is the Deep GP architecture most commonly explored in this paper.

To calculate the covariance between $f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}))$ and $f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}'))$, we first note that
$\tau_i \triangleq f_1^{(i)}(\ensuremath{\mathbf{x}}) - f_1^{(i)}(\ensuremath{\mathbf{x}}')$ is Gaussian distributed:
\begin{align}
  \tau_i = f_1^{(i)}(\ensuremath{\mathbf{x}}) - f_1^{(i)}(\ensuremath{\mathbf{x}}') \sim \normaldist{0}{ \sigma^2 },
  \qquad
  \sigma^2 \triangleq k_1(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}) + k_1(\ensuremath{\mathbf{x}}', \ensuremath{\mathbf{x}}') - 2 k_1(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}').
  \label{eqn:tau_diff}
\end{align}
The covariance of the Deep GP is therefore given as:
\begin{align*}
  \Evover{\text{RBF} + \text{add-RBF}}{f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})) f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) }
  &=
  \Evover{\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')}{k_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}') }
  \\
  &=
  \Evover{\tau_i}{\frac{o_2^2}{H_1} \sum_{i=1}^{H_1} \exp \left( - \frac{\tau_i^2}{ 2 \ell_2^2 } \right),}
  \tag{$\tau_i \triangleq f_1^{(i)}(\ensuremath{\mathbf{x}}) - f_1^{(i)}(\ensuremath{\mathbf{x}}')$} \nonumber
  \\
  &=
  \Evover{\tau_1}{\exp \left( - \frac{\tau_1^2}{ 2 \ell_2^2 } \right),}
  \tag{$\tau_i$ are i.i.d.}
  \\
  &= \frac{o_2^2}{\sqrt{2 \pi \sigma^2}}
    \int_{-\infty}^{\infty}
      \exp\left( \frac{ -\tau_1^2 }{2 \ell_2^2}  \right)
      \exp\left( \frac{ -\tau_1^2 }{2 \sigma^2}  \right)
    \intd \tau_1
  \nonumber
  \\
  &= \frac{o_2^2}{\sqrt{2 \pi \sigma^2}}
    \int_{-\infty}^{\infty}
      \exp\left( - \tau_1^2 \frac{ \sigma^2 + \ell_2^2}{2 \sigma^2 \ell_2^2} \right)
    \intd \tau_1
  \nonumber
  \\
  &= \frac{o_2^2}{\sqrt{2 \pi \sigma^2}}
     \sqrt{ \frac{2 \pi \sigma^2 \ell_2^2}{ \sigma^2 + \ell_2^2 }}.
  \tag{Gaussian normalizing constant}
\end{align*}
Plugging in $\sigma^2$ from \cref{eqn:tau_diff}, we have
\begin{equation}
  \Evover{\text{RBF} + \text{add-RBF}}{f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})) f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) } = o_2^2 \left( 1 + \frac{2 o_1^2 \left( 1 - \exp\left(
    -\frac{\Vert \ensuremath{\mathbf{x}} - \ensuremath{\mathbf{x}}' \Vert^2_2}{2 \ell_1^2} \right ) \right)}{\ell_2^2}
  \right)^{-1/2}.
  \label{eqn:add_rbf_dgp}
\end{equation}
Note that this covariance is the same, regardless of the Deep GP width $H_1$.
Therefore, it is also the covariance of the infinite width GP limit.
More generally, if we replace the first RBF covariance with an arbitrary covariance function $k_1(\cdot, \cdot)$
we have:
\begin{equation}
  \Evover{k_1 + \text{add-RBF}}{f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})) f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) } = o_2^2 \left( 1 + \frac%
    {k_1(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}) + k_1(\ensuremath{\mathbf{x}}', \ensuremath{\mathbf{x}}') - 2 k_1(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}')}%
    {\ell_2^2}
  \right)^{-1/2}.
  \label{eqn:add_rbf_dgp_general}
\end{equation}
A similar derivation can be found in \citep{lu2020interpretable}.

\subsection{RBF + (Non-Additive) RBF.}
Now consider a two layer Deep GP $f_2(\ensuremath{\mathbf{f}}_1(\cdot))$
where the first and second layers both use \emph{non-additive} RBF prior covariance functions:
\begin{align*}
  k_1( \ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}' ) &= o_1^2 \exp \left( \frac{- \Vert \ensuremath{\mathbf{x}} - \ensuremath{\mathbf{x}}' \Vert^2_2}{2 \ell_1^2} \right),
  \\
  k_2( \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}') ) &= o_2^2 \exp \left(
    - \frac{\left\Vert \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}) - \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}') \right\Vert_2^2}%
    { 2 H_1 \ell_2^2 }
  \right),
\end{align*}
where the $1 / H_1$ factor is included to reduce the impact of the dimensionality of $\ensuremath{\mathbf{f}}_1(\cdot)$.
This is a very common Deep GP architecture \citep[e.g.][]{damianou2013deep,bui2016deep,cutajar2017random,salimbeni2017doubly},
and it is the architecture in the \cref{sec:posterior} example.

Crucially, the RBF kernel decomposes as a product across its dimensions:
\begin{align*}
  k_2( \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}') ) =
  o_2^2 \exp \left(
    - \frac{\left\Vert \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}) - \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}') \right\Vert_2^2}%
    { 2 H_1 \ell_2^2 }
  \right)
  =
  o_2^2 \prod_{i=1}^{H_1} \exp \left(
    - \frac{\left( f_1^{(i)}(\ensuremath{\mathbf{x}}) - f_1^{(i)}(\ensuremath{\mathbf{x}}') \right)^2}%
    { 2 H_1 \ell_2^2 }
  \right)
\end{align*}
Since the $f_1^{(i)}(\cdot)$ are independent, we have:
\begin{align*}
  \Evover{\text{RBF} + \text{RBF}}{f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})) f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) }
  &=
  \Evover{\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')}{
    o_2^2 \prod_{i=1}^{H_1} \exp \left(
      - \frac{\left( f_1^{(i)}(\ensuremath{\mathbf{x}}) - f_1^{(i)}(\ensuremath{\mathbf{x}}') \right)^2}%
      { 2 H_1 \ell_2^2 }
    \right)
  }
  \\
  &=
  o_2^2 \prod_{i=1}^{H_1} \Evover{f_1^{(i)}(\ensuremath{\mathbf{x}}), f_1^{(i)}(\ensuremath{\mathbf{x}}')}{
    \exp \left(
      - \frac{\left( f_1^{(i)}(\ensuremath{\mathbf{x}}) - f_1^{(i)}(\ensuremath{\mathbf{x}}') \right)^2}%
      { 2 H_1 \ell_2^2 }
    \right)
  }
  \\
  &=
  o_2^2 \prod_{i=1}^{H_1}
  \Evover{\text{RBF} + \text{add-RBF}}{f_2 \left(\frac{\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})}{\sqrt{H_1}}\right) f_2 \left(\frac{\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')}{\sqrt{H_1}}\right) },
\end{align*}
Plugging in \cref{eqn:add_rbf_dgp_general}, we have
\begin{align}
  \Evover{\text{RBF} + \text{RBF}}{f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})) f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) }
  &= o_2^2 \left( 1 + \frac%
    {k_1(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}) + k_1(\ensuremath{\mathbf{x}}', \ensuremath{\mathbf{x}}') - 2 k_1(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}')}%
    {H_1 \ell_2^2}
  \right)^{-H_1/2}.
  \nonumber
  \\
  &= o_2^2 \left( 1 + \frac%
    {2o_1^2 \left( 1 - \exp\left(
      -\frac{\Vert \ensuremath{\mathbf{x}} - \ensuremath{\mathbf{x}}' \Vert_2^2}{2 \ell_1^2}
    \right) \right)}%
    {H_1 \ell_2^2}
  \right)^{-H_1/2}.
\end{align}
In the limit as $H_1 \to \infty$, this second moment becomes:
\begin{align}
  \lim_{H_1 \to \infty}
  \Evover{\text{RBF} + \text{RBF}}{f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})) f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) }
  &= o_2^2 \exp \left( -\frac{ k_1(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}) + k_1(\ensuremath{\mathbf{x}}', \ensuremath{\mathbf{x}}') - 2 k_1(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}') } { 2 \ell_2^2 } \right).
  \nonumber
  \\
  &= o_2^2 \exp \left( \frac{o_1^2}{\ell_2^2} \exp \left( -\frac{\Vert \ensuremath{\mathbf{x}} - \ensuremath{\mathbf{x}}' \Vert_2^2}{2 \ell^2_1} \right) - 1 \right).
  \label{eqn:rbf_dgp_infty}
\end{align}

\subsection{RBF + Additive RBF + Additive RBF.}
\label{sec:rbf3l}

Now consider the three layer Deep GP $f_3(\ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\cdot)))$,
where the first layer uses an RBF covariance and the other layers use sums of 1-dimensional RBF covariances.
\begin{align*}
  k_1( \ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}' ) &= o_1^2 \exp \left( \frac{- \Vert \ensuremath{\mathbf{x}} - \ensuremath{\mathbf{x}}' \Vert^2_2}{2 \ell_1^2} \right),
  \\
  k_2( \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}') ) &= \frac{o_2^2}{H_1} \sum_{i=1}^{H_1} \exp \left( - \frac{( f_1^{(i)}(\ensuremath{\mathbf{x}}) - f_1^{(i)}(\ensuremath{\mathbf{x}}') )^2}{ 2 \ell_2^2 } \right),
  \\
  k_3( \ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})), \ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) ) &= \frac{o_3^2}{H_2} \sum_{i=1}^{H_2} \exp \left( - \frac{\left( f_2^{(i)}(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})) - f_2^{(i)}(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) \right)^2}{ 2 \ell_3^2 } \right),
\end{align*}
where $H_1$, $H_2$ are the widths of the first and second layers,
and $o_1$, $\ell_1$, $o_2$, $\ell_2$, $o_3$ and $\ell_3$ are hyperparameters.
We use this architecture in \cref{sec:tails,sec:experiments_bayesian}.

Unfortunately, it is intractable to compute the second moment of this Deep GP in closed form.
To see why this is the case, note that:
\begin{align}
  \Evover{\text{RBF} + \text{add-RBF} + \text{add-RBF}}{f_3(\ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}))) \:\: f_3(\ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}'))) }
  \:
  =
  \:
  \Evover{\ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})), \ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}'))}{k_3(\ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})), \: \ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}'))) }
  \label{eqn:3layer_covar}
\end{align}
In other words, computing the covariance requires taking the expectation over the Deep GP marginal $\ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})), \ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}'))$ which is intractable to compute.
We do note that we can approximate this marginal with Gauss-Hermite quadrature if $H_1$ is sufficiently small,
Moreover, unlike the 2-layer case, the width $H_1$ affects the second moment of this 3-layer Deep GP.
This is because changing $H_1$ changes the marginal distribution $\ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})), \ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}'))$, which ultimately impacts the expectation in \cref{eqn:3layer_covar}.
Changing the value of $H_2$ does not affect the covariance by linearity of expectation, assuming that we hold $H_1$ constant.

As $H_1 \to \infty$ and $\ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\cdot))$ converges to a Gaussian process,
the expectation in \cref{eqn:3layer_covar} becomes tractable again.
$f_3(\ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\cdot)))$ effectively becomes a 2-layer Deep GP,
where the first layer has covariance given by \cref{eqn:add_rbf_dgp}
and the second layer is the sum of 1-dimensional RBF covariances.
Thus, combining \cref{eqn:add_rbf_dgp} and \cref{eqn:add_rbf_dgp_general},
we can compute the covariance of the limiting GP:

\begin{align}
  &\phantom{=} \lim_{H_2 \to \infty}{H_1 \to \infty} \Evover{\text{RBF} + \text{add-RBF} + \text{add-RBF}}{f_3(\ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}))) \:\: f_3(\ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}'))) }
  \nonumber
  \:
  \\
  &= o_3^2 \left( 1 + \frac{
    2 o_2 \left( 1 -  \left( 1 + \frac{2 o_1^2 \left( 1 - \exp\left(
      -\frac{\Vert \ensuremath{\mathbf{x}} - \ensuremath{\mathbf{x}}' \Vert^2_2}{2 \ell_1^2} \right ) \right)}{\ell_2^2}
    \right)^{-1/2} \right)
  }{\ell_3^2} \right)^{-1/2}
  \nonumber
\end{align}

A similar derivation can be found in \citep{lu2020interpretable}.

\subsection{Neural networks.}
The prior second moment of a neural network with ReLU activations and a single hidden layer
(i.e. the construction in Eq.~\ref{eqn:2layer_nn})
is given by the arc-cosine kernel \citep{cho2009kernel}:
\begin{equation}
  \Evover{\text{2-layer NN}}{
    f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})) f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}'))
  } = \beta^2 + \frac{1}{2 \pi} \Vert \ensuremath{\mathbf{x}} \Vert \: \Vert \ensuremath{\mathbf{x}}' \Vert \left(
    \sin(\theta) + ( \pi - \theta) \cos(\theta)
  \right),
\end{equation}
where $\theta = \cos^{-1} \left( (\ensuremath{\mathbf{x}}^\top \ensuremath{\mathbf{x}}') / (\Vert \ensuremath{\mathbf{x}} \Vert \: \Vert \ensuremath{\mathbf{x}}' \Vert) \right)$.
Note that this covariance is constant regardless of $H_1$.

As with the 3-layer RBF Deep GP, we cannot compute the prior second moment of deeper neural networks in closed form.
Moreover, once neural networks have more than 1 hidden layer, then the width of hidden layers affects the covariance.
Nevertheless, we can compute the limiting infinite width covariance using the recursive formula defined in \citep{cho2009kernel,lee2017deep}.

\section{Proof of Theorem~\ref{thm:dgp_gp}}
\label{sec:proof_dgp_gp}

Here we prove one of our main results (\cref{thm:dgp_gp}), which states that Deep GP converge to (single-layer) GP in the infinite width limit.
Following \citet{matthews2018gaussian}, we will prove that random processes with countable index sets converge in distribution to GP with countable index sets.
Thus, it is sufficient to prove that the marginals of the random process converge in distribution to multivariate Gaussians \citep[e.g.][]{billingsley2013convergence}.

Before arriving at a general result, we begin with specialized proofs for two subclasses of Deep GP:
\begin{enumerate*}
  \item those that use \emph{additively-decomposing prior covariance functions} in each GP layer (\cref{lemma:additive_dgp}), and
  \item those that use \emph{isotropic prior covariance functions} (\cref{lemma:isotropic_dgp}).
\end{enumerate*}
We note that these two cases include many ``textbook'' covariance functions (i.e. the kernels described by \citet{genton2001classes} or \citet[][Ch. 4]{rasmussen2006gaussian}).
Afterwards, we present a more general result.
We note that the assumptions required for the general result are rather minimal, and are indeed satisfied by many Deep GP architectures (or arbitrarily-precise approximations thereof).

\subsection{Warmup 1: Deep GP with Additive and/or Isotropic Covariance Functions}
\label{sec:additive_dgp_proof}

The first case we will explore is Deep GP with covariance functions that decompose additively.
We will assume that the output of the additive composition is scaled to account for the input dimensionality.
This additive decomposition suggests a straightforward application of the strong law of large numbers.
\begin{observation}
  \label{lemma:additive_dgp}
  Let $f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}))$ be a 2-layer zero-mean Deep GP, where $k_2(\cdot, \cdot): \ensuremath{\mathbb{R}}^{H_1} \times \ensuremath{\mathbb{R}}^{H_1} \to \ensuremath{\mathbb{R}}$
  can be written in the form:
  \begin{equation}
    k_2(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}') = \frac{1}{H_1} \sum_{i=1}^{H_1} k_2^{(\text{comp})} \left( z_i, z'_i \right),
    \label{eqn:additive_covar}
  \end{equation}
  where $k_2^{(\text{comp})}(\cdot, \cdot) : \ensuremath{\mathbb{R}} \times \ensuremath{\mathbb{R}} \to \ensuremath{\mathbb{R}}$ is a positive definite function that is bounded by some polynomial:
  %
  \[
    | k_2^{(\text{comp})}(z, z') | \leq \sum_{i=0}^R \sum_{j=0}^{i} a_{i,j} | z^{i-j} z^{\prime j} |
  \]
  %
  for some $R < \infty$ and constants $a_{i,j} > 0$.
  Additionally, assume that $| k_1(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}') | < \infty$ for all finite $\ensuremath{\mathbf{x}}$, $\ensuremath{\mathbf{x}}'$.
  Then the conditional covariance $\Ev{ f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})) f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) \mid \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')}$ becomes almost surely constant as $H_1 \to \infty$ for finite $\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}'$.
\end{observation}
\begin{proof}
  We have that
  %
  \begin{align}
    &\phantom{=} \: \lim_{H_1 \to \infty} \Ev{f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})) f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) \mid \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}) \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')}
    \nonumber
    \\
    &=
    \lim_{H_1 \to \infty} k_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}'))
    =
    \lim_{H_1 \to \infty} \frac{1}{H_1}
    \sum_{i=1}^{H_1} k_2^{(\text{comp})}(f^{(i)}(\ensuremath{\mathbf{x}}), f^{(i)}(\ensuremath{\mathbf{x}}')).
    \label{eqn:additive_limit}
  \end{align}
  %
  Note that all the $k_2^{(\text{comp})}(f^{(i)}(\ensuremath{\mathbf{x}}), f^{(i)}(\ensuremath{\mathbf{x}}'))$ terms are i.i.d. by construction.
  Moreover, since $k_1(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}')$ is finite for all finite $\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}'$,
  all moments of $f_1^{(i)}(\ensuremath{\mathbf{x}}), f_1^{(i)}(\ensuremath{\mathbf{x}}')$ will also be finite.
  Using our assumptions on $k_2^{(\text{comp})}(\cdot, \cdot)$, we have:
  %
  \begin{align*}
    \left\vert \Evover{f^{(i)}(\cdot)}{ k_2^{(\text{comp})}(f^{(i)}(\ensuremath{\mathbf{x}}), f^{(i)}(\ensuremath{\mathbf{x}}'))} \right\vert
    &\leq
    \Evover{f^{(i)}(\cdot)}{ \left\vert k_2^{(\text{comp})}(f^{(i)}(\ensuremath{\mathbf{x}}), f^{(i)}(\ensuremath{\mathbf{x}}')) \right\vert}
    \\
    &\leq
    \sum_{j=0}^R \sum_{k=0}^{j} a_{j,k}
    \Evover{f^{(i)}(\cdot)}{ \left\vert f^{(i)}(\ensuremath{\mathbf{x}})^{j-k} f^{(i)}(\ensuremath{\mathbf{x}}')^k \right\vert }
    \\
    &\leq
    \sum_{j=0}^R \sum_{k=0}^{j} a_{j,k}
    \sqrt{\Evover{f^{(i)}(\cdot)}{ \left\vert f^{(i)}(\ensuremath{\mathbf{x}})^{2j-2k} f^{(i)}(\ensuremath{\mathbf{x}}')^{2k} \right\vert }}
    \\
    &< \infty.
    \tag{moments of a $f_1^{(i)}(\ensuremath{\mathbf{x}}), f_1^{(i)}(\ensuremath{\mathbf{x}}')$ are finite}
  \end{align*}
  %
  Therefore, we can apply the strong law of large numbers to the limit in \cref{eqn:additive_limit}
  which thus is almost surely constant.
\end{proof}

Neural networks are one common case of a Deep GP with additive covariance functions.
Combining \cref{lemma:additive_dgp} with \cref{lemma:nn_cf} gives the classic GP convergence result first discovered by \citet{neal1995bayesian}.
We note however that, when additive structure exists, we do not need to rely on the covariance perspective of \cref{lemma:nn_cf,lemma:dgp_cf}, as we can prove GP convergence using the central limit analysis of \citep{neal1995bayesian,lee2017deep,matthews2018gaussian}.

\subsection{Warmup 2: Deep GP with Isotropic Covariance Functions}
\label{sec:isotropic_dgp_proof}

The most common Deep GP architectures use RBF or Mat\'ern covariance functions for the GP layers \citep[e.g.][]{damianou2013deep,damianou2015deep,bui2016deep,salimbeni2017doubly,havasi2018inference}.
These covariance functions belong to the class of \emph{isotropic kernels}, which are covariances that can be written as a function of Euclidean distance:
\[
  k(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}') = \varphi( \Vert \ensuremath{\mathbf{z}} - \ensuremath{\mathbf{z}}' \Vert_2^2 ).
\]
Similar to \cref{lemma:additive_dgp}, our analysis of isotropic Deep GP relies on the strong law of large numbers.
Again, we will assume that the covariance functions are scaled to account for the input dimensionality.

\begin{observation}
  \label{lemma:isotropic_dgp}
  Define $\mathbb{A} \triangleq \cup_{d=1}^\infty ( \ensuremath{\mathbb{R}}^d \times \ensuremath{\mathbb{R}}^d)$.
  Let $f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}))$ be a 2-layer zero-mean Deep GP, where $k_2(\cdot, \cdot): \mathbb A \to \ensuremath{\mathbb{R}}$
  is a continuous isotropic kernel that can be written in the form:
  \begin{equation}
    k_2(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}') = \begin{cases}
      \varphi \left( (\ensuremath{\mathbf{z}} - \ensuremath{\mathbf{z}}')^2 \right)  & \ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}' \in \ensuremath{\mathbb{R}} \\
      \varphi \left( \frac{1}{2} \Vert \ensuremath{\mathbf{z}} - \ensuremath{\mathbf{z}}' \Vert_2^2 \right)  & \ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}' \in \ensuremath{\mathbb{R}}^2 \\
      \varphi \left( \frac{1}{3} \Vert \ensuremath{\mathbf{z}} - \ensuremath{\mathbf{z}}' \Vert_2^2 \right)  & \ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}' \in \ensuremath{\mathbb{R}}^3 \\
      \varphi \left( \frac{1}{4} \Vert \ensuremath{\mathbf{z}} - \ensuremath{\mathbf{z}}' \Vert_2^2 \right)  & \ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}' \in \ensuremath{\mathbb{R}}^4 \\
      \vdots
    \end{cases}.
    \label{eqn:isotropic_covar}
  \end{equation}
  Additionally, assume that $| k_1(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}') | < \infty$ for all finite $\ensuremath{\mathbf{x}}$, $\ensuremath{\mathbf{x}}'$.
  Then the conditional covariance $\Ev{ f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})) f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) \mid \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')}$ becomes almost surely constant as $H_1 \to \infty$ for finite $\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}'$.
\end{observation}
\begin{proof}
  We have that
  %
  \begin{align}
    &\phantom{=} \: \lim_{H_1 \to \infty} \Ev{f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})) f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) \mid \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}) \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')}
    \nonumber
    \\
    &=
    \lim_{H_1 \to \infty} k_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}'))
    =
    \lim_{H_1 \to \infty} \varphi \left( \frac{1}{H_1} \Vert \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}) - \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}') \Vert_2^2 \right)
    \nonumber
    \\
    &=
    \varphi \left( \lim_{H_1 \to \infty} \sum_{i=1}^{H_1} \left(
      f_1^{(i)}(\ensuremath{\mathbf{x}}) - f_1^{(i)}(\ensuremath{\mathbf{x}}')
    \right)^2 \right)
    \label{eqn:isotropic_limit}
  \end{align}
  %
  where we can move the limit inside $\varphi(\cdot)$ by the continuity assumption.
  Note that $f_1^{(i)}(\ensuremath{\mathbf{x}}) - f_1^{(i)}(\ensuremath{\mathbf{x}}') \stackrel{\text{i.i.d}}{\sim} \normaldist{ 0 }{ \sigma^2 }$,
  where $\sigma^2 = k_1(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}) + k_1(\ensuremath{\mathbf{x}}', \ensuremath{\mathbf{x}}') - 2 k_1(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}')$.
  Since $k_1(\ensuremath{\mathbf{x}}, \ensuremath{\mathbf{x}}')$ is finite---%
  and thus $\sigma^2 = \Ev{(f_1^{(i)}(\ensuremath{\mathbf{x}}) - f_1^{(i)}(\ensuremath{\mathbf{x}}'))^2}$ is finite---%
  \cref{eqn:isotropic_limit} is almost surely constant by the strong law of large numbers.
\end{proof}
Combining \cref{lemma:isotropic_dgp} with \cref{lemma:dgp_cf}, we have that Deep GP with isotropic covariance functions
become GP in their infinite width limit.

\subsection{Assumptions Required for a General Result}
\label{sec:assumptions}

We note that the most common Deep GP/neural network architectures either decompose additively or use isotropic covariance functions \citep{damianou2013deep,bui2016deep,salimbeni2017doubly,agrawal2020wide,aitchison2020bigger}.
In these cases, \cref{lemma:additive_dgp,lemma:isotropic_dgp} are sufficient to prove GP convergence.
To prove a more general result, we will need to make an additional set of assumptions about the GP covariance functions.
We emphasize that these assumptions are sufficient but not necessary conditions,
yet they hold for most Deep GP architectures (or arbitrarily-accurate approximations thereof).
Informally, these assumptions are:
\begin{enumerate}
  \item each covariance function \emph{``scales reasonably''} with the dimensionality of its inputs;
  \item each covariance function has a \emph{Lesbegue-Stieltjes representation}; and
  \item each covariance function is \emph{bounded}.
\end{enumerate}
These assumptions are---in practice---very minimal.
The first item ensures that the prior covariance doesn't ``blow up'' for high-dimensional data (e.g. the covariance does not converge to a constant or a Dirac delta).
The second item admits all but the most pathological covariance functions \citep{yaglom1987correlation}.
The last item may at first seem unreasonable, since some common covariance functions are unbounded (e.g. linear kernel, polynomial kernel, etc.).
However, we note that many covariance functions are bounded on any compact domain, and therefore from a practical perspective we can approximate any of these unbounded covariances to any arbitrary precision.

\begin{assumption}[Covariance functions have Lesbegue-Stieltjes representations with compact spectral support]
  \label{ass:kernels_harmonizable}
  \normalfont
    We assume that all covariance functions $k(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}'): \ensuremath{\mathbb{R}}^D \times \ensuremath{\mathbb{R}}^D \to \ensuremath{\mathbb{R}}$ can be represented by a Lesbegue-Stieltjes integral of the following form:
    %
    \begin{equation}
      k(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}') = \int \rho \left( \frac{1}{D}
      \sum_{i=1}^D \phi \left( z_i \xi_i - z'_i \xi'_i \right) \right)
      \intd \mu(\ensuremath{\boldsymbol{\xi}}, \ensuremath{\boldsymbol{\xi}}'),
      \label{eqn:lesbegue_steiltjes}
    \end{equation}
    %
    where
    \begin{itemize}
      \item $\mu(\ensuremath{\boldsymbol{\xi}}, \ensuremath{\boldsymbol{\xi}}')$ is a positive definite function of bounded variation and compact support: i.e.:
        there exists some constant $C$ such that $\mu( (\ensuremath{\mathbb{R}}^D \times \ensuremath{\mathbb{R}}^D) - ( [-C, C]^D \times [-C, C]^D) ) = 0$;
      \item $\phi : \ensuremath{\mathbb{R}} \to \ensuremath{\mathbb{R}}$ is bounded above and below by a polynomial; and
      \item $\rho(z) : \ensuremath{\mathbb{R}} \to \ensuremath{\mathbb{R}}$ is a continuous function that is finite for all $|z| < \infty$.
    \end{itemize}
    %
    We will show that this formulation---though complex---admits additive kernels, isotropic kernels, and most other non-pathological kernels (or arbitrarily-precise approximations thereof).
\end{assumption}

\begin{assumption}[Covariance functions scale with dimensionality]
  \label{ass:kernels_dimensionality}
  \normalfont
    Let $\mathbb{A} = \cup_{d=1}^\infty ( \ensuremath{\mathbb{R}}^d \times \ensuremath{\mathbb{R}}^d)$.
    We assume that the covariance function $k(\cdot, \cdot): \mathbb{A} \to \ensuremath{\mathbb{R}}$ scales \emph{reasonably with dimensionality};
    that is, it can be written in the following form:
    %
    \[
      k(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}') = \begin{cases}
        k^{(1)}(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}'), & \ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}' \in \ensuremath{\mathbb{R}}, \\
        k^{(2)}(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}'), & \ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}' \in \ensuremath{\mathbb{R}}^2, \\
        k^{(3)}(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}'), & \ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}' \in \ensuremath{\mathbb{R}}^3, \\
        \vdots
      \end{cases}
    \]
    where the $k^{(j)}(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}')$ satisfy \cref{ass:kernels_harmonizable} and are defined recursively:
    %
    \begin{equation}
      \begin{gathered}

        k^{(j)}(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}') = \int \rho \left( \frac{1}{j}
        \sum_{i=1}^D \phi \left( z_i \xi_i - z'_i \xi'_i \right) \right)
        \intd \mu_j(\ensuremath{\boldsymbol{\xi}}, \ensuremath{\boldsymbol{\xi}}'),
        \qquad
        \int_{\xi_j, \xi'_j}
        \intd \mu_j(\ensuremath{\boldsymbol{\xi}}, \ensuremath{\boldsymbol{\xi}}') = \intd \mu_{j-1}(\ensuremath{\boldsymbol{\xi}}_{-j}, \ensuremath{\boldsymbol{\xi}}_{-j}),
      \end{gathered}
      \nonumber
    \end{equation}
    %
    In other words, marginalizing out $\xi_j, \xi_j'$ out of the $2j$-dimensional product measure $\intd \mu_j(\ensuremath{\boldsymbol{\xi}}, \ensuremath{\boldsymbol{\xi}}')$
    recovers the $2(j-1)$-dimensional product measure of $k^{(j-1)}(\cdot, \cdot)$.
\end{assumption}

\subsection{An Exhaustive Discussion About Assumptions~\ref{ass:kernels_harmonizable} and \ref{ass:kernels_dimensionality}}
We again reiterate that \cref{ass:kernels_harmonizable,ass:kernels_dimensionality} are not necessary conditions.
If more is known about the Deep GP architecture (e.g. all covariance functions are additive or isotropic, etc.), then it is possible to use a smaller and simpler set of assumptions.
We nevertheless argue that---in practice, \cref{ass:kernels_harmonizable,ass:kernels_dimensionality} are indeed very general.

\paragraph{All (mean-square continuous) stationary covariance functions can be expressed by \cref{ass:kernels_harmonizable}.}
Note that---if we let $\intd \mu(\cdot, \cdot)$ be atomic---\cref{eqn:lesbegue_steiltjes} admits covariance functions of the form:
\begin{equation}
  k(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}') = \varphi \left( \frac{1}{D}
  \left( \ensuremath{\mathbf{z}} - \ensuremath{\mathbf{z}}' \right) \right),
  \label{eqn:stationary}
\end{equation}
where $\varphi(\cdot) : \ensuremath{\mathbb{R}}^D \to \ensuremath{\mathbb{R}}$ is a bounded and continuous positive definite function.
This is the form of a mean-square continuous \emph{stationary covariance function} \citep[e.g.][Ch. 4]{rasmussen2006gaussian}.

\paragraph{All but the most pathological non-stationary covariance functions can be expressed (or well-approximated) by \cref{ass:kernels_harmonizable}.}
Alternatively, if we instead choose $\rho(\cdot) = \cos(\cdot)$, $\phi(\cdot)$ to be the identity function, and $\mu(\cdot, \cdot)$ to be non-atomic, then \cref{eqn:lesbegue_steiltjes} reduces to:
\begin{equation}
  k(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}') = \int \cos \left( \frac{1}{D}
  \left( \ensuremath{\mathbf{z}}_i \ensuremath{\boldsymbol{\xi}}_i - \ensuremath{\mathbf{z}}'_i \ensuremath{\boldsymbol{\xi}}'_i \right) \right)
  \intd \mu(\ensuremath{\boldsymbol{\xi}}, \ensuremath{\boldsymbol{\xi}}'),
  \label{eqn:fourier_steiltjes}
\end{equation}
This is a Fourier-Stieljes integral, and almost all (bounded) covariance functions that are encountered in the machine learning literature can be written in this form \citep{genton2001classes}.
Such covariance functions are known as \emph{harmonizable kernels}.
\citet[][Sec. 26.4]{yaglom1987correlation} argues that (bounded) covariance functions that \emph{cannot} be expressed as Fourier-Stieltjes integrals tend to be pathological in nature.

Of course, any covariance function that can be expressed as \cref{eqn:fourier_steiltjes} is necessarily \emph{bounded},
since $\cos(\cdot)$ is bounded and the measure $\mu(\cdot, \cdot)$ has bounded variation by assumption.
The most common unbounded covariance functions are dot-product kernels, such as the linear kernel ($\beta^2 + \ensuremath{\mathbf{x}}^\top \ensuremath{\mathbf{x}}'$ for some constant $\beta > 0$)
or the ``ReLU kernel'' ($\beta^2 + \ensuremath{\boldsymbol{\sigma}}(\ensuremath{\mathbf{x}})^\top \ensuremath{\boldsymbol{\sigma}}(\ensuremath{\mathbf{x}}')$, where $\ensuremath{\boldsymbol{\sigma}}(\cdot) = \max\{ \ensuremath{\mathbf{0}}, \cdot \}$).
Importantly, these covariance functions meet the additive structure condition of \cref{lemma:additive_dgp}, and so they do not require the general treatment of \cref{ass:kernels_harmonizable,ass:kernels_dimensionality}.
However, we would also note that these covariances are bounded on any compact domain, so we can approximate them to arbitrary precision by replacing $\ensuremath{\mathbf{x}}$ with $\frac{\ensuremath{\mathbf{x}}}{\Vert \ensuremath{\mathbf{x}} \Vert} \min\{ \Vert \ensuremath{\mathbf{x}} \Vert, B\}$ for any $B < \infty$ and similarly for $\ensuremath{\mathbf{x}}'$.

The other simplifying assumption is that the spectral measure $\mu(\cdot, \cdot)$ has compact support.
Again, even if $k(\cdot, \cdot)$ corresponds to a spectral measure with infinite support, it can often be approximated to an arbitrary precision by replacing $\ensuremath{\boldsymbol{\xi}}$ with $\frac{\ensuremath{\boldsymbol{\xi}} }{\Vert \ensuremath{\boldsymbol{\xi}} \Vert} \min \{ \Vert \ensuremath{\boldsymbol{\xi}} \Vert, B \}$ for some constant $B$.

\paragraph{\cref{ass:kernels_dimensionality} captures natural ways of scaling to dimensionality.}
The recursive formula in \cref{ass:kernels_dimensionality} is very close to the general form defined in \cref{ass:kernels_harmonizable}.
The $1/j$ term simply prevents the covariance from becoming unbounded or degenerate as $j \to \infty$.
We note that---by choosing the appropriate $\rho(\cdot)$ and $\phi(\cdot)$ function in \cref{eqn:lesbegue_steiltjes}---this $1/j$ term can correspond to ``natural'' scaling rates.
For example:
\begin{itemize}
  \item If $k(\cdot, \cdot)$ is isotropic, then \cref{eqn:lesbegue_steiltjes} can take the form:
    \[
      k(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}') = \varphi \left( \frac 1 D \sum_{i=1}^D \left( z_i - z_i' \right)^2 \right),
    \]
    where $\varphi(\cdot)$ is a continuous positive definite function.
    We get this form by setting $\rho(\cdot) = \varphi(\cdot)$, $\phi(\cdot) = (\cdot)^2$, and by setting $\mu(\cdot, \cdot)$ to be atomic.
    This is the scaling of isotropic covariances studied in \cref{sec:isotropic_dgp_proof}.

  \item If $k(\cdot, \cdot)$ is additive, then \cref{eqn:lesbegue_steiltjes} can take the form:
    \[
      k(\ensuremath{\mathbf{z}}, \ensuremath{\mathbf{z}}') = \frac 1 D \sum_{i=1}^D \int \cos \left( z_i \xi_i - z_i' \xi_i' \right) \intd \mu(\xi_i, \xi_i').
    \]
    We get this by setting $\rho(\cdot)$ to be the identity and by setting $\phi(\cdot) = \cos(\cdot)$.
    This is now the sum of 1D harmonizable covariance functions using the scaling studied in \cref{sec:additive_dgp_proof}.
    Again, harmonizable covariances functions include all but the most pathological bounded covariances.
\end{itemize}

\subsection{A General Result}
With \cref{ass:kernels_harmonizable,ass:kernels_dimensionality},
we show that the conditional covariance of a 2-layer zero-mean Deep GP becomes almost surely constant in the limit of infinite-width.

\begin{lemma}\label{lemma:dgp_cond_covar}
  Let $f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}))$ be a 2-layer zero-mean Deep GP, where $k_2(\cdot, \cdot)$ satisfies \cref{ass:kernels_harmonizable,ass:kernels_dimensionality}.
  The conditional covariance $\Ev{ f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})) f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) \mid \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')}$ becomes almost surely constant as $H_1 \to \infty$.
\end{lemma}
\begin{proof}
  By \cref{ass:kernels_harmonizable,ass:kernels_dimensionality}, the limiting conditional covariance can be written as:
  %
  \begin{align}
    &\phantom{=}\:
    \lim_{H_1 \to \infty} \Ev{ f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})) f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) \mid \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')}
    \nonumber
    \\
    &=
    \lim_{H_{1} \to \infty} k_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}'))
    \nonumber
    \\
    &= \lim_{H_{1} \to \infty} \int \rho \left( \frac{1}{H_{1}} \sum_{i=1}^{H_1} \phi \left(  f_1^{(i)}(\ensuremath{\mathbf{x}}) \xi_i - f^{(i)}(\ensuremath{\mathbf{x}}') \xi_i' \right) \right)
      \intd \mu_{(H_{1})}(\ensuremath{\boldsymbol{\xi}}, \ensuremath{\boldsymbol{\xi}}'),
    \tag{\cref{ass:kernels_harmonizable,ass:kernels_dimensionality}}
    \\
    &=  \int \rho \left( \lim_{H_{1} \to \infty} \frac{1}{H_{1}} \sum_{i=1}^{H_1} \phi \left(  f_1^{(i)}(\ensuremath{\mathbf{x}}) \xi_i - f^{(i)}(\ensuremath{\mathbf{x}}') \xi_i' \right) \right)
      \intd \mu_{(H_1)}(\ensuremath{\boldsymbol{\xi}}, \ensuremath{\boldsymbol{\xi}}'),
    \tag{dominated convergence, continuity of $\rho(\cdot)$}
  \end{align}
  %
  where dominated convergence holds because $\rho(\cdot)$ is finite for all finite inputs, $\phi(\cdot)$ is bounded above and below by a polynomial, and $\mu_{(H_{1})}(\ensuremath{\boldsymbol{\xi}}, \ensuremath{\boldsymbol{\xi}}')$ has bounded variation and compact support.
  Now consider the exponentiated term:
  %
  \begin{align}
    \lim_{H_{1} \to \infty} \frac{1}{H_{1}} \sum_{i=1}^{H_{1}} \phi \left( f_1^{(i)}(\ensuremath{\mathbf{x}}) \xi_i - f_1^{(i)}(\ensuremath{\mathbf{x}}') \xi'_i \right).
    \label{eqn:kernel_spectral_limit}
  \end{align}
  %
  The $f^{(i)}(\cdot)$ terms are i.i.d. zero-mean Gaussian by construction, and have finite variance because $k_1(\cdot, \cdot)$ is finite almost everywhere (\cref{ass:kernels_harmonizable}).
  Moreover, the $\xi_i$ and $\xi_i'$ terms are bounded since $\intd \mu_{(H_1)}(\ensuremath{\boldsymbol{\xi}}, \ensuremath{\boldsymbol{\xi}}')$ has compact support (\cref{ass:kernels_harmonizable}).
  Consequentially, $ f_1^{(i)}(\ensuremath{\mathbf{x}}) \xi_i - f_1^{(i)}(\ensuremath{\mathbf{x}}') \xi'_i $ are Gaussian random variables with zero mean and bounded variance.
  Moreover, since $\phi(\cdot)$ is bounded above and below by a polynomial, and the moments of a Gaussian are positive polynomial functions of the covariance,
  we have that
  $\Var{ \phi \left( f_1^{(i)}(\ensuremath{\mathbf{x}}) \xi_i - f_1^{(i)}(\ensuremath{\mathbf{x}}') \xi'_i \right) }$
  is bounded, and thus
  \[
    \left\vert \Ev{ \phi \left( f_1^{(i)}(\ensuremath{\mathbf{x}}) \xi_i - f_1^{(i)}(\ensuremath{\mathbf{x}}') \xi'_i \right) } \right\vert < \infty,
    \qquad
    \sum_{i=1}^\infty \frac 1 {i^2} \Var{ \phi \left( f_1^{(i)}(\ensuremath{\mathbf{x}}) \xi_i - f_1^{(i)}(\ensuremath{\mathbf{x}}') \xi'_i \right) } < \infty,
  \]
  Applying the strong law of large numbers, we have that \cref{eqn:kernel_spectral_limit},
  and thus $\Ev{ f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}})) f_2(\ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')) \mid \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}), \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}')}$,
  converge to a constant almost surely.
\end{proof}

The proof of \cref{thm:dgp_gp} follows from applying \cref{lemma:dgp_cf,lemma:dgp_cond_covar}.
\newtheorem*{dgpthm}{Theorem~\ref{thm:dgp_gp} (Restated)}
\begin{dgpthm}
  \vspace{0.5em}
  Let $f_{L} \circ \ldots \circ \ensuremath{\mathbf{f}}_{1} \left( \ensuremath{\mathbf{x}} \right)$ be a zero-mean Deep GP (Eq.~\ref{eqn:dgp_def}),
  where each layer satisfies \cref{ass:kernels_harmonizable,ass:kernels_dimensionality}.
  Then $\lim_{H_L \to \infty} \cdots \lim_{H_1 \to \infty} f_{L} \circ \ldots \circ \ensuremath{\mathbf{f}}_{1} \left( \ensuremath{\mathbf{x}} \right)$
  converges in distribution to a (single-layer) GP.
\end{dgpthm}
\begin{proof}
In the two layer case, comdinbing \cref{lemma:dgp_cf,lemma:dgp_cond_covar} gives us:
\begin{equation}
  \lim_{H_1 \to \infty} \Ev{ \exp \left( i \ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_2 \right) }
  =
  \exp \left(
    -\frac 1 2 \ensuremath{\mathbf{t}}^\top \Ev{ \ensuremath{\mathbf{f}}_2 \ensuremath{\mathbf{f}}_2^\top } \ensuremath{\mathbf{t}}
  \right)
  \quad
  \text{for all } \ensuremath{\mathbf{t}} \in \ensuremath{\mathbb{R}}^N.
\end{equation}
Note that this is the characteristic function of a zero-mean multivariate Gaussian with covariance $\Ev{ \ensuremath{\mathbf{f}}_2 \ensuremath{\mathbf{f}}_2^\top }.$
Thus by L\'evy's continuity theorem, $\ensuremath{\mathbf{f}}_2$ converges in distribution to $\normaldist{\ensuremath{\mathbf{0}}}{\Ev{ \ensuremath{\mathbf{f}}_2 \ensuremath{\mathbf{f}}_2^\top }}.$
Since this is true for any finite marginal $\ensuremath{\mathbf{f}}_2$, the Deep GP $f_2(\ensuremath{\mathbf{f}}_1(\cdot))$ converges in distribution to a (single-layer) Gaussian process.
A simple induction extends this to multiple layers.
\end{proof}

\subsection{Comparison to \citet{agrawal2020wide}.}
\citet[][Thm. 8]{agrawal2020wide} also study infinite-width limits of Deep GP, though their analysis is restricted to a sub-class of models.
Specifically, they focus on infinitely-wide neural networks with finite bottleneck layers---a specific class of Deep GP
that they refer to as \emph{bottleneck NNGP}.
As the width of the bottleneck layers grow, these models become neural networks with infinite width in all layers.
Coupling this with the analysis of infinitely-wide neural networks \citep{lee2017deep,matthews2018gaussian}, we have that bottleneck NNGP converge to standard GP in the limit of infinite width.
However, the authors note that not every Deep GP can be expressed by the bottleneck NNGP architecture \citep[][Remark 7]{agrawal2020wide},
and so their analysis is not sufficient to prove that all Deep GP converge to GP.

It is worth considering whether this strategy can be applied to other architectures---i.e. what Deep GP can be reduced to infinite-width neural networks with bottlenecks.
For example, \citet{cutajar2017random} study Deep GP with isotropic covariances.
They convert each GP layer into neural network-like layers using random Fourier features \citep{rahimi2007random}.
However, their model is not exactly equivalent to a neural network, and so the analysis of \citet{agrawal2020wide} does not immediately apply.
Moreover, it is not obvious how to express nonstationary GP as neural network-like architectures with modular width.

Finally, we remark that the strategy of \citet[][Thm. 8]{agrawal2020wide} is in some sense the opposite of what is explored in this paper.
They and others \citep{duvenaud2014avoiding,cutajar2017random,dutordoir2021deep} reduce certain classes of Deep GP to infinitely-wide neural networks with bottlenecks;
conversely, we reduce neural networks to Deep GP.

\section{Proofs of Theorems~\ref{thm:mean_concentration} and \ref{thm:tails}}
\label{sec:proof_tails}

Both \cref{thm:mean_concentration,thm:tails} use the same two-step strategy presented for \cref{lemma:nn_cf}.
We will decompose an expectation using the law of total expectation, and then apply Jensen's inequality.

\newtheorem*{thmpeak}{Theorem~\ref{thm:mean_concentration} (Restated)}
\begin{thmpeak}
  Let $f_L \circ \ldots \circ \ensuremath{\mathbf{f}}_1(\cdot)$ be a zero-mean Deep GP.

  Given a finite set of inputs $\ensuremath{\boldmatrix{X}} = [\ensuremath{\mathbf{x}}_1, \ldots, \ensuremath{\mathbf{x}}_N]$, define
  $\ensuremath{\mathbf{f}}_\ell = [(f_\ell \circ \ldots \circ \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_1)), \ldots, (f_\ell \circ \ldots \circ \ensuremath{\mathbf{f}}_1(\ensuremath{\mathbf{x}}_N))]$ for $\ell \in [1, L]$,
  and define $\ensuremath{\boldmatrix{K}}_\text{lim} = \Evover{\ensuremath{\mathbf{f}}_L}{ \ensuremath{\mathbf{f}}_L \ensuremath{\mathbf{f}}_L^\top }$.
  If $\Evover{\ensuremath{\mathbf{f}}_\ell \mid \ensuremath{\mathbf{f}}_{\ell - 1}}{ \ensuremath{\mathbf{f}}_\ell \ensuremath{\mathbf{f}}_\ell^\top }$
  is not constant a.s. for any $\ell \in [2, L]$,
  then,
  $
  p( \ensuremath{\mathbf{f}}_L = \ensuremath{\mathbf{0}}) > \normaldist{\ensuremath{\mathbf{g}} = \ensuremath{\mathbf{0}}; \ensuremath{\mathbf{0}}}{ \ensuremath{\boldmatrix{K}}_\text{lim} }.
  $
  %
\end{thmpeak}

\begin{proof}
  We first produce a characteristic function bound of $\ensuremath{\mathbf{f}}_L$,
  extending the proof technique in \cref{lemma:nn_cf} to multi-layer models:

  \begin{align}
    \Evover{ \ensuremath{\mathbf{f}}_L } { \exp( i \ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_L )}
    &=
    \Evover{ \ensuremath{\boldmatrix{F}}_1 }{ \Evover{\ensuremath{\boldmatrix{F}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \ldots \Evover{\ensuremath{\boldmatrix{F}}_{L-1} \mid \ensuremath{\boldmatrix{F}}_{L-2}}{ \Evover{ \ensuremath{\boldmatrix{F}}_L \mid \ensuremath{\boldmatrix{F}}_{L-1} }{
        \exp( i \ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_L )
    }}}}
    \tag{law of total expectation}
    \\
    &=
    \Evover{ \ensuremath{\boldmatrix{F}}_1 }{ \Evover{\ensuremath{\boldmatrix{F}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \ldots \Evover{ \ensuremath{\boldmatrix{F}}_{L-1} \mid \ensuremath{\boldmatrix{F}}_{L-2} }{
      \exp \left( - \frac 1 2 \ensuremath{\mathbf{t}}^\top \ensuremath{\boldmatrix{K}}_L\left( \ensuremath{\boldmatrix{F}}_{L-1}, \ensuremath{\boldmatrix{F}}_{L-1} \right) \ensuremath{\mathbf{t}} \right)
    }}}
    \tag{Gaussian characteristic function}
    \\
    &\geq
    \exp\left( -\frac 1 2 \ensuremath{\mathbf{t}}^\top
      \Evover{ \ensuremath{\boldmatrix{F}}_1 }{ \Evover{\ensuremath{\boldmatrix{F}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \ldots \Evover{ \ensuremath{\boldmatrix{F}}_{L-1} \mid \ensuremath{\boldmatrix{F}}_{L-2} }{
        \ensuremath{\boldmatrix{K}}_L\left( \ensuremath{\boldmatrix{F}}_{L-1}, \ensuremath{\boldmatrix{F}}_{L-1} \right)
      }}}
    \ensuremath{\mathbf{t}} \right)
    \tag{Jensen's inequality, strict convexity of $\exp$}
    \\
    &=
    \exp\left( -\frac 1 2 \ensuremath{\mathbf{t}}^\top
      \Evover{ \ensuremath{\boldmatrix{F}}_{L-1} }{
        \ensuremath{\boldmatrix{K}}_L\left( \ensuremath{\boldmatrix{F}}_{L-1}, \ensuremath{\boldmatrix{F}}_{L-1} \right)
      }
    \ensuremath{\mathbf{t}} \right)
    =
    \exp\left( -\frac 1 2 \ensuremath{\mathbf{t}}^\top
      \Evover{ \ensuremath{\mathbf{f}}_{L} }{
        \ensuremath{\mathbf{f}}_L \ensuremath{\mathbf{f}}_L^\top
      }
    \ensuremath{\mathbf{t}} \right)
    \tag{law of total expectation}
    \\
    &=
    \exp\left( -\frac 1 2 \ensuremath{\mathbf{t}}^\top
      \Evover{ \ensuremath{\mathbf{f}}_{L} }{
        \ensuremath{\mathbf{f}}_L \ensuremath{\mathbf{f}}_L^\top
      }
    \ensuremath{\mathbf{t}} \right)
    \nonumber
    \\
    &=
    \exp\left( -\frac 1 2 \ensuremath{\mathbf{t}}^\top
      \ensuremath{\boldmatrix{K}}_\text{lim}
    \ensuremath{\mathbf{t}} \right)
    =
    \Evover{\ensuremath{\mathbf{g}} \sim \normaldist{\ensuremath{\mathbf{0}}}{\ensuremath{\boldmatrix{K}}_\text{lim}}} { \exp( i \ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{g}} ) }.
    \label{eqn:cf_bound}
  \end{align}
  %
  Note that the inequality is strict for some $\ensuremath{\mathbf{t}} > \ensuremath{\mathbf{0}}$ because $\Evover{\ensuremath{\mathbf{f}}_\ell \mid \ensuremath{\mathbf{f}}_{\ell - 1}}{ \ensuremath{\mathbf{f}}_\ell \ensuremath{\mathbf{f}}_\ell^\top}$ is almost surely not constant by assumption.
  Thus, we have
  %
  \begin{align*}
    p( \ensuremath{\mathbf{f}}_L = \ensuremath{\mathbf{0}})
    &= \int \Evover{ \ensuremath{\mathbf{f}}_L }{ \exp( i \ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_L ) } \intd \ensuremath{\mathbf{t}}
    \\
    &> \int \Evover{ \ensuremath{\mathbf{g}} \sim \normaldist{\ensuremath{\mathbf{0}}}{\ensuremath{\boldmatrix{K}}_\text{lim}} }{ \exp( i \ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{g}} ) } \intd \ensuremath{\mathbf{t}}
    \:\: = \:\:  \normaldist{\ensuremath{\mathbf{g}} = \ensuremath{\mathbf{0}}; \ensuremath{\mathbf{0}}}{ \ensuremath{\boldmatrix{K}}_\text{lim} }.
  \end{align*}
\end{proof}

It is worth noting that the Jensen gap of the characteristic function cascades with depth.
For example, given the 3-layer model $f_3(\ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\cdot)))$, we have:
\begin{align*}
  \underbracket{\Evover{\ensuremath{\boldmatrix{F}}_1}{\Evover{\ensuremath{\boldmatrix{F}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \exp \left( -\frac 1 2 \ensuremath{\mathbf{t}}^\top \ensuremath{\boldmatrix{K}}_3 \ensuremath{\mathbf{t}} \right) }}}_{\text{CF of 3-layer Deep GP marginal}}
  &\geq
  \underbracket{\Evover{\ensuremath{\boldmatrix{F}}_1}{ \exp \left( -\frac 1 2 \ensuremath{\mathbf{t}}^\top \!\! \Evover{\ensuremath{\boldmatrix{F}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \ensuremath{\boldmatrix{K}}_3 } \ensuremath{\mathbf{t}} \right)}}_{\text{CF of 2-layer Deep GP marginal}}
  \\
  &\geq
  \underbracket{\exp\left( -\frac 1 2 \ensuremath{\mathbf{t}}^\top \!\! \Evover{\ensuremath{\boldmatrix{F}}_1}{\Evover{\ensuremath{\boldmatrix{F}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \ensuremath{\boldmatrix{K}}_3 }} \ensuremath{\mathbf{t}} \right)}_{\text{CF of $\normaldist{\ensuremath{\mathbf{0}}}{\Evover{\ensuremath{\boldmatrix{F}}_1}{\Evover{\ensuremath{\boldmatrix{F}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \ensuremath{\boldmatrix{K}}_3 }}}$}},
\end{align*}
Consequentially, the peaks of deeper models will be sharper than those of shallower models.
This is analogous to the tail effects of depth analyzed in \cref{sec:tails}.

\newtheorem*{thmtails}{Theorem~\ref{thm:tails} (Restated)}
\begin{thmtails}
  Let $\ensuremath{\mathbf{t}} \in \ensuremath{\mathbb{R}}^N$.
  Using the same setup, notation, and assumptions as \cref{thm:mean_concentration},
  the odd moments of $\ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_L$ are zero and the even moments larger than $2$ are super-Gaussian,
  i.e. $\Evover{\ensuremath{\mathbf{f}}_L}{ (\ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_L)^{r} } \geq \Evover{ \ensuremath{\mathbf{g}} \sim \normaldist{\ensuremath{\mathbf{0}}}{\ensuremath{\boldmatrix{K}}_\text{lim}} }{ (\ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{g}})^r }$
  for all even $r \geq 4$.
  Moreover, if $k_L(\cdot, \cdot)$ is bounded almost everywhere, the moment generating function $\Evover{\ensuremath{\mathbf{f}}_L}{ \exp( \ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_L) }$ exists and is similarly super-Gaussian.
\end{thmtails}
\begin{proof}
  We can express the moments of $\ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_L$ as:
  \begin{align*}
    \Evover{\ensuremath{\mathbf{f}}_L}{ \left( \ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_L \right)^r }
    &=
    \Evover{ \ensuremath{\boldmatrix{F}}_1 }{ \Evover{\ensuremath{\boldmatrix{F}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \ldots \Evover{\ensuremath{\boldmatrix{F}}_{L-1} \mid \ensuremath{\boldmatrix{F}}_{L-2}}{ \Evover{ \ensuremath{\boldmatrix{F}}_L \mid \ensuremath{\boldmatrix{F}}_{L-1} }{
       \left( \ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_L \right)^r
    }}}},
  \end{align*}
  and note that the innermost expectation can be simplified to:
  \begin{align*}
    \Evover{\ensuremath{\mathbf{f}}_L \mid \ensuremath{\boldmatrix{F}}_{L-1}}{ \left( \ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_L \right)^r }
    = \begin{cases}
      0 & r \text{ is odd} \\
      \left( \ensuremath{\mathbf{t}}^\top \ensuremath{\boldmatrix{K}}_L(\ensuremath{\boldmatrix{F}}_{L-1}, \ensuremath{\boldmatrix{F}}_{L-1}) \ensuremath{\mathbf{t}} \right)^{\frac r 2} (r - 1)!! & r \text{ is even}.
    \end{cases}
    \tag{Gaussian moments}
  \end{align*}
  For odd $r$, note that this implies $\Evover{\ensuremath{\mathbf{f}}_L}{ (\ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_L)^r } = 0$.
  For even $r \geq 4$, note that $\ensuremath{\mathbf{t}}^\top \ensuremath{\boldmatrix{K}}_L(\ensuremath{\boldmatrix{F}}_{L-1}, \ensuremath{\boldmatrix{F}}_{L-1}) \ensuremath{\mathbf{t}} \geq 0$ by positive definiteness of kernels,
  and note that $(z)^{r/2}$ is convex for all $z \geq 0$.
  Following the same logic as in the proof for \cref{thm:mean_concentration}, we have:
  %
  \begin{align*}
    \Evover{\ensuremath{\mathbf{f}}_L}{ \left( \ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_L \right)^r }
    &=
    \Evover{ \ensuremath{\boldmatrix{F}}_1 }{ \Evover{\ensuremath{\boldmatrix{F}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \ldots \Evover{\ensuremath{\boldmatrix{F}}_{L-1} \mid \ensuremath{\boldmatrix{F}}_{L-2}}{
        \left( \ensuremath{\mathbf{t}}^\top \ensuremath{\boldmatrix{K}}_L(\ensuremath{\boldmatrix{F}}_{L-1}, \ensuremath{\boldmatrix{F}}_{L-1}) \ensuremath{\mathbf{t}} \right)^{\frac r 2} (r - 1)!!
    }}}
    \\
    &\geq
    \left( \ensuremath{\mathbf{t}}^\top
      \Evover{ \ensuremath{\boldmatrix{F}}_1 }{ \Evover{\ensuremath{\boldmatrix{F}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \ldots \Evover{\ensuremath{\boldmatrix{F}}_{L-1} \mid \ensuremath{\boldmatrix{F}}_{L-2}}{
        \ensuremath{\boldmatrix{K}}_L(\ensuremath{\boldmatrix{F}}_{L-1}, \ensuremath{\boldmatrix{F}}_{L-1})
      }}}
    \ensuremath{\mathbf{t}} \right)^{\frac r 2} (r - 1)!!
    \tag{Jensen's inequality, strict convexity}
    \\
    &=
    \left( \ensuremath{\mathbf{t}}^\top \ensuremath{\boldmatrix{K}}_\text{lim} \ensuremath{\mathbf{t}} \right)^{\frac r 2} (r - 1)!!
    =
    \Evover{\normaldist{\ensuremath{\mathbf{g}}; \ensuremath{\mathbf{0}}}{\ensuremath{\boldmatrix{K}}_\text{DGP}(\ensuremath{\boldmatrix{X}}, \ensuremath{\boldmatrix{X}})}}{ (\ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{g}})^r }
  \end{align*}
  %
  A similar proof will show that the moment generating function is similarly super-Gaussian:
  \begin{align}
    \Evover{ \ensuremath{\mathbf{f}}_L } { \exp( \ensuremath{\mathbf{t}}^\top \ensuremath{\mathbf{f}}_L )}
    &=
    \Evover{ \ensuremath{\boldmatrix{F}}_{L-1} }{
      \exp \left( \frac 1 2 \ensuremath{\mathbf{t}}^\top \ensuremath{\boldmatrix{K}}_L\left( \ensuremath{\boldmatrix{F}}_{L-1}, \ensuremath{\boldmatrix{F}}_{L-1} \right) \ensuremath{\mathbf{t}} \right)
    }
    \label{eqn:dgp_mgf_expanded}
    \\
    &=
    \Evover{ \ensuremath{\boldmatrix{F}}_1 }{ \Evover{\ensuremath{\boldmatrix{F}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \ldots \Evover{ \ensuremath{\boldmatrix{F}}_{L-1} \mid \ensuremath{\boldmatrix{F}}_{L-2} }{
      \exp \left( \frac 1 2 \ensuremath{\mathbf{t}}^\top \ensuremath{\boldmatrix{K}}_L\left( \ensuremath{\boldmatrix{F}}_{L-1}, \ensuremath{\boldmatrix{F}}_{L-1} \right) \ensuremath{\mathbf{t}} \right)
    }}}
    \nonumber
    \\
    &\geq
    \exp\left( \frac 1 2 \ensuremath{\mathbf{t}}^\top
      \Evover{ \ensuremath{\boldmatrix{F}}_1 }{ \Evover{\ensuremath{\boldmatrix{F}}_2 \mid \ensuremath{\boldmatrix{F}}_1}{ \ldots \Evover{ \ensuremath{\boldmatrix{F}}_{L-1} \mid \ensuremath{\boldmatrix{F}}_{L-2} }{
        \ensuremath{\boldmatrix{K}}_L\left( \ensuremath{\boldmatrix{F}}_{L-1}, \ensuremath{\boldmatrix{F}}_{L-1} \right)
      }}}
    \ensuremath{\mathbf{t}} \right)
    \nonumber
    \\
    &=  \exp \left( \frac 1 2 \ensuremath{\mathbf{t}}^\top \ensuremath{\boldmatrix{K}}_\text{lim} \ensuremath{\mathbf{t}} \right)
    = \Evover{\normaldist{\ensuremath{\mathbf{g}}; \ensuremath{\mathbf{0}}}{\ensuremath{\boldmatrix{K}}_\text{lim}}}{ \exp( \ensuremath{\mathbf{g}}^\top \ensuremath{\mathbf{t}} ) }.
    \nonumber
  \end{align}
  We know that the moment generating function exists because, by assumption, $k_L(\cdot, \cdot)$ is bounded almost everywhere, and thus the integral defined by the expectation in \cref{eqn:dgp_mgf_expanded} is finite.
\end{proof}

\section{Additional Results}
\label{sec:additional_results}

\subsection{Comparing Tails of Wider versus Deeper Models}
In \cref{sec:tails}, we plot $N=2$ marginal densities of Deep GP to demonstrate how depth and width affect tail properties (\cref{fig:density_depth_width}).
To further demonstrate that Deep GP are heavy tailed and sharply peaked, \cref{fig:gp_diff_depth_width} displays the \emph{difference} between Deep GP marginal densities and the limiting GP marginal density, i.e.:
\[
  p_\text{DGP}(y_1, y_2 \mid \ensuremath{\mathbf{x}}_1, \ensuremath{\mathbf{x}}_2) - p_\text{Lim. GP}(y_1, y_2 \mid \ensuremath{\mathbf{x}}_1, \ensuremath{\mathbf{x}}_2).
\]
Red areas correspond to values of $\ensuremath{\mathbf{y}}$ where the Deep GP has more density, while blue areas correspond to values where the limiting GP has more density.
Note that Deep GP of all widths and depths have red values near the $[0, 0]$ mean (corresponding to a sharper peak than the limiting GP) and red values in the upper left and lower right quadrants (corresponding to heavier tails than the limiting GP).

We also note that deeper/narrower models have heavier tails and sharper peaks than shallower/wider models (\cref{fig:width_diff_depth_width}).
The left plot shows the difference between the marginal densities of Depth-3 and Depth-2 Deep GP (with the same first and second moments).
The remaining plots show the difference between Depth-2 Deep GP of varying width (again, with the same first and second moments).

\subsection{Control Experiment: How Well Does NUTS Sample Deep GP Posteriors?}
In order to determine if the NUTS sampler accurately captures Deep GP performance,
we perform a control experiment on synthetic data.
Specifically, we generate $N=1000$ datasets from $\text{width}=1$, $\text{width}=2$, $\text{width}=4$, $\text{width}=8$ Deep GP,
sampling these models at randomly-generated $\ensuremath{\mathbf{x}}$ values in a $4$-dimensional input space.
Each generating Deep GP has two layers: the first uses a RBF covariance, and the second uses the sum of 1-dimensional RBF covariances.
We then train $\text{width}=1$, $\text{width}=2$, $\text{width}=4$, $\text{width}=8$ Deep GP on each of the generated datasets,
using half the data for training and half for testing.
Our hypothesis is that $\text{width}=j$ models should at least achieve good test set performance on $\text{width}=j$ generated datasets.

\cref{fig:control_experiment} displays the test set log likelihood on the generated datasets.
We see that $\text{width} \in \{2, 4, 8\}$ models tend to achieve similar performance on each of the datasets.
On the other hand, the $\text{width}=1$ models achieve significantly worse test set performance,
even on the dataset generated by a $\text{width}=1$ model.
This suggests that the NUTS sampler is unable to converge to good posterior samples for $\text{width}=1$ models,
which may potentially explain the superior performance of $\text{width}=2$ Deep GP observed in the \cref{sec:experiments_bayesian} experiments.

\subsection{3-Layer Deep GP and 3-Layer Bayesian Neural Networks}
We extend the experiments from \cref{sec:experiments_bayesian} to 3-layer Deep GP and Bayesian neural networks.
Specifically, we measure the test set log likelihood (\cref{fig:dgp3l_width} top) and training set kernel fit (\cref{fig:dgp3l_width} bottom) on 6 regression datasets.
The Deep GP models use a standard RBF covariance function for the first layer and sums of 1-dimensional RBF covariances for the second and third layers.
The neural networks add an additional $f_3(\cdot) = {\textstyle \frac{1}{\sqrt{H_2}}} \ensuremath{\mathbf{w}}_3^\top \ensuremath{\boldsymbol{\sigma}}(\cdot) + \beta b_3$
layer on top of the \cref{eqn:2layer_nn} construction, where $H_2$ is the width of the second layer and the entries of $\ensuremath{\mathbf{w}}_3, b_3$ are i.i.d. unit Gaussian.

We compare models of different width, increasing the width of both hidden layers simultaneously.
It is worth noting that changing the width of the first hidden layer affects the prior second moment of the Deep GP/neural network.
Therefore, unlike the the experiments in \cref{sec:experiments_bayesian}, we are no longer ensuring that all models are moment-matched.
Nevertheless, width has the same effect for 3-layer models as it does for 2-layer models.
Width-$2$ Deep GP almost always outperform all other Deep GP, and neural networks achieve best log likelihood and kernel fit with $\leq 8$ hidden units per layer.

\subsection{Additional Figures and Tables}
We report additional metrics for the experiments in \cref{sec:experiments_bayesian}.
\cref{fig:dgp_width_rmse,fig:dgp3l_width_rmse} report the test set root mean squared error (RMSE) of 2-layer and 3-layer models as a function of width (lower is better).
We find that additional width generally harms RMSE.
\cref{tab:dgp_depth_rmse} reports the test set RMSE for Deep GP of various depth (controlling the first and second prior moments, as in \cref{tab:dgp_depth}).
Again, depth is generally beneficial with regards to RMSE.

\begin{figure}[ht!]
  \centering
  \includegraphics[width=\linewidth]{figures/gp_diff_depth_width.pdf}
  \caption{
    We depict the \emph{difference} between the Deep GP marginal density $p_{\text{DGP}}(y_1, y_2 \mid \ensuremath{\mathbf{x}}_1, \ensuremath{\mathbf{x}}_2)$
    and the limiting GP marginal density $p_{\text{Lim. GP}}(y_1, y_2 \mid \ensuremath{\mathbf{x}}_1, \ensuremath{\mathbf{x}}_2)$
    on the $N=2$ dataset $\ensuremath{\mathbf{x}}_1 = -0.5$, $\ensuremath{\mathbf{x}}_2 = 0.5$.
    Red regions correspond to values of $y_1, y_2$ where the Deep GP has more density, and vice versa for the blue regions.
    All Deep GP have heavier tails and a sharper peak than the limiting GP.
    \label{fig:gp_diff_depth_width}
  }
\end{figure}

\begin{figure}[ht!]
  \centering
  \includegraphics[width=\linewidth]{figures/width_diff_depth_width.pdf}
  \caption{
    We depict the \emph{difference} between the marginal densities of various Deep GP on the $N=2$ dataset $\ensuremath{\mathbf{x}}_1 = -0.5$, $\ensuremath{\mathbf{x}}_2 = 0.5$.
    Red regions correspond to values of $y_1, y_2$ where the deeper/narrower Deep GP has more density, and vice versa for the blue regions.
    {\bf Left:} Comparing Deep GP of different depth.
    The 3-layer (width-1) Deep GP has a sharper peak and heavier tails than the 2-layer model, as indicated by the red regions.
    {\bf Right:} Comparing 2-layer Deep GP of different width.
    Width-$j$ models have sharper peaks and heavier tails than width-$j+1$ models, as indicated by the red regions.
    All models have the same first and second moments.
    \label{fig:width_diff_depth_width}
  }
\end{figure}

\begin{figure}[ht!]
  \centering
  \includegraphics[width=0.6\linewidth]{figures/control_experiment.pdf}
  \caption{
    Control experiment to test how well NUTS sampling captures the Deep GP posterior.
    We generate datasets from $\text{width} \in \{ 1, 2, 4, 8 \}$ Deep GP,
    and then fit $\text{width} \in \{ 1, 2, 4, 8 \}$ Deep GP to these datasets.
    The $\text{width} \in \{ 2, 4, 8 \}$ models achieve roughly the same test set log likelihood on each dataset (higher is better).
    $\text{width}=1$ Deep GP tend to achieve worse log likelihood, even on a dataset generated from a $\text{width}=1$ Deep GP.
    \label{fig:control_experiment}
  }
\end{figure}

\begin{figure}[ht!]
  \centering
  \includegraphics[width=\linewidth]{figures/dgp3l_width.pdf}
  \caption{
    {\bf Top:} Test set log likelihood (LL) of {\bf 3-layer Deep GP} (and neural networks) as a function of width on regression datasets (higher is better).
    Numbers are shifted so that $0$ corresponds to the limiting GP log likelihood.
    {\bf Bottom:} Fit of the posterior kernel $k(\ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\cdot)), \ensuremath{\mathbf{f}}_2(\ensuremath{\mathbf{f}}_1(\cdot)))$ on the training data, as measured by Gaussian log marginal likelihood (higher is better).
    $0$ corresponds to the limiting GP log marginal likelihood.
  }
  \label{fig:dgp3l_width}
\end{figure}

\clearpage

\begin{figure}[ht!]
  \centering
  \includegraphics[width=\linewidth]{figures/dgp_width_rmse.pdf}
  \caption{
    Test set root mean squared error (RMSE) of {\bf 2-layer} Deep GP (and neural networks) as a function of width on regression datasets (lower is better).
    Numbers are scaled so that $1$ corresponds to the limiting GP RMSE.
  }
  \label{fig:dgp_width_rmse}
\end{figure}

\begin{figure}[ht!]
  \centering
  \includegraphics[width=\linewidth]{figures/dgp3l_width_rmse.pdf}
  \caption{
    Test set root mean squared error (RMSE) of {\bf 3-layer} Deep GP (and neural networks) as a function of width on regression datasets (lower is better).
    Numbers are scaled so that $1$ corresponds to the limiting GP RMSE.
  }
  \label{fig:dgp3l_width_rmse}
\end{figure}

\begin{table}[ht!]
  \caption{%
    Root mean squared error (RMSE) of Deep GP on UCI regression datasets as a function of depth (lower is better).
    All models for a given dataset have the same prior covariance.
  }
  \label{tab:dgp_depth_rmse}
  \centering
  \resizebox{\linewidth}{!}{%
    \input tables/uci_depth_rmse.tex
  }
\end{table}