commit 2c88600f7a817437dd08b1125c5d9c25ee6c9394
parent 1e8f4f61931768f795eb54d71d844eb495ab3e05
Author: Silas Brack <s174433@student.dtu.dk>
Date: Sun, 5 Feb 2023 17:10:56 +0100
Progress.
Diffstat:
14 files changed, 171 insertions(+), 86 deletions(-)
diff --git a/chapters/abstract.tex b/chapters/abstract.tex
@@ -3,24 +3,30 @@
% https://www.student.unsw.edu.au/writing-abstracts-honours-theses
-Bayesian methods promise to provide a principled way to quantify uncertainty in neural networks.
-These methods rely on Bayesian inference to learn the posterior distribution of the model parameters given the data.
-In doing so, they allow for uncertainty estimates, which is important for many applications in machine learning, such as those involving safety-critical decisions which rely on risk assessment and interpretability.
-However, Bayesian inference is often computationally intractable, and as such, approximate methods which compromise performance for feasibility are used.
-The Laplace approximation is one such method which enables post-hoc estimation of the posterior distribution by assuming a normal distribution for the posterior and using the Hessian to approximate the posterior precision.
-Additionally, since the Laplace approximation deduces uncertainty rather than learning it, it has many advantages over other Bayesian methods that attempt to learn variance.
-Due to its construction, the Laplace approximation allows for training by maximising the marginal likelihood, which facilitates model selection and hyperparameter optimisation.
-However, the Laplace approximation typically relies on crude approximations to the posterior precision to make it computationally feasible.
-When we do not factor this precision matrix into more practical approximations, the full posterior precision must be instantiated, which is intractable for all but toy problems, since this matrix scales quadratically with the number of parameters.
-When we perform the Laplace approximation using this full posterior precision, we call this the full Laplace approximation.
-% Conventional full Laplace uses the full posterior precision matrix, which cannot be instantiated for all but toy problems, since it scales quadratically with the number of parameters.
-In this work, we propose a method for performing the Laplace approximation using only Hessian-vector products, which allows us to perform marginal training and inference using the full Laplace approximation without storing the entire Hessian.
-To accomplish this, we estimate the Hessian's log-determinant and inverse square root using only Hessian-vector products.
-We also suggest a technique for evaluating the quality of posterior samples without instantiating nor inverting the Hessian based on results from traditional statistics.
+Bayesian methods promise to provide a principled way to quantify uncertainty in neural networks
+This is important for many applications in machine learning, such as those involving safety-critical decisions which rely on risk assessment and interpretability.
+However, Bayesian inference is often computationally intractable, so approximate methods which compromise performance for practicality are used.
+The Laplace approximation (LA) is one such method, though it itself typically relies on crude approximations to the posterior precision matrix to make it computationally feasible.
+When we do not factor the posterior precision into more practical approximations, the full posterior precision must be instantiated.
+We call this the full Laplace approximation.
+For all but toy problems, however, the full Laplace approximation is intractable, since the posterior precision matrix scales quadratically with the number of parameters.
+
+In this work, we propose a method for computing the Laplace approximation using only Hessian-vector products, which allows us to perform marginal training and posterior sampling using the full Laplace approximation without storing the entire Hessian.
+To accomplish this, we show that it is possible to estimate the Hessian's log-determinant and inverse square root using only Hessian-vector products.
+Next, we suggest a technique for evaluating the quality of posterior samples without instantiating nor inverting the Hessian based on results from traditional statistics.
To implement this method, we use JAX, a library for automatic differentiation which enables us to efficiently perform Hessian-vector products without explicitly storing the entire Hessian.
-We show that our method is able to perform the full Laplace approximation on both a sine function and MNIST and that training the marginal likelihood using our method is feasible and produces meaningful results.
+
+We perform the full Laplace approximation on both a sine function and MNIST using our method.
+This consists two steps, (a) training a neural network by maximising either its posterior probability or marginal likelihood and (b) sampling from this posterior.
+We find that our approximate maximum marginal training procedure is able to learn a set of parameters which yields performance comparable to that of the maximum posterior and maximum likelihood training procedures.
+By analysing quantile--quantile plots of our posterior samples and visualising our posterior predictive samples, we find that our approximate sampling method produces samples which are correctly distributed.
+We then benchmark the performance of Hessian-vector products and estimate that our method yields a 10\,000x memory reduction over the conventional full Laplace approximation, since it does not store the quadratically-scaling full Hessian.
This is the first method for performing the Laplace approximation while only accessing the Hessian implicitly.
-Future work will focus on carrying out hyperparameter optimisation and large-scale experiments using this method.
+
+% Our method yields a 10\,000x memory reduction over the conventional full Laplace approximation, since it does not store the quadratically-scaling full Hessian.
+% We compute the full Laplace approximation on both a sine function and MNIST.
+% To do this, we demonstrate that the samples generated by our approximate sampling procedure are correctly distributed and that training the marginal likelihood using our method is feasible and produces meaningful results.
+% This is the first method for performing the Laplace approximation while only accessing the Hessian implicitly.
% \chapter*{Resum\'e}
% \addcontentsline{toc}{chapter}{Resum\'e}
diff --git a/chapters/boring_experiments.tex b/chapters/boring_experiments.tex
@@ -43,7 +43,7 @@ These experiments were run on a single NVIDIA A100 GPU.
The results for this experiment have been visualised in \cref{fig:hessian-profile}.
Additionally, the raw results for this experiment have been summarised in the appendix, in \cref{tab:hessian-profile}.
We can see that the explicit Hessian-vector product fails for \(D > 1 M\) parameters, while the explicit inversion fails for \(D > 10 k\) parameters.
-This is not surprising, as the explicit Hessian is a \(D \times D\) matrix, and so the memory requirements for this matrix are \(D^2\) parameters.
+This is not surprising, as the explicit Hessian is a \(D \times D\) matrix, and so the memory requirements for this matrix depend on the \(D^2\) parameters.
Additionally, while the wall clock time for computing the Hessian-vector product is sublinear with \(D\), explicit Hessian-vector product is over an order of magnitude slower for \(D = 1 \mathrm{M}\) than for \(D = 100 \mathrm{k}\).
This may be due to the explicit Hessian, at this size, being too large to fit in the cache of the GPU.
@@ -115,7 +115,7 @@ In practice, it is likely that only a small number of samples will be obtained (
However, the sampling procedure still takes over 20 minutes to compute 10 samples.
As such, the time required to compute the samples may be a significant factor in the overall time required to compute the Laplace approximation.
In particular, for models with a very large number of parameters, the time required to compute the samples may be prohibitive when compared when the near-instant time required to sample from the diagonal Laplace approximation.
-Even if the sampling time scales sublinearly with the number of parameters, it would still prevent the application in methods such as online Laplace, which require the Laplace approximation to be computed at each iteration, from being used in practice.
+Even if the sampling time scales sublinearly with the number of parameters, it would still prevent the application in Laplace methods that require the Laplace approximation to be computed at each iteration from being used in practice.
However, this sampling time constraint is also significant for MCMC methods, which are the most common method for sampling from the posterior distribution of a BNN.
\marginnote{Results for the experiments in this chapter run on CPU are shown in the appendix, in \cref{fig:hessian-profile-cpu}.}
diff --git a/chapters/conclusion.tex b/chapters/conclusion.tex
@@ -18,7 +18,7 @@ However, we also found that the marginal training procedure to optimise the hype
Finally, we showed that it is possible to perform the full Laplace approximation with marginal likelihood training, and sample from its posterior.
We demonstrated on a simple sine wave regression problem and an MNIST classification problem that our method can be used to train the full Laplace approximation and yield predictive performance which rivals deterministic training.
-However, we found that the sampling procedure is too slow for online Laplace, and may become a significant bottleneck for inference using post-hoc Laplace on large-scale problems.
+However, we found that the sampling procedure is too slow too sample outside of a post-hoc setting, and may become a significant bottleneck for inference even under post-hoc Laplace on large-scale problems.
Overall, while we have developed novel methods for training and inference in the full Laplace approximation, we have not yet demonstrated its improved performance over the diagonal Laplace approximation nor its practical application to large-scale problems.
diff --git a/chapters/experiments.tex b/chapters/experiments.tex
@@ -2,8 +2,6 @@
\chapter{Cool Experiments}
\label{ch:cool-experiments}
-WARNING: THIS CHAPTER IS NOT FINISHED
-
\section{The Sine Function}
The choice of \(\alpha\) is a non-trivial one
@@ -11,8 +9,8 @@ Show plot of predictions for alpha 100 and alpha 1, to show that alpha 1 looks s
\begin{marginfigure}
\centering
- \includegraphics{sin.pdf}
- \caption[Laplace predictions for a sine curve.]{Visualisation of the predicted output of multilayer perceptron (MLP) modelling a sine curve. REPLACE THIS WITH NEW MARGINAL PLOTS}
+ \includegraphics{sin_posterior_predictive_joint.pdf}
+ \caption[Laplace predictions for a sine curve.]{Visualisation of the predicted output of multilayer perceptron (MLP) modelling a sine curve.}
\label{fig:sin}
\end{marginfigure}
diff --git a/chapters/introduction.tex b/chapters/introduction.tex
@@ -10,13 +10,22 @@
% Specific applications of Laplace approximation in Bayesian neural networks.
% Why this problem is important.
+% The Laplace approximation is one such method which enables post-hoc estimation of the posterior distribution by assuming a normal distribution for the posterior and using the Hessian to approximate the posterior precision.
+% Additionally, since the Laplace approximation deduces uncertainty rather than learning it, it has many advantages over other Bayesian methods that attempt to learn variance.
+% Due to its construction, the Laplace approximation allows for training by maximising the marginal likelihood, which facilitates model selection and hyperparameter optimisation.
+% However, the Laplace approximation typically relies on crude approximations to the posterior precision to make it computationally feasible.
+% When we perform the Laplace approximation using this full posterior precision, we call this the full Laplace approximation.
+% Conventional full Laplace uses the full posterior precision matrix, which cannot be instantiated for all but toy problems, since it scales quadratically with the number of parameters.
+
Bayesian methods allow for estimates of uncertainty which enable more efficient usage of data (e.g., via active learning) and avoid overfitting.
-They accomplish this by modelling a distribution on the model parameters, which is then used to make predictions.
+They accomplish this by using Bayes' theorem to model a distribution on the model parameters, which is then used to make predictions.
From here we can sample from this distribution to make predictions, thereby yielding uncertainty estimates, improving model interpretability and enabling the assessment of model predictive confidence.
+In doing so, they allow for uncertainty estimates, which is important for many applications in machine learning, such as those involving safety-critical decisions which rely on understanding risk and interpretability, where traditional deterministic neural networks are overconfident or fail to provide uncertainty estimates at all.
+The estimation of model confidence is particularly important in domains where silent failures can have catastrophic consequences~\cite{jospin2022hands}, such as autonomous driving~\cite{rao2018deep}, medical diagnosis~\cite{ker2017deep}, and financial markets~\cite{cavalcante2016computational}.
+The improved interpretability, however, is very valuable in the context of deep learning, where models are black-boxes which are difficult to interpret.
These methods have been successfully applied to a wide range of classical problems in statistics, though attempts to apply them to deep learning have had limited success.
-
For deep learning, the Bayesian approach is often limited by the computational cost of Bayesian inference.
-This is because Bayesian inference requires computing the posterior distribution of the model parameters given the data, which, due to the nonlinear complexity of the neural network architecture, is often intractable.\sidenote{MEH. Would like to mention some applications, to argue that this is an important problem problem worth solving. However, I'm struggling to find solid applications, i.e., large-scale problems solved by BNNs. \textcite{wenzel2020good} suggest there haven't really been any.}
+This is because Bayesian inference requires computing the posterior distribution of the model parameters given the data, which, due to the nonlinear complexity of the neural network architecture, is often intractable.
Typically, the Bayesian approach to deep learning is to use a prior distribution over the model parameters and then compute the posterior distribution of the model parameters given the data, where the posterior is maximised to obtain the optimal model parameters for the available data.
This posterior distribution can then be sampled from to make predictions.
@@ -30,8 +39,8 @@ However, this is computationally infeasible, and so approximate Bayesian methods
\section{Current Methods}
-Currently, approximate Bayesian methods are either expensive to compute (Markov Chain Monte Carlo), are significantly more difficult to implement (such as variational inference), or simply perform poorly and are limited in their Bayesian interpretation (Monte Carlo dropout, deep ensembles, SWAG, etc.).
-Of these, MC dropout and Markov Chain Monte Carlo are the most widely used, though both have significant drawbacks.
+Currently, approximate Bayesian methods are either expensive to compute (Markov chain Monte Carlo) or are limited in their Bayesian interpretation (e.g., Monte Carlo dropout~\cite{gal2016dropout}, deep ensembles~\cite{lakshminarayanan2017simple}, SWAG~\cite{maddox2019simple}).
+Of these, MC dropout and Markov chain Monte Carlo are the most widely used, though both have significant drawbacks.
As such, there is demand for a method which exhibits the same computational cost as the optimization algorithms for deterministic neural networks while providing accurate posterior approximations and working out-of-the-box for any given architecture.
The Laplace approximation~\cite{daxberger2021laplace,laplace1774memoire} is a simple yet theoretically well-supported posterior approximation suitable for Bayesian modeling.
@@ -82,9 +91,15 @@ An overview of the Laplace approximation and our contributions is shown in \cref
\begin{figure}
\centering
\input{figures/overview.tex}
- \caption[Overview of the Laplace approximation.]{Overview of the Laplace approximation (LA) for Bayesian neural networks. We highlight our contributions in green, i.e., the computations which we will approximate using only Hessian-vector products.}
+ \caption[Overview of the Laplace approximation.]{Overview of the Laplace approximation (LA) for Bayesian neural networks. We highlight in green the computations which we approximate in this project using only Hessian-vector products, namely the log-determinant (see \cref{ch:training}) and inverse square root (see \cref{ch:sampling}) of the Hessian. Our contributions allow us to forgo the typical crude approximations of the Hessian by performing the full Laplace approximation without explicitly storing the Hessian matrix.}
\label{fig:overview}
\end{figure}
+% \begin{marginfigure}
+% \centering
+% \resizebox{\linewidth}{!}{\input{figures/overview.tex}}
+% \caption[Overview of the Laplace approximation.]{Overview of the Laplace approximation (LA) for Bayesian neural networks. We highlight our contributions in green, i.e., the computations which we will approximate using only Hessian-vector products.}
+% \label{fig:overview}
+% \end{marginfigure}
% This should be discussed in a bit more detail in a thesis, since there
% are certain objectives discussed in the project plan.
diff --git a/chapters/laplace.tex b/chapters/laplace.tex
@@ -6,7 +6,7 @@
\section[Optimisation in Deep Learning]{Optimisation in Deep Learning}[Optimisation]
\label{sec:optimisation-frequentist}
-Suppose a neural network is a real-valued function \(f: \reals^N \times \reals^D \rightarrow \reals^O\) parameterised in \(\bm\theta\) which maps an input \(\bm x\) to an output \(f(\bm x, \bm\theta) \equiv f_{\bm\theta}(\bm x)\).
+Suppose a neural network is a real-valued function \(f: \reals^N \times \reals^D \rightarrow \reals^O\) parametrised in \(\bm\theta\) which maps an input \(\bm x\) to an output \(f(\bm x, \bm\theta) \equiv f_{\bm\theta}(\bm x)\).
Our goal is to find the parameters \(\bm\theta^*\) which best model the observed data.
From a frequentist perspective, we can define a loss function on the model \(\mathcal{L}(\bm\theta): \reals^D \rightarrow \reals\) such that the optimal parameters \(\bm\theta^*\) minimise this loss function for the given model.
@@ -57,7 +57,7 @@ It doesn't take into account the curvature of the loss function, and so can over
% Natural Gradient Descent is an approximate second-order optimisation method. It has an interpretation as optimizing over a Riemannian manifold using an intrinsic distance metric, which implies the updates are invariant to transformations such as whitening.
% By using the positive semi-definite (PSD) Gauss-Newton matrix to approximate the (possibly negative definite) Hessian, NGD can often work better than exact second-order methods.
-% \textcite{rattray1998natural} introduced the natural gradient as a way to optimize a function \(f\) parameterised by \(\bm\theta\) by following the direction of the steepest descent in the Fisher information metric.
+% \textcite{rattray1998natural} introduced the natural gradient as a way to optimize a function \(f\) parametrised by \(\bm\theta\) by following the direction of the steepest descent in the Fisher information metric.
% It can be interpreted as
% % The algorithm can be seen in \cref{natural-gradient}.
% %
@@ -98,7 +98,8 @@ The moving averages are weighted by the parameters \(\beta_1\) and \(\beta_2\) r
The learning rate is then scaled by the ratio of the first and second moments.
The update rule for Adam is thus given by
\begin{align}
- \bm\theta_{t+1} ={} & \bm\theta_t - \eta_t \bm m_t \odot \frac{1}{\sqrt{\bm v_t} + \varepsilon},
+ % \bm\theta_{t+1} ={} & \bm\theta_t - \eta_t \bm m_t \odot \frac{1}{\sqrt{\bm v_t} + \varepsilon},
+ \bm\theta_{t+1} ={} & \bm\theta_t - \eta_t \bm m_t \odot \left(\sqrt{\bm v_t} + \varepsilon\right)^{-1},
\\ \bm m_t ={}& (1 - \beta_1^t) \left[ \beta_1 \bm m_{t-1} + (1 - \beta_1) \nabla_{\bm\theta} \mathcal{L}(\bm\theta_t) \right],
\\ \bm v_t ={}& (1 - \beta_2^t) \left[ \beta_2 \bm v_{t-1} + (1 - \beta_2) \nabla_{\bm\theta} \mathcal{L}(\bm\theta_t) \odot \nabla_{\bm\theta} \mathcal{L}(\bm\theta_t) \right],
\end{align}
@@ -151,7 +152,7 @@ However, in some cases, it is useful to have a closed-form expression for the po
There are several methods for approximating the posterior distribution, which we will discuss in the following sections.
Some, like Markov chain Monte Carlo (MCMC)~\cite{gelman1995bayesian}, are exact methods, which can be used to sample from the posterior distribution directly (though they are often computationally expensive).
However, they cannot be used to compute the posterior distribution itself directly.
-Others, like variational inference (VI)~\cite{jordan1999introduction,wainwright2008graphical}, are methods which attempt to find a tractable approximation to the posterior distribution by parameterising a family of approximate posteriors and optimising the parameters to minimise the KL divergence between the approximate posterior and the true posterior.
+Others, like variational inference (VI)~\cite{jordan1999introduction,wainwright2008graphical}, are methods which attempt to find a tractable approximation to the posterior distribution by parametrising a family of approximate posteriors and optimising the parameters to minimise the KL divergence between the approximate posterior and the true posterior.
The performance of these methods depends on the variational family used to approximate the posterior and the choice of the variational parameters.
One common choice for selecting the variational family is to use a Gaussian distribution with diagonal covariance matrix, which is known as mean-field VI.\sidenote{The mean-field assumption refers to the assumption that the parameters are independent of each other which leads to a diagonal covariance matrix.}
Many variational methods suffer from the difficulty of their optimisation problem, since they often attempt to train variance parameters, which are difficult to optimise~\cite{skafte2019reliable}.
@@ -178,7 +179,7 @@ Normalising it by the normalisation constant \(p(\bm y)\) gives the normalised p
\end{align}
The Laplace approximation can simply be trained as an MAP solution, where the Hessian is computed to obtain the posterior precision at inference time.
This method is known as post-hoc Laplace.
-However, it is also possible to compute the posterior precision during training, which is known as online Laplace.
+However, it is also possible to compute the posterior precision during training.
This can be done either to sample from the neural network at each iteration of training and evaluating the Hessian at each sample~\cite{miani2022laplacian} or to estimate the log-marginal likelihood~\cite{daxberger2021laplace,immer2021scalable}, enabling training of hyperparameters and improved model selection.
We approach the latter, as well as post-hoc Laplace, in this work.
@@ -192,7 +193,7 @@ In the next sections, we will go into the computation of the Hessian for the neg
The Hessian is a matrix of second-order partial derivatives of a scalar function.
From now on, we will use the notation \(\nabla^2_{\bm\theta} f\) to represent the Hessian matrix of \(f\), where each element is given by \(\left(\nabla^2_{\bm\theta} f\right)_{i, j} = \frac{\partial^2 f}{\partial\theta_i \partial\theta_j}\).
-Suppose we have a function \(\mathcal{L}: \reals^D \rightarrow \reals\) parameterised by \(\bm \theta \in \reals^D\).
+Suppose we have a function \(\mathcal{L}: \reals^D \rightarrow \reals\) parametrised by \(\bm \theta \in \reals^D\).
Then the Hessian can be interpreted as the Jacobian matrix of the gradient of the function, as per \(J_{\bm \theta}(\nabla_{\bm\theta} \mathcal{L})\).
We are interested in the Hessian of the loss function \(\mathcal{L}(\bm\theta)\) with respect to the parameters \(\bm\theta\).
@@ -331,22 +332,22 @@ The rank of the sum of these matrices thus depends on the eigenvectors of each \
If we add \(\bm J_n\T \bm H_n \bm J_n\) into the sum \(\sum_{i=1}^{n-1} \bm J_i\T \bm H_i \bm J_i\), then, if any of the eigenvectors of \(\bm J_n\T \bm H_n \bm J_n\) are not contained in the span of the eigenvectors of the sum, then the rank of the sum will increase.
This is because the rank of a matrix is the number of linearly independent columns.
Thus, non-rigorously, we have that the ``more similar'' the eigenvectors (and assuming that the eigenvectors have the same ordering between the two matrices) are between terms in the sum, the closer the largest eigenvalue will be to the sum of the largest eigenvalues of each term.\sidenote{By ``same ordering'' we mean that they are eigenvectors with respect to the same relative eigenvalue.}
-This means that the rank of the sum of the GGN matrices can increase as we add more data points, and will not be greater than \(N \cdot O\).
+Practically speaking, this means that the rank of the sum of the GGN matrices can increase as we add more data points, and will not be greater than \(N \cdot O\).
Furthermore, as we add more data points, the eigenvalues of the sum of the GGN matrices will increase.
Specifically, the largest eigenvalue of the sum of the GGN matrices will be the less than or equal to the sum of the largest eigenvalues of the GGN matrices.
Additionally, the non-zero spectrum of the sum of the GGN matrices will itself get steeper as we add more data points.
-If the \(K\) non-zero eigenvectors are identical across the GGN matrices, then this spectrum of the sum of the GGN matrices will just be the sum of the spectrum of each GGN matrix and should thus not get steeper.
+If the \(O\) non-zero eigenvectors are identical across the GGN matrices, then this non-zero spectrum of the sum of the GGN matrices will just be the sum of the spectrum of each GGN matrix and should thus not get steeper.\sidenote{If the GGN matrices \(\bm J\T \bm H \bm J\) have an identical spectrum
+ \begin{align*}
+ \{ \underbrace{s_1, s_2, \ldots, s_O}_{O}, \underbrace{0, \ldots, 0}_{D - O} \},
+ \end{align*}
+ then the sum of GGN matrices as per \(\sum^N \bm J\T \bm H \bm J = N \bm J\T \bm H \bm J\) has the spectrum
+ \begin{align*}
+ \{ \underbrace{N s_1, \ldots, N s_O}_{O}, \underbrace{0, \ldots, 0}_{D - O} \}.
+ \end{align*}
+}
However, if these eigenvectors are not identical, then as the largest eigenvalue of the sum of the GGN increases and the number of non-zero eigenvalues of the sum of the GGN increases, the non-zero spectrum of the sum of the GGN matrices will get steeper.
For these reasons, as we add more data points, the posterior precision will become more ill-conditioned.\sidenote{A matrix is considered ill-conditioned when the ratio of its largest eigenvalue to its lowest eigenvalue, its \emph{condition number} (\(\kappa \coloneqq \lambda_{\mathrm{max}} / \lambda_{\mathrm{min}}\)), is very large.}
-ADD ANY OF THIS?
-\begin{align*}
- \lambda\left( J\T J + \alpha I \right) ={} & \{ \underbrace{s_1, s_2, \ldots, s_O}_{O}, \underbrace{\alpha, \ldots, \alpha}_{D - O} \}
- \\ \lambda\left( N J\T J + \alpha I \right) ={}& \{ \underbrace{N s_1, \ldots, N s_O}_{O}, \underbrace{\alpha, \ldots, \alpha}_{D - O} \}
- \\ \lambda\left( \frac{N}{B} \textstyle{\sum^B_i} J_i\T J_i + \alpha I \right) ={}& \{ \underbrace{s_1, \ldots, s_K}_{K \leq B \cdot O}, \underbrace{\alpha, \ldots, \alpha}_{D - K} \}
- % \Span_\lambda\left( J\T J + \alpha \identity \right)
-\end{align*}
-
\section{Limitations}
One limitation of the Laplace approximation is that it can only approximate one mode of the posterior distribution (the MAP).
diff --git a/chapters/sampling.tex b/chapters/sampling.tex
@@ -4,7 +4,7 @@
\section{Motivation}
-In many methods, such as the Laplace approximation (\cref{sec:laplace}), we sample from a multivariate normal distribution parameterised by a mean vector \(\bm \mu\) and a covariance matrix \(\bm \Sigma\).
+In many methods, such as the Laplace approximation (\cref{sec:laplace}), we sample from a multivariate normal distribution parametrised by a mean vector \(\bm \mu\) and a covariance matrix \(\bm \Sigma\).
To sample from a Gaussian distribution, you can apply the reparametrisation trick to compute a vector of samples as
\begin{align}\label{eq:sample-reparam}
\bm \varepsilon ={} & \bm \mu + \bm \Sigma^{1/2} \bm \varepsilon_0,
@@ -37,7 +37,8 @@ However, these methods are tricky to tune and tend to converge very slowly for h
As such, we do not use these methods for sampling from the Laplace approximation, and instead sample using the reparametrisation trick.
For this, we need a method to compute the inverse square root using only matrix-vector products.
-\section[Contour Integral Quadrature]{Contour Integral Quadrature}[Contour Integral Quad]
+\section{Contour Integral Quadrature}[Contour Integral Quad]
+\label{sec:ciq}
\textcite{pleiss2020fast} propose one such method.
Contour integral quadrature (CIQ) attempts to solve the problem of computing the inverse square root of a matrix, up to a rotation, by approximating the expression \(f(\bm K) \bm \varepsilon_0 = \bm K^{-1/2} \bm \varepsilon_0\) where \(\bm K\) is a positive semi-definite matrix using Cauchy's integral formula.
@@ -95,7 +96,7 @@ Cauchy's integral formula is a central theorem in complex analysis which states
\begin{align}\label{eq:cauchy-integral-formula}
f(\bm K) ={} & \frac{1}{2 \pi i} \oint_\Gamma f(\tau) \left(\tau \identity - \bm K\right)^{-1}\,d\tau,
\end{align}
-where \(\tau\) is a complex variable and \(\oint_\Gamma\) is the contour integral along the contour \(\Gamma\).
+where \(\oint_\Gamma\) is the contour integral along the contour \(\Gamma\) parametrised in the complex variable \(\tau\).
Since, in our case, the eigenvalues of \(\bm K\) will all be real-valued, a circle centred on the real axis which encloses the minimum and maximum eigenvalues of \(\bm K\) will suffice.\marginnote{If a real-valued matrix is symmetric, its eigenvalues are real. In our case, \(\bm K\) will be the precision matrix obtained from the Laplace approximation.}
To estimate these values, we can use the Lanczos algorithm, a Krylov subspace method which adapts the power iteration method, to compute a lower bound on the maximum eigenvalue and an upper bound on the minimum eigenvalue of \(\bm K\).\sidenote{Krylov subspace methods are a class of iterative methods which estimate \(f(\bm K) \bm v\) for some matrix \(\bm K\) and vector \(\bm v\) by using a Krylov subspace. The Krylov subspace is a subspace spanned by the set of images of \(\bm v\) under the first \(j\) powers of \(\bm K\), given by \(\mathcal{K}_j( \bm K, \bm v) = \mathrm{span}\left\{ \bm v, \bm K \bm v, \bm K^2 \bm v, \ldots, \bm K^{j-1} \bm v \right\}\).}
diff --git a/chapters/training.tex b/chapters/training.tex
@@ -5,6 +5,8 @@
\section{Maximising the Evidence}
\label{sec:maximising-the-evidence}
+% In Online Laplace, we seek to maximise the evidence, which is the marginal likelihood of the data given the model parameters.
+
As seen in \cref{eq:bayes-theorem}, the posterior probability for a given model is equal to
\begin{align}
p(\bm \theta \given \bm y) = \frac{p(\bm y \given \bm \theta)\,p(\bm \theta)}{p(\bm y)},
@@ -19,7 +21,7 @@ For this reason, we typically approximate the posterior by some tractable distri
Since we are integrating over the posterior, maximising the marginal likelihood will seek minima that are flat, since the posterior will be very broad in these regions.
This is a desirable property, since it means that the model will be less likely to overfit the data, both from a theoretical standpoint~\cite{hochreiter1997flat} and in practice~\cite{keskar2016large,jiang2019fantastic,maddox2019simple}.
-Furthermore, \textcite{fong2020marginal} show that maximising the marginal (and consequently the log-marginal) is equivalent to performing k-fold cross validation for all values \(k = 1, \ldots, \infty\) and choosing the model with the highest average posterior probability (across each of \(k\) folds and across all values of \(k\)). TODO: FIX THIS TO BE MORE ACCURATE.
+\textcite{fong2020marginal} show that maximising the marginal (and consequently the log-marginal) is equivalent to performing leave-p-out cross validation for all values \(p = 1, \ldots, \infty\) and choosing the model with the highest average posterior probability (across each of \(p\) folds and across all values of \(p\)).
This result shows that the marginal likelihood is a good proxy for the true cross-validation performance of a model.
Because of this, it can be used for model selection, since it will choose the model that performs best on average across all permutations of the data.
@@ -52,7 +54,7 @@ For the normal likelihood and normal prior, we have
p(\bm y \given \bm \theta) ={} & \normal(\bm y \given f_{\bm\theta}(\bm x), \rho^{-1} \identity) = \prod_{i=1}^N \normal(y_i \given f_{\bm\theta}(\bm x_i), \rho^{-1} \identity),
\\ \log p(\bm y \given \bm \theta) ={}& \sum_{i=1}^N \log \normal(y_i \given f_{\bm\theta}(\bm x_i), \rho^{-1} \identity) \label{eq:normal-log-likelihood}
\\ ={}& \sum_{i=1}^N \left( - \frac{O}{2} \log(2 \pi) - \frac{1}{2} \log \det (\rho^{-1} \identity) - \frac{1}{2} \rho \norm{y_i - f_{\bm\theta}(\bm x_i)}^2\right)\nonumber
- \\={}& - \frac{N O}{2} \log(2 \pi) + \frac{N O}{2} \log \rho - \frac{1}{2} \rho \sum_{i=1}^N \norm{y_i - f_{\bm \theta}(\bm x_i)}^2\nonumber
+ \\={}& - \frac{N O}{2} \log(2 \pi) + \frac{N O}{2} \log \rho - \frac{1}{2} \rho \sum_{i=1}^N \norm{y_i - f_{\bm \theta}(\bm x_i)}^2, \nonumber
\end{align}
and
\begin{align}\label{eq:normal-prior}
diff --git a/figures/overview.tex b/figures/overview.tex
@@ -1,26 +1,36 @@
\begin{tikzpicture}
[
- % font=\sffamily,
+ font=\sffamily,
box/.style = {draw, rounded corners, minimum width=22mm, minimum height=5mm, align=center},
]
+ % \definecolor{color1}{rgb}{0.86, 0.82, 1.0}
+ % \definecolor{color2}{rgb}{0.97, 0.96, 1.0}
+ \definecolor{color1}{rgb}{0.74, 0.83, 0.9}
+ \definecolor{color2}{rgb}{0.94, 0.97, 1.0}
\definecolor{kaogreen}{HTML}{91C889}
- \node (nn) [box] {Neural Network};
+ \node (nn) [box,fill=color1] {Neural Network};
\node (max) [below=of nn,yshift=0.5cm] {\(\argmax_{\bm \theta}\)};
- \node (posterior) [box,minimum width=3.8cm,minimum height=2cm,below=of max,xshift=-2cm,yshift=0.5cm] {\(\log p(\bm \theta \given \bm y)\)};
- \node (marginal) [box,minimum width=3.8cm,minimum height=2cm,below=of max,xshift=2cm,yshift=0.5cm] {\(\overbrace{\log p(\bm \theta \given \bm y) \quad \text{\colorbox{kaogreen}{$\log \det \bm \Lambda$}}}^{\text{\normalsize \(\log p(\bm y)\)}}\)};
+ \node (posterior) [box,fill=color2,minimum width=3.8cm,minimum height=2cm,below=of max,xshift=-2cm,yshift=0.5cm] {\(\log p(\bm \theta \given \bm y)\)};
+ \node (marginal) [box,fill=color2,minimum width=3.8cm,minimum height=2cm,below=of max,xshift=2cm,yshift=0.5cm] {\(\overbrace{\log p(\bm \theta \given \bm y) \quad \text{\colorbox{kaogreen}{$\log \det \bm \Lambda$}}}^{\text{\normalsize \(\log p(\bm y)\)}}\)};
\draw[dashed] (nn) to (max);
\draw[dashed,->] (max) to [bend right] (posterior);
\draw[dashed,->] (max) to [bend left] (marginal);
- \node (training) [box,minimum width=8.05cm,minimum height=3.2cm,fit={(max)(posterior)(marginal)}] {};
- \node [above=0cm and 0cm of training.north west,anchor=south west] {\textsc{Training}};
- \node [above=0cm and 0cm of posterior.north west,anchor=south west] {\textsc{post-hoc}};
- \node [above=0cm and 0cm of marginal.north east,anchor=south east] {\textsc{online}};
+ \begin{scope}[on background layer]
+ \node (training) [box,fill=color1,minimum width=8.05cm,minimum height=3.2cm,fit={(max)(posterior)(marginal)}] {};
+ \end{scope}
+ \node [above=0cm and 0cm of training.north west,anchor=south west] {Training};
+ \node [above=0cm and 0cm of posterior.north west,anchor=south west] {{Post-Hoc}};
+ \node [above=0cm and 0cm of marginal.north east,anchor=south east] {Online};
+ % \node [above=0cm and 0cm of training.north east,anchor=south east] {\cref{ch:training}};
%
\node (map) [below=of training,yshift=0.7cm] {\(\bm \theta_{\textsc{map}}\)};
\draw[dashed] (training) to (map);
- \node (sample) [below=of map] {Sample};
+ \node (sample) [below=of map,yshift=0.5cm] {Sample};
\node (samplingeq) [below=0cm and 0cm of sample] {\(\begin{aligned}\bm \varepsilon \sim{}& \normal(\bm \theta_{\textsc{map}}, \bm \Lambda^{-1}) \\ ={}& \bm \theta_{\textsc{map}} + \text{\colorbox{kaogreen}{$\bm \Lambda^{-1/2} \bm \varepsilon_0$}}\end{aligned}\)};
- \node (inference) [box,minimum width=8.05cm,minimum height=3.2cm,fit={(sample)(samplingeq)}] {};
- \node [above=0cm and 0cm of inference.north west,anchor=south west] {\textsc{Inference}};
+ \begin{scope}[on background layer]
+ \node (inference) [box,fill=color1,minimum width=5.05cm,fit={(sample)(samplingeq)}] {}; % minimum width=8.05cm,minimum height=3.2cm,
+ \end{scope}
+ \node [above=0cm and 0cm of inference.north west,anchor=south west] {Inference};
\draw[dashed,->] (map) to (inference);
+ % \node [above=0cm and 0cm of inference.north east,anchor=south east] {\cref{ch:sampling}};
\end{tikzpicture}
diff --git a/figures/sin_posterior_predictive.pdf b/figures/sin_posterior_predictive.pdf
Binary files differ.
diff --git a/figures/sin_posterior_predictive_joint.pdf b/figures/sin_posterior_predictive_joint.pdf
Binary files differ.
diff --git a/figures/sin_posterior_predictive_samples.pdf b/figures/sin_posterior_predictive_samples.pdf
Binary files differ.
diff --git a/main.bib b/main.bib
@@ -29,6 +29,16 @@
year = {2017}
}
+@article{cavalcante2016computational,
+ title = {Computational Intelligence and Financial Markets: A Survey and Future Directions},
+ author = {Cavalcante, Rodolfo C and Brasileiro, Rodrigo C and Souza, Victor LF and Nobrega, Jarley P and Oliveira, Adriano LI},
+ journal = {Expert Systems with Applications},
+ volume = {55},
+ pages = {194--211},
+ year = {2016},
+ publisher = {Elsevier}
+}
+
@inproceedings{damianou2013deep,
title = {Deep {G}aussian Processes},
author = {Damianou, Andreas and Lawrence, Neil D},
@@ -45,11 +55,11 @@
year = {2021}
}
-@article{denker1990transforming,
- title = {Transforming Neural-Net Output Levels to Probability Distributions},
- author = {Denker, John and LeCun, Yann},
- journal = nips,
- year = {1990}
+@inproceedings{denker1990transforming,
+ title = {Transforming Neural-Net Output Levels to Probability Distributions},
+ author = {Denker, John and LeCun, Yann},
+ booktitle = nips,
+ year = {1990}
}
@article{eschenhagen2021mixtures,
@@ -80,6 +90,13 @@
publisher = {Oxford University Press}
}
+@inproceedings{gal2016dropout,
+ title = {Dropout as a {B}ayesian Approximation: Representing Model Uncertainty in Deep Learning},
+ author = {Gal, Yarin and Ghahramani, Zoubin},
+ booktitle = icml,
+ year = {2016}
+}
+
@inproceedings{gardner2018gpytorch,
title = {{GP}yTorch: Blackbox Matrix-Matrix Gaussian Process Inference With {GPU} Acceleration},
author = {Gardner, Jacob and Pleiss, Geoff and Weinberger, Kilian Q and Bindel, David and Wilson, Andrew G},
@@ -95,7 +112,7 @@
}
@article{giordano2018covariances,
- title = {Covariances, Robustness and Variational Bayes},
+ title = {Covariances, Robustness and Variational {B}ayes},
author = {Giordano, Ryan and Broderick, Tamara and Jordan, Michael I},
journal = jmlr,
volume = {19},
@@ -201,6 +218,27 @@
publisher = {Springer}
}
+@article{jospin2022hands,
+ title = {Hands-on {B}ayesian Neural Networks---A Tutorial for Deep Learning Users},
+ author = {Jospin, Laurent Valentin and Laga, Hamid and Boussaid, Farid and Buntine, Wray and Bennamoun, Mohammed},
+ journal = {IEEE Computational Intelligence Magazine},
+ volume = {17},
+ number = {2},
+ pages = {29--48},
+ year = {2022},
+ publisher = {IEEE}
+}
+
+@article{ker2017deep,
+ title = {Deep Learning Applications in Medical Image Analysis},
+ author = {Ker, Justin and Wang, Lipo and Rao, Jai and Lim, Tchoyoson},
+ journal = {IEEE Access},
+ volume = {6},
+ pages = {9375--9389},
+ year = {2017},
+ publisher = {IEEE}
+}
+
@article{keskar2016large,
title = {On Large-Batch Training for Deep Learning: Generalization Gap and Sharp Minima},
author = {Keskar, Nitish Shirish and Mudigere, Dheevatsa and Nocedal, Jorge and Smelyanskiy, Mikhail and Tang, Ping Tak Peter},
@@ -222,6 +260,13 @@
year = {2019}
}
+@inproceedings{lakshminarayanan2017simple,
+ title = {Simple and Scalable Predictive Uncertainty Estimation Using Deep Ensembles},
+ author = {Lakshminarayanan, Balaji and Pritzel, Alexander and Blundell, Charles},
+ booktitle = nips,
+ year = {2017}
+}
+
@article{laplace1774memoire,
title = {M\'emoire Sur La Probabilit\'e Des Causes Par Les \'Ev\'enements},
author = {Laplace, Pierre-Simon},
@@ -231,11 +276,11 @@
year = {1774}
}
-@article{lecun1989optimal,
- title = {Optimal Brain Damage},
- author = {LeCun, Yann and Denker, John and Solla, Sara},
- journal = nips,
- year = {1989}
+@inproceedings{lecun1989optimal,
+ title = {Optimal Brain Damage},
+ author = {LeCun, Yann and Denker, John and Solla, Sara},
+ booktitle = nips,
+ year = {1989}
}
@book{mackay2003information,
@@ -245,12 +290,11 @@
publisher = {Cambridge University Press}
}
-@article{maddox2019simple,
- title = {A Simple Baseline for {B}ayesian Uncertainty in Deep Learning},
- author = {Maddox, Wesley J and Izmailov, Pavel and Garipov, Timur and Vetrov, Dmitry P and Wilson, Andrew Gordon},
- journal = nips,
- volume = {32},
- year = {2019}
+@inproceedings{maddox2019simple,
+ title = {A Simple Baseline for {B}ayesian Uncertainty in Deep Learning},
+ author = {Maddox, Wesley J and Izmailov, Pavel and Garipov, Timur and Vetrov, Dmitry P and Wilson, Andrew Gordon},
+ booktitle = nips,
+ year = {2019}
}
@inproceedings{martens2015optimizing,
@@ -292,7 +336,7 @@
}
@inproceedings{pleiss2020fast,
- title = {Fast Matrix Square Roots With Applications to {G}aussian Processes and Bayesian Optimization},
+ title = {Fast Matrix Square Roots With Applications to {G}aussian Processes and {B}ayesian Optimization},
author = {Pleiss, Geoff and Jankowiak, Martin and Eriksson, David and Damle, Anil and Gardner, Jacob},
booktitle = nips,
year = {2020}
@@ -309,6 +353,14 @@
publisher = {Elsevier}
}
+@inproceedings{rao2018deep,
+ title = {Deep Learning for Self-Driving Cars: Chances and Challenges},
+ author = {Rao, Qing and Frtunikj, Jelena},
+ booktitle = {Proceedings of the 1st International Workshop on Software Engineering for AI in Autonomous Systems},
+ pages = {35--38},
+ year = {2018}
+}
+
@article{rattray1998natural,
title = {Natural Gradient Descent for On-Line Learning},
author = {Rattray, Magnus and Saad, David and Amari, Shun-ichi},
@@ -320,12 +372,12 @@
publisher = {American Physical Society}
}
-@article{ritter2018online,
- title = {Online Structured {L}aplace Approximations for Overcoming Catastrophic Forgetting},
- author = {Ritter, Hippolyt and Botev, Aleksandar and Barber, David},
- journal = nips,
- volume = {31},
- year = {2018}
+@inproceedings{ritter2018online,
+ title = {Online Structured {L}aplace Approximations for Overcoming Catastrophic Forgetting},
+ author = {Ritter, Hippolyt and Botev, Aleksandar and Barber, David},
+ booktitle = nips,
+ volume = {31},
+ year = {2018}
}
@inproceedings{ritter2018scalable,
diff --git a/main.tex b/main.tex
@@ -56,7 +56,7 @@
\usepackage{censor}
\usepackage{tikz}
-\usetikzlibrary{fit,patterns,shapes.arrows,quotes,positioning}
+\usetikzlibrary{fit,patterns,shapes.arrows,quotes,positioning,backgrounds}
\usepackage{pgfplots}
\pgfplotsset{compat=newest}
\DeclareUnicodeCharacter{2212}{−}
