commit 0b1eb6b8410b08e8d413f1d5da92cac3b7fd9c4d
parent 8dc33277c1db430818a7204578b09624ba4f059f
Author: Silas Brack <s174433@student.dtu.dk>
Date: Sat, 21 Jan 2023 15:10:35 +0100
More writing.
Diffstat:
3 files changed, 113 insertions(+), 2 deletions(-)
diff --git a/chapters/theory.tex b/chapters/theory.tex
@@ -94,9 +94,21 @@ Instead, it is often approximated by the diagonal of the Hessian.
For a recent, detailed discussion of the natural gradient, see \sidetextcite{martens2020new}.
\sidecite{wu2019logan}
+However, these optimisation methods all identify a single set of parameters as the optimal solution.
+In the next section, we will discuss how to learn a distribution over parameters instead.
+
\section{Bayesian Deep Learning}
\label{sec:bayesian-deep-learning}
+Bayesian deep learning is a framework for deep learning that allows for modelling the parameters as random variables instead of as single optima in the loss landscape.
+It built from Bayes' theorem, which allows for uncertainty quantification by
+\begin{align}\label{eq:bayes-theorem}
+ p(\bm \theta \given \bm x) = \frac{p(\bm x \given \bm \theta)\,p(\bm \theta)}{p(\bm x)}
+\end{align}
+Instead of learning a single set of parameters, Bayesian deep learning learns a distribution over parameters.
+
+However, the posterior distribution is typically intractable, so approximations to the posterior are used instead.
+
\section{Laplace Approximation}
\label{sec:laplace}
@@ -129,6 +141,10 @@ GGN APPROXIMATION FOR POSITIVE DEFINITENESS
EIGENVALUES?
minimum eigenvalue determined by the perturbation \(\alpha\) which guarantees positive definiteness
+As we have seen, the Laplace approximation is obtained from a second-order Taylor expansion of the log-posterior around the MAP solution, which corresponds to approximating the posterior with a Gaussian distribution with a covariance matrix given by the inverse of the Hessian of the negative log-posterior.
+However, this approximation relies on the positive definiteness of the Hessian, which is not guaranteed for non-convex losses.
+We will now show that the GGN approximation can be used as a positive definite Hessian approximate the posterior even when the Hessian itself is not positive definite.
+
\section{The Hessian}
The Hessian is a matrix of second-order partial derivatives of a scalar function.
diff --git a/chapters/training.tex b/chapters/training.tex
@@ -8,15 +8,108 @@ According to Bayes' rule, the posterior probability for a given model is equal t
\begin{align}
p(\bm \theta \given \bm x) = \frac{p(\bm x \given \bm \theta) p(\bm \theta)}{p(\bm x)},
\end{align}
-where \(p(\bm x)\) is known as the evidence.\sidenote{It's also known as the \emph{marginal likelihood}, since it's obtained by marginalising out \(\bm \theta\) from the likelihood, as per \(p(\bm x) = \int_{\bm \theta} p(\bm x \given \bm \theta) p(\bm \theta)\,d\bm \theta\).}
+where \(p(\bm x)\) is known as the evidence. It's also known as the \emph{marginal likelihood}, since it's obtained by marginalising out \(\bm \theta\) from the likelihood, as per
+\begin{align}
+ p(\bm x) ={} & \int_{\bm \theta} p(\bm \theta \given \bm x)\,d\bm \theta.
+\end{align}
+This term is typically intractable, since it involves integrating over the entire parameter space.
+However, certain
+For the Laplace approximation, we approximate the posterior as \(p(\bm \theta \given \bm x) \approx \normal(\bm \theta \given \bm \theta_{\textsc{map}}, \bm \Lambda^{-1})\), where \(\bm \Lambda = -\nabla^2_{\bm\theta} \mathcal L(\bm \theta_{\textsc{map}})\) is the Hessian of the negative log-posterior at the maximum a posteriori (MAP) estimate \(\bm \theta_{\textsc{map}}\).
+We thus obtain the approximate Laplace posterior
+\begin{align}\label{eq:laplace-posterior}
+ \log p(\bm \theta \given \bm x) \stackrel{\textsc{la}}{\approx}{} & \log p(\bm \theta_{\textsc{map}} \given \bm x) + \nabla_{\bm \theta} \log p(\bm \theta_{\textsc{map}} \given \bm x) \cdot (\bm \theta - \bm \theta_{\textsc{map}})
+ \\ & + \frac{1}{2} (\bm \theta - \bm \theta_{\textsc{map}}) \nabla^2_{\bm \theta} \log p(\bm \theta_{\textsc{map}} \given \bm x) (\bm \theta - \bm \theta_{\textsc{map}})\nonumber
+ \\={}& \log p(\bm \theta_{\textsc{map}} \given \bm x) - \frac{1}{2} (\bm \theta - \bm \theta_{\textsc{map}}) \bm \Lambda (\bm \theta - \bm \theta_{\textsc{map}})\nonumber
+\end{align}
+The approximate Laplace marginal is then computed as per
+\begin{align}\label{eq:laplace-marginal}
+ p(\bm x) ={} \stackrel{\textsc{la}}{\approx}{} & \int_{\bm \theta} p(\bm \theta_{\textsc{map}} \given \bm x) \exp\left( -\frac{1}{2} (\bm \theta - \bm \theta_{\textsc{map}}) \bm \Lambda (\bm \theta - \bm \theta_{\textsc{map}}) \right)\,d \bm \theta
+ \\ ={}& p(\bm \theta_{\textsc{map}} \given \bm x) \int_{\bm \theta} \exp\left( -\frac{1}{2} (\bm \theta - \bm \theta_{\textsc{map}}) \bm \Lambda (\bm \theta - \bm \theta_{\textsc{map}})\right)\,d \bm \theta\nonumber
+ \\ ={}& p(\bm \theta_{\textsc{map}} \given \bm x)\,(2 \pi)^{D / 2}\,(\det \bm \Lambda)^{-1/2}.\nonumber
+\end{align}
+Taking the log of the approximate Laplace marginal, we obtain
+\begin{align}\label{eq:laplace-log-marginal}
+ \log p(\bm x) \stackrel{\textsc{la}}{\approx}{} & \log p(\bm \theta_{\textsc{map}} \given \bm x) + \frac{D}{2} \log (2 \pi) - \frac{1}{2} \log \det \bm \Lambda
+ \\ ={}& \log p(\bm x \given \bm \theta_{\textsc{map}}) + \log p(\bm \theta_{\textsc{map}}) + \frac{D}{2} \log (2 \pi) - \frac{1}{2} \log \det \bm \Lambda. \nonumber
+\end{align}
+For the normal likelihood and normal prior, we have
+\begin{align}\label{eq:normal-likelihood}
+ p(\bm x \given \bm \theta) ={} & \normal(\bm y \given f_{\bm\theta}(\bm x), \rho^{-1} \identity) = \prod_{i=1}^N \normal(y_i \given f_{\bm\theta}(\bm x_i), \rho^{-1} \identity)
+ \\ \log p(\bm x \given \bm \theta) ={}& \sum_{i=1}^N \log \normal(y_i \given f_{\bm\theta}(\bm x_i), \rho^{-1} \identity)
+ \\ ={}& \sum_{i=1}^N \left( - \frac{O}{2} \log(2 \pi) - \frac{1}{2} \log \det (\rho^{-1} \identity) - \frac{1}{2} \rho \norm{\bm y_i - f_{\bm\theta}(\bm x_i)}^2\right)\nonumber
+ \\={}& - \frac{N O}{2} \log(2 \pi) + \frac{N O}{2} \log \rho - \frac{1}{2} \rho \sum_{i=1}^N \norm{\bm y_i - f_{\bm \theta}(\bm x_i)}^2\nonumber
+\end{align}
+and
+\begin{align}\label{eq:normal-prior}
+ p(\bm \theta) ={} & \normal(\bm \theta \given \bm 0, \alpha^{-1} \identity)
+ \\ \log p(\bm \theta) ={}& - \frac{D}{2} \log(2 \pi) + \frac{D}{2} \log \alpha - \frac{1}{2} \alpha \norm{\bm \theta}^2,
+\end{align}
+where \(\alpha\) is the precision of the prior and \(\rho\) is the precision of the likelihood.
+
+Combining \cref{eq:laplace-log-marginal}, \cref{eq:normal-likelihood}, and \cref{eq:normal-prior}, we obtain
+\begin{align}\label{eq:expanded-log-marginal}
+ \log p(\bm x) \stackrel{\textsc{la}}{\approx}{} & - \frac{N O}{2} \log(2 \pi) + \frac{N O}{2} \log \rho - \frac{1}{2} \rho \sum_{i=1}^N \norm{\bm y_i - f_{\bm \theta}(\bm x_i)}^2
+ \\ & - \frac{D}{2} \log(2 \pi) + \frac{D}{2} \log \alpha - \frac{1}{2} \alpha \norm{\bm \theta}^2 \nonumber
+ \\ & + \frac{D}{2} \log (2 \pi) - \frac{1}{2} \log \det \bm \Lambda \nonumber
+\end{align}
+
+The \texttt{laplace} Python library provides a function to compute the log-marginal likelihood of the Laplace approximation. The function is called \texttt{log\_marginal\_likelihood} and is given by
+\begin{align}
+ \log p(\bm x) \mathrel{\stackrel{\textsc{la}}{\approx}}{} & \frac{N O}{2} \log \rho - \frac{1}{2} \rho \sum_{i=1}^N \norm{\bm y_i - f_{\bm \theta}(\bm x_i)}^2
+ \\ & - \frac{1}{2} \left( \log \det \bm \Lambda - D \log \alpha + \alpha \norm{\bm \theta}^2 \right). \nonumber
+\end{align}
+As compared to the exact log-marginal from \cref{eq:expanded-log-marginal}, this implementation is obtained by omitting the constant terms which do not depend on \(\bm \theta\), \(\alpha\), or \(\rho\).
+However, in doing so, the authors ignore the terms \(D / 2 \log(2 \pi)\)
+While these terms do not affect the optimisation problem, they are useful for comparing models with different numbers of parameters.
+
+\begin{tabular}{ll}
+ Minimise & Maximise
+ \\\addlinespace 1. \(\rho / 2 \sum_{i=1}^N \norm{\bm y_i - f_{\bm \theta}(\bm x_i)}^2\) & 1. \(N O / 2 \log \rho\)
+ \\ 2. \(1 / 2 \log \det \bm \Lambda\) & 2. \(D \log \alpha\)
+ \\ 3. \(\alpha / 2 \norm{\bm \theta}^2\)
+\end{tabular}
\sidetextcite{fong2020marginal} argue that maximising the marginal (and consequently the log-marginal) is equivalent to performing k-fold cross validation for all values \(k = 1, \ldots, \infty\) and choosing the model with the highest average posterior probability (across each of \(k\) folds and across all values of \(k\)).
\section[The Determinant Lower Bound]{The Determinant Lower Bound}[The Lower Bound]
-\subsection[Hutchinson's Trace Estimator]{Hutchinson's Trace Estimator}[The Trace Estimator]
+We compute an \emph{upper} bound on the log-determinant of the Hessian of the negative log-posterior \(\bm \Lambda \in \reals^{D \times D}\), i.e.,
+\begin{align}
+ \log \det \bm \Lambda \leq{} & B_D(\mu_1, \mu_2, \beta)
+ \\ ={}& \begin{pmatrix} \log \beta & \log t \end{pmatrix}
+ \begin{pmatrix} \beta & t \\ \beta^2 & t^2 \end{pmatrix}^{-1}
+ \begin{pmatrix} \mu_1 \\ \mu_2 \end{pmatrix},
+\end{align}
+where
+\begin{align}\label{eq:mu1-mu2}
+ \mu_1 ={} & \Tr(\bm \Lambda)
+ \\ \mu_2 ={}& \norm{\bm \Lambda}^2_F
+ \\ t ={}& \frac{\beta \mu_1 - \mu_2}{\beta D - \mu_1},
+\end{align}
+where \(\beta\) is an upper bound on the largest eigenvalue of \(\bm \Lambda\).
+Unfortunately, well-known methods such as the Lanczos algorithm compute a \emph{lower} bound on the largest eigenvalue.
+MMM THIS IS NOT THAT CLEAN WITHOUT RANK O:
+Since \(\bm \Lambda\) is a low-rank matrix given by \(\sum_{i=1}^N \bm J_i\T \bm H_i \bm J_i + \alpha \identity\), it has eigenvalues \(\{\lambda_1 + \alpha, \ldots, \lambda_{O \cdot N} + \alpha, \alpha, \ldots, \alpha\}\).
+\begin{align}
+ \log \det \bm \Lambda = \sum_{i=1}^{O} \log(\lambda_i + \alpha) + \sum_{}
+\end{align}
+where \(\lambda_1, \ldots, \lambda_O\) are the eigenvalues of \(\bm \Lambda\).
+Furthermore, the trace of
+\begin{align}
+ \Tr(\bm \Lambda) ={} & \Tr({\textstyle{\sum_{i=1}^N}} \bm J_i\T \bm H_i \bm J_i) = \sum_{i=1}^N \Tr(\bm J_i\T \bm H_i \bm J_i)
+ \\ ={}& \sum_{i=1}^N \Tr(\bm H_i \bm J_i \bm J_i\T)
+\end{align}
+
+Since \(B_D(\mu_1, \mu_2, \beta)\) is an upper bound on the log-determinant of \(\bm \Lambda\), we can use this to compute a lower bound on the log-marginal likelihood, as per \cref{eq:expanded-log-marginal}.
+This is effective because we want to maximise the log-marginal.
+
+WHERE DOES THE BOUND COME FROM? WHICH UNCERTAINTY? THE UPPER BOUND ON THE EIGENVALUE?
+
+We then need to compute the trace of the Hessian matrix, as per \cref{eq:mu1-mu2}.
+
+\subsection[Hutchinson's Trace Estimator]{Hutchinson's Trace Estimator}[The Trace Estimator]
\subsection{Scaling}
diff --git a/macros.tex b/macros.tex
@@ -37,6 +37,8 @@
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
+\DeclareMathOperator{\Tr}{Tr}
+
\DeclareMathOperator{\sn}{sn}
\DeclareMathOperator{\cn}{cn}
\DeclareMathOperator{\dn}{dn}
