commit 4bc2e66b1d4db5e99a63e5bcf53db33d3878a9a1
parent 7344f0fca9dbbd40b47e97004b163ec3ceab9cdc
Author: Silas Brack <s174433@student.dtu.dk>
Date: Thu, 2 Feb 2023 10:05:39 +0100
Added more to appendix, added warnings to results.
Diffstat:
11 files changed, 69 insertions(+), 15 deletions(-)
diff --git a/chapters/appendix.tex b/chapters/appendix.tex
@@ -1,5 +1,6 @@
-\setchapterstyle{lines}
-\labch{appendix}
+\chapter{Algorithms}
+% \setchapterstyle{lines}
+\label{ch:appendix-algorithms}
% \setchapterpreamble[u]{\margintoc}
\begin{algorithm}
@@ -52,3 +53,51 @@
$\bm c_q \gets \bm c^{(q)}_J \norm{\bm b}_2$\;
}
\end{algorithm}
+
+\chapter{Additional Results}
+% \setchapterstyle{lines}
+\label{ch:appendix-results}
+
+\begin{figure*}
+ \centering
+ \begin{subfigure}[b]{0.7\textwidth}
+ \centering
+ \includegraphics{ablation/chisq_55k_0.1_1q_100precondpoints_cg.pdf}
+ \caption{\(Q=1\)}
+ \end{subfigure}
+ \begin{subfigure}[b]{0.7\textwidth}
+ \centering
+ \includegraphics{ablation/chisq_55k_0.1_2q_100precondpoints_cg.pdf}
+ \caption{\(Q=2\)}
+ \end{subfigure}
+
+ \begin{subfigure}[b]{0.7\textwidth}
+ \centering
+ \includegraphics{ablation/chisq_55k_0.1_3q_100precondpoints_cg.pdf}
+ \caption{\(Q=3\)}
+ \end{subfigure}
+ \begin{subfigure}[b]{0.7\textwidth}
+ \centering
+ \includegraphics{ablation/chisq_55k_0.1_4q_100precondpoints_cg.pdf}
+ \caption{\(Q=4\)}
+ \end{subfigure}
+
+ \begin{subfigure}[b]{0.7\textwidth}
+ \centering
+ \includegraphics{ablation/chisq_55k_0.1_5q_100precondpoints_cg.pdf}
+ \caption{\(Q=5\)}
+ \end{subfigure}
+ \begin{subfigure}[b]{0.7\textwidth}
+ \centering
+ \includegraphics{ablation/chisq_55k_0.1_10q_100precondpoints_cg.pdf}
+ \caption{\(Q=10\)}
+ \end{subfigure}
+
+ \begin{subfigure}[b]{0.7\textwidth}
+ \centering
+ \includegraphics{ablation/chisq_55k_0.1_20q_100precondpoints_cg.pdf}
+ \caption{\(Q=20\)}
+ \end{subfigure}
+ \caption[Chi-squared samples plots for ablation on MNIST.]{Chi-squared sample plots for visualising normality of 200 posterior samples from the full Laplace approximation on MNIST for a varying number of quadrature points \(Q\). Samples appear correctly distributed for \(Q \geq 5\), incorrectly distributed for \(Q < 4\), and borderline for \(Q = 4\).}
+ \label{fig:ablation-chisq}
+\end{figure*}
diff --git a/chapters/experiments.tex b/chapters/experiments.tex
@@ -2,6 +2,8 @@
\chapter{Experiments}
\labch{experiments}
+WARNING: THIS SECTION IS BY NO MEANS FINISHED
+
\section[Hessian-Vector Products]{Hessian-Vector Products}[Hessian-Vector Products]
In this project, we propose performing the Laplace approximation by only storing the computational graph for the implicit Hessian-vector product, thereby avoiding the explicit computation and storage of the quadratically-scaling Hessian matrix.
@@ -50,7 +52,7 @@ Notably, however, these implicit Hessian-vector product methods allowed for comp
some analysis
comment on performance of manual products
-comment on weird dip in inverse, possibly something to do with caching or something like that? like the size of your cache
+comment on weird dip in inverse, possibly something to do with caching or something like that? like related to the size of the L2 cache
\begin{marginfigure}%[-4cm]
\centering
@@ -72,8 +74,8 @@ This is in line with the theory, as noted in \cref{sec:practical-ggn} (SHOULD IT
\begin{marginfigure}
\centering
- \includegraphics{mnist_ablation.pdf}
- \caption[Ablation over quadrature points for MNIST.]{Ablation over quadrature points for MNIST. We can see that sampling fails categorically for \(Q < 5\) and succeeds categorically for \(Q \geq 10\).}
+ \includegraphics{ablation/mnist_ablation.pdf}
+ \caption[Ablation over quadrature points for MNIST.]{Ablation over quadrature points for MNIST. We can see that sampling fails categorically for \(Q < 5\) and succeeds categorically for \(Q \geq 10\). FIX THIS PLOT.}
\label{fig:mnist-ablation}
\end{marginfigure}
@@ -83,33 +85,33 @@ We then investigated the number of quadrature points \(Q\) required to determine
-We vary \(q \in \{1, \ldots, 20\}\) to determine the number of quadrature points required for the Laplace approximation.
+We vary \(Q \in \{1, \ldots, 20\}\) to determine the number of quadrature points required for the Laplace approximation.
We then compute the Mahalanobis distance between the samples and the true posterior mean, and compare this to the chi-squared distribution with \(D\) degrees of freedom.
-We then compute the p-value of the Kolmogorov-Smirnov test for each value of \(q\) by comparing the empirical CDF of the Mahalanobis distances to the theoretical \(\chi^2_D\) CDF.
+We then compute the p-value of the Kolmogorov-Smirnov test for each value of \(Q\) by comparing the empirical CDF of the Mahalanobis distances to the theoretical \(\chi^2_D\) CDF.
This p-value is then used to determine whether the null hypothesis that the samples are multivariate normal can be rejected based on some significance threshold.
Sample evaluation is explained in more detail in \cref{sec:sampling-evaluation}.
The results of this experiment are shown in \cref{tab:mnist-ablation,fig:mnist-ablation}.
\begin{margintable}
\centering
- \caption[Ablation over quadrature points for MNIST.]{Sampling time and p-values for Kolmogorov-Smirnov with varying quadrature points \(Q\) on MNIST with prior precision \(\alpha = 0.1\), for 200 posterior samples. Lower p-value indicates a higher probability of rejecting normality. Bold indicates sampling was successful based on visual inspection of quantile--quantile plots. Since, for values of \(Q \geq 5\), samples appear visually to be multivariate normal, we can conclude that we should choose a significance threshold of 0.01 or lower.}
+ \caption[Ablation over quadrature points for MNIST.]{Sampling time and p-values for Kolmogorov-Smirnov with varying quadrature points \(Q\) on MNIST with prior precision \(\alpha = 0.1\), for 200 posterior samples. Lower p-value indicates a higher probability of rejecting normality. Bold indicates sampling was successful based on visual inspection of quantile--quantile plots (see \cref{ch:appendix-results}, \cref{fig:ablation-chisq}). Since, for values of \(Q \geq 5\), samples appear visually to be multivariate normal, we can conclude that we should choose a significance threshold of 0.01 or lower.}
\label{tab:mnist-ablation}
\include{tables/mnist_ablation.tex}
\end{margintable}
% RESULTS
-Based on \cref{fig:mnist-ablation}, sampling appears to be successful (based on a significance threshold of \(0.05\)) for \(q \in \{10, \ldots, 20\}\).
+Based on \cref{fig:mnist-ablation}, sampling appears to be successful (based on a significance threshold of \(0.05\)) for \(Q \in \{10, \ldots, 20\}\).
% DISCUSSION
-However, inspection of CHISQ PLOTS suggests that there is not a significant difference between these values and \(q = 5\).
+However, inspection of CHISQ PLOTS suggests that there is not a significant difference between these values and \(Q = 5\).
In general, significance testing can be sensitive to the number of samples used, and, for large numbers of samples, these tests can excessively reject the null hypothesis.
However, the number of samples is generally fixed as a function of the number of parameters in the model and the amount of memory available.
As such, it is important to select an appropriate significance threshold for the test based on the number of posterior samples that are required.
-For 200 posterior samples, a more appropriate significance threshold of \(0.01\) or lower would reject sampling for \(q \leq 3\), since the cases where sampling completely fails have p-values that are below single floating point precision.
-Furthermore, the plots in CHISQ PLOTS suggest that the performance of the sampling method is effectively equal for \(q \in \{5, \ldots, 20\}\).
+For 200 posterior samples, a more appropriate significance threshold of \(0.01\) or lower would reject sampling for \(Q \leq 3\), since the cases where sampling completely fails have p-values that are below single floating point precision.
+Furthermore, the plots in CHISQ PLOTS suggest that the performance of the sampling method is effectively equal for \(Q \in \{5, \ldots, 20\}\).
-In terms of the time required to compute the samples, the time required to compute the samples is XXXX in \(q\).
+In terms of the time required to compute the samples, the time required to compute the samples is XXXX in \(Q\).
However, in taking as much as 20 to 30 minutes to compute 200 samples of an MNIST model with 15\,000 parameters, the time required to compute the samples is not negligible.
In practice, it is likely that only a small number of samples will be obtained (around 5-15), since for each sample it is necessary to store a parameter vector of size \(D\).
However, for 10 samples, it still takes XXXX minutes.
@@ -125,7 +127,7 @@ The choice of \(\alpha\) is a non-trivial one
\begin{marginfigure}
\centering
\includegraphics{sin.pdf}
- \caption[Laplace predictions for a sine curve.]{Visualisation of the predicted output of multilayer perceptron (MLP) modelling a sine curve.}
+ \caption[Laplace predictions for a sine curve.]{Visualisation of the predicted output of multilayer perceptron (MLP) modelling a sine curve. REPLACE THIS WITH NEW MARGINAL PLOTS}
\label{fig:sin}
\end{marginfigure}
@@ -141,6 +143,6 @@ The choice of \(\alpha\) is a non-trivial one
\begin{marginfigure}
\centering
\includegraphics{eigenvalues_mnist_55k_0.1_old.pdf}
- \caption[Spectrum of the posterior precision on MNIST.]{Hyperexponential distribution of eigenvalues of Laplace posterior precision on MNIST with a convolutional NN with 17k parameters and a 3 by 3 filter.}
+ \caption[Spectrum of the posterior precision on MNIST.]{Hyperexponential distribution of eigenvalues of Laplace posterior precision on MNIST with a convolutional NN with 17k parameters and a 3 by 3 filter. There are two distinct clusters in the spectrum, with the first cluster being over an order of magnitude larger than the second. This first cluster consists of nine eigenvalues, and so we can conclude that the posterior covariance has an effective rank close to nine. FIX THIS PLOT.}
\label{fig:spectrum-mnist}
\end{marginfigure}
diff --git a/figures/ablation/chisq_55k_0.1_10q_100precondpoints_cg.pdf b/figures/ablation/chisq_55k_0.1_10q_100precondpoints_cg.pdf
Binary files differ.
diff --git a/figures/ablation/chisq_55k_0.1_1q_100precondpoints_cg.pdf b/figures/ablation/chisq_55k_0.1_1q_100precondpoints_cg.pdf
Binary files differ.
diff --git a/figures/ablation/chisq_55k_0.1_20q_100precondpoints_cg.pdf b/figures/ablation/chisq_55k_0.1_20q_100precondpoints_cg.pdf
Binary files differ.
diff --git a/figures/ablation/chisq_55k_0.1_2q_100precondpoints_cg.pdf b/figures/ablation/chisq_55k_0.1_2q_100precondpoints_cg.pdf
Binary files differ.
diff --git a/figures/ablation/chisq_55k_0.1_3q_100precondpoints_cg.pdf b/figures/ablation/chisq_55k_0.1_3q_100precondpoints_cg.pdf
Binary files differ.
diff --git a/figures/ablation/chisq_55k_0.1_4q_100precondpoints_cg.pdf b/figures/ablation/chisq_55k_0.1_4q_100precondpoints_cg.pdf
Binary files differ.
diff --git a/figures/ablation/chisq_55k_0.1_5q_100precondpoints_cg.pdf b/figures/ablation/chisq_55k_0.1_5q_100precondpoints_cg.pdf
Binary files differ.
diff --git a/figures/mnist_ablation.pdf b/figures/ablation/mnist_ablation.pdf
Binary files differ.
diff --git a/main.tex b/main.tex
@@ -71,6 +71,9 @@
\tikzstyle{mydashed}=[dash pattern=on 2pt off 2pt]
\newcommand{\tick}[2]{\draw[thick] (#1) ++ (#2:0.1) --++ (#2-180:0.2)} %0.03*\xmax
+\usepackage{caption}
+\usepackage{subcaption}
+
\input{macros.tex}
\graphicspath{{figures/}{./}} % Paths in which to look for images
