commit a238e2050ebc9d87bce5bb4cd696d25666a36233
parent 0e31212b9a128bac2882af690a4b3ca19381828a
Author: Silas Brack <s174433@student.dtu.dk>
Date: Wed, 1 Feb 2023 18:30:12 +0100
First draft, excluding experiments.
Diffstat:
8 files changed, 79 insertions(+), 67 deletions(-)
diff --git a/chapters/conclusion.tex b/chapters/conclusion.tex
@@ -7,45 +7,46 @@
In this project, we aimed to develop a scalable full-rank Laplace approximation for Bayesian neural networks.
In doing so, we developed a novel method for approximately sampling from the full-rank Laplace posterior and methods for preconditioning the Hessian of the negative log posterior, which involved attempting to better understand the spectrum of the Generalised Gauss-Newton Laplace posterior precision.
To test the quality of our sampling procedure, we showed how to evaluate the quality of the samples it produces while only accessing the samples themselves and the desired posterior precision, enabling us and to choose a suitable preconditioner for it.
-This evaluation method showed that, with an appropriate preconditioner, we can produce samples which are Gaussian-distributed with the correct mean and covariance, as desired.
+This evaluation method showed that, with an appropriate preconditioner, we can reliably produce samples which are Gaussian-distributed with the correct mean and covariance, as desired.
As such, we have shown that our approximate sampling method is a valid (albeit slow) method for sampling from the full-rank Laplace posterior.
-Furthermore,
+We also developed a novel method for training the full-rank Laplace approximation by maximising the Laplace-approximate marginal likelihood of the data.
+To compute this evidence term, we developed a method for computing an upper bound on the log determinant of the Laplace posterior precision without storing the posterior precision itself.
+This method allows us to compute a differentiable log-marginal likelihood bound, which we can train with standard gradient descent optimisation methods.
+We showed that this method is a valid method for training the full-rank Laplace approximation, and that it can be used to train the full-rank Laplace approximation on a convolutional neural network.
+However, we also found that the marginal training procedure to optimise the hyperparameters of the full-rank Laplace approximation is not yet feasible, since the hyperparameters converge to unreasonable values.
-% Which quantitative results do I have which I can put here?
+Finally, we showed that it is possible to perform the full Laplace approximation on a convolutional neural network, train its weights with marginal likelihood training, and sample from its posterior.
+We demonstrated on a simple sine wave regression problem and an MNIST classification problem that our method can be used to train the full Laplace approximation and yield predictive performance which rivals deterministic training.
+However, we found that the sampling procedure is too slow for online Laplace, and may become a significant bottleneck for inference using post-hoc Laplace on large-scale problems.
-How did we get these results? What methods did we use?
-
-% How did our findings contribute to the field? What changes?
-Overall, while we have developed novel methods for training and inference in the full-rank Laplace approximation, we have not yet demonstrated its improved performance over the diagonal Laplace approximation nor its feasibility.
+Overall, while we have developed novel methods for training and inference in the full-rank Laplace approximation, we have not yet demonstrated its improved performance over the diagonal Laplace approximation nor its practical application to large-scale problems.
\section[Outlook and Future Developments]{Outlook and Future Developments}[Future Work]
% OUR ISSUES
% What were our limitations?
-
-Fix marginal training, specifically hyperparameter optimisation
-Fix convolution performance in marginal
-Rank estimation, and (very closely related) determining if the eigenvectors are relatively stationary across different observations
-
-Furthermore, while we have improved the understanding of the spectrum of the Generalised Gauss-Newton Laplace posterior precision, we have not yet fully understood it.
-In particular, ...
-As such, it would be interesting to further investigate the spectrum of the Generalised Gauss-Newton Laplace posterior precision, in particular to better understand both the spectrum and the Hessian structure and their relationship to ...
+As mentioned in the previous section, we have not yet demonstrated the practicality of our method for full Laplace approximation, particularly for large-scale problems.
+Additionally, hyperparameter selection typically requires performing a grid search over a range of hyperparameters, which is computationally expensive.
+As such, the practicality of our method depends on the ability to perform marginal training to optimise hyperparameters via backpropagation.
+Additionally, marginal training is prohibitively slow for training convolutions, and we have not yet found a way to speed it up.
+Furthermore, estimation of the rank of the posterior precision is important to guarantee the closeness of the upper bound on the log-determinant term in the marginal, and this is still an open problem.
+This is closely related to the fact that we do not have a solid understanding of how much the eigenvectors of the posterior precision matrix vary across different observations nor what affects the shape of the spectrum of this Hessian.
+This point is also related to the structure of the Hessian in the Laplace approximation more generally.
+As such, there is still much work to be done to understand the methods we have developed and to improve them.
% Also diagonal raw values maybe, compare the two?
% Diagonal lets you know exactly which parameters contribute to the largest eigenvalues
-
-We did not show that the full-rank Laplace approximation shows improved predictive performance or uncertainty calibration when compared to the diagonal or Kronecker-factored Laplace approximations.
-...
+We also have not shown that the full-rank Laplace approximation shows improved predictive performance or uncertainty calibration when compared to the diagonal or Kronecker-factored Laplace approximations.
Furthermore, the full-rank Laplace approximation should be compared to other methods for Bayesian inference, such as variational approximations, deep ensembles, Monte Carlo dropout, and Markov chain Monte Carlo --- both in terms of performance and runtime.
% RELATED TO ACTUAL USEFULNESS OF THE METHOD
Furthermore, the feasibility of the full-rank Laplace approximation must be demonstrated on a large-scale problem.
-It is not clear how the full-rank Laplace approximation will scale with the number of parameters, data points, and model outputs.
-This is a factor both regarding the actual performance of the method and the post-hoc sampling time for the approximate posterior.
+It is not clear how our method for the full Laplace approximation will scale with the number of parameters, data points, and model outputs.
+This regards not only the actual predictive performance and uncertainty estimation of the method, but also the post-hoc inference time for the approximate posterior.
As such, important experiments must be performed to determine the scalability of the full-rank Laplace approximation.
-This includes experiments on a large-scale problem, such as a large-scale image classification problem on a dataset with hundreds of thousands to millions of rows, with significantly higher resolution than the MNIST dataset, and more than 10 classes, such as for zero-shot learning.
+This includes experiments on a large-scale problem, such as a large-scale image classification problem on a dataset with hundreds of thousands to millions of rows, with significantly higher resolution than the MNIST dataset, and more than 10 classes, such as in the case of zero-shot learning.
These experiments would thus also require models with many parameters, such as a ResNet.
% WHERE THE PROBLEM SHOULD BE EXPLORED NEXT. NOT SURE THERES MUCH FOR ME TO SAY
diff --git a/chapters/introduction.tex b/chapters/introduction.tex
@@ -16,7 +16,7 @@ These predictions will then have uncertainty estimates, thus improving model int
These methods have been successfully applied to a wide range of classical problems in statistics, though attempts to apply them to deep learning have had limited success.
For deep learning, the Bayesian approach is often limited by the computational cost of Bayesian inference.
-This is because Bayesian inference requires computing the posterior distribution of the model parameters given the data, which, due to the nonlinear complexity of the neural network architecture, is often intractable.
+This is because Bayesian inference requires computing the posterior distribution of the model parameters given the data, which, due to the nonlinear complexity of the neural network architecture, is often intractable.\sidenote{MEH. Would like to mention some applications, to argue that this is an important problem problem worth solving. However, I'm struggling to find solid applications, i.e., large-scale problems solved by BNNs. \textcite{wenzel2020good} suggest there haven't really been any.}
Typically, the Bayesian approach to deep learning is to use a prior distribution over the model parameters and then compute the posterior distribution of the model parameters given the data, where the posterior is maximised to obtain the optimal model parameters for the available data.
This posterior distribution can then be sampled from to make predictions.
diff --git a/chapters/results.tex b/chapters/results.tex
@@ -1,5 +0,0 @@
-\setchapterpreamble[u]{\margintoc}
-\chapter{Results}
-\labch{results}
-
-
diff --git a/chapters/sampling.tex b/chapters/sampling.tex
@@ -403,14 +403,14 @@ The preconditioning of \(\bm J_i \bm H_i \bm J_i\T\) (which should be low-rank)
Regarding the eigenvalues and eigenvectors, we consider a best-case scenario.
Assume that the pivoted Cholesky factorisation finds the \(K\) largest eigenvectors.
-If the eigenspectrum of this factorisation is steep enough, then the largest eigenvalue \(s_1\) will be significantly larger than the smallest eigenvalue \(s_1\) (i.e., there will be a steep drop-off from the largest to the smallest non-zero eigenvalue).
-Since the eigenvectors of each per-observation Hessian will not be the same, when we sum over them, the eigenvalues of the sum will not be the same as the eigenvalues of the individual Hessians.
-However, some of the eigenvectors may be close to each other, and so those eigenvalues will be approximately summed in the summed Hessian's spectrum.
+If the eigenspectrum of this factorisation is steep enough, then the largest eigenvalue \(s_1\) will be significantly larger than the smallest eigenvalue \(s_K\) (i.e., there will be a steep drop-off from the largest to the smallest non-zero eigenvalue).
+Since the eigenvectors of each per-observation Hessian will not be the same, when we sum over them, the eigenvalues of the sum will not be the sum of the eigenvalues of the individual Hessians.
+However, some of the eigenvectors may be close to each other, and so the spectrum of the GGN sum may be approximately the sum of the GGN eigenvalues.
However, the rank of the total Hessian will increase by the number of observations and so the number of non-zero eigenvalues consequently increase as well.
These eigenvalues will be smaller than the summed eigenvalues (because the eigenvectors may not be contained in the same subspace), but they will be larger than the smallest eigenvalue of the per-observation Hessian.
-This will then lead to a steeper eigenspectrum and a larger condition number of the total Hessian.
-Thus, when the batch size \(B\) is large, the condition number will increase and the preconditioner will become less effective.
-How much the condition number increases with the batch size depends on the eigenspectrum of the per-observation Hessians (i.e., how close the eigenvectors are to each other).\sidenote{I SHOULD REWRITE THIS PARAGRAPH, IT'S KINDA HARD TO EXPLAIN.}
+This will then lead to a steeper spectrum and a larger condition number of the total Hessian.
+Thus, when the batch size \(B\) is large, the condition number of the sum of GGN Hessians will increase and the preconditioner will become less effective.
+How much the condition number increases with the batch size depends on the eigenspectrum of the per-observation Hessians (i.e., how close the eigenvectors are to each other).\sidenote{MAYBE I SHOULD REWRITE THIS PARAGRAPH, IT'S KINDA HARD TO EXPLAIN.}
Unfortunately, if we want to use the pivoted Cholesky decomposition to approximate \(\sum^B_i \bm J_i\T \bm H_i \bm J_i\), we are largely trying to approximate the higher eigenvectors of the sum.
In order for this to be the case, we will want to oversample vectors from this matrix which correspond to the largest eigenvalues.
@@ -487,7 +487,9 @@ Since the chi-squared distribution is continuous, we can use the Kolmogorov-Smir
This test is performed by computing the Kolmogorov-Smirnov statistic, which is the maximum difference between the empirical and theoretical CDFs.
The null hypothesis of the test is that the samples are drawn from the same distribution, and so the test is rejected if the Kolmogorov-Smirnov statistic is greater than the critical value for the test.
This means that sample normality is rejected if the p-value of the test is less than the significance level of the test.
-However, this test is sensitive to the number of samples used to compute the statistic, and so a large number of samples will lead to a very low p-value even if the samples are drawn from the same distribution.
+However, this test is very sensitive to the number of samples used to compute the statistic, and so a large number of samples will lead to a very low p-value even if the samples are drawn from the same distribution.
+Since we can sample an arbitrarily large number of samples from the approximate posterior, this may lead to low p-values even when sampling succeeds.
+\marginnote{MAYBE ADD A LIMITATIONS / CONCLUSION SECTION AT THE END OF THIS CHAPTER?}
% \section{Limitations}
diff --git a/chapters/theory.tex b/chapters/theory.tex
@@ -19,7 +19,7 @@ These two cases correspond to the mean-squared error (MSE) and cross-entropy (CE
Due to the assumption of independence between observations, we can factorise the likelihood and the negative log-likelihood loss as
\begin{align}\label{eq:negative-log-likelihood}
- p(\bm y \given \bm \theta) ={} & \prod_i^N p(x_i \given \bm \theta) \nonumber
+ p(\bm y \given \bm \theta) ={} & \prod_i^N p(x_i \given \bm \theta), \nonumber
\\ \mathcal{L}(\bm\theta) \equiv \mathcal{L}(\bm \theta, \bm y) \coloneqq -\log p(\bm y \given \bm \theta) ={}& - \sum_i^N \log p(y_i \given \bm \theta).
\end{align}
This allows us to compute the loss as the sum of the losses for each individual observation.
@@ -32,19 +32,20 @@ Stochastic gradient descent (SGD) is a first-order optimisation algorithm which
The update rule is given by
%
\begin{align}
- \bm\theta_{t+1} = \bm\theta_t - \eta_t \nabla_{\bm\theta} \mathcal{L}(\bm\theta_t)
+ \bm\theta_{t+1} = \bm\theta_t - \eta_t \nabla_{\bm\theta} \mathcal{L}(\bm\theta_t),
\end{align}
%
where \(\eta_t\) is the learning rate at iteration \(t\).
-The learning rate \(\eta_t\) can be constant or adaptive, and is often chosen to be a decreasing function of \(t\).
+The learning rate \(\eta_t\) can be constant or adaptive, and is often chosen to be a non-increasing function of \(t\).
The gradient \(\nabla_{\bm\theta} \mathcal{L}(\bm\theta_t, \bm x_t, y_t)\) can be computed separately for each single training example \(\bm x_t, y_t\), as seen in \cref{eq:negative-log-likelihood}.
-The gradient is computed using the chain rule as
+Using the chain rule on the composition of the loss into the loss as a function of the model output \(\mathcal{L}\) and the model output as a function of the parameters \(f\) such that \(\mathcal{L}(\bm \theta_t, \bm x_t, y_t) = \mathcal{L}(y_t, f(\bm x_t; \bm \theta_t))\),we compute the gradient as
%
\begin{align}\label{eq:sgd-gradient}
- \nabla_{\bm\theta} \mathcal{L}(\bm\theta_t, \bm x_t, y_t) ={} & \nabla_{\bm\theta} \log p(y_t \given \bm \theta) \nabla_{\bm\theta} f_{\bm\theta_t}(\bm x_t)
+ \nabla_{\bm\theta} \mathcal{L}(\bm\theta_t, \bm x_t, y_t) ={} & \nabla_{\bm\theta} \log p(y_t \given \bm \theta) \nabla_{\bm\theta} f_{\bm\theta_t}(\bm x_t),
\end{align}
%
where \(\nabla_{\bm\theta} f_{\bm\theta_t}(\bm x_t)\) is the gradient of the neural network output with respect to its parameters at iteration \(t\).
+Note that in this case, we perform one gradient update for each data point, which is why we iterate over the data points and update the parameters in the same step \(t\).
Thus, SGD only requires the gradient of the loss function with respect to the parameters of the neural network, which can be computed efficiently using automatic differentiation for a similar cost to a single forward pass through the network.
This makes SGD a popular choice for training neural networks.
However, SGD is prone to getting stuck in local minima, and can be slow to converge.
@@ -196,8 +197,9 @@ Then the Hessian can be interpreted as the Jacobian matrix of the gradient of th
We are interested in the Hessian of the loss function \(\mathcal{L}(\bm\theta)\) with respect to the parameters \(\bm\theta\).
For a neural network with \(D\) parameters, the Hessian is therefore a \(D \times D\) square matrix.
-Furthermore, if all the neural network's second partial derivatives are continuous, then the Hessian is symmetric and positive definite.
+Furthermore, if all the neural network's second partial derivatives are continuous, then the Hessian is symmetric.
In this case, the Hessian is generally dominated by the block-diagonal~\cite{martens2015optimizing}.
+However, the Hessian will only be positive definite if the loss function is a convex function of the model parameters \(\bm \theta\).
Typically, the loss function is considered as the negative log-likelihood or negative log-posterior.
In this framing, if considering, e.g., the negative log-posterior loss from \cref{eq:negative-log-posterior}, by the linearity of the derivative, we get
@@ -218,11 +220,17 @@ From the equation of the Hessian we can apply the chain rule twice and the produ
\begin{align}\label{eq:ggn-whole-hessian}
\nabla^2_{\bm \theta} \mathcal{L}
={} & J_{\bm \theta}(\nabla_{\bm \theta} \mathcal{L})
- = J_{\bm \theta}\left(\nabla_f \mathcal{L} \cdot \nabla_{\bm \theta} f\right) \nonumber
- \\ ={}& J_{\bm \theta} (\nabla_{\bm \theta} f) \cdot \nabla_f \mathcal{L} + J_{\bm \theta} \left(\nabla_f \mathcal{L}\right) \cdot \nabla_{\bm \theta} f \nonumber
- \\ ={}& \nabla^2_{\bm \theta} f \cdot \nabla_f \mathcal{L} + \nabla_{\bm \theta} f \T \cdot \nabla^2_f \mathcal{L} \cdot \nabla_{\bm \theta} f,
+ = J_{\bm \theta}\left(\nabla_f \mathcal{L} \cdot J_{\bm \theta} f\right) \nonumber
+ \\ ={}& J_{\bm \theta} (J_{\bm \theta} f) \cdot \nabla_f \mathcal{L} + J_{\bm \theta} \left(\nabla_f \mathcal{L}\right) \cdot J_{\bm \theta} f \nonumber
+ \\ ={}& \nabla^2_{\bm \theta} f \cdot \nabla_f \mathcal{L} + J_{\bm \theta} f \T \cdot \nabla^2_f \mathcal{L} \cdot J_{\bm \theta} f,
\end{align}
where \(\nabla_f\) is the gradient with respect to \(f(\bm x, \bm \theta)\) and \(J_{\bm \theta}\) is the Jacobian with respect to \(\bm \theta\).
+More precisely, we obtain
+\begin{align}
+ \nabla^2_{\bm \theta} \mathcal{L}_{ij} = \sum_{n=1}^O \frac{\partial^2}{\partial \theta_i \theta_j} f_n \cdot \frac{\partial}{\partial f_n} \mathcal{L} + \sum_{n,m=1}^O \frac{\partial}{\partial \theta_i} f_n \cdot \frac{\partial^2}{\partial f_n f_m} \mathcal{L} \cdot \frac{\partial}{\partial \theta_j} f_m,
+\end{align}
+where \(f_n\) is the n\textsuperscript{th} output of the neural network function \(f(\bm x, \bm \theta)\).
+% Mention that it's evaluated?
% The first term is the GGN approximation of the Hessian of the loss function.
Assume now we will approximate \(f\) using a first-order Taylor expansion.
@@ -231,9 +239,9 @@ This approximation is given by
%
\begin{align}\label{eq:nn-taylor}
f_{\bm \theta}(\bm x) \approx{} f_{\bm \theta_0}(\bm x) + \nabla_{\bm \theta} f_{\bm \theta_0}(\bm x) \cdot (\bm \theta - \bm \theta_0) .
- \\ \text{Notice that } \nabla^2_{\bm \theta} f_{\bm \theta_0}(\bm x) = 0. \nonumber
\end{align}
%
+Notice that \(\nabla^2_{\bm \theta} f_{\bm \theta_0}(\bm x) = 0\) under the Taylor expansion assumption of neural network linearity.
By combining \cref{eq:ggn-whole-hessian,eq:nn-taylor} we then obtain the generalised Gauss-Newton approximation of the Hessian,
%
\begin{align*}
@@ -333,7 +341,15 @@ If the \(K\) non-zero eigenvectors are identical across the GGN matrices, then t
However, if these eigenvectors are not identical, then as the largest eigenvalue of the sum of the GGN increases and the number of non-zero eigenvalues of the sum of the GGN increases, the non-zero spectrum of the sum of the GGN matrices will get steeper.
For these reasons, as we add more data points, the posterior precision will become more ill-conditioned.\sidenote{A matrix is considered ill-conditioned when the ratio of its largest eigenvalue to its lowest eigenvalue, its \emph{condition number} (\(\kappa \coloneqq \lambda_{\mathrm{max}} / \lambda_{\mathrm{min}}\)), is very large.}
-\section{Limitations and Large-Scale Laplace}[Limitations]
+ADD ANY OF THIS?
+\begin{align*}
+ \lambda\left( J\T J + \alpha I \right) ={} & \{ \underbrace{s_1, s_2, \ldots, s_O}_{O}, \underbrace{\alpha, \ldots, \alpha}_{D - O} \}
+ \\ \lambda\left( N J\T J + \alpha I \right) ={}& \{ \underbrace{N s_1, \ldots, N s_O}_{O}, \underbrace{\alpha, \ldots, \alpha}_{D - O} \}
+ \\ \lambda\left( \frac{N}{B} \textstyle{\sum^B_i} J_i\T J_i + \alpha I \right) ={}& \{ \underbrace{s_1, \ldots, s_K}_{K \leq B \cdot O}, \underbrace{\alpha, \ldots, \alpha}_{D - K} \}
+ % \Span_\lambda\left( J\T J + \alpha \identity \right)
+\end{align*}
+
+\section{Limitations}
One limitation of the Laplace approximation is that it can only approximate one mode of the posterior distribution (the MAP).
However, most approximate posterior distributions, such as MCMC and variational inference, also struggle to approximate multimodal distributions.
@@ -351,14 +367,6 @@ However, it is also possible to perform the Laplace approximation by training a
This would reduce overfitting and allow us to optimise hyperparameters via backpropagation.
However, we must find a differentiable way to compute the Laplace approximate log-marginal likelihood without explicitly storing the posterior precision.
-ADD ANY OF THIS?
-\begin{align}
- \lambda\left( J\T J + \alpha I \right) ={} & \{ \underbrace{s_1, s_2, \ldots, s_k}_{k}, \underbrace{\alpha, \ldots, \alpha}_{n - k} \}
- \\ \lambda\left( N J\T J + \alpha I \right) ={}& \{ \underbrace{N s_1, N s_2, \ldots, N s_k}_{k}, \underbrace{\alpha, \ldots, \alpha}_{n - k} \}
- \\ \lambda\left( \frac{N}{B} \textstyle{\sum^B_i} J_i\T J_i + \alpha I \right) ={}& \{ \underbrace{?, \ldots, ?}_{\leq B \cdot k}, \underbrace{\alpha, \ldots, \alpha}_{\geq n - B \cdot k} \}
- % \Span_\lambda\left( J\T J + \alpha \identity \right)
-\end{align}
-
% \section[The Fisher Information Matrix]{The Fisher Information Matrix}[Fisher Information]
% \label{sec:fisher-information}
% % https://agustinus.kristia.de/techblog/2018/03/11/fisher-information/
diff --git a/chapters/training.tex b/chapters/training.tex
@@ -188,7 +188,7 @@ We do this by scaling the trace and square traces for \cref{eq:laplace-log-deter
\\ \Tr\left(\left(\frac{N}{B} \textstyle{\sum_{i=1}^B} \bm J_i \bm H_i \bm J_i\right)^2\right) ={}& \frac{N^2}{B^2} \Tr\left(\left(\textstyle{\sum_{i=1}^B} \bm J_i \bm H_i \bm J_i\right)^2\right). \label{eq:laplace-square-trace-scaling}
\end{align}
By combining \cref{eq:laplace-log-determinant-bound,eq:bai-golub-bound,eq:hutchinson-trace,eq:hutchinson-square-trace,eq:trace-rearrangement,eq:laplace-trace-scaling,eq:laplace-square-trace-scaling}, we can therefore compute a differentiable upper bound on the log-marginal likelihood under the Laplace approximation using only Hessian-vector products without the need to instantiate the posterior precision.
-
+\marginnote{MAYBE ADD A LIMITATIONS / CONCLUSION SECTION AT THE END OF THIS CHAPTER?}
% \subsection{Limitations}
% Estimating the trace and square trace of the Hessian matrix is the most computationally expensive part of estimating the log-marginal under the Laplace approximation, since this involves computing the GGN-vector product.
diff --git a/main.bib b/main.bib
@@ -378,6 +378,13 @@
publisher = {Now Publishers, Inc.}
}
+@inproceedings{wenzel2020good,
+ title = {How Good is the {B}ayes Posterior in Deep Neural Networks Really?},
+ author = {Wenzel, Florian and Roth, Kevin and Veeling, Bastiaan S and {\'S}wi{\k{a}}tkowski, Jakub and Tran, Linh and Mandt, Stephan and Snoek, Jasper and Salimans, Tim and Jenatton, Rodolphe and Nowozin, Sebastian},
+ booktitle = icml,
+ year = {2020}
+}
+
@book{williams2006gaussian,
title = {Gaussian Processes for Machine Learning},
author = {Williams, Christopher KI and Rasmussen, Carl Edward},
diff --git a/main.tex b/main.tex
@@ -177,12 +177,12 @@
% DEDICATION
%----------------------------------------------------------------------------------------
-\dedication{
- % \blackout
- % {It’s an incredibly exciting thing, this one, meaningless life of yours.}\\
- {You don't have to have a dream.}\\
- \flushright -- Tim Minchin
-}
+% \dedication{
+% % \blackout
+% % {It’s an incredibly exciting thing, this one, meaningless life of yours.}\\
+% {You don't have to have a dream.}\\
+% \flushright -- Tim Minchin
+% }
%----------------------------------------------------------------------------------------
% OUTPUT TITLE PAGE AND PREVIOUS
@@ -196,14 +196,14 @@
% PREFACE
%----------------------------------------------------------------------------------------
-\input{chapters/preface.tex}
-\index{preface}
+% \input{chapters/preface.tex}
+% \index{preface}
-\input{chapters/acknowledgments.tex}
-\index{acknowledgments}
+% \input{chapters/acknowledgments.tex}
+% \index{acknowledgments}
-\input{chapters/abstract.tex}
-\index{abstract}
+% \input{chapters/abstract.tex}
+% \index{abstract}
%----------------------------------------------------------------------------------------
% TABLE OF CONTENTS & LIST OF FIGURES/TABLES
@@ -246,7 +246,6 @@
\input{chapters/training.tex}
\input{chapters/sampling.tex}
\input{chapters/experiments.tex}
-\input{chapters/results.tex}
\input{chapters/conclusion.tex}
%----------------------------------------------------------------------------------------
