commit 48d9222e0ee4c1da83778b985381ada380c69df7
parent 0203f4439e2cb0420e6b58aa9a939322e2ffdc32
Author: Silas Brack <s174433@student.dtu.dk>
Date: Mon, 2 Jan 2023 01:11:49 +0100
Changed \epsilon to \varepsilon.
Diffstat:
6 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/chapters/appendix.tex b/chapters/appendix.tex
@@ -31,7 +31,7 @@
$\bm z_j \gets \bm z_j / \beta_{j}$\;
$\bm q_j \gets \bm q_j / \beta_{j}$\;
\For{$q \gets 1$ \KwTo $Q$}{
- $\epsilon^{(q)}_{j} \gets \sin^{(q)}_{j-2} \beta_{j-1}$\;%\Comment{Start Givens rotation}
+ $\varepsilon^{(q)}_{j} \gets \sin^{(q)}_{j-2} \beta_{j-1}$\;%\Comment{Start Givens rotation}
$\zeta^{(q)}_{j} \gets \cos^{(q)}_{j-2} \beta_{j-1}$\;
$\alpha^{(q)}_{j} \gets \alpha_{j} + t_q$\;%\Comment{Compute shifted alpha}
$\eta^{(q)}_j \gets \cos^{(q)}_{j-1} \alpha^{(q)}_{j} - \sin^{(q)}_{j-1} \zeta^{(q)}_{j}$\;%\Comment{Givens rotation from 1 step ago}
@@ -43,7 +43,7 @@
$\eta^{(q)}_{j} \gets \eta^{(q)}_{j} \cos^{(q)}_{j} + \sin^{(q)}_{j} \beta_{j}$\;%\Comment{Apply current Givens rotation}
$\varphi^{(q)}_{j} \gets - \varphi^{(q)}_{j-1} \sin^{(q)}_{j}$\;%\Comment{Apply the latest Givens rotation to the Lanczos-rhs, computing the scale terms for the search vectors}
$\varphi^{(q)}_{j-1} \gets \varphi^{(q)}_{j-1} \cos^{(q)}_{j}$\;
- $\bm d^{(q)}_j \gets \left( \bm q_{j-1} - \zeta^{(q)}_{j} \bm d^{(q)}_{j-1} - \epsilon^{(q)}_{j} \bm d^{(q)}_{j-2} \right) / \eta^{(q)}_{j}$\;%\Comment{Get the new search vector}
+ $\bm d^{(q)}_j \gets \left( \bm q_{j-1} - \zeta^{(q)}_{j} \bm d^{(q)}_{j-1} - \varepsilon^{(q)}_{j} \bm d^{(q)}_{j-2} \right) / \eta^{(q)}_{j}$\;%\Comment{Get the new search vector}
$\Delta \bm c^{(q)}_{j} \gets \bm d^{(q)}_{j} \varphi^{(q)}_{j-1}$\;
$\bm c^{(q)}_{j} \gets \bm c^{(q)}_{j-1} + \Delta \bm c^{(q)}_{j}$\;%\Comment{Update the solution}
}
diff --git a/chapters/literature.tex b/chapters/literature.tex
@@ -169,9 +169,9 @@ In this case, we get \(\bm \Lambda = \left. - \nabla^2_{\bm \theta} \log p(\bm \
% In mean-field VI, model weights are sampled from an approximate posterior \(q(\bm \theta) \approx p(\bm{\theta} \given \bm y)\) as
% \begin{align}
% q(\bm \theta) ={} & \normal(\bm \mu, \bm \sigma) \Leftrightarrow \nonumber \\
-% \bm{\theta} ={} & \bm{\mu} + \bm{\sigma} \circ \bm{\epsilon}\label{eq:meanfield}
+% \bm{\theta} ={} & \bm{\mu} + \bm{\sigma} \circ \bm{\varepsilon}\label{eq:meanfield}
% \end{align}
-% where \(\bm{\epsilon} \sim \normal(\bm 0, \identity)\).
+% where \(\bm{\varepsilon} \sim \normal(\bm 0, \identity)\).
% For this type of variational approximation, the entropy term is given by \(\entropy{q(\theta)} = - \sum_i \log \sigma_i\) and the cross-entropy of the prior relative to the variational approximation is calculated via Monte Carlo simulation by taking the prior log probability with respect to mean-field posterior samples as
% \begin{align}
% \crossentropy{q(\bm\theta)}{p(\bm\theta)} ={} & - \int q(\bm\theta) \log(p(\bm\theta))\,d\bm\theta \nonumber \\
@@ -208,13 +208,13 @@ In this case, we get \(\bm \Lambda = \left. - \nabla^2_{\bm \theta} \log p(\bm \
% \subsection{Radial Variational Approximation}
% \textcite{farquhar2020radial} introduces the Radial approximate posterior,
-% For each NN layer, this posterior is sampled similarly to the mean-field (\cref{eq:meanfield}), but by normalising the \(\bm{\epsilon}\) term (projecting it onto a hypersphere) yielding a direction term \(\bm{\epsilon}/\norm{\bm{\epsilon}}\), and scaling it by the distance \(r\).
+% For each NN layer, this posterior is sampled similarly to the mean-field (\cref{eq:meanfield}), but by normalising the \(\bm{\varepsilon}\) term (projecting it onto a hypersphere) yielding a direction term \(\bm{\varepsilon}/\norm{\bm{\varepsilon}}\), and scaling it by the distance \(r\).
% Therefore, for each layer, the radial posterior is sampled as
% \begin{align}\label{eq:radial}
% q(\bm \theta) ={} & \mathrm{Radial}(\bm{\mu}, \bm{\sigma}) \nonumber \\
-% \bm{\theta} ={} & \bm{\mu} + \bm{\sigma} \circ \frac{\bm{\epsilon}}{\norm{\bm{\epsilon}}} r
+% \bm{\theta} ={} & \bm{\mu} + \bm{\sigma} \circ \frac{\bm{\varepsilon}}{\norm{\bm{\varepsilon}}} r
% \end{align}
-% where \(\bm{\epsilon} \sim \normal(\bm{0}, \identity)\), \(r \sim \normal(0, 1)\).
+% where \(\bm{\varepsilon} \sim \normal(\bm{0}, \identity)\), \(r \sim \normal(0, 1)\).
% The entropy term of the Radial approximate posterior is given
% \(\entropy{q(\theta)} = - \sum_i \log \sigma_i + \mathrm{const}\) (and is therefore approximately equal to the entropy of the mean-field approximation up to a constant) and the cross-entropy term is calculated via Monte Carlo simulation as for mean-field VI (\cref{ssec:mfvi}), but by sampling from a radial posterior \(\bm{\theta}^{(s)} \sim \mathrm{Radial}(\bm{\mu}, \bm{\sigma})\) as in \cref{eq:radial}.
diff --git a/chapters/mixture.tex b/chapters/mixture.tex
@@ -10,7 +10,7 @@ such that \(\sum^N_i w_i \bm \mu_i = \bm \mu_{\textsc{map}}\) and \(\sum_i^N w_i
Sampling from this model would then involve sampling an index
\begin{align}
i \sim{} & \mathrm{Cat}(N, \bm w)
- \\ \bm \epsilon \sim{}& \normal(\bm \mu_i, \bm \Sigma_i)
+ \\ \bm \varepsilon \sim{}& \normal(\bm \mu_i, \bm \Sigma_i)
\end{align}
where \(\bm \mu_i\) is computed by shifting the MAP solution based on ??? and \(\bm \Sigma_i\) is computed as the inverse precision matrix from the Laplace approximation, as per \cref{eq:shifted-map}.
\begin{align}\label{eq:shifted-map}
@@ -19,7 +19,7 @@ where \(\bm \mu_i\) is computed by shifting the MAP solution based on ??? and \(
\end{align}
The reparameterisation trick can then be performed using only a single observation to determine the covariance matrix
\begin{align}
- \bm \epsilon ={} & \bm \mu_i + \bm \Sigma^{-1/2}_i \bm \epsilon_0,
+ \bm \varepsilon ={} & \bm \mu_i + \bm \Sigma^{-1/2}_i \bm \varepsilon_0,
\end{align}
where, since \(\bm \Sigma^{-1}_i = J_i\T H_i J_i + \alpha / N \identity\) is low-rank with rank \(o\), the Woodbury identity from \cref{sec:woodbury} can be used.
This method depends on the conditioning number of the outer product \(J H J\T\), which is significantly lower than that of the inner product \(J\T H J\).
diff --git a/chapters/sampling.tex b/chapters/sampling.tex
@@ -7,24 +7,24 @@
In many methods, such as the Laplace approximation (\cref{sec:laplace}), we need to sample from a multivariate normal distribution parameterised by a mean vector \(\bm \mu\) and a covariance matrix \(\bm \Sigma\).
To sample from a Gaussian distribution, you can apply the \emph{reparameterisation trick} to compute a vector of samples as
\begin{align}\label{eq:sample-reparam}
- \bm \epsilon ={} & \bm \mu + \bm \Sigma^{1/2} \bm \epsilon_0,
+ \bm \varepsilon ={} & \bm \mu + \bm \Sigma^{1/2} \bm \varepsilon_0,
\end{align}
-where \(\bm \epsilon_0\) are samples drawn from a standard normal distribution, i.e., \(\bm \epsilon_0 \sim \normal(\bm 0, \identity)\).
+where \(\bm \varepsilon_0\) are samples drawn from a standard normal distribution, i.e., \(\bm \varepsilon_0 \sim \normal(\bm 0, \identity)\).
Using the Laplace approximation, we estimate the precision matrix as the Hessian matrix of the loss with respect to the parameters, as discussed in \cref{sec:laplace}.
Because of this, the covariance matrix is given by the inverse of this Hessian and we thus sample as per
\begin{align}\label{eq:sample-laplace}
- \bm \epsilon = \bm \mu + \bm \Lambda^{-1/2} \bm \epsilon_0,
+ \bm \varepsilon = \bm \mu + \bm \Lambda^{-1/2} \bm \varepsilon_0,
\end{align}
where \(\Lambda = \nabla^2_{\bm\theta} \log p(\bm \theta \given \bm x)\).
In \cref{eq:sample-reparam}, the square root of a matrix yields another matrix which, when multiplied by itself, is equal to the original matrix.
-However, since a Gaussian distribution is invariant to rotation\sidenote{That is, if we have a vector of standard normal samples \(\bm \epsilon \sim \normal(\bm 0, \identity)\), then, for an orthogonal matrix \(\bm A\) (a rotation matrix), \(\bm A\T \bm \epsilon \sim \normal(\bm 0, \identity)\). We then have that we can sample \emph{non} standard normal samples as \(\bm \Lambda^{-1/2} \bm A\T \bm \epsilon \sim \normal(\bm 0, \identity)\).}, in this special case, we simply need to obtain a matrix which returns the original matrix \emph{up to a rotation}.
+However, since a Gaussian distribution is invariant to rotation\sidenote{That is, if we have a vector of standard normal samples \(\bm \varepsilon \sim \normal(\bm 0, \identity)\), then, for an orthogonal matrix \(\bm A\) (a rotation matrix), \(\bm A\T \bm \varepsilon \sim \normal(\bm 0, \identity)\). We then have that we can sample \emph{non} standard normal samples as \(\bm \Lambda^{-1/2} \bm A\T \bm \varepsilon \sim \normal(\bm 0, \identity)\).}, in this special case, we simply need to obtain a matrix which returns the original matrix \emph{up to a rotation}.
We thus need to find a method which computes the inverse square root of a high-dimensional matrix up to a rotation.
\section{Contour Integral Quadrature}
\textcite{pleiss2020fast} propose one such algorithm.
-Contour integral quadrature (CIQ) attempts to solve the problem of computing the inverse-square root of a matrix, up to a rotation, by approximating the expression \(f(\bm K) \bm \epsilon_0 = \bm K^{-1/2} \bm \epsilon_0\) where \(\bm K\) is a positive semi-definite matrix using Cauchy's integral formula.
+Contour integral quadrature (CIQ) attempts to solve the problem of computing the inverse-square root of a matrix, up to a rotation, by approximating the expression \(f(\bm K) \bm \varepsilon_0 = \bm K^{-1/2} \bm \varepsilon_0\) where \(\bm K\) is a positive semi-definite matrix using Cauchy's integral formula.
To do this, it exploits the Lanczos algorithm, Gaussian quadrature, and the msMINRES algorithm, methods which only require the computation of matrix-vector products \(\bm K \bm v\).
There are then essentially three steps to CIQ, namely:
\begin{enumerate}
@@ -49,13 +49,13 @@ There are then essentially three steps to CIQ, namely:
% $\mathrm{samples} \gets \sum^Q_{q=1} w_q \left( t_q \identity + H \right)^{-1} b$
\end{algorithm}
-This procedure (see \cref{alg:ciq}) computes the approximate inverse square root product \(\bm K\) by \(\bm \epsilon_0\) up to a rotation, i.e.,
+This procedure (see \cref{alg:ciq}) computes the approximate inverse square root product \(\bm K\) by \(\bm \varepsilon_0\) up to a rotation, i.e.,
\begin{align}\label{eq:ciq-asdf}
- \bm K^{-1/2} \bm \epsilon_0.
+ \bm K^{-1/2} \bm \varepsilon_0.
\end{align}
From here, it is then trivial to also compute the product of a matrix square root by multiplying \(\bm K\) onto this vector, as per
\begin{align}
- \bm K^{1/2} \bm \epsilon_0 = \bm K \cdot \underbrace{\bm K^{-1/2} \bm \epsilon_0}_{\text{\cref{eq:ciq-asdf}}}.
+ \bm K^{1/2} \bm \varepsilon_0 = \bm K \cdot \underbrace{\bm K^{-1/2} \bm \varepsilon_0}_{\text{\cref{eq:ciq-asdf}}}.
\end{align}
\def\ymax{2.0}
@@ -176,7 +176,7 @@ and shifts as
\section{Preconditioning}
-One drawback of contour integral quadrature if that the accuracy its of approximation of \(\bm \Lambda^{-1/2} \bm \epsilon_0\) suffers when the precision matrix \(\bm \Lambda\) is poorly conditioned\sidenote{Recall that a matrix is considered poorly conditioned when the ratio of its largest eigenvalue to its lowest eigenvalue, its \emph{conditioning number} (\(\kappa := \lambda_{\mathrm{max}} / \lambda_{\mathrm{min}}\)), is very large.}.
+One drawback of contour integral quadrature if that the accuracy its of approximation of \(\bm \Lambda^{-1/2} \bm \varepsilon_0\) suffers when the precision matrix \(\bm \Lambda\) is poorly conditioned\sidenote{Recall that a matrix is considered poorly conditioned when the ratio of its largest eigenvalue to its lowest eigenvalue, its \emph{conditioning number} (\(\kappa := \lambda_{\mathrm{max}} / \lambda_{\mathrm{min}}\)), is very large.}.
In the case of the Laplace approximation, as per \cref{eq:ggn-laplace}, we have that the precision matrix is given by
\begin{align}\label{eq:ggn-laplace}
@@ -361,19 +361,19 @@ In general, it is worth noting the flaw with scaling the term in the \(\sum J\T
\section{Sampling Evaluation}
\begin{align}
- \epsilon_0\T \epsilon_0 \sim \chi^2(D)
+ \varepsilon_0\T \varepsilon_0 \sim \chi^2(D)
\end{align}
-Samples \(\epsilon\) which are normally distributed but do not have a mean of zero and a standard deviation of one can be normalised and their sum of square deviations be calculated by computing their Mahalanobis distances.
+Samples \(\varepsilon\) which are normally distributed but do not have a mean of zero and a standard deviation of one can be normalised and their sum of square deviations be calculated by computing their Mahalanobis distances.
\begin{align}
- d ={} & \sqrt{(\epsilon - \mu)\T \Sigma^{-1} (\epsilon - \mu)}\label{eq:mahalanobis-distance}
+ d ={} & \sqrt{(\varepsilon - \mu)\T \Sigma^{-1} (\varepsilon - \mu)}\label{eq:mahalanobis-distance}
\\ \Rightarrow{}& d\T d \sim \chi^2(D) \nonumber
\end{align}
This has the advantage of only requiring computation of the precision matrix \(\Sigma^{-1}\).
For the Laplace approximation, this can easily be computed as \(\Sigma^{-1} = - \nabla^2_\theta \log p(\theta \given x)\).
-Since \(- \nabla^2_\theta \log p(\theta \given x)\) can be computed exactly, this calculation can be performed to evaluate the whether a set of samples \(\epsilon\) are drawn from a distribution \(\normal(\mu, \Sigma)\) while only accessing the precision matrix in the evaluation.
+Since \(- \nabla^2_\theta \log p(\theta \given x)\) can be computed exactly, this calculation can be performed to evaluate the whether a set of samples \(\varepsilon\) are drawn from a distribution \(\normal(\mu, \Sigma)\) while only accessing the precision matrix in the evaluation.
As such, this method can be used to evaluate whether the approximate normal samples obtained from CIQ are correctly distributed.
\section{Limitations}
diff --git a/chapters/theory.tex b/chapters/theory.tex
@@ -381,13 +381,13 @@ The MINRES algorithm can then be trivially extended to solve for multiple shifts
% $v_j \gets \gamma_j q_j$\;
% $\delta_j \gets \norm{v_j}$\;
% \For{$q \gets 1$ \KwTo $Q$}{
-% $\epsilon_j^{(q)} \gets \delta_{j-1} \left( \delta_{j-2} / \sqrt{\delta_{j-2}^2 + \eta_{j-2}^{(q)2}} \right)$\;
+% $\varepsilon_j^{(q)} \gets \delta_{j-1} \left( \delta_{j-2} / \sqrt{\delta_{j-2}^2 + \eta_{j-2}^{(q)2}} \right)$\;
% $\zeta_j^{(q)} \gets \delta_{j-1} \left( \eta_{j-2}^{(q)} / \sqrt{\delta_{j-2}^2 + \eta_{j-2}^{(q)2}} \right)$\;
% $\eta_j^{(q)} \gets (\gamma_j + t_q) \left( \eta_{j-1}^{(q)} / \sqrt{\delta_{j-1}^2 + \eta_{j-1}^{(q)2}} \right) + \zeta_j^{(q)} \left( \delta_{j-1} / \sqrt{\delta_{j-1}^2 + \eta_{j-1}^{(q)2}} \right)$\;
% $\zeta_j^{(q)} \gets \zeta_j^{(q)} \left( \eta_{j-1}^{(q)} / \sqrt{\delta_{j-1}^2 + \eta_{j-1}^{(q)2}} \right) + (\gamma_j + t_q) \left( \delta_{j-1} / \sqrt{\delta_{j-1}^2 + \eta_{j-1}^{(q)2}} \right)$\;
% $\eta_j^{(q)} \gets \eta_j^{(q)} \left( \eta_{j}^{(q)} / \sqrt{\delta_{j}^2 + \eta_{j}^{(q)2}} \right)$\;
% $\varphi_j^{(q)} \gets \varphi_{j-1}^{(q)} \left( \delta_{j-1} / \sqrt{\delta_{j-1}^2 + \eta_{j-1}^{(q)2}} \right) \left( \eta_{j}^{(q)} / \sqrt{\delta_{j}^2 + \eta_{j}^{(q)2}} \right)$\;
-% $d_j^{(q)} \gets \left( q - \zeta_j^{(q)} d_{j-1}^{(q)} - \epsilon_{j}^{(q)} d_{j-2}^{(q)} \right) / \eta_{j}^{(q)}$\;
+% $d_j^{(q)} \gets \left( q - \zeta_j^{(q)} d_{j-1}^{(q)} - \varepsilon_{j}^{(q)} d_{j-2}^{(q)} \right) / \eta_{j}^{(q)}$\;
% $c_j^{(q)} \gets c_{j-1}^{(q)} + \varphi_{j}^{(q)} + d_{j}^{(q)}$\;
% }
% }
diff --git a/main.tex b/main.tex
@@ -322,7 +322,7 @@
% $\beta$ & beta \pronounced{BAY-tuh} & $\xi$, $\Xi$ & xi \pronounced{KSIGH} \\
% $\gamma$, $\Gamma$ & gamma \pronounced{GAM-muh} & o & omicron \pronounced{OM-uh-CRON} \\
% $\delta$, $\Delta$ & delta \pronounced{DEL-tuh} & $\pi$, $\Pi$ & pi \pronounced{PIE} \\
-% $\epsilon$ & epsilon \pronounced{EP-suh-lon} & $\rho$ & rho \pronounced{ROW} \\
+% $\varepsilon$ & epsilon \pronounced{EP-suh-lon} & $\rho$ & rho \pronounced{ROW} \\
% $\zeta$ & zeta \pronounced{ZAY-tuh} & $\sigma$, $\Sigma$ & sigma \pronounced{SIG-muh} \\
% $\eta$ & eta \pronounced{AY-tuh} & $\tau$ & tau \pronounced{TOW (as in cow)} \\
% $\theta$, $\Theta$ & theta \pronounced{THAY-tuh} & $\upsilon$, $\Upsilon$ & upsilon \pronounced{OOP-suh-LON} \\
