mirror of
https://github.com/janishutz/eth-summaries.git
synced 2026-09-10 19:15:25 +02:00
[IML] Kernel PCA
This commit is contained in:
RobinB27
@@ -1,3 +1,7 @@
|
||||
\smalltext{This is a summary of the 2026 IML Script (FS26). All content from the script is (briefly) covered. There may be errors. Images belong to the IML team.}
|
||||
|
||||
\subtext{Wherever $\Vert\cdot\Vert_p$ isn't specified, $p=2$.}
|
||||
\subtext{Wherever $\Vert\cdot\Vert_p$ isn't specified, $p=2$.}
|
||||
|
||||
{\scriptsize
|
||||
\tableofcontents
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
\textbf{Problem}: How can we evaluate model performance?
|
||||
|
||||
\smalltext{Intuitively, we consider a model good if it performs well on non-training data, i.e. how well it generalizes.}
|
||||
|
||||
New terminology:
|
||||
|
||||
\begin{tabular}{ll}
|
||||
$M$ & Method \\
|
||||
$M(\mathcal{D}) = \hat{f}_\mathcal{D}$ & Model trained on $\mathcal{D}$ \\
|
||||
$f^*$ & Ideal function ("ground truth")
|
||||
\end{tabular}
|
||||
|
||||
{\footnotesize
|
||||
\remark The terms model and method are often used interchangeably.
|
||||
}
|
||||
|
||||
\subsection{Error types}
|
||||
|
||||
\definition \textbf{Estimation Error}\\
|
||||
\subtext{$x$ is sampled i.i.d. from unseen data, not in $\mathcal{D}$}
|
||||
$$
|
||||
l\Bigl( \hat{f}(x), f^*(x) \Bigr) = \Bigl( \hat{f}(x)-f^*(x) \Bigr)^2
|
||||
$$
|
||||
|
||||
As $f^*$ is usually unknown, this is used instead:
|
||||
|
||||
\definition \textbf{Prediction Error}\\
|
||||
\subtext{$\hat{f}\in F,\quad (x,y) \in \mathcal{D}'$}
|
||||
$$
|
||||
l\Bigl(\hat{f}(x),y\Bigr) = \Bigl(\hat{f}(x) - y\Bigr)^2
|
||||
$$
|
||||
|
||||
To model generalization we use probability theory.
|
||||
|
||||
\textbf{Assumption}: $X \overset{\text{i.i.d.}}{\sim} \P_X$, $Y \overset{\text{i.i.d.}}{\sim} \P_Y$
|
||||
|
||||
Our goal is now to approximate $\E_X\Bigl[ l\bigl( f(X), f^*(X) \bigr) \Bigr]$ for possible choices of $f$.
|
||||
|
||||
\textbf{Assumption}: $y = f^*(x) + \epsilon$ (Noise model)
|
||||
\begin{align*}
|
||||
\epsilon &\sim \P_\epsilon \\
|
||||
\E[\epsilon] &= 0 \\
|
||||
\V[\epsilon] &= \E[\epsilon^2] - \underbrace{\E[\epsilon]^2}_{=0} = \E[\epsilon^2]
|
||||
\end{align*}
|
||||
|
||||
This defines a cond. distribution of $Y$ given $X$:
|
||||
$$
|
||||
\P_{Y|X}\Bigl[ Y \leq a \Bigr] = \P_\epsilon\Bigl[ \epsilon \leq a - f^*(x) \Bigr]
|
||||
$$
|
||||
|
||||
\definition \textbf{Generalization Error} (Population Risk)\\
|
||||
\subtext{$f: \mathcal{X}\to\R,\quad (X,Y) \sim \P_{X,Y}$}
|
||||
$$
|
||||
L\Bigl( f;\P_{X,Y} \Bigr) = \E_{X,Y}\Bigl[ l\bigl(f(X), Y\bigr) \Bigr]
|
||||
$$
|
||||
|
||||
The advantage here is that $f^*$ isn't required to exist. If a $f^*$ is still assumed, the result is very intuitive:
|
||||
$$
|
||||
\E_{X,Y}\Bigl[ f^*(X),Y \Bigr] = \E_{X,Y}\Bigl[ \bigl(f^*(X) - Y\bigr)^2 \Bigr] = \V[\epsilon]
|
||||
$$
|
||||
|
||||
Another advantage is that a good generalization error of $\hat{f}$ implies it is a good estimation of $f^*$, i.e. minimalizes expected estimation error.
|
||||
|
||||
\lemma \textbf{Generalization Error approximates Estimation error}\\
|
||||
\subtext{$f^*(x) = \E[Y \sep X=x]$}
|
||||
$$
|
||||
L\Bigl( \hat{f}; \P_{X,Y}\Bigr) = \underbrace{\E_X\Bigl[ \bigl(\hat{f}(X)-f^*(X)\bigr)^2 \Bigr]}_\text{Estimation error} + \underbrace{\E_{X,Y}\Bigl[ \bigl(f^*(X)-Y\bigr)^2 \Bigr]}_\text{Irreducible noise error, $\V[\epsilon]$}
|
||||
$$
|
||||
|
||||
\subsection{Approximating generalization error}
|
||||
|
||||
\textbf{Problem}: We don't have $\P_{X,Y}$ and can't evaluate $L\Bigl(\hat{f};\P_{X,Y}\Bigr)$.
|
||||
|
||||
\textbf{Solution:} Approximation, using $\mathcal{D}$.
|
||||
|
||||
\textbf{Assumption}: $\forall (x_i,y_i) \in \mathcal{D}:\quad (x_i,y_i) \overset{\text{i.i.d.}}{\sim} \P_{X,Y}$\\
|
||||
\subtext{In practice, this might not be true. $\mathcal{D}$ can be biased.}
|
||||
$$
|
||||
\forall (x_i,y_i) \in \mathcal{D}:\quad y_0 = f^*(x_i) + \epsilon_i \quad \text{\footnotesize\color{gray} (Follows from assumption)}
|
||||
$$
|
||||
|
||||
\method \textbf{Using Training Error}\\
|
||||
\smalltext{An intuitive approach is using the training error.}
|
||||
$$
|
||||
\hat{f}_\mathcal{D} = \underset{f\in F}{\text{arg min}}\Bigl( L(f;\mathcal{D}) \Bigr) = \underset{f\in F}{\text{arg min}}\Biggl( \frac{1}{n}\sum_{i=1}^{n}l\Bigl(f(x_i), y_i\Bigr) \Biggr)
|
||||
$$
|
||||
Generally, this isn't a good estimation and for $\mathcal{D}' \neq \mathcal{D}$:\\
|
||||
$L(\hat{f}_\mathcal{D};\mathcal{D}') > L(\hat{f}_\mathcal{D};\mathcal{D})$ usually, since $\hat{f}_\mathcal{D}$ is biased for $\mathcal{D}$.
|
||||
|
||||
\newpage
|
||||
|
||||
\method \textbf{Training/Test Split}\\
|
||||
\smalltext{Split $\mathcal{D}$ into $\mathcal{D}_\text{train},\mathcal{D}_\text{test}$. Find $\hat{f}_{\mathcal{D}_\text{train}}$}
|
||||
$$
|
||||
L(\hat{f}_{\mathcal{D}_\text{train}};\mathcal{D}_\text{test}) = \frac{1}{|\mathcal{D}_\text{test}|}\sum_{(x,y)\in\mathcal{D}_\text{test}} l\Bigl( \hat{f}_{\mathcal{D}_\text{train}}(x), y \Bigr)
|
||||
$$
|
||||
|
||||
We can apply the law of large numbers:
|
||||
{\footnotesize
|
||||
$$
|
||||
\underset{|\mathcal{D}_\text{test}|\to\infty}{\lim}\Biggl( \frac{1}{|\mathcal{D}_\text{test}|}\sum_{(x,y)\in\mathcal{D}_\text{test}} l\Bigl( \hat{f}_{\mathcal{D}_\text{train}}(x), y \Bigr)\Biggr) = \E_{X,Y}\Bigl[ l\bigl( \hat{f}_{\mathcal{D}_\text{train}}(X),Y \bigr) \Bigr]
|
||||
$$
|
||||
}
|
||||
|
||||
{\footnotesize
|
||||
\remark Assume $\hat{f}$ is trained on $\mathcal{D}_\text{train}$ and chosen (among other $f$) for its low error on $\mathcal{D}_\text{test}$. Note how now $\hat{f}$ depends on \textit{both} $\mathcal{D_\text{test}}$ and $\mathcal{D}_\text{train}$. Now, the prerequisites for the law of large numbers is no longer satisfied. $\hat{f}$ is \textit{not} independent from $\mathcal{D}_\text{test}$ and
|
||||
$$
|
||||
\frac{1}{|\mathcal{D}_\text{test}|}\sum_{(x,y)\in\mathcal{D}_\text{test}} l\Bigl(\hat{f}(x),y\Bigr)
|
||||
$$
|
||||
is \textit{not} a sum of indep. quantities. It might not converge to $\E_{X,Y}\bigl[ f(X),Y \bigr]$ anymore.
|
||||
}
|
||||
|
||||
\textbf{Problem}: We can't model gen. error anymore if we use $\mathcal{D}_\text{test}$ for model selection.
|
||||
|
||||
\textbf{Solution}: We add a third split $\mathcal{D}_\text{valid}$.
|
||||
|
||||
{\footnotesize
|
||||
\remark This approach works. We can use $\mathcal{D}_\text{test}$ to find a good model and can estimate the gen. error using $\mathcal{D}_\text{valid}$. The problem now is we're losing a lot of data to $\mathcal{D}_\text{test}, \mathcal{D}_\text{valid}$ that isn't used for training.
|
||||
}
|
||||
|
||||
\subsection{Cross Validation}
|
||||
|
||||
Cross Validation doesn't require $\mathcal{D}_\text{test}$ anymore.
|
||||
@@ -361,3 +361,62 @@ $$
|
||||
|
||||
\remark \textbf{k-Means}. For k-means, $\mathbf{W}$ contains cluster centroids and $\mathbf{Z}$ the cluster assignments.
|
||||
}
|
||||
\subsection{Kernel PCA}
|
||||
|
||||
Like in supervised learning, we can again use non-linear feature maps and kernels.\\
|
||||
\subtext{Note this isn't covered in the IML script, only in the lectures.}
|
||||
|
||||
\subsubsection{Kernel PCA in one dimension}
|
||||
|
||||
The optimal solution for PCA with $k=1$ was:\\
|
||||
\subtext{$x_i$ are rows of $\mathbf{X}$}
|
||||
$$
|
||||
w^* = \underset{\Vert w\Vert_2=1}{\text{arg max}}\ w^\top \mathbf{X}^\top \mathbf{X}w = \underset{\Vert w\Vert_2=1}{\text{arg max}} \sum_{i=1}^{n}\bigl( w^\top x_i \bigr)
|
||||
$$
|
||||
\subtext{Remember $\Sigma = \frac{1}{n}\mathbf{X}^\top\mathbf{X}$ and that $\frac{1}{n}$ isn't relevant for optimization.}
|
||||
|
||||
\definition \textbf{Kernel PCA} $k=1$
|
||||
$$
|
||||
\underset{\alpha^\top \mathbf{K} \alpha=1}{\text{arg max}}\ \alpha^\top \mathbf{K}^\top\mathbf{K}\alpha
|
||||
$$
|
||||
The derivation for this is straightforward:
|
||||
|
||||
\newpage
|
||||
|
||||
We can apply feature maps: $\displaystyle w = \sum_{i=1}^{n}\alpha_i \phi(x_i)$ and find:
|
||||
\begin{align*}
|
||||
&\underset{\Vert w\Vert_2=1}{\text{arg max}} \sum_{i=1}^{n}\bigl( w^\top x_i \bigr) \\
|
||||
&= \underset{\Vert w\Vert_2=1}{\text{arg max}} \sum_{i=1}^{n}\biggl( \sum_{j=1}^{n} \alpha_j\phi(x_k)^\top \phi(x_i) \biggr) & \text{(def. $w$)} \\
|
||||
&= \underset{\Vert w\Vert_2=1}{\text{arg max}} \sum_{i=1}^{n}\biggl( \sum_{j=1}^{n} \alpha_j k(x_j, x_i) \biggr) & \text{(introduce $k$)} \\
|
||||
&= \underset{\Vert w\Vert_2=1}{\text{arg max}} \sum_{i=1}^{n}\Bigl( \alpha^\top \mathbf{K}_i \Bigr)^2 & \text{(notation)} \\
|
||||
&= \underset{\Vert w\Vert_2=1}{\text{arg max}}\ \alpha^\top \mathbf{K}^\top\mathbf{K}\alpha
|
||||
\end{align*}
|
||||
Finally we can use that $\Vert w \Vert^2 = \alpha^\top \mathbf{K} \alpha$ to get:
|
||||
|
||||
The optimal solution uses the Eigendecomposition for $\mathbf{K}$.\\
|
||||
\subtext{Analogous to PCA, where we used the Eigendecomposition for $\Sigma$.}
|
||||
|
||||
\lemma \textbf{Eigendecomposition of $\mathbf{K}$} $\quad \mathbf{K} = \lambda_i v_i v_i^\top$
|
||||
|
||||
\theorem \textbf{Kernel PCA solution for} $k=1$
|
||||
$$
|
||||
\alpha^* = \frac{1}{\sqrt{\lambda_1}} v_1
|
||||
$$
|
||||
|
||||
\subsubsection{Kernel PCA in general}
|
||||
|
||||
Again, the result from $k=1$ generalizes.
|
||||
|
||||
\theorem \textbf{Kernel PCA General Solution}\\
|
||||
\smalltext{$v_i$ are the EV of $\mathbf{K}$, $\lambda_1 \geq \ldots \geq \lambda_n \geq 0$}
|
||||
$$
|
||||
\alpha^{(i)} = \frac{1}{\sqrt{\lambda_i}}v_i
|
||||
$$
|
||||
{\footnotesize
|
||||
\remark The $\alpha^{(i)}$ are called \textit{Kernel Principal Components}.
|
||||
}
|
||||
|
||||
To project new points $x \mapsto z$ we can use:
|
||||
$$
|
||||
z_i = \sum_{i=1}^{n}\alpha_j^{(i)}k(x_j,x)
|
||||
$$
|
||||
Reference in New Issue
Block a user