diff --git a/semester6/iml/main.pdf b/semester6/iml/main.pdf index 3aa57ac..a884245 100644 Binary files a/semester6/iml/main.pdf and b/semester6/iml/main.pdf differ diff --git a/semester6/iml/parts/06_kernels.tex b/semester6/iml/parts/06_kernels.tex index 26222ce..002c67d 100644 --- a/semester6/iml/parts/06_kernels.tex +++ b/semester6/iml/parts/06_kernels.tex @@ -86,12 +86,18 @@ $$ \end{rcases*} \quad k(x,x') = \Bigl\langle \psi\bigl( \phi(x) \bigr), \psi\bigl( \phi(x') \bigr) \Bigr\rangle $$ - \item Kernels can be added in 2 ways, yielding a kernel + \item Kernels can be added in 2 ways, yielding a kernel\\ + \subtext{$k_1: \R^d \times \R^d \to \R,\ k_2: \R^{d'}\times\R^{d'}\to\R,\quad$(ii) assumes $d=d'$} \begin{align*} \text{(i)}\quad & k\Bigl( (x,y),(x',y') \Bigr) &= k_1(x,x') + k_2(y,y') \\ \text{(ii)}\quad & k(x,x') &= k_1(x,x') + k_2(x,x') \end{align*} - \item Kernels can be multiplied in 2 ways, yielding a kernel + \item Kernels can be multiplied in 2 ways, yielding a kernel\\ + \subtext{$k_1: \R^d \times \R^d \to \R,\ k_2: \R^{d'}\times\R^{d'}\to\R,\quad$(ii) assumes $d=d'$} + \begin{align*} + \text{(i)}\quad & k\Bigl( (x,y),(x',y') \Bigr) &= k_1(x,x') \cdot k_2(y,y') \\ + \text{(ii)}\quad & k(x,x') &= k_1(x,x') \cdot k_2(x,x') + \end{align*} \end{enumerate} \lemma \textbf{Non-negative Taylor Series}\\ @@ -100,4 +106,40 @@ $$ k(x,x') = g(\langle x,x' \rangle) \text{ is a valid kernel} $$ +\subsubsection{Commonly used Kernels} + +\definition \textbf{Polynomial} $g\bigl(\langle x, x' \rangle\bigr) = (1+x)^m$\\ +\subtext{Where $m$ decides the maximum polynomial degree.} + +\definition \textbf{Radial Base Function} (RBF)\\ +\subtext{$p,\alpha$ are parameters, $\tau$ is called \textit{bandwith parameter}.} +$$ + k(x,x') = \exp\biggl( -\frac{\Vert x-x'\Vert_p^\alpha}{\tau} \biggr) +$$ + +The RBF kernel for some special $p,\alpha$ is named: + +\definition \textbf{Gaussian}\\ +\subtext{Sometimes used synonymously as just "RBF".} +$$ + k(x,x') = \exp\biggl( -\frac{\Vert x-x'\Vert_2^2}{\tau} \biggr) +$$ + +\definition \textbf{Laplacian}\\ +\subtext{Sometimes defined with $\alpha=1$, e.g. in \textit{scikit-learn}.} +$$ + k(x,x') = \exp\biggl( -\frac{\Vert x-x'\Vert_1^2}{\tau} \biggr) +$$ + +Gaussian (top) and Laplacian (bottom): + +\begin{center} + \includegraphics[width=0.4\linewidth]{resources/RBFkernels.png}\\ + \subtext{\textit{Introduction to Machine Learning (2026), p. 173}} +\end{center} + +{\footnotesize + \remark The feature space of the RBF kernels has $\dim(\mathcal S_\text{RBF})=\infty$. +} + % Add Polynomial, RBF (Gaussian, Laplacian) \ No newline at end of file diff --git a/semester6/iml/parts/08_unsupervised.tex b/semester6/iml/parts/08_unsupervised.tex index c133ee5..13646bb 100644 --- a/semester6/iml/parts/08_unsupervised.tex +++ b/semester6/iml/parts/08_unsupervised.tex @@ -165,4 +165,102 @@ $$ There are several other methods to do this, based e.g. on concepts from information theory. -\subsection{Principal Component Analysis} \ No newline at end of file +\newpage + +\subsection{Principal Component Analysis} + +\textbf{Motivation}: For $\mathcal D = \{x_i\}_{i=1}^n,x_i\in\R^d$, we'd like a low-dimentional representation with $k \ll d$, which we call \textit{embeddings} $\{z_i\}_{i=1}^n, z_i\in\R^k$.\\ +\subtext{e.g. for performance, memory, noise removal, visualizations...} + +{\footnotesize + \textbf{Intuition}: High dimensional data usually has many redundancies \& highly correlated features. Dimensionality Reduction in practice preserves most substantial data. This idea is formalized as the \textit{Manifold Hypothesis}. +} + +\subsubsection{PCA in one dimension} + +\method \textbf{PCA $k=1$}\\ +\smalltext{Find $w^*\in\R^d$ and $z_1^*,\ldots,z_2^*\in\R$ for dimensionality reduction $d=1$:} +$$ + w^*,z_1^*,\ldots,z_n^* = \underset{w\in\R^d : \Vert w\Vert_2 = 1}{\min}\sum_{i=1}^{n}\underbrace{\Bigl\Vert x_i-\overbrace{z_iw}^{\approx x} \Bigr\Vert}_\text{Reconstruction Error} +$$ + +\begin{center} + \includegraphics[width=0.35\linewidth]{resources/pca1d.png}\\ + \subtext{Introduction to Machine Learning (2026), p. 223} +\end{center} + +\lemma \textbf{PCA $k=1$ (Variance Matrix)}\\ +\smalltext{Alternative formulation.} +$$ + w^* = \underset{\Vert w \Vert_2=1}{\text{arg max}}\ w^\top \Sigma w +$$ + +{\footnotesize + \textbf{Intuition}: $w$ can be thought of as the subspace (a line for $d=1$) we want to project onto. The optimal $w^*$ aligns with the direction maximizing the \textit{empirical variance} $\Sigma$ of the projected data. +} + +\definition \textbf{Empirical Covariance} $\displaystyle\Sigma = \frac{1}{n}\sum_{i=1}^{n}x_ix_i^\top = \frac{1}{n}X^\top X$ + +\lemma \textbf{Eigendecomposition} $\displaystyle\Sigma = \sum_{i=1}^{d}\lambda_iv_iv_i^\top$\\ +\subtext{Eigenvalues $\lambda_1 > \ldots > \lambda_d > 0$,\\ +eigenvectors $v_i$ forming an orthonormal Basis of $\R^d$} + +\theorem \textbf{PCA Solution for $k=1$}\\ +\smalltext{$v$ is the Eigenvector associated with the largest $\lambda$ for $\Sigma$.} +$$ + w^* = v_1 \quad \text{solves} \quad w^* = \underset{\Vert w \Vert_2=1}{\text{arg max}}\ w^\top \Sigma w +$$ + +\subsubsection{Generalized PCA} + +Luckily, the case $k=1$ generalizes: + +\theorem \textbf{PCA General Solution}\\ +\smalltext{Here, $\mathbf{W}\in\R^{d\times k}$ and $z_i^* \in \R^k$ for parameter $k$} +$$ + \mathbf{W}^*,z_1^*,\ldots,z_n^* = \underset{\mathbf{W}\in\R^{d\times k}: \mathbf{W}^\top\mathbf{W}=\mathbf{I}}{\text{arg min}} \sum_{i=1}^{n} \Bigl\Vert x_i - \mathbf{W}z_i \Bigr\Vert^2 +$$ +is solved using +$$ + \mathbf{W}^* = \Bigl( v_1 \| \cdots \| v_k \Bigr), \qquad z_i^* = \mathbf{W}^{*\top}x_i +$$ + +{\footnotesize + \textbf{Intuition}: In words, the basis for the $k$-dim. subspace minimizing reconstruction error is given using the first $k$ eigenvectors of $\Sigma$. +} + +\lemma \textbf{PCA is an orthohgonal projection} +$$ + P: \R^d\to\R^d,\qquad x \mapsto \mathbf{W}^*\mathbf{W}^*x +$$ + +\lemma \textbf{Connection to SVD}\\ +\smalltext{The First $k$ EV of $\Sigma$ are the first $k$ right singular vectors of $\mathbf{X}$.} +$$ + \mathbf{X} = \mathbf{USV}^\top \qquad n\cdot\Sigma = \mathbf{VS}^\top \mathbf{SV}^\top +$$ + +{\footnotesize + \remark Non-linear methods and kernels can be applied to PCA the same way they are applied to regression. +} + +\newpage +\subsubsection{Connection to k-Means} + +k-Means (Clustering) can be reformulated: $z_1,\ldots,z_n \in E_k$ where $E_k = \{e_1,\ldots,e_n\}$ holds unit vectors $e_i$. +$$ + \mathbf{W}^*,z_1^*,\ldots,z_n^* = \underset{\mathbf{W}\in\R^{d\times k}: \mathbf{W}^\top\mathbf{W}=\mathbf{I}}{\text{arg min}} \sum_{i=1}^{n} \Bigl\Vert x_i - \mathbf{W}z_i \Bigr\Vert^2 +$$ +This is the same as PCA, only the space for $z^*$ changes. Both PCA and k-Means are special forms of a more general technique: + +\definition \textbf{Matrix Factorization Methods}\\ +\smalltext{General class of techniques for decomp. of $\mathbf{X}$} +$$ + \mathbf{X} \approx \mathbf{WZ} +$$ + +{\footnotesize + \remark \textbf{PCA}. For PCA, $\mathbf{W}$ is an orthonormal Basis of the optimal $k$-dimenional subspace, $\mathbf{Z}$ are the coefficients for the projection. + + \remark \textbf{k-Means}. For k-means, $\mathbf{W}$ contains cluster centroids and $\mathbf{Z}$ the cluster assignments. +} diff --git a/semester6/iml/resources/RBFkernels.png b/semester6/iml/resources/RBFkernels.png new file mode 100644 index 0000000..d171dfa Binary files /dev/null and b/semester6/iml/resources/RBFkernels.png differ diff --git a/semester6/iml/resources/pca1d.png b/semester6/iml/resources/pca1d.png new file mode 100644 index 0000000..ad7245f Binary files /dev/null and b/semester6/iml/resources/pca1d.png differ