diff --git a/semester6/iml/main.pdf b/semester6/iml/main.pdf index a4eb1a4..2d87072 100644 Binary files a/semester6/iml/main.pdf and b/semester6/iml/main.pdf differ diff --git a/semester6/iml/main.tex b/semester6/iml/main.tex index 3048283..ad4e164 100644 --- a/semester6/iml/main.tex +++ b/semester6/iml/main.tex @@ -15,31 +15,42 @@ \input{parts/01_regression.tex} \newpage -\section{Classification} -\input{parts/02_classification.tex} +\section{Optimization} +\input{parts/02_optimization.tex} \newpage +\section{Model Selection} +\input{parts/03_model_selection.tex} + +\newpage +\section{Regularization} +\input{parts/04_regularization.tex} + +\newpage +\section{Classification} +\input{parts/05_classification.tex} + \section{Kernels} -\input{parts/03_kernels.tex} +\input{parts/06_kernels.tex} \newpage \section{Neural Networks} -\input{parts/04_networks.tex} +\input{parts/07_networks.tex} \newpage \section{Unsupervised Learning} -\input{parts/05_unsupervised.tex} +\input{parts/08_unsupervised.tex} \newpage \section{Probabilistic Modelling} -\input{parts/06_probabilistic.tex} +\input{parts/09_probabilistic.tex} \newpage \section{Gaussian Mixture Models} -\input{parts/07_GMM.tex} +\input{parts/10_GMM.tex} \newpage \section{Language Modeling} -\input{parts/08_LLM.tex} +\input{parts/11_LLM.tex} \end{document} diff --git a/semester6/iml/parts/01_regression.tex b/semester6/iml/parts/01_regression.tex index 8e1dbdd..2ecc993 100644 --- a/semester6/iml/parts/01_regression.tex +++ b/semester6/iml/parts/01_regression.tex @@ -32,6 +32,8 @@ $$ L(f) := \frac{1}{n}\sum_{i=1}^{n} l\bigl( f(x_i), y \bigr) $$ +\newpage + \subsection{Multiple Linear Regression} \textbf{Multiple Linear Regression} directly uses the $x \in \R^d$. \\ @@ -44,9 +46,7 @@ Here, $F_\text{affine} = \bigl\{ f(x) = w^\top x + w_0 \big| w \in \R^d, w_0 \in instead search in $F_\text{linear} = \{ f(x) = \hat{w}^\top x | \hat{w} \in \R^{d+1} \}$ } -\newpage - -\textbf{Loss Functions} +\subsection{Loss Functions} \definition \textbf{Squared Loss} $\quad l\bigl( f(x),y \bigr) := \bigl( f(x) - y \bigr)^2$\\ \subtext{Most common Loss Function, but sensitive to outliers.} @@ -73,9 +73,10 @@ $$ \newpage -\textbf{Linear Regression} +\subsection{The Normal Equation} +The normal equation is the basis for the closed form solution of linear regression. (square loss) -To find $\hat{f} := \underset{f \in F_\text{linear}}{\text{arg min}} L(f)$ we just look for $w \in \R^d$. +To find $\hat{f} := \underset{f \in F_\text{linear}}{\text{arg min}} L(f)$ we look for ideal weights $\hat{w} \in \R^d$. $$ \hat{w} := \underset{w \in \R^d}{\text{arg min}} L(f_w) = \frac{1}{n}\sum_{i=1}^{n}\underbrace{\Bigl( y_i - w^\top x_i \Bigr)^2}_{l\bigl(f(x_i), y_i\bigr)} $$ @@ -87,15 +88,74 @@ $$ $$ \subtext{The factor $\frac{1}{n}$ is irrelevant for Optimization, it doesn't depend on $w$} -So we find the usual problem: +So we find a problem familiar from linear algebra: $$ \hat{w} = \underset{w \in \R^d}{\text{arg min}} \bigl\Vert y-Xw \bigr\Vert^2 $$ The solution is a stationary point, so: $$ - \nabla_w \bigl\Vert y-Xw \bigr\Vert^2 = 2X^\top(X\hat{w}-y) = 0 + \nabla_w \bigl\Vert y-Xw \bigr\Vert^2 = 2X^\top(X\hat{w}-y) \overset{!}{=} 0 $$ -Which yields the \textbf{Normal Equation} from linear algebra. +Which yields the \textbf{Normal Equation}. $$ - X^\top X\hat{w} = X^\top y + \mathbf{X}^\top\mathbf{X}\hat{w} = \mathbf{X}^\top y $$ + +\theorem \textbf{Geometric Interpretation}\\ +$\hat{y} = \mathbf{X}\hat{w}$ for $\hat{w}$ solving $\mathbf{X}^\top\mathbf{X}\hat{w} = \mathbf{X}^\top y$ is the orthogonal projection of $y$ onto $\text{span}(\mathbf{X})$. + +\begin{center} + \includegraphics[width=0.275\textwidth]{resources/normalEquation.png}\\ + \subtext{Introduction to Machine Learning (2026), p. 74} +\end{center} + +\newpage + +\subsection{Closed Form Solution} + +\theorem \textbf{Minimum-Norm Solution}\\ +$$ + \hat{w} = \Bigl(\mathbf{X}^\top\mathbf{X}\Bigr)^\dagger\mathbf{X}^\top y = \mathbf{X}^\top\Bigl(\mathbf{X}^\top\mathbf{X}\Bigr)^{-1} \mathbf{X}^\top y +$$ +$$ + \text{for} \qquad \hat{w} = \underset{w \in \R^d}{\text{arg min}} \bigl\Vert w \bigr\Vert^2 +$$ + +{\footnotesize + \remark The computational cost for this is $\mathcal O (nd^2+d^3)$. +} + +The closed form solution depends on $\text{rank}(\mathbf{X})$. + +Assuming $d \leq n$ and $\text{rank}(\mathbf{X}) = d$: $(\mathbf{X}^\top\mathbf{X})^{-1}$ exists. +$$ + \hat{w} = (\mathbf{X}^\top\mathbf{X})^{-1}\mathbf{X}^\top y \qquad (\text{unique}) +$$ +Assuming $d > n$ or $\text{rank}(\mathbf{X}) < d$ we have $|\ker(\mathbf{X})|=\infty$ and there are infinite solutions. The pseudo-inverse provides the minimum-norm solution: +$$ + \hat{w} = \Bigl(\mathbf{X}^\top\mathbf{X}\Bigr)^\dagger \mathbf{X}^\top y +$$ +{\footnotesize + \remark If $\text{rank}(\mathbf{X})=d$, then $\bigl(\mathbf{X}^\top\mathbf{X}\bigr)^\dagger = \bigl(\mathbf{X}^\top\mathbf{X}\bigr)^{-1}$. +} + +\subsection{Non-Linear Least Squares} + +To expand Linear Regression to Non-linear functions, feature maps are used on $x$: $\phi: \R^d \to \R^p$. +$$ + f_w(x) = \sum_{j=1}^{p} w_j^\top \phi_j(x) +$$ +This induces a function class different from $F_\text{linear}$: +$$ + F_\phi = \biggl\{ f_w(x) = \sum_{j=1}^{p} w_j^\top \phi_j(x) \ \bigg|\ w \in \R^p \biggr\} +$$ +But the optimization problem remains the same:\\ +\subtext{$\Phi \in \R^{n \times p}$ now replaces $\mathbf{X} \in \R^{n \times d}$.} +$$ + \hat{w} = \underset{w\in\R^p}{\text{arg min}} \Bigl\Vert y - \Phi w \Bigr\Vert^2 +$$ + +\begin{center} + \includegraphics[width=0.2\textwidth]{resources/nonlinearLeastSquares.png}\\ + \subtext{Introduction to Machine Learning (2026), p. 79} +\end{center} \ No newline at end of file diff --git a/semester6/iml/parts/02_optimization.tex b/semester6/iml/parts/02_optimization.tex new file mode 100644 index 0000000..260b347 --- /dev/null +++ b/semester6/iml/parts/02_optimization.tex @@ -0,0 +1,54 @@ +\textbf{Problem}: Finding $\hat{w}$ for $l$ with no closed form solution.\\ +\subtext{or if the closed form solution is too expensive to compute.} + +\textbf{Solution}: Iterative optimization methods. + +\begin{algorithm} + \caption{Iterative Optimization} + $t \gets 0$ \; + $w^{(0)} \gets w_\text{initial}$ \; + \SetKwRepeat{Do}{repeat}{until} + \Do{\text{Stopping Criterion}}{ + $w^{(t+1)} \gets w^{(t)} + \tilde{\eta}_t v^{(t)}$ \; + $t \gets t+1$ + } + \textbf{return} $w^{(t)}$ +\end{algorithm} + +{\footnotesize + \notation The update takes the form $\tilde{\eta}_t v^{(t)}$. $v^{(t)}$ is the update direction, $\tilde{\eta}_t$ is the step size. +} + +\subsection{Gradient Descent} + +Intuitively: go in the direction $v^{(t)}$ where $L$ decreases most. + +\lemma $-\nabla L(w^{(t)})$ is the direction of steepest descent.\\ +\subtext{Assuming diff.-able $L$. Provable via Taylor expansion \& Cauchy-Schwarz.} + +\definition \textbf{Gradient Descent Update Step} +\begin{align*} + w^{(t+1)} &= w^{(t)} - \tilde{\eta}_t\cdot \frac{\nabla L(w^{(t)})}{\Vert \nabla L(w^{(t)}) \Vert} & (\text{Normalized}) \\ + w^{(t+1)} &= w^{(t)} - \eta\cdot \nabla L(w^{(t)}) & (\text{Unnormalized}) +\end{align*} + +Unnormalized gradient descent takes advantage of $\Vert \nabla L(w^{(t)}) \Vert$: +{\small + \begin{itemize} + \item $\Vert \nabla L(w^{(t)}) \Vert$ small $\mapsto$ close to stat. point $\mapsto$ small steps. + \item $\Vert \nabla L(w^{(t)}) \Vert$ large $\mapsto$ far from stat. point $\mapsto$ large steps. + \end{itemize} +} +Stopping criterion uses the same idea: $\Vert w^{t} - w^{t+1} \Vert < \epsilon$ or equivalently $\Vert \nabla L(w^{(t)}) \Vert < \epsilon$. + +\begin{algorithm} + \caption{Gradient Descent} + $t \gets 0$ \; + $w^{(0)} \gets w_\text{initial}$ \; + \SetKwRepeat{Do}{repeat}{until} + \Do{\text{$\Vert w^{t} - w^{t+1} \Vert < \epsilon$}}{ + $w^{(t+1)} \gets w^{(t)} - \eta \nabla L(w^{(t)})$ \; + $t \gets t+1$ + } + \textbf{return} $w^{(t)}$ +\end{algorithm} \ No newline at end of file diff --git a/semester6/iml/parts/03_model_selection.tex b/semester6/iml/parts/03_model_selection.tex new file mode 100644 index 0000000..e69de29 diff --git a/semester6/iml/parts/04_regularization.tex b/semester6/iml/parts/04_regularization.tex new file mode 100644 index 0000000..e69de29 diff --git a/semester6/iml/parts/02_classification.tex b/semester6/iml/parts/05_classification.tex similarity index 100% rename from semester6/iml/parts/02_classification.tex rename to semester6/iml/parts/05_classification.tex diff --git a/semester6/iml/parts/03_kernels.tex b/semester6/iml/parts/06_kernels.tex similarity index 100% rename from semester6/iml/parts/03_kernels.tex rename to semester6/iml/parts/06_kernels.tex diff --git a/semester6/iml/parts/04_networks.tex b/semester6/iml/parts/07_networks.tex similarity index 100% rename from semester6/iml/parts/04_networks.tex rename to semester6/iml/parts/07_networks.tex diff --git a/semester6/iml/parts/05_unsupervised.tex b/semester6/iml/parts/08_unsupervised.tex similarity index 100% rename from semester6/iml/parts/05_unsupervised.tex rename to semester6/iml/parts/08_unsupervised.tex diff --git a/semester6/iml/parts/06_probabilistic.tex b/semester6/iml/parts/09_probabilistic.tex similarity index 100% rename from semester6/iml/parts/06_probabilistic.tex rename to semester6/iml/parts/09_probabilistic.tex diff --git a/semester6/iml/parts/07_GMM.tex b/semester6/iml/parts/10_GMM.tex similarity index 100% rename from semester6/iml/parts/07_GMM.tex rename to semester6/iml/parts/10_GMM.tex diff --git a/semester6/iml/parts/08_LLM.tex b/semester6/iml/parts/11_LLM.tex similarity index 100% rename from semester6/iml/parts/08_LLM.tex rename to semester6/iml/parts/11_LLM.tex diff --git a/semester6/iml/resources/nonlinearLeastSquares.png b/semester6/iml/resources/nonlinearLeastSquares.png new file mode 100644 index 0000000..dbcdb70 Binary files /dev/null and b/semester6/iml/resources/nonlinearLeastSquares.png differ diff --git a/semester6/iml/resources/normalEquation.png b/semester6/iml/resources/normalEquation.png new file mode 100644 index 0000000..2c0b770 Binary files /dev/null and b/semester6/iml/resources/normalEquation.png differ