[IML] Examples

semester6/iml/parts/00_intro.tex

@@ -1,3 +1,3 @@
-\smalltext{This is a summary of the 2026 IML Script (FS26). All content from the script is (briefly) covered. There may be errors.}
+\smalltext{This is a summary of the 2026 IML Script (FS26). All content from the script is (briefly) covered. There may be errors. Images belong to the IML team.}
 
 \subtext{Wherever $\Vert\cdot\Vert_p$ isn't specified, $p=2$.}

semester6/iml/parts/07_networks.tex

@@ -3,7 +3,7 @@ Note how the features $x, \phi(x)$ are predetermined. Why not learn them?
 
 \textbf{New Optimization Problem}:
 
-The new join-optimization problem, for $w$ and $\phi$:\\
+The new joint-optimization problem, for $w$ and $\phi$:\\
 \subtext{$\Theta$ is a set of parameters for $\phi$}
 $$
     \hat{w} = \underset{w\in\R^m,\Theta\in\R^{m\times d}}{\text{arg min}}\Biggl( \frac{1}{n}\sum_{i=1}^n l\Bigl( w^\top \phi(x_i;\Theta),y_i \Bigr) \Biggr)

semester6/iml/parts/08_unsupervised.tex

@@ -95,6 +95,97 @@ This is an iterative method to find the cluster centers.
     \remark Each iteration is in $\mathcal{O}\bigl( nkd \bigr)$.
 }
 
+{\footnotesize
+    \textbf{Example}: Consider $x_{1,2,3} = \begin{bmatrix}
+        2 \\
+        1
+    \end{bmatrix},
+    \begin{bmatrix}
+        -1 \\
+        -1
+    \end{bmatrix},
+    \begin{bmatrix}
+        -2 \\
+        -4
+    \end{bmatrix}$ for which we'd like $k=2$ clusters. We choose: 
+    $$\mu_1^{(0)}= \begin{bmatrix}
+        1 \\
+        1
+    \end{bmatrix} \qquad \mu_2^{(0)} = \begin{bmatrix}
+        -1 \\
+        -1
+    \end{bmatrix}
+    $$
+    For Lloyd's Heuristic, the initial cluster assignments $z^{(1)}_{1,2,3}$:
+    $$
+        z_1^{(1)} = \underset{j\in\{1,2\}}{\text{arg min}} \Biggl( \Biggl\Vert \begin{bmatrix}
+            2 \\
+            1
+        \end{bmatrix} - \begin{bmatrix}
+            1 \\
+            1
+        \end{bmatrix} \Biggr\Vert, \Biggl\Vert \begin{bmatrix}
+            2 \\
+            1
+        \end{bmatrix} - \begin{bmatrix}
+            -1 \\
+            -1
+        \end{bmatrix} \Biggr\Vert  \Biggr) = 1
+    $$
+    $$
+        z_2^{(1)} = \underset{j\in\{1,2\}}{\text{arg min}} \Biggl( \Biggl\Vert \begin{bmatrix}
+            -1 \\
+            -1
+        \end{bmatrix} - \begin{bmatrix}
+            1 \\
+            1
+        \end{bmatrix} \Biggr\Vert, \Biggl\Vert \begin{bmatrix}
+            -1 \\
+            -1
+        \end{bmatrix} - \begin{bmatrix}
+            -1 \\
+            -1
+        \end{bmatrix} \Biggr\Vert  \Biggr) = 2
+    $$
+        $$
+        z_2^{(1)} = \underset{j\in\{1,2\}}{\text{arg min}} \Biggl( \Biggl\Vert \begin{bmatrix}
+            -2 \\
+            -4
+        \end{bmatrix} - \begin{bmatrix}
+            1 \\
+            1
+        \end{bmatrix} \Biggr\Vert, \Biggl\Vert \begin{bmatrix}
+            -2 \\
+            -4
+        \end{bmatrix} - \begin{bmatrix}
+            -1 \\
+            -1
+        \end{bmatrix} \Biggr\Vert  \Biggr) = 2
+    $$
+    And then the updated cluster centers:
+    $$
+        \mu_1^{(1)} = \frac{1}{1}\Biggl( \begin{bmatrix}
+            -1 \\
+            -1
+        \end{bmatrix} \Biggr) = \begin{bmatrix}
+            -1 \\
+            -1
+        \end{bmatrix}
+    $$
+    $$
+        \mu_2^{(1)} = \frac{1}{2}\Biggl( \begin{bmatrix}
+            -1 \\
+            -1
+        \end{bmatrix} + \begin{bmatrix}
+            -2 \\
+            -4
+        \end{bmatrix} \Biggr) = \frac{1}{2}\begin{bmatrix}
+            -3  \\
+            -5
+        \end{bmatrix}
+    $$
+}
+
 % Continue with convergence analysis, k-means++
 
 \subsubsection{Convergence}

semester6/iml/parts/10_GMM.tex

@@ -83,6 +83,20 @@ $$
     \end{align*}
 \end{itemize}
 
+{\footnotesize
+    \textbf{Example}: Let $Y \in \{1,2,3\}$ be the clusters, we'd like to assign $X \in \{0,1\}^5$. We're using \textit{Soft Expectation Maximization} to find the cluster assignments $\gamma_j(x_i)$. We assume $\P[X_i = 1 \sep Y]$ follows a $\text{Ber}(p)$ distribution.
+    
+    We'd like to find $\gamma_3(x_2) = \P[Y=3 \sep X=x_2]$. The E-step is:
+    $$
+    \P[Y=3 \sep X=x_2] = \frac{\P[X=x_2\sep Y=3]\cdot \overbrace{\P[Y=3]}^{w_3}}{\sum_{i=1}^{3}\P[X=x_2\sep Y=i]\cdot\P[Y=i]}
+    $$
+    For the M-step, we update the assignments via MLE. For $\text{Ber}(p)$, this is the \textit{sample mean}.
+    $$
+    \P[X_i=1 \sep Y=1] = \frac{\sum_{k=1}^{3}\mathbb{I}_{\{x_k^i=1\}}\cdot\P[Y=1\sep X=x_k]}{\sum_{k=1}^3\P[Y=1\sep X=x_k]}
+    $$
+    Where we use $x\in \R^5$
+}
+
 \subsubsection{Convergence}
 
 EM does not generally guarantee global convergence.