eth-summaries/semester6/iml/parts/08_LLM.tex

\textbf{Problem}: How to build ChatGPT?

\textbf{Assumption}: Producing \& Understanding language are strongly related: If a model can predict how a text continues, it must have understood \textit{something} about language \& meaning.

\definition \textbf{Language Model}: A prob. distrib. over words/sentences

The model should assign high prob. to \textit{plausible} texts: 
$$
    \P\bigl[\text{"I like to eat pasta"}\bigr] > \P\bigl[\text{"I rocks like eat to"}\bigr]
$$

A simple model, exponential in $m$, would be:
$$
    P\bigl( \underbrace{\text{Sentence}}_\text{Length $m$} \bigr) = P\Bigl(\underbrace{X_1=x_1}_\text{1st word}, \underbrace{X_2=x_2}_\text{2nd word},\ldots,X_m=x_m \Bigr)
$$

{\footnotesize
    \remark In practice, \textit{Tokens} are used instead of words. Tokens are "units of meaning" obtained from a sentence, e.g. seperating "don't" into "do" and "n't", to improve processing.
}

\subsection{Autoregressive models}

\textbf{Idea}: Abuse the chain rule of probability.
$$
    P(x_1,x_2,\ldots,x_m) = P(x_1)P(x_2|x_2)\cdots P(x_m|x_1,\ldots,x_{m-1})
$$
This lends itself very naturally to sentence generation:
\begin{align*}
    x_1 \sim P(X_1) \quad x_2 \sim P(X_2\sep X_1=x_1) \quad \ldots
\end{align*}
Essentially: Sample a token (next word) from the cond. distrib. (sentence so far) at each step. This avoids computing the entire joint distribution.

{\footnotesize
    \remark Model outputs are inputs for next prediction: \textit{Autoregressive}.
}

\begin{center}
    \includegraphics[width=0.6\linewidth]{resources/Autoregressor.png}\\
    \footnotesize\color{gray}\textit{Introduction to Machine Learning (2026), p. 277}
\end{center}

\subsubsection{Neural Network Language Models}

Instead of looking at all previous words, only the recent $k$:\\
\subtext{This is called \textit{Context length}}
\begin{align*}
            &P\bigl( X_t=x\sep X_{1:t-1} = x_{1:t-1} \bigr)                                     \\
    \approx &P\bigl(X_t=x\sep X_{t-k:t-1} = x_{t-k:t-1}, \theta\bigr)                           \\
    :=      & \text{Cat}\bigl( x \sep \text{softmax}\bigl( f(x_{t-k:t-1}, \theta) \bigr) \bigr)
\end{align*}

\textbf{Optimization Problem}:
$$
    \underset{\theta}{\min} L(\theta) := \underbrace{-\sum_{t}\log\Bigl( P\bigl( X_t=x_t\sep X_{t-k:t-1} = x_{t-k:t-1}, \theta \bigr) \Bigr)}_\text{Negative Log-Likelihood}  
$$

This is a form of \textit{Self-Supervision}: No labels are needed.\\
In Datasets, the next word $x_t$ is the prediction target.\\
\subtext{Thus: \textit{Any text} can be used as a training set}

\subsubsection{Input Representation}

Preparing inputs roughly follows this idea:
$$
    \text{Language} \underset{\text{Tokenization}}{\mapsto} \text{Tokens} \underset{\text{Vectorization}}{\mapsto} \text{Vector}
$$

\definition \textbf{Word Embedding Matrix} $\quad \mathbf{W}_e \in \R^{N\times d}, d \ll N$\\ 
\subtext{$N$ is the vocabulary size}

Each token receives a position in $d$-dimensional space: Semantically similar tokens are closer.

{\footnotesize
    \remark \textbf{One-Hot Encoding}\\
    Bad for many reasons: dimension $\propto$ vocabulary size, dot product is always zero (every input equally "dissimilar")
}

\newpage
\subsection{Transformers \& Attention}

\textbf{Problem}: How to efficiently model sequences?

{\footnotesize
    \remark Historically, Recurrent Neural Networks (RNNs) were used.\\
    \subtext{RNNS run sequentially: too slow for sequence modelling.}
}

\subsubsection{Single-Head Attention}
The model weighs previous tokens relevant for the next one.\\
\subtext{$w_{e,i}$ is $i$-th row of $\mathbf{W}_e$, $w_{p,i}$ is the positional encoding.}
$$
    z_i = w_{e,i} + w_{p,i}
$$
$\mathbf{W}_q, \mathbf{W}_k, \mathbf{W}_v$ are learnable parameter matrices.\\ 
Intuitively, the model should learn to produce queries \& keys with high scores for relevant token pairs.
$$
    \underbrace{q_i = z_i \mathbf{W}_q}_\text{Query} \quad \underbrace{k_i = z_i \mathbf{W}_k}_\text{Key} \quad \underbrace{v_i = z_i \mathbf{W}_v}_\text{Value}
$$
\definition \textbf{Attention Score}\\
\smalltext{Measures how relevant token $j$ is for predicting token $i$.}
$$
    \text{score}_{i,j} \propto \exp\Biggl( \frac{q_i k_j^\top}{\sqrt{d_k}} + m_{i,j} \Biggr) 
    \qquad m_{i,j} = \begin{cases}
        0 & j \leq i \\
        -\infty & j > i
    \end{cases}
$$
\subtext{$\sqrt{d_k}$ is a scaling factor, $m_{i,j}$ is a mask to prevent scoring future tokens.}

\definition \textbf{Token Update}
$$
    z_i' = \sum_{j=1}^n \text{score}_{i,j} v_j \quad \text{(Weighted sum of values)}
$$
$$
    \mathbf{Z}' = \text{softmax}\Bigl( \frac{\mathbf{Q}\mathbf{K}^\top}{\sqrt{d_k}} + \mathbf{M} \Bigr)\mathbf{V}
$$
\subtext{$\mathbf{Q},\mathbf{K},\mathbf{V} \in \R^{n \times d_k}$ are queries, keys, values. $\mathbf{M}$ is the mask matrix.}

{\footnotesize
    \remark Unlike in RNNs, all tokens can be processed in parallel.
}

\newpage
\subsubsection{Multi-Head Attention}

In practice, multiple attention "heads" are used to capture different aspects of the input.\\
\subtext{Each head $r = 1, \ldots, h$ has its own $\mathbf{W}_r^Q, \mathbf{W}_r^K, \mathbf{W}_r^V$ matrices.}

\definition \textbf{Multi-Head Attention Update}\\
\smalltext{Use $h$ heads in parallel. The concatenation collects all heads' information, and $\mathbf{W}^O$ projects it back to the original dimension $d$.}
$$
    \mathbf{Z}'_r = \text{softmax}\Bigl( \frac{\mathbf{Q}_r\mathbf{K}_r^\top}{\sqrt{d_k}} + \mathbf{M} \Bigr)\mathbf{V}_r
$$
$$
    \text{MultiHead}(\mathbf{Z}) = \text{Concat}(\mathbf{Z}'_1, \ldots, \mathbf{Z}'_h)\mathbf{W}^O
$$
\subtext{For $\mathbf{Q}_r = \mathbf{Z}\mathbf{W}_r^Q$, $\mathbf{K}_r = \mathbf{Z}\mathbf{W}_r^K$, $\mathbf{V}_r = \mathbf{Z}\mathbf{W}_r^V$}

\subsubsection{Transformer}

To build a network from multi-head attention, the transformer block is used. 

\definition \textbf{Transformer Block}\\
\smalltext{A Transformer block consists of a Multi-Head Attention layer followed by a Feed-Forward Neural Network layer. Each layer has residual connections ($\oplus$) and layer normalization.}

\begin{center}
    \includegraphics[width=0.3\linewidth]{resources/Transfomer.png}\\
    \footnotesize\color{gray}\textit{Introduction to Machine Learning (2026), p. 285}
\end{center}

Normalization \& residual connections help prevent vanishing gradients and stabilize training. Intuitively: only learn small corrections instead of relearning the whole token.

\newpage
\subsubsection{Decoding}

This is the process of generating text from the model.

At each step, all tokens (so far) are used as input, returning a probability distribution over the vocabulary. The next token is sampled from it.

{\footnotesize
    \remark During training, the entire sequence is used for the foreward pass (with causal masking). At inference, each token requires a new foreward pass.
}

\method \textbf{Greedy Decoding}\\
\smalltext{At each step, choose the token with the highest probability.}
$$
    x_t = \underset{x}{\text{arg max}}\ P\Bigl(X_t=x\sep X_{1:t-1}=x_{1:t-1}, \theta\Bigr)
$$
\subtext{Fast, but leads to repetitive text.}

\method \textbf{Beam Search}\\
\smalltext{Keep track of the $b$ most likely sequences at each step, then expand by each by one token, keep the best $b$ again.}

\method \textbf{Sampling}\\
\smalltext{Sample the next token from the predicted distribution.}\\
$$
    P\Bigl(X_t=x\sep X_{1:t-1}=x_{1:t-1}\Bigr) = \text{softmax}\Bigl( \frac{f(x_{1:t-1})_x}{\tau} \Bigr)_x
$$
\subtext{$\tau$ is the temperature parameter, controlling the sharpness of the distribution. $\tau \to 0$ mimics greedy decoding, $\tau \to \infty$ leads to uniform sampling.}