diff --git a/semester4/dmdb/data-modelling-databases-summary.pdf b/semester4/dmdb/data-modelling-databases-summary.pdf index 5825fb9..78eeba6 100644 Binary files a/semester4/dmdb/data-modelling-databases-summary.pdf and b/semester4/dmdb/data-modelling-databases-summary.pdf differ diff --git a/semester4/dmdb/parts/04_query-processing/00_sorting/00_intro.tex b/semester4/dmdb/parts/04_query-processing/00_sorting/00_intro.tex index 388fe4f..699d69d 100644 --- a/semester4/dmdb/parts/04_query-processing/00_sorting/00_intro.tex +++ b/semester4/dmdb/parts/04_query-processing/00_sorting/00_intro.tex @@ -1,3 +1,4 @@ +\label{sec:external-sort} Sorting is crucial in DBMS due to the possibility that the user requests sorted data using an \texttt{ORDER BY} clause. In addtion, it can help speed up some operators. diff --git a/semester4/dmdb/parts/04_query-processing/04_join/00_simple-join.tex b/semester4/dmdb/parts/04_query-processing/04_join/00_simple-join.tex index 8756a8f..e0e49ba 100644 --- a/semester4/dmdb/parts/04_query-processing/04_join/00_simple-join.tex +++ b/semester4/dmdb/parts/04_query-processing/04_join/00_simple-join.tex @@ -3,12 +3,14 @@ \caption{SimpleJoin Algorithm} \begin{algorithmic}[1] \Procedure{SimpleJoin}{$R$, $S$} - \For{each tuple $t_R \in R$} - \For{each tuple $t_S \in S$} - \If{$\texttt{pred}(t_R, t_S)$} - \State $\texttt{add}\{ t_R, t_S \}$ to the join output - \EndFor - \EndFor + \For{each tuple $t_R \in R$} + \For{each tuple $t_S \in S$} + \If{$\texttt{pred}(t_R, t_S)$} + \State $\texttt{add}\{ t_R, t_S \}$ to the join output + \EndIf + \EndFor + \EndFor \EndProcedure \end{algorithmic} \end{algorithm} +Here, the max output size is $T_R \cdot T_S$, where $T_X$ is the number of tuples for realtion $X$. diff --git a/semester4/dmdb/parts/04_query-processing/04_join/01_nested-loops.tex b/semester4/dmdb/parts/04_query-processing/04_join/01_nested-loops.tex index e69de29..c018ea1 100644 --- a/semester4/dmdb/parts/04_query-processing/04_join/01_nested-loops.tex +++ b/semester4/dmdb/parts/04_query-processing/04_join/01_nested-loops.tex @@ -0,0 +1,17 @@ +\subsubsection{Nested Loops Join (NLJ)} +\begin{algorithm} + \caption{Nested Loops Join Algorithm} + \begin{algorithmic}[1] + \Procedure{NestedLoopsJoin}{$R$, $S$} + \For{each page $p_R \in R$} + \For{each page $p_S \in S$} + \State Join the tuples on page $p_R$ with the tuples on page $p_S$ (evaluate the predicate for all pairs) + \State Add matching tuples to the join output + \EndFor + \EndFor + \EndProcedure + \end{algorithmic} +\end{algorithm} + +The cost here is $P_R + P_R \cdot P_S$ I/Os, where $P_X$ is the number of pages in the relation $X$. +Furthermore, the DBMS should put the smaller relation on the outer loop to improve performance. diff --git a/semester4/dmdb/parts/04_query-processing/04_join/02_index-nested-loops.tex b/semester4/dmdb/parts/04_query-processing/04_join/02_index-nested-loops.tex index e69de29..a85c83c 100644 --- a/semester4/dmdb/parts/04_query-processing/04_join/02_index-nested-loops.tex +++ b/semester4/dmdb/parts/04_query-processing/04_join/02_index-nested-loops.tex @@ -0,0 +1,19 @@ +\subsubsection{Index Nested Loops Join (INLJ)} +\begin{algorithm} + \caption{Nested Loops Join Algorithm} + \begin{algorithmic}[1] + \Procedure{IndexNestedLoopsJoin}{$R$, $S$} + \For{each page $p_R \in R$} \Comment{Outer loop} + \For{each tuple $t_r \in p_R$} + \State Probe the index on the join attribute of $S$ \Comment{Inner loop} + \State Add matching tuples to the join output + \EndFor + \EndFor + \EndProcedure + \end{algorithmic} +\end{algorithm} + +The cost here is $P_R + T_R \cdot C(I_S)$ I/Os, with $P_X$ the number of pages in the relation $X$, $T_X$ the number of tuples in said relation +and $C(I_X)$ is the cost to probe the index $I_X$ of relation $X$ and depends on the clustering of data and type of index. + +This operation needs three buffer frames. $1$ for $R$, $1$ for output and $1$ to traverse $I_S$ diff --git a/semester4/dmdb/parts/04_query-processing/04_join/03_block-nested-loops.tex b/semester4/dmdb/parts/04_query-processing/04_join/03_block-nested-loops.tex index e69de29..1170aaf 100644 --- a/semester4/dmdb/parts/04_query-processing/04_join/03_block-nested-loops.tex +++ b/semester4/dmdb/parts/04_query-processing/04_join/03_block-nested-loops.tex @@ -0,0 +1,22 @@ +\subsubsection{Block Nested Loops Join (BNLJ)} +\begin{algorithm} + \caption{Block Nested Loops Join Algorithm} + \begin{algorithmic}[1] + \Procedure{BlockNestedLoopsJoin}{$R$, $S$} + \For{all $B - 2$ blocks of pages of $R$} + \For{each page $p_S \in S$} + \State Join the tuples on page $p_S$ with the tuples in the block + \State Add matching tuples to the join output + \EndFor + \EndFor + \EndProcedure + \end{algorithmic} +\end{algorithm} + +We have $B$ buffer frames, of which one is reserved for the pages of $S$ and one is reserved for the output, thus the $B - 2$ blocks. +Each of these blocks houses $P_R / (B - 2))$ pages of $R$, reducing search costs by only scanning $S$ once for every block. + +The cost here is $P_R + (P_R / (B - 2) \cdot P_S$ I/Os. +If however, $R$ fits into memory, we have cost $P_R + P_S$ I/Os. +We can reduce that further by building as second step (right after the first for loop) a in-memory hash table for the tuples in the $R$ pages block. +This reduces the cost of comparisons, however, we need $(B - 2) \cdot f$ memory now. diff --git a/semester4/dmdb/parts/04_query-processing/04_join/04_block-index-nested-loops.tex b/semester4/dmdb/parts/04_query-processing/04_join/04_block-index-nested-loops.tex index e69de29..cc2d981 100644 --- a/semester4/dmdb/parts/04_query-processing/04_join/04_block-index-nested-loops.tex +++ b/semester4/dmdb/parts/04_query-processing/04_join/04_block-index-nested-loops.tex @@ -0,0 +1,28 @@ +\subsubsection{Block Index Nested Loops Join (BINLJ)} +\begin{algorithm} + \caption{Block Index Nested Loops Join Algorithm} + \begin{algorithmic}[1] + \Procedure{BlockIndexNestedLoopsJoin}{$R$, $S$} + \For{all $B - 2$ blocks of pages of $R$} + \State sort the tuples in this block. + \For{each tuple $t_R$ in $B - 2$ block of $R$} + \State Probe the index $I_S$ on the join attribute of $S$ + \State Add matching tuples to the join output + \EndFor + \EndFor + \EndProcedure + \end{algorithmic} +\end{algorithm} + +The cost here is $P_R + T_R \cdot C(I_S)$ I/Os. +We sort the tuples in each buffer frame on the predicate key, which leads to tuples in $R$ with the same key only requiring a single index search. +In addition, tuples with ``similar'' values in $R$ will likely match with keys on the same pages of $I_S$, which will already be in the buffer pool. + +In practice, the difference between BNLJ and NJS is fairly stark. + +\inlineexample $P_R = 500$, $P_S = 1000$, $100$ tuples per page and we have $B = 12$ frames. Then: +\begin{itemize} + \item $\texttt{Cost(NLJ)} = P_R + P_R \cdot P_S = 500 + 500 \cdot 1000 = 500,500$IOs + \item $\texttt{Cost(BNLJ)} = P_R + (P_R / (B - 2)) \cdot P_S = 500 + (500 / 10) * 1000 = 50,500$IOs +\end{itemize} +We only need very little more memory for BNLJ, with a big improvement in performance. diff --git a/semester4/dmdb/parts/04_query-processing/04_join/05_sort-merge-join.tex b/semester4/dmdb/parts/04_query-processing/04_join/05_sort-merge-join.tex index e69de29..854c223 100644 --- a/semester4/dmdb/parts/04_query-processing/04_join/05_sort-merge-join.tex +++ b/semester4/dmdb/parts/04_query-processing/04_join/05_sort-merge-join.tex @@ -0,0 +1,12 @@ +\subsubsection{(Simple) Sort Merge Join (SMJ)} +We first sort $R$ and $S$ on the join attributes with external merge sort (see Section~\ref{sec:external-sort}). We then read the sorted files and merge. +This is of course not great for performance, but if the relations are already sorted, we can skip the first step. + +The cost (if there are \bi{no duplicates}) is $\texttt{Sort}(S) + \texttt{Sort}(R) + P_R + P_S$ I/Os. + +If there are duplicates (i.e. the keys are not unique, say we are joining on a column that doesn't have the UNIQUE constraint) +then we need to move the pointer for the join back on the second relation to make sure we don't miss any tuples. + +The cost now of course isn't linear anymore and worst case quadratic, $\tco{\texttt{Sort}(S) + \texttt{Sort}(R) + P_R \cdot P_S}$. + +Compared to BNLJ, its performance doesn't depend on the amount of memory available, but tends to be a tad slower than BNLJ, if BNLJ is provided with ample memory. diff --git a/semester4/dmdb/parts/04_query-processing/04_join/06_optimized-smj.tex b/semester4/dmdb/parts/04_query-processing/04_join/06_optimized-smj.tex index e69de29..76bcb36 100644 --- a/semester4/dmdb/parts/04_query-processing/04_join/06_optimized-smj.tex +++ b/semester4/dmdb/parts/04_query-processing/04_join/06_optimized-smj.tex @@ -0,0 +1,10 @@ +\subsubsection{Optimized Sort Merge Join ((O)SMJ)} +In this section, we are exploring how we can optimize sort and merge with extra memory. + +Sorting can be optimized using replacement sort, the output is $\frac{P_R + P_S}{2B}$ sorted runs of length $2B$. + +That of course means that the logical optimization is to do a $K$-way merge to merge the sorted runs and join the relations at the same time. +If we have $B$ frames, so we can do a $B-1$ way merge, otherwise, we merge runs from each relation separately to reduce the number of runs +until we have less than $B$ sorted runs, then we perform the join. + +The cost drops to $3 \cdot (P_R + P_S)$ I/Os, but we need memory such that $B^2 \geq \max\{ P_R, P_S \}$ diff --git a/semester4/dmdb/parts/04_query-processing/04_join/07_hash-join.tex b/semester4/dmdb/parts/04_query-processing/04_join/07_hash-join.tex index e69de29..234c5a5 100644 --- a/semester4/dmdb/parts/04_query-processing/04_join/07_hash-join.tex +++ b/semester4/dmdb/parts/04_query-processing/04_join/07_hash-join.tex @@ -0,0 +1,11 @@ +\subsubsection{Hash Join (HJ)} +This works by using a hash function that maps values into $B = P_R \cdot f$ buckets, with $f$ being the Minister of Magic (up until, and including book 5) factor, +also known as the fudge factor, which is additional hash table information stored alongside the tuples. + +We first partition the table $R$ into these buckets. Then, we probe the hash table for all tuples of $S$. + +Ideally, we would want each bucket to contain exactly the same number of pages ($N / (B - 1)$ pages), but due to to hash collisions and duplicate keys, +this is not likely to happen. +To take care of this imbalance, we do another pass with a different hash function. + +We should \textit{NOT} use hash join if the hash table doesn't fit in memory because then it will become VERY inefficient. diff --git a/semester4/dmdb/parts/04_query-processing/04_join/08_partitioned-hash-join.tex b/semester4/dmdb/parts/04_query-processing/04_join/08_partitioned-hash-join.tex new file mode 100644 index 0000000..a7776f9 --- /dev/null +++ b/semester4/dmdb/parts/04_query-processing/04_join/08_partitioned-hash-join.tex @@ -0,0 +1,23 @@ +\subsubsection{Partitioned Hash Join (PHJ)} +If the table doesn't fit into memory, then we can't use a normal hash join because that would be terribly inefficient. + +Instead, we partition both tables into buckets and join the corresponding partitions. + +\begin{algorithm} + \caption{Partitioned Hash Join} + \begin{algorithmic}[1] + \Procedure{PartitionedHashJoin}{$R$, $S$} + \State Partition $R$ into $RP$ partitions, using hash function $h$ on the join key + \State Partition $S$ into $SP$ partitions, using hash function $h$ on the join key + \State Join each partition $RP$ with the corresponding $SP$ partition using BNJL or building a hash table $SP$ or $RP$ in memory. + This hash function should be different from $h$. + \EndProcedure + \end{algorithmic} +\end{algorithm} + +The benefit of this approach is that in the third step, we are only operating on small numbers of records each, which can be done efficiently in memory. + +What we ideally do in the above algorithm is to do \textit{recursive partitioning}, i.e. as already \textit{somewhat} described, we partition each partition again, +if it has more than one page. This further reduces access times. + +The cost for this is $3 \cdot (P_S + P_R)$, thus both OSMJ and PHJ are very similar in performance. diff --git a/semester4/dmdb/parts/04_query-processing/04_join/09_conclusion.tex b/semester4/dmdb/parts/04_query-processing/04_join/09_conclusion.tex index e69de29..183d018 100644 --- a/semester4/dmdb/parts/04_query-processing/04_join/09_conclusion.tex +++ b/semester4/dmdb/parts/04_query-processing/04_join/09_conclusion.tex @@ -0,0 +1,14 @@ +\subsubsection{Conclusion} +We can use all algorithms for equality joins (equi-joins), but if we have equalities over multiple attributes, then: +\begin{itemize} + \item PNL, BNL: work the same + \item INL, BINL: build indexes on one, or all attributes of the equality conditions + \item SMJ, HJ: Sort or hash on the combination of the attributes +\end{itemize} + +For inequality joins: +\begin{itemize} + \item PNL, BNL: work the same + \item INL, BINL: need clustered B+ tree index + \item SMJ, HJ: don't work at all +\end{itemize} diff --git a/semester4/dmdb/parts/04_query-processing/main.tex b/semester4/dmdb/parts/04_query-processing/main.tex index 153a32c..63b54f2 100644 --- a/semester4/dmdb/parts/04_query-processing/main.tex +++ b/semester4/dmdb/parts/04_query-processing/main.tex @@ -21,6 +21,7 @@ \subsection{Aggregation} \input{parts/04_query-processing/03_aggregation.tex} +\newpage \subsection{Join} \input{parts/04_query-processing/04_join/00_simple-join.tex} \input{parts/04_query-processing/04_join/01_nested-loops.tex} @@ -30,6 +31,6 @@ \input{parts/04_query-processing/04_join/05_sort-merge-join.tex} \input{parts/04_query-processing/04_join/06_optimized-smj.tex} \input{parts/04_query-processing/04_join/07_hash-join.tex} -\input{parts/04_query-processing/04_join/08_recursive-partitioning.tex} +\input{parts/04_query-processing/04_join/08_partitioned-hash-join.tex} \input{parts/04_query-processing/04_join/09_conclusion.tex} % \input{parts/04_query-processing/} diff --git a/semester4/dmdb/parts/05_vector-search/00_rag.tex b/semester4/dmdb/parts/05_vector-search/00_rag.tex new file mode 100644 index 0000000..48c13ed --- /dev/null +++ b/semester4/dmdb/parts/05_vector-search/00_rag.tex @@ -0,0 +1 @@ +\subsection{Retrieval Augmented Generation} diff --git a/semester4/dmdb/parts/04_query-processing/04_join/07_partitioned-hash-join.tex b/semester4/dmdb/parts/06_data/00_formats/00_intro.tex similarity index 100% rename from semester4/dmdb/parts/04_query-processing/04_join/07_partitioned-hash-join.tex rename to semester4/dmdb/parts/06_data/00_formats/00_intro.tex diff --git a/semester4/dmdb/parts/04_query-processing/04_join/08_recursive-partitioning.tex b/semester4/dmdb/parts/06_data/01_cloud-systems/00_intro.tex similarity index 100% rename from semester4/dmdb/parts/04_query-processing/04_join/08_recursive-partitioning.tex rename to semester4/dmdb/parts/06_data/01_cloud-systems/00_intro.tex