Skip to content

Commit f609a86

Browse files
fs446fs446
committed
Update ddasp_exercise_slides.tex
- ex 06 vs. 06 mods - PCA typo -> improved equations
1 parent 5a5b477 commit f609a86

File tree

1 file changed

+125
-53
lines changed

1 file changed

+125
-53
lines changed

‎slides/ddasp_exercise_slides.tex

+125-53
Original file line numberDiff line numberDiff line change
@@ -2289,9 +2289,52 @@ \subsection{Exercise 04}
22892289

22902290

22912291

2292+
2293+
22922294
\subsection{Exercise 05}
22932295

2294-
\begin{frame}{Ex05: Condition Number / Regularization}
2296+
\begin{frame}{Ex05: Least Squares / Left Inverse / Projection into Subspaces / Linear Regression}
2297+
%
2298+
Toy Example 1
2299+
2300+
$
2301+
\bm{X}=
2302+
\begin{bmatrix}
2303+
2 & 0 \\ 0 & 1 \\ 0 & 0 \\ 0 & 0
2304+
\end{bmatrix}\quad
2305+
\bm{y} =
2306+
\begin{bmatrix}
2307+
4 \\ 1 \\ -1 \\ -2
2308+
\end{bmatrix}
2309+
$
2310+
%
2311+
\qquad
2312+
%
2313+
$\min_{\text{wrt }\bm{\theta}} \lVert\bm{e}\rVert_2^2 = \min_{\text{wrt }\bm{\theta}} \lVert\bm{y} - \bm{X} \bm{\theta}\rVert_2^2
2314+
\leftrightarrow
2315+
\operatorname*{argmin}_{\bm{\theta}} \lVert\bm{y} - \bm{X} \bm{\theta}\rVert_2^2
2316+
$
2317+
2318+
\addvspace{100mm}
2319+
2320+
\end{frame}
2321+
2322+
2323+
\begin{frame}{Linear Regression}
2324+
Toy Example 2
2325+
2326+
only on Clever Touch
2327+
\end{frame}
2328+
2329+
2330+
2331+
2332+
2333+
2334+
2335+
2336+
\subsection{Exercise 06}
2337+
\begin{frame}{Ex06: Condition Number / Regularization}
22952338
Objectives
22962339
\begin{itemize}
22972340
\item concept of the condition number in terms of singular values
@@ -2387,7 +2430,7 @@ \subsection{Exercise 05}
23872430

23882431

23892432

2390-
\begin{frame}[t]{Ex05: Regularization of the LS-Problem}
2433+
\begin{frame}[t]{Regularization of the LS-Problem}
23912434
$\cdot$ full column rank inverse problem $\rightarrow$ solve with left inverse $\bm{X}^{\dagger_l} \bm{y} = \bm{\theta}$
23922435
$$
23932436
\bm{X}^{\dagger_l}_{N \times M}
@@ -2462,8 +2505,8 @@ \subsection{Exercise 05}
24622505
\begin{frame}[t]{L-Curve to Find Optimum Regularization Parameter $\lambda$}
24632506
$$
24642507
\hat{\bm{\theta}}(\textcolor{C0}{\lambda}) \quad=\quad
2465-
\left[\bm{V} (\bm{\Sigma}^\mathrm{T} \bm{\Sigma} + \textcolor{C0}{\lambda}\bm{I})^{-1} \bm{\Sigma}^\mathrm{T} \bm{U}^\mathrm{H}\right] \bm{y} \quad=\quad
2466-
\left[(\bm{X}^\mathrm{H}\bm{X} + \textcolor{C0}{\lambda}\bm{I})^{-1} \bm{X}^\mathrm{H}\right] \bm{y}
2508+
\left[\bm{V} (\bm{\Sigma}^\mathrm{T} \bm{\Sigma} + \textcolor{C0}{\lambda}\bm{I})^{-1} \bm{\Sigma}^\mathrm{T} \bm{U}^\mathrm{T}\right] \bm{y} \quad=\quad
2509+
\left[(\bm{X}^\mathrm{T}\bm{X} + \textcolor{C0}{\lambda}\bm{I})^{-1} \bm{X}^\mathrm{T}\right] \bm{y}
24672510
$$
24682511
\begin{center}
24692512
\begin{tikzpicture}[scale=1]
@@ -2519,73 +2562,102 @@ \subsection{Exercise 05}
25192562
\lVert\bm{y} - \bm{X} \bm{\theta}\rVert_2^2 + \textcolor{C0}{\lambda} \lVert \bm{\theta} \rVert_2^2
25202563
$$
25212564

2522-
$\cdot$ and the plain Least Squares Error Problem (special case for $\textcolor{C0}{\lambda}=0$)
2565+
$\cdot$ and the ordinary Least Squares Error Problem (i.e. special case for $\textcolor{C0}{\lambda}=0$)
25232566
$$
25242567
\min_{\text{wrt }\bm{\theta}} J(\bm{\theta}) \quad\text{with cost function}\quad
25252568
J(\bm{\theta}) = \lVert\bm{y} - \bm{X} \bm{\theta}\rVert_2^2
25262569
$$
25272570

2528-
have the closed form solution using the (regularized) left inverse of $\bm{X} = \bm{U}\bm{\Sigma}\bm{V}^H$:
2571+
have the closed form solution using the (regularized) left inverse of $\bm{X} = \bm{U}\bm{\Sigma}\bm{V}^\mathrm{T}$:
25292572
$$
25302573
\hat{\bm{\theta}} \quad=\quad
2531-
\left[\bm{V} (\bm{\Sigma}^\mathrm{T} \bm{\Sigma} + \textcolor{C0}{\lambda \bm{I})}^{-1} \bm{\Sigma}^\mathrm{T} \bm{U}^\mathrm{H}\right] \bm{y} \quad=\quad
2532-
\left[(\bm{X}^\mathrm{H}\bm{X} + \textcolor{C0}{\lambda \bm{I}})^{-1} \bm{X}^\mathrm{H}\right] \bm{y}
2574+
\left[\bm{V} (\bm{\Sigma}^\mathrm{T} \bm{\Sigma} + \textcolor{C0}{\lambda \bm{I})}^{-1} \bm{\Sigma}^\mathrm{T} \bm{U}^\mathrm{T}\right] \bm{y} \quad=\quad
2575+
\left[(\bm{X}^\mathrm{T}\bm{X} + \textcolor{C0}{\lambda \bm{I}})^{-1} \bm{X}^\mathrm{T}\right] \bm{y}
25332576
$$
25342577

25352578
Such a model has $N$ \underline{model parameters} $\bm{\theta} = [\theta_1, \theta_2 \dots \theta_N]^T$ and one \underline{hyper parameter} $\textcolor{C0}{\lambda}$ to be learned from the data $\bm{y}_{M \times 1}$ and the feature matrix $\bm{X}_{M \times N}$ with full column rank $R=N$.
25362579

25372580
\end{frame}
25382581

2582+
\begin{frame}[t]{Normal Equations of Tikhonov Regularization in SDV Domain}
25392583

2584+
The ridge regression optimization problem
2585+
$$
2586+
\min_{\text{wrt }\bm{\theta}} J(\bm{\theta}) \quad\text{with cost function}\quad
2587+
J(\bm{\theta}) =
2588+
\lVert\bm{y} - \bm{X} \bm{\theta}\rVert_2^2 + \textcolor{C0}{\lambda} \lVert \bm{\theta} \rVert_2^2
2589+
$$
2590+
yields the normal equations
2591+
$$(\bm{X}^\mathrm{T}\bm{X} + \lambda \bm{I}) \hat{\bm{\theta}} = \bm{X}^\mathrm{T} \bm{y}$$
2592+
and can be solved in SVD domain
2593+
$$((\bm{U} \bm{\Sigma} \bm{V}^\mathrm{T})^\mathrm{T} (\bm{U} \bm{\Sigma} \bm{V}^\mathrm{T}) + \lambda \bm{I}) \hat{\bm{\theta}}=
2594+
(\bm{U} \bm{\Sigma} \bm{V}^\mathrm{T})^\mathrm{T} \bm{y}$$
25402595

2541-
\subsection{Exercise 06}
2596+
%$$(\bm{V} \bm{\Sigma}^\mathrm{T} \bm{U}^\mathrm{T} \bm{U} \bm{\Sigma} \bm{V}^\mathrm{T} + \lambda \bm{I}) \hat{\bm{\theta}}=
2597+
%\bm{V} \bm{\Sigma}^\mathrm{T} \bm{U}^\mathrm{T} \bm{y}$$
25422598

2543-
\begin{frame}{Ex06: Audio Toy Example for Linear Regression and SVD}
2544-
Objectives
2545-
\begin{itemize}
2546-
\item audio multitrack data (stems) arranged as data matrix
2547-
\item the SVD of this matrix allows to listen to the U space, i.e. to the orthogonal audio signals (which is some source separation approach)
2548-
\item try to find the mixing gains of a mix that is corrupted by noise
2549-
\end{itemize}
2550-
\end{frame}
2599+
$$(\bm{V} \bm{\Sigma}^\mathrm{T} \bm{\Sigma} \bm{V}^\mathrm{T} + \lambda \bm{V}\bm{I}\bm{V}^\mathrm{T}) \hat{\bm{\theta}}=
2600+
\bm{V} \bm{\Sigma}^\mathrm{T} \bm{U}^\mathrm{T} \bm{y}$$
25512601

2552-
\begin{frame}[t]{Ex06: Audio Toy Example for Linear Regression and SVD}
2553-
\begin{center}
2554-
$
2555-
\def\L{0.5}
2556-
\def\F{1}
2557-
\def\N{3}
2558-
\def\rank{0.999999}
2559-
\drawmatrix[fill=none, height=\N, width=0]{y}_\mathtt{N \times 1} =
2560-
\drawmatrix[fill=none, height=\N, width=\F]{X}_\mathtt{N \times F}
2561-
\drawmatrix[fill=none, height=\F, width=0]\theta_\mathtt{F \times 1}+
2562-
\drawmatrix[fill=none, height=\N, width=0]{\nu}_\mathtt{N \times 1}
2563-
=
2564-
\drawmatrix[bbox style={fill=C4}, bbox height=\N, bbox width=\N, fill=C0, height=\N, width=\rank\F]U_\mathtt{N \times N}
2565-
\drawmatrix[bbox style={fill=gray!50}, bbox height=\N, bbox width=\F, fill=white, height=\rank\F, width=\rank\F]\Sigma_\mathtt{N \times F}
2566-
\drawmatrix[bbox style={fill=C1}, bbox height=\F, bbox width=\F, fill=C2, height=\F, width=\rank\F]{V}_\mathtt{F \times F}^H
2567-
\drawmatrix[fill=none, height=\F, width=0]\theta_\mathtt{F \times 1}+
2568-
\drawmatrix[fill=none, height=\N, width=0]{\nu}_\mathtt{N \times 1}
2569-
$
2570-
\end{center}
2571-
\end{frame}
2602+
$$[\bm{V} (\bm{\Sigma}^\mathrm{T} \bm{\Sigma} + \lambda \bm{I})\bm{V}^\mathrm{T}] \hat{\bm{\theta}}=
2603+
\bm{V} \bm{\Sigma}^\mathrm{T} \bm{U}^\mathrm{T} \bm{y}$$
2604+
using left inverse
2605+
$$\hat{\bm{\theta}}=
2606+
[\bm{V} (\bm{\Sigma}^\mathrm{T} \bm{\Sigma} + \lambda \bm{I})\bm{V}^\mathrm{T}]^{-1} \bm{V} \bm{\Sigma}^\mathrm{T} \bm{U}^\mathrm{T} \bm{y}$$
25722607

2608+
$$\hat{\bm{\theta}}=
2609+
\bm{V} (\bm{\Sigma}^\mathrm{T} \bm{\Sigma} + \lambda \bm{I})^{-1} \bm{\Sigma}^\mathrm{T} \bm{U}^\mathrm{T} \bm{y}$$
25732610

2611+
\end{frame}
25742612

2575-
\begin{frame}{Ex06: Audio Toy Example for Linear Regression and SVD}
2576-
Consider the following linear combinations
2577-
$$\bm{X} \bm{\theta} + \bm{\nu} = \bm{y}$$
2578-
where $\bm{\theta}=[\theta_1, \theta_2, \theta_3, ..., \theta_{F}]^\mathrm{T}$ are typical variables for the model parameter vector.
2613+
% \subsection{Exercise 06}
25792614
%
2580-
\begin{itemize}
2581-
\item $\bm{X}_{N \times F}$ matrix with $N$ audio samples for each column, $f$-th column represents the $f$-th audiotrack / feature
2582-
\item $\bm{\theta}_{F \times 1}$ column vector of scalar values that represent a dedicated gain for each audiotrack
2583-
\item $\bm{\nu}_{N \times 1}$ column vector that represents an $N$-sample long noise signal added to the mixdown $\bm{X} \bm{\theta}$
2584-
\item $\bm{y}_{N \times 1}$ audio signal with $N$ samples as a result of the linear model's linear combination plus noise
2585-
\end{itemize}
2615+
% \begin{frame}{Ex06: Audio Toy Example for Linear Regression and SVD}
2616+
% Objectives
2617+
% \begin{itemize}
2618+
% \item audio multitrack data (stems) arranged as data matrix
2619+
% \item the SVD of this matrix allows to listen to the U space, i.e. to the orthogonal audio signals (which is some source separation approach)
2620+
% \item try to find the mixing gains of a mix that is corrupted by noise
2621+
% \end{itemize}
2622+
% \end{frame}
25862623
%
2587-
Let us assume that a) we know $\bm{X}$ (i.e. the individual audio tracks) and $\bm{y}$ (i.e. the noise-corrupted final mixdown), b) that we do not know the noise $\bm{\nu}$ and c) that we want to estimate the 'real world' mixing gains $\bm{\theta}$
2588-
\end{frame}
2624+
% \begin{frame}[t]{Ex06: Audio Toy Example for Linear Regression and SVD}
2625+
% \begin{center}
2626+
% $
2627+
% \def\L{0.5}
2628+
% \def\F{1}
2629+
% \def\N{3}
2630+
% \def\rank{0.999999}
2631+
% \drawmatrix[fill=none, height=\N, width=0]{y}_\mathtt{N \times 1} =
2632+
% \drawmatrix[fill=none, height=\N, width=\F]{X}_\mathtt{N \times F}
2633+
% \drawmatrix[fill=none, height=\F, width=0]\theta_\mathtt{F \times 1}+
2634+
% \drawmatrix[fill=none, height=\N, width=0]{\nu}_\mathtt{N \times 1}
2635+
% =
2636+
% \drawmatrix[bbox style={fill=C4}, bbox height=\N, bbox width=\N, fill=C0, height=\N, width=\rank\F]U_\mathtt{N \times N}
2637+
% \drawmatrix[bbox style={fill=gray!50}, bbox height=\N, bbox width=\F, fill=white, height=\rank\F, width=\rank\F]\Sigma_\mathtt{N \times F}
2638+
% \drawmatrix[bbox style={fill=C1}, bbox height=\F, bbox width=\F, fill=C2, height=\F, width=\rank\F]{V}_\mathtt{F \times F}^H
2639+
% \drawmatrix[fill=none, height=\F, width=0]\theta_\mathtt{F \times 1}+
2640+
% \drawmatrix[fill=none, height=\N, width=0]{\nu}_\mathtt{N \times 1}
2641+
% $
2642+
% \end{center}
2643+
% \end{frame}
2644+
%
2645+
%
2646+
%
2647+
% \begin{frame}{Ex06: Audio Toy Example for Linear Regression and SVD}
2648+
% Consider the following linear combinations
2649+
% $$\bm{X} \bm{\theta} + \bm{\nu} = \bm{y}$$
2650+
% where $\bm{\theta}=[\theta_1, \theta_2, \theta_3, ..., \theta_{F}]^\mathrm{T}$ are typical variables for the model parameter vector.
2651+
% %
2652+
% \begin{itemize}
2653+
% \item $\bm{X}_{N \times F}$ matrix with $N$ audio samples for each column, $f$-th column represents the $f$-th audiotrack / feature
2654+
% \item $\bm{\theta}_{F \times 1}$ column vector of scalar values that represent a dedicated gain for each audiotrack
2655+
% \item $\bm{\nu}_{N \times 1}$ column vector that represents an $N$-sample long noise signal added to the mixdown $\bm{X} \bm{\theta}$
2656+
% \item $\bm{y}_{N \times 1}$ audio signal with $N$ samples as a result of the linear model's linear combination plus noise
2657+
% \end{itemize}
2658+
% %
2659+
% Let us assume that a) we know $\bm{X}$ (i.e. the individual audio tracks) and $\bm{y}$ (i.e. the noise-corrupted final mixdown), b) that we do not know the noise $\bm{\nu}$ and c) that we want to estimate the 'real world' mixing gains $\bm{\theta}$
2660+
% \end{frame}
25892661

25902662

25912663

@@ -2680,7 +2752,7 @@ \subsection{Exercise 08}
26802752

26812753
Mapping
26822754

2683-
$\bm{X}_c= \bm{U} \bm{\Sigma} \bm{V}^\mathrm{T} = \bm{F}_c \bm{L}^\mathrm{T}$
2755+
$\bm{X}_c= \bm{U} \bm{\Sigma} \cdot \bm{V}^\mathrm{T} = \bm{F}_c \cdot \bm{L}^\mathrm{T}$
26842756

26852757
$\bm{F}_c = \bm{X}_c \bm{L} = \bm{X}_c \bm{V} = \bm{U} \bm{\Sigma}$
26862758

@@ -2701,9 +2773,9 @@ \subsection{Exercise 08}
27012773

27022774
Mapping
27032775

2704-
$\bm{X}_r = \bm{V} \bm{\Sigma} \bm{U}^\mathrm{T} = \bm{L} \bm{F}_r$
2776+
$\bm{X}_r = \bm{V} \cdot \bm{\Sigma}^\mathrm{T} \bm{U}^\mathrm{T} = \bm{L} \cdot \bm{F}_r$
27052777

2706-
$\bm{F}_r = \bm{L}^\mathrm{T} \bm{X}_r = \bm{V}^\mathrm{T} \bm{X}_r = \bm{\Sigma} \bm{U}^\mathrm{T}$
2778+
$\bm{F}_r = \bm{L}^\mathrm{T} \bm{X}_r = \bm{V}^\mathrm{T} \bm{X}_r = \bm{\Sigma}^\mathrm{T} \bm{U}^\mathrm{T} = (\bm{\Sigma} \bm{U})^\mathrm{T}$
27072779

27082780
PC scores $\bm{F}_r =
27092781
\begin{bmatrix}
@@ -2713,7 +2785,7 @@ \subsection{Exercise 08}
27132785
- \bm{f}_F -
27142786
\end{bmatrix}
27152787
=
2716-
\bm{\Sigma} \bm{U}^\mathrm{T}$
2788+
(\bm{\Sigma} \bm{U})^\mathrm{T}$
27172789

27182790
PC loadings $\bm{L} = \bm{V} $
27192791

0 commit comments

Comments
 (0)