% Template based on:
% https://www.cs.columbia.edu/~djhsu/coms4774-s21/scribe.html

\documentclass[12pt]{article}

\usepackage{amscd}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amstext}
\usepackage{amsthm}
\usepackage{bbold}
\usepackage{bm}
\usepackage{colonequals}
\usepackage[dvips,letterpaper,margin=1in]{geometry}
\usepackage{mathtools}
\usepackage{graphicx}
\usepackage{xcolor}

\usepackage{csquotes}
\MakeOuterQuote{"}

\definecolor{cblack}{rgb}{0,0,0}
\definecolor{cblue}{rgb}{0.121569,0.466667,0.705882}
\definecolor{corange}{rgb}{1.000000,0.498039,0.054902}
\definecolor{cgreen}{rgb}{0.172549,0.627451,0.172549}
\definecolor{cred}{rgb}{0.839216,0.152941,0.156863}
\definecolor{cpurple}{rgb}{0.580392,0.403922,0.741176}
\definecolor{cbrown}{rgb}{0.549020,0.337255,0.294118}
\definecolor{cpink}{rgb}{0.890196,0.466667,0.760784}
\definecolor{cgray}{rgb}{0.498039,0.498039,0.498039}

\usepackage{hyperref}
\hypersetup{
  linkcolor  = cblue,
  citecolor  = cgreen,
  urlcolor   = corange,
  colorlinks = true,
}

% Lecture information
\newcommand\coursename{CPSC 664, Spring 2023}
\newcommand\scribe{Abhinav Bhardwaj}
\newcommand\lecturer{Tim Kunisky}
\newcommand\lecturedate{February 28}
\newcommand\lecturetitle{Lecture 13: Free Probability and Kac-Rice }

% Define theorem environments here.
\newtheorem{claim}{Claim}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{remark}[theorem]{Remark}
\newtheorem{assumption}[theorem]{Assumption}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{question}[theorem]{Question}
\newtheorem{problem}[theorem]{Problem}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{example}[theorem]{Example}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{conjecture}[theorem]{Conjecture}

% Some of my macros
\renewcommand{\AA}{\mathbb{A}}
\newcommand{\CC}{\mathbb{C}}
\newcommand{\DD}{\mathbb{D}}
\newcommand{\EE}{\mathbb{E}}
\newcommand{\FF}{\mathbb{F}}
\newcommand{\HH}{\mathbb{H}}
\newcommand{\NN}{\mathbb{N}}
\newcommand{\PP}{\mathbb{P}}
\newcommand{\QQ}{\mathbb{Q}}
\newcommand{\RR}{\mathbb{R}}
\renewcommand{\SS}{\mathbb{S}}
\newcommand{\ZZ}{\mathbb{Z}}
\DeclareSymbolFont{bbold}{U}{bbold}{m}{n}
\DeclareSymbolFontAlphabet{\mathbbold}{bbold}
\newcommand{\One}{\mathbbold{1}}

\newcommand{\ba}{\bm a}
\newcommand{\bb}{\bm b}
\newcommand{\bc}{\bm c}
\newcommand{\bd}{\bm d}
\newcommand{\be}{\bm e}
\newcommand{\bg}{\bm g}
\newcommand{\bh}{\bm h}
\newcommand{\bi}{\bm i}
\newcommand{\bj}{\bm j}
\newcommand{\bk}{\bm k}
\newcommand{\bbm}{\bm m}
\newcommand{\bp}{\bm p}
\newcommand{\bq}{\bm q}
\newcommand{\br}{\bm r}
\newcommand{\bs}{\bm s}
\newcommand{\bt}{\bm t}
\newcommand{\bu}{\bm u}
\newcommand{\bv}{\bm v}
\newcommand{\bw}{\bm w}
\newcommand{\bx}{\bm x}
\newcommand{\by}{\bm y}
\newcommand{\bz}{\bm z}

\newcommand{\bA}{\bm A}
\newcommand{\bB}{\bm B}
\newcommand{\bC}{\bm C}
\newcommand{\bD}{\bm D}
\newcommand{\bE}{\bm E}
\newcommand{\bF}{\bm F}
\newcommand{\bG}{\bm G}
\newcommand{\bH}{\bm H}
\newcommand{\bI}{\bm I}
\newcommand{\bL}{\bm L}
\newcommand{\bM}{\bm M}
\newcommand{\bN}{\bm N}
\newcommand{\bP}{\bm P}
\newcommand{\bQ}{\bm Q}
\newcommand{\bR}{\bm R}
\newcommand{\bS}{\bm S}
\newcommand{\bT}{\bm T}
\newcommand{\bU}{\bm U}
\newcommand{\bV}{\bm V}
\newcommand{\bW}{\bm W}
\newcommand{\bX}{\bm X}
\newcommand{\bY}{\bm Y}
\newcommand{\bZ}{\bm Z}

\newcommand{\zero}{\bm{0}}
\newcommand{\one}{\bm{1}}

\newcommand{\sA}{\mathcal{A}}
\newcommand{\sB}{\mathcal{B}}
\newcommand{\sC}{\mathcal{C}}
\newcommand{\sD}{\mathcal{D}}
\newcommand{\sE}{\mathcal{E}}
\newcommand{\sF}{\mathcal{F}}
\newcommand{\sG}{\mathcal{G}}
\newcommand{\sH}{\mathcal{H}}
\newcommand{\sI}{\mathcal{I}}
\newcommand{\sJ}{\mathcal{J}}
\newcommand{\sK}{\mathcal{K}}
\newcommand{\sL}{\mathcal{L}}
\newcommand{\sM}{\mathcal{M}}
\newcommand{\sN}{\mathcal{N}}
\newcommand{\sO}{\mathcal{O}}
\newcommand{\sP}{\mathcal{P}}
\newcommand{\sQ}{\mathcal{Q}}
\newcommand{\sR}{\mathcal{R}}
\newcommand{\sS}{\mathcal{S}}
\newcommand{\sT}{\mathcal{T}}
\newcommand{\sU}{\mathcal{U}}
\newcommand{\sV}{\mathcal{V}}
\newcommand{\sX}{\mathcal{X}}
\newcommand{\sY}{\mathcal{Y}}

\newcommand{\fB}{\mathscr{B}}
\newcommand{\fC}{\mathscr{C}}
\newcommand{\fE}{\mathscr{E}}
\newcommand{\fH}{\mathscr{H}}
\newcommand{\fI}{\mathscr{I}}
\newcommand{\fP}{\mathscr{P}}
\newcommand{\fV}{\mathscr{V}}

\DeclareSymbolFont{sfoperators}{OT1}{cmss}{m}{n}
% don't waste a math group
\DeclareSymbolFontAlphabet{\mathsf}{sfoperators}
% tell LaTeX to use sfoperators for names of operators
\makeatletter
\renewcommand{\operator@font}{\mathgroup\symsfoperators}
\makeatother

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\GOE}{GOE}
\DeclareMathOperator{\sym}{sym}
\DeclareMathOperator{\Part}{Part}
\DeclareMathOperator{\poly}{poly}
\DeclareMathOperator{\Unif}{Unif}
\DeclareMathOperator{\Var}{Var}
\DeclareMathOperator{\Cov}{Cov}
\DeclareMathOperator{\Tr}{Tr}
\DeclareMathOperator{\rank}{rank}
\DeclareMathOperator{\obj}{obj}
\DeclareMathOperator{\diag}{diag}
\DeclareMathOperator{\sgn}{sgn}

\renewcommand\Re{\operatorname{Re}}
\renewcommand\Im{\operatorname{Im}}
% These put symbols below.
\newcommand{\Px}{\mathop{\mathbb{P}}}
\newcommand{\Ex}{\mathop{\mathbb{E}}}
\newcommand{\Varx}{\mathop{\mathsf{Var}}}

\newcommand\numberthis{\addtocounter{equation}{1}\tag{\theequation}}

%
% Put your custom macros here if you want.
%
\usepackage{amsmath}
\newcommand{\Crit}{\mathsf{Crit}}
\newcommand{\what}{\widehat}

\begin{filecontents}{2023-02-28.bib}
@misc{MAB,
      title={Landscape Complexity for the Empirical Risk of Generalized Linear Models}, 
      author={Antoine Maillard and G\'{e}rard Ben Arous and Giulio Biroli},
      year={2023},
      eprint={1912.02143},
      archivePrefix={arXiv},
      primaryClass={stat.ML}
}

@misc{arous2021landscape,
      title={Landscape complexity beyond invariance and the elastic manifold}, 
      author={G\'{e}rard Ben Arous and Paul Bourgade and Benjamin McKenna},
      year={2021},
      eprint={2105.05051},
      archivePrefix={arXiv},
      primaryClass={math.PR}
}
\end{filecontents}

\begin{document}

% This creates the header for scribe notes.
\noindent \coursename \hfill Scribe: \scribe \\
\lecturedate \hfill Lecturer: \lecturer
\vspace{1em}

\hrule

\vspace{1.5em}
\begin{center}
  {\Large\lecturetitle}
\end{center}

\section{Introduction}

	This lecture covers applying free probability to landscape complexity. First, we revisit the Kac-Rice analysis of the "noisy well" and solve for asymptotics of critical points using free convolution. We will discuss the role of large deviations reasoning and discuss a model used to describe the behavior of simple neural networks.
 
\section{Noisy Well Model}

Details can be found in \cite{arous2021landscape}.

The main idea of this section is to obtain a closed form approximation of the total number of critical points of $f$ in al of $\mathbb{R}^{n}$.

Recall the noisy well model.

\begin{equation}
    \label{noisywell}
    f(x) = \frac{\alpha}{2}\|x\|_2^2+ g(x)
\end{equation}

Where $g(x)$ is the random function from Lecture \#8, Continuous Counting with the Kac-Rice Formula. We do not reproduce $g$ here, but we established that for a set $A,$
\begin{equation}
     \EE\Crit(f, A) = \frac{1}{\alpha^{n}}\EE[\lvert \det(H) \rvert]\mathbb{P}[\mathcal{N} \in A],
\end{equation}
where $\mathcal{N}$ was some normal random variable whose parameters do not matter. For details, see Lecture \#8. Taking $A := \RR^n$, the probability on the right hand side is $1$ and we are left analyzing the expected (absolute) determinant. 

$H$ was the matrix
\begin{equation}
    H \colonequals \alpha I_n + \sqrt{d(d-1)N^{d-2}}hI_{N} + \sqrt{d(d-1)N^{d-1}}W + \sqrt{dN^{d-1}}D.
\end{equation}
where $h \sim \mathcal{N}(0,1)$ is independent of $D$ and $W$. $D$ is a random diagonal matrix with independent standard normal random variables on the diagonal. $W$ is a matrix drawn from the \textit{Gaussian orthogonal ensemble} (GOE), normalized to have spectral norm at most $2$. For the precise definition of the GOE, see Lecture 8. The most important observation is that the eigenvalue distribution of $W$ follows the semicircle law and that $D$ and $W$ are asymptotically free by Voiculescu's theorem, since the GOE is rotationally invariant.

Define $\widehat{\alpha} \colonequals \alpha / N^{\frac{(d-1)}{2}}$, and $\widehat{H} \colonequals H / N^{(d-1)/2}$. Set $\widehat{H}^{(0)} \colonequals \sqrt{d(d-1)}W + \sqrt{d}D$, the last two terms in $\widehat{H}$. We have, for some constant $C$, 
\begin{equation}
\EE[\lvert \det(H) \rvert] = \frac{C}{\what{\alpha}^{n}}\int e^{-N\frac{h^2}{2}}\EE[\lvert \det(hI + \what{\alpha}I + \what{H}^{(0)})\rvert ]dh
\end{equation}
We can rewrite the integral as 

\begin{equation}
 C\int \exp\left(N\left[-\log\what{\alpha} - \frac{h^2}{2} + \frac{1}{N}\log\EE[\lvert \det(hI + \what{\alpha}I + \what{H}^{(0)})\rvert ]\right]\right)dh
\end{equation}

This form motivates us to use the Laplace approximation, which gives
\begin{align*}
    \EE[\lvert \det(H) \rvert]  &\approx C\exp\left(N \sup_h\left\{ -\log\what{\alpha} - \frac{h^2}{2} + \frac{1}{N}\log\EE[\lvert \det(hI + \what{\alpha}I + \what{H}^{(0)})\rvert ] \right\}\right) \\
                                &= C\exp\left(N \sup_h\left\{ -\log\what{\alpha} - \frac{h^2}{2} + \frac{1}{N}\log\EE\left[ \exp\left(\sum_{i = 1}^N \log |h + \what{\alpha} + \lambda_i|\right)\right] \right\}\right) \numberthis
\end{align*}
where $\lambda_i$ are the eigenvalues of $\what{H}^{(0)}$.

It turns out that by concentration/large deviation bounds,  we can very closely approximate this last term  by commuting log and the expectation:
\begin{align*}
  \EE[\lvert \det(H) \rvert] &\approx C\exp\left(N \sup_h\left\{ -\log\what{\alpha} - \frac{h^2}{2} + \frac{1}{N}\EE\left[\sum_{i = 1}^N \log |h + \what{\alpha} + \lambda_i|\right] \right\}\right) \intertext{and we recognize this last term as an integral with respect to the spectral measure of $\what{H}^{(0)}$,}
                               &= C\exp\left(N \sup_h\left\{ -\log\what{\alpha} - \frac{h^2}{2} + \int \log|h + \what{\alpha} + t| d\what{\mu}_N(t)\right\}\right). \numberthis
\end{align*}
Because $D$ and $W$ are asymptotically free, the spectral measure $\hat{\mu_{N}}$ is approximately the additive free convolution of the spectral measures of $D$ and $W$,
\begin{equation}
    \hat{\mu_{N}} \approx \mu_{sc} \boxplus \mu_{D},
\end{equation}
where $\mu_{sc}$ is the semicircle distribution (the spectral measure of $W$, see Lecture 10) and $\mu_{D}$ is the spectral measure given by the entries along the diagonal of $D$ (which means it is a Gaussian measure). Therefore, we have a closed form approximation for $\EE[\lvert \det(H) \rvert]$, which allows us to approximate the number of critical points of $f$:
\begin{equation}
    \EE[\lvert \det(H) \rvert] \approx C\exp\left(N \sup_h\left\{ -\log\what{\alpha} - \frac{h^2}{2} + \int \log|h + \what{\alpha} + t| d(\mu_{sc} \boxplus \mu_D)(t)\right\}\right).
\end{equation}
We note that, while not quite a closed form in terms of $\what{\alpha}$, this is at least a function that we can approximate numerically, by computing an approximation to the integral term as a function of $h$ and $\what{\alpha}$ and then solving an optimization problem over $h$.

\section{A Basic Neural Network Model}

The next model we consider is the following.
\begin{equation}
L(x) = \frac{1}{m}\sum_{i = 1}^m\phi(\langle \zeta_i, x\rangle).
\end{equation}
We will only sketch how to apply the Kac-Rice formula to this model.

Think of $\phi$ as an activation function, and assume it is smooth. $x \in \SS^{n-1}$ is a unit vector and $\zeta_i$ are random vectors whose components are independent, identically distributed Gaussians.

Under some technical assumptions, one can establish a limiting formula for the number of critical points of $L$ in $B \subset \RR$.

The main result is that the expected number of critical points has a (convoluted) closed form in terms of the multiplicative free convolution. Another main idea is that here, we need the theory of large deviations in order to get the result. In particular, concentration of the expected determinant will not hold sufficiently to exchange the log and expectation as we did in the previous argument.

\subsection{Step 1: Kac-Rice Formula}

The Kac-Rice formula on the sphere for this case gives, for a set $B \subset \mathbb{R}$,
\begin{equation}
     \EE \Crit(L, B) = \int_{\SS^{n-1}} \rho_{\nabla L(x)}(0)\EE\left[\One\{L(x) \in B\} \lvert \det \nabla^2 L(x) \rvert \Big\lvert \nabla L(x) = 0\right]d\mu_{\SS}(x),
\end{equation}
where $\mu_{\SS}$ is the uniform measure on the sphere, $\rho_{\nabla L(x)}$ is a density, and the gradients and Hessians must be taken as their Riemannian versions on the sphere, which amounts to introducing some projections to the direction orthogonal to $x$.

We may take $B = \SS^{n - 1}$ for the sake of simplicity.
In this case, as usual, the main difficulty is controlling the expectation of the absolute value of the determinant.
This will involve the corrections for the spherical setting mentioned above, but essentially will boil down to a computation like
\begin{equation}
    \EE \left| \det\left(\sum_{i = 1}^m \phi^{\prime\prime}(\langle \zeta_i, x \rangle)\frac{(P_{x^{\perp}}\zeta_i) (P_{x^{\perp}}\zeta_i)^{\top}}{n}\right)\right|,
\end{equation}
where $P_{x^{\perp}} = I - xx^{\top}$ is the projection to the direction orthogonal to $x$.
Conveniently, if we set $u_i = \langle \zeta_i, x \rangle$ and $v_i$ to be an $(n - 1)$-dimensional embedding of $P_{x^{\perp}}\zeta_i$, then $u_i$ and all coordinates of $v_i$ are independent standard Gaussians, and the above is
\begin{equation}
    \EE \left| \det\left(\sum_{i = 1}^m \phi^{\prime\prime}(u_i)\frac{v_iv_i^{\top}}{n}\right)\right|,
\end{equation}
a much nicer random matrix than it initially appears.

\subsection{Step 2: Large Deviation Bound}

We can write the matrix above as $\frac{1}{n}VDV^{\top}$ for $V$ an i.i.d.\ Gaussian matrix and $D$ a diagonal matrix with entries i.i.d.\ distributed as $\phi^{\prime\prime}(\mathcal{N}(0, 1))$ (i.e., $\phi^{\prime\prime}$ applied to i.i.d.\ standard Gaussians).
If $\frac{m}{n} \to \lambda > 0$ for some parameter $\lambda$, then the e.s.d.\ of $\frac{1}{n}VV^{\top}$ converges to $\rho_{\lambda}$ the Marcenko-Pastur distribution with parameter $\lambda$.
Thus, the spectrum of $\frac{1}{n}VDV^{\top}$ converges to $\rho_{\lambda} \boxtimes \phi^{\prime\prime}(\mathcal{N}(0, 1))$, the multiplicative free convolution.
We might then expect
\begin{equation}
    \EE \left| \det\left(\sum_{i = 1}^m \phi^{\prime\prime}(u_i)\frac{v_iv_i^{\top}}{n}\right)\right| \stackrel{?}{\approx} \exp\left(m\int \log |t| d(\rho_{\lambda} \boxtimes \phi^{\prime\prime}(\mathcal{N}(0, 1)))(t) \right).
\end{equation}

As before, this would require sufficiently strong concentration of a particular function of the spectrum of the left-hand side around its typical value.
But, in this case, this does \emph{not} hold.
Instead, we need to consider large deviations probabilities: by \emph{Sanov's theorem}, the probability that $u_1, \dots, u_m$ have an empirical distribution close to some measure $\nu$ is roughly $\exp(-m H(\nu | \sN(0, 1)))$, where $H(\cdot | \cdot)$ is the relative entropy.
Applying a Laplace method approximation, we get instead
\begin{equation}
    \EE \left| \det\left(\sum_{i = 1}^m \phi^{\prime\prime}(u_i)\frac{v_iv_i^{\top}}{n}\right)\right| \approx \exp\left(m \sup_{\nu}\left\{\int \log |t| d(\rho_{\lambda} \boxtimes \phi^{\prime\prime}(\nu))(t) - H(\nu | \sN(0, 1)) \right\} \right).
\end{equation}

Most of the work in \cite{MAB} is dedicated to showing that a version of this holds for a slightly different random matrix.
They then arrive at a similar variational problem over measures that controls the expected number of critical points of $L$.

\bibliographystyle{alpha}
\bibliography{2023-02-28}

\end{document}
