% https://www.cs.columbia.edu/~djhsu/coms4774-s21/scribe.html

\documentclass[12pt]{article}

\usepackage{amscd}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amstext}
\usepackage{amsthm}
\usepackage{bbold}
\usepackage{bm}
\usepackage{colonequals}
\usepackage[dvips,letterpaper,margin=1in]{geometry}
\usepackage{mathtools}
\usepackage{graphicx}
\usepackage{xcolor}
\usepackage{float}
\usepackage{cancel}

\usepackage{csquotes}
\MakeOuterQuote{"}

\definecolor{cblack}{rgb}{0,0,0}
\definecolor{cblue}{rgb}{0.121569,0.466667,0.705882}
\definecolor{corange}{rgb}{1.000000,0.498039,0.054902}
\definecolor{cgreen}{rgb}{0.172549,0.627451,0.172549}
\definecolor{cred}{rgb}{0.839216,0.152941,0.156863}
\definecolor{cpurple}{rgb}{0.580392,0.403922,0.741176}
\definecolor{cbrown}{rgb}{0.549020,0.337255,0.294118}
\definecolor{cpink}{rgb}{0.890196,0.466667,0.760784}
\definecolor{cgray}{rgb}{0.498039,0.498039,0.498039}

\usepackage{hyperref}
\hypersetup{
  linkcolor  = cblue,
  citecolor  = cgreen,
  urlcolor   = corange,
  colorlinks = true,
}

% Lecture information
\newcommand\coursename{CPSC 664, Spring 2023}
\newcommand\scribe{Jane Lee}
\newcommand\lecturer{Tim Kunisky}
\newcommand\lecturedate{February 9, 2023}
\newcommand\lecturetitle{Lecture 8: Continuous Counting with the Kac-Rice Formula}

% Define theorem environments here.
\newtheorem{claim}{Claim}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{remark}[theorem]{Remark}
\newtheorem{assumption}[theorem]{Assumption}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{question}[theorem]{Question}
\newtheorem{problem}[theorem]{Problem}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{example}[theorem]{Example}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{conjecture}[theorem]{Conjecture}

% Some of my macros
\renewcommand{\AA}{\mathbb{A}}
\newcommand{\CC}{\mathbb{C}}
\newcommand{\DD}{\mathbb{D}}
\newcommand{\EE}{\mathbb{E}}
\newcommand{\FF}{\mathbb{F}}
\newcommand{\HH}{\mathbb{H}}
\newcommand{\NN}{\mathbb{N}}
\newcommand{\PP}{\mathbb{P}}
\newcommand{\QQ}{\mathbb{Q}}
\newcommand{\RR}{\mathbb{R}}
\renewcommand{\SS}{\mathbb{S}}
\newcommand{\ZZ}{\mathbb{Z}}
\DeclareSymbolFont{bbold}{U}{bbold}{m}{n}
\DeclareSymbolFontAlphabet{\mathbbold}{bbold}
\newcommand{\One}{\mathbbold{1}}

\newcommand{\ba}{\bm a}
\newcommand{\bb}{\bm b}
\newcommand{\bc}{\bm c}
\newcommand{\bd}{\bm d}
\newcommand{\be}{\bm e}
\newcommand{\bg}{\bm g}
\newcommand{\bh}{\bm h}
\newcommand{\bi}{\bm i}
\newcommand{\bj}{\bm j}
\newcommand{\bk}{\bm k}
\newcommand{\bbm}{\bm m}
\newcommand{\bp}{\bm p}
\newcommand{\bq}{\bm q}
\newcommand{\br}{\bm r}
\newcommand{\bs}{\bm s}
\newcommand{\bt}{\bm t}
\newcommand{\bu}{\bm u}
\newcommand{\bv}{\bm v}
\newcommand{\bw}{\bm w}
\newcommand{\bx}{\bm x}
\newcommand{\by}{\bm y}
\newcommand{\bz}{\bm z}

\newcommand{\bA}{\bm A}
\newcommand{\bB}{\bm B}
\newcommand{\bC}{\bm C}
\newcommand{\bD}{\bm D}
\newcommand{\bE}{\bm E}
\newcommand{\bF}{\bm F}
\newcommand{\bG}{\bm G}
\newcommand{\bH}{\bm H}
\newcommand{\bI}{\bm I}
\newcommand{\bL}{\bm L}
\newcommand{\bM}{\bm M}
\newcommand{\bN}{\bm N}
\newcommand{\bP}{\bm P}
\newcommand{\bQ}{\bm Q}
\newcommand{\bR}{\bm R}
\newcommand{\bS}{\bm S}
\newcommand{\bT}{\bm T}
\newcommand{\bU}{\bm U}
\newcommand{\bV}{\bm V}
\newcommand{\bW}{\bm W}
\newcommand{\bX}{\bm X}
\newcommand{\bY}{\bm Y}
\newcommand{\bZ}{\bm Z}

\newcommand{\zero}{\bm{0}}
\newcommand{\one}{\bm{1}}

\newcommand{\sA}{\mathcal{A}}
\newcommand{\sB}{\mathcal{B}}
\newcommand{\sC}{\mathcal{C}}
\newcommand{\sD}{\mathcal{D}}
\newcommand{\sE}{\mathcal{E}}
\newcommand{\sF}{\mathcal{F}}
\newcommand{\sG}{\mathcal{G}}
\newcommand{\sH}{\mathcal{H}}
\newcommand{\sI}{\mathcal{I}}
\newcommand{\sJ}{\mathcal{J}}
\newcommand{\sK}{\mathcal{K}}
\newcommand{\sL}{\mathcal{L}}
\newcommand{\sM}{\mathcal{M}}
\newcommand{\sN}{\mathcal{N}}
\newcommand{\sO}{\mathcal{O}}
\newcommand{\sP}{\mathcal{P}}
\newcommand{\sQ}{\mathcal{Q}}
\newcommand{\sR}{\mathcal{R}}
\newcommand{\sS}{\mathcal{S}}
\newcommand{\sT}{\mathcal{T}}
\newcommand{\sU}{\mathcal{U}}
\newcommand{\sV}{\mathcal{V}}
\newcommand{\sX}{\mathcal{X}}
\newcommand{\sY}{\mathcal{Y}}

\newcommand{\fB}{\mathscr{B}}
\newcommand{\fC}{\mathscr{C}}
\newcommand{\fE}{\mathscr{E}}
\newcommand{\fH}{\mathscr{H}}
\newcommand{\fI}{\mathscr{I}}
\newcommand{\fP}{\mathscr{P}}
\newcommand{\fV}{\mathscr{V}}

\DeclareSymbolFont{sfoperators}{OT1}{cmss}{m}{n}
% don't waste a math group
\DeclareSymbolFontAlphabet{\mathsf}{sfoperators}
% tell LaTeX to use sfoperators for names of operators
\makeatletter
\renewcommand{\operator@font}{\mathgroup\symsfoperators}
\makeatother

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\GOE}{GOE}
\DeclareMathOperator{\sym}{sym}
\DeclareMathOperator{\Part}{Part}
\DeclareMathOperator{\poly}{poly}
\DeclareMathOperator{\Unif}{Unif}
\DeclareMathOperator{\Var}{Var}
\DeclareMathOperator{\Cov}{Cov}
\DeclareMathOperator{\Tr}{Tr}
\DeclareMathOperator{\rank}{rank}
\DeclareMathOperator{\obj}{obj}
\DeclareMathOperator{\diag}{diag}
\DeclareMathOperator{\sgn}{sgn}

\renewcommand\Re{\operatorname{Re}}
\renewcommand\Im{\operatorname{Im}}
% These put symbols below.
\newcommand{\Px}{\mathop{\mathbb{P}}}
\newcommand{\Ex}{\mathop{\mathbb{E}}}
\newcommand{\Varx}{\mathop{\mathsf{Var}}}

\newcommand\numberthis{\addtocounter{equation}{1}\tag{\theequation}}

%
% Put your custom macros here if you want.
%



\begin{document}

% This creates the header for scribe notes.
\noindent \coursename \hfill Scribe: \scribe \\
\lecturedate \hfill Lecturer: \lecturer
\vspace{1em}

\hrule

\vspace{1.5em}
\begin{center}
  {\Large\lecturetitle}
\end{center}

\section{Problem Statement}
So far in the class, we've applied moment methods to a random variable of interest $Z$ which counts discrete things, such as $Z \coloneqq \#\{x \text{ satisfying }F\}$ for the random $k$-SAT problem. Here, $F: \{0,1\}^n \rightarrow \{0,1\}$ was the random Boolean formula acting on $n$ Boolean input values. Today, we're turning our attention to continuous landscapes and solving counting problems over a continuous set.

\begin{itemize}
    \item {\bf Given:} A random function $F: \RR \rightarrow \RR$.
    \item {\bf Goal:} For $N = N(F,A) \coloneqq \#\{x \in A \subseteq \RR : F(x) = 0 \}$, calculate $\EE N$, $\EE N^2$.
\end{itemize}
Examples of such a function $F$ include:
\begin{itemize}
    \item $F(x) = \sum_{a=0}^d g_a x^a$, where each $g_a \sim \sN(0, \sigma_a^2)$ (Kac)
    \item $F(x) = \sum_{a=0}^d g_a \cos(ax)$, again each $g_a \sim \sN(0, \sigma_a^2)$ (Rice)
\end{itemize}
If we think of $A$ as some interval in $\RR$, like $A=[a,b]$, we can think of $N$ as counting the number of roots of, say, a random polynomial in this interval. However, $A$ is some uncountable set (for instance, not something like the unit hypercube $\{0,1\}^n$), so we cannot write $N$ in the form
\[ N \neq \sum_{x \in A} \One \{ F(x) = 0 \}.\]
Instead, we need to express it in integral form
\[ N = \int_A (\,\, \cdots) ~dx. \]
We will use the Dirac delta function $\delta(\cdot)$, which is characterized by the formal property ``$\int_{-\infty}^\infty f(x) \delta(x) dx = f(0)$.''

\paragraph{Aside:} The Dirac delta function is not a function in the traditional sense. We can define the Dirac measure which accepts a subset $A$ of the real line as an argument and returns $\delta(A) = 1$ if $0 \in A$ and $\delta(A) = 0$ otherwise. Then the Lebesgue integral with respect to $\delta$ satisfies
\[ \int_{-\infty}^\infty f(x) \delta(dx) = f(0). \]
The measure $\delta$ is not absolutely continuous with respect to the Lebesgue measure, so no Radon-Nikodym derivative with respect to Lebesgue measure (that is, no density) exists, and so we cannot actually write
\[ \int_{-\infty}^\infty f(x) \delta(x) dx = f(0),\]
but if we understand this expression as the former, the latter is a slight abuse of notation.

\section{Deriving the Formula}
\subsection{One Dimension}
\paragraph{Intuitive motivation.} While considering how to write what $N$ is, we also want to consider invariances in the problem (e.g., we do not want the formula to depend on stretching the interval). For this, recall that we can represent a usual integral which is the area over a flat surface, as a line integral which is the area over a curved surface.

\begin{figure}[H]
    \centering
    \includegraphics[width=0.8\columnwidth]{2023-02-09-fig1.png}
    \caption{(Left) Integral of a function over $x$ between $x=0$ and $x=5$ can be represented as a line integral over the curve $\gamma$ (Right).}
    \label{fig:line_integral_1d}
\end{figure}

 Define
 \[ N \stackrel{\Large{\star}}{=} \int_\gamma \delta(\omega) d\omega = \int_a^b \delta(F(x))\underbrace{\left| F'(x) \right|}_{\text{``length''}} \, dx. \]
 Then we have
 \[ \EE N = \int_a^b \EE\big[ \underbrace{\delta(F(x))\left| F'(x) \right|}_{\text{r.v. parameterized by $x$}} \big] dx. \]
 Conditioning on $F(x) = y$:
 \begin{align*}
     \EE N &= \int_a^b \int_{-\infty}^\infty \EE\left[ \delta(F(x)) \left|F'(x)\right| \,\bigg|\, F(x) = y \right] p_{F(x)}(y) \, dy \, dx  \tag{$p_{F(x)}$ is density of $F(x)$} \\
     &= \int_a^b \int_{-\infty}^\infty \delta(F(y)) \EE \left[ \left| F'(x) \right| \,\bigg|\, F(x) = y \right]p_{F(x)}(y) \, dy \, dx \\
     &= \boxed{\int_a^b \EE \left[ \left| F'(x) \right| \,\bigg|\, F(x) = 0 \right] p_{F(x)}(0) dx} \tag{1D Kac-Rice Formula}
 \end{align*}

 \paragraph{More carefully (Kac counting formula).}
Suppose $F(x)$ satisfies the following conditions:
\begin{itemize}
    \item $F \in \mathcal{C}^1$, that is $F$ continuously differentiable.
    \item $F(a) \neq 0$ and $F(b) \neq 0$. (Otherwise, the contribution around $F(a)$ or $F(b)$ would be $\epsilon$, and thus contribute $1/2$ to $N$ which ideally should be a whole number.)
    \item $F(x) = 0 \Rightarrow F'(x) \neq 0$. That is, we need the root to be a ``full crossing,'' e.g., not a flat landscape.
\end{itemize}

Then, the \emph{Kac counting formula} says that, for all $\epsilon > 0$ sufficiently small, we have
 \[ N = \frac{1}{2\epsilon} \int_a^b \One_{[-\epsilon, \epsilon]}(F(x)) \left| F'(x) \right| \, dx. \]
See Figure~\ref{fig:esp_neighborhood}.

\begin{figure}[H]
    \centering
    \includegraphics[width=0.8\columnwidth]{2023-02-09-fig2.png}
    \caption{As long as the function $F(x)$ is ``nicely behaved,'' the contribution of each root is, by FTC, $\int_{a'}^{b'} \left| F'(x) \right| dx = 2\epsilon $. Thus, it contributes 1 to $N$.}
    \label{fig:esp_neighborhood}
\end{figure}

\subsection{Multiple Dimensions}

Generalizing the 1-dimensional formula, suppose instead that $F$ is a function
\[ F: \RR^N \to \RR^N. \]
In this case, zeros of $F$ are ``generically isolated,'' so it still makes sense to count them. In addition, the following changes occur:
\begin{itemize}
    \item The set $A$ is some measurable set in $\RR^N$.
    \item The curve $\gamma$ becomes a manifold embedded in $\RR^N$.
    \item The curve length $|F'(x)|$ becomes the determinants of the Jacobian of $F$, i.e., $\left|\det\left( (DF)(x) \right) \right|$ which can be seen as infinitesimal area or $N$-volume.
\end{itemize}
Then our $N$ becomes now
\[ N = \int_\gamma \delta(\omega) d\omega = \int_A \delta(F(x)) \left| \det\left( (DF)(x) \right) \right| dx. \]
And subsequently,
\begin{align*}
    \EE N &= \int_A \EE \left[ \left| \det\left( (DF)(x) \right) \right| \,\bigg|\, F(x) = 0 \right] p_{F(x)}(0) \, dx \tag{$N$-dim Kac-Rice formula}
\end{align*}
Note that $0$ here is the $N$-dimensional $0$ vector.

\section{Applications}
One important application of the Kac-Rice formula is to counting the critical points of a random function.
\begin{itemize}
    \item {\bf Given: } A function $f: \RR^N \mapsto \RR$. Define $F(x) \coloneqq \nabla f(x)$. Note that $F: \RR^N \mapsto \RR^N$.
    \item {\bf Quantity of interest: } $N(f, A) = \#\{\text{critical points of $f$ in $A$} \} \coloneqq \text{Crit}(f, A)$.
\end{itemize}
In this case, the Jacobian is the Hessian $(DF)(x) = \nabla^2 f(x)$, so the $N$-dimensional Kac-Rice formula yields the following:
\[ \EE \text{Crit}(f, A) = \int_A \EE \left[ \left| \det\left(\nabla^2 f(x) \right) \right| \,\bigg|\, \nabla f(x) = 0 \right] p_{\nabla f(x)}(0)\, dx. \]
This is the Kac-Rice formula for critical points.

\paragraph{Remarks.} We often want to find critical points subject to some further constraints, such as:
\begin{itemize}
    \item Constraints on the value of $f(x)$, like $f(x) \in B \subseteq \RR$.
    \item Specific ``index'' of the critical point, e.g., ``saddleness'' of the critical point (the number of positive eigenvalues of the Hessian).
\end{itemize}
For these, we can add an indicator in the formula for $N$ under the integral, and the same calculations will go through. We can also in principle write similar integral formulas for the higher moments $\EE \text{Crit}(f, A)^k$ is, but in practice this is very hard to compute except in special very symmetric settings.

\paragraph{Example.}
Let \[f(x) = \frac{\alpha}{2}\|x\|^2 + g(x),\] where
\[ g(x) = \sum_{\substack{i_1, \dots, i_d \in [N] \\ s_1, \dots, s_d \in \{0, 1\}}} \underbrace{w_{i_1, \dots, i_d, s_1, \dots, s_d}}_{\sim \mathcal{N}(0, 1)} \begin{pmatrix}
\cos(x_{i_1}) &\text{if }s_1 = 0 \\
\sin(x_{i_1}) &\text{if }s_1 = 1
\end{pmatrix}
\cdots
\begin{pmatrix}
\cos(x_{i_d}) &\text{if }s_d = 0 \\
\sin(x_{i_d}) &\text{if }s_d = 1
\end{pmatrix}.
\]
Note that $g(x)$ is a Gaussian process with $\EE g(x) = 0$. If we define
\[ K(x,y) \coloneqq \EE g(x) g(y) = \left(\cos(x_1 - y_1) \cdots + \cos(x_N - y_N) \right)^d, \]
then this is also equal to $K(x-y)$, which says that the process is ``stationary,'' which will be explored more in the next lecture.

Intuitively, $f(x)$ looks like a paraboloid plus some ``spatially uniform'' noise, illustrated below. Asking whether this $f$ has critical points is asking whether the well is ``steep'' enough, so we can overcome the noise to reach 0 by a process like gradient descent.
\begin{figure}[H]
    \centering
    \includegraphics[width=0.8\columnwidth]{2023-02-09-fig3.png}
    \caption{Intuitive illustration of what $f(x)$ looks like.}
    \label{fig:fx_ex}
\end{figure}

\paragraph{Next time.} To complete our calculation, we will calculate correlations between the derivatives of $g$ in terms of the covariance by differentiating under the expectation.
This will give identities that look like:
\[ \EE \left[ \frac{\partial g}{\partial x_i}(x) \frac{\partial g}{\partial x_j \partial x_k}(x) \right] = \frac{\partial^3 K}{\partial x_i \partial y_j, \partial y_k}(x,x). \]

We will find that, for any fixed $x$, $\nabla f(x)$ and $\nabla^2 f(x)$ are independent. This means that the conditioning in the Kac-Rice formula has no effect:
\[ \EE \text{Crit}(f, A) = \int_A \EE \left[ \left| \det \nabla^2 f(x) \right| \mid \cancel{\nabla f(x) = 0} \right] p_{\nabla f(x)}(0) dx. \]
Most of our work will then be the analysis of the random matrix $\nabla^2 f(x)$.

\end{document}
