% Template based on:
% https://www.cs.columbia.edu/~djhsu/coms4774-s21/scribe.html

\documentclass[12pt]{article}

\usepackage{amscd}
\usepackage{amsmath}
\usepackage{float}
\usepackage{amssymb}
\usepackage{amstext}
\usepackage{amsthm}
\usepackage{bbold}
\usepackage{bm}
\usepackage{colonequals}
\usepackage[dvips,letterpaper,margin=1in]{geometry}
\usepackage{mathtools}
\usepackage{graphicx}
\usepackage{xcolor}

\usepackage{csquotes}
\MakeOuterQuote{"}

\definecolor{cblack}{rgb}{0,0,0}
\definecolor{cblue}{rgb}{0.121569,0.466667,0.705882}
\definecolor{corange}{rgb}{1.000000,0.498039,0.054902}
\definecolor{cgreen}{rgb}{0.172549,0.627451,0.172549}
\definecolor{cred}{rgb}{0.839216,0.152941,0.156863}
\definecolor{cpurple}{rgb}{0.580392,0.403922,0.741176}
\definecolor{cbrown}{rgb}{0.549020,0.337255,0.294118}
\definecolor{cpink}{rgb}{0.890196,0.466667,0.760784}
\definecolor{cgray}{rgb}{0.498039,0.498039,0.498039}

\usepackage{hyperref}
\hypersetup{
  linkcolor  = cblue,
  citecolor  = cgreen,
  urlcolor   = corange,
  colorlinks = true,
}

% Lecture information
\newcommand\coursename{CPSC 664, Spring 2023}
\newcommand\scribe{Curtis McDonald}
\newcommand\lecturer{Tim Kunisky}
\newcommand\lecturedate{April 11, 2023}
\newcommand\lecturetitle{Lecture 21: Rigorous Analysis of SK Model}

% Define theorem environments here.
\newtheorem{claim}{Claim}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{remark}[theorem]{Remark}
\newtheorem{assumption}[theorem]{Assumption}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{question}[theorem]{Question}
\newtheorem{problem}[theorem]{Problem}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{example}[theorem]{Example}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{conjecture}[theorem]{Conjecture}

% Some of my macros
\renewcommand{\AA}{\mathbb{A}}
\newcommand{\CC}{\mathbb{C}}
\newcommand{\DD}{\mathbb{D}}
\newcommand{\EE}{\mathbb{E}}
\newcommand{\FF}{\mathbb{F}}
\newcommand{\HH}{\mathbb{H}}
\newcommand{\NN}{\mathbb{N}}
\newcommand{\PP}{\mathbb{P}}
\newcommand{\QQ}{\mathbb{Q}}
\newcommand{\RR}{\mathbb{R}}
\renewcommand{\SS}{\mathbb{S}}
\newcommand{\ZZ}{\mathbb{Z}}
\DeclareSymbolFont{bbold}{U}{bbold}{m}{n}
\DeclareSymbolFontAlphabet{\mathbbold}{bbold}
\newcommand{\One}{\mathbbold{1}}

\newcommand{\ba}{\bm a}
\newcommand{\bb}{\bm b}
\newcommand{\bc}{\bm c}
\newcommand{\bd}{\bm d}
\newcommand{\be}{\bm e}
\newcommand{\bg}{\bm g}
\newcommand{\bh}{\bm h}
\newcommand{\bi}{\bm i}
\newcommand{\bj}{\bm j}
\newcommand{\bk}{\bm k}
\newcommand{\bbm}{\bm m}
\newcommand{\bp}{\bm p}
\newcommand{\bq}{\bm q}
\newcommand{\br}{\bm r}
\newcommand{\bs}{\bm s}
\newcommand{\bt}{\bm t}
\newcommand{\bu}{\bm u}
\newcommand{\bv}{\bm v}
\newcommand{\bw}{\bm w}
\newcommand{\bx}{\bm x}
\newcommand{\by}{\bm y}
\newcommand{\bz}{\bm z}

\newcommand{\bA}{\bm A}
\newcommand{\bB}{\bm B}
\newcommand{\bC}{\bm C}
\newcommand{\bD}{\bm D}
\newcommand{\bE}{\bm E}
\newcommand{\bF}{\bm F}
\newcommand{\bG}{\bm G}
\newcommand{\bH}{\bm H}
\newcommand{\bI}{\bm I}
\newcommand{\bL}{\bm L}
\newcommand{\bM}{\bm M}
\newcommand{\bN}{\bm N}
\newcommand{\bP}{\bm P}
\newcommand{\bQ}{\bm Q}
\newcommand{\bR}{\bm R}
\newcommand{\bS}{\bm S}
\newcommand{\bT}{\bm T}
\newcommand{\bU}{\bm U}
\newcommand{\bV}{\bm V}
\newcommand{\bW}{\bm W}
\newcommand{\bX}{\bm X}
\newcommand{\bY}{\bm Y}
\newcommand{\bZ}{\bm Z}

\newcommand{\zero}{\bm{0}}
\newcommand{\one}{\bm{1}}

\newcommand{\sA}{\mathcal{A}}
\newcommand{\sB}{\mathcal{B}}
\newcommand{\sC}{\mathcal{C}}
\newcommand{\sD}{\mathcal{D}}
\newcommand{\sE}{\mathcal{E}}
\newcommand{\sF}{\mathcal{F}}
\newcommand{\sG}{\mathcal{G}}
\newcommand{\sH}{\mathcal{H}}
\newcommand{\sI}{\mathcal{I}}
\newcommand{\sJ}{\mathcal{J}}
\newcommand{\sK}{\mathcal{K}}
\newcommand{\sL}{\mathcal{L}}
\newcommand{\sM}{\mathcal{M}}
\newcommand{\sN}{\mathcal{N}}
\newcommand{\sO}{\mathcal{O}}
\newcommand{\sP}{\mathcal{P}}
\newcommand{\sQ}{\mathcal{Q}}
\newcommand{\sR}{\mathcal{R}}
\newcommand{\sS}{\mathcal{S}}
\newcommand{\sT}{\mathcal{T}}
\newcommand{\sU}{\mathcal{U}}
\newcommand{\sV}{\mathcal{V}}
\newcommand{\sX}{\mathcal{X}}
\newcommand{\sY}{\mathcal{Y}}

\newcommand{\fB}{\mathscr{B}}
\newcommand{\fC}{\mathscr{C}}
\newcommand{\fE}{\mathscr{E}}
\newcommand{\fH}{\mathscr{H}}
\newcommand{\fI}{\mathscr{I}}
\newcommand{\fP}{\mathscr{P}}
\newcommand{\fV}{\mathscr{V}}

\DeclareSymbolFont{sfoperators}{OT1}{cmss}{m}{n}
% don't waste a math group
\DeclareSymbolFontAlphabet{\mathsf}{sfoperators}
% tell LaTeX to use sfoperators for names of operators
\makeatletter
\renewcommand{\operator@font}{\mathgroup\symsfoperators}
\makeatother

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\GOE}{GOE}
\DeclareMathOperator{\sym}{sym}
\DeclareMathOperator{\Part}{Part}
\DeclareMathOperator{\poly}{poly}
\DeclareMathOperator{\Unif}{Unif}
\DeclareMathOperator{\Var}{Var}
\DeclareMathOperator{\Cov}{Cov}
\DeclareMathOperator{\Tr}{Tr}
\DeclareMathOperator{\rank}{rank}
\DeclareMathOperator{\obj}{obj}
\DeclareMathOperator{\diag}{diag}
\DeclareMathOperator{\sgn}{sgn}

\renewcommand\Re{\operatorname{Re}}
\renewcommand\Im{\operatorname{Im}}
% These put symbols below.
\newcommand{\Px}{\mathop{\mathbb{P}}}
\newcommand{\Ex}{\mathop{\mathbb{E}}}
\newcommand{\Varx}{\mathop{\mathsf{Var}}}

\newcommand\numberthis{\addtocounter{equation}{1}\tag{\theequation}}

%
% Put your custom macros here if you want.
%

\begin{filecontents}{2023-01-17.bib}
@book{MM-2011-NatureOfComputation,
  title={The nature of computation},
  author={Moore, Cristopher and Mertens, Stephan},
  year={2011},
  publisher={OUP Oxford}
}

@book{Durrett-2019-ProbabilityTheoryExamples,
  title={Probability: theory and examples},
  author={Durrett, Rick},
  volume={49},
  year={2019},
  publisher={Cambridge University Press}
}
\end{filecontents}

\begin{document}

% This creates the header for scribe notes.
\noindent \coursename \hfill Scribe: \scribe \\
\lecturedate \hfill Lecturer: \lecturer
\vspace{1em}

\hrule

\vspace{1.5em}
\begin{center}
  {\Large\lecturetitle}
\end{center}

\section{Proof Ideas for SK Model}

\subsection*{More about Parisi Formula}
\subsubsection*{Overlap Density}
Last class, we discussed the Parisi formula for analyzing the limit of the free energy as $n \to 0$. We saw this unusually ``flipping'' of the order of indexes $m_{j}$ to now be between 0 and 1. Here we review the conclusions:

Recall the block replica symmetry breaking ansatz matrix $Q^{*} \in \mathbb{R}^{n \times n}$ with parameters $n \geq m_{1} \geq \cdots, m_{k} >1$ and $a_{0} \leq a_{1}\leq a_{k} \leq \cdots 1$. The entry $a_{j}$ appears $n(m_{j} - m_{j+1})$ times, thus the proportion of the matrix which is entry $a_{j}$ is $\frac{n(m_{j}-m_{j+1})}{n(n-1)}$. As $n \to 0$, this approaches $m_{j+1} - m_{j}$, i.e. the ``probability'' of drawing $a_{j}$ from the matrix approaches $m_{j+1} - m_{j}$.

\begin{figure}[H]
\begin{center}
\includegraphics[scale=0.1]{2023-04-11-fig1.jpg}
\end{center}
\end{figure}

Thus, consider $n = 0 = m_{0} \leq m_{1} \leq \cdots, \leq m_{k+1} = 1$ and define the step function
\begin{align*}
a:[0,1] &\to [0,1]\\
	x &\mapsto \sum_{j=1}^{k+1} 1\{ m_{j-1} \leq x < m_{j}\} a_{j}
\end{align*}
that is $a(x)$ is constant value $a_{j}$ on interval $[m_{j-1}, m_{j})$.


Consider this as the cumulative distribution function (CDF) of the overlap distribution $\mu$ which is a probability measure over the interval $[0,1]$ defined by
\begin{align*}
\mu([B,A]) = a^{-1}(B) - a^{-1}(A)
\end{align*}

As the number of replica symmetry breaking terms $k \to \infty$, the step function approaches a continuous CDF function which defines the overlapp density $\mu$ via this CDF.

\begin{figure}[H]
\begin{center}
\includegraphics[scale=0.1]{2023-04-11-fig2.jpg}
\end{center}
\end{figure}

\subsubsection*{Connections to Heat Formula}

In the Parisi formula, we have second derivative terms $\frac{\partial^{2}}{\partial h^{2}}$ appearing from the expression $E_{g \sim N(0,1)}[f(g+h)]$. This is a consequence of the heat equation.

The heat equation tells us the solution of the PDE
\begin{align}
\frac{\partial}{\partial t} u(t,h)&= \frac{1}{2}\frac{\partial^{2}}{\partial h^{2}} u(t,h)\label{heat_eqn}
\end{align}
with initial condition $u(0,h) = f(h)$. I.e., the time derivative of $u$ is equal to half the second derivative in space, with initial condition at time 0. The solution to the heat equation is
\begin{align}
u(t,h)&= E_{g \sim N(0,t)}[f(g+h)]
\end{align}
thus by the heat equation,
\begin{align*}
\frac{\partial}{\partial t}E_{g \sim N(0,t)}[f(g+h)]&= \frac{\partial^{2}}{\partial h^{2}}E_{g \sim N(0,t)}[f(g+h)]
\end{align*}

\subsection*{Talagrand Proof Ideas}
Consider $x \in \{\pm \frac{1}{\sqrt{N}}\}^{N}$ (so hyper cube normalized to be unit vectors) and we are given a random matrix $W$. For each entry $x$ in unit hypercube, we have value $x^{T}Wx$. Put these together in a vector $G^{0}_{x} \in \mathbb{R}^{2^{N}}$
\begin{align*}
G^{0}_{x} = \begin{pmatrix}
x_{1}^{T}Wx_{1}\\
x_{2}^{T}Wx_{2}\\
\vdots\\
x_{2^{N}}^{T}Wx_{2^{N}}
\end{pmatrix}
\end{align*}
can also think of $G^{0}_{x}$ being defined by the choice of $W$, thus is a mapping from $R^{N \times N} \to R^{N}$, maps a matrix to a vector. When we write $G^{0}_{x}, G^{0}_{y}$ we refer to single entries of the vector $G^{0}$ at values $x$ and $y$. We note a few properties of the mean and covariance of the vector $G^{0}_{x}$,
\begin{align*}
E_{W}[G^{0}_{x}] &= 0\\
E_{W}[G^{0}_{x}G^{0}_{y}] &= E_{W}[x^{T}Wxy^{T}Wy]\\
&=E_{W}[\left(\sum_{i=1}^{N}x_{i}^{2}W_{i,i}+2\sum_{j>i}W_{i,j}x_{i}x_{j}\right)\left(\sum_{i=1}^{N}y_{i}^{2}W_{i,i}+2\sum_{j>i}W_{i,j}y_{i}y_{j} \right)]\\
&=\frac{2}{N} \sum_{i=1}^{N}x_{i}^{2}y_{i}^{2}+\frac{4}{N} \sum_{j > i}x_{i}x_{j}y_{i}y_{j}\\
&= \frac{2}{N} \langle x, y\rangle^{2}
\end{align*}
The quantity we are interested in studying is the expected maximum of $\frac{1}{N}x^{T}Wx$ over the original hypercube, which is equal to
\begin{align}
E_{W}[M(W)] = E_{W}[\max_{x \in \{\pm \frac{1}{\sqrt{N}}\}} G^{0}_{x}]
\end{align}
that is, the expected maximum entry of the $G^{0}_{x}$ vector. We will provide an upper bound on this quantity.

\section{Gaussian Interpolation}
Gaussian interpolation is a method to bound the expected value of a function over a Gaussian by comparing the value to an expectation over a simpler Gaussian which we can compute.

For some function $F:\mathbb{R}^{M} \to \mathbb{R}$, we want to study $E_{g \sim N(0, \Sigma^{0})}[F(g)]$ that is the expected value of $F$ over a Gaussian with covariance $\Sigma^{0}$. We construct a continuous evolution from covariance matrix $\Sigma^{(0)} \to \Sigma^{(1)}$ by $\Sigma: [0,1] \to R^{M \times M}_{\geq 0}$. Define the function of time
\begin{align*}
f(t)&= E_{g \sim N(0, \Sigma(t))}[F(g)]
\end{align*}
by fundamental theorem of calculus,
\begin{align*}
f(1) - f(0)&= \int_{0}^{1}f'(t)dt\\
f'(t)&= \frac{\partial}{\partial t} E_{g \sim N(0, \Sigma(t))}[F(g)]
\end{align*}
if we can show this derivative $f'(t)>0$ for all $t$, then $f(1) > f(0)$ and solving $f(1)$ provides an upper bound on the quantity $f(0)$ we want to study.
Pick a matrix $A(t)$ such that the covariance matrix is the square of this matrix $\Sigma(t) =A(t)A(t)$. Then
\begin{align*}
f(t)&= E_{g \sim N(0, I)}[F(A(t) g)]
\end{align*}
all $t$ dependence is inside $F$ function and we can take derivatives more easily.
\begin{align*}
f'(t)&= \frac{\partial}{\partial t}E_{g \sim N(0, I)}[F(A(t)g)]\\
&=E_{g \sim N(0, I)}[ \frac{\partial}{\partial t}F(A(t)g)]\\
&=E_{g}[\langle \nabla F(A(t)g), A'(t)g \rangle]\\
&=E_{g}[\sum_{i=1}^{M}\partial_{i}F(A(t)g)\sum_{j}A'(t)_{i,j}g_{j}]\\
&=\sum_{j}E_{g}[\sum_{i=1}^{M}\partial_{i}F(A(t)g)A'(t)_{i,j}g_{j}]\\
&=\sum_{j}E_{g}[\sum_{i=1}^{M}\sum_{k=1}^{M}\partial_{i}\partial_{k}F(A(t)g)A(t)_{k,j}A'(t)_{i,j}]\quad\text{(Integration by parts)}\\
&=\sum_{i}\sum_{k}E_{g}[\partial_{i}\partial_{k}F(A(t) g) (A(t)A'(t))_{i,k}]\\
&= E_{g}[ \langle \nabla^{2}F(A(t)g), A(t)A'(t) \rangle]
\end{align*}
that is, expected sum of entry wise multiplication of hessian of $F$ and $A(t)A'(t)$.
We can ``symmetrize'' the inner product, for any matrix $H$
\begin{align*}
\langle H, A(t)A'(t) \rangle = \langle H,  \frac{A(t)A'(t)+A'(t)^{T}A(t)^{T}}{2}\rangle
\end{align*}
thus define the symmetrized matrix
\begin{align*}
\Sigma'(t)&= A(t)A'(t)+A'(t)^{T}A(t)^{T}
\end{align*}
which yields the following lemma.
\begin{lemma}
\begin{align*}
f'(t)&= \frac{1}{2} \langle \Sigma'(t), E_{g \sim N(0, I)}[\nabla^{2}F(g) ]\rangle
\end{align*}
\end{lemma}

\subsection*{Consequences of Lemma}
We want to deduce situations where the above  inner product value is positive, thus $f(1)$ provides an upper bound on $f(0)$. Say we use Gaussian interpolation
\begin{align*}
\Sigma(t)&= (1-t)\Sigma^{(0)} + t \Sigma^{(1)}\\
\Sigma'(t)&= \Sigma^{(1)} - \Sigma^{(0)}
\end{align*}
constant time change for any $t$. Thus as a corollary
\begin{corollary}
If $\langle \nabla^{2}F(x), \Sigma^{(1)} - \Sigma^{(0)} \rangle \geq 0~\forall ~x$, then we have that $f(0) \leq f(1)$.
\end{corollary}
Note that we only want the expected value to be positive, so ensuring the inner product is positive at all locations is a more restrictive condition then we need but implies the expectation is positive.
Furthermore, if $F(x)$ is convex, it's Hessian is positive definite, thus we would only require $\Sigma^{(1)} - \Sigma^{(0)}$ to be PSD as well.
\begin{corollary}
If $F$ is convex and $\Sigma^{(1)} - \Sigma^{(0)}$ is PSD, then $f(1) \geq f(0)$.
\end{corollary}

Finally, as a most restrictive condition of all, we can ensure that entry wise, the multiplication of $\nabla^{2}F$ and $\Sigma^{(1)}- \Sigma^{(0)}$ is positive
\begin{corollary}\label{elementwise_cor}
If
\begin{align*}
\begin{cases}
\partial_{i}\partial_{j} F(x) \geq 0 &\Sigma^{(1)}_{i,j} \geq \Sigma^{(0)}_{i,j}\\
\partial_{i}\partial_{j} F(x) \leq 0 &\Sigma^{(1)}_{i,j} < \Sigma^{(0)}_{i,j}
\end{cases}
\end{align*}
for all $x$, then $f(1) \geq f(0)$.
\end{corollary}

\section{Application of Gaussian Interpolation to Maximum Value Problem}
Our goal is to study the function
\begin{align*}
F(g)&= \max_{x \in \{\pm \frac{1}{\sqrt{N}}\}}g_{x}
\end{align*}
but instead we study the differentiable soft max function and take $\beta \to \infty$,
\begin{align*}
F_{\beta}(g)&= \frac{1}{\beta}\log (\sum_{x}e^{\beta g_{x}})
\end{align*}
called the ``Free Energy.''
\begin{align*}
\partial_{i} F_{\beta}(g)&= \frac{e^{\beta g_{i}}}{\sum_{j}e^{\beta g_{j}}}
\end{align*}
denote the value $p_{i}(g) = \frac{e^{\beta g_{i}}}{\sum_{j}e^{\beta g_{j}}}$. Note $p$ as a vector is positive and sums to one, can be thought of as a probability density or weighting over values $i$. Second derivative is then
\begin{align*}
\partial_{i}\partial_{j} F_{\beta}(g)&= \begin{cases}
p_{i}(g)-p_{i}(g)^{2}& i = j\\
-p_{i}(g)p_{j}(g)& i \neq j\
\end{cases}
\end{align*}
since the $p_{i}(x)$ are all positive and less than $1$, we have that $p_{i}(g) - p_{i}(g)^{2} \geq 0$ and $-p_{i}(g)p_{j}(g) \leq 0$. Thus, by element wise corollary \ref{elementwise_cor}, we want a matrix $\Sigma^{(1)}$ which is larger than $\Sigma^{(0)}$ on all diagonal entries, and smaller on all off diagonal entries.
\begin{lemma}[Weak Sudakov-Fernique Inequality] If $\Sigma^{(1)}_{i,i} \geq \Sigma^{(0)}_{i,i}$, $\Sigma^{(1)}_{i,j} \leq \Sigma^{(0)}_{i,j} i \neq j$ then
\begin{align*}
E_{\Sigma^{(0)}}[\max_{x} g_{x}] \leq E_{\Sigma^{(1)}}[\max_{x} g_{x}]
\end{align*}
\end{lemma}
For example, consider a rank 1 matrix $\Sigma = 11^{T}$. By above result, max value under this covariance is less than max value under identity covariance $\Sigma = I_{M \times M}$. Furthermore, expected value is even larger for matrix with diagonal entries $1$ and non-diagonal entries constant $-\frac{1}{M-1}$. Thus weak Sudakov inequality gives a convenient way to compare expected max values simply by properties of covariance matrix entries.

\section{Preview of Next Class}
We will cover the Strong Sudakov Fernique inequality, which covers the covariance matrices defined by
\begin{align*}
\Sigma^{(0)}_{xy} &= \frac{2}{N} \langle x, y \rangle^{2}\\
\Sigma^{(1)}_{xy}&=\frac{4}{N} \langle x, y \rangle
\end{align*}
turns out $\Sigma^{(1)}$ above is covariance matrix of Gaussian process
\begin{align*}
G^{(1)}_{x} = \frac{2}{\sqrt{n}} \langle g, x \rangle
\end{align*}
under these results
\begin{align*}
E(\frac{M(W)}{N}) \leq E( \max_{x \in \{\pm \frac{1}{\sqrt{N}}\}} \langle g, x \rangle]&= \frac{2}{N}E[\|g\|_{1}]\\
&=2 E_{g \sim N(0,1)}[|g|]\\
&= 2 \sqrt{\frac{2}{\pi}}
\end{align*}
which gives same upper bound provided by the replica-symmetric solution of Sherrington and Kirkpatrick,
\begin{align*}
2P^{*}&\leq 2 \sqrt{\frac{2}{\pi}}
\end{align*}
but as an upper bound and with a more rigorous formulation.

\end{document}
