% Template based on:
% https://www.cs.columbia.edu/~djhsu/coms4774-s21/scribe.html

\documentclass[12pt]{article}

\usepackage{amscd}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amstext}
\usepackage{amsthm}
\usepackage{bbold}
\usepackage{bm}
\usepackage{colonequals}
\usepackage[dvips,letterpaper,margin=1in]{geometry}
\usepackage{mathtools}
\usepackage{graphicx}
\usepackage{xcolor}

\usepackage{csquotes}
\MakeOuterQuote{"}

\definecolor{cblack}{rgb}{0,0,0}
\definecolor{cblue}{rgb}{0.121569,0.466667,0.705882}
\definecolor{corange}{rgb}{1.000000,0.498039,0.054902}
\definecolor{cgreen}{rgb}{0.172549,0.627451,0.172549}
\definecolor{cred}{rgb}{0.839216,0.152941,0.156863}
\definecolor{cpurple}{rgb}{0.580392,0.403922,0.741176}
\definecolor{cbrown}{rgb}{0.549020,0.337255,0.294118}
\definecolor{cpink}{rgb}{0.890196,0.466667,0.760784}
\definecolor{cgray}{rgb}{0.498039,0.498039,0.498039}

\usepackage{hyperref}
\hypersetup{
  linkcolor  = cblue,
  citecolor  = cgreen,
  urlcolor   = corange,
  colorlinks = true,
}

% Lecture information
\newcommand\coursename{CPSC 664, Spring 2023}
\newcommand\scribe{Peixin You}
\newcommand\lecturer{Tim Kunisky}
\newcommand\lecturedate{March 30, 2023}
\newcommand\lecturetitle{Lecture 18: Replica analysis of SK model}

% Define theorem environments here.
\newtheorem{claim}{Claim}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{remark}[theorem]{Remark}
\newtheorem{assumption}[theorem]{Assumption}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{question}[theorem]{Question}
\newtheorem{problem}[theorem]{Problem}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{example}[theorem]{Example}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{conjecture}[theorem]{Conjecture}

% Some of my macros
\renewcommand{\AA}{\mathbb{A}}
\newcommand{\CC}{\mathbb{C}}
\newcommand{\DD}{\mathbb{D}}
\newcommand{\EE}{\mathbb{E}}
\newcommand{\FF}{\mathbb{F}}
\newcommand{\HH}{\mathbb{H}}
\newcommand{\NN}{\mathbb{N}}
\newcommand{\PP}{\mathbb{P}}
\newcommand{\QQ}{\mathbb{Q}}
\newcommand{\RR}{\mathbb{R}}
\renewcommand{\SS}{\mathbb{S}}
\newcommand{\ZZ}{\mathbb{Z}}
\DeclareSymbolFont{bbold}{U}{bbold}{m}{n}
\DeclareSymbolFontAlphabet{\mathbbold}{bbold}
\newcommand{\One}{\mathbbold{1}}

\newcommand{\ba}{\bm a}
\newcommand{\bb}{\bm b}
\newcommand{\bc}{\bm c}
\newcommand{\bd}{\bm d}
\newcommand{\be}{\bm e}
\newcommand{\bg}{\bm g}
\newcommand{\bh}{\bm h}
\newcommand{\bi}{\bm i}
\newcommand{\bj}{\bm j}
\newcommand{\bk}{\bm k}
\newcommand{\bbm}{\bm m}
\newcommand{\bp}{\bm p}
\newcommand{\bq}{\bm q}
\newcommand{\br}{\bm r}
\newcommand{\bs}{\bm s}
\newcommand{\bt}{\bm t}
\newcommand{\bu}{\bm u}
\newcommand{\bv}{\bm v}
\newcommand{\bw}{\bm w}
\newcommand{\bx}{\bm x}
\newcommand{\by}{\bm y}
\newcommand{\bz}{\bm z}

\newcommand{\bA}{\bm A}
\newcommand{\bB}{\bm B}
\newcommand{\bC}{\bm C}
\newcommand{\bD}{\bm D}
\newcommand{\bE}{\bm E}
\newcommand{\bF}{\bm F}
\newcommand{\bG}{\bm G}
\newcommand{\bH}{\bm H}
\newcommand{\bI}{\bm I}
\newcommand{\bL}{\bm L}
\newcommand{\bM}{\bm M}
\newcommand{\bN}{\bm N}
\newcommand{\bP}{\bm P}
\newcommand{\bQ}{\bm Q}
\newcommand{\bR}{\bm R}
\newcommand{\bS}{\bm S}
\newcommand{\bT}{\bm T}
\newcommand{\bU}{\bm U}
\newcommand{\bV}{\bm V}
\newcommand{\bW}{\bm W}
\newcommand{\bX}{\bm X}
\newcommand{\bY}{\bm Y}
\newcommand{\bZ}{\bm Z}

\newcommand{\zero}{\bm{0}}
\newcommand{\one}{\bm{1}}

\newcommand{\sA}{\mathcal{A}}
\newcommand{\sB}{\mathcal{B}}
\newcommand{\sC}{\mathcal{C}}
\newcommand{\sD}{\mathcal{D}}
\newcommand{\sE}{\mathcal{E}}
\newcommand{\sF}{\mathcal{F}}
\newcommand{\sG}{\mathcal{G}}
\newcommand{\sH}{\mathcal{H}}
\newcommand{\sI}{\mathcal{I}}
\newcommand{\sJ}{\mathcal{J}}
\newcommand{\sK}{\mathcal{K}}
\newcommand{\sL}{\mathcal{L}}
\newcommand{\sM}{\mathcal{M}}
\newcommand{\sN}{\mathcal{N}}
\newcommand{\sO}{\mathcal{O}}
\newcommand{\sP}{\mathcal{P}}
\newcommand{\sQ}{\mathcal{Q}}
\newcommand{\sR}{\mathcal{R}}
\newcommand{\sS}{\mathcal{S}}
\newcommand{\sT}{\mathcal{T}}
\newcommand{\sU}{\mathcal{U}}
\newcommand{\sV}{\mathcal{V}}
\newcommand{\sX}{\mathcal{X}}
\newcommand{\sY}{\mathcal{Y}}

\newcommand{\fB}{\mathscr{B}}
\newcommand{\fC}{\mathscr{C}}
\newcommand{\fE}{\mathscr{E}}
\newcommand{\fH}{\mathscr{H}}
\newcommand{\fI}{\mathscr{I}}
\newcommand{\fP}{\mathscr{P}}
\newcommand{\fV}{\mathscr{V}}

\DeclareSymbolFont{sfoperators}{OT1}{cmss}{m}{n}
% don't waste a math group
\DeclareSymbolFontAlphabet{\mathsf}{sfoperators}
% tell LaTeX to use sfoperators for names of operators
\makeatletter
\renewcommand{\operator@font}{\mathgroup\symsfoperators}
\makeatother

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\GOE}{GOE}
\DeclareMathOperator{\sym}{sym}
\DeclareMathOperator{\Part}{Part}
\DeclareMathOperator{\poly}{poly}
\DeclareMathOperator{\Unif}{Unif}
\DeclareMathOperator{\Var}{Var}
\DeclareMathOperator{\Cov}{Cov}
\DeclareMathOperator{\Tr}{Tr}
\DeclareMathOperator{\rank}{rank}
\DeclareMathOperator{\obj}{obj}
\DeclareMathOperator{\diag}{diag}
\DeclareMathOperator{\sgn}{sgn}

\renewcommand\Re{\operatorname{Re}}
\renewcommand\Im{\operatorname{Im}}
% These put symbols below.
\newcommand{\Px}{\mathop{\mathbb{P}}}
\newcommand{\Ex}{\mathop{\mathbb{E}}}
\newcommand{\Varx}{\mathop{\mathsf{Var}}}

\newcommand\numberthis{\addtocounter{equation}{1}\tag{\theequation}}

%
% Put your custom macros here if you want.
%

\begin{document}

% This creates the header for scribe notes.
\noindent \coursename \hfill Scribe: \scribe \\
\lecturedate \hfill Lecturer: \lecturer
\vspace{1em}

\hrule

\vspace{1.5em}
\begin{center}
  {\Large\lecturetitle}
\end{center}

\section{Problem Statement}

Suppose we have a normalized GOE matrix $W\sim GOE(N,1/N)$ (so that the empirical spectral distribution converges to the semicircle distribution from $-2$ to $2$) and we have the objective function $$H(x)=x^TWx\text{ over }x\in\{\pm 1\}^N.$$ 
In this lecture, we will try to use replica analysis to solve this question.

The usual physics setup is as follows: we consider the Gibbs measure
$$\mu_{\beta}(x)=\frac{1}{Z(\beta)\exp{(\beta H(x)})}$$ which is the probability distribution over the hypercube, when $\beta$ goes bigger it weights more and more the objective function.
We define the related
$$Z_W(\beta) = \sum_{x} \exp{(\beta H(x))}, \qquad F_W(\beta)=\log{(Z(\beta))}.$$

From last time, we know $F_W$ will scale with $N$ so we want to catch that number by defining:
$$f(\beta) = \lim_{N\to \infty} \frac{1}{N}\EE_{W} F_W(\beta)$$

We can show $$2P_*:=\lim_{N\to\infty}\frac{1}{N}\EE \max_x H(x)=\lim_{\beta\to\infty} \frac{1}{\beta}f(\beta),$$
and we will try to make a prediction for $f(\beta)$ to compute the right-hand side.

\section{Idea of Replica trick}

The idea of the replica trick is that $$\EE \log Z=\lim_{n\to 0}\frac{\log \EE Z^n}{n}$$
So we could move the expectation inside the log. We want to apply the replica method on $\EE_{W} F_W(\beta)$. 
We have
\begin{align*}
    \EE F(\beta) &= \EE \log Z(\beta)\\
    &= \lim_{n\to 0}\frac{\log \EE Z(\beta)^n}{n}
\end{align*}

Also notice that
\begin{align*}
    \EE_W Z(\beta)^n &= \EE_W \sum_{x_1,\dots,x_n} \exp\left(\beta \sum_{i=1}^n H(x_i)\right) \\
    &= \sum_{x_1,\dots,x_n} \EE_W \exp\left(\beta \sum_{i=1}^n H(x_i)\right)
\end{align*}

Now we want to apply the following identity.
\begin{theorem}
    $\EE_{W\sim GOE(N,\sigma^2)} \exp{(\langle W, A\rangle)}=\exp{(\sigma^2 \| A\|^2_F)}$.
\end{theorem}
We rewrite $\exp(\beta \sum_{i=1}^n H(x_i))$ to $\langle W, \beta \sum x_i x_i^T\rangle$.
For convenience, let $X=[x_1,\dots,x_n]$, then $\sum x_i x_i^T= XX^T$.

So now we have
\begin{align*}
    \EE_W Z(\beta)^n &= \sum_{x_1,\dots,x_n}\exp\left(\frac{1}{N}\| \beta XX^T\|_F^2\right) \\
    &= \sum_{x_1,\dots,x_n}\exp{\left(N\beta^2 \left\|\frac{1}{N}X^TX\right\|^2_F\right)}.
\end{align*}
Let $Q(x_1,\dots,x_n) \colonequals \frac{1}{N}X^TX$, so $Q(x_1,\dots,x_n)_{i,j}=\frac{1}{N}\langle x_i,x_j\rangle \in [-1,1]$.
In particular, $Q\succeq 0$ and $Q_{i,i}=1$.

We then have
\begin{align*}
    \EE_W Z(\beta)^n &= \sum_{Q \text{ achievable}} \exp{(N\beta^2 \|Q\|_F^2)}\cdot \#\{(x_1,\dots,x_n)\in(\{\pm 1\}^N)^n: Q(x_1,\dots,x_n)=Q\}.
\end{align*}
We can see $$\#\{(x_1,\dots,x_n)\in(\{\pm 1\}^N)^n: Q(x_1,\dots,x_n)=Q\}=2^{nN}\PP_{x_1,\dots,x_n \sim \mathrm{Unif}(\{\pm-1\}^N)}[Q(x_1,\dots,x_n)=Q]$$
Then, by a Laplace method approximation,
\begin{align*}
    \EE_W Z(\beta)^n &\approx \exp{\left(N \sup_{Q\succeq 0,Q_{ii=1}}\left\{n\log 2+\beta^2||Q||_F^2+\frac{1}{N}\log \PP[Q(x_1,\dots,x_n)\approx Q]\right\}\right)}
\end{align*}

We give a heuristic for computing the latter probability.
Let $y_i$ be the $i$th row of $X$. so we could rewrite $Q$ to $\frac{1}{N}\sum y_i y_i^T$.
Then, we have
\begin{align*}
  \PP[Q(x_1,\dots,x_n)\approx Q] &= \PP \left[\frac{1}{N} \sum_1^N y_iy_i^T\approx Q\right]
                                   \intertext{This should be viewed as a \emph{large deviations} probability. Following scalar large deviations theory, we consider a Chernoff-type bound parametrized by a matrix $\Lambda$,}
    &\leq \PP[\exp \langle \Lambda, \sum y_iy_i^T \rangle \geq \exp \langle \Lambda, NQ \rangle] \\
    &\leq \frac{\EE \exp(\langle \Lambda, \sum y_iy_i^T)}{\exp N\langle \Lambda, Q\rangle} \\
    &= \left(\frac{\EE_y \exp y^T \Lambda y}{\exp \langle \Lambda, Q \rangle}\right)^N \\
    &:= B(\Lambda)^N.
\end{align*}
As in the scalar case, one can show (in a version of \emph{Cram\'{e}r's theorem}) that $$\PP[Q(x_1,\dots,x_n)\approx Q] \approx (\inf_{\Lambda} B(\Lambda))^N.$$
Now we have
\begin{align*}
    \EE Z(\beta)^n \approx \exp\left(N \sup_{Q}\inf_{\Lambda} \left\{n\log 2+\beta^2 ||Q||_F^2+\log \EE_y \exp(y^T\Lambda y) -\langle\Lambda,Q\rangle\right\}\right).
\end{align*}

If we don't have the restriction of $y$ in the hypercube but in the full sphere, we could compute the $\EE_y$ term explicitly.
But in our case, this term creates many complications.
As before, our approach will be to make a guess for $Q$, which will lead us to also make a guess for $\Lambda$ and simplify this expression to something more tractable.

\section{Replica Symmetric ansatz [SK]}
We will assume the optimizer $Q^{\star}$ looks like
\begin{equation*}
Q^* = 
\begin{pmatrix}
1 & a & \cdots & a \\
a & 1 & \cdots & a \\
\vdots  & \vdots  & \ddots & \vdots  \\
a & a & \cdots & 1 
\end{pmatrix}.
\end{equation*}

This means the diagonal is $1$, and all other place is constant $a$.
We may think of this as the Gram matrix describing the typical relative configuration of $n$ independent draws from the Gibbs measure $\mu_{\beta}$.
For example, when $\beta=0$, these draws will be uniform in the hypercube and we will find that this ``overlap matrix'' should look like the identity.

What we are asking is how $Q^*$ is going to change when we increase $\beta$, which means we care more about our objective function. Basically, the above assumption says inside the hypercube, there are some regions where all the good solutions are and which look like ``geometrically simple'' lower-dimensional hypercubes or spheres, and the Gibbs measure is uniform over those regions.
We will see that this guess will ultimately be wrong, but we will follow the historical development and give these details first.

In order to approximate our $\EE Z(\beta)^n$, we also want to know what $\Lambda$ could be under the assumption of our optimal $Q$. Function $h(\Lambda)=\log \EE \exp y^T\Lambda y$ is convex, so if we replace function $\Lambda$ with $\bar \Lambda_{i,i}$ be the average of the diagonal entry which is $\frac{1}{n}\sum \Lambda_{i,i}$ and $\bar \Lambda_{i,j}=\frac{1}{n(n-1)}\sum_{k\neq l}\Lambda_{k,l}$, then we can show $h(\bar\Lambda)\leq h(\Lambda)$ and $\langle \Lambda,Q\rangle =\langle\bar\Lambda,Q\rangle$. So we can assume:
\begin{equation*}
\Lambda^{\star} = 
\begin{pmatrix}
c & b & \cdots & b \\
b & c & \cdots & b \\
\vdots  & \vdots  & \ddots & \vdots  \\
b & b & \cdots & c .
\end{pmatrix}
\end{equation*}

If we look at the value of the diagonal of $\EE\exp y^T\Lambda y$ and $\langle \lambda, Q \rangle$, both of them are equal $\sum \Lambda_{i,i}$ and they have different signs in the objective function, which means the value of the diagonal won't affect the result.
So, we may assume $c = b$ so that $\Lambda^{\star} = b11^{\top}$.

Then, we have
\begin{align*}
    h(\Lambda^{\star})&= \log \EE_y \exp(b(\sum y_i)^2) \\
    &= \log \EE_{y\sim \text{ Unif}(\{\pm 1\}^n, g\sim N(0,1))} \exp(g\sqrt{2b}\sum y_i) \\
    &= \log \EE_g (\EE_{y\sim \text{Unif}(\{\pm 1\})} \exp (g\sqrt{2b}y))^n \\
    &= \log \EE_g(\cosh g\sqrt{2b})^n
\end{align*}

So, we finally find
\begin{align*}
    \EE Z(\beta)^n &\approx \exp\left(N\sup_a\inf_b \left\{n\log 2 + \beta^2(n+n(n-1)a^2) + \EE_g\cosh(g\sqrt{2b})^n-nb-abn(n-1)\right\}\right).
\end{align*}
We will continue the calculation from here in the next lecture.

\end{document}
