% Template based on:
% https://www.cs.columbia.edu/~djhsu/coms4774-s21/scribe.html

\documentclass[12pt]{article}

\usepackage{physics}
\usepackage{dsfont}
\usepackage{amscd}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amstext}
\usepackage{amsthm}
\usepackage{bbold}
\usepackage{bm}
\usepackage{colonequals}
\usepackage[dvips,letterpaper,margin=1in]{geometry}
\usepackage{mathtools}
\usepackage{graphicx}
\usepackage{xcolor}

\usepackage{csquotes}
\MakeOuterQuote{"}

\definecolor{cblack}{rgb}{0,0,0}
\definecolor{cblue}{rgb}{0.121569,0.466667,0.705882}
\definecolor{corange}{rgb}{1.000000,0.498039,0.054902}
\definecolor{cgreen}{rgb}{0.172549,0.627451,0.172549}
\definecolor{cred}{rgb}{0.839216,0.152941,0.156863}
\definecolor{cpurple}{rgb}{0.580392,0.403922,0.741176}
\definecolor{cbrown}{rgb}{0.549020,0.337255,0.294118}
\definecolor{cpink}{rgb}{0.890196,0.466667,0.760784}
\definecolor{cgray}{rgb}{0.498039,0.498039,0.498039}

\usepackage{hyperref}
\hypersetup{
  linkcolor  = cblue,
  citecolor  = cgreen,
  urlcolor   = corange,
  colorlinks = true,
}

\usepackage{float}
\usepackage[capitalise,noabbrev,nameinlink]{cleveref}

% Lecture information
\newcommand\coursename{CPSC 664, Spring 2023}
\newcommand\scribe{Dantong Li}
\newcommand\lecturer{Tim Kunisky}
\newcommand\lecturedate{March 28, 2023}
\newcommand\lecturetitle{Lecture 17: Introduction to Sherrington-Kirkpatrick (SK) model}

% Define theorem environments here.
\newtheorem{claim}{Claim}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{remark}[theorem]{Remark}
\newtheorem{assumption}[theorem]{Assumption}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{question}[theorem]{Question}
\newtheorem{problem}[theorem]{Problem}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{example}[theorem]{Example}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{conjecture}[theorem]{Conjecture}

% Some of my macros
\renewcommand{\AA}{\mathbb{A}}
\newcommand{\CC}{\mathbb{C}}
\newcommand{\DD}{\mathbb{D}}
\newcommand{\EE}{\mathbb{E}}
\newcommand{\FF}{\mathbb{F}}
\newcommand{\HH}{\mathbb{H}}
\newcommand{\NN}{\mathbb{N}}
\newcommand{\PP}{\mathbb{P}}
\newcommand{\QQ}{\mathbb{Q}}
\newcommand{\RR}{\mathbb{R}}
\renewcommand{\SS}{\mathbb{S}}
\newcommand{\ZZ}{\mathbb{Z}}
\DeclareSymbolFont{bbold}{U}{bbold}{m}{n}
\DeclareSymbolFontAlphabet{\mathbbold}{bbold}
\newcommand{\One}{\mathbbold{1}}

\newcommand{\ba}{\bm a}
\newcommand{\bb}{\bm b}
\newcommand{\bc}{\bm c}
\newcommand{\bd}{\bm d}
\newcommand{\be}{\bm e}
\newcommand{\bg}{\bm g}
\newcommand{\bh}{\bm h}
\newcommand{\bi}{\bm i}
\newcommand{\bj}{\bm j}
\newcommand{\bk}{\bm k}
\newcommand{\bbm}{\bm m}
\newcommand{\bp}{\bm p}
\newcommand{\bq}{\bm q}
\newcommand{\br}{\bm r}
\newcommand{\bs}{\bm s}
\newcommand{\bt}{\bm t}
\newcommand{\bu}{\bm u}
\newcommand{\bv}{\bm v}
\newcommand{\bw}{\bm w}
\newcommand{\bx}{\bm x}
\newcommand{\by}{\bm y}
\newcommand{\bz}{\bm z}

\newcommand{\bA}{\bm A}
\newcommand{\bB}{\bm B}
\newcommand{\bC}{\bm C}
\newcommand{\bD}{\bm D}
\newcommand{\bE}{\bm E}
\newcommand{\bF}{\bm F}
\newcommand{\bG}{\bm G}
\newcommand{\bH}{\bm H}
\newcommand{\bI}{\bm I}
\newcommand{\bL}{\bm L}
\newcommand{\bM}{\bm M}
\newcommand{\bN}{\bm N}
\newcommand{\bP}{\bm P}
\newcommand{\bQ}{\bm Q}
\newcommand{\bR}{\bm R}
\newcommand{\bS}{\bm S}
\newcommand{\bT}{\bm T}
\newcommand{\bU}{\bm U}
\newcommand{\bV}{\bm V}
\newcommand{\bW}{\bm W}
\newcommand{\bX}{\bm X}
\newcommand{\bY}{\bm Y}
\newcommand{\bZ}{\bm Z}

\newcommand{\zero}{\bm{0}}
\newcommand{\one}{\bm{1}}

\newcommand{\sA}{\mathcal{A}}
\newcommand{\sB}{\mathcal{B}}
\newcommand{\sC}{\mathcal{C}}
\newcommand{\sD}{\mathcal{D}}
\newcommand{\sE}{\mathcal{E}}
\newcommand{\sF}{\mathcal{F}}
\newcommand{\sG}{\mathcal{G}}
\newcommand{\sH}{\mathcal{H}}
\newcommand{\sI}{\mathcal{I}}
\newcommand{\sJ}{\mathcal{J}}
\newcommand{\sK}{\mathcal{K}}
\newcommand{\sL}{\mathcal{L}}
\newcommand{\sM}{\mathcal{M}}
\newcommand{\sN}{\mathcal{N}}
\newcommand{\sO}{\mathcal{O}}
\newcommand{\sP}{\mathcal{P}}
\newcommand{\sQ}{\mathcal{Q}}
\newcommand{\sR}{\mathcal{R}}
\newcommand{\sS}{\mathcal{S}}
\newcommand{\sT}{\mathcal{T}}
\newcommand{\sU}{\mathcal{U}}
\newcommand{\sV}{\mathcal{V}}
\newcommand{\sX}{\mathcal{X}}
\newcommand{\sY}{\mathcal{Y}}

\newcommand{\fB}{\mathscr{B}}
\newcommand{\fC}{\mathscr{C}}
\newcommand{\fE}{\mathscr{E}}
\newcommand{\fH}{\mathscr{H}}
\newcommand{\fI}{\mathscr{I}}
\newcommand{\fP}{\mathscr{P}}
\newcommand{\fV}{\mathscr{V}}

\DeclareSymbolFont{sfoperators}{OT1}{cmss}{m}{n}
% don't waste a math group
\DeclareSymbolFontAlphabet{\mathsf}{sfoperators}
% tell LaTeX to use sfoperators for names of operators
\makeatletter
\renewcommand{\operator@font}{\mathgroup\symsfoperators}
\makeatother

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\GOE}{GOE}
\DeclareMathOperator{\sym}{sym}
\DeclareMathOperator{\Part}{Part}
\DeclareMathOperator{\poly}{poly}
\DeclareMathOperator{\Unif}{Unif}
\DeclareMathOperator{\Var}{Var}
\DeclareMathOperator{\Cov}{Cov}
% \DeclareMathOperator{\Tr}{Tr}
% \DeclareMathOperator{\rank}{rank}
\DeclareMathOperator{\obj}{obj}
\DeclareMathOperator{\diag}{diag}
\DeclareMathOperator{\sgn}{sgn}

\renewcommand\Re{\operatorname{Re}}
\renewcommand\Im{\operatorname{Im}}
% These put symbols below.
\newcommand{\Px}{\mathop{\mathbb{P}}}
\newcommand{\Ex}{\mathop{\mathbb{E}}}
\newcommand{\Varx}{\mathop{\mathsf{Var}}}

\newcommand\numberthis{\addtocounter{equation}{1}\tag{\theequation}}

%
% Put your custom macros here if you want.
%
% floor, ceiling, set
\DeclarePairedDelimiter{\ceil}{\lceil}{\rceil}
\DeclarePairedDelimiter{\floor}{\lfloor}{\rfloor}
\DeclarePairedDelimiter{\set}{\lbrace}{\rbrace}
\DeclarePairedDelimiter{\iprod}{\langle}{\rangle}
\DeclarePairedDelimiter{\card}{\lvert}{\rvert}
\let\abs\relax
\DeclarePairedDelimiter{\abs}{\lvert}{\rvert}
\DeclarePairedDelimiter{\level}{\llbracket}{\rrbracket}

\DeclareMathOperator{\Int}{int}
\DeclareMathOperator{\bdy}{bdy}
\DeclareMathOperator{\Lim}{Lim}
\DeclareMathOperator{\mean}{mean}
\DeclareMathOperator{\col}{col}
\DeclareMathOperator{\proj}{proj}
\DeclareMathOperator{\dual}{dual}
\DeclareMathOperator{\opt}{opt}
\DeclareMathOperator{\cone}{cone}
\DeclareMathOperator{\conv}{conv}
\DeclareMathOperator{\supp}{supp}
% \DeclareMathOperator{\poly}{poly}
% \DeclareMathOperator{\sgn}{sgn}
\DeclareMathOperator{\depth}{depth}
\DeclareMathOperator{\OPT}{OPT}
\DeclareMathOperator{\Set}{set}
\DeclareMathOperator{\pred}{pred}
\DeclareMathOperator{\SAT}{SAT}
\DeclareMathOperator{\indeg}{indeg}
\DeclareMathOperator{\outdeg}{outdeg}
\DeclareMathOperator{\tw}{tw}
% \DeclareMathOperator{\bw}{bw}
\DeclareMathOperator{\pw}{pw}
\DeclareMathOperator{\cutwidth}{cutwidth}
\DeclareMathOperator{\Cut}{Cut}
\DeclareMathOperator{\Vs}{Vs}
\DeclareMathOperator{\vs}{vs}
\DeclareMathOperator{\adj}{adj}
\DeclareMathOperator{\Sp}{Sp}
% \DeclareMathOperator{\argmax}{argmax}
% \DeclareMathOperator{\argmin}{argmin}
\DeclareMathOperator{\dom}{dom}
\DeclareMathOperator{\MLE}{MLE}
\DeclareMathOperator{\MAP}{MAP}
\DeclareMathOperator{\KL}{KL}
\DeclareMathOperator{\LK}{LK}
\DeclareMathOperator{\Be}{Be}
\DeclareMathOperator{\Bin}{Bin}
\DeclareMathOperator{\Geo}{Geo}
\DeclareMathOperator{\Po}{Po}
\DeclareMathOperator{\Exp}{Exp}
\DeclareMathOperator{\Mult}{Mult}
% \DeclareMathOperator{\Var}{Var}
% \DeclareMathOperator{\Cov}{Cov}
\DeclareMathOperator{\Cauchy}{Cauchy}
\DeclareMathOperator{\Diag}{Diag}
% \DeclareMathOperator{\diag}{diag}
\DeclareMathOperator{\DB}{DB}
\DeclareMathOperator{\BB}{BB}
\DeclareMathOperator{\OU}{OU}
\DeclareMathOperator{\GWN}{GWN}
\DeclareMathOperator{\Crit}{Crit}
\DeclareMathOperator{\vol}{vol}

% commonly used sets
\newcommand{\R}{\mathbb{R}}
\renewcommand{\S}{\mathbb{S}}
\newcommand{\Z}{\mathbb{Z}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\Q}{\mathbb{Q}}
\newcommand{\C}{\mathbb{C}}
\newcommand{\E}{\mathbb{E}}
\newcommand{\B}{\mathcal{B}}
\newcommand{\F}{\mathcal{F}}
\newcommand{\G}{\mathcal{G}}
\renewcommand{\L}{\mathcal{L}}
\renewcommand{\H}{\mathscr{H}}
\renewcommand{\P}{\mathbb{P}}

\newcommand{\h}{\vec{h}}
\newcommand{\p}{\vec{p}}
\newcommand{\x}{\vec{x}}
\newcommand{\y}{\vec{y}}
\newcommand{\z}{\vec{z}}
\renewcommand{\a}{\vec{a}}
\renewcommand{\b}{\vec{b}}
\renewcommand{\c}{\vec{c}}
\renewcommand{\t}{\vec{t}}
\renewcommand{\u}{\vec{u}}
\renewcommand{\v}[1]{\vec{#1}}

\newcommand{\sset}{\subseteq}
\newcommand{\mcal}{\mathcal}
\newcommand{\mscr}{\mathscr}
\newcommand{\mfrak}{\mathfrak}
\newcommand{\up}{\uparrow}
\newcommand{\down}{\downarrow}
\newcommand{\zeros}{\mathds{0}}
\newcommand{\ones}{\mathds{1}}
\newcommand{\tends}[1]{\xrightarrow{#1}}
\newcommand{\sdnet}[1]{\xleftarrow{#1}}
\newcommand{\eq}[1]{\stackrel{#1}{=}}
\newcommand{\Geq}[1]{\stackrel{#1}{\geq}}
\newcommand{\Leq}[1]{\stackrel{#1}{\leq}}
\newcommand{\Gt}[1]{\stackrel{#1}{>}}
\newcommand{\Lt}[1]{\stackrel{#1}{<}}
\newcommand{\Add}[1]{\stackrel{#1}{+}}
\newcommand{\Sim}[1]{\stackrel{#1}{\sim}}
\newcommand{\mat}[1]{\begin{bmatrix}#1\end{bmatrix}}

\begin{document}

% This creates the header for scribe notes.
\noindent \coursename \hfill Scribe: \scribe \\
\lecturedate \hfill Lecturer: \lecturer
\vspace{1em}

\hrule

\vspace{1.5em}
\begin{center}
  {\Large\lecturetitle}
\end{center}

\section{Problem Definition}

This is a very important model in statistical physics and we will motivate this problem with a pure combinatorial problem, the max cut problem.

We've seen the following in Homework 1

\begin{align*}
     &W \Sim{} GOE(N, \frac{1}{N})
\end{align*}

$|| W || = O(1)$ w.h.p., then we define the following optimization problem

\begin{align*}
    M(W) = \max_{x \in \{\pm 1\}^N} x^TWx
\end{align*}

$||x|| = \sqrt{N}$ w.h.p., this is like the eigenvalue problem but over the hypercube, which changes the problem drastically and this constraint makes it harder but more interesting.

\section{Motivation}

This problem is motivated by a combinatorial problem, the MaxCut problem, specified by a graph $G = (V, E)$. We devide the set $V$ into two sets and maximize the number of edges between these two sets. This is a well-known NP-complete problem.

\begin{claim}
    The normalized MaxCut is a special case of our optimization problem $M$.
\end{claim}

\begin{proof}

Given a $G$, we have two matrices: $A = A_G$ the adjacency matrix and a diagonal matrix $D = D_G$ where $D_{xx} = deg_G(x)$. Then we can build the Laplacian matrix $L = L_G = D - A$. If we look at the quadratic form:

\begin{align*}
    v^TLv &= \sum_x deg_G(x) v_x^2 - 2\sum_{(x,y)\in E}v_xv_y \\
    &= \sum_{(x,y)\in E} (v_x^2 + v_y^2) - 2\sum_{(x,y)\in E}v_xv_y \\
    &= \sum_{(x,y)\in E} (v_x - v_y)^2
\end{align*}

which can be regarded as the "gradient" of $v$: $||\nabla_v||^2$. If we plug in $x$ in the hypercube so that $x \in \{\pm 1\}^V$ corresponds to a division of the vertices set $U \subset V$, then $x^TLx = 4 \times \{\# \text{ of edges cut by } U\}$. So $MaxCut(G) = M(\frac{1}{4}L)$.

\end{proof}

\begin{claim}
    Plugging in these "smooth" Gaussian matrices $W$ corresponds to meaningful MaxCut problem.
\end{claim}

Let's look at MaxCut for random graphs.

For $d \geq 3$, $G \sim Unif(\{\text{d-regular graph on N vertices}\})$.

\begin{remark}
    One can utilize configuration models to efficiently sample from this distribution.
\end{remark}

Since $d$ is independent from $N$, $G$ is a sparse graph: $|E| = \frac{dN}{2} = O(N)$. One could ask what is the typical MaxCut value for this graph.

Since it's $d-regular$, $D = dI$ so $x^TLx = dN - x^TAx$ and:

\begin{align*}
    MaxCut(G) &= \left[ \frac{1}{4} dN - \frac{1}{4} \min_x x^TAx \right] \\
    Normalized\_MaxCut(G) &= \left[ \frac{1}{2} - \frac{1}{2dN} \min_x x^TAx \right]
\end{align*}

For simplicity we will use $MaxCut$ to denote $Normalized\_MaxCut$.

\section{Bounds for MaxCut}
\subsection{Upper Bound}

We can upper bound it with
\begin{align*}
    MaxCut(G) &\leq \left[ \frac{1}{2} - \frac{1}{2dN} \lambda_{min}(A) ||x||^2 \right] \\
    &= \left[ \frac{1}{2} - \frac{1}{2dN} \lambda_{min}(A)N \right] \\
    &= \left[ \frac{1}{2} + \frac{||\lambda_{min}(A)||}{2d}  \right]
\end{align*}

Let's denote the eigenvalues of $A$ as: $\lambda_1 \geq \lambda_2 \geq \cdots \geq \lambda_N$, then $\lambda_1 = d$ with eigenvector being all $1$'s, which is evidence that the results about the empirical spectral distribution from the free probability tool do not describe the outliers of the eigenvalues. ($d > 2\sqrt{d-1}$)

\begin{theorem}
    W.h.p., $-2\sqrt{d-1} - o_{N\rightarrow \infty}(1) \leq \lambda_N \leq \lambda_1 \leq 2\sqrt{d-1} + o_{N\rightarrow \infty}(1)$
\end{theorem}

And by the above theorem, one can show that w.h.p.:

\begin{align*}
    MaxCut(G) \leq \frac{1}{2} + \frac{\sqrt{d-1}}{d} + o(1)
\end{align*}

\subsection{Lower Bound}

We can also lower bound this by analyzing some "local algorithms" that given $G$, output a cut. W.h.p.:

\begin{align*}
    MaxCut(G) \geq \frac{1}{2} + \frac{c}{\sqrt{d}}
\end{align*}.

\subsection{Problems to Study}

\begin{itemize}
    \item \textbf{Q1}: For small constants $d$ (i.e. $d=3$), find $\lim_N \E MaxCut(G)$
    \item \textbf{Q2}: For large $d$, find the threshold.
\end{itemize}

We focus on the second problem. Specifically, we are interested in showing the existence of such constant $c$, so that $\lim_N \E MaxCut(G) = \frac{1}{2} + \frac{c}{\sqrt{d}} + o(\frac{1}{\sqrt{d}})$ as $d \rightarrow \infty$. And if it exists, we want to find out what it is.

\begin{remark}
    Same for sparse Erdos–Renyi graphs (edges are present with probability $\frac{d}{N}$). The same argument does not apply but the same conclusion can be drawn.
\end{remark}

\begin{remark}
    Same for minimum bisection (i.e. $||U|| = \frac{N}{2}$) and maximum bisection problem. In simulations, it is close to MaxCut.
\end{remark}

\begin{conjecture}
$\lim_{N \rightarrow \infty} MaxCut(G) - \frac{1}{2} = \frac{1}{2} - \lim_{N \rightarrow \infty} MinBis(G) $ even for fixed $d$.
\end{conjecture}

\section{Find Constant c}

Recall that we have:

\begin{align*}
    \frac{1}{2} - \frac{1}{2dN} \min_x x^TAx = MaxCut(G) = \frac{1}{2} + \frac{c}{\sqrt{d}} + o(\frac{1}{\sqrt{d}})
\end{align*}

It's equivalent to that w.h.p.:

\begin{align*}
     & \frac{1}{2\sqrt{d}N} \min_x x^TAx =  -c + o(1) \\
     \Leftrightarrow & \frac{1}{2N} \min_x x^T\frac{A}{\sqrt{d}}x =  -c + o(1)
\end{align*}

Recall that "free CLT" says that e.s.d. of $\frac{A}{\sqrt{d}}$ is the semi-circle, which is the e.s.d. of $GOE(N, \frac{1}{N})$, so heuristically

\begin{align*}
     \frac{1}{2N} \min_x x^T\frac{A}{\sqrt{d}}x &\approx  \frac{1}{2N} \min_x x^TWx \\
     &=  -\frac{1}{2N} \max_x x^T(-W)x \\
     &=  -\frac{1}{2N} \max_x x^TWx
\end{align*}

Heuristic guess of the constant:
\begin{align*}
    c = \lim_{N \rightarrow \infty} \frac{1}{2N} \E_{W \sim GOE(N, \frac{1}{N})} M(W)
\end{align*}
, which is the ground state of this SK model.

\begin{theorem}
    The Parisi number $P^*$ (i.e. the above c value) exists and has some complicated formula. The number is 0.7632.
\end{theorem}

\begin{theorem}
    $\lim_{N \rightarrow \infty}\E MaxCut(G) = \frac{1}{2} + \frac{P^*}{\sqrt{d}} + o(\frac{1}{\sqrt{d}})$
\end{theorem}

\section{Replica Method for $P^*$}

In physics, the SK Hamiltonian $H(x) = x^TWx$. Then we can define:

\begin{itemize}
    \item $\mu_{\beta}(x) = \frac{1}{Z(\beta)} e^{\beta H(x)}$, where $Z(\beta) = \sum_x e^{\beta H(x)} $
    \item Free energy: $F(\beta) = \log(Z(\beta))$; Free energy density: $f(\beta) = \lim_{N \rightarrow \infty} \frac{1}{N} F(\beta)$
    \item Ground state: $\lim_{\beta \rightarrow \infty} \frac{1}{\beta} f(\beta)$
\end{itemize}

Note that $\E F(\beta) = \E \log Z(\beta) = \lim_{n \rightarrow 0} \frac{\E Z(\beta)^n}{n}$

while $\E Z(\beta)^n = \sum_{x_1, \cdots, x_n} \E_{W} e^{\beta (\langle W, x_1x_1^T + \cdots + x_nx_n^T \rangle)}$.

The exponent could be written as the Frobenius norm of $XX^T$ and then what we did previously is to transform this to the Frobenius norm of  $X^TX$, which is a small matrix. And in the end one could get some results by making some naive assumption that the matrix has some certain number on the diagonal and some other number on the off-diagonal entries. But one would find out that this result would imply some "negative entropy", which could not happen. We will see in the next lectures that some replica-symmetry-breaking could be used to repair this technique to calculate the $P^*$ value.






\end{document}