% Template based on:
% https://www.cs.columbia.edu/~djhsu/coms4774-s21/scribe.html

\documentclass[12pt]{article}

\usepackage{amscd}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amstext}
\usepackage{amsthm}
\usepackage{bbold}
\usepackage{bm}
\usepackage{colonequals}
\usepackage{enumitem}
\usepackage[dvips,letterpaper,margin=1in]{geometry}
\usepackage{mathtools}
\usepackage{graphicx}
\usepackage{xcolor}

\usepackage{csquotes}
\MakeOuterQuote{"}

\definecolor{cblack}{rgb}{0,0,0}
\definecolor{cblue}{rgb}{0.121569,0.466667,0.705882}
\definecolor{corange}{rgb}{1.000000,0.498039,0.054902}
\definecolor{cgreen}{rgb}{0.172549,0.627451,0.172549}
\definecolor{cred}{rgb}{0.839216,0.152941,0.156863}
\definecolor{cpurple}{rgb}{0.580392,0.403922,0.741176}
\definecolor{cbrown}{rgb}{0.549020,0.337255,0.294118}
\definecolor{cpink}{rgb}{0.890196,0.466667,0.760784}
\definecolor{cgray}{rgb}{0.498039,0.498039,0.498039}

\usepackage{hyperref}
\hypersetup{
  linkcolor  = cblue,
  citecolor  = cgreen,
  urlcolor   = corange,
  colorlinks = true,
}

% Lecture information
\newcommand\coursename{CPSC 664, Spring 2023}
\newcommand\scribe{Psi Vesely}
\newcommand\lecturer{Tim Kunisky}
\newcommand\lecturedate{April 18, 2023}
\newcommand\lecturetitle{Lecture $23$: Cavity Method I}

% Define theorem environments here.
\newtheorem{claim}{Claim}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{remark}[theorem]{Remark}
\newtheorem{assumption}[theorem]{Assumption}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{question}[theorem]{Question}
\newtheorem{problem}[theorem]{Problem}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{example}[theorem]{Example}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{conjecture}[theorem]{Conjecture}

% Some of my macros
\renewcommand{\AA}{\mathbb{A}}
\newcommand{\CC}{\mathbb{C}}
\newcommand{\DD}{\mathbb{D}}
\newcommand{\EE}{\mathbb{E}}
\newcommand{\FF}{\mathbb{F}}
\newcommand{\HH}{\mathbb{H}}
\newcommand{\NN}{\mathbb{N}}
\newcommand{\PP}{\mathbb{P}}
\newcommand{\QQ}{\mathbb{Q}}
\newcommand{\RR}{\mathbb{R}}
\renewcommand{\SS}{\mathbb{S}}
\newcommand{\ZZ}{\mathbb{Z}}
\DeclareSymbolFont{bbold}{U}{bbold}{m}{n}
\DeclareSymbolFontAlphabet{\mathbbold}{bbold}
\newcommand{\One}{\mathbbold{1}}

\newcommand{\ba}{\bm a}
\newcommand{\bb}{\bm b}
\newcommand{\bc}{\bm c}
\newcommand{\bd}{\bm d}
\newcommand{\be}{\bm e}
\newcommand{\bg}{\bm g}
\newcommand{\bh}{\bm h}
\newcommand{\bi}{\bm i}
\newcommand{\bj}{\bm j}
\newcommand{\bk}{\bm k}
\newcommand{\bbm}{\bm m}
\newcommand{\bp}{\bm p}
\newcommand{\bq}{\bm q}
\newcommand{\br}{\bm r}
\newcommand{\bs}{\bm s}
\newcommand{\bt}{\bm t}
\newcommand{\bu}{\bm u}
\newcommand{\bv}{\bm v}
\newcommand{\bw}{\bm w}
\newcommand{\bx}{\bm x}
\newcommand{\by}{\bm y}
\newcommand{\bz}{\bm z}

\newcommand{\bA}{\bm A}
\newcommand{\bB}{\bm B}
\newcommand{\bC}{\bm C}
\newcommand{\bD}{\bm D}
\newcommand{\bE}{\bm E}
\newcommand{\bF}{\bm F}
\newcommand{\bG}{\bm G}
\newcommand{\bH}{\bm H}
\newcommand{\bI}{\bm I}
\newcommand{\bL}{\bm L}
\newcommand{\bM}{\bm M}
\newcommand{\bN}{\bm N}
\newcommand{\bP}{\bm P}
\newcommand{\bQ}{\bm Q}
\newcommand{\bR}{\bm R}
\newcommand{\bS}{\bm S}
\newcommand{\bT}{\bm T}
\newcommand{\bU}{\bm U}
\newcommand{\bV}{\bm V}
\newcommand{\bW}{\bm W}
\newcommand{\bX}{\bm X}
\newcommand{\bY}{\bm Y}
\newcommand{\bZ}{\bm Z}

\newcommand{\zero}{\bm{0}}
\newcommand{\one}{\bm{1}}

\newcommand{\sA}{\mathcal{A}}
\newcommand{\sB}{\mathcal{B}}
\newcommand{\sC}{\mathcal{C}}
\newcommand{\sD}{\mathcal{D}}
\newcommand{\sE}{\mathcal{E}}
\newcommand{\sF}{\mathcal{F}}
\newcommand{\sG}{\mathcal{G}}
\newcommand{\sH}{\mathcal{H}}
\newcommand{\sI}{\mathcal{I}}
\newcommand{\sJ}{\mathcal{J}}
\newcommand{\sK}{\mathcal{K}}
\newcommand{\sL}{\mathcal{L}}
\newcommand{\sM}{\mathcal{M}}
\newcommand{\sN}{\mathcal{N}}
\newcommand{\sO}{\mathcal{O}}
\newcommand{\sP}{\mathcal{P}}
\newcommand{\sQ}{\mathcal{Q}}
\newcommand{\sR}{\mathcal{R}}
\newcommand{\sS}{\mathcal{S}}
\newcommand{\sT}{\mathcal{T}}
\newcommand{\sU}{\mathcal{U}}
\newcommand{\sV}{\mathcal{V}}
\newcommand{\sX}{\mathcal{X}}
\newcommand{\sY}{\mathcal{Y}}

\newcommand{\fB}{\mathscr{B}}
\newcommand{\fC}{\mathscr{C}}
\newcommand{\fE}{\mathscr{E}}
\newcommand{\fH}{\mathscr{H}}
\newcommand{\fI}{\mathscr{I}}
\newcommand{\fP}{\mathscr{P}}
\newcommand{\fV}{\mathscr{V}}

\DeclareSymbolFont{sfoperators}{OT1}{cmss}{m}{n}
% don't waste a math group
\DeclareSymbolFontAlphabet{\mathsf}{sfoperators}
% tell LaTeX to use sfoperators for names of operators
\makeatletter
\renewcommand{\operator@font}{\mathgroup\symsfoperators}
\makeatother

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\GOE}{GOE}
\DeclareMathOperator{\sym}{sym}
\DeclareMathOperator{\Part}{Part}
\DeclareMathOperator{\poly}{poly}
\DeclareMathOperator{\Unif}{Unif}
\DeclareMathOperator{\Var}{Var}
\DeclareMathOperator{\Cov}{Cov}
\DeclareMathOperator{\Tr}{Tr}
\DeclareMathOperator{\rank}{rank}
\DeclareMathOperator{\obj}{obj}
\DeclareMathOperator{\diag}{diag}
\DeclareMathOperator{\sgn}{sgn}

\renewcommand\Re{\operatorname{Re}}
\renewcommand\Im{\operatorname{Im}}
% These put symbols below.
\newcommand{\Px}{\mathop{\mathbb{P}}}
\newcommand{\Ex}{\mathop{\mathbb{E}}}
\newcommand{\Varx}{\mathop{\mathsf{Var}}}

\newcommand\numberthis{\addtocounter{equation}{1}\tag{\theequation}}

%
% Put your custom macros here if you want.
%

\begin{document}

% This creates the header for scribe notes.
\noindent \coursename \hfill Scribe: \scribe \\
\lecturedate \hfill Lecturer: \lecturer
\vspace{1em}

\hrule

\vspace{1.5em}
\begin{center}
  {\Large\lecturetitle}
\end{center}

RS regime of SK model ($\mu_\beta, \beta$ small). Gibbs measure $\mu(\beta) \propto \exp(\beta x^\intercal W x$). Idea: ``reverse induction'' on N (``cavity'' left from removing $x_i$). At size $N+1$ $x=(x_0,\ldots,x_N)$. Let $\hat{x} = (x_1,\ldots,x_N)$ and $\hat{W}$ be the $N\times N$ bottom-right submatrix of $W$. Say $W_{ii} = 0$. Then
\[
 x^\intercal W x = \hat{x}^\intercal \hat{W} \hat{x} + x_0 \sum_{1}^N W_{0i} :=  \hat{x}^\intercal \hat{W} \hat{x} + h(x_1,\ldots,x_N)
\]
We say $h$ is the ``cavity field,'' the effect $x_0$ feels from the other $x_i$.
\begin{align*}
  \Pr_{(x_0,\ldots,x_N)\sim \mu_{\beta,N+1}}[x_0 = s \hat h(x_1,\ldots,x_N) = h] &\propto \sum_{\substack{x:x_0 = s,\\ h(x_1,\ldots,x_N) = s}} \exp(\beta x^\intercal W x)  \\
  &=e^{\beta s h} \Pr_{(x_1,\ldots,x_N)\sim \mu_{\beta,N}}[h(x_1,\dotsc,x_N) = h]
\end{align*}
With the replica method, we began taking expectations over $W$ immediately, whereas with the cavity method we wait. The next question we may ask is what is the distribution of $h(x_1,\dotsc,x_N)$ under $\mu_{\beta,N}$? We will answer this by:
\begin{enumerate}
  \item Conditioning on which cluster $x \sim \mu_{\beta,N}$ lands in.
  \[
    \langle f(x) \rangle_N = \EE_{\substack{x\sim\mu_{\beta,N},\\ \text{cond. on one cluster}}} f(x)
  \]
  \item Assume under $\langle \cdot \rangle_N$ (the conditional distribution inside a cluster) there are weak correlations between $x_i$. Then $h(x_1,\dotsc,x_N)$ would satisfy CLT when
 \[
  \sum_{1}^N W_{0i} x_i \approx \sN(\mu,\sigma^2)
 \]
 for $\mu = \mu(\beta,\hat{W}), \sigma^2 = \sigma^2(\beta,\hat{W})$.
 \[
  \mu = \langle h \rangle = \sum W_{0i} \langle x_i \rangle_{N} := \sum W_{0i} m_i
 \]
 \begin{align*}
  \sigma^2 = \langle h^2 \rangle_N - \langle h \rangle_{N}^2 &= \sum_{ij} W_{0i}W_{0j}(\langle x_i x_j \rangle_N - \langle x_i \rangle_N \langle x_j \rangle_N) \\
  &\approx \sum W_{0i} (1- \langle x_i \rangle_{N}^2)
  \approx \frac{1}{N} \sum (1- \langle x_i \rangle_{N}^2)  \\
  &= 1 -\frac{1}{N} \sum \langle x_{i}^{(1)} x_{i}^{(2)} \rangle_{N}^2 = 1 - \left\langle \frac{x^{(1)}x^{(2)}}{N}\right\rangle \approx 1-q
 \end{align*}
\end{enumerate}
For $x\sim\mu_{\beta,N}$ we have $h(x) \stackrel{d}{\approx} \sN(\sum_{1}^N W_{0i}m_i, 1-q)$. Thus we obtain a nice joint distribution of $(x_0,h(x_1,\dotsc,x_N))$ under $\mu_{\beta,N+1}$. We then calculate
\[
  \langle x_0 \rangle_{N+1} = \Pr[x_0 = 1] = \Pr[x_0=-1] = \frac{e^\beta\mu-e^{-\beta\mu}}{e^\beta\mu+e^{-\beta\mu}} = \tanh(\beta\mu) = \tanh(\beta\sum W_{0i} \langle x_i \rangle_N)
\]
\begin{align*}
  \Pr[x_0 = x] &\propto \int e^{\beta h x}\exp(-\frac{(\beta-\mu)^2}{2\sigma^2}) dh \\
  &= \int \exp(-\frac{(h-\mu-\beta\sigma^2x)^2}{2\sigma^2}) \exp(\frac{-\mu^2+(\beta\sigma^2x+\mu)^2}{2\sigma^2}) dh \propto \exp(\beta \mu x)
\end{align*}
We can then solve for the overlay:
\[
  q= \EE_W \frac{1}{N} \sum_{i=0}^{N+1} \langle x_i \rangle_{N+1}^2 = \EE_W \langle x_0 \rangle_{N}^2 = \EE \tanh(\beta\sum W_{0i} \langle x_0 \rangle_N) := \EE \tanh(\beta\langle h \rangle_N)^2
\]
where the $W_{0i}, \langle x_i \rangle_N$ are independent. We then get
\[
  \langle h \rangle \approx \sN(0,\frac{1}{N} \sum \langle x_i \rangle_{N}^2) = \sN(0,q)
\]
So
\[
 q = \EE \tanh(\beta t)^2  = \EE_{t \sim \sN(0,1)} \tanh(\beta\sqrt{q}\cdot t)^2
\]
giving us an equation for the typical overlay from the replica method. To locate clusters
\[
  \langle x_0 \rangle_{N+1} = \tanh(\beta\sum W_{0i} \langle x_i \rangle_N) = \tanh(\beta\langle h \rangle_N)
\]
almost works.
\[
  P_{N+1}(h) \propto \cosh(\beta h) P_N(h)
\]
The normalizaing constant of $P_{W+1}(h)$ is
\[
  \int \frac{e^{\beta h} + e^{-\beta h}}{2} \cdot (\text{density of } \sN(\langle h \rangle_N), 1-q) dh = e^{\frac{1-q}{2} \beta^2} \cosh(\beta\langle h \rangle_N)
\]
Implying
\[
 p_{N+1}(h) = \frac{e^{{\frac{-1-q}{2}\beta^2}}}{\cosh(\beta\langle h \rangle_N)} \sN(\langle h \rangle_{N},1-q)
\]
Let $t:=\cosh(\beta\langle h \rangle_N)$. Then
\begin{align*}
 \langle h \rangle_{N+1} = t \EE h \cosh(\beta h) \\
  &= t \EE_{h\sim\sN(0,1)}(h+\langle h \rangle_N) \cosh(\beta(\sqrt{1-q}\cdot h + \langle h \rangle_N)) \\
  &= \langle h \rangle_N t \sqrt{1-q} \EE_{h}\cosh(\beta(\sqrt{1-q}\cdot h + \langle h \rangle_N)) \\
  &= \langle h \rangle_N t \beta (1-q) \EE_{h}\sin(\beta h) \\
  &= \langle h \rangle_N + \beta (1-q) \tanh(\beta\langle h \rangle_N) = \langle h \rangle_N + \beta (1-q) \langle x_0 \rangle_{N+1}
\end{align*}
Then
\[
 \langle h \rangle_N =  \langle h \rangle_{N+1} - \beta(1-q)\langle x_0 \rangle_{N+1} = \sum J_{0i} \langle x_i \rangle_{N+1} - \beta(1-q)\langle x_0\rangle_{N+1}
\]
It follows
\[
  \langle x_0 \rangle = \tanh(\beta\sum w_{0i} \langle x_i \rangle - \beta^2(1-q)\langle x_0\rangle)
\]
Then for all $i$
\[
  \langle x_i \rangle = \tanh(\beta\sum w_{ij} \langle x_j \rangle - \beta^2(1-q)\langle x_i\rangle)
\]
We can find the center of the these clusters with a nonlinear power method.

\end{document}
