% Template based on:
% https://www.cs.columbia.edu/~djhsu/coms4774-s21/scribe.html

\documentclass[12pt]{article}

\usepackage{amscd}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amstext}
\usepackage{amsthm}
\usepackage{bbold}
\usepackage{bm}
\usepackage{colonequals}
\usepackage[dvips,letterpaper,margin=1in]{geometry}
\usepackage{mathtools}
\usepackage{graphicx}
\usepackage{xcolor}

\usepackage{csquotes}
\MakeOuterQuote{"}

\definecolor{cblack}{rgb}{0,0,0}
\definecolor{cblue}{rgb}{0.121569,0.466667,0.705882}
\definecolor{corange}{rgb}{1.000000,0.498039,0.054902}
\definecolor{cgreen}{rgb}{0.172549,0.627451,0.172549}
\definecolor{cred}{rgb}{0.839216,0.152941,0.156863}
\definecolor{cpurple}{rgb}{0.580392,0.403922,0.741176}
\definecolor{cbrown}{rgb}{0.549020,0.337255,0.294118}
\definecolor{cpink}{rgb}{0.890196,0.466667,0.760784}
\definecolor{cgray}{rgb}{0.498039,0.498039,0.498039}

\usepackage{hyperref}
\hypersetup{
  linkcolor  = cblue,
  citecolor  = cgreen,
  urlcolor   = corange,
  colorlinks = true,
}

% Lecture information
\newcommand\coursename{CPSC 664, Spring 2023}
\newcommand\scribe{Abhinav Bhardwaj}
\newcommand\lecturer{Tim Kunisky}
\newcommand\lecturedate{April 6th}
\newcommand\lecturetitle{Lecture 20: Replica Symmetry Breaking II }

% Define theorem environments here.
\newtheorem{claim}{Claim}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{remark}[theorem]{Remark}
\newtheorem{assumption}[theorem]{Assumption}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{question}[theorem]{Question}
\newtheorem{problem}[theorem]{Problem}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{example}[theorem]{Example}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{conjecture}[theorem]{Conjecture}

% Some of my macros
\renewcommand{\AA}{\mathbb{A}}
\newcommand{\CC}{\mathbb{C}}
\newcommand{\DD}{\mathbb{D}}
\newcommand{\EE}{\mathbb{E}}
\newcommand{\FF}{\mathbb{F}}
\newcommand{\HH}{\mathbb{H}}
\newcommand{\NN}{\mathbb{N}}
\newcommand{\PP}{\mathbb{P}}
\newcommand{\QQ}{\mathbb{Q}}
\newcommand{\RR}{\mathbb{R}}
\renewcommand{\SS}{\mathbb{S}}
\newcommand{\ZZ}{\mathbb{Z}}
\DeclareSymbolFont{bbold}{U}{bbold}{m}{n}
\DeclareSymbolFontAlphabet{\mathbbold}{bbold}
\newcommand{\One}{\mathbbold{1}}

\newcommand{\ba}{\bm a}
\newcommand{\bb}{\bm b}
\newcommand{\bc}{\bm c}
\newcommand{\bd}{\bm d}
\newcommand{\be}{\bm e}
\newcommand{\bg}{\bm g}
\newcommand{\bh}{\bm h}
\newcommand{\bi}{\bm i}
\newcommand{\bj}{\bm j}
\newcommand{\bk}{\bm k}
\newcommand{\bbm}{\bm m}
\newcommand{\bp}{\bm p}
\newcommand{\bq}{\bm q}
\newcommand{\br}{\bm r}
\newcommand{\bs}{\bm s}
\newcommand{\bt}{\bm t}
\newcommand{\bu}{\bm u}
\newcommand{\bv}{\bm v}
\newcommand{\bw}{\bm w}
\newcommand{\bx}{\bm x}
\newcommand{\by}{\bm y}
\newcommand{\bz}{\bm z}

\newcommand{\bA}{\bm A}
\newcommand{\bB}{\bm B}
\newcommand{\bC}{\bm C}
\newcommand{\bD}{\bm D}
\newcommand{\bE}{\bm E}
\newcommand{\bF}{\bm F}
\newcommand{\bG}{\bm G}
\newcommand{\bH}{\bm H}
\newcommand{\bI}{\bm I}
\newcommand{\bL}{\bm L}
\newcommand{\bM}{\bm M}
\newcommand{\bN}{\bm N}
\newcommand{\bP}{\bm P}
\newcommand{\bQ}{\bm Q}
\newcommand{\bR}{\bm R}
\newcommand{\bS}{\bm S}
\newcommand{\bT}{\bm T}
\newcommand{\bU}{\bm U}
\newcommand{\bV}{\bm V}
\newcommand{\bW}{\bm W}
\newcommand{\bX}{\bm X}
\newcommand{\bY}{\bm Y}
\newcommand{\bZ}{\bm Z}

\newcommand{\zero}{\bm{0}}
\newcommand{\one}{\bm{1}}

\newcommand{\sA}{\mathcal{A}}
\newcommand{\sB}{\mathcal{B}}
\newcommand{\sC}{\mathcal{C}}
\newcommand{\sD}{\mathcal{D}}
\newcommand{\sE}{\mathcal{E}}
\newcommand{\sF}{\mathcal{F}}
\newcommand{\sG}{\mathcal{G}}
\newcommand{\sH}{\mathcal{H}}
\newcommand{\sI}{\mathcal{I}}
\newcommand{\sJ}{\mathcal{J}}
\newcommand{\sK}{\mathcal{K}}
\newcommand{\sL}{\mathcal{L}}
\newcommand{\sM}{\mathcal{M}}
\newcommand{\sN}{\mathcal{N}}
\newcommand{\sO}{\mathcal{O}}
\newcommand{\sP}{\mathcal{P}}
\newcommand{\sQ}{\mathcal{Q}}
\newcommand{\sR}{\mathcal{R}}
\newcommand{\sS}{\mathcal{S}}
\newcommand{\sT}{\mathcal{T}}
\newcommand{\sU}{\mathcal{U}}
\newcommand{\sV}{\mathcal{V}}
\newcommand{\sX}{\mathcal{X}}
\newcommand{\sY}{\mathcal{Y}}

\newcommand{\fB}{\mathscr{B}}
\newcommand{\fC}{\mathscr{C}}
\newcommand{\fE}{\mathscr{E}}
\newcommand{\fH}{\mathscr{H}}
\newcommand{\fI}{\mathscr{I}}
\newcommand{\fP}{\mathscr{P}}
\newcommand{\fV}{\mathscr{V}}

\DeclareSymbolFont{sfoperators}{OT1}{cmss}{m}{n}
% don't waste a math group
\DeclareSymbolFontAlphabet{\mathsf}{sfoperators}
% tell LaTeX to use sfoperators for names of operators
\makeatletter
\renewcommand{\operator@font}{\mathgroup\symsfoperators}
\makeatother

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\GOE}{GOE}
\DeclareMathOperator{\sym}{sym}
\DeclareMathOperator{\Part}{Part}
\DeclareMathOperator{\poly}{poly}
\DeclareMathOperator{\Unif}{Unif}
\DeclareMathOperator{\Var}{Var}
\DeclareMathOperator{\Cov}{Cov}
\DeclareMathOperator{\Tr}{Tr}
\DeclareMathOperator{\rank}{rank}
\DeclareMathOperator{\obj}{obj}
\DeclareMathOperator{\diag}{diag}
\DeclareMathOperator{\sgn}{sgn}
\newcommand{\norm}[1]{\left\lVert#1\right\rVert}

\renewcommand\Re{\operatorname{Re}}
\renewcommand\Im{\operatorname{Im}}
% These put symbols below.
\newcommand{\Px}{\mathop{\mathbb{P}}}
\newcommand{\Ex}{\mathop{\mathbb{E}}}
\newcommand{\Varx}{\mathop{\mathsf{Var}}}

\newcommand\numberthis{\addtocounter{equation}{1}\tag{\theequation}}

%
% Put your custom macros here if you want.
%
\usepackage{amsmath}
\newcommand{\Crit}{Crit}

\begin{document}

% This creates the header for scribe notes.
\noindent \coursename \hfill Scribe: \scribe \\
\lecturedate \hfill Lecturer: \lecturer
\vspace{1em}

\hrule

\vspace{1.5em}
\begin{center}
  {\Large\lecturetitle}
\end{center}

\section{Introduction}

	This lecture covers the the second half of the calculations of Giorgio Parisi's famous formula derived by use of replica symmetry breaking.

\begin{proposition}[MGF of a Gaussian]
\label{gaussian-mgf}
\begin{equation}
\EE_{g \sim \mathcal{N}(0,1)} \exp(ga) = \exp(a^{2}/2)
\end{equation}
\end{proposition}

\section{The RSB ansatz: Frobenius norm of $Q$}

Recall that we left off with, using the replica trick,

\begin{equation}
\label{replica-expansion}
    \EE Z(\beta)^{n} \approx \exp\Big(
    N\sup_{Q}\inf_{\Lambda}\Big[n\log 2 + \beta^2 \norm{Q}_{F}^{2} + \log\EE_{y \sim Unif(\{1, -1\}^{n})}\exp(y^{T}\Lambda y - \langle \Lambda, Q \rangle]
    \Big)
\end{equation}

By taking matrix derivatives, one can solve for the minimizing $\Lambda$ in the expression as $2\beta^{2}Q$.

\begin{equation}
    \EE Z(\beta)^{n} \approx \exp\Big(
    N\sup_{Q}\Big[n\log 2 + \beta^2 \norm{Q}_{F}^{2} + n + \log\EE_{y}\exp(2\beta^2y^{T}Qy)\Big]
    \Big)
\end{equation}


The replica symmetry breaking ansatz has parameters $n \geq m_{1} \geq m_{2} > ... m_{k} \geq 1$. Each $a_{i}$ is the value of the block between each pair of levels. The idea of Parisi is that for an integer number of replicas, the entries of the replica matrix $Q$ can take only $k$ positive values.

\begin{equation}
    Q_{i, j} = a_{i} \text{ if } \Big[\frac{i}{m_k}\Big] \neq \Big[\frac{j}{m_k}\Big] \text{ and } \Big[\frac{i}{m_{k+1}}\Big] = \Big[\frac{j}{m_{k+1}}\Big]
\end{equation}



We need $Q^{*}$ to be positive semidefinite. Physically, this corresponds to stability of the free energy under infinitesimal perturbations. So, we have a condition on the $a_{i}$'s to make this happen. It turns out that $Q^{*}$ is PSD if and only if $a_{0} \leq a_{1} \leq \dots \leq a_{k} \leq 1$.


This block structure of $Q^{*}$ allows us to directly calculate the Frobenius norm of $Q^{*}$.

\begin{equation}
\label{norm}
    \norm{Q^{*}}_{F}^{2} = n[1 + \sum_{j = 0}^{k}a_{j}^2(m_{j} - m_{j+1})]
\end{equation}

if we define $m_0 = n$ and $m_{k+1} = 1$.

Recall that $f(\beta) = \lim_{N \to \infty} \frac{1}{n}\lim_{n \to 0} \frac{1}{n}\log\EE Z(\beta)^{n}$. As $n \to \infty, \frac{1}{n}\norm{Q^{*}}_{F}^{2} = 1 + \sum_{j = 0}^{k}a_j^2(m_j - m_{j+1}$. {\it Weird physics alert: } Also as $n \to \infty$, the chain of inequalities $n \geq m_1 \geq \dots ... \geq 1$ reverses to $m_0 \leq m_1 \leq \dots \leq m_{k+1} = n$. See the Parisi paper. It has something to do with the symmetric group of 0 elements.

The sum in \eqref{norm} can be interpreted as a Riemann sum as the $m_{i}'s$ fill the interval; as $k \to \infty$. This sum turns into $1 - \int_{0}^{1}a(x)^{2}dx$.

So we have

\begin{equation}
    \frac{1}{n}\norm{Q}_{F}^{2} = 1 - \int_{0}^{1}a(x)^{2}dx
\end{equation}


In the replica symmetric version, we ended up with an optimization over $a$. Now, for each $x$, we optimize over $a(x)$. So we write the $\log\EE_{y}\exp(2\beta^2y^{T}Qy)$ term in terms of $a(x)$. In the literature, this function $a$ (often seen as $q$), is called the {\it Parisi order parameter}.

\section{RSB Ansatz: Entropy Term}

Now we deal with the entropy term in \eqref{replica-expansion}.

\begin{proposition}
\begin{equation}
    \EE_{y \sim Unif(\{\pm 1\}^{n}} \exp\Big(\frac{1}{2}y^{T}\Sigma y + \langle \mu, y \rangle \Big) = \EE_{g \sim \mathcal{N}(\mu,\Sigma)} \prod_{i}\cosh(g_i)
\end{equation}
\end{proposition}

\begin{proof}
Use the spectral decomposition of $\Sigma$, which we take to be positive definite.

\begin{equation}
    \Sigma = V\Lambda V^{T} = \sum_{i = 1}^{n}\lambda_{i}v_{i}v_{i}^{T}
\end{equation}

Then the LHS becomes

\begin{equation}
    \EE_{y \sim Unif(\{\pm 1\}^{n}} \exp\Big(\frac{1}{2}y^{T}\Sigma y + \langle \mu, y \rangle \Big) = \EE_{y} \exp\Big(\frac{1}{2}\sum\lambda_{i}\langle v_{i}, y \rangle^{2} + \langle \mu, y \rangle \Big)
\end{equation}

Now we use the proposition, the MGF of a Gaussian, because we can write the term in the exponential as $\sum_{j}y_{j}(\mu_{j} + \sum_{i}v_{i}\sqrt{\lambda_{i}}y_i)$. So we can write this as

\begin{equation}
    \EE_{g_{i} \sim N(0, 1)}\exp(\sum_{j}y_{j}(\mu_{j} + \sum_{i}v_{i}\sqrt{\lambda_{i}}g_i)) =  \EE_{g \sim N(\mu, \Sigma)}\cosh(\mu_j + (V\sqrt{D} g)_{j})
\end{equation}

\end{proof}

This yields the following recursion

\begin{equation}
\EE_{y}\exp(\frac{1}{2}y^{T}Q^{(k-1)}y) = \EE_{g \sim N(0, 1)}\prod_{i}\cosh(g_i)
\end{equation}

Let $\mathbb{1}^{(j)}$ be the matrix of $j^{2}$ square blocks that is one on the diagonal.

$Q^{(k)} = a_{0}\mathbb{1}^{(1)} + (a_1 - a_0)\mathbb{1}^{(3)} + ...$


Starting from the first step RSB, we have, letting $g = h*1 + g_1$,

\begin{equation}
\EE_{y}\exp(2\beta^2y^{T}Qy) = \EE_{h \sim N(0, a_0)}\EE_{g \sim N(0, \mathbb{1}^{(3)}}\prod_{i = 1}^{n}\cosh(h + g_i) = \EE_{h \sim N(0,a_0)}[\EE_{y}\exp(\frac{1}{2}y^{T}Q^{(0)}y + h\mathbb{1}^{T}y)]^{(n/m_1)}
\end{equation}


This procedure can be iterated. It is closely related to the heat equation, which comes up in Parisi's formula.


\end{document}