% Template based on:
% https://www.cs.columbia.edu/~djhsu/coms4774-s21/scribe.html

\documentclass[12pt]{article}

\usepackage{amscd}
\usepackage{amsmath}
\usepackage{float}
\usepackage{amssymb}
\usepackage{amstext}
\usepackage{amsthm}
\usepackage{bbold}
\usepackage{bm}
\usepackage{colonequals}
\usepackage[dvips,letterpaper,margin=1in]{geometry}
\usepackage{mathtools}
\usepackage{graphicx}
\usepackage{xcolor}

\usepackage{csquotes}
\MakeOuterQuote{"}

\definecolor{cblack}{rgb}{0,0,0}
\definecolor{cblue}{rgb}{0.121569,0.466667,0.705882}
\definecolor{corange}{rgb}{1.000000,0.498039,0.054902}
\definecolor{cgreen}{rgb}{0.172549,0.627451,0.172549}
\definecolor{cred}{rgb}{0.839216,0.152941,0.156863}
\definecolor{cpurple}{rgb}{0.580392,0.403922,0.741176}
\definecolor{cbrown}{rgb}{0.549020,0.337255,0.294118}
\definecolor{cpink}{rgb}{0.890196,0.466667,0.760784}
\definecolor{cgray}{rgb}{0.498039,0.498039,0.498039}

\usepackage{hyperref}
\hypersetup{
  linkcolor  = cblue,
  citecolor  = cgreen,
  urlcolor   = corange,
  colorlinks = true,
}

% Lecture information
\newcommand\coursename{CPSC 664, Spring 2023}
\newcommand\scribe{Alex Wardle-Solano}
\newcommand\lecturer{Tim Kunisky}
\newcommand\lecturedate{January 24, 2023}
\newcommand\lecturetitle{Lecture 3: Integer Partitioning III; Random $k$-SAT}

% Define theorem environments here.
\newtheorem{claim}{Claim}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{remark}[theorem]{Remark}
\newtheorem{assumption}[theorem]{Assumption}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{question}[theorem]{Question}
\newtheorem{problem}[theorem]{Problem}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{example}[theorem]{Example}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{conjecture}[theorem]{Conjecture}

% Some of my macros
\renewcommand{\AA}{\mathbb{A}}
\newcommand{\CC}{\mathbb{C}}
\newcommand{\DD}{\mathbb{D}}
\newcommand{\EE}{\mathbb{E}}
\newcommand{\FF}{\mathbb{F}}
\newcommand{\HH}{\mathbb{H}}
\newcommand{\NN}{\mathbb{N}}
\newcommand{\PP}{\mathbb{P}}
\newcommand{\QQ}{\mathbb{Q}}
\newcommand{\RR}{\mathbb{R}}
\renewcommand{\SS}{\mathbb{S}}
\newcommand{\ZZ}{\mathbb{Z}}
\DeclareSymbolFont{bbold}{U}{bbold}{m}{n}
\DeclareSymbolFontAlphabet{\mathbbold}{bbold}
\newcommand{\One}{\mathbbold{1}}

\newcommand{\ba}{\bm a}
\newcommand{\bb}{\bm b}
\newcommand{\bc}{\bm c}
\newcommand{\bd}{\bm d}
\newcommand{\be}{\bm e}
\newcommand{\bg}{\bm g}
\newcommand{\bh}{\bm h}
\newcommand{\bi}{\bm i}
\newcommand{\bj}{\bm j}
\newcommand{\bk}{\bm k}
\newcommand{\bbm}{\bm m}
\newcommand{\bp}{\bm p}
\newcommand{\bq}{\bm q}
\newcommand{\br}{\bm r}
\newcommand{\bs}{\bm s}
\newcommand{\bt}{\bm t}
\newcommand{\bu}{\bm u}
\newcommand{\bv}{\bm v}
\newcommand{\bw}{\bm w}
\newcommand{\bx}{\bm x}
\newcommand{\by}{\bm y}
\newcommand{\bz}{\bm z}

\newcommand{\bA}{\bm A}
\newcommand{\bB}{\bm B}
\newcommand{\bC}{\bm C}
\newcommand{\bD}{\bm D}
\newcommand{\bE}{\bm E}
\newcommand{\bF}{\bm F}
\newcommand{\bG}{\bm G}
\newcommand{\bH}{\bm H}
\newcommand{\bI}{\bm I}
\newcommand{\bL}{\bm L}
\newcommand{\bM}{\bm M}
\newcommand{\bN}{\bm N}
\newcommand{\bP}{\bm P}
\newcommand{\bQ}{\bm Q}
\newcommand{\bR}{\bm R}
\newcommand{\bS}{\bm S}
\newcommand{\bT}{\bm T}
\newcommand{\bU}{\bm U}
\newcommand{\bV}{\bm V}
\newcommand{\bW}{\bm W}
\newcommand{\bX}{\bm X}
\newcommand{\bY}{\bm Y}
\newcommand{\bZ}{\bm Z}

\newcommand{\zero}{\bm{0}}
\newcommand{\one}{\bm{1}}

\newcommand{\sA}{\mathcal{A}}
\newcommand{\sB}{\mathcal{B}}
\newcommand{\sC}{\mathcal{C}}
\newcommand{\sD}{\mathcal{D}}
\newcommand{\sE}{\mathcal{E}}
\newcommand{\sF}{\mathcal{F}}
\newcommand{\sG}{\mathcal{G}}
\newcommand{\sH}{\mathcal{H}}
\newcommand{\sI}{\mathcal{I}}
\newcommand{\sJ}{\mathcal{J}}
\newcommand{\sK}{\mathcal{K}}
\newcommand{\sL}{\mathcal{L}}
\newcommand{\sM}{\mathcal{M}}
\newcommand{\sN}{\mathcal{N}}
\newcommand{\sO}{\mathcal{O}}
\newcommand{\sP}{\mathcal{P}}
\newcommand{\sQ}{\mathcal{Q}}
\newcommand{\sR}{\mathcal{R}}
\newcommand{\sS}{\mathcal{S}}
\newcommand{\sT}{\mathcal{T}}
\newcommand{\sU}{\mathcal{U}}
\newcommand{\sV}{\mathcal{V}}
\newcommand{\sX}{\mathcal{X}}
\newcommand{\sY}{\mathcal{Y}}

\newcommand{\fB}{\mathscr{B}}
\newcommand{\fC}{\mathscr{C}}
\newcommand{\fE}{\mathscr{E}}
\newcommand{\fH}{\mathscr{H}}
\newcommand{\fI}{\mathscr{I}}
\newcommand{\fP}{\mathscr{P}}
\newcommand{\fV}{\mathscr{V}}

\DeclareSymbolFont{sfoperators}{OT1}{cmss}{m}{n}
% don't waste a math group
\DeclareSymbolFontAlphabet{\mathsf}{sfoperators}
% tell LaTeX to use sfoperators for names of operators
\makeatletter
\renewcommand{\operator@font}{\mathgroup\symsfoperators}
\makeatother

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\GOE}{GOE}
\DeclareMathOperator{\sym}{sym}
\DeclareMathOperator{\Part}{Part}
\DeclareMathOperator{\poly}{poly}
\DeclareMathOperator{\Unif}{Unif}
\DeclareMathOperator{\Var}{Var}
\DeclareMathOperator{\Cov}{Cov}
\DeclareMathOperator{\Tr}{Tr}
\DeclareMathOperator{\rank}{rank}
\DeclareMathOperator{\obj}{obj}
\DeclareMathOperator{\diag}{diag}
\DeclareMathOperator{\sgn}{sgn}

\renewcommand\Re{\operatorname{Re}}
\renewcommand\Im{\operatorname{Im}}
% These put symbols below.
\newcommand{\Px}{\mathop{\mathbb{P}}}
\newcommand{\Ex}{\mathop{\mathbb{E}}}
\newcommand{\Varx}{\mathop{\mathsf{Var}}}

\newcommand{\obj}{\mathsf{obj}}

\newcommand\numberthis{\addtocounter{equation}{1}\tag{\theequation}}

%
% Put your custom macros here if you want.
%

\begin{document}

% This creates the header for scribe notes.
\noindent \coursename \hfill Scribe: \scribe \\
\lecturedate \hfill Lecturer: \lecturer
\vspace{1em}

\hrule

\vspace{1.5em}
\begin{center}
  {\Large\lecturetitle}
\end{center}

\section{Number Partitioning in the Satisfiable Regime}

Recall that in the last lecture we showed, by the second moment method, that when $2^n / B\sqrt{n} \to \infty$, then with high probability there exist perfect partitions of $a_1, \dots, a_n$.
We call this the ``satisfiable'' regime of the number partitioning problem.
Let us give a few more details about how the problem behaves in this case.

Define the objective function that we are interested in optimizing:
\begin{equation}
    \obj(x) \colonequals \bigg|\sum_{i=1}^{n}x_{i}a_{i}\bigg| \text{ for } x \in \{-1,1\}.
\end{equation}

We can then split up the range of our objective function into two-wide bins, $\{0, 1\}, \{2, 3\}$, and so forth.
Let $Y_{2k}$ equal the number of $x$ that achieve objective value in $\{2k, 2k + 1\}$.
We recall that we observed that, for any particular draw of $a_i$, only even or only odd values of $\obj(x)$ can be achieved by \emph{any} $x$.
But, binning the objective values this way, one can show that the $Y_{2k}$ concentrate; in particular, when $2^n / B\sqrt{n} \to \infty$, then there are many $x$'s achieving values in each bin, and so each $Y_{2k}$ is large with high probability.

\begin{figure}[hbt!]
    \label{fig:f}
    \begin{center}
        \includegraphics[scale=0.23]{2023-01-24-fig2.jpeg}
    \end{center}
    \caption{A plot of the achievable values of $\obj(x)$ in the satisfiable regime.}
\end{figure}

\section{Number Partitioning in the Unsatisfiable Regime}

The far more interesting case from the point of view of the optimization landscape is the ``unsatisfiable'' regime, when $2^n / B\sqrt{n} \to 0$.
In this regime, with high probability, the $Y_{2k} = 0$ for all $k$ up to any constant value (by the first moment method).
We are still interested in the value of the problem, i.e., the smallest achievable $\obj(x)$ over all $x$:
\begin{equation}
    \min_{x \in \{\pm 1\}^n} \obj(x) \equalscolon d_0.
\end{equation}

Since $\obj(x) = \obj(-x)$, each $Y_{2k}$ is even, so heuristically we might expect that $d_{0}$ should be the smallest value such that
\begin{equation}
    \mathbbold{E}[Y_{0} + Y_{2} + ... + Y_{d_0}] = 2.
\end{equation}

Since each $Y_{2k}$ is a sum of four of the $Z_k$ (as defined in previous lectures), and $\EE[Z_k] \approx \sqrt{3 / 2\pi} 2^n / B\sqrt{n}$ for each small $k$, we expect to have

\begin{align*}
  \mathbbold{E}[Y_{0} + Y_{2} + ... + Y_{d_1}] &\approx \frac{d_{0}}{2} \cdot 4 \cdot \mathbbold{E}[Z_0] \\
  &\approx \frac{d_{0}}{2} \cdot 4 \cdot \sqrt{\frac{3}{2\pi}} \frac{2^n}{B\sqrt{n}} \\
  &= d_{0}\sqrt{\frac{6}{\pi}}\frac{2^{n}}{B\sqrt{n}}
\end{align*}
This gives us a heuristic value for the bottom value of our objective function over all $x$,
\begin{equation}
    d_0 \stackrel{?}{=} \sqrt{\frac{2\pi}{3}} \frac{B\sqrt{n}}{2^n}.
\end{equation}

\begin{figure}[hbt!]
    \label{fig:g}
    \begin{center}
        \includegraphics[scale=0.16]{2023-01-24-fig1.jpeg}
    \end{center}
    \caption{A plot of the achievable values of $\obj(x)$ in the unsatisfiable regime.}
\end{figure}

In fact, surprisingly, one can show that this heuristic holds along with much stronger statements about the $x$ achieving small objective values.
Consider not just the minimizing value of $\obj(x)$ but the ordered set of smallest values of the objective function. We can plot these minimal values, normalizing our axis to order constant. Here, $\widehat{d}_{i}$ denotes the $i$th smallest value of the objective function. We then denote the sequence of $\Delta_{i}$'s to represent the normalized distances between the minimal values of the objective function. One may show the following with some involved moment calculations.

\begin{theorem}
    Fix $k \in \NN$.
    Let $v = (\Delta_{1}, ..., \Delta_{k}) \in \mathbb{R}_{\geq 0}^{k}$.
    Then, for $A \subset \mathbb{R}_{\geq 0}^k$,
    \begin{equation}
        \lim_{n\to\infty}\mathbb{P}((\Delta_{1}, ..., \Delta_{k}) \in A) = \mathbb{P}((E_{1}, ... , E_{k}), \in A)
    \end{equation}
    where the $E_i$ are distributed as
    \begin{equation}
        E_{i} \overset{\mathrm{iid}}{\sim} \mathsf{Exp}\bigg(\sqrt\frac{3}{2\pi}\bigg).
    \end{equation}
\end{theorem}

That is, as $n$ tends to infinity, the vector of normalized distances between the $k$ minimal values of our objective function tends to a vector of independent and identically distributed exponential variables (that is, the minimal values form a \emph{Poisson process}).
Note that, in particular, we have
\begin{equation}
    \EE[\widehat{d}_0] \to \EE[E_1] = \sqrt{\frac{2\pi}{3}},
\end{equation}
confirming our heuristic prediction.

For further information about the optimization landscape, we may consider the configuration of the vectors $x$ that achieve these small values of $\obj(x)$.
In particular, one can show (again using modified moment calculations) that, with high probability,
\begin{equation}
    \EE \max_{1\le i\le j\le k} |\langle x_{i}, x_{j} \rangle| = O(\sqrt{n})
\end{equation}

In other words, the $k$ minimizing $x$'s are distributed with the same correlation in the hypercube of dimension $n$ as $k$ random points would be, suggesting that finding these points should be computationally hard.

\section{Analogy to the Random Energy Model}

The above results were predicted in the physics literature before they were proved mathematically by the following simple heuristic.
The integer partitioning problem is specified by the random mapping $\obj: \{\pm 1\}^n \to \RR_{\geq 0}$, with the values of $\obj(x)$ being correlated in complicated ways since they all depend on the underlying $a_i$.

In the \emph{random energy model}, we consider a related but different random mapping $\obj^{\prime}: \{\pm 1\}^n \to \RR_{\geq 0}$, where the values of $\obj^{\prime}(x)$ are \emph{independent}.
Consider the value of the original function $\obj(x)$ for a \emph{random} $x$.
This is the sum of $n$ i.i.d.\ random variables distributed as $\Unif(\{-B, \dots, B\})$, in particular having variance roughly $\frac{1}{3}B^2$.
So, by the central limit theorem, we expect
\begin{equation}
    \obj(x) \stackrel{(\mathrm{d})}{\approx} \left| \sN\left(0, \frac{1}{3}B^2n\right) \right|,
\end{equation}
with $|\sN(\cdot, \cdot)|$ denoting the ``folded'' normal distribution given by the absolute value of a normal variable.

In the random energy model, we \emph{define}
\begin{equation}
    \obj^{\prime}(x) \stackrel{\text{iid}}{\sim} \left| \sN\left(0, \frac{1}{3}B^2n\right) \right|.
\end{equation}
Since the objective values of this model are just i.i.d.\ random variables, it is much easier to compute all of the statistics from the previous section for $\obj^{\prime}$.
Surprisingly, all of the results above hold for this new objective function!

This suggests that the computational problem of minimizing $\obj$ should behave like that of minimizing $\obj^{\prime}$.
But, the latter is just the problem of finding the smallest of $2^n$ i.i.d.\ draws of a particular distribution, which is obviously computationally hard.
So, this analogy gives some evidence that we should expect the number partitioning problem to be very hard in the unsatisfiable regime.

\section{Introduction to Random $k$-SAT}

The random $k$-SAT problem refers to the problem of finding a satisfying truth assignment for a Boolean formula in $z_{1}, ..., z_{n} \in {0, 1}$. A $k$-SAT instance consists of $m$ \emph{clauses}, each consisting of $k$ literals joined by inclusive disjunctions. Such a $k$-SAT formula is satisfied when all of the constituent clauses are satisfied, and looks like:
\begin{align}
    F(z) = &((\neg)z_{11} \vee (\neg)z_{12} \cdots (\neg)z_{1k}) \wedge \\
    &\vdots  \\
    &((\neg)z_{m1} \vee (\neg)z_{m2} \cdots (\neg)z_{mk}),
\end{align}
with each negation $\neg$ possibly occurring or not occurring.

In the \emph{random} $k$-SAT problem, we choose each of the $m$ clauses independently and uniformly at random from the set of all $\binom{n}{k} 2^k$ possible clauses.
The problem we are concerned with is whether or not there exists some assignment $x \in {0,1}^{n}$ such that $F(x) = 1$.
Numerical experiments suggest the following \emph{phase transition} or \emph{threshold} phenomenon.

\begin{conjecture}
    Suppose that $k$ and some $\alpha \in \mathbbold{R}_{\geq 0}$ are fixed, and $m = \alpha n$ with $n \to \infty$.
    Then, there exists $\alpha^* = \alpha^*(k)$ such that
    \begin{equation}
        \begin{array}{ll} \alpha > \alpha^* & \implies F \text{ is unsatisfiable with high probability,} \\
          \alpha < \alpha^* & \implies F \text{ is satisfiable with high probability.} \end{array}
    \end{equation}
\end{conjecture}

We leave the proof of this conjecture for future lectures but consider some of the important questions we may want to ask ourselves going forward.
The partial proof of this conjecture is very involved, but we will see some of the basic ideas involved in the coming few lectures and in the remainder of the course.
As a preview, we will be concerned with the following questions: first, what is this value of $\alpha^*$?
And, what is the structure of satisfying $x$'s when $\alpha < \alpha^*$?

\end{document}
