```python
!pip install nangs
```

    Requirement already satisfied: nangs in /usr/local/lib/python3.7/dist-packages (0.1.2)

```python
# autoreload nangs
%reload_ext autoreload
%autoreload 2
%matplotlib inline
```

```python

```

```python
#imports

import numpy as np
import matplotlib.pyplot as plt
import torch
import pandas as pd
import nangs
from nangs import *

device = "cuda" if torch.cuda.is_available() else "cpu"

nangs.__version__, torch.__version__
```

    ('0.1.2', '1.9.0+cu102')

```python
class Sine(torch.nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, x):
        return torch.sin(x)

def block(i, o):
    fc = torch.nn.Linear(i, o)
    return torch.nn.Sequential(
        #Sine(),
        torch.nn.Dropout(p=0.),
        torch.nn.GELU(),
        torch.nn.Linear(i, o)
    )

class MLP(torch.nn.Module):
    def __init__(self, inputs, outputs, layers, neurons):
        super().__init__()
        fc_in = torch.nn.Linear(inputs, neurons)
        fc_hidden = [
            block(neurons, neurons)
            for layer in range(layers-1)
        ]
        fc_out = block(neurons, outputs)

        self.mlp = torch.nn.Sequential(
            fc_in,
            *fc_hidden,
            fc_out
        )

    def forward(self, x):
        return self.mlp(x)
```

```python
device

```

    'cuda'

```python
alpha=10
class Difussion2d(PDE):
    def computePDELoss(self, inputs, outputs):
        p = outputs[:, 0]
        grads = self.computeGrads(p, inputs)
        dpdx, dpdt = grads[:, 0], grads[:, 1]
        # second order derivatives
        dp2dx2 = self.computeGrads(dpdx, inputs)[:, 0]
        #dp2dy2 = self.computeGrads(dpdy, inputs)[:, 1]
        return {'pde':dpdt - dp2dx2}
```

```python
class NeumannX(Neumann):
    def computeBocoLoss(self, inputs, outputs):
        p = outputs[:, 0]
        grads = self.computeGrads(p, inputs)
        dpdx = grads[:, 0]
        return {'gradX': dpdx}
```

```python
class PBC0(Neumann):
    def computeBocoLoss(self, inputs, outputs):
        pbc = 0
        p = outputs[:, 0]
        return {'PWF': 100*(p-pbc)}
```

```python
pde = Difussion2d(inputs=('x', 't'), outputs='p')
```

```python
def append(arr, values, axis=None):
    arr = np.asanyarray(arr)
    if axis is None:
        if arr.ndim != 1:
            arr = arr.ravel()
        values = np.ravel(values)
        axis = arr.ndim-1
    return np.concatenate((arr, values), axis=axis)
```

```python
# mesh

x = np.linspace(0,1,100)

t = np.linspace(0,1,100)
#t = append(t,np.linspace(0.2,1,10))
#t=np.array([0])
#tt=np.logspace(-2,0,20)
#t = append(t,tt)
mesh = Mesh({'x': x, 't':t}, device=device)
pde.set_mesh(mesh)
```

```python
from nangs import Dirichlet
```

```python
t0 = np.array([0])
```

```python
_x = np.meshgrid(x)
#p0 = np.sin(2*np.pi*_x)*np.sin(2*np.pi*_y)
```

```python

_x[0].shape
```

    (100,)

```python
p0=np.zeros((100))
p0[:]=1
p0[0]=0
#p0[47:53,47:53]=1
```

```python
p0.shape
```

    (100,)

```python
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
from matplotlib import cm
fig = plt.figure()
#ax = fig.gca(projection='3d')
plt.plot(_x[0], p0)
#fig.colorbar(surf, shrink=0.5, aspect=5)
#plt.show()
```

```python
initial_condition = Dirichlet({'x': x,'t':t0}, {'p': p0.reshape(-1)}, device=device, name="initial")
```

```python
p0.shape
```

    (100,)

```python
pde.add_boco(initial_condition)
#p0=np.ones(len(y))
#p1=np.zeros(len(y))
```

```python
pwf = PBC0({'x': np.array([0]),'t':t}, name='pwf', device=device)
#left = Dirichlet({'x': x, 'y': y}, {'p': p0.reshape(-1)}, device=device, name="left")
```

```python

pde.add_boco(pwf)
```

```python
#right = PBC0({'x': np.array([1]), 'y': y, 't':t}, name='right', device=device)
#right = Dirichlet({'x': x[-1], 'y': y}, {'p': p1.reshape(-1)}, device=device, name="right")
right =  NeumannX({'x': x[-1],'t':t}, name='right', device=device)
```

```python
pde.add_boco(right)
```

```python
#from nangs import MLP

BATCH_SIZE = 128
LR = 1e-2
EPOCHS = 2
NUM_LAYERS = 3
NUM_HIDDEN = 50

mlp = MLP(len(pde.inputs), len(pde.outputs), NUM_LAYERS, NUM_HIDDEN).to(device)
optimizer = torch.optim.Adam(mlp.parameters())
scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=LR, pct_start=0.1, total_steps=EPOCHS)
```

```python
mlp
```

    MLP(
      (mlp): Sequential(
        (0): Linear(in_features=2, out_features=50, bias=True)
        (1): Sequential(
          (0): Dropout(p=0.0, inplace=False)
          (1): GELU()
          (2): Linear(in_features=50, out_features=50, bias=True)
        )
        (2): Sequential(
          (0): Dropout(p=0.0, inplace=False)
          (1): GELU()
          (2): Linear(in_features=50, out_features=50, bias=True)
        )
        (3): Sequential(
          (0): Dropout(p=0.0, inplace=False)
          (1): GELU()
          (2): Linear(in_features=50, out_features=1, bias=True)
        )
      )
    )

```python
EPOCHS = 300
scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=LR, pct_start=0.1, total_steps=EPOCHS)
```

```python
pde.compile(mlp, optimizer, scheduler)
```

```python

%time hist = pde.solve(EPOCHS, BATCH_SIZE)
```

```python

```

```python
t = 0.0
x = np.linspace(0,1,100)
#y = np.linspace(0,1,100)

_x, _t = np.meshgrid(x, t)
eval_mesh = Mesh({'x': x,  't':t}, device=device)
p = pde.eval(eval_mesh)
p = p.cpu().numpy()
#p5 = p.reshape((_y.shape[0],_x.shape[1]))
```

```python
p,grad,loss=pde.eval_with_grad(eval_mesh)
```

```python
ppp=[]
loss = []
x = np.linspace(0.0,1,100)
for t in [0.01,0.015,0.02,0.1,1,2]:#np.linspace(0.5,1,2):
  #_x,_t = np.meshgrid(x,t)
  eval_mesh = Mesh({'x': x,  't':t}, device=device)
  pp,g,l = pde.eval_with_grad(eval_mesh)
  loss.append(l)
  #pp = pp.cpu().numpy()
  ppp.append(pp)
```

```python
for l in loss:
  plt.plot(np.abs(l))
```

```python
for l in loss:
  plt.plot(np.abs(l))
```

```python
from sympy import Sum, factorial, oo, IndexedBase, Function, sin,exp
from sympy.abc import n
```

```python
def diff_sol(x,t,alpha=0.1):
    sol= Sum(4/((2*n + 1)*np.pi)*sin((2*n+1)*np.pi/2*x)*exp(-((2*n + 1)*np.pi/2)**2*t*alpha),(n,0,50))

    return sol #,sol_n
```

```python
xxx=np.linspace(0,1,100)
for t in np.linspace(0.1,1,11):
    d=[]
    for xx in xxx:
        d.append(diff_sol(xx,t,alpha=1).doit())
    plt.plot(d)
    eval_mesh = Mesh({'x': x,  't':t}, device=device)
    pp = pde.eval(eval_mesh)
    pp = pp.cpu().numpy()
    plt.plot(p)
```

```python
xxx=np.linspace(0,1,10)
x = np.linspace(0,1,10)
for t in np.linspace(0.02,1,11):
    d=[]
    for xx in xxx:
        d.append(diff_sol(xx,t,alpha=1).doit())
    plt.plot(d)
    eval_mesh = Mesh({'x': x,  't':t}, device=device)
    pp = pde.eval(eval_mesh)
    pp = pp.cpu().numpy()
    plt.plot(pp,marker='o',linestyle='None')
```

```python

out=pde.eval_with_grad(eval_mesh)
```

```python
out_np=out[1].detach().cpu().numpy()
```

```python
out_np.shape
```

    (100, 2)

```python
xxx=np.linspace(0,1,10)
x = np.linspace(0,1,10)
for t in np.linspace(0.5,1.2,3):
    d=[]
    for xx in xxx:
        d.append(diff_sol(xx,t,alpha=1).doit())
    plt.plot(d)
    eval_mesh = Mesh({'x': x,  't':t}, device=device)
    pp = pde.eval(eval_mesh)
    pp = pp.cpu().numpy()
    plt.plot(pp,marker='o',linestyle='None')
```

```python
class PDE():

    def __init__(self, inputs, outputs):

        # check lists of unique strings, non-repeated
        if isinstance(inputs, str):
            inputs = tuple(inputs)
        if isinstance(outputs, str):
            outputs = tuple(outputs)

        checkIsListOfStr(inputs)
        checkIsListOfStr(outputs)
        checkUnique(inputs)
        checkUnique(outputs)
        checkNoRepeated(inputs, outputs)

        self.inputs = inputs
        self.outputs = outputs
        self.mesh = None
        self.bocos = []

    def set_mesh(self, mesh):
        assert mesh.vars == self.inputs, "your data does not match the PDE inputs"
        self.mesh = mesh

    def add_boco(self, boco):
        assert boco.name not in [
            boco.name for boco in self.bocos], f'Boco {boco.name} already exists, use another name'
        boco.validate(self.inputs, self.outputs)
        self.bocos.append(boco)

    def compile(self, model, optimizer, scheduler=None, criterion=None):
        self.model = model
        self.optimizer = optimizer
        self.criterion = criterion if criterion else torch.nn.MSELoss()
        self.scheduler = scheduler

    def computePDELoss(self, vars, grads):
        print("This function need to be overloaded !!!")

    def solve(self, epochs=50, batch_size=None, shuffle=True, graph=True):
        dataloaders = self.set_dataloaders(batch_size, shuffle)
        if graph:
            self.graph_fig, (self.graph_ax1, self.graph_ax2) = plt.subplots(
                1, 2, figsize=(15, 5))
            self.graph_out = display(self.graph_fig, display_id=True)
        # solve PDE
        history = History()
        mb = master_bar(range(1, epochs+1))
        for epoch in mb:
            history.add({'lr': get_lr(self.optimizer)})
            # iterate over the internal points in batches
            for batch in progress_bar(dataloaders['inner'], parent=mb):
                X = batch
                self.optimizer.zero_grad()
                # optimize for boundary points
                for boco in self.bocos:
                    for batch in dataloaders['bocos'][boco.name]:
                        loss = boco.computeLoss(
                            batch, self.model, self.criterion)
                        for name, l in loss.items():
                            l.backward()
                            history.add_step({name: l.item()})
                # optimize for internal points
                X.requires_grad = True
                p = self.model(X)
                loss = self.computePDELoss(X, p)
                assert isinstance(
                    loss, dict), "you should return a dict with the name of the equation and the corresponding loss"
                for name, l in loss.items():
                    l = self.criterion(l, torch.zeros(
                        l.shape).to(self.mesh.device))
                    l.backward(retain_graph=True)
                    history.add_step({name: l.item()})
                self.optimizer.step()
                mb.child.comment = str(history.average())
            history.step()
            mb.main_bar.comment = str(history)
            if graph:
                self.plot_history(history)
            # mb.write(f"Epoch {epoch}/{epochs} {history}")
            if self.scheduler:
                self.scheduler.step()
        if graph:
            plt.close()
        return history.history

    def plot_history(self, history):
        self.graph_ax1.clear()
        self.graph_ax2.clear()
        for name, metric in history.history.items():
            if name != 'lr':
                self.graph_ax1.plot(metric, label=name)
            else:
                self.graph_ax2.plot(metric, label=name)
        self.graph_ax1.legend(loc='upper right')
        self.graph_ax2.legend(loc='upper right')
        self.graph_ax1.grid(True)
        self.graph_ax2.grid(True)
        self.graph_ax1.set_yscale("log")
        self.graph_out.update(self.graph_fig)

    def set_dataloaders(self, batch_size, shuffle):
        dataloaders = {
            'inner': self.mesh.build_dataloader(batch_size, shuffle),
            'bocos': {}
        }
        for boco in self.bocos:
            dataloaders['bocos'][boco.name] = boco.build_dataloader(
                batch_size, shuffle)
        return dataloaders

    def computeGrads(self, outputs, inputs):
        grads, = torch.autograd.grad(outputs, inputs,
                                     grad_outputs=outputs.data.new(
                                         outputs.shape).fill_(1),
                                     create_graph=True, only_inputs=True)
        return grads

    def eval(self, mesh, batch_size=None):
        dataloader = mesh.build_dataloader(batch_size, shuffle=False)
        outputs = torch.tensor([]).to(mesh.device)
        self.model.eval()
        with torch.no_grad():
            for batch in dataloader:
                outputs = torch.cat([outputs, self.model(batch)])
        return outputs

    def eval_with_grad(self, mesh, batch_size=None):
        dataloader = mesh.build_dataloader(batch_size, shuffle=False)
        outputs = torch.tensor([]).to(mesh.device)
        self.model.eval()
        #with torch.no_grad():
        for batch in dataloader:
            batch.requires_grad = True
            outputs = self.model(batch)
        #for batch in dataloader:

            #outputs = self.model(batch)
            p=outputs
            X=batch
            X.requires_grad = True
            grad , = torch.autograd.grad(p, X,
                    grad_outputs=p.data.new(p.shape).fill_(1),
                    create_graph=True, only_inputs=True)
            loss = self.computePDELoss(X,p)
        return outputs,grad,loss['pde'].detach().cpu().numpy()

    def get_loss(self, mesh):
      batch_size = None
      dataloader = mesh.build_dataloader(batch_size, shuffle=False)
      self.model.eval()
      for batch in dataloader:
          batch.requires_grad = True
          outputs = self.model(batch)
          loss = self.computePDELoss(batch,outputs)
      return loss
```

```python
def checkIsListOfStr(l):
    "Make sure that l is a list containing only strings"
    if isinstance(l, tuple):
        for i in l:
            if not isinstance(i, str):
                raise Exception(str(i) + ' must be a string')

def checkUnique(l):
    "Make sure that l does not contain repeated elements"
    for i, item1 in enumerate(l):
        for j, item2 in enumerate(l):
            if i != j and item1 == item2:
                raise Exception('Repeated item ' + str(item1))

def checkNoRepeated(l1, l2):
    "Make sure there are no repeated elements in both lists"
    for i in l1:
        if i in l2:
            raise Exception('Repeated item ' + str(i))
```

```python
import numpy as np

class History():
    def __init__(self, precision=5):
        self.history = {}
        self.current = {}
        self.precision = precision

    def add(self, d):
        for name, metric in d.items():
            if not name in self.history:
                self.history[name] = []
            self.history[name].append(metric)

    def add_step(self, d):
        for name, metric in d.items():
            if name not in self.current:
                self.current[name] = []
            self.current[name].append(metric)

    def average(self):
        return {name: round(np.mean(self.current[name]), self.precision) for name in self.current}

    def step(self):
        for name in self.current:
            self.add({name: np.mean(self.current[name])})
        self.current = {}

    def __str__(self):
        s = ''
        for name, value in self.history.items():
            s += f' | {name} {round(value[-1], self.precision)}'
        return s
```

```python
from fastprogress import master_bar, progress_bar
```

```python
def get_lr(optimizer):
    for param_group in optimizer.param_groups:
        return param_group['lr']
```

```python

```


# Nystrom Method Coding

### Resources

###### Nystrom

* Scikit-Learn Implementation - [Github](https://github.com/scikit-learn/scikit-learn/blob/a24c8b46/sklearn/kernel_approximation.py#L367)

###### Randomized SVD (rSVD)

* Numpy svd Implementation - [Docs](https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.svd.html)
* Scikit-Learn rSVD Implementation - [Github](https://github.com/scikit-learn/scikit-learn/blob/a24c8b464d094d2c468a16ea9f8bf8d42d949f84/sklearn/utils/extmath.py#L228)
* Facebook PCA/rSVD Implementation - [Webpage](https://research.fb.com/fast-randomized-svd/) | [Docs](http://fbpca.readthedocs.io/en/latest/) | [Github](https://github.com/facebook/fbpca/blob/master/fbpca.py#L1393)

###### Papers

* Using The Nystrom Method to Speed Up Kernel Machines - [paper](https://papers.nips.cc/paper/1866-using-the-nystrom-method-to-speed-up-kernel-machines.pdf)
*

###### Research

* Nystrom + rSVD - [Paper](http://www.cs.cmu.edu/~muli/file/nystrom_icml10.pdf) | [Github](https://github.com/mli/nystrom)
    + Nystrom - [Github](https://github.com/mli/nystrom/blob/master/nys.m)
    + rSVD - [Github](https://github.com/mli/nystrom/blob/master/rsvd.m)
    + Nystrom + rSVD - [Github](https://github.com/mli/nystrom/blob/master/rnys.m)
* Locally Linear Landmarks - [Webpage](https://eng.ucmerced.edu/people/vladymyrov/) | [Paper](http://www.ecmlpkdd2013.org/wp-content/uploads/2013/07/131.pdf)
* Variational Nystrom Method - [Webpage](https://eng.ucmerced.edu/people/vladymyrov/)

```python
from sklearn.datasets import make_swiss_roll
from sklearn.metrics.pairwise import rbf_kernel
from sklearn.decomposition import TruncatedSVD
from sklearn.utils.extmath import randomized_svd
import fbpca

from scipy.spatial.distance import pdist
from scipy.linalg import cholesky
import numpy as np
import time as time

import matplotlib.pyplot as plt
import mpl_toolkits.mplot3d.axes3d as p3
%matplotlib inline
```

```python
# make some sample data
n_samples = 1500
noise = 0.01

X, y = make_swiss_roll(n_samples, noise)
```

```python
# plot the original data
fig = plt.figure()
ax = p3.Axes3D(fig)
ax.view_init(10, -50)
ax.scatter(X[:,0], X[:,1], y)
plt.title('Original Data')
ax.set_xlabel(r'$X_1$')
ax.set_ylabel(r'$X_2$')
ax.set_zlabel(r'$Y$')
plt.show()
```

```python
n_train_samples = 1e4
n_test_samples = 1e3
d_dims = 1
random_state = 123
from sklearn.utils import check_random_state

generator = check_random_state(random_state)
x_train = generator.randn(int(n_train_samples))
y_train = np.sin(x_train) * 0.1 * generator.randn(int(n_train_samples))

x_test = generator.randn(int(n_test_samples))
y_test = np.sin(x_test) * 0.1 * generator.randn(int(n_test_samples))

x_train = x_train[:, np.newaxis]
x_test = x_test[:, np.newaxis]
y_train = y_train[:, np.newaxis]
y_test = y_test[:, np.newaxis]

print('Size of x_train: {}'.format(x_train.shape))
print('Size of x_train: {}'.format(x_test.shape))
```

    Size of x_train: (10000, 1)
    Size of x_train: (1000, 1)

```python
# construct kernel matrix
gamma = 1 / (2 * np.mean(pdist(x_train, metric='euclidean')))
K = rbf_kernel(x_train, gamma=gamma)

print('Construct RBF matrix.')
print('Size of K: {}'.format(K.shape))
```

    Construct RBF matrix.
    Size of K: (10000, 10000)

## Rank-K Nystrom Approximation to PSD matrix A

We are looking to perform an approximation of matrix $K$ such that:

$$K \approx U \times D \times U$$

#### Parameters

* err = approximation error from the frobenius norm
* $n$ is the number if components (rank-k approximation) to be used in the approximation
* $m$ samples $m$ columns of matrix $K$
* idx can specify which columns to take indices of
* random state for the random components chosen

#### Checks

* $K$ should be square and symmetric
* Default $m$ should equal $k$.
* The $m \geq k$

#### SVD Terminology and Notation

The full **Singular Value Decomposition**:

$$A = U \Sigma V^T,$$

where $A \in \mathcal{R}^{m \times n}$

---
The **Singular Values** of $A$:

$\Sigma \in \mathcal{R}^{m \times n}$ where

* $\Sigma = \text{diag}\left( \sigma_1, \sigma_2, \ldots, \sigma_{\rho} \right)$
* $\rho=(m,n)$
* $\sigma_1 \geq \sigma_2 \geq \ldots \geq \sigma_{\rho} \geq 0$

---
The **Singular Vectors** of $A$:

$$U \in \mathcal{R}^{m \times m}, U^{T}U=I_m$$
$$V \in \mathcal{R}^{n \times n}, V^T V=I_n$$

---

The Best rank-k approximation of $A$:

$$A_k = U_k \Sigma_k V_k^T = \sum_{i=1}^{i=k} \rho_i U^{i} (V^{i})^{T}$$

---

The **Pseudo-Inverse** of $A$:

$$A^{\dagger} = V \Sigma^{-1} U^{T}$$

---

---

##### Nystrom Approximation

\begin{align}
K \approx \tilde{K} &= \tilde{U} \tilde{D} \tilde{U} \\
&= \left( \sqrt{\frac{m}{n}} C U_w \Sigma_{w}^{-1} \right) \left( \frac{n}{m} \Sigma_w \right)\left( \sqrt{\frac{m}{n}} C U_w \Sigma_{w}^{-1} \right)^{T} \\
&= C W_{r}^{\dagger} C^{T}
\end{align}

##### Nystrom Woodbury Approximation

Using the Matrix Inversion Lemma:

\begin{align}
\left( \lambda I + K\right)^{-1} &\approx \left(\lambda I + \tilde{K} \right)^{-1} \\
&= \left( \lambda I + C W_k^{\dagger} C^{T}  \right)^{-1} \\
&= \frac{1}{\lambda} \left( I - C \left[ \lambda I + W_k^{\dagger}C^{T}C \right]^{-1} W_{k}^{-1}C^{T} \right)
\end{align}

##### Applications

* Kernel Ridge Regression (KRR) / Gaussian Process Regression (GPR)
* Gaussian Process Classification (GPC)
* Support Vector Machines (SVMs)
* Kernel Logistic Regression
* Spectral Clustering
* Manifold Learning

### Nystrom Approximnation of K (samples, samples)

```python
# Nystrom approximation of K
eps = 1e-12 # lowest error approximation

# size of K
n_samples, _ = K.shape
col_components = int(np.floor(n_samples/4))
rank_components = int(np.floor(n_samples/6))

print('Size of K: {}'.format(K.shape))
print('Number of Samples: {}'.format(n_samples))
print('Number of Components: {}'.format(rank_components))

# perform sampling (uniform sampling without replacement)

indices = np.random.permutation(n_samples)

# column indices
col_indices = indices[:col_components]
C = K[:, col_indices]
W = C[col_indices, :]                          # m-by-m intersection matrix

# Perform the truncated SVD
print('\nPerform Normal SVD...\n')
U, D, V = np.linalg.svd(W, full_matrices=False)

print('Shape of U: {}'.format(U.shape))
print('Shape of S: {}'.format(D.shape))
print('Shape of V: {}'.format(V.shape))

# Perform the randomized SVD
print('\nPerform FB RSVD...\n')
# U_rand, D_rand, V_rand = fbpca.pca(W, k=rank_components, raw=True)
U_rand, D_rand, V_rand = randomized_svd(W, n_components=rank_components,
                                 random_state=123)

print('Shape of U rand: {}'.format(U_rand.shape))
print('Shape of S rand: {}'.format(D_rand.shape))
print('Shape of V rand: {}'.format(V_rand.shape))
```

    Size of K: (10000, 10000)
    Number of Samples: 10000
    Number of Components: 1666

    Perform Normal SVD...

    Shape of U: (2500, 2500)
    Shape of S: (2500,)
    Shape of V: (2500, 2500)

    Perform FB RSVD...

    Shape of U rand: (2500, 1666)
    Shape of S rand: (1666,)
    Shape of V rand: (1666, 2500)

```python
print('Time Experiment for normal SVD.')
%time _, _, _ = np.linalg.svd(W, full_matrices=True)

print('\nTime Experiment for randomized SVD (fb).')
%time _, _, _ = fbpca.pca(W, k=rank_components, raw=True, n_iter=2)
%time _, _, _ = randomized_svd(W, n_components=n_components,random_state=123)
```

    Time Experiment for normal SVD.
    CPU times: user 9.52 s, sys: 288 ms, total: 9.81 s
    Wall time: 5.39 s

    Time Experiment for randomized SVD (fb).
    CPU times: user 10.8 s, sys: 366 ms, total: 11.2 s
    Wall time: 5.67 s
    CPU times: user 4.78 s, sys: 179 ms, total: 4.95 s
    Wall time: 2.53 s

```python
# perform the approximation (Regular)
U_approx = np.sqrt(col_components / n_samples) * C @ U @ np.diag(D**(-1))
D_approx = (n_samples / col_components) * np.diag(D)

print('Size of U approximated: {}'.format(U_approx.shape))
print('Size of D approximated: {}'.format(D_approx.shape))

# perform the approximation (randomized)
U_rapprox = np.sqrt(col_components / n_samples) * C  @ U_rand
D_rapprox = (n_samples / col_components) * np.diag(D_rand**(-1))

print('Size of U rand approximated: {}'.format(U_rapprox.shape))
print('Size of D rand approximated: {}'.format(D_rapprox.shape))
```

    Size of U approximated: (10000, 2500)
    Size of D approximated: (2500, 2500)
    Size of U rand approximated: (10000, 1666)
    Size of D rand approximated: (1666, 1666)

```python
W_r = U @ np.diag(D) @ U.T
W_rand = U_rand @ np.diag(D_rand) @ U_rand.T

err = np.linalg.norm(K - U_approx @ D_approx @ U_approx.T, 'fro')
rerr = np.linalg.norm(K - U_rapprox @ D_rapprox @ U_rapprox.T, 'fro')
print('Error (Normal): {:.3e}'.format(err))
print('Error (Randomized - fb): {:.3e}'.format(rerr))

K_approx = C @ np.linalg.pinv(W_r) @ C.T
K_rapprox = C @ np.linalg.pinv(W_rand) @ C.T

err = np.linalg.norm(K - K_approx, 'fro')
rerr = np.linalg.norm(K - K_rapprox, 'fro')
print('Error (Normal): {:.3e}'.format(err))
print('Error (Randomized - fb): {:.3e}'.format(rerr))
```

    Error (Normal): 1.169e-10
    Error (Randomized - fb): 2.435e-11
    Error (Normal): 8.297e-02
    Error (Randomized - fb): 8.380e-02

```python
fig, ax = plt.subplots(nrows=1, ncols=3, figsize=(12, 9))

ax[0].imshow(K)
ax[1].imshow(K_approx)
ax[2].imshow(K_rapprox)
ax[0].set_title('Original Kernel Matrix')
ax[1].set_title('Approximated Kernel Matrix')
ax[2].set_title('Approximated (random) Kernel Matrix')

plt.show()
```

### Nystrom Approximation of X (samples, features)

```python
# Nystrom approximation of K
eps = 1e-12 # lowest error approximation

# size of K
n_samples = X.shape[0]

# chose number of components
n_components= int(np.floor(n_samples/3))
n_components = min(n_samples, n_components)    # check to make sure components are less than samples

# perform random sampling (uniform sampling without replacement)
indices = np.random.permutation(n_samples)
basis_indices = indices[:n_components]
basis = X[basis_indices]

print('Size of indices: {}'.format(indices.shape))
print('Size of basis indices: {}'.format(basis_indices.shape))
print('Size of basis: {}'.format(basis.shape))

# construct the basis kernel
basis_gamma = 1 / (2 * np.mean(pdist(basis, metric='euclidean')))
basis_kernel = rbf_kernel(basis, gamma=basis_gamma)

print('Size of basis Kernel: {}'.format(basis_kernel.shape))

# get basis vectors
U, S, V = np.linalg.svd(basis_kernel)

S = np.maximum(S, eps)

normalization = np.matmul(U / np.sqrt(S), V)

components = basis
component_indices = basis_indices

print('Size of normalization: {}'.format(normalization.shape))
print('Size of components: {}'.format(components.shape))
print('Size of component indices: {}'.format(component_indices.shape))

# transform new data
```

    Size of indices: (1500,)
    Size of basis indices: (500,)
    Size of basis: (500, 3)
    Size of basis Kernel: (500, 500)
    Size of normalization: (500, 500)
    Size of components: (500, 3)
    Size of component indices: (500,)

```python

```


# MNIST database classification
This example is based on the work of quantum neural networks used as classifiers by Farhi et al (https://arxiv.org/abs/1802.06002).

A quantum and a classical neural network are compared on how they can classify pictures that represent two possible handwritten digits (a 3 and a 6).

```python
import tensorflow as tf
import tensorflow_quantum as tfq

import cirq
import sympy
import numpy as np
import seaborn as sns
import collections

# visualization tools
%matplotlib inline
import matplotlib.pyplot as plt
from cirq.contrib.svg import SVGCircuit

```

## Load the data
We upload the MNIST database from Keras

```python
(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()

# Rescale the images from [0,255] to the [0.0,1.0] range.
x_train, x_test = x_train[..., np.newaxis]/255.0, x_test[..., np.newaxis]/255.0

print("Number of original training examples:", len(x_train))
print("Number of original test examples:", len(x_test))

```

    Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz
    11493376/11490434 [==============================] - 0s 0us/step
    Number of original training examples: 60000
    Number of original test examples: 10000

Now we can filter the pictures that only represent a 3 or a 6 and relabel them for binary classification, such that 3 corresponds to `True` ans 6 to `False`.

```python
def filter_36(x, y):
    keep = (y == 3) | (y == 6)
    x, y = x[keep], y[keep]
    y = y == 3
    return x,y
```

```python
x_train, y_train = filter_36(x_train, y_train)
x_test, y_test = filter_36(x_test, y_test)

print("Number of filtered training examples:", len(x_train))
print("Number of filtered test examples:", len(x_test))

```

    Number of filtered training examples: 12049
    Number of filtered test examples: 1968

```python
print(y_train[0])

plt.imshow(x_train[29, :, :, 0])
plt.colorbar()

```

### 1.1 Downscale the image
The images in MNIST are 28x28, which means that they're too big for a quantum device. We rescale them to 4x4

```python
x_train_small = tf.image.resize(x_train, (4,4)).numpy()
x_test_small = tf.image.resize(x_test, (4,4)).numpy()

```

```python
print(y_train[0])

plt.imshow(x_train_small[0,:,:,0], vmin=0, vmax=1)
plt.colorbar()

```

### 1.2 Removing contradictory examples
Because of the pixellation due to going from 28x28 to 4x4, some of the samples corresponding to the same unique picture are labelled sometimes as a 3 and sometimes as a 6. This is contradictory for classifiers and it is handled by the next function.

```python
def remove_contradicting(xs, ys):
    mapping = collections.defaultdict(set)
    orig_x = {}
    # Determine the set of labels for each unique image:
    for x,y in zip(xs,ys):
       orig_x[tuple(x.flatten())] = x
       mapping[tuple(x.flatten())].add(y)

    new_x = []
    new_y = []
    for flatten_x in mapping:
      x = orig_x[flatten_x]
      labels = mapping[flatten_x]
      if len(labels) == 1:
          new_x.append(x)
          new_y.append(next(iter(labels)))
      else:
          # Throw out images that match more than one label.
          pass

    num_uniq_3 = sum(1 for value in mapping.values() if len(value) == 1 and True in value)
    num_uniq_6 = sum(1 for value in mapping.values() if len(value) == 1 and False in value)
    num_uniq_both = sum(1 for value in mapping.values() if len(value) == 2)

    print("Number of unique images:", len(mapping.values()))
    print("Number of unique 3s: ", num_uniq_3)
    print("Number of unique 6s: ", num_uniq_6)
    print("Number of unique contradicting labels (both 3 and 6): ", num_uniq_both)
    print()
    print("Initial number of images: ", len(xs))
    print("Remaining non-contradicting unique images: ", len(new_x))

    return np.array(new_x), np.array(new_y)

```

```python
x_train_nocon, y_train_nocon = remove_contradicting(x_train_small, y_train)

```

    Number of unique images: 10387
    Number of unique 3s:  4912
    Number of unique 6s:  5426
    Number of unique contradicting labels (both 3 and 6):  49

    Initial number of images:  12049
    Remaining non-contradicting unique images:  10338

### 1.3 Encode the data as quantum circuits
Here we convert the remaining pictures to binary encoding. This uses a threshold, which intuitively means that the pixel is zero or one depending on its intensity being higher or lower than the threshold.

```python
THRESHOLD = 0.5

x_train_bin = np.array(x_train_nocon > THRESHOLD, dtype=np.float32)
x_test_bin = np.array(x_test_small > THRESHOLD, dtype=np.float32)

# print(x_train_bin)
```

```python
_ = remove_contradicting(x_train_bin, y_train_nocon)

```

    Number of unique images: 193
    Number of unique 3s:  80
    Number of unique 6s:  69
    Number of unique contradicting labels (both 3 and 6):  44

    Initial number of images:  10338
    Remaining non-contradicting unique images:  149

Now we can encode each of the strings (corresponding to images) with a circuit that applies a $X$ gate depending on whethere the bit intensity is higher than a given threshold.

Note: the encoding circuit must have as many qubits as there are pixels.

```python
def convert_to_circuit(image):
    """Encode truncated classical image into quantum datapoint."""
    values = np.ndarray.flatten(image)
    qubits = cirq.GridQubit.rect(4, 4)
    circuit = cirq.Circuit()
    for i, value in enumerate(values):
        if value:
            circuit.append(cirq.X(qubits[i]))
    return circuit

x_train_circ = [convert_to_circuit(x) for x in x_train_bin]
x_test_circ = [convert_to_circuit(x) for x in x_test_bin]

```

```python
SVGCircuit(x_train_circ[0])

```

How do we know that the previous circuit properly encodes the bitstring? Well, we can show the indices of the bistring-image which are non-zero.

```python
bin_img = x_train_bin[0,:,:,0]
indices = np.array(np.where(bin_img)).T
indices
```

    array([[2, 2],
           [3, 1]])

Finally, the quantum circuits can be converted to tensors for `tfq`.

```python
x_train_tfcirc = tfq.convert_to_tensor(x_train_circ)
x_test_tfcirc = tfq.convert_to_tensor(x_test_circ)

```

## Quantum neural network
### Build the circuit
This uses the idea of building a parametrized circuit of $e^{iXX}$ and $e^{iZZ}$ layers. In each layer the data qubits act on the readout qubit, of which, eventually, we want to know the expectation value.

```python
class CircuitLayerBuilder():
    def __init__(self, data_qubits, readout):
        self.data_qubits = data_qubits
        self.readout = readout

    def add_layer(self, circuit, gate, prefix):
        for i, qubit in enumerate(self.data_qubits):
            symbol = sympy.Symbol(prefix + '-' + str(i))
            # the readout is always the target
            circuit.append(gate(qubit, self.readout)**symbol)

```

Now we can build an example circuit

```python
demo_builder = CircuitLayerBuilder(data_qubits = cirq.GridQubit.rect(4,1),
                                   readout=cirq.GridQubit(-1,-1))

circuit = cirq.Circuit()
demo_builder.add_layer(circuit, gate = cirq.XX, prefix='xx')
SVGCircuit(circuit)

```

We can now make a function for our parametrized circuit. In doing this we need to decide on a number of layers (e.g. 2).

```python
def create_quantum_model():
    """Create a QNN model circuit and readout operation to go along with it."""
    data_qubits = cirq.GridQubit.rect(4, 4)  # a 4x4 grid.
    readout = cirq.GridQubit(-1, -1)         # a single qubit at [-1,-1]
    circuit = cirq.Circuit()

    # Prepare the readout qubit.
    circuit.append(cirq.X(readout))
    circuit.append(cirq.H(readout))

    builder = CircuitLayerBuilder(
        data_qubits = data_qubits,
        readout=readout)

    # Then add layers (experiment by adding more).
    builder.add_layer(circuit, cirq.XX, "xx1")
    builder.add_layer(circuit, cirq.ZZ, "zz1")

    # Finally, prepare the readout qubit.
    circuit.append(cirq.H(readout))

    return circuit, cirq.Z(readout)

```

```python
model_circuit, model_readout = create_quantum_model()
# SVGCircuit(model_circuit)
```

### Wrap the model-circuit in a tfq-keras model

The circuit model we created can be wrapped into a tfw-keras one, which will allow to define a cost and an accuracy and to train the parameters.

```python
# Build the keras model
model = tf.keras.Sequential([
    # The input layer is the data-circuit, encoded as a tf.string
    tf.keras.layers.Input(shape=(), dtype=tf.string),
    # The PQC layer returns the expected value of the readout gate, range [-1, 1].
    tfq.layers.PQC(model_circuit, model_readout)
])
```

Since we want to minimize the expectation value of the readout (range = \[-1, 1\]), we can use the *hinge loss* as a cost function. For this we must convert the labels of the images we still have

```python
y_train_hinge = 2. * y_train_nocon - 1.
y_test_hinge = 2. * y_test - 1.
```

Now for the accuracy metric. We want to use the `hinge_accuracy` as defined below

```python
def hinge_accuracy(y_true, y_pred):
    y_true = tf.squeeze(y_true) > 0.
    y_pred = tf.squeeze(y_pred) > 0.
    result = tf.cast(y_true == y_pred, tf.float32)

    return tf.reduce_mean(result)
```

The following line creates the quantum machine learning model (loss + optimizer + metrics).

```python
model.compile(
    loss=tf.keras.losses.Hinge(),
    optimizer=tf.keras.optimizers.Adam(),
    metrics=[hinge_accuracy]
)
```

```python
print(model.summary())
```

    Model: "sequential"
    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    pqc (PQC)                    (None, 1)                 32
    =================================================================
    Total params: 32
    Trainable params: 32
    Non-trainable params: 0
    _________________________________________________________________
    None

Now the model can be trained. For the sake of brevity, we reduce the number of training examples (32 parameters can be trained well enough with 500 samples) and define the number of epochs and the batch size.

```python
EPOCHS = 3
BATCH_SIZE = 32

NUM_EXAMPLES = 500
```

```python
x_train_tfcirc_sub = x_train_tfcirc[:NUM_EXAMPLES]
y_train_hinge_sub = y_train_hinge[:NUM_EXAMPLES]
```

```python
qnn_history = model.fit(
    x_train_tfcirc_sub, y_train_hinge_sub,
    batch_size=BATCH_SIZE,
    epochs=EPOCHS,
    verbose=1,
    validation_data=(x_test_tfcirc, y_test)
)

qnn_results = model.evaluate(x_test_tfcirc, y_test)
```

    2021-07-12 15:46:28.953821: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
    2021-07-12 15:46:28.978068: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2899885000 Hz
    Epoch 1/3
    157/157 [==============================] - 88s 555ms/step - loss: 0.9808 - hinge_accuracy: 0.5614 - val_loss: 0.7020 - val_hinge_accuracy: 0.7979
    Epoch 2/3
    157/157 [==============================] - 87s 554ms/step - loss: 0.5951 - hinge_accuracy: 0.8125 - val_loss: 0.4304 - val_hinge_accuracy: 0.8029
    Epoch 3/3
    157/157 [==============================] - 87s 555ms/step - loss: 0.3838 - hinge_accuracy: 0.8550 - val_loss: 0.3615 - val_hinge_accuracy: 0.8992
    62/62 [==============================] - 4s 69ms/step - loss: 0.3615 - hinge_accuracy: 0.8992

## Classical Neural Network

In the following, we will see how a classical neural network can outperform a quantum one for classical data classification. In the first case, we won't reduce the dimensionality of the data and keep the images as 28 x 28. In the second case, we will use the 4 x 4 reduced images with just 37 parameter, which is a model more similar to the quantum one.

```python
def create_classical_model():
    # A simple model based off LeNet from https://keras.io/examples/mnist_cnn/
    model = tf.keras.Sequential()
    model.add(tf.keras.layers.Conv2D(32, [3, 3], activation='relu', input_shape=(28,28,1)))
    model.add(tf.keras.layers.Conv2D(64, [3, 3], activation='relu'))
    model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
    model.add(tf.keras.layers.Dropout(0.25))
    model.add(tf.keras.layers.Flatten())
    model.add(tf.keras.layers.Dense(128, activation='relu'))
    model.add(tf.keras.layers.Dropout(0.5))
    model.add(tf.keras.layers.Dense(1))
    return model

model = create_classical_model()
model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
              optimizer=tf.keras.optimizers.Adam(),
              metrics=['accuracy'])

model.summary()
```

    Model: "sequential_1"
    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    conv2d (Conv2D)              (None, 26, 26, 32)        320
    _________________________________________________________________
    conv2d_1 (Conv2D)            (None, 24, 24, 64)        18496
    _________________________________________________________________
    max_pooling2d (MaxPooling2D) (None, 12, 12, 64)        0
    _________________________________________________________________
    dropout (Dropout)            (None, 12, 12, 64)        0
    _________________________________________________________________
    flatten (Flatten)            (None, 9216)              0
    _________________________________________________________________
    dense (Dense)                (None, 128)               1179776
    _________________________________________________________________
    dropout_1 (Dropout)          (None, 128)               0
    _________________________________________________________________
    dense_1 (Dense)              (None, 1)                 129
    =================================================================
    Total params: 1,198,721
    Trainable params: 1,198,721
    Non-trainable params: 0
    _________________________________________________________________

```python
model.fit(x_train,
          y_train,
          batch_size=128,
          epochs=1,
          verbose=1,
          validation_data=(x_test, y_test))

cnn_results = model.evaluate(x_test, y_test)
```

    95/95 [==============================] - 7s 74ms/step - loss: 0.1106 - accuracy: 0.9507 - val_loss: 0.0037 - val_accuracy: 0.9985
    62/62 [==============================] - 0s 4ms/step - loss: 0.0037 - accuracy: 0.9985

Now for the 'fair' model

```python
def create_fair_classical_model():
    # A simple model based off LeNet from https://keras.io/examples/mnist_cnn/
    model = tf.keras.Sequential()
    model.add(tf.keras.layers.Flatten(input_shape=(4,4,1)))
    model.add(tf.keras.layers.Dense(2, activation='relu'))
    model.add(tf.keras.layers.Dense(1))
    return model

model = create_fair_classical_model()
model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
              optimizer=tf.keras.optimizers.Adam(),
              metrics=['accuracy'])

model.summary()
```

    Model: "sequential_2"
    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    flatten_1 (Flatten)          (None, 16)                0
    _________________________________________________________________
    dense_2 (Dense)              (None, 2)                 34
    _________________________________________________________________
    dense_3 (Dense)              (None, 1)                 3
    =================================================================
    Total params: 37
    Trainable params: 37
    Non-trainable params: 0
    _________________________________________________________________

```python
model.fit(x_train_bin,
          y_train_nocon,
          batch_size=128,
          epochs=20,
          verbose=2,
          validation_data=(x_test_bin, y_test))

fair_nn_results = model.evaluate(x_test_bin, y_test)
```

    Epoch 1/20
    81/81 - 0s - loss: 0.6750 - accuracy: 0.5584 - val_loss: 0.6577 - val_accuracy: 0.5152
    Epoch 2/20
    81/81 - 0s - loss: 0.6278 - accuracy: 0.6163 - val_loss: 0.6108 - val_accuracy: 0.6509
    Epoch 3/20
    81/81 - 0s - loss: 0.5823 - accuracy: 0.7453 - val_loss: 0.5623 - val_accuracy: 0.7932
    Epoch 4/20
    81/81 - 0s - loss: 0.5397 - accuracy: 0.8167 - val_loss: 0.5210 - val_accuracy: 0.8028
    Epoch 5/20
    81/81 - 0s - loss: 0.5032 - accuracy: 0.8358 - val_loss: 0.4859 - val_accuracy: 0.8186
    Epoch 6/20
    81/81 - 0s - loss: 0.4714 - accuracy: 0.8522 - val_loss: 0.4556 - val_accuracy: 0.8323
    Epoch 7/20
    81/81 - 0s - loss: 0.4441 - accuracy: 0.8565 - val_loss: 0.4298 - val_accuracy: 0.8328
    Epoch 8/20
    81/81 - 0s - loss: 0.4205 - accuracy: 0.8573 - val_loss: 0.4076 - val_accuracy: 0.8328
    Epoch 9/20
    81/81 - 0s - loss: 0.4001 - accuracy: 0.8572 - val_loss: 0.3883 - val_accuracy: 0.8323
    Epoch 10/20
    81/81 - 0s - loss: 0.3823 - accuracy: 0.8576 - val_loss: 0.3714 - val_accuracy: 0.8323
    Epoch 11/20
    81/81 - 0s - loss: 0.3667 - accuracy: 0.8728 - val_loss: 0.3569 - val_accuracy: 0.8709
    Epoch 12/20
    81/81 - 0s - loss: 0.3531 - accuracy: 0.8802 - val_loss: 0.3443 - val_accuracy: 0.8709
    Epoch 13/20
    81/81 - 0s - loss: 0.3411 - accuracy: 0.8802 - val_loss: 0.3331 - val_accuracy: 0.8709
    Epoch 14/20
    81/81 - 0s - loss: 0.3305 - accuracy: 0.8804 - val_loss: 0.3232 - val_accuracy: 0.8725
    Epoch 15/20
    81/81 - 0s - loss: 0.3211 - accuracy: 0.8813 - val_loss: 0.3145 - val_accuracy: 0.8725
    Epoch 16/20
    81/81 - 0s - loss: 0.3127 - accuracy: 0.8813 - val_loss: 0.3068 - val_accuracy: 0.8725
    Epoch 17/20
    81/81 - 0s - loss: 0.3051 - accuracy: 0.8813 - val_loss: 0.2997 - val_accuracy: 0.8725
    Epoch 18/20
    81/81 - 0s - loss: 0.2983 - accuracy: 0.8815 - val_loss: 0.2936 - val_accuracy: 0.8725
    Epoch 19/20
    81/81 - 0s - loss: 0.2922 - accuracy: 0.8816 - val_loss: 0.2881 - val_accuracy: 0.8730
    Epoch 20/20
    81/81 - 0s - loss: 0.2866 - accuracy: 0.8819 - val_loss: 0.2830 - val_accuracy: 0.8730
    62/62 [==============================] - 0s 465us/step - loss: 0.2830 - accuracy: 0.8730

## Comparison

```python
qnn_accuracy = qnn_results[1]
cnn_accuracy = cnn_results[1]
fair_nn_accuracy = fair_nn_results[1]

sns.barplot(["Quantum", "Classical, full", "Classical, fair"],
            [qnn_accuracy, cnn_accuracy, fair_nn_accuracy])
```


```python
from resources.workspace import *
```

$
% START OF MACRO DEF
% DO NOT EDIT IN INDIVIDUAL NOTEBOOKS, BUT IN macros.py
%
\newcommand{\Reals}{\mathbb{R}}
\newcommand{\Expect}[0]{\mathbb{E}}
\newcommand{\NormDist}{\mathcal{N}}
%
\newcommand{\DynMod}[0]{\mathscr{M}}
\newcommand{\ObsMod}[0]{\mathscr{H}}
%
\newcommand{\mat}[1]{{\mathbf{{#1}}}}
%\newcommand{\mat}[1]{{\pmb{\mathsf{#1}}}}
\newcommand{\bvec}[1]{{\mathbf{#1}}}
%
\newcommand{\trsign}{{\mathsf{T}}}
\newcommand{\tr}{^{\trsign}}
\newcommand{\tn}[1]{#1}
\newcommand{\ceq}[0]{\mathrel{≔}}
%
\newcommand{\I}[0]{\mat{I}}
\newcommand{\K}[0]{\mat{K}}
\newcommand{\bP}[0]{\mat{P}}
\newcommand{\bH}[0]{\mat{H}}
\newcommand{\bF}[0]{\mat{F}}
\newcommand{\R}[0]{\mat{R}}
\newcommand{\Q}[0]{\mat{Q}}
\newcommand{\B}[0]{\mat{B}}
\newcommand{\C}[0]{\mat{C}}
\newcommand{\Ri}[0]{\R^{-1}}
\newcommand{\Bi}[0]{\B^{-1}}
\newcommand{\X}[0]{\mat{X}}
\newcommand{\A}[0]{\mat{A}}
\newcommand{\Y}[0]{\mat{Y}}
\newcommand{\E}[0]{\mat{E}}
\newcommand{\U}[0]{\mat{U}}
\newcommand{\V}[0]{\mat{V}}
%
\newcommand{\x}[0]{\bvec{x}}
\newcommand{\y}[0]{\bvec{y}}
\newcommand{\z}[0]{\bvec{z}}
\newcommand{\q}[0]{\bvec{q}}
\newcommand{\br}[0]{\bvec{r}}
\newcommand{\bb}[0]{\bvec{b}}
%
\newcommand{\bx}[0]{\bvec{\bar{x}}}
\newcommand{\by}[0]{\bvec{\bar{y}}}
\newcommand{\barB}[0]{\mat{\bar{B}}}
\newcommand{\barP}[0]{\mat{\bar{P}}}
\newcommand{\barC}[0]{\mat{\bar{C}}}
\newcommand{\barK}[0]{\mat{\bar{K}}}
%
\newcommand{\D}[0]{\mat{D}}
\newcommand{\Dobs}[0]{\mat{D}_{\text{obs}}}
\newcommand{\Dmod}[0]{\mat{D}_{\text{obs}}}
%
\newcommand{\ones}[0]{\bvec{1}}
\newcommand{\AN}[0]{\big( \I_N - \ones \ones\tr / N \big)}
%
% END OF MACRO DEF
$
# The Gaussian (i.e. Normal) distribution

Consider the Gaussian random variable $x \sim \mathcal{N}(b,B)$;
 - the first parameter, $b$, indicates its mean;
 - the second parameter, $B>0$, indicates its variance.

Sometimes it is more convenient to write
$$\begin{align}
p(x) = \mathcal{N}(x \mid b,B) \, ,
\end{align}$$
for its probability density function (**pdf**), which is given by
$$\begin{align}
\mathcal{N}(x \mid b,B) = (2 \pi B)^{-1/2} e^{-(x-b)^2/2 B} \, , \tag{G1}
\end{align}$$
for $x \in (-\infty, +\infty)$.

**Exc 2.2:** Code it up (complete the code below)! Hints:
* Note that `**` is the power operator in Python.
* As in Matlab, $e^x$ is available as `exp(x)`

```python
def pdf_G1(x,b,B):
    "Univariate (scalar), Gaussian pdf"
    # pdf_values = ### INSERT ANSWER HERE ###
    return pdf_values
```

```python
#show_answer('pdf_G1')
```

Let's plot the pdf.

```python
# Density parameters
b = 0  # mean     of distribution
B = 25 # variance of distribution

# Grid computations
N  = 201                # num of grid points
xx = linspace(-20,20,N) # grid
dx = xx[1]-xx[0]        # grid spacing
pp = pdf_G1(xx,b,B)     # pdf values

# Plot
plt.figure(figsize=(6,2))
plt.plot(xx,pp);
```

This could for example be the pdf of a stochastic noise variable. It could also describe our *quantitative belief* and uncertainty about a parameter (or state), which we model as randomness in Bayesian statistics.

**Exc 2.4:** Play around with `b` and `B` (and re-run the above code) and answer these questions by looking at the resulting figure:
 * How does the pdf curve change when `b` changes?
 * How does the pdf curve change when you increase `B`?
 * In a few words, describe the shape of the Gaussian pdf curve. Does this ring a bell?

NB: Restore `B=25` and re-run the above cell (this is a convienient value for the below examples)

**Exc 2.6:** Recall $p(x) = \mathcal{N}(x \mid b,B)$ from eqn (G1).
Use pen, paper, and calculus to answer the following questions,
which derive some helpful mnemonics about the distribution.

 * (i) Find $x$ such that $p(x) = 0$.
 * (ii) Where is the location of the mode (maximum) of the distribution?
I.e. find $x$ such that $\frac{d p}{d x}(x) = 0$.
Hint: it's easier to analyse $\log p(x)$ rather than $p(x)$ itself.
 * (iii) Where is the inflection point? I.e. where $\frac{d^2 p}{d x^2}(x) = 0$.
 * (iv) Show that $\int_{-\infty}^{\infty} e^{-(x-b)^2/2 B} \, d x  =  (2 \pi B)^{1/2}$.
 * (v) Some forms of "sensitivy analysis" (a basic form of uncertainty quantification) consist in evaluating $\frac{d^2 p}{d x^2}(x)$ at the mode.
Explain this by reference to the Gaussian shape.
Hint: calculate and interpret $\frac{d^2 p}{d x^2}(b)$

### The multivariate (i.e. vector) case
Here's the pdf of the *multivariate* Gaussian:
$$\begin{align}
\NormDist(x \mid  b,B)
&=
|2 \pi B|^{-1/2} \, \exp\Big(-\frac{1}{2}\|x-b\|^2_B\Big) \, , \tag{GM}
\end{align}$$
where $|.|$ represents the determinant, and $\|.\|_W$ represents the norm with weighting: $\|x\|^2_W = x^T W^{-1} x$.
In this multivariate case, $B$ is called the *covariance* (matrix).

The following implements this pdf. Take a moment to digest the code. Don't worry if you don't understand all of the details. Hints:
 * `@` produces matrix multiplication (`*` in `Matlab`);
 * `*` produces array multiplication (`.*` in `Matlab`);
 * `axis=-1` makes `np.sum()` work along the last dimension of an ND-array.

```python
from numpy.linalg import det, inv

def weighted_norm22(xx,W):
    "Computes the norm of each vector (on the last axis) of xx, weighted by W."
    ww = np.sum( (xx @ inv(W)) * xx, axis=-1)
    return ww

def pdf_GM(xx,b,B):
    "pdf -- Gaussian, Multivariate: N(x|b,B)"
    return 1/sqrt(det(2*pi*B)) * exp(-0.5*weighted_norm22(xx-b,B))
```

The following code plots the pdf as contour (iso-density) curves.

```python
# Grid
XX,YY = np.meshgrid(xx,xx)
grid  = np.dstack((XX,YY))

# Covariance specification
corr = 0.7
var1 = 1
var2 = 1
cov12 = sqrt(var1*var2)*corr
Cov = B * array([[var1  , cov12],
                 [cov12 , var2]])
# Eval
pp = pdf_GM(grid, b=0, B=Cov)

# Plot
plt.figure(figsize=(4,4))
plt.contour(XX,YY,pp)
plt.axis('equal');
```

**Exc 2.7:** How do the contours look? Try to understand why. Cases:
 * (a) correlation=0.
 * (b) correlation=0.99.
 * (c) correlation=0.5. (Note that we've used `plt.axis('equal')`).
 * (d) correlation=0.5, but with non-equal variances.

**Exc 2.8:** Play the [correlation game](http://guessthecorrelation.com/) (doesn't work right in Chrome) until you get a score (shown as gold coins) of 5 or more.

**Exc 2.9:**
* What's the difference between correlation and covariance?
* What's the difference between correlation (or covariance) and dependence?
* Does correlation imply causation?
* Can you use correlation to in making predictions? Sunglasses

# Bayes' rule
Bayes' rule is how we do inference.
It defines how we should merge our prior (quantitative belief) about $x$,
when given an observation $y$ somehow related to $x$.

For continuous random variables, $x$ and $y$, it reads:

$$\begin{align}
p(x|y) = \frac{p(x) \, p(y|x)}{p(y)} \, , \tag{2}
\end{align}$$

or, in words:

$$
\text{"posterior" (pdf of $x$ given $y$)}
\; = \;
\frac{\text{"prior" (pdf of $x$)}
\; \times \;
\text{"likelihood" (pdf of $y$ given $x$)}}
{\text{"normalization" (pdf of $y$)}} \, .
$$

```python
#show_example('BR')
```

**Exc 2.10:** Derive Bayes' rule from the definition of [conditional pdf's](https://en.wikipedia.org/wiki/Conditional_probability#Kolmogorov_definition).

```python
#show_answer('BR derivation')
```

Exercises marked with an asterisk (*) are optional.

**Exc 2.11*:** Slightly after reverend T. Bayes, P. S. Laplace also independently (and more clearly) developed Bayes' rule, published in 1774. Some time therafter, what we now call "statistical inference" came to be known as the reasoning of "inverse probability". Nowadays, "inverse problems" are often given a statistical interpretation. Considering this context, why do you think we use $x$ for the "unknown", and $y$ for the known/given/fixed data?

```python
#show_answer('inverse')
```

Computers generally work with discrete, numerical representations of mathematical entities.
Numerically, pdfs may be represented by their `values` on a grid, such as `xx` from above. Bayes' rule (2) then consists of *grid-point-wise* multiplication, as shown below.

```python
def Bayes_rule(prior_values,lklhd_values,dx):
    "Numerical (pointwise) implementation of Bayes' rule."
    pp = prior_values * lklhd_values   # pointwise multiplication
    posterior_values = pp/(sum(pp)*dx) # normalization
    return posterior_values
```

The code below shows Bayes' rule in action.
Again, remember that the only thing it's doing is multiplying the `prior value` and `likelihood value` at each gridpoint.
Move the sliders with the arrow keys to animate it.

```python
# Fix the prior's parameters
b = 0 # mean
B = 1 # variance

@interact(y=(-10,10,1),R=(0.01,20,0.2))
def animate_Bayes(y=4.0,R=1):
    prior_vals = pdf_G1(xx,b,B)
    lklhd_vals = pdf_G1(y,xx,R)

    postr_vals = Bayes_rule(prior_vals, lklhd_vals, xx[1]-xx[0])

    plt.figure(figsize=(10,4))
    plt.plot(xx,prior_vals, label='prior $\mathcal{N}(x|b,B)$')
    plt.plot(xx,lklhd_vals, label='likelihood $\mathcal{N}(y|x,R)$')
    plt.plot(xx,postr_vals, label='posterior - pointwise')

    ### Uncomment this block AFTER doing the Exc 2.24 ###
    # xhat, P = Bayes_rule_G1(b,B,y,R)
    # postr_vals2 = pdf_G1(xx,xhat,P)
    # plt.plot(xx,postr_vals2,'--',label='posterior - parametric\n $\mathcal{N}(x|\hat{x},P)$')

    plt.ylim(ymax=0.6)
    plt.legend()
    plt.show()
```

**Exc 2.12:** This excercise serves to make you acquainted with how Bayes' rule blends information.
Move the sliders to see what happens, and answer the following:
 * What happens to the posterior when $R \rightarrow \infty$ ?
 * What happens to the posterior when $R \rightarrow 0$ ?
 * Move around $y$. What is the posterior's location (mean/mode) when $R = B$ ?
 * Does the posterior scale (width) depend on $y$?
 * Consider the shape (ignoring location & scale) of the posterior. Does it depend on $R$ or $y$?
 * Can you see a shortcut to computing this posterior rather than having to do the pointwise multiplication?

```python
#show_answer('Posterior behaviour')
```

**Exc 2.14:** Show that the normalization in `Bayes_rule()` amounts to the same as dividing by $p(y)$.

```python
#show_answer('BR normalization')
```

In fact, since $p(y)$ is implicitly known (c.f. Exc 2.14),
we often don't bother to write it down, simplifying Bayes' rule (2) to
$$\begin{align}
p(x|y) \propto p(x) \, p(y|x) \, .  \tag{3}
\end{align}$$

**Exc 2.15*:**
* (a) Implement a "uniform" (or "flat" or "box") distribution pdf and call it `pdf_U1(x,b,B)`. These formulae for its mean/variance will be useful.
In the above animations, replace `pdf_G1` with your new `pdf_U1` (both for the prior and likelihood). Ensure that everything is working correctly.

```python
#show_answer('pdf_U1')
```

* (b)
 - Why (in the figure) are the walls of the pdf (ever so slightly) inclined?
 - What happens when you move the prior and likelihood too far apart? Is the fault of the implementation, the math, or the problem statement?

```python
#show_answer('BR U1')
```

* (c)*:
 - Re-do Exc 2.12, now with `pdf_U1`.
* (d)*:
 - Now test a Gaussian prior with a uniform likelihood.

NB: At the end of this excercise, restore `pdf_G1` (both the prior and likelihood) in the above animation (for later use).

### Gaussian-Gaussian Bayes

The above animation shows Bayes' rule in 1 dimension. Previously, we saw how a Gaussian looks in 2 dimensions. Can you imagine how Bayes' rule looks in 2 dimensions? In higher dimensions, these things get difficult to imagine, let alone visualize.

Similarly, the size of the calculations required for Bayes' rule poses a difficulty. Indeed, the following exercise shows that (pointwise) multiplication for all grid points becomes preposterious in high dimensions.

**Exc 2.16:**
 * (a) How many point-multiplications are needed on a grid with $N$ points in $M$ dimensions? (Imagine an $M$-dimensional cube where each side has a grid with $N$ points on it)
 * (b) Suppose we model 15 physical quanitites, on each grid point, on a discretized surface model of Earth. Assume the resolution is $1^\circ$ for latitude (110km), $1^\circ$ for longitude. How many variables are there in total? This is the dimensionality ($M$) of the problem.
 * (c) Suppose each variable is has a pdf represented with a grid using only $N=10$ points. How many multiplications are necessary to calculate Bayes rule (jointly) for all variables on our Earth model?

```python
#show_answer('Dimensionality a')
#show_answer('Dimensionality b')
#show_answer('Dimensionality c')
```

In response to this computational difficulty, we try to be smart and do something more analytical ("pen-and-paper"): we only compute the parameters (mean and (co)variance) of the posterior pdf.

This is doable and quite simple in the Gaussian-Gaussian case:
With a prior $p(x) = \mathcal{N}(x \mid b,B)$ and a likelihood $p(y|x) = \mathcal{N}(y \mid x,R)$,
the posterior is
$$\begin{align}
p(x|y)
&= \mathcal{N}(x \mid \hat{x},P) \tag{4} \, ,
\end{align}$$
where, in the univarite (1-dimensional) case:
$$\begin{align}
    P &= 1/(1/B + 1/R) \, , \tag{5} \\\
  \hat{x} &= P(b/B + y/R) \, .  \tag{6}
\end{align}$$

The multivariate case is discussed in a later tutorial; for now, try to tackle exc 2.18.

#### Exc  2.18 'Gaussian Bayes':
Derive the above expressions for $P$ and $\hat{x}$
from Bayes' rule (3) and the expression for a Gaussian pdf (G1).

```python
#show_answer('BR Gauss')
```

**Exc 2.20:** Do some light algebra to show that eqns (5) and (6) can be rewritten as
$$\begin{align}
    P &= (1-K)B \, ,  \tag{8} \\\
  \hat{x} &= b + K (y-b) \tag{9} \, ,
\end{align}$$
where $K = B/(B+R)$, which is called the "Kalman gain". This form of the update becomes highly interesting in the multivariate case (later).

**Exc 2.22*:** Consider the formula for $K$ and its role in the previous couple of equations. Why do you think $K$ is called a "gain"?

```python
#show_answer('KG intuition')
```

**Exc 2.24:** Implement a Gaussian-Gaussian Bayes' rule (eqns 5 and 6, or eqns 8 and 9) by completing the code below.

```python
def Bayes_rule_G1(b,B,y,R):
    ### INSERT ANSWER HERE ###
    return xhat,P
```

```python
#show_answer('BR Gauss code')
```

**Exc 2.26:** Go back to the above animation code, and uncomment the block that uses `Bayes_rule_G1()`. Re-run.
Make sure its curve coincides with that which uses pointwise multiplication (i.e. `Bayes_rule()`).
This is the main secret of the "Kalman filter".

**Exc 2.28:** Questions related to the above animation:
 * Does the width (i.e. scale) for the posterior depend on the location $y$ of the likelihood?
 * Note that the width (i.e. scale) for the posterior always smaller than that of prior and likelihood.
   * What does this mean [information-wise](https://en.wikipedia.org/wiki/Differential_entropy#Differential_entropies_for_various_distributions)?
   * Is this ordering also always true for non-Gaussian distributions?
 * Imagine that you're pretty sure about something, but then you get a wildly different indication (observation).
   What is your posterior uncertainty? Has it decreased?
   So, are you a Gaussian thinker?

```python
#show_answer('Posterior cov')
```

**Exc 2.30*:** Why are we so fond of the Gaussian assumption?

```python
#show_answer('Why Gaussian')
```

### Next: [Univariate (scalar) Kalman filtering](T3%20-%20Univariate%20Kalman%20filtering.ipynb)


###### Content under Creative Commons Attribution license CC-BY 4.0, code under MIT license (c) 2019 Daniel Koehn, based on (c)2018 L.A. Barba, G.F. Forsyth [CFD Python](https://github.com/barbagroup/CFDPython#cfd-python), (c)2014 L.A. Barba, I. Hawke, B. Knaepen [Practical Numerical Methods with Python](https://github.com/numerical-mooc/numerical-mooc#practical-numerical-methods-with-python), also under CC-BY.

```python
from IPython.core.display import HTML
css_file = '../style/custom.css'
HTML(open(css_file, 'r').read())
```

<link href="https://fonts.googleapis.com/css?family=Merriweather:300,300i,400,400i,700,700i,900,900i" rel='stylesheet' >
<link href="https://fonts.googleapis.com/css?family=Source+Sans+Pro:300,300i,400,400i,700,700i" rel='stylesheet' >
<link href='http://fonts.googleapis.com/css?family=Source+Code+Pro:300,400' rel='stylesheet' >
<style>

@font-face {
    font-family: "Computer Modern";
    src: url('http://mirrors.ctan.org/fonts/cm-unicode/fonts/otf/cmunss.otf');
}

#notebook_panel { /* main background */
    background: rgb(245,245,245);
}

div.cell { /* set cell width */
    width: 800px;
}

div #notebook { /* centre the content */
    background: #fff; /* white background for content */
    width: 1000px;
    margin: auto;
    padding-left: 0em;
}

#notebook li { /* More space between bullet points */
margin-top:0.5em;
}

/* draw border around running cells */
div.cell.border-box-sizing.code_cell.running {
    border: 1px solid #111;
}

/* Put a solid color box around each cell and its output, visually linking them*/
div.cell.code_cell {
    background-color: rgb(256,256,256);
    border-radius: 0px;
    padding: 0.5em;
    margin-left:1em;
    margin-top: 1em;
}

div.text_cell_render{
    font-family: 'Source Sans Pro', sans-serif;
    line-height: 140%;
    font-size: 110%;
    width:680px;
    margin-left:auto;
    margin-right:auto;
}

/* Formatting for header cells */
.text_cell_render h1 {
    font-family: 'Merriweather', serif;
    font-style:regular;
    font-weight: bold;
    font-size: 250%;
    line-height: 100%;
    color: #004065;
    margin-bottom: 1em;
    margin-top: 0.5em;
    display: block;
}
.text_cell_render h2 {
    font-family: 'Merriweather', serif;
    font-weight: bold;
    font-size: 180%;
    line-height: 100%;
    color: #0096d6;
    margin-bottom: 0.5em;
    margin-top: 0.5em;
    display: block;
}

.text_cell_render h3 {
    font-family: 'Merriweather', serif;
	font-size: 150%;
    margin-top:12px;
    margin-bottom: 3px;
    font-style: regular;
    color: #008367;
}

.text_cell_render h4 {    /*Use this for captions*/
    font-family: 'Merriweather', serif;
    font-weight: 300;
    font-size: 100%;
    line-height: 120%;
    text-align: left;
    width:500px;
    margin-top: 1em;
    margin-bottom: 2em;
    margin-left: 80pt;
    font-style: regular;
}

.text_cell_render h5 {  /*Use this for small titles*/
    font-family: 'Source Sans Pro', sans-serif;
    font-weight: regular;
    font-size: 130%;
    color: #e31937;
    font-style: italic;
    margin-bottom: .5em;
    margin-top: 1em;
    display: block;
}

.text_cell_render h6 { /*use this for copyright note*/
    font-family: 'Source Code Pro', sans-serif;
    font-weight: 300;
    font-size: 9pt;
    line-height: 100%;
    color: grey;
    margin-bottom: 1px;
    margin-top: 1px;
}

    .CodeMirror{
            font-family: "Source Code Pro";
			font-size: 90%;
    }
/*    .prompt{
        display: None;
    }*/

    .warning{
        color: rgb( 240, 20, 20 )
        }
</style>

# Iterative Methods to Solve 2D Boundary Value Problems

In [Lesson 1](http://nbviewer.ipython.org/urls/github.com/daniel-koehn/Differential-equations-earth-system/tree/master/07_BVP_2D/01_2D_Laplace_Equation.ipynb) and [Lesson 2](http://nbviewer.ipython.org/urls/github.com/daniel-koehn/Differential-equations-earth-system/tree/master/07_BVP_2D/02_2D_Poisson_Equation.ipynb) of this module we used the Jacobi method (a relaxation scheme) to iteratively find solutions to Laplace and Poisson equations.

And it worked, so why are we still talking about it?  Because the Jacobi method is slow, very slow to converge.  It might not have seemed that way in the first two notebooks because we were using small grids, but we did need more than 3,000 iterations to reach the exit criterion while solving the Poisson equation on a $41\times 41$ grid.

You can confirm this below: using `nx,ny=` $128$ on the Laplace problem of Lesson 1, the Jacobi method requires nearly *20,000* iterations before we reach $10^{-8}$ for the L2-norm of the difference between two iterates. That's a *lot* of iterations!

Now, consider this application:  an incompressible Navier-Stokes solver has to ensure that the velocity field is divergence-free at every time step.  One of the most common ways to ensure this is to solve a Poisson equation for the pressure field. In fact, the pressure Poisson equation is responsible for the majority of the computational expense of an incompressible Navier-Stokes solver. Imagine having to do 20,000 Jacobi iterations for *every* time step in a fluid-flow problem with many thousands or perhaps millions of grid points!

The Jacobi method is the slowest of all relaxation schemes, so let's learn how to improve on it. In this lesson, we'll study the Gauss-Seidel method—twice as fast as Jacobi, in theory—and the successive over-relaxation (SOR) method. We also have some neat Python tricks lined up for you to get to the solution even faster. Let's go!

### Test problem

Let's use the same example problem as in [Lesson 1](http://nbviewer.ipython.org/urls/github.com/daniel-koehn/Differential-equations-earth-system/tree/master/07_BVP_2D/01_2D_Laplace_Equation.ipynb): Laplace's equation with boundary conditions

$$
\begin{equation}
  \begin{gathered}
p=0 \text{ at } x=0\\
\frac{\partial p}{\partial x} = 0 \text{ at } x = L_x\\
p = 0 \text{ at }y = 0 \\
p = \sin \left(  \frac{\frac{3}{2}\pi x}{L_x} \right) \text{ at } y = L_y
  \end{gathered}\notag
\end{equation}
$$

We import our favorite Python libraries, and also some custom functions that we wrote in [Lesson 1](http://nbviewer.ipython.org/urls/github.com/daniel-koehn/Differential-equations-earth-system/tree/master/07_BVP_2D/01_2D_Laplace_Equation.ipynb), which we have saved in a 'helper' Python file for re-use.

```python
import numpy
from matplotlib import pyplot
%matplotlib inline

# import JIT from Numba
from numba import jit
```

```python
# Set the font family and size to use for Matplotlib figures.
pyplot.rcParams['font.family'] = 'serif'
pyplot.rcParams['font.size'] = 16
```

```python
from helper import laplace_solution, l2_norm, plot_3d
```

We now have the functions `laplace_solution`, `l2_norm`, and `plot_3d` in our namespace. If you can't remember how they work, just use the Python built-in function `help()` and take advantage of the docstrings.  It's a good habit to always write docstrings in your functions, and now you see why!

In this notebook, we are going to use larger grids than before, to better illustrate the speed increases we achieve with different iterative methods. Let's create a $128\times128$ grid and initialize.

```python
# Set parameters.
nx = 128  # number of points in the x direction
ny = 128  # number of points in the y direction
Lx = 5.0  # domain length in the x direction
Ly = 5.0  # domain length in the y direction
dx = Lx / (nx - 1)  # grid spacing in x direction
dy = Ly / (ny - 1)  # grid spacing in y direction

# Create the gridline locations.
x = numpy.linspace(0.0, Lx, num=nx)
y = numpy.linspace(0.0, Ly, num=ny)

# Set the initial conditions.
p0 = numpy.zeros((ny, nx))
p0[-1, :] = numpy.sin(1.5 * numpy.pi * x / Lx)
```

We said above that the Jacobi method takes nearly $20,000$ iterations before it satisfies our exit criterion of $10^{-8}$ (L2-norm difference between two consecutive iterations). You'll just have to confirm that now. Have a seat!

```python
@jit(nopython=True) # use Just-In-Time (JIT) Compilation for C-performance
def laplace_jac(p,pn,nx,ny):

    for i in range(1,nx-1):
        for j in range(1,ny-1):

            # Laplace equation
            p[j, i] = 0.25 * (pn[j+1, i] + pn[j, i+1] +
                              pn[j-1, i] + pn[j, i-1])
    return p
```

```python
def laplace_2d_jacobi(p0, maxiter=20000, rtol=1e-6):
    """
    Solves the 2D Laplace equation on a uniform grid
    with equal grid spacing in both directions
    using Jacobi relaxation method.

    The exit criterion of the solver is based on the relative L2-norm
    of the solution difference between two consecutive iterations.

    Parameters
    ----------
    p0 : numpy.ndarray
        The initial solution as a 2D array of floats.
    maxiter : integer, optional
        Maximum number of iterations to perform;
        default: 20000.
    rtol : float, optional
        Relative tolerance for convergence;
        default: 1e-6.

    Returns
    -------
    p : numpy.ndarray
        The solution after relaxation as a 2D array of floats.
    ite : integer
        The number of iterations performed.
    diff : float
        The final relative L2-norm of the difference.
    """

    # estimate nx, ny
    ny, nx = numpy.shape(p0)

    # copy initial guess p0
    p = p0.copy()

    conv = []  # convergence history
    diff = rtol + 1.0  # initial difference
    ite = 0  # iteration index

    while diff > rtol and ite < maxiter:

        # store old p-values
        pn = p.copy()

        # Update the solution at interior points of the Laplace equation
        p = laplace_jac(p,pn,nx,ny)

        # Apply 2nd-order Neumann condition (zero-gradient)
        # at the right boundary.
        p[1:-1, -1] = 0.25 * (2.0 * pn[1:-1, -2] +
                              pn[2:, -1] + pn[:-2, -1])

        # Compute the relative L2-norm of the difference.
        diff = numpy.sqrt(numpy.sum((p - pn)**2) / numpy.sum(pn**2))
        conv.append(diff)
        ite += 1

    return p, ite, conv
```

```python
# Compute the solution using Jacobi relaxation method.
p, ites, conv_jacobi = laplace_2d_jacobi(p0, maxiter=20000, rtol=1e-8)
print('Jacobi relaxation: {} iterations '.format(ites) +
      'to reach a relative difference of {}'.format(conv_jacobi[-1]))
```

    Jacobi relaxation: 19993 iterations to reach a relative difference of 9.998616840939269e-09

Would we lie to you? $19,993$ iterations before we reach the exit criterion of $10^{-8}$. Yikes!

We can also time how long the Jacobi method takes using the `%%timeit` cell-magic.  Go make some tea, because this can take a while—the `%%timeit` magic runs the function a few times and then averages their runtimes to give a more accurate result.

- - -
##### Notes

1. When using `%%timeit`, the return values of a function (`p` and `iterations` in this case) *won't* be saved.

2. We document our timings below, but your timings can vary quite a lot, depending on your hardware. In fact, you may not even see the same trends (some recent hardware can play some fancy tricks with optimizations that you have no control over).
- - -

With those caveats, let's give it a shot:

```python
%%timeit
laplace_2d_jacobi(p0, maxiter=20000, rtol=1e-8)
```

    1.72 s ± 27.9 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

The printed result above (and others to come later) is from a 2017 desktop PC, powered by a 4 GHz Intel Core i7-6700K CPU. We tried also on more modern machines, and got conflicting results—like the Gauss-Seidel method being slightly slower than Jacobi, even though it required fewer iterations. Don't get too hung up on this: the hardware optimizations applied by more modern CPUs are varied and make a big difference sometimes.

Meanwhile, let's check the overall accuracy of the numerical calculation by comparing it to the analytical solution.

```python
# Compute the analytical solution.
p_exact = laplace_solution(x, y, Lx, Ly)

# Compute the relative L2-norm of the error.
l2_norm(p, p_exact)
```

    6.17355133528859e-05

That's a pretty small error. Let's assume it is good enough and focus on speeding up the process.

## Gauss-Seidel

You will recall from [Lesson 1](http://nbviewer.ipython.org/urls/github.com/daniel-koehn/Differential-equations-earth-system/tree/master/07_BVP_2D/01_2D_Laplace_Equation.ipynb) that a single Jacobi iteration is written as:

$$
\begin{equation}
p^{k+1}_{i,j} = \frac{1}{4} \left(p^{k}_{i,j-1} + p^k_{i,j+1} + p^{k}_{i-1,j} + p^k_{i+1,j} \right) \tag{1}
\end{equation}
$$

The Gauss-Seidel method is a simple tweak to this idea: use updated values of the solution as soon as they are available, instead of waiting for the values in the whole grid to be updated.

If you imagine that we progress through the grid points in the order shown by the arrow in Figure 1, then you can see that the updated values $p^{k+1}_{i-1,j}$ and $p^{k+1}_{i,j-1}$ can be used to calculate $p^{k+1}_{i,j}$.

#### Figure 1. Assumed order of updates on a grid.

The iteration formula for Gauss-Seidel is thus:

$$
\begin{equation}
p^{k+1}_{i,j} = \frac{1}{4} \left(p^{k+1}_{i,j-1} + p^k_{i,j+1} + p^{k+1}_{i-1,j} + p^k_{i+1,j} \right) \tag{2}
\end{equation}
$$

In the function `laplace_gauss`, we compute the Gauss-Seidel updates. Compare the code with the `laplace_jac` function.

```python
@jit(nopython=True) # use Just-In-Time (JIT) Compilation for C-performance
def laplace_gauss(p,nx,ny):

    for i in range(1,nx-1):
        for j in range(1,ny-1):

            # Solution of the Laplace equation using the Gauss-Seidel method
            p[j, i] = 0.25 * (p[j+1, i] + p[j, i+1] +
                              p[j-1, i] + p[j, i-1])

    return p
```

To solve the Laplace problem with the Gauss-Seidel method, we use the code `laplace_2d_gauss_seidel`

```python
def laplace_2d_gauss_seidel(p0, maxiter=20000, rtol=1e-6):
    """
    Solves the 2D Laplace equation on a uniform grid
    with equal grid spacing in both directions
    using Gauss-Seidel relaxation method.

    The exit criterion of the solver is based on the relative L2-norm
    of the solution difference between two consecutive iterations.

    Parameters
    ----------
    p0 : numpy.ndarray
        The initial solution as a 2D array of floats.
    maxiter : integer, optional
        Maximum number of iterations to perform;
        default: 20000.
    rtol : float, optional
        Relative tolerance for convergence;
        default: 1e-6.

    Returns
    -------
    p : numpy.ndarray
        The solution after relaxation as a 2D array of floats.
    ite : integer
        The number of iterations performed.
    diff : float
        The final relative L2-norm of the difference.
    """

    # estimate nx, ny
    ny, nx = numpy.shape(p0)

    # copy initial guess p0
    p = p0.copy()

    conv = []  # convergence history
    diff = rtol + 1.0  # initial difference
    ite = 0  # iteration index

    while diff > rtol and ite < maxiter:

        # store old value p
        pn = p.copy()

        # Update the solution at interior points of the Laplace equation
        p = laplace_gauss(p,nx,ny)

        # Apply 2nd-order Neumann condition (zero-gradient)
        # at the right boundary.
        for j in range(1, ny - 1):
            p[j, -1] = 0.25 * (2.0 * p[j, -2] +
                               p[j - 1, -1] + p[j + 1, -1])

        # Compute the relative L2-norm of the difference.
        diff = l2_norm(p, pn)
        conv.append(diff)
        ite += 1

    return p, ite, conv
```

Let's test the code performance

```python
# Compute the solution using Gauss-Seidel relaxation method.
p, ites, conv_gs = laplace_2d_gauss_seidel(p0,maxiter=20000, rtol=1e-8)
print('Gauss-Seidel relaxation: {} iterations '.format(ites) +
      'to reach a relative difference of {}'.format(conv_gs[-1]))
```

    Gauss-Seidel relaxation: 13939 iterations to reach a relative difference of 9.997635653131014e-09

Cool! Using the most recently updated values of the solution in the Gauss-Seidel method saved 6,000 iterations! Now we can see how much faster than Jacobi this is, because both methods are implemented the same way:

```python
%%timeit
laplace_2d_gauss_seidel(p0, maxiter=20000, rtol=1e-8)
```

    2.74 s ± 18.8 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

We get no speed-up over the Numba version of Jacobi, and you may see different results. This can be confusing, and hard to explain without getting into the nitty grity of hardware optimizations.

Don't lose hope! We have another trick up our sleeve!

## Successive Over-Relaxation (SOR)

Successive over-relaxation is able to improve on the Gauss-Seidel method by using in the update a linear combination of the previous and the current solution, as follows:

$$
\begin{equation}
p^{k+1}_{i,j} = (1 - \omega)p^k_{i,j} + \frac{\omega}{4} \left(p^{k+1}_{i,j-1} + p^k_{i,j+1} + p^{k+1}_{i-1,j} + p^k_{i+1,j} \right) \tag{3}
\end{equation}
$$

The relaxation parameter $\omega$ will determine how much faster SOR will be than Gauss-Seidel. SOR iterations are only stable for $0 < \omega < 2$. Note that for $\omega = 1$, SOR reduces to the Gauss-Seidel method.

If $\omega < 1$, that is technically an "under-relaxation" and it will be slower than Gauss-Seidel.

If $\omega > 1$, that's the over-relaxation and it should converge faster than Gauss-Seidel.

Let's write a function for SOR iterations of the Laplace equation, using Numba to get high performance.

```python
@jit(nopython=True) # use Just-In-Time (JIT) Compilation for C-performance
def laplace_SOR(p,nx,ny,omega):

    for i in range(1,nx-1):
        for j in range(1,ny-1):

            # Solution of the Laplace equation using the Gauss-Seidel method
            p[j, i] = ((1.0 - omega) * p[j, i] +
                        omega * 0.25 *(p[j, i - 1] + p[j, i + 1] +
                                       p[j - 1, i] + p[j + 1, i]))

    return p
```

```python
def laplace_2d_sor(p0, omega, maxiter=20000, rtol=1e-6):
    """
    Solves the 2D Laplace equation on a uniform grid
    with equal grid spacing in both directions
    using successive over-relaxation (SOR) method.

    The exit criterion of the solver is based on the relative L2-norm
    of the solution difference between two consecutive iterations.

    Parameters
    ----------
    p0 : numpy.ndarray
        The initial solution as a 2D array of floats.
    omega : float
        Relaxation parameter.
    maxiter : integer, optional
        Maximum number of iterations to perform;
        default: 20000.
    rtol : float, optional
        Relative tolerance for convergence;
        default: 1e-6.

    Returns
    -------
    p : numpy.ndarray
        The solution after relaxation as a 2D array of floats.
    ite : integer
        The number of iterations performed.
    conv : list
        The convergence history as a list of floats.
    """

    # estimate nx, ny
    ny, nx = p0.shape

    # copy initial guess p0
    p = p0.copy()

    conv = []  # convergence history
    diff = rtol + 1.0  # initial difference
    ite = 0  # iteration index

    while diff > rtol and ite < maxiter:

        # store old value p
        pn = p.copy()

        # Update the solution at interior points of the Laplace equation
        p = laplace_SOR(p,nx,ny,omega)

        # Apply 2nd-order Neumann condition (zero-gradient)
        # at the right boundary.
        for j in range(1, ny - 1):
            p[j, -1] = 0.25 * (2.0 * p[j, -2] +
                               p[j - 1, -1] + p[j + 1, -1])

        # Compute the relative L2-norm of the difference.
        diff = numpy.sqrt(numpy.sum((p - pn)**2) / numpy.sum(pn**2))
        conv.append(diff)
        ite += 1

    return p, ite, conv
```

That wasn't too bad at all.  Let's try this out first with $\omega = 1$ and check that it matches the Gauss-Seidel results from above.

```python
# Compute the solution using SOR method.
omega = 1.0
p, ites, conv_sor = laplace_2d_sor(p0, omega, maxiter=20000, rtol=1e-8)
print('SOR (omega={}): {} iterations '.format(omega, ites) +
      'to reach a relative difference of {}'.format(conv_sor[-1]))
```

    SOR (omega=1.0): 13939 iterations to reach a relative difference of 9.997635651907095e-09

We have the exact same number of iterations as Gauss-Seidel.  That's a good sign that things are working as expected.

Now let's try to over-relax the solution and see what happens.  To start, let's try $\omega = 1.5$.

```python
# Compute the solution using SOR method.
omega = 1.5
p, ites, conv_sor = laplace_2d_sor(p0, omega, maxiter=20000, rtol=1e-8)
print('SOR (omega={}): {} iterations '.format(omega, ites) +
      'to reach a relative difference of {}'.format(conv_sor[-1]))
```

    SOR (omega=1.5): 7108 iterations to reach a relative difference of 9.991011444054642e-09

Wow!  That really did the trick!  We dropped from $13,939$ iterations down to $7,108$. Now we're really cooking! Let's try `%%timeit` on SOR.

```python
%%timeit
laplace_2d_sor(p0, omega, maxiter=20000, rtol=1e-8)
```

    1.48 s ± 6.03 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

Things continue to speed up. But we can do even better!

### Tuned SOR

Above, we picked $\omega=1.5$ arbitrarily, but we would like to over-relax the solution as much as possible without introducing instability, as that will result in the fewest number of iterations.

For square domains, it turns out that the ideal factor $\omega$ can be computed as a function of the number of nodes in one direction, e.g., `nx`.

$$
\begin{equation}
\omega \approx \frac{2}{1+\frac{\pi}{nx}}
\end{equation}
$$

This is not some arbitrary formula, but its derivation lies outside the scope of this course. (If you're curious and have some serious math chops, you can check out Reference 3 for more information). For now, let's try it out and see how it works.

```python
# Compute the solution using tuned SOR method.
omega = 2.0 / (1.0 + numpy.pi / nx)
p, ites, conv_opt_sor = laplace_2d_sor(p0, omega, maxiter=20000, rtol=1e-8)
print('SOR (omega={:.4f}): {} iterations '.format(omega, ites) +
      'to reach a relative difference of {}'.format(conv_opt_sor[-1]))
```

    SOR (omega=1.9521): 1110 iterations to reach a relative difference of 9.964283921218147e-09

Wow! That's *very* fast. Also, $\omega$ is very close to the upper limit of 2. SOR tends to work fastest when $\omega$ approaches 2, but don't be tempted to push it.  Set $\omega = 2$ and the walls will come crumbling down.

Let's see what `%%timeit` has for us now.

```python
%%timeit
laplace_2d_sor(p0, omega, maxiter=20000, rtol=1e-8)
```

    232 ms ± 4.31 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

Regardless of the hardware in which we tried this, the tuned SOR gave *big* speed-ups, compared to the Jacobi method (whether implemented with NumPy or Numba). Now you know why we told you at the end of [Lesson 1](http://nbviewer.ipython.org/urls/github.com/daniel-koehn/Differential-equations-earth-system/tree/master/07_BVP_2D/01_2D_Laplace_Equation.ipynb) that the Jacobi method is the *worst* iterative solver and almost never used.

Just to convince ourselves that everything is OK, let's check the error after the $1,110$ iterations of tuned SOR:

```python
# Compute the relative L2-norm of the error.
l2_norm(p, p_exact)
```

    7.792743355043485e-05

Looking very good, indeed.

We didn't explain it in any detail, but notice the very interesting implication of Equation $(6)$: the ideal relaxation factor is a function of the grid size.
Also keep in mind that the formula only works for square domains with uniform grids. If your problem has an irregular geometry, you will need to find a good value of $\omega$ by numerical experiments.

## Decay of the difference between iterates

In the [Poisson Equation notebook](http://nbviewer.ipython.org/urls/github.com/daniel-koehn/Differential-equations-earth-system/tree/master/07_BVP_2D/02_2D_Poisson_Equation.ipynb), we noticed how the norm of the difference between consecutive iterations first dropped quite fast, then settled for a more moderate decay rate. With Gauss-Seidel, SOR and tuned SOR, we reduced the number of iterations required to reach the stopping criterion. Let's see how that reflects on the time history of the difference between consecutive solutions.

```python
# Plot the convergence history for different methods.
pyplot.figure(figsize=(9.0, 4.0))
pyplot.xlabel('Iterations')
pyplot.ylabel('Relative $L_2$-norm\nof the difference')
pyplot.grid()
pyplot.semilogy(conv_jacobi, label='Jacobi')
pyplot.semilogy(conv_gs, label='Gauss-Seidel')
pyplot.semilogy(conv_sor, label='SOR')
pyplot.semilogy(conv_opt_sor, label='Optimized SOR')
pyplot.legend()
pyplot.xlim(0, 20000);
```

The Jacobi method starts out with very fast convergence, but then it settles into a slower rate. Gauss-Seidel shows a faster rate in the first few thousand iterations, but it seems to be slowing down towards the end. SOR is a lot faster to converge, though, and optimized SOR just plunges down!

## References

1.  [Gonsalves, Richard J.  Computational Physics I. State University of New York, Buffalo: (2011): Section 3.1 ](http://www.physics.buffalo.edu/phy410-505/2011/index.html)

2. Moin, Parviz, "Fundamentals of Engineering Numerical Analysis," Cambridge University Press, 2nd edition (2010).

3. Young, David M. "A bound for the optimum relaxation factor for the successive overrelaxation method." Numerische Mathematik 16.5 (1971): 408-413.


```python
%load_ext autoreload
%autoreload 2

%matplotlib notebook
```

```python
import math

from kelly_criterion.kelly import log_return
from kelly_criterion.samplers import make_bernoulli_return_sampler
from kelly_criterion.samplers import make_uniform_return_sampler
from kelly_criterion.plots import plot_func
from kelly_criterion.plots_3d import plot_3d_func_on_simplex
```

# Multivariate Kelly Goes All-In

## Objective

The goal of this lab is to explore the feature observed in \[Finkelstein et al, 1981\] that while a Kelly strategy in a univariate setting will never invest the full fortune, it can do in a multivariate setting.

## Background

Example 3 of \[Finkelstein et al, 1981\] describes:

>The surprising fact is that for one variable X1 a gambler does not bet all his fortune no matter what b is, but he does bet all his fortune on two independent copies of X1 for b large enough.

The example they give is $X, Y \sim U(-1, b)$ IID. As usual, we want to maximize $G(\mu, \nu) = \mathbb{E}\big[ \log(1+\mu X + \nu Y \big]$. They show that $G$ is maximized at $(\frac{1}{2},\frac{1}{2})$ for $b > \log(16) - 1 \approx 1.77$. That is, the Kelly strategy allocates the full capital.

## Bivariate Uniform

In the case considered in \[Finkelstein et al, 1981\], we plot $G$ first for $b < 1.77$:

```python
samples = make_uniform_return_sampler(1.1, n=2)(1000)
plot_3d_func_on_simplex(lambda x, y: log_return([x, y], samples))
```

    Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …

the behaviour looks similar to the univariate case. When we increase $b$, though, we see

```python
samples = make_uniform_return_sampler(2, n=2)(1000)
plot_3d_func_on_simplex(lambda x, y: log_return([x, y], samples))
```

    Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …

as described.

Varying $b$, we see the issue is about hitting the boundary of the domain within the domain of the log.

Despite investing the full capital, the probability of a total loss is still 0.

## Bivariate Bernoulli

The above example raises the question of what this looks like in the case of bivariate Bernoullis, where there is a finite probability of total loss.

Again, for small $p$ it looks as previously

```python
samples = make_bernoulli_return_sampler(0.6, n=2)(1000)
plot_3d_func_on_simplex(lambda x, y: log_return([x, y], samples))
```

    Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …

as $p$ grows we see similar behaviour to the uniform case:

```python
samples = make_bernoulli_return_sampler(0.9, n=2)(1000)
plot_3d_func_on_simplex(lambda x, y: log_return([x, y], samples))
```

    Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …

But to see what happens near the boundary, let $G(\lambda_1, \lambda_2) = \mathbb{E}\big[1+\lambda_1 X_1 + \lambda_2 X_2 \big]$ for $X_i \sim Bernoulli(p)$ IID.

Then by the same argument from \[Finkelstein et al, 1981\], if there's an optimum then convexity of $G$ means it must occur along the ridge $\lambda_1 = \lambda_2$. So consider:

$$
\begin{align}
G(\lambda) &= \mathbb{E}\big[ \log(1+\lambda (X_1 + X_2)) \big] \\
    &= p^2 \log(1+2\lambda) + q^2\log(1-2\lambda) \\
\\
G'(\lambda) &= \frac{p^2}{1+2\lambda} - \frac{q^2}{1-2\lambda} \\
\\
G''(\lambda) &= 0 \\
\end{align}
$$

which gives an optimum at $\lambda^*(p) = \frac{1}{2}\Big( \frac{2p^2}{2p^2-2p+1} - 1 \Big)$. So we have $\lambda^*(p) \uparrow \frac{1}{2} \text{ as } p \uparrow 1$, but $\lambda^*(p) < \frac{1}{2}$

```python
plot_func(lambda p: 0.5*(2*p*p/(2*p*p-2*p+1) - 1), x_lower=0.5, x_upper=1)
```

So the null probability of ruin is indeed preserved, as expected.

And it's become clear I had mis-interpretted the quote starting this off. Obviously, the claim that a univariate Kelly strategy never goes all in is false. An obvious counter-example being $X_1 > 0$.

The significance must lie more in the way convergence happens at the boundary in this continuous case and how that drives the behaviour there.

## References

[\[Finkelstein et al, 1981\]](https://www.jstor.org/stable/1426692?seq=1) Finkelstein, Mark, and Robert Whitley. "Optimal Strategies for Repeated Games." Advances in Applied Probability 13, no. 2 (1981): 415-28. Accessed December 9, 2020. doi:10.2307/1426692. https://escholarship.org/uc/item/4w69h2xx

notebooks/3. Multivariate Kelly Goes All-In.ipynb