# Gaussian Process Regression
## Part II - Kernels, Hyperparameters, and all of that...
## 2nd Machine Learning in Heliophysics
## Boulder, CO
### 21 - 25 March 2022

### Enrico Camporeale (University of Colorado, Boulder & NOAA Space Weather Prediction Center)
#### enrico.camporeale@noaa.gov
This work is licensed under a Creative Commons Attribution 4.0 International License.

## Recap

\begin{equation}\boxed{\boldsymbol\mu_{a|b}=\boldsymbol\mu_a+\boldsymbol\Sigma_{ab}\boldsymbol\Sigma_{bb}^{-1}(\boldsymbol x_b - \boldsymbol\mu_b) \\
\boldsymbol\Sigma_{a|b} = \boldsymbol\Sigma_{aa} - \boldsymbol\Sigma_{ab}\boldsymbol\Sigma_{bb}^{-1}\boldsymbol\Sigma_{ba}} \end{equation} or

\begin{equation}\boxed{\boldsymbol\mu_{a|b}=\boldsymbol\mu_a-\boldsymbol\Lambda_{aa}^{-1}\boldsymbol\Lambda_{ab}(\boldsymbol x_b - \boldsymbol\mu_b) \\
\boldsymbol\Sigma_{a|b} = \boldsymbol\Lambda_{aa}^{-1}} \end{equation}

```python
from matplotlib import pyplot as plt
import numpy as np
from scipy.sparse import spdiags
import seaborn as sns
import pandas as pd
from ipywidgets import interactive, interact_manual
import time
import matplotlib.animation as animation

plt.rcParams.update({'font.size': 26})
%matplotlib inline
%load_ext autoreload
%autoreload 2
```

## Sampling from a multivariate normal distribution

Every modern programming language has built-in functions to sample from a multivariate normal distribution

```python
# define dimension
dim = 5

# define mean
mu = ([0,0,0,0,0])

# define covariance matrix
# for now let's do this 'by hand'

data = np.array([1*np.ones(dim), 0.9*np.ones(dim), 0.8*np.ones(dim), 0.7*np.ones(dim), 0.1*np.ones(dim)])
diags = np.array([0, 1, 2, 3, 4])
Sigma = spdiags(data, diags, 5, 5)
Sigma = (Sigma+ np.transpose(Sigma)).toarray()
print('Covariance matrix = \n', Sigma)
# Here we take 1000 samples from the multivariate normal with mean mu and covariance Sigma
samples = np.random.multivariate_normal(mu,Sigma,1000)

# Visualization
df = pd.DataFrame(data=samples) # transform np.array to pandas dataframe

## this is nice but slow...
#g = sns.PairGrid(df)
#g.map_diag(sns.kdeplot)
#g.map_offdiag(sns.kdeplot, n_levels=6);

## this is faster
g = sns.pairplot(df, diag_kind="kde", corner=False)
g.set(xlim=(-5,5), ylim=(-5,5))

```

## An alternative way of representing multidimensional samples

```python
fig = plt.figure(figsize=(7, 7))
ax = fig.gca()
x = range(5) # label for the dimension

# plot 10 samples of my 5-dimensional distribution
plt.plot(x, np.transpose(samples[1:10,:]))
ax.set_xlabel('Dimension label')
plt.show()
```

Each dimension is strongly correlated with the adjacent but the correlation decreases with the dimension distance

## Let us change the covariance between dimension 0 and dimension 1

```python
Sigma[0,1]=0; Sigma[1,0]=0
print(Sigma)
samples = np.random.multivariate_normal(mu,Sigma,5)

fig = plt.figure(figsize=(7, 7))
ax = fig.gca()
x = range(5)
plt.plot(x, np.transpose(samples[0:5,:]))
ax.set_xlabel('Dimension label')
plt.show()

```

# Let's move to higher dimension

```python
# Define the covariance matrix
def cov_matrix(a, b):
    """ GP squared exponential kernel """
    sigma_sq = 10
    sqdist = np.sum(a**2,1).reshape(-1,1) + np.sum(b**2,1) - 2*np.dot(a, b.T) # this is (a-b)^2
    return np.exp(-.5 * (1/sigma_sq) * sqdist)

# plot an example of covariance matrix in 20 dimensions

dim = 20
x=np.array(range(dim)).reshape(dim,1)
Sigma = cov_matrix(x,x)
df = pd.DataFrame(data=Sigma) # transform np.array to pandas dataframe
fig = plt.figure(figsize=(7, 7))
ax = fig.gca()
ax = sns.heatmap(df)
ax.set_aspect('equal')

```

```python
def f(dim):
    x=np.array(range(dim)).reshape(dim,1)
    mu = 0.0*np.ones(dim)
    Sigma = cov_matrix(x,x)
    samples = np.random.multivariate_normal(mu,Sigma,1)

    fig = plt.figure(figsize=(7, 7))
    ax = fig.gca()
    plt.plot(x, np.transpose(samples))
    ax.set_xlabel('Dimension label')
    return(0)
interactive_plot = interact_manual(f, dim=(10,100))
output = interactive_plot

```

    interactive(children=(IntSlider(value=55, description='dim', min=10), Button(description='Run Interact', style…

Again, all points are strongly correlated with the neighbours resulting in smooth curves

## What if now we take the distribution conditioned on all except one variable?

```python
%matplotlib inline
# Number of dimensions
dim = 50
x=np.array(range(dim)).reshape(dim,1)

# Define covariance matrix and mean and draw one sample
Sigma = cov_matrix(x,x)
mu = 0.0*np.ones(dim)
samples = np.random.multivariate_normal(mu,Sigma,1)

# Choose one variable. We condition on all the others
dim_a = 30 # label of the variable we want to calculate the distribution of

dim_b = np.concatenate((np.arange(0,dim_a),np.arange(dim_a+1,dim))) # all variables except dim_a

Lambda = np.linalg.inv(Sigma+spdiags(1e-2*np.ones(dim), 0, dim, dim).toarray())  ## add a little noise on the main diag
Lambda_aa= Lambda[dim_a][dim_a] # this is now a scalar since we are conditioning on all except just one variable
Sigma_cond = 1/Lambda_aa  # because Lambda_aa is a scalar. This is Sigma of a conditioned on b
std = np.sqrt(Sigma_cond) # Standard deviation

Lambda_ab = Lambda[dim_b,:][:,dim_a].reshape(1,dim-1)  # dimension 1 x (dim-1)

xb = samples[0][dim_b]

mu_ab = -np.dot(Lambda_ab,xb)/Lambda_aa # mean of a conditioned on b

# Draw 100 samples from the conditioned distribution (this is a 1D normal)
samples_1D = np.random.normal(mu_ab, std, 100)

# calculate the distribution over 100 points (for plotting)
y= np.linspace(-2,2,num=100)
f = dim_a+1/np.sqrt(2*np.pi)/std * np.exp(-0.5*(y-mu_ab)**2/std**2)

# visualization
fig = plt.figure(figsize=(7, 7))
ax = fig.gca()
ax.plot(x, np.transpose(samples),'.-',label='Points sampled from full distribution')
ax.plot(f,y,label='Conditional distribution')
ax.plot(dim_a*np.ones(100),samples_1D,'.r',label='Samples from cond. distr.')
ax.set_ylim(mu_ab-2, mu_ab+2)
ax.set_xlabel('Dimension label')
ax.legend(fontsize=16)
plt.show()
```

## Another example. We take out more than one variable and sample curves from the conditional distribution

```python
## %matplotlib inline
# Number of dimensions
dim = 20
x=np.array(range(dim)).reshape(dim,1)
Sigma = cov_matrix(x,x)
mu = 0.0*np.ones(dim)

samples = np.random.multivariate_normal(mu,Sigma,1)
dim_a = np.arange(0,dim)

# Choose a set of variables dim_b on which we condition the distribution
dim_b = np.arange(0,dim,5)

Sigma_aa= Sigma[dim_a,:][:,dim_a]
Sigma_ab= Sigma[dim_a,:][:,dim_b]
Sigma_ba= Sigma[dim_b,:][:,dim_a] # This is really just the transpose of Sigma_ab !!
Sigma_bb= Sigma[dim_b,:][:,dim_b]

xb = samples[0][dim_b]

# GP Algorithm
z = np.linalg.solve(Sigma_bb, xb)
mu_ab = np.matmul(Sigma_ab,z)

z = np.linalg.solve(Sigma_bb, Sigma_ba)
Sigma_cond = Sigma_aa - np.matmul(Sigma_ab,z)

V = np.sqrt(np.diag(Sigma_cond)) # Standard deviation

#Visualization
fig = plt.figure(figsize=(7, 7))
ax = fig.gca()
samples_1D = np.transpose(np.random.multivariate_normal(mu_ab,Sigma_cond,100))
ax.plot(dim_a,samples_1D,'',color='grey')
ax.plot(x, np.transpose(samples),'.-b',label='Points sampled from full distribution')
ax.plot(dim_a,mu_ab, 'k.-', label='Mean of cond. distr.')
ax.plot(dim_a,mu_ab+2*V, 'k--', label='2 Standard deviations (95%)')
ax.plot(dim_a,mu_ab-2*V, 'k--', )

ax.plot(dim_b,np.transpose(samples[:,dim_b]),'or', label='Points conditioned on')
ax.set_xlabel('Dimension label')
ax.legend(fontsize=16)
plt.show()

```

## The aha! moment

We now have a recipe to construct multivariate normal distribution conditional on a subset of variables. The recipe is fully analytical and relies on linear algebra only.
Most importantly, the mean and covariance of the conditional distribution are known in closed form.

_Nonlinear regression task (1D)_: we have a number of observations/data $D$ for which we know corresponding input/output: $\{x_1,y_1\},\{x_2,y_2\},\ldots,\{x_D,y_D\}$

We want to find the function $f(x)$ that generalizes the input/output relationship, assuming that $y=f(x)+\varepsilon$

D observations $\longrightarrow$ $D-$dimensional multivariate Gaussian

Dimensional label $i\longrightarrow x_i$

Value sampled for the dimension $i\longrightarrow y_i$

### What else do we need to be able to use this formalism for nonlinear regression?

+ Our input data is not an integer (ie we need to be able to construct functions $y=f(x)$ for a real-valued x
+ We need the covariance matrix between observations and observations
+ We need the covariance matrix between observations and predictions
+ We need the covariance matrix between predictions and predictions

Remember that $cov(x,y)=\mathbb{E}( (x-\mathbb{E}(x))(y-\mathbb{E}(y)) = \mathbb{E}(xy)-\mathbb{E}(x)\mathbb{E}(y)$

# We need a Covariance function (aka Kernel) !!

# Covariance functions

## Reference: http://www.gaussianprocess.org/gpml/chapters/RW4.pdf

A Covariance function is the crucial ingredient in GP regression, because it encodes all of our assumptions about the underlying functions we are trying to learn

A general name for a function $k$ of two arguments mapping a pair of inputs $x\in \mathcal{X} , x'\in \mathcal{X}\rightarrow\mathcal{R}$ is a kernel.

Not any generic kernel is a valid covariance function. The main constraint it has to satisfy is that the Covariance matrix generated must be positive semi-definite (all its eigenvalues are positive).

# The most important concept in Gaussian Process Regression

### We assume that the covariance between two data points $(x_1,y_1)$ and $(x_2,y_2)$ is a function of their inputs (in general multidimensional):

$$\boxed{cov(y_1,y_2) = k(x_1,x_2)}$$

Because the covariance encodes the _similarity_ between data points, this is a statement about similar inputs having similar outputs.

Also, assuming the covariance function to be of a given pre-defined functional form allows us to contruct the full Covariance Matrix $\boldsymbol\Sigma$ where each element is defined as:

$\Sigma_{ij}=k(x_i,x_j)$

Once again: the Covariance Matrix entries are only functions of the inputs $x$, not the outputs $y$. This is how we can construct also the elements corresponding to unseen data (predictions)

## Stationary Covariance Functions

Because we want to encode the _similarity_ between the output $y_1$ observed at $x_1$ and the output $y_2$ observed at $x_2$, it makes sense to use a kernel that is a function of the distance $|x_1-x_2|$ only.

Those are called Stationary Covariance Functions

### Squared Exponential Covariance Function (or Radial Basis Function, RBF)

$r=x_1-x_2$

$k(x_1,x_2) = \sigma\exp\left(-\frac{r^2}{l^2}\right)$  (defined and used here at the top of the notebook)

From C. E. Rasmussen &ISBN 026218253X.C. K. I. Williams, Gaussian Processes for Machine Learning, the MIT Press, 2006, www.GaussianProcess.org/gpml

### Hyperparameters

Most covariance functions depend on some free parameters, called Hyperparameters.

For instance the squared exponential function has (in 1D) two free parameters: $\sigma$ and $l$.

$k(x_1,x_2) = \sigma\exp\left(-\frac{(x_1-x_2)^2}{l^2}\right)$

### Role of hyperparameters

```python
# Define the covariance matrix
def cov_matrix(a, b, sigma, l):
    """ GP squared exponential kernel """
    l_sq = l**2
    sqdist = np.sum(a**2,1).reshape(-1,1) + np.sum(b**2,1) - 2*np.dot(a, b.T) # this is (a-b)^2
    return sigma*np.exp(-.5 * (1/l_sq) * sqdist)

# dimensions
dim = 20

def f(sigma,l):
    x=np.array(range(dim)).reshape(dim,1)
    mu = 0.0*np.ones(dim)
    Sigma = cov_matrix(x,x,sigma,l)
    samples = np.random.multivariate_normal(mu,Sigma,1)
#    fig, (ax1, ax2) = plt.subplots(1, 2,figsize=(7, 7))
#    fig = plt.figure()
#    ax = fig.gca()
    fig = plt.figure(figsize=(15,4))
    ax1 = fig.add_subplot(121)
    ax2 = fig.add_subplot(122)
    ax1.plot(x, np.transpose(samples))
    ax1.set_xlabel('Dimension label')
    df = pd.DataFrame(data=Sigma) # transform np.array to pandas dataframe
    ax2 = sns.heatmap(df)
    ax2.set_aspect('equal')

    return(0)
interactive_plot = interactive(f, sigma=(1,10),l=(0.1,10))
output = interactive_plot.children[-1]
interactive_plot

```

    interactive(children=(IntSlider(value=5, description='sigma', max=10, min=1), FloatSlider(value=5.05, descript…

## How do we choose the hyperparameters?

### We fit the data!

That is, we choose the hyperparameters that when used in the kernel generate a covariance matrix that is most consistent with the observed data.

Basically, we try to answer the following: given a functional form for the the covariance function, what are the hyperparameters such that our data is likely to be generated as a sample from the resulting D-dimensional Gaussian distribution ?

This turns out to be an optimization problem, where we want to maximize the probability associated to the data (one sample from the distribution). This is the  training  phase of a GP

# Gaussian Process

## ... and finally a definition:

### A Gaussian Process is a collection of random variables, any finite number of which have a joint Gaussian distribution. It can be thought as a generalization of a Gaussian distribution that works over functions.

A Gaussian Process is completely specified by its mean and covariance.

Mean function $m(\boldsymbol{x}) = \mathbb{E}(f(x)))$

Covariance function $k(\boldsymbol x_i, \boldsymbol x_j) = \mathbb{E}[(f(\boldsymbol x_i) -m(\boldsymbol x_i))(f(\boldsymbol x_j) -m(\boldsymbol x_j))]$

This is usually specified as $f(\boldsymbol x)\sim\mathcal{GP}(m(\boldsymbol x),k)$

## Cookbook

$ y(\boldsymbol x) = f(\boldsymbol x) + \varepsilon$

We have $D$ data points $(\boldsymbol x_i,y_i)$

We use the shortcut $f_* = f(\boldsymbol x_*) $ for (unknown) predicted values for input $\boldsymbol x_*$

We assume that the joint distribution of observed and unobserved values is Gaussian:

$\begin{bmatrix} y \\ f_* \end{bmatrix}\sim\mathcal{N}\left(\boldsymbol 0,\begin{bmatrix} k(\boldsymbol x,\boldsymbol x) & k(\boldsymbol x,\boldsymbol x_*) \\ k(\boldsymbol x_*,\boldsymbol x) & k(\boldsymbol x_*,\boldsymbol x_*)\end{bmatrix}\right)$

Note that a mean $m(\boldsymbol x)=0$ is typically chosen when no further information is available. This choice does not assume anything about the data and holds upon shifting (ie subtract the mean).

1) Choose a kernel that encodes our information about the data

2) Optimize the hyperparameters to fit the data

3) Construct the Covariance matrix

4) Calculate mean and (if you want to estimate the uncertainty) standard deviation at the inputs $\boldsymbol x_*$ using:

\begin{equation}\boldsymbol\mu_{a|b}=\boldsymbol\mu_a+\boldsymbol\Sigma_{ab}\boldsymbol\Sigma_{bb}^{-1}(\boldsymbol x_b - \boldsymbol\mu_b) \\
\boldsymbol\Sigma_{a|b} = \boldsymbol\Sigma_{aa} - \boldsymbol\Sigma_{ab}\boldsymbol\Sigma_{bb}^{-1}\boldsymbol\Sigma_{ba} \end{equation}

## Computational considerations

The algorithm involves inverting the matrix $\Sigma_{bb}$, which is the partition of the covariance matrix $k(x,x)$ (the observed data). This means that the method does not scale well with large number of datapoints.

Also, $\Sigma_{bb}$ might be ill-conditioned, meaning that its inverse can be computationally unstable. This is typically fixed by adding a small noise terms to the diagonal, which can be interpreted as a Gaussian noise term $\varepsilon$ on the observations.
Techniques have been developed to accelerate GPs, based on computing sparse approximation of the covariance matrix.

## Numerical algorithm (zero mean)

$$\begin{equation}\boldsymbol\mu_{f_*|y}=\boldsymbol k(x_*,x)\boldsymbol k(x,x)^{-1} \boldsymbol y \\
\boldsymbol\Sigma_{f_*|y} = \boldsymbol k(x_*,x_*) - \boldsymbol k(x_*,x) \boldsymbol k(x,x)^{-1} \boldsymbol k(x,x_*) \end{equation}$$

inputs: $\boldsymbol x$(inputs), $\boldsymbol  y$(targets), $\boldsymbol k$ (covariance function), $\sigma^2$ (noise level), $x_∗$ (test input)

outputs: $f_*$ (output)

We make use of the Cholesky decomposition to solve for $\boldsymbol k(x,x)^{-1} \boldsymbol y$

$\boldsymbol L\boldsymbol L^T =\boldsymbol k(x,x)+\sigma^2I$

Using an auxilary variable $\boldsymbol z = \boldsymbol k(x,x)^{-1}\boldsymbol  y$:

solve: $\boldsymbol L(\boldsymbol L^T \boldsymbol z) = \boldsymbol y$

$\boldsymbol f_* = \boldsymbol k(x_*,x)\cdot \boldsymbol z$

solve $\boldsymbol L \boldsymbol v=\boldsymbol k(x,x_*)$

full covariance matrix $\rightarrow\boldsymbol \Sigma_{f_*|y} = \boldsymbol k(x_*,x_*) -\boldsymbol v^T\boldsymbol v$

## Why non-parametric?

## And why the assumption that the data is drawn from a multivariate normal distribution makes sense ?

### A quick connection to Bayesian linear regression

In linear regression we have:  $y = \boldsymbol{w}^T\boldsymbol{x} + \varepsilon$, with $\varepsilon\sim\mathcal{N}(0,\sigma^2)$.

$P(y|x,w) \sim \mathcal{N}(\boldsymbol{w}^T\boldsymbol{x},\sigma^2)$

If we now assume that $P(w|Data) \sim\mathcal{N}(0,S)$ (is a Gaussian) we can derive the $P(y|x,Data)$ by marginalizing out the weights $w$:

$P(y|x,Data) = \int P(y|x,w,Data)P(w|Data) dw$

Therefore we have integrated over all possible values of $w$.

Non-parametric == infinite number of parameters

The assumptions of $P(w|Data)$ and $\varepsilon$ being Gaussian then imply that $P(y|x,Data)$ is Gaussian as well.

The GP regression approach starts directly from this assumption, with no prior model (linear or not) assumed

## Advanced topics

### Warped Gaussian Process (generalization to non-Gaussian distributions)

### Deep Gaussian Process

### GP with Heteroskedastic noise

### GP with physics-based kernels

# End of Part 2

Gaussian Process Regression Part 2.ipynb


    Run in Google Colab

    View on GitHub

    Download notebook

Author: [Tony Yum](https://www.linkedin.com/in/tonyyum/)

# Sympy and hedging

In this notebook, we first play with Sympy and see what symbolic maths can do that
cannot be done with numerical methods. Next, using sympy we can calculate derivatives
of a function and we leverage that to work out the delta of an option. We then do some delta
hedging to see if the hedging cost indeed replicates the price of an option and get some
understanding regarding the process.

## Imports

```python
%matplotlib inline

import sympy as sy
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import math
import pandas as pd
from datetime import datetime
from scipy.stats import norm
from sympy.stats import Normal, cdf
```

## Integration and infinity

First let's define an integration. The formula displays nicely in notebook.

```python
x = sy.Symbol('x')
sy.Integral(sy.exp(-x), (x, 0, sy.oo))
```

$\displaystyle \int\limits_{0}^{\infty} e^{- x}\, dx$

Let's actually integrate the function with interval [0, ∞]

Using numerical integration, it will never get to 1.

```python
sy.integrate(sy.exp(-x), (x, 0, sy.oo))
```

$\displaystyle 1$

Let's just convince ourselves that the area under the function is around 1.

Blue line is the function and red box is the 1x1 square (even though it doesn't look square)

```python
import matplotlib.patches as patches

x = np.linspace(0, 10)
y = np.exp(-x)

fig, ax = plt.subplots()
ax.plot(x, y)
rect = patches.Rectangle((0, 0), 1, 1,
                        linewidth=1, edgecolor='r', facecolor='none')

# Add the patch to the Axes
ax.add_patch(rect)
```

## Black Model

Here is the Black formula https://en.wikipedia.org/wiki/Black_model

I have taken the code from https://gist.github.com/raddy/bd0e977dc8437a4f8276

We now have Price as well as derivatives for put and call options.

```python
#spot, strike, vol, days till expiry, interest rate, call or put (1,-1)
S, K, sigma, T, r,cp = sy.symbols('S,K,sigma,T,r,cp')

N = Normal('W', 0.0, 1.0)

d1 = (sy.ln(S / K) + (0.5 * sigma ** 2) * T) / (sigma * sy.sqrt(T))
d2 = d1 - sigma * sy.sqrt(T)

tv = sy.exp(-r * T) * (cp *S * cdf(N)(cp*d1) - cp * K  * cdf(N)(cp*d2))

bs_tv = sy.lambdify((S, K, sigma, T, r,cp),tv)

#1st Order Greeks
delta = sy.lambdify((S, K, sigma, T, r,cp),tv.diff(S))
vega = sy.lambdify((S, K, sigma, T, r,cp),tv.diff(sigma)/100.) #WATCH UNITS
theta = sy.lambdify((S, K, sigma, T, r,cp),tv.diff(T))
rho = sy.lambdify((S, K, sigma, T, r,cp),tv.diff(r))

#2nd Order Greeks
gamma = sy.lambdify((S, K, sigma, T, r,cp),tv.diff(S,S))
vanna = sy.lambdify((S, K, sigma, T, r,cp),tv.diff(S,sigma)/100.)
vomma = sy.lambdify((S, K, sigma, T, r,cp),tv.diff(sigma,sigma)/1e4) #IN TICKS
charm = sy.lambdify((S, K, sigma, T, r,cp),tv.diff(S,T)) #DELTA DECAY

#3rd Order -- Who cares about anything about dGamma?
speed = sy.lambdify((S, K, sigma, T, r,cp),tv.diff(S,3))
```

Again we can display those formulas

```python
d1
```

$\displaystyle \frac{0.5 T \sigma^{2} + \log{\left(\frac{S}{K} \right)}}{\sqrt{T} \sigma}$

```python
d2
```

$\displaystyle - \sqrt{T} \sigma + \frac{0.5 T \sigma^{2} + \log{\left(\frac{S}{K} \right)}}{\sqrt{T} \sigma}$

```python
tv
```

$\displaystyle \left(- K cp \left(\frac{\operatorname{erf}{\left(0.5 \sqrt{2} cp \left(- \sqrt{T} \sigma + \frac{0.5 T \sigma^{2} + \log{\left(\frac{S}{K} \right)}}{\sqrt{T} \sigma}\right) \right)}}{2} + \frac{1}{2}\right) + S cp \left(\frac{\operatorname{erf}{\left(\frac{0.5 \sqrt{2} cp \left(0.5 T \sigma^{2} + \log{\left(\frac{S}{K} \right)}\right)}{\sqrt{T} \sigma} \right)}}{2} + \frac{1}{2}\right)\right) e^{- T r}$

```python
tv.diff(S)
```

$\displaystyle \left(- \frac{0.5 \sqrt{2} K cp^{2} e^{- 0.5 cp^{2} \left(- \sqrt{T} \sigma + \frac{0.5 T \sigma^{2} + \log{\left(\frac{S}{K} \right)}}{\sqrt{T} \sigma}\right)^{2}}}{\sqrt{\pi} S \sqrt{T} \sigma} + cp \left(\frac{\operatorname{erf}{\left(\frac{0.5 \sqrt{2} cp \left(0.5 T \sigma^{2} + \log{\left(\frac{S}{K} \right)}\right)}{\sqrt{T} \sigma} \right)}}{2} + \frac{1}{2}\right) + \frac{0.5 \sqrt{2} cp^{2} e^{- \frac{0.5 cp^{2} \left(0.5 T \sigma^{2} + \log{\left(\frac{S}{K} \right)}\right)^{2}}{T \sigma^{2}}}}{\sqrt{\pi} \sqrt{T} \sigma}\right) e^{- T r}$

## Visualising Black Model

### Difference expiration date

```python
x = np.linspace(50, 150)
y = [bs_tv(s, 100, 0.1, 1, 0.02, 1) for s in x]
plt.plot(x, y)
y = [bs_tv(s, 100, 0.1, 2, 0.02, 1) for s in x]
plt.plot(x, y)
y = [bs_tv(s, 100, 0.1, 3, 0.02, 1) for s in x]
plt.plot(x, y)
y = [bs_tv(s, 100, 0.1, 4, 0.02, 1) for s in x]
plt.plot(x, y)
```

### First order derivatives

```python
fig, axs = plt.subplots(2, 2)
fig.set_size_inches(14, 8)

x = np.linspace(50, 150)
y = [delta(a, 100, 0.1, 1, 0.02, 1) for a in x]
ax = axs[0, 0]
ax.plot(x, y)
ax.title.set_text('Delta')

x = np.linspace(0.02, 0.2)
y = [vega(100, 100, a, 1, 0.02, 1) for a in x]
ax = axs[0, 1]
ax.plot(x, y)
ax.title.set_text('Vega')

x = np.linspace(0, 1)
y = [theta(100, 100, 0.1, a, 0.02, 1) for a in x]
ax = axs[1, 0]
ax.plot(x, y)
ax.title.set_text('Theta')

x = np.linspace(0, 0.05)
y = [rho(100, 100, 0.1, 0.1, a, 1) for a in x]
ax = axs[1, 1]
ax.plot(x, y)
ax.title.set_text('Rho')
```

### Second order derivatives

```python
fig, axs = plt.subplots(2, 2)
fig.set_size_inches(14, 8)

x = np.linspace(50, 150)
y = [gamma(a, 100, 0.1, 1, 0.02, 1) for a in x]
ax = axs[0, 0]
ax.plot(x, y)
ax.title.set_text('Gamma')

x = np.linspace(50, 150)
y = [vanna(a, 100, 0.1, 1, 0.02, 1) for a in x]
ax = axs[0, 1]
ax.plot(x, y)
ax.title.set_text('Vanna')

x = np.linspace(0, 0.5)
y = [vomma(100, 100, a, 1, 0.02, 1) for a in x]
ax = axs[1, 0]
ax.plot(x, y)
ax.title.set_text('Vomma')

x = np.linspace(50, 150)
y = [charm(a, 100, 0.1, 1, 0.02, 1) for a in x]
ax = axs[1, 1]
ax.plot(x, y)
ax.title.set_text('Charm')
```

## Heding Portfolio

Let's say we're a bank and sold an option. We now need to short delta
amount of Forward with same underlying as the option.

We would think that the discounted hedging cost should be close to the option
premium at $T_0$. Let's see if that's the case

```python
days_in_year = 260 # Business days
p_S0 = 100
p_K = 100
p_sigma = 0.1
p_T = 1
n_days = days_in_year+1
dt = 1. / n_days

def simulate_one_path(p_r=0.02):
    mu = 0. ## Under Q-measure, doesn't really matter this is, we'll prove it later
    dW = np.random.normal((mu - 0.5 * p_sigma ** 2) * dt,
                          math.sqrt(dt) * p_sigma, days_in_year)
    dW = np.insert(dW, 0, 0)

    df = pd.DataFrame({
        'day': np.linspace(0, days_in_year, n_days).astype(int),
        't': np.linspace(0, 1, n_days),
        'S': p_S0 * np.exp(dW.cumsum())
    })

    df['V'] = df[['S', 't']].apply(
        lambda x: bs_tv(x[0], p_K, p_sigma, p_T - x[1], p_r, 1) , axis=1)
    df['prev_V'] = df.V.shift()

    df['delta'] = df[['S', 't']].apply(
        lambda x: delta(x[0], p_K, p_sigma, p_T - x[1], p_r, 1) , axis=1)
    df.loc[df.shape[0] - 1, 'delta'] = 0

    df['stock'] = df.delta

    df['prev_S'] = df.S.shift()

    df['prev_stock'] = df.stock.shift()
    df.loc[0, 'prev_stock'] = 0

    df['trade'] = df.stock - df.prev_stock
    df['option_pnl'] = -(df.V - df.prev_V)
    df.loc[0, 'option_pnl'] = 0.
    df['stock_pnl'] = df.prev_stock * (df.S - df.prev_S)
    df.loc[0, 'stock_pnl'] = 0.
    payoff = -max(df.loc[df.index[-1], 'S'] - p_K, 0)
    df['pnl'] = df.option_pnl + df.stock_pnl
    assert abs(payoff + df.loc[df.index[-1], 'V']) < 0.0000001, 'BS Price at Maturity shold equal payoff'

    return df

def calc_total_pnl(df):
    row = df.iloc[-1]
    return -max(row.S - p_K, 0) + df.stock_pnl.sum()

pd.options.display.float_format = '{:,.3f}'.format
p_r = 0.02

df = simulate_one_path(p_r)

print((calc_total_pnl(df) * math.exp(-p_r * p_T), bs_tv(p_S0, p_K, p_sigma, p_T, p_r, 1)))
df[[x for x in df.columns if not x.startswith('prev')]]
```

    (-3.945426603026897, 3.908798206018734)

    <lambdifygenerated-11>:2: RuntimeWarning: divide by zero encountered in double_scalars
      return ((-K*cp*((1/2)*erf(0.5*sqrt(2)*cp*(-sqrt(T)*sigma + (0.5*T*sigma**2 + log(S/K))/(sqrt(T)*sigma))) + 1/2) + S*cp*((1/2)*erf(0.5*sqrt(2)*cp*(0.5*T*sigma**2 + log(S/K))/(sqrt(T)*sigma)) + 1/2))*exp(-T*r))
    <lambdifygenerated-12>:2: RuntimeWarning: divide by zero encountered in double_scalars
      return ((-0.5*sqrt(2)*K*cp**2*exp(-0.5*cp**2*(-sqrt(T)*sigma + (0.5*T*sigma**2 + log(S/K))/(sqrt(T)*sigma))**2)/(sqrt(pi)*S*sqrt(T)*sigma) + cp*((1/2)*erf(0.5*sqrt(2)*cp*(0.5*T*sigma**2 + log(S/K))/(sqrt(T)*sigma)) + 1/2) + 0.5*sqrt(2)*cp**2*exp(-0.5*cp**2*(0.5*T*sigma**2 + log(S/K))**2/(T*sigma**2))/(sqrt(pi)*sqrt(T)*sigma))*exp(-T*r))
    <lambdifygenerated-12>:2: RuntimeWarning: invalid value encountered in double_scalars
      return ((-0.5*sqrt(2)*K*cp**2*exp(-0.5*cp**2*(-sqrt(T)*sigma + (0.5*T*sigma**2 + log(S/K))/(sqrt(T)*sigma))**2)/(sqrt(pi)*S*sqrt(T)*sigma) + cp*((1/2)*erf(0.5*sqrt(2)*cp*(0.5*T*sigma**2 + log(S/K))/(sqrt(T)*sigma)) + 1/2) + 0.5*sqrt(2)*cp**2*exp(-0.5*cp**2*(0.5*T*sigma**2 + log(S/K))**2/(T*sigma**2))/(sqrt(pi)*sqrt(T)*sigma))*exp(-T*r))

      day
      t
      S
      V
      delta
      stock
      trade
      option_pnl
      stock_pnl
      pnl

      0
      0
      0.000
      100.000
      3.909
      0.510
      0.510
      0.510
      0.000
      0.000
      0.000

      1
      1
      0.004
      100.252
      4.031
      0.520
      0.520
      0.010
      -0.123
      0.129
      0.006

      2
      2
      0.008
      101.045
      4.448
      0.550
      0.550
      0.031
      -0.417
      0.412
      -0.005

      3
      3
      0.012
      101.326
      4.597
      0.561
      0.561
      0.011
      -0.149
      0.155
      0.006

      4
      4
      0.015
      101.349
      4.603
      0.562
      0.562
      0.001
      -0.006
      0.013
      0.007

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      256
      256
      0.985
      120.232
      20.226
      1.000
      1.000
      0.000
      0.217
      -0.218
      -0.002

      257
      257
      0.988
      121.464
      21.459
      1.000
      1.000
      0.000
      -1.233
      1.232
      -0.002

      258
      258
      0.992
      121.534
      21.531
      1.000
      1.000
      0.000
      -0.072
      0.070
      -0.002

      259
      259
      0.996
      122.422
      22.420
      1.000
      1.000
      0.000
      -0.890
      0.888
      -0.002

      260
      260
      1.000
      121.532
      21.532
      0.000
      0.000
      -1.000
      0.888
      -0.890
      -0.002

261 rows × 10 columns

### Visualising the Hedge Portfolio

```python
df = simulate_one_path(0.02)

fig, axs = plt.subplots(2, 2)
fig.set_size_inches(14, 8)

row = df.iloc[-1]
tot_pnl = calc_total_pnl(df)

fig.suptitle('Payoff = {:.2f}, PNL_T = {:.2f}, discounted = {:.2f}, BS Price = {:.2f}'.format(
    df.loc[df.index[-1], 'V'],
    tot_pnl,
    tot_pnl * math.exp(-p_r * p_T),
    bs_tv(p_S0, p_K, p_sigma, p_T, p_r, 1)
), fontsize=16)

ax = axs[0, 0]
ax.plot(df.S)
ax.title.set_text('S0 = {:.2f}, S_T = {:.2f}'.format(
    df.loc[0, 'S'], df.loc[df.shape[0] - 1, 'S']
))

ax = axs[0, 1]
ax.plot(df.delta)
ax.title.set_text('Delta')

ax = axs[1, 0]
ax.plot(df.option_pnl.cumsum())
ax.title.set_text('Option PNL')

ax = axs[1, 1]
ax.plot(df.stock_pnl.cumsum())
ax.title.set_text('Hedge PNL')

```

## Gamma likes big moves

Look at the green line. Everytime there is a big movement in the underlying in either direction we have a big loss.

We lose because we are short gamma. On the other hand we are long theta meaning if there are not much movement in
the market the short option position we have would decay to zero (win for us)

```python
fig, ax1 = plt.subplots()
fig.set_size_inches(20, 8)
fig.suptitle('Why short gamma gives negative PNL on big swings', fontsize=16)
ax1.plot(df.t, df.stock_pnl, 'b-')
ax1.plot(df.t, df.option_pnl, 'r-')
ax1.set_xlabel('t')
ax2 = ax1.twinx()
ax2.plot(df.t, df.pnl, 'g-')

ax1.set_ylabel('Total PNL', color='b')
ax2.set_ylabel('Stock PNL', color='g')
```

## Let's do some simulation

Instead of looking at one particular scenario, let's run it 1,000 times and see if the hedging cost is indeed the option premium.

```python
def get_port_pnl(r):
    df = simulate_one_path(r)
    return calc_total_pnl(df) * math.exp(-r * p_T)

```

```python
def plot_graph_with_r(p_r, n_sims):
    before = datetime.now()
    port_pnl = [get_port_pnl(p_r) for _ in range(n_sims)]
    avg_pnl = sum(port_pnl) / n_sims
    plt.hist(port_pnl, bins=20)
    bs_price = bs_tv(p_S0, p_K, p_sigma, p_T, p_r, 1)
    plt.title(f'r: {p_r * 100:.0f}% Mean Pnl: {avg_pnl:.2f}, BS Pirce: {bs_price:.2f}')
    print('Time taken {}s'.format((datetime.now() - before).seconds))

plot_graph_with_r(0., 1000)
```

```python
plot_graph_with_r(0.05, 1000)
```

Looks good, bit this is too slow and boring to wait for if we want to ask more questions.

## Vectorise the delta

Instead we can calculate the delta for every day using vectorisation to speed things up.

We can go from 25s to 0.75s on my machine

First let's write the delta in numpy and check it against the sympy version

```python
def bs_delta(S, K, sigma, r, T):
    n_steps = S.shape[0]
    d1 = (np.log(S/K) + (0.5 * sigma**2)*T)/(sigma*np.sqrt(T))
    d2 = d1-sigma*np.sqrt(T)
    DF = np.exp(-r * T)
    return DF * norm.cdf(d1)

dummy_s = np.array([90., 100., 110., 90., 100., 110.])
dummy_t = np.array([1., 1., 0.6, 0.6, 0.3, 0.3])

p_r = 0.02
sym_v = [bs_tv(s, p_K, p_sigma, t, p_r, 1) for s, t in zip(dummy_s, dummy_t)]
sym_delta = [delta(s, p_K, p_sigma, t, p_r, 1) for s, t in zip(dummy_s, dummy_t)]

res_delta = bs_delta(dummy_s, p_K, p_sigma, p_r, dummy_t)

df = pd.DataFrame({
    't': dummy_t,
    'S': dummy_s,
    'delta': res_delta,
    'sym_delta': sym_delta,
})

np.testing.assert_array_almost_equal(df.delta, df.sym_delta)

df
```

      t
      S
      delta
      sym_delta

      0
      1.000
      90.000
      0.155
      0.155

      1
      1.000
      100.000
      0.510
      0.510

      2
      0.600
      110.000
      0.887
      0.887

      3
      0.600
      90.000
      0.092
      0.092

      4
      0.300
      100.000
      0.508
      0.508

      5
      0.300
      110.000
      0.956
      0.956

## Numpy delta hedging

```python
def simulate_hedging_cost(S0, K, sigma, r, T, mu, n_steps):
    dt = 1. / n_steps
    dW = np.random.normal((mu - 0.5 * sigma ** 2) * dt,
        math.sqrt(dt) * sigma, n_steps-1)
    dW = np.insert(dW, 0, 0)

    S = S0 * np.exp(dW.cumsum())
    prev_S = np.roll(S, 1)

    t = T - np.linspace(0, 1-dt, n_steps-1)

    S_T = S[-1]
    V_T = max(S_T - K, 0)
    delta = bs_delta(S[:-1], K, sigma, r, t)
    delta = np.append(delta, 0.)

    pos = delta
    pos[-1] = 0
    prev_pos = np.roll(pos, 1)
    prev_pos[0] = 0

    pos_pnl = prev_pos * (S - prev_S)
    pos_pnl[0] = 0.
    return pos_pnl.sum() - V_T

n_steps = 260
p_r = 0.
p_mu = 0.
simulate_hedging_cost(p_S0, p_K, p_sigma, p_r, p_T, p_mu, n_steps)
```

    -4.230757562475581

## Plotting numpy hedging

```python
def plot_sims(r, sigma, S0, mu=0., n_steps=1000):
    before = datetime.now()
    n_sims = 1000
    DF = math.exp(-r * p_T)
    all_costs = [simulate_hedging_cost(S0, p_K, sigma, r, p_T, mu, n_steps) * DF for _ in range(n_sims)]
    avg_cost = sum(all_costs) / n_sims
    plt.hist(all_costs, bins=20)
    plt.title('BS Price = {:.2f}, Avg Cost = {:.2f}'.format(
        bs_tv(S0, p_K, sigma, p_T, r, 1),
        avg_cost
    ))

    time_taken = (datetime.now() - before)
    print('Time taken {}.{}s'.format(time_taken.seconds, int(time_taken.microseconds / 1e4)))

plot_sims(0., 0.1, 100.)
```

What if we set $r = 0.05$

```python
plot_sims(0.05, 0.1, 100.)
```

What if we set $mu = 0.08$

This is an interesting scenario as it shows that pricing under risk neutral drift works even if reality has a different dift.

```python
plot_sims(0., 0.1, 100., mu=0.05)
```

Now we have $r = 0.02$ and $\sigma = 0.2$

See how it is now skewed

```python
plot_sims(0.02, 0.2, 100.)
```

Set $S_0 = 110$ i.e. in the money

```python
plot_sims(0.02, 0.1, 110.)
```

Set S_0 = 90 i.e. out of money

```python
plot_sims(0.02, 0.1, 90.)
```

There are much we can learn from looking at the hedging cost as we change different parameters.

In the next section, we would setup some distributed compute to run more simulation and get more insight.


### Example 3 , part B: Diffusion for non uniform material properties

In this example we will look at the diffusion equation for non uniform material properties and how to handle second-order derivatives. For this, we will reuse Devito's `.laplace` short-hand expression outlined in the previous example and demonstrate it using the examples from step 7 of the original tutorial. This example is an enhancement of `03_diffusion` in terms of having non-uniform viscosity as opposed to the constant $\nu$. This example introduces the use of `Function` in order to create this non-uniform space.

So, the equation we are now trying to implement is

$$\frac{\partial u}{\partial t} = \nu(x,y) \frac{\partial ^2 u}{\partial x^2} + \nu(x,y) \frac{\partial ^2 u}{\partial y^2}$$

In our case $\nu$ is not uniform and $\nu(x,y)$ represents spatially variable viscosity.
To discretize this equation we will use central differences and reorganizing the terms yields

\begin{align}
u_{i,j}^{n+1} = u_{i,j}^n &+ \frac{\nu(x,y) \Delta t}{\Delta x^2}(u_{i+1,j}^n - 2 u_{i,j}^n + u_{i-1,j}^n) \\
&+ \frac{\nu(x,y) \Delta t}{\Delta y^2}(u_{i,j+1}^n-2 u_{i,j}^n + u_{i,j-1}^n)
\end{align}

As usual, we establish our baseline experiment by re-creating some of the original example runs. So let's start by defining some parameters.

```python
from examples.cfd import plot_field, init_hat
import numpy as np
%matplotlib inline

# Some variable declarations
nx = 100
ny = 100
nt = 1000

nu = 0.15 #the value of base viscosity

offset = 1 # Used for field definition

visc =  np.full((nx, ny), nu) # Initialize viscosity
visc[nx//4-offset:nx//4+offset, 1:-1] = 0.0001 # Adding a material with different viscosity
visc[1:-1,nx//4-offset:nx//4+offset ] = 0.0001
visc[3*nx//4-offset:3*nx//4+offset, 1:-1] = 0.0001

visc_nb = visc[1:-1,1:-1]

dx = 2. / (nx - 1)
dy = 2. / (ny - 1)
sigma = .25
dt = sigma * dx * dy / nu

# Initialize our field

# Initialise u with hat function
u_init = np.empty((nx, ny))
init_hat(field=u_init, dx=dx, dy=dy, value=1)
u_init[10:-10, 10:-10] = 1.5

zmax = 2.5 # zmax for plotting
```

We now set up the diffusion operator as a separate function, so that we can re-use if for several runs.

```python
def diffuse(u, nt ,visc):
    for n in range(nt + 1):
        un = u.copy()
        u[1:-1, 1:-1] = (un[1:-1,1:-1] +
                        visc*dt / dy**2 * (un[1:-1, 2:] - 2 * un[1:-1, 1:-1] + un[1:-1, 0:-2]) +
                        visc*dt / dx**2 * (un[2:,1: -1] - 2 * un[1:-1, 1:-1] + un[0:-2, 1:-1]))
        u[0, :] = 1
        u[-1, :] = 1
        u[:, 0] = 1
        u[:, -1] = 1
```

Now let's take this for a spin. In the next two cells we run the same diffusion operator for a varying number of timesteps to see our "hat function" dissipate to varying degrees.

```python
#NBVAL_IGNORE_OUTPUT

# Plot material according to viscosity, uncomment to plot
import matplotlib.pyplot as plt
plt.imshow(visc_nb, cmap='Greys',  interpolation='nearest')

# Field initialization
u = u_init

print ("Initial state")
plot_field(u, zmax=zmax)

diffuse(u, nt , visc_nb )
print ("After", nt, "timesteps")
plot_field(u, zmax=zmax)

diffuse(u, nt, visc_nb)
print ("After another", nt, "timesteps")
plot_field(u, zmax=zmax)
```

You can notice that the area with lower viscosity is not diffusing its heat as quickly as the area with higher viscosity.

```python
#NBVAL_IGNORE_OUTPUT

# Field initialization
u = u_init

diffuse(u, nt , visc_nb)
print ("After", nt, "timesteps")
plot_field(u, zmax=zmax)
```

Excellent. Now for the Devito part, we need to note one important detail to our previous examples: we now have a second-order derivative. So, when creating our `TimeFunction` object we need to tell it about our spatial discretization by setting `space_order=2`. We also use the notation `u.laplace` outlined previously to denote all second order derivatives in space, allowing us to reuse this code for 2D and 3D examples.

```python
from devito import Grid, TimeFunction, Eq, solve, Function
from sympy.abc import a

# Initialize `u` for space order 2
grid = Grid(shape=(nx, ny), extent=(2., 2.))

# Create an operator with second-order derivatives
a = Function(name='a',grid = grid) # Define as Function
a.data[:]= visc  # Pass the viscosity in order to be used in the operator.

u = TimeFunction(name='u', grid=grid, space_order=2)

# Create an equation with second-order derivatives
eq = Eq(u.dt, a * u.laplace)
stencil = solve(eq, u.forward)
eq_stencil = Eq(u.forward, stencil)

eq_stencil
```

$\displaystyle u{\left(t + dt,x,y \right)} = dt \left(\left(\frac{\partial^{2}}{\partial x^{2}} u{\left(t,x,y \right)} + \frac{\partial^{2}}{\partial y^{2}} u{\left(t,x,y \right)}\right) a{\left(x,y \right)} + \frac{u{\left(t,x,y \right)}}{dt}\right)$

Great. Now all that is left is to put it all together to build the operator and use it on our examples. For illustration purposes we will do this in one cell, including update equation and boundary conditions.

```python
#NBVAL_IGNORE_OUTPUT
from devito import Operator, Constant, Eq, solve, Function

# Reset our data field and ICs
init_hat(field=u.data[0], dx=dx, dy=dy, value=1.)

# Field initialization
u.data[0] = u_init

# Create an operator with second-order derivatives
a = Function(name='a',grid = grid)
a.data[:]= visc

eq = Eq(u.dt, a * u.laplace, subdomain=grid.interior)
stencil = solve(eq, u.forward)
eq_stencil = Eq(u.forward, stencil)

# Create boundary condition expressions
x, y = grid.dimensions
t = grid.stepping_dim
bc = [Eq(u[t+1, 0, y], 1.)]  # left
bc += [Eq(u[t+1, nx-1, y], 1.)]  # right
bc += [Eq(u[t+1, x, ny-1], 1.)]  # top
bc += [Eq(u[t+1, x, 0], 1.)]  # bottom

op = Operator([eq_stencil] + bc)
op(time=nt, dt=dt, a = a)

print ("After", nt, "timesteps")
plot_field(u.data[0], zmax=zmax)

op(time=nt, dt=dt, a = a)
print ("After another", nt, "timesteps")
plot_field(u.data[0], zmax=zmax)
```


```python
import matplotlib.pyplot as plt
import numpy as np
from scipy import optimize
```

To solve the complicated, typically transcendental equations we come across, we can use the `scipy.optimize.root_scalar()` function. To do this, we need to create a function that returns the difference between the left-hand and right-hand sides of the equation we're solving.

In this example, let's solve the equation relating $M_1$, $M_2$, $A_2/A_1$, and $\gamma$:
\begin{equation}
\frac{A_2}{A_1} = \frac{M_1}{M_2} \left[ \frac{1 + \frac{\gamma-1}{2} M_2^2}{1 + \frac{\gamma-1}{2} M_1^2} \right]^{\frac{\gamma+1}{2(\gamma-1)}} e^{\Delta s/R}
\end{equation}
for $M_2$ if we know $M_1$, $A_2/A_1$, and $\gamma$.

To do this, we'll create a function `fA()` that takes as arguments $M_2$, $M_1$, $A_2/A_1$, and $\gamma$. It codes the Equation 1 as
\begin{equation}
\frac{A_2}{A_1} - \frac{M_1}{M_2} \left[ \frac{1 + \frac{\gamma-1}{2} M_2^2}{1 + \frac{\gamma-1}{2} M_1^2} \right]^{\frac{\gamma+1}{2(\gamma-1)}} e^{\Delta s/R} = 0
\end{equation}

```python
def fA(M2, M1, A_ratio, gamma):
    '''Function for equation relating Mach numbers and area ratio.
    '''
    return A_ratio - (M1/M2) * ((1 + ((gamma-1)/2.)*M2**2)/(1 + ((gamma-1)/2.)*M1**2))**((gamma + 1)/(2*(gamma-1)))
```

Then, we'll specify the known quantities and call the `root_scalar()` function, specifying the function to solve `fA` as the first parameter.

`root_scalar()` expects the first parameter in the function `fA` to be the unknown it is solving for. We can also pass through the other arguments as known constants using the `args` parameter. We can also help it find the correct solution by indicating a subsonic or supersonic solution with the `bracket` parameter.

```python
M1 = 0.5
A_ratio = 5/2
gamma = 1.4
root = optimize.root_scalar(fA, # This is the function we are solving
                            args=(M1, A_ratio, gamma),
                            # give a range for the solution
                            bracket=[1.0001, 10.0]
                            )
print(root)
```

          converged: True
               flag: 'converged'
     function_calls: 15
         iterations: 14
               root: 2.753763176695055

Instead of solving equations directly, we can use the helpful tabulated results available in the back of the textbook (as long as we are dealing with air, or at least an ideal diatomic gas where $\gamma = 1.4$).

However, often our problems don't fall exactly on the numbers provided, so we need to use interpolation for an accurate answer. We can do this easily with the `numpy.interp()` function.

In this example, we want to find the ratio $\frac{p}{p_t}$ associated with $M = 1.715$. To do that, we create lists of $M$ and $\frac{p}{p_t}$ surrounding the point of interest, t

```python
machs = [1.70, 1.71, 1.72, 1.73, 1.74]
p_ratios = [0.20259, 0.19956, 0.19656, 0.19361, 0.19070]
np.interp(1.715, machs, p_ratios)
```

    0.19805999999999996

You can also work the other way, but the `np.interp()` function expects the x list to be *increasing*, so we need to reverse the order of both lists.

```python
np.interp(0.198, p_ratio[::-1], M[::-1])
```

    1.7151999999999998

```python

```


# Solving Linear Systems via Gaussian Elimination

```
%matplotlib inline
```

We remember that the problem we are trying to solve is the system of linear equations written as

\begin{equation}
  A {\bf x} = {\bf b}.
\end{equation}

This is equivalent to $n$ simultaneous linear equations for the $n$ unknowns $x_i$.

The first class of methods we consider are **direct**: a finite number of steps are followed which, in principle, result in the correct answer.

Most direct methods use the same core approach: convert the linear system to an equivalent problem that is easy to solve.

So the first thing we should do is consider which linear systems are easy to solve. Obviously if the matrix is diagonal it is easy to solve. More generally, it is straightforward to solve a linear system if the matrix is *triangular*.

As an example, consider

\begin{equation}
  \begin{pmatrix} 1 & 2 & 3 \\ 0 & 4 & 5 \\ 0 & 0 & 6 \end{pmatrix} \begin{pmatrix} x_1 \\ x_2 \\ x_3 \end{pmatrix} = \begin{pmatrix} 6 \\ 9 \\ 6 \end{pmatrix}.
\end{equation}

Perform **back substitution** by solving from the bottom up. That is, read off the equation defined by the bottom row of the matrix:

\begin{equation}
  6 x_3 = 6 \qquad \implies \qquad x_3 = 1.
\end{equation}

Then read off the equation defined by the second row of the matrix, using the information found for $x_3$:

\begin{equation}
  4 x_2 + 5 x_3 = 9 \qquad \implies \qquad 4 x_2 = 4 \qquad \implies \qquad x_2 = 1.
\end{equation}

Finally, read off the equation defined by the first row of the matrix, using the information found for $x_2$ and $x_3$:

\begin{equation}
  x_1 + 2 x_2 + 3 x_3 = 6 \qquad \implies \qquad x_1 = 1.
\end{equation}

Back substitution will work for any *upper* triangular matrix, provided that no diagonal element is zero (guaranteed if the matrix is non-singular). If the matrix is lower triangular we would start at the top and work down, using **forward** substitution.

## Gaussian elimination

So, we need to be able to convert our original linear system to an equivalent system in triangular form - here we will aim to construct an upper triangular matrix. By equivalent, we mean that the solution is the same.

Think of each row corresponding to a single linear equation, and each column corresponding to a single unknown variable. Then we see that there are three basic operations that can be performed without changing the solution:

1. Multiply any row by a non-zero scalar (equivalent to scaling the whole equation).
2. Swap two rows (equivalent to changing the order of the equations).
3. Swap two columns (equivalent to changing the order of the unknown variables).

We will concentrate on the first two operations, as swapping the columns requires us to keep track of which entries of the final solution vector correspond to which unknown variables.

To ensure that the operations are performed consistently to both the entries of the matrix $A$ *and* the right hand side vector ${\bf b}$, it is best to construct the **augmented matrix** $\left( A | {\bf b} \right)$. All operations are applied to this matrix, which has $n+1$ columns and $n$ rows.

### Basic algorithm

The basic algorithm takes the augmented matrix with arbitrary coefficients and returns an upper triangular augemented matrix. This can then be split into the form of the original system and solved by back substitution.

The algorithm considers each row in turn. Assume

1. We are looking at row $i$, where $1 \le i \le n$;
2. The previous steps of the algorithm mean that the coefficients in the first $i-1$ columns of row $i$, and all rows below row $i$ are zero.
3. The $i^{\text{th}}$ entry of row $i$, $a_{i, i}$ is non-zero.

Then we can eliminate all coefficients in column $i$ in the rows below $i$ by subtracting a suitable of row $i$. Explicitly, consider row $j$ where $i < j \le n$. For all coefficients in row $j$, i.e. for $a_{j, k}$ with $1 \le k \le n$, we replace the coefficient by

\begin{equation}
  a_{j, k} \rightarrow a_{j, k} - \frac{a_{j, i}}{a_{i, i}} a_{i, k}.
\end{equation}

We see that when $k = i$ the coefficient $a_{j, i}$ will be set to zero, as required.

If we repeat this algorithm for all rows $i$ then we will have constructed a triangular matrix.

Let us write this algorithm explicitly in code.

```
import numpy as np

def GaussianEliminationReduction(A, b):
    """
    Simple Gaussian Elimination algorithm. Solves linear system A x = b, assuming well-conditioned square matrix A.

    Parameters
    ----------

    A : float, numpy array, 2d.
        Square matrix, size n x n.
    b : float, numpy array, 1d.
        RHS vector, size n (must conform with A).
    """

    # Store size of system
    n = len(b)
    assert(np.all(A.shape == (n, n)))

    # Form augmented matrix
    aug = np.hstack((A, b.reshape(n,1)));

    # Loop over rows
    for i in range(n):
        # Loop over rows below i
        for j in range(i+1, n):
            aug[j, :] = aug[j, :] - aug[j, i] / aug[i, i] * aug[i, :]

    # Return the separated, reduced, matrix and right hand side vector.
    return (aug[:,:-1], aug[:, -1])
```

Now test the algorithm on some simple examples. The first is already in triangular form as above - just checks that it doesn't do anything stupid!

```
A = np.array([[1.0, 2.0, 3.0], \
              [0.0, 4.0, 5.0], \
              [0.0, 0.0, 6.0]])
b = np.array([6.0, 9.0, 6.0])

(A1, b1) = GaussianEliminationReduction(A, b)
print("Resulting triangular matrix is \n{}.\n Right hand side vector becomes {}.\n".format(A1, b1))

C = np.array([[3.0, 0.0, 1.0], \
              [6.0, 2.0, 4.0], \
              [9.0, 2.0, 6.0]])
d = np.array([4.0, 10.0, 15.0])

(A2, b2) = GaussianEliminationReduction(C, d)
print("Resulting triangular matrix is \n{}.\n Right hand side vector becomes {}.\n".format(A2, b2))
```

    Resulting triangular matrix is
    [[ 1.  2.  3.]
     [ 0.  4.  5.]
     [ 0.  0.  6.]].
     Right hand side vector becomes [ 6.  9.  6.].

    Resulting triangular matrix is
    [[ 3.  0.  1.]
     [ 0.  2.  2.]
     [ 0.  0.  1.]].
     Right hand side vector becomes [ 4.  2.  1.].

### Pivoting

However, the simple algorithm above does not work robustly. As a simple example, consider the system below.

```
E = np.array([[1.0, 1.0, 1.0], \
              [0.0, 0.0, 2.0], \
              [0.0, 1.0, 1.0]])
f = np.array([1.0, 1.0, 2.0])

(A3, b3) = GaussianEliminationReduction(E, f)
print("Resulting triangular matrix is \n{}.\n Right hand side vector becomes {}.\n".format(A3, b3))
```

    Resulting triangular matrix is
    [[  1.   1.   1.]
     [  0.   0.   2.]
     [ nan  nan -inf]].
     Right hand side vector becomes [  1.   1. -inf].

    -c:27: RuntimeWarning: divide by zero encountered in double_scalars
    -c:27: RuntimeWarning: invalid value encountered in multiply

The algorithm has failed as the matrix does not satisfy the assumptions necessary: in particular, the element $a_{2, 2}$ is zero.

We can simply avoid this problem by swapping rows. In this instance we would swap the second and third row and the algorithm would work (in this case, swapping the second and third rows puts the matrix in triangular form).

However, there is a similar problem if an entry is very small, but non-zero. Consider the problem

\begin{equation}
  \left( \begin{array}{c c|c} \varepsilon & 1 & 1 \\ 1 & 1 & 2 \end{array} \right) \rightarrow \left( \begin{array}{c c|c} \varepsilon & 1 & 1 \\ 0 & 1 - \varepsilon^{-1} & 2 - \varepsilon^{-1} \end{array} \right)
\end{equation}

which has been reduced to triangular form using Gaussian elimination. If $\varepsilon$ is very small then the terms $\varepsilon^{-1}$ will dominate, making the problem approximately

\begin{equation}
  \left( \begin{array}{c c|c} \varepsilon & 1 & 1 \\ 0 & - \varepsilon^{-1} & - \varepsilon^{-1} \end{array} \right) \quad \implies \quad {\bf x} = \begin{pmatrix} 0 \\ 1 \end{pmatrix}.
\end{equation}

The correct answer *should* be ${\bf x} = (1, 1)^T$. There is nothing wrong with the condition number. The problem is the Gaussian Elimination step where it divides through by $\varepsilon$, a very small number.

To avoid both these problems, the standard technique is to **pivot**, or use **pivoting**. At each stage in the algorithm we are working with row $i$ and looking to eliminate the entries in rows $j > i$. The first step is to check if there is a row $j > i$ with a larger (in magnitude) coefficient in column $i$. If so, we swap the rows. Note that we cannot consider rows above $i$ in the matrix, as these may have non-zero entries in columns $<i$.

We modify the algorithm to include pivoting.

```
def GaussianEliminationPivotingReduction(A, b):
    """
    Simple Gaussian Elimination algorithm with pivoting. Solves linear system A x = b,
    assuming well-conditioned square matrix A.

    Parameters
    ----------

    A : float, numpy array, 2d.
        Square matrix, size n x n.
    b : float, numpy array, 1d.
        RHS vector, size n (must conform with A).
    """

    # Store size of system
    n = len(b)
    assert(np.all(A.shape == (n, n)))

    # Form augmented matrix
    aug = np.hstack((A, b.reshape(n,1)));

    # Loop over rows
    for i in range(n):
        # Find the row with largest magnitude, and then swap the rows.
        max_row = np.argmax(aug[i:, i])
        if (max_row): # Only swap rows if the maximum is not this row. \
                      # NOTE: the max_row is counted relative to i, so max_row = 0 => row i.
            tmp               = np.copy(aug[i, :])
            aug[i, :]         = np.copy(aug[i+max_row, :])
            aug[i+max_row, :] = np.copy(tmp)
        # Loop over rows below i
        for j in range(i+1, n):
            aug[j, :] = aug[j, :] - aug[j, i] / aug[i, i] * aug[i, :]

    # Return the separated, reduced, matrix and right hand side vector.
    return (aug[:,:-1], aug[:, -1])
```

Check on the previously failing case.

```
(A4, b4) = GaussianEliminationPivotingReduction(E, f)
print("Resulting triangular matrix is \n{}.\n Right hand side vector becomes {}.\n".format(A4, b4))
```

    Resulting triangular matrix is
    [[ 1.  1.  1.]
     [ 0.  1.  1.]
     [ 0.  0.  2.]].
     Right hand side vector becomes [ 1.  2.  1.].

## Special case - tridiagonal systems

Tridiagonal systems turn up frequently in the solution of differential equations, particularly when using finite difference methods.

Consider the simple $4 \times 4$ system

\begin{equation}
        \begin{pmatrix}
          2 & 1 & 0 & 0 \\
          1 & 2 & 1 & 0 \\
          0 & 1 & 2 & 1 \\
          0 & 0 & 1 & 2
        \end{pmatrix}
        \begin{pmatrix}
          x_1 \\ x_2 \\ x_3 \\ x_4
        \end{pmatrix}
        =
        \begin{pmatrix}
          -1 \\ 0 \\ 0 \\ -1
        \end{pmatrix}.
\end{equation}

We proceed as we would with Gaussian elimination: remove the (single) coefficient below the diagonal first, rescaling as we go:

\begin{equation}
        \begin{pmatrix}
          1 & \tfrac{1}{2} & 0 & 0 \\
          0 & 1 & \tfrac{2}{3} & 0 \\
          0 & 0 & 1 & \tfrac{3}{4} \\
          0 & 0 & 0 & 1
        \end{pmatrix}
        \begin{pmatrix}
          x_1 \\ x_2 \\ x_3 \\ x_4
        \end{pmatrix}
        =
        \begin{pmatrix}
          -\tfrac{1}{2} \\ \tfrac{1}{3} \\ -\tfrac{1}{4} \\ -\tfrac{3}{5}
        \end{pmatrix},
\end{equation}

and then use back-substitution to get

\begin{equation}
        \left\{
          \begin{aligned}
            x_4 & =-\tfrac{3}{5} \\
            x_3 & = \tfrac{1}{5} \\
            x_2 & = \tfrac{1}{5} \\
            x_1 & =-\tfrac{3}{5}
          \end{aligned} \right. .
\end{equation}

The general formulation of this is the **Thomas algorithm**. It has the significant advantage that only the three non-trivial diagonals need to be stored and worked with. This can massively reduce the memory required.

Lectures/02 - Gaussian Elimination.ipynb


# 13 Linear Algebra: Singular Value Decomposition

One can always decompose a matrix $\mathsf{A}$

\begin{gather}
\mathsf{A} = \mathsf{U}\,\text{diag}(w_j)\,\mathsf{V}^{T}\\
\mathsf{U}^T \mathsf{U} = \mathsf{U} \mathsf{U}^T = 1\\
\mathsf{V}^T \mathsf{V} = \mathsf{V} \mathsf{V}^T = 1
\end{gather}

where $\mathsf{U}$ and $\mathsf{V}$ are orthogonal matrices and the $w_j$ are the _singular values_ that are assembled into a diagonal matrix $\mathsf{W}$.

$$
\mathsf{W} = \text{diag}(w_j)
$$

The inverse (if it exists) can be directly calculated from the SVD:

$$
\mathsf{A}^{-1} = \mathsf{V} \text{diag}(1/w_j) \mathsf{U}^T
$$

## Solving ill-conditioned coupled linear equations

```python
import numpy as np
```

### Non-singular matrix
Solve the linear system of equations

$$
\mathsf{A}\mathbf{x} = \mathbf{b}
$$

Using the standard linear solver in numpy:

```python
A = np.array([
        [1, 2, 3],
        [3, 2, 1],
        [-1, -2, -6],
    ])
b = np.array([0, 1, -1])
```

```python
np.linalg.solve(A, b)
```

    array([ 0.83333333, -0.91666667,  0.33333333])

Using the inverse from SVD:

$$
\mathbf{x} = \mathsf{A}^{-1} \mathbf{b}
$$

```python
U, w, VT = np.linalg.svd(A)
print(w)
```

    [ 7.74140616  2.96605874  0.52261473]

First check that the SVD really factors $\mathsf{A} = \mathsf{U}\,\text{diag}(w_j)\,\mathsf{V}^{T}$:

```python
U.dot(np.diag(w).dot(VT))
```

    array([[ 1.,  2.,  3.],
           [ 3.,  2.,  1.],
           [-1., -2., -6.]])

```python
np.allclose(A, U.dot(np.diag(w).dot(VT)))
```

    True

Now calculate the matrix inverse $\mathsf{A}^{-1} = \mathsf{V} \text{diag}(1/w_j) \mathsf{U}^T$:

```python
inv_w = 1/w
print(inv_w)
```

    [ 0.1291755   0.33714774  1.91345545]

```python
A_inv = VT.T.dot(np.diag(inv_w)).dot(U.T)
print(A_inv)
```

    [[ -8.33333333e-01   5.00000000e-01  -3.33333333e-01]
     [  1.41666667e+00  -2.50000000e-01   6.66666667e-01]
     [ -3.33333333e-01  -1.08335035e-16  -3.33333333e-01]]

Check that this is the same that we get from `numpy.linalg.inv()`:

```python
np.allclose(A_inv, np.linalg.inv(A))
```

    True

Now, *finally* solve (and check against `numpy.linalg.solve()`):

```python
x = A_inv.dot(b)
print(x)
np.allclose(x, np.linalg.solve(A, b))
```

    [ 0.83333333 -0.91666667  0.33333333]

    True

```python
A.dot(x)
```

    array([ -7.77156117e-16,   1.00000000e+00,  -1.00000000e+00])

```python
np.allclose(A.dot(x), b)
```

    True

### Singular matrix

If the matrix $\mathsf{A}$ is *singular* (i.e., its rank (linearly independent rows or columns) is less than its dimension and hence the linear system of equation does not have a unique solution):

For example, the following matrix has the same row twice:

```python
C = np.array([
     [ 0.87119148,  0.9330127,  -0.9330127],
     [ 1.1160254,   0.04736717, -0.04736717],
     [ 1.1160254,   0.04736717, -0.04736717],
    ])
b1 = np.array([ 2.3674474,  -0.24813392, -0.24813392])
b2 = np.array([0, 1, 1])
```

```python
np.linalg.solve(C, b1)
```

NOTE: failure is not always that obvious: numerically, a matrix can be *almost* singular.

Try solving the linear system of equations

$$
\mathsf{D}\mathbf{x} = \mathbf{b}_1
$$
with matrix $\mathsf{D}$ below:

```python
D = C.copy()
D[2, :] = C[0] - 3*C[1]
D
```

    array([[ 0.87119148,  0.9330127 , -0.9330127 ],
           [ 1.1160254 ,  0.04736717, -0.04736717],
           [-2.47688472,  0.79091119, -0.79091119]])

```python
np.linalg.solve(D, b1)
```

    array([  1.61493184e+00,   2.69013663e+16,   2.69013663e+16])

Note that some of the values are huge, and suspiciously like the inverse of machine precision? Sign of a nearly singular matrix.

**Note**: *Just because a function did not throw an exception it does not mean that the answer is correct.* **Always check your output!**

Now back to the example with $\mathsf{C}$:

#### SVD for singular matrices
If a matrix is *singular* or *near singular* then one can *still* apply SVD.

One can then compute the *pseudo inverse*

\begin{align}
\mathsf{A}^{-1} &= \mathsf{V} \text{diag}(\alpha_j) \mathsf{U}^T \\
\alpha_j &= \begin{cases}
 \frac{1}{w_j}, &\quad\text{if}\ w_j \neq 0\\
 0, &\quad\text{if}\ w_j = 0
 \end{cases}
\end{align}

i.e., any singular $w_j = 0$ is being "augmented" by setting

$$
\frac{1}{w_j} \rightarrow 0 \quad\text{if}\quad w_j = 0
$$

in $\text{diag}(1/w_j)$.

Perform the SVD for the singular matrix $\mathsf{C}$:

```python
U, w, VT = np.linalg.svd(C)
print(w)
```

    [  1.99999999e+00   1.00000000e+00   2.46519033e-32]

Note the third value $w_2 \approx 0$: sign of a singular matrix.

Test that the SVD really decomposes $\mathsf{A} = \mathsf{U}\,\text{diag}(w_j)\,\mathsf{V}^{T}$:

```python
U.dot(np.diag(w).dot(VT))
```

    array([[ 0.87119148,  0.9330127 , -0.9330127 ],
           [ 1.1160254 ,  0.04736717, -0.04736717],
           [ 1.1160254 ,  0.04736717, -0.04736717]])

```python
np.allclose(C, U.dot(np.diag(w).dot(VT)))
```

    True

There are the **singular values** (let's say, $|w_i| < 10^{-12}$):

```python
singular_values = np.abs(w) < 1e-12
print(singular_values)
```

    [False False  True]

#### Pseudo-inverse

Calculate the **pseudo-inverse** from the SVD

\begin{align}
\mathsf{A}^{-1} &= \mathsf{V} \text{diag}(\alpha_j) \mathsf{U}^T \\
\alpha_j &= \begin{cases}
 \frac{1}{w_j}, &\quad\text{if}\ w_j \neq 0\\
 0, &\quad\text{if}\ w_j = 0
 \end{cases}
\end{align}

Augment:

```python
inv_w = 1/w
inv_w[singular_values] = 0
print(inv_w)
```

    [ 0.5  1.   0. ]

```python
C_inv = VT.T.dot(np.diag(inv_w)).dot(U.T)
print(C_inv)
```

    [[-0.04736717  0.46650635  0.46650635]
     [ 0.5580127  -0.21779787 -0.21779787]
     [-0.5580127   0.21779787  0.21779787]]

#### Solution for $\mathbf{b}_1$

Now solve the linear problem with SVD:

```python
x1 = C_inv.dot(b1)
print(x1)
```

    [-0.34365138  1.4291518  -1.4291518 ]

```python
C.dot(x1)
```

    array([ 2.3674474 , -0.24813392, -0.24813392])

```python
np.allclose(C.dot(x1), b1)
```

    True

Thus, using the pseudo-inverse $\mathsf{C}^{-1}$ we can obtain solutions to the equation

$$
\mathsf{C} \mathbf{x}_1 = \mathbf{b}_1
$$

However, $\mathbf{x}_1$ is not the only solution: there's a whole line of solutions that are formed by the special solution and a combination of the basis vectors in the *null space* of the matrix:

The (right) *kernel* or *null space* contains all vectors $\mathbf{x^0}$ for which

$$
\mathsf{C} \mathbf{x^0} = 0
$$

(The dimension of the null space corresponds to the number of singular values.) You can find a basis that spans the null space. Any linear combination of null space basis vectors will also end up in the null space when $\mathbf{A}$ is applied to it.

Specifically, if $\mathbf{x}_1$ is a special solution and $\lambda_1 \mathbf{x}^0_1 + \lambda_2 \mathbf{x}^0_2 + \dots$ is a vector in the null space then

$$
\mathbf{x} = \mathbf{x}_1 + ( \lambda_1 \mathbf{x}^0_1 + \lambda_2 \mathbf{x}^0_2 + \dots )
$$

is **also a solution** because

$$
\mathsf{C} \mathbf{x} = \mathsf{C} \mathbf{x^0} + \mathsf{C} ( \lambda_1 \mathbf{x}^0_1 + \lambda_2 \mathbf{x}^0_2 + \dots ) = \mathsf{C} \mathbf{x^0} + 0 = \mathbf{b}_1 + 0 = \mathbf{b}_1
$$

The $\lambda_i$ are arbitrary real numbers and hence there is an infinite number of solutions.

In SVD:

* The columns $U_{\cdot, i}$ of $\mathsf{U}$ (i.e. `U.T[i]` or `U[:, i]`) corresponding to non-zero $w_i$, i.e. $\{i : w_i \neq 0\}$, form the basis for the _range_ of the matrix $\mathsf{A}$.
* The columns $V_{\cdot, i}$ of $\mathsf{V}$ (i.e. `V.T[i]` or `V[:, i]`) corresponding to zero $w_i$, i.e. $\{i : w_i = 0\}$, form the basis for the _null space_ of the matrix $\mathsf{A}$.

```python
x1
```

    array([-0.34365138,  1.4291518 , -1.4291518 ])

The rank space comes from $\mathsf{U}^T$:

```python
U.T
```

    array([[ -7.07106782e-01,  -4.99999999e-01,  -4.99999999e-01],
           [  7.07106780e-01,  -5.00000001e-01,  -5.00000001e-01],
           [ -2.47010760e-16,  -7.07106781e-01,   7.07106781e-01]])

The basis vectors for the rank space (``~ bool_array`` applies a logical ``NOT`` operation to the entries in the boolean array so that we can pick out "not singular values"):

```python
U.T[~singular_values]
```

    array([[-0.70710678, -0.5       , -0.5       ],
           [ 0.70710678, -0.5       , -0.5       ]])

The null space comes from $\mathsf{V}^T$:

```python
VT
```

    array([[-0.8660254 , -0.35355339,  0.35355339],
           [-0.5       ,  0.61237244, -0.61237244],
           [-0.        , -0.70710678, -0.70710678]])

The basis vector for the null space:

```python
VT[singular_values]
```

    array([[-0.        , -0.70710678, -0.70710678]])

The component of $\mathbf{x}_1$ along the basis vector of the null space of $\mathsf{C}$ (here a 1D space) – note that this component is zero, i.e., the special solution lives in the rank space:

```python
x1.dot(VT[singular_values][0])
```

    2.2204460492503131e-16

We can create a family of solutions by adding vectors in the null space to the special solution $\mathbf{x}_1$, e.g. $\lambda_1 = 2$:

```python
lambda_1 = 2
x1_1 = x1 + lambda_1 * VT[2]
print(x1_1)

np.allclose(C.dot(x1_1), b1)
```

    [-0.34365138  0.01493824 -2.84336536]

    True

Thus, **all** solutions are
```
x1 + lambda * VT[2]
```

#### Solution for  $\mathbf{b}_2$

The solution vector $x_2$ solves
$$
\mathsf{C}\mathbf{x}_2 = \mathbf{b}_2
$$

```python
b2
```

    array([0, 1, 1])

```python
x2 = C_inv.dot(b2)
print(x2)
print(C.dot(x2))
np.allclose(C.dot(x2), b2)
```

    [ 0.9330127  -0.43559574  0.43559574]
    [ -4.44089210e-16   1.00000000e+00   1.00000000e+00]

    True

... and the general solution will again be obtained by adding any multiple of the null space basis vector.

#### Null space

The Null space is spanned by the following basis vectors (just one in this example):

```python
null_basis = VT[singular_values]
null_basis
```

    array([[-0.        , -0.70710678, -0.70710678]])

Show that
$$
\mathsf{C}\mathbf{x}^0 = 0
$$

```python
C.dot(null_basis.T)
```

    array([[  0.00000000e+00],
           [ -6.93889390e-18],
           [ -6.93889390e-18]])

## SVD for fewer equations than unknowns
$N$ equations for $M$ unknowns with $N < M$:

* no unique solutions (underdetermined)
* $M-N$ dimensional family of solutions
* SVD: at least $M-N$ zero or negligible $w_j$: columns of $\mathsf{V}$ corresponding to singular $w_j$ span the solution space when added to a particular solution.

Same as the above [**Solving ill-conditioned coupled linear equations**](#Solving-ill-conditioned-coupled-linear-equations).

## SVD for more equations than unknowns
$N$ equations for $M$ unknowns with $N > M$:

* no exact solutions in general (overdetermined)
* but: SVD can provide best solution in the least-square sense
  $$
  \mathbf{x} = \mathsf{V}\, \text{diag}(1/w_j)\, \mathsf{U}^{T}\, \mathbf{b}
  $$
  where

  * $\mathbf{x}$ is a $M$-dimensional vector of the unknowns (parameters of the fit),
  * $\mathsf{V}$ is a $M \times N$ matrix
  * the $w_j$ form a square $M \times M$ matrix,
  * $\mathsf{U}$ is a $M \times N$ matrix (and $\mathsf{U}^T$ is a $N \times M$ matrix), and
  * $\mathbf{b}$ is the $N$-dimensional vector of the given values (data)

It can be shown that $\mathbf{x}$ minimizes the residual

$$
\mathbf{r} := |\mathsf{A}\mathbf{x} - \mathbf{b}|.
$$

where the matrix $\mathsf{A}$ will be described below and will contain the evaluation of the fit function for each data point in $\mathbf{b}$.

(For a $N \le M$, one can find $\mathbf{x}$ so that $\mathbf{r} = 0$ – see above.)

(In the following, we will switch notation and denote the vector of $M$ unknown parameters of the model as $\mathbf{a}$; this $\mathbf{a}$ corresponds to $\mathbf{x}$ above. $N$ is the number of observations.)

### Linear least-squares fitting

This is the *liner least-squares fitting problem*: Given $N$ data points $(x_i, y_i)$ (where $1 \le i \le N$), fit to a linear model $y(x)$, which can be any linear combination of $M$ functions of $x$.

For example, if we have $N$ functions $x^k$ with parameters $a_k$
$$
y(x) = a_1 + a_2 x + a_3 x^2 + \dots + a_M x^{M-1}
$$
or in general
$$
y(x) = \sum_{k=1}^M a_k X_k(x)
$$

The goal is to determine the $M$ coefficients $a_k$.

Define the **merit function**
$$
\chi^2 = \sum_{i=1}^N \left[ \frac{y_i - \sum_{k=1}^M a_k X_k(x_i)}{\sigma_i}\right]^2
$$
(sum of squared deviations, weighted with standard deviations $\sigma_i$ on the $y_i$).

Best parameters $a_k$ are the ones that *minimize $\chi^2$*.

*Design matrix* $\mathsf{A}$ ($N \times M$, $N \geq M$), vector of measurements $\mathbf{b}$ ($N$-dim) and parameter vector $\mathbf{a}$ ($M$-dim):

\begin{align}
A_{ij} &= \frac{X_j(x_i)}{\sigma_i}\\
b_i &= \frac{y_i}{\sigma_i}\\
\mathbf{a} &= (a_1, a_2, \dots, a_M)
\end{align}

The design matrix $\mathsf{A}$ contains the *predicted* values from the basis functions for all values $x_i$ of the independent variable $x$ for which we have measured data $y_i$.

Minimum occurs when the derivative vanishes:
$$
0 = \frac{\partial\chi^2}{\partial a_k} = \sum_{i=1}^N {\sigma_i}^{-2} \left[ y_i - \sum_{j=1}^M a_j X_j(x_i) \right] X_k(x_i), \quad 1 \leq k \leq M
$$
($M$ coupled equations)

To simplify the notation, define the $M \times M$ matrix
\begin{align}
\alpha_{kj} &= \sum_{i=1}^N \frac{X_k(x_i) X_j(x_i)}{\sigma_i^2}\\
\mathsf{\alpha} &= \mathsf{A}^T \mathsf{A}
\end{align}
and the vector of length $M$
\begin{align}
\beta_{k} &= \sum_{i=1}^N \frac{y_i X_k(x_i)}{\sigma_i^2}\\
\boldsymbol{\beta} &= \mathsf{A}^T \mathbf{b}
\end{align}

Then the $M$ coupled equations can be compactly written as
\begin{align}
\sum_{j=1}^{M} \alpha_{kj} a_j &= \beta_k\\
\mathsf{\alpha}\mathbf{a} = \boldsymbol{\beta}
\end{align}

$\mathsf{\alpha}$ and  $\boldsymbol{\beta}$ are known, so we have to solve this matrix equation for the vector of the unknown parameters $\mathbf{a}$.

#### Error estimates for the parameters

The inverse of $\mathsf{\alpha}$ is related to the uncertainties in the parameters:
$$
\mathsf{C} := \mathsf{\alpha}^{-1}
$$
in particular
$$
\sigma(a_i) = C_{ii}
$$
(and the $C_{ij}$ are the co-variances).

#### Solution of the linear least-squares fitting problem with SVD
We need to solve the overdetermined system of $M$ coupled equations
\begin{align}
\sum_{j=1}^{M} \alpha_{kj} a_j &= \beta_k\\
\mathsf{\alpha}\mathbf{a} = \boldsymbol{\beta}
\end{align}

SVD finds $\mathbf{a}$ that minimizes
$$
\chi^2 = |\mathsf{A}\mathbf{a} - \mathbf{b}|
$$

The errors are
$$
\sigma^2(a_j) = \sum_{i=1}^{M} \left(\frac{V_{ji}}{w_i}\right)^2
$$

#### Example
Synthetic data

$$
y(x) = 3\sin x - 2\sin 3x + \sin 4x
$$

with noise $r$ added (uniform in range $-5 < r < 5$).

```python
import matplotlib
import matplotlib.pyplot as plt
%matplotlib inline
matplotlib.style.use('ggplot')

import numpy as np
```

```python
def signal(x, noise=0):
    r = np.random.uniform(-noise, noise, len(x))
    return 3*np.sin(x) - 2*np.sin(3*x) + np.sin(4*x) + r
```

```python
X = np.linspace(-10, 10, 500)
Y = signal(X, noise=5)
```

```python
plt.plot(X, Y, 'r-', X, signal(X, noise=0), 'k--')
```

Define our fit function (the model) and the basis functions. We need the basis functions for setting up the problem and we will later use the fitfunction together with our parameter estimates to compare our fit to the true underlying function.

```python
def fitfunc(x, a):
    return a[0]*np.cos(x) + a[1]*np.sin(x) + \
           a[2]*np.cos(2*x) + a[3]*np.sin(2*x) + \
           a[4]*np.cos(3*x) + a[5]*np.sin(3*x) + \
           a[6]*np.cos(4*x) + a[7]*np.sin(4*x)

def basisfuncs(x):
    return np.array([np.cos(x), np.sin(x),
                     np.cos(2*x), np.sin(2*x),
                     np.cos(3*x), np.sin(3*x),
                     np.cos(4*x), np.sin(4*x)])
```

(Note that we could have used the `basisfuncs()` in `fitfunc()` – left as an exercise for the keen reader...)

Set up the $\mathsf{\alpha}$ matrix and the $\boldsymbol{\beta}$ vector (here we assume that all observations have the same error $\sigma = 1$):

```python
M = 8
sigma = 1.
alpha = np.zeros((M, M))
beta = np.zeros(M)
for x in X:
    Xk = basisfuncs(x)
    for k in range(M):
        for j in range(M):
            alpha[k, j] += Xk[k]*Xk[j]
for x, y in zip(X, Y):
    beta += y * basisfuncs(x)/sigma
```

Finally, solving the problem follows the same procedure as before:

Get the SVD:

```python
U, w, VT = np.linalg.svd(alpha)
V = VT.T
```

In this case, the singular values do not immediately show if any basis functions are superfluous (this would be the case for values close to 0).

```python
w
```

    array([ 296.92809624,  282.94804954,  243.7895787 ,  235.7300808 ,
            235.15938555,  235.14838812,  235.14821093,  235.14821013])

... nevertheless, remember to routinely mask any singular values or close to singular values:

```python
w_inv = 1/w
w_inv[np.abs(w) < 1e-12] = 0
alpha_inv = V.dot(np.diag(w_inv)).dot(U.T)
```

Solve the system of equations with the pseudo-inverse:

```python
a_values = alpha_inv.dot(beta)
print(a_values)
```

    [ 0.02941343  3.15273275  0.22893881  0.14290046  0.30121258 -2.04230627
      0.28692984  1.08197408]

Compare the fitted values to the original parameters $a_j = 0, +3, 0, 0, 0, -2, 0, +1$.

The original parameters show up as 3.15, -2.04 and 1.08 but the other parameters also have appreciable values. Given that the noise was sizable, this is not unreasonable.

Compare the plot of the underlying true function ("signal", dashed line) to the model ("fit", solid line):

```python
plt.plot(X, fitfunc(X, a_values), 'b-', label="fit")
plt.plot(X, signal(X, noise=0), 'k--', label="signal")
plt.legend(loc="best", fontsize="small")
```

We get some spurious oscillations but overall the result looks reasonable.

```python

```


```python
from sympy import *         # import sympy library into our development environment
x = symbols('x')            # x  becomes a non-type sympy symbols that can be substited for python objects later
f = sin(x)**tan(x)          # storing our function as variable f
f                           # outputting our function in LaTex
```

$\displaystyle \sin^{\tan{\left(x \right)}}{\left(x \right)}$

```python
from sympy.plotting import plot3d
for i in range(0,6,1):       # range(start, finish, steps)
    if i != 0:
        plot3d(f, (x, 0, i), (y, 0, i), title = i)
```

```python
for i in range(0,6,1):       # range(start, finish, steps)
    if i != 0:
        plot3d(-f, (x, 0, i), (y, 0, i), title = i)   # inversed the loot by making our function negative
```

```python

```


# Parameter Estimation for Linear Dynamical Systems
This notebook details an algorithm for estimating the parameters of a linear dynamical system using the Expectation Maximization (EM) algorithm.

Given a set of observations (measurements), this offline algorithm iterates between the E-step that fixes the current parameters and computes posterior probabilities over the hidden states and an M-step that maximizes the expected log-likelihood of the parameters using the posterior distribution computed in the E-step.

For linear Gaussian systems, the E-step is the Kalman smoother and the M-step is a linear regression problem.

## Model
The state space is defined by the following:
$$ \begin{align*} x_{t+1} &= A x_t + w_t \\
y_t &= C x_t + v_t\end{align*}$$
where $w_t\sim \mathcal{N}(0,Q)$ is the process noise and $v_t\sim \mathcal{N}(0,R)$ is the measurement noise. It is assumed that $x_t \in \mathbb{R}^k$ and $y_t \in \mathbb{R}^p$

It is assumed that only the output $y_t$ is measured and the state variables and noise are hidden.

To simplify the analysis, we combine the state variable with the process noise and measurement with its noise into single Gaussian random variables.

$$ \begin{align*} P(x_{t+1}|x_t) &= \exp{\{-\frac{1}{2} \left[x_{t+1}-A x_t\right]^T Q^{-1} \left[x_{t+1}-A x_t\right]\}} (2 \pi)^{-\frac{k}{2}}|Q|^{-\frac{1}{2}} \\
P(y_{t}|x_t) &= \exp{\{-\frac{1}{2} \left[y_{t}-C x_t\right]^T R^{-1} \left[y_{t}-C x_t\right]\}} (2 \pi)^{-\frac{p}{2}}|R|^{-\frac{1}{2}} \end{align*}$$

## Kalman Smoother Estimator
The Kalman Smoother Estimator is an algorithm that seeks to estimate the trajectory of the latent state variables given a sequence of measurements. The state evolution and the output equations are modeled by the state space equation described above.

The optimal estimator $x_t^n = \mathbb{E}(x_t|y_1,y_2,...,y_n)$ by minimizing the mean squared error $P_t^n = \mathbb{E}((x_t-x_t^n)^T (x_t-x_t^n) | y_1,y_2,...,y_n)$.

### Forward Recursion
The forward recursion follows the Kalman Filter equations where the state estimate is updated based on current measurement.
#### Prediction
1. Predicted State Estimate: $x_{t|t-1} = A x_{t-1|t-1}$
2. Predicted Covariance Estimate: $V_{t|t-1} = A V_{t-1|t-1} A^T + Q $

#### Update
1. Measurement Residual: $z_t = y_{t} - C x_{t|t-1}$
2. Innovation Covariance: $S_t = C V_{t|t-1} C^T + R$
3. Optimal Kalman Gain: $K_t = P_{t|t-1} C^T S_t^{-1}$
4. Updated State Estimate: $x_{t|t} = x_{t|t-1} + K_t z_t$
5. Covariance Update: $V_{t|t} = (I-K_t C) V_{t|t-1}$

### Backward Recursion
The backward recursion performs another update to the state estimates based on future measurements.

Starting from time $t=n$, with $x_{n|n}$ & $V_{n|n}$ obtained from the Kalman filter:
1. Define $J_{t-1} = V_{t-1|t-1} A V_{t|t-1}^{-1}$
2. $x_{t-1|n} = x_{t-1|t-1} + J_{t-1}(x_{t|n}-A x_{t-1|t-1})$
3. $V_{t-1|n} = V_{t-1|t-1} + J_{t-1}(V_{t|n}-V_{t|t-1})J_{t-1}^T$

---

A useful quantity that is not used in the Kalman smoother, but is used in the EM algorithm is $V_{t,t-1}$. This can be computed during the backward recursion for convenience.
Starting from time $t=n$, with $V_{n,n-1|n} = (I-K_n C)A V_{n-1|n-1}$:
$$V_{i-1,i-2|n} = V_{i-1|i-1}J_{i-2}^T + J_{i-1}(V_{i,i-1|n}-A V_{i-1|i-1})J_{i-2}^T$$

## EM Algorithm
Assuming Markov property, and defining ${x} \equiv {x_1,x_2,...,x_n}$ & ${y} \equiv {y_1,y_2,...,y_n}$
$$ \begin{align} P(\{x\},\{y\}) = &P(x_0) \prod_{t=0}^{n-1} P(x_{t+1}|x_t) \prod_{t=0}^n P(y_t|x_t) \\
\Rightarrow \log{P(\{x\},\{y\})} = &-\sum_{t=1}^n{\frac{1}{2} \left[x_{t+1}-A x_t\right]^T Q^{-1} \left[x_{t+1}-A x_t\right]} \\
 &-\sum_{t=0}^n{\frac{1}{2} \left[y_{t}-C x_t\right]^T R^{-1} \left[y_{t}-C x_t\right]}  \\
 &-\frac{1}{2} \left[x_{0}- \pi_0\right]^T V_0^{-1} \left[x_{0}- \pi_0\right]  \\
 &-\frac{n-1}{2}\log{|Q|} - \frac{n}{2}\log{|R|} - \frac{n(p+k)}{2}\log{2 \pi}\end{align}$$
 where $P(x_0) \sim \mathcal{N}(\pi_0,V_0)$ with $\pi_0$ & $V_0$ to be estimated.

### E Step
The E-step of the algorithm requires computing the expected log likelihood $\mathcal{Q} = \mathbb{E}\left[\log P(\{x\},\{y\})|\{y\} \right]$ (Note: Its $\mathbb{E}\left(\log P(\cdot)|\{y\}\right)$).

To compute the above Expectation, we define:
$$\begin{align} \hat{x}_t &= \mathbb{E}(x_t|{y}) = x_{t|n}\\
P_t &= \mathbb{E}(x_t x_t^T|{y}) = V_{t|n} + x_{t|n} x_{t|n}^T \\
P_{t,t-1} &= \mathbb{E}(x_t x_{t-1}^T|{y}) = V_{t,t-1|n} + x_{t|n} x_{t-1|n}^T \end{align} $$
 and can be computed as part of the Kalman smoother estimator described above.

### M Step
The parameters of this system are $A$ , $C$ , $Q$ , $R$ , $\pi_0$ , $V_0$. Each of this can be computed by setting the corresponding partial derivative of the expected log likelihood to zero.

#### Output Matrix:
$$ \begin{align}\frac{\partial \mathcal{Q}}{\partial C} &= \sum_{t=0}^n R^{-1} \left[y_t - C x_t \right] (-x_t)^T \\
&= \sum_{t=0}^n \Bigg( R^{-1} \left[ y_t x_t^T - C P_t\right]\Bigg) \\
&= 0 \iff C_{\text{new}} = \Bigg(\sum_{t=0}^n y_t x_t^T \Bigg) \Bigg(\sum_{t=0}^n P_t \Bigg)^{-1}\end{align}$$

#### Output Noise Covariance:
Here we make use of the following property: $\frac{\partial{\log|R|}} {\partial{R^{-1}}} = R$
$$ \begin{align}\frac{\partial \mathcal{Q}}{\partial R^{-1}} &= \frac{n}{2} R -\sum_{t=0}^n{\left[ \frac{y_t y_t^T}{2} - C x_t y_t^T + \frac{Cx_t x_t^T C^T}{2}\right]} \\ &= \frac{n}{2} R -\sum_{t=0}^n{\left[ \frac{y_t y_t^T}{2} - C x_t y_t^T + \frac{C P_t C^T}{2}\right]}\\ &= \frac{n}{2} R -\sum_{t=0}^n{\left[ \frac{y_t y_t^T}{2} - \frac{C x_t y_t^T}{2} - \frac{C}{2} (x_t y_t^T - P_t C^T)\right]} \\
&= \frac{n}{2} R -\sum_{t=0}^n{\left[ \frac{y_t y_t^T}{2} - \frac{C x_t y_t^T}{2} \right] }\\
&= 0 \iff R_{\text{new}} =  \frac{1}{n} \left[\sum_{t=0}^n{ \left(y_t y_t^T - C_{\text{new}} x_t y_t^T\right)} \right] \end{align}$$

#### State Transition Matrix:
$$ \begin{align}\frac{\partial \mathcal{Q}}{\partial A} &= \sum_{t=0}^{n-1} Q^{-1} \left[x_{t+1} - A x_t \right] (-x_t)^T \\
&= \sum_{t=0}^{n-1} \Bigg( Q^{-1} \left[ x_{t+1} x_t^T - A P_t\right]\Bigg) \\
&= 0 \iff A_{\text{new}} = \Bigg(\sum_{t=0}^{n-1} P_{t+1,t} \Bigg) \Bigg(\sum_{t=0}^{n-1} P_t \Bigg)^{-1}\end{align}$$

#### Process Noise Covariance:
$$ \begin{align}\frac{\partial \mathcal{Q}}{\partial Q^{-1}} &= \frac{n-1}{2}Q -\sum_{t=0}^{n-1}{\left[ \frac{x_{t+1} x_{t+1}^T}{2} - \frac{A x_t x_{t+1}^T}{2}  - \frac{x_{t+1}x_t^T A^T}{2} + \frac{Ax_t x_t^T A^T}{2}\right]} \\ &= \frac{n-1}{2}Q - \frac{1}{2}\sum_{t=0}^{n-1}{\left[ P_{t+1} - A P_{t,t+1}  - P_{t+1,t} A^T + A P_t A^T\right]} \\
&= \frac{n-1}{2}Q - \frac{1}{2}\sum_{t=0}^{n-1}{\left[ P_{t+1} - A P_{t,t+1}  - (P_{t+1,t} - A P_t) A^T\right]} \\
&= \frac{n-1}{2}Q - \frac{1}{2}\sum_{t=0}^{n-1}{\left[ P_{t+1} - A P_{t,t+1} \right]} \\
&= 0 \iff Q_{\text{new}} = \frac{1}{n-1} \left(\sum_{t=0}^{n-1}{P_{t+1} }- A_{\text{new}}\sum_{t=0}^{n-1}{P_{t,t+1}}\right)\end{align}$$

#### Initial State Mean
$$ \begin{align}\frac{\partial \mathcal{Q}}{\partial \pi_0} &= (x_0-\pi_0)V_0^{-1} \\
&= 0 \iff \pi_{0,\text{new}} = x_0 \end{align}$$

#### Initial State Covariance
$$ \begin{align}\frac{\partial \mathcal{Q}}{\partial V_{-1}} &= \frac{V_1}{2} - \frac{1}{2}(P_0-x_0\pi_0^T-\pi_0x_p^T+\pi_0\pi_0^T) \\
&= 0 \iff V_{0,\text{new}} = P_0 - x_0 x_0^T \end{align}$$

## References
1. Ghahramani, Z., & Hinton, G. E. (1996a). Parameter estimation for linear dynamical systems (Tech. Rep. No. CRG-TR-96-2). Toronto: Department of Computer Science, University ofToronto. Available [online](http://www.gatsby.ucl.ac.uk/»zoubin/papers/tr-96-2.ps.gz).
2. Shumway, R. H., & Stoffer, D. S. (1982). An approach to time series smoothing and forecasting using the EM algorithm. J. Time Series Analysis, 3(4), 253–264.


```python
# https://socratic.org/questions/a-fence-8-ft-tall-runs-parallel-to-a-tall-building-at-a-distance-of-4-ft-from-th-1
from IPython.display import Image
from IPython.core.display import HTML
from sympy import *; x,h,y,n,t,d,c,A,r   = symbols("x h y n t d c A r"); C, D = symbols("C D", real=True)
Image(url= "https://i.imgur.com/XYvZXIq.png")
```

```python
Image(url= "https://i.imgur.com/2DeA9ZT.png")
# change 4 to 5
```

```python
Image(url= "https://i.imgur.com/um3ZnlW.png")
```

```python
h = factor(8*(5+x))/x
h
```

$\displaystyle \frac{8 \left(x + 5\right)}{x}$

```python
L = h**2+(x+5)**2
Eq(symbols("L")**2,L)
```

$\displaystyle L^{2} = \left(x + 5\right)^{2} + \frac{64 \left(x + 5\right)^{2}}{x^{2}}$

```python
diff(L)
```

$\displaystyle 2 x + 10 + \frac{64 \left(2 x + 10\right)}{x^{2}} - \frac{128 \left(x + 5\right)^{2}}{x^{3}}$

```python
solve(diff(L),x)
```

    [-5,
     4*5**(1/3),
     -2*5**(1/3) - 2*sqrt(3)*5**(1/3)*I,
     -2*5**(1/3) + 2*sqrt(3)*5**(1/3)*I]

```python
L.subs(x, 4*5**(1/3))
```

$\displaystyle 331.951408234819$

```python
sqrt(L.subs(x, 4*5**(1/3)))
```

$\displaystyle 18.2195336997087$

```python
Image(url= "https://i.imgur.com/yafiIYG.png")
```


```python
from sympy import *
init_printing(use_latex='mathjax')
from IPython.display import display
```

```python
delta = Symbol("Delta")
x = Symbol("x")
y = Symbol("y")
V = Symbol("V")
H0 = Matrix([[-delta/2,0],[0,delta/2]])
V = Matrix([[0,V],[V,0]])
H = H0 + V
```

```python
display(H)
```

$$\left[\begin{matrix}- \frac{\Delta}{2} & V\\V & \frac{\Delta}{2}\end{matrix}\right]$$

```python
evecs = H.eigenvects()
eigvals = [evecs[0][0],evecs[1][0]]
eigvecs = [evecs[0][2],evecs[1][2]]
```

```python
eigvals
```

$$\left [ - \frac{1}{2} \sqrt{\Delta^{2} + 4 V^{2}}, \quad \frac{1}{2} \sqrt{\Delta^{2} + 4 V^{2}}\right ]$$

```python
eigvecs
```

$$\left [ \left [ \left[\begin{matrix}- \frac{V}{- \frac{\Delta}{2} + \frac{1}{2} \sqrt{\Delta^{2} + 4 V^{2}}}\\1\end{matrix}\right]\right ], \quad \left [ \left[\begin{matrix}- \frac{V}{- \frac{\Delta}{2} - \frac{1}{2} \sqrt{\Delta^{2} + 4 V^{2}}}\\1\end{matrix}\right]\right ]\right ]$$

```python

```


# [Naive Bayes from scratch](https://philippmuens.com/naive-bayes-from-scratch/)

# Bayes Theorem Example
#### Problem
- What's the probability that it'll be sunny throughout the day given that we saw clouds? Given
    + The chance to see clouds is: 60%
    + Only 7 out of 30 days can be considered sunny
    + 50% of the sunny days started out with clouds

#### Solution
- P(Cloud) = 0.6
- P(Sun) = 0.23
- P(Cloud | Sun) = 0.5
- Probability that it'll be sunny throughout the day given that we saw clouds:
    + P(Sun | Cloud) = $\frac{P(Cloud\ |\ Sun)\ *\ P(Sun)}{P(Cloud)} = \frac{0.5*0.23}{0.6} = 19.17 \%$

# Bayes Theorem and spammy words
## 1. Detect if an message given a word is a spam mail?
- Assume that we're working with the word "prize" and want to figure out what the probability is that a given message which contains the word "prize" is spam
- Formulating this problem via Bayes Theorem: $P(Spam\ |\ Prize)\ =\ \frac{P(Prize\ |\ Spam)\ *\ P(Spam)}{P(Prize)}$
    - $P(Spam\ |\ Prize)$: probability of a message being spam given that we see the word "Prize" in it
    - $P(Prize\ |\ Spam)$: the probability of finding the word "Prize" in our already seen spam messages
    - $P(Spam)$: the probability of the message being spam
    - $P(Prize)$: probability of finding the word "Prize" in any of our already seen messages

- Expanded Bayes Theorem: $P(Spam\ |\ Prize)\ =\ \frac{P(Prize\ |\ Spam)\ *\ P(Spam)}{P(Prize\ ∣\ Spam)\ *\ P(Spam)\ +\ P(Prize\ ∣\ Ham)\ *\ P(Ham)}$
- Initially we dont know the the probability of the message being spam or ham: $P(Spam)\ =\ P(Ham)\ =\ 0.5$

$$\begin{equation}
    \begin{split}
        P(Spam\ |\ Prize)\ &=\ \frac{P(Prize\ |\ Spam)\ *\ 0.5}{P(Prize\ ∣\ Spam)\ *\ 0.5\ +\ P(Prize\ ∣\ Ham)\ *\ 0.5} \\
            & = \frac{P(Prize\ |\ Spam)}{P(Prize\ ∣\ Spam)\ +\ P(Prize\ ∣\ Ham)}
    \end{split}
\end{equation}$$

## 2. Scaling it up to whole sentences
- A naive approach = insert the whole message as the conditional probability part
    - A sentence = a succession of single words
    + Apply single-word model to every word we come across in our message
    + Computing the overall probability for the whole message = multiplication of all those individual probabilities

- Example:
    $$\begin{split}
        P(Message\ |\ Spam)\ &=\ P(word_1, word_2, ..., word_n\ |\ Spam) \\
                            &=\ P(word_1\ |\ Spam)\ *\ P(word_2\ |\ Spam)\ *\ ...\ *\ P(word_n\ |\ Spam)
    \end{split}$$

## 3. Difficulties
- Difficulty 1: Vanishing Probility in computation
    + Multiply a lot of small probabilities
    + Computer architectures are only capable to deal with a certain amount of precision
        $\to$ "arithmetic underflow": eventually turn our number into a 0
    - Trick:
        + $log(a\ *\ b) = log(a)\ +\ log(b)$
        + $exp(log(a\ *\ b)) = a\ *\ b$

- Difficulty 2: Vanishing Probility in unseen words
    + If we've never seen the word W in a spam message, then P(W ∣ Spam) = 0
    - Trick: Introduce factor k
        + $P(W\ |\ S)\ =\ \frac{k\ +\ \text{spam that containing W}}{(2\ *\ k)\ +\ \text{total spam}}$
        + Example: 100 spam examples but found the word W 0 times
            + k = 0: No factor
                + $P(W\ |\ S)\ =\ \frac{0}{100}\ = 0$
            + k = 1
                + $P(W\ |\ S)\ =\ \frac{k\ +\ 0}{(2*k)\ +\ 100}\ = 0.001$

## 4. Implementation
- We need to "train" our Naive Bayes classifier with training data

### 1. Dataset

#### Download

```bash
%%bash
if [[ ! -d "data/enron1" ]]; then
    wget -nc \
        -P data \
        http://nlp.cs.aueb.gr/software_and_datasets/Enron-Spam/preprocessed/enron1.tar.gz 2> /dev/null
    tar -xzf data/enron1.tar.gz -C data 2> /dev/null
fi
```

```bash
%%bash
ls data/enron1
```

    ham
    spam
    Summary.txt

```bash
%%bash
ls data/enron1/ham | head -5
```

    0001.1999-12-10.farmer.ham.txt
    0002.1999-12-13.farmer.ham.txt
    0003.1999-12-14.farmer.ham.txt
    0004.1999-12-14.farmer.ham.txt
    0005.1999-12-14.farmer.ham.txt

```bash
%%bash
ls data/enron1/spam | head -5
```

    0006.2003-12-18.GP.spam.txt
    0008.2003-12-18.GP.spam.txt
    0017.2003-12-18.GP.spam.txt
    0018.2003-12-18.GP.spam.txt
    0026.2003-12-18.GP.spam.txt

#### Preprocessing data

```python
from glob import glob
from typing import List

spam_message_paths: List[str] = glob('data/enron1/spam/*.txt')
ham_message_paths: List[str] = glob('data/enron1/ham/*.txt')
message_paths: List[str] = spam_message_paths + ham_message_paths

class Message():
    def __init__(self, text: str, is_spam: bool) -> None:
        self.text: str = text
        self.is_spam: bool = is_spam

messages: List[Message] = []
for path in message_paths:
    with open(path, errors='ignore') as file:
        is_spam: bool = True if 'spam' in path else False
        text: str = ''
        for line in file.readlines():
            text += line.replace('Subject:', '').strip()
            text += ' '
        messages.append(
            Message(text, is_spam))
```

#### Stats

```python
len(messages)
```

    5172

```python
spam_messages = 0
for message in messages:
    if message.is_spam == True: spam_messages += 1
print("Number of spam messages: {}".format(spam_messages))

print('''---- Example spam ----
{}'''.format(messages[0].text))
```

    Number of spam messages: 1500
    ---- Example spam ----
    we sell regalis for an affordable price hi , regalis , also known as superviagra or cialis - half a pill lasts all weekend - has less sideeffects - has higher success rate now you can buy regalis , for over 70 % cheaper than the equivilent brand for sale in us we ship world wide , and no prescription is required ! ! even if you ' re not impotent , regalis will increase size , pleasure and power ! try it today you wont regret ! get it here : http : / / koolrx . com / sup / best regards , jeremy stones no thanks : http : / / koolrx . com / rm . html

```python
ham_messages = 0
for message in messages:
    if message.is_spam == False: ham_messages += 1
print("Number of ham messages: {}".format(spam_messages))

print('''---- Example ham ----
{}'''.format(messages[1505].text))
```

    Number of ham messages: 1500
    ---- Example ham ----
    re : killing ena to ena deals in sitara jay , if a deal is killed it poses a problem for us in unify if there are any paths associated with the deal ; therefore , we request the deals be zeroed out . call me if this is a problem . also , we would appreciate further details on why these deals are being killed . in addition , i have copied rita and mark from volume management for their input . regards , tammy x 35375 - - - - - original message - - - - - from : pena , matt sent : thursday , december 13 , 2001 3 : 39 pm to : krishnaswamy , jayant ; pinion , richard ; jaquet , tammy cc : severson , russ ; truong , dat ; aybar , luis ; ma , felicia subject : re : killing ena to ena deals in sitara thanks jay ! tammy / richard : you may want to let the schedulers know , although they may already . - - - - - original message - - - - - from : krishnaswamy , jayant sent : thursday , december 13 , 2001 3 : 38 pm to : pinion , richard ; jaquet , tammy cc : severson , russ ; pena , matt ; truong , dat ; aybar , luis ; ma , felicia subject : killing ena to ena deals in sitara richars / tammy : we will be killing about 2000 deals in sitara tonight . whenever a deal is touched in sitara , it will bridge over to unify . these are desk 2 desk deals , and should have minimal impact on you .

#### train_test_split

```python
from typing import List, Tuple
from random import shuffle

def train_test_split(messages: List[Message], pct=0.8) -> Tuple[List[Message], List[Message]]:
    shuffle(messages)
    num_train = int(round(len(messages) * pct, 0))
    return (messages[:num_train], messages[num_train:])
```

```python
train: List[Message]
test: List[Message]

train, test = train_test_split(messages, 0.85)
print(len(train))
print(len(test))
```

    4396
    776

### 2. Model
#### Tokenize
- Input: a raw message
- Output: a list of words

```python
import re

def tokenize(text: str) -> set():
    words = []
    for word in re.findall(r'[A-Za-z0-9\']+', text):
        words.append(word.lower())
    return set(words)
```

```python
assert tokenize(
    'Is this a text? If so, Tokenize this text!...') == \
    {'is', 'this', 'a', 'text', 'if', 'so', 'tokenize'}

tokenize(
    'Is this a text? If so, Tokenize this text!...')
```

    {'a', 'if', 'is', 'so', 'text', 'this', 'tokenize'}

#### Naive Bayes model

```python
from collections import defaultdict
from typing import Dict, List, Set
from math import log, exp

class NaiveBayes:
    def __init__(self, k=1) -> None:
        self._k: int = k
        self._num_spam_messages: int = 0
        self._num_ham_messages: int = 0
        self._num_word_in_spam: Dict[int] = defaultdict(int)
        self._num_word_in_ham: Dict[int] = defaultdict(int)
        self._spam_words: Set[str] = set()
        self._ham_words: Set[str] = set()
        self._words: Set[str] = set()

    def train(self, messages: List[Message]) -> None:
        for msg in messages:
            tokens: Set[str] = tokenize(msg.text)
            self._words.update(tokens)
            if msg.is_spam:
                self._num_spam_messages += 1
                self._spam_words.update(tokens)
                for token in tokens:
                    self._num_word_in_spam[token] += 1
            else:
                self._num_ham_messages += 1
                self._ham_words.update(tokens)
                for token in tokens:
                    self._num_word_in_ham[token] += 1

    def _p_word_spam(self, word: str) -> float:
        return (self._k + self._num_word_in_spam[word]) / ((2 * self._k) + self._num_spam_messages)

    def _p_word_ham(self, word: str) -> float:
        return (self._k + self._num_word_in_ham[word]) / ((2 * self._k) + self._num_ham_messages)

    def predict(self, text: str) -> float:
        text_words: Set[str] = tokenize(text)
        log_p_spam: float = 0.0
        log_p_ham: float = 0.0

        for word in self._words:
            p_spam: float = self._p_word_spam(word)
            p_ham: float = self._p_word_ham(word)
            if word in text_words:
                log_p_spam += log(p_spam)
                log_p_ham += log(p_ham)
            else:
                log_p_spam += log(1 - p_spam)
                log_p_ham += log(1 - p_ham)

        p_if_spam: float = exp(log_p_spam)
        p_if_ham: float = exp(log_p_ham)

        try:
            return p_if_spam / (p_if_spam + p_if_ham)
        except:
            return 1.0
```

### 3. Train and Test

#### Train

```python
nb: NaiveBayes = NaiveBayes()
nb.train(train)
```

#### Test

```python
true_positives: int = 0
true_negatives: int = 0

false_negatives: int = 0
false_positives: int = 0

for cnt, message in enumerate(test, 1):
    if cnt%100 == 0 or cnt == len(test): print("Progress: {}/{}".format(cnt, len(test)))

    scr: int = nb.predict(message.text)
    test_result: bool = True if scr > 0.5 else False

    if test_result == True and message.is_spam == True:
        true_positives += 1
    elif test_result == False and message.is_spam == False:
        true_negatives += 1
    elif test_result == True and message.is_spam == False:
        false_negatives += 1
    elif test_result == False and message.is_spam == True:
        false_positives += 1
```

    Progress: 100/776
    Progress: 200/776
    Progress: 300/776
    Progress: 400/776
    Progress: 500/776
    Progress: 600/776
    Progress: 700/776
    Progress: 776/776

```python
print('Accuracy: {:2f} %'.format((true_positives + true_negatives)*100.0 / len(test)))
```

    Accuracy: 82.345361 %

```python
print('''Confusion matrix:
    {}\t{}
    {}\t{}'''.format(true_positives, false_positives, false_negatives, true_negatives))
```

    Confusion matrix:
        117	93
        44	522


# First Order System in Pyomo

## First-Order Differential Equation with Initial Condition

The following cell implements a solution to a first-order linear model in the form

\begin{align}
\tau\frac{dy}{dt} + y & = K u(t) \\
\end{align}

where $\tau$ and $K$ are model parameters, and $u(t)$ is an external process input.

```python
%matplotlib inline
from pyomo.environ import *
from pyomo.dae import *
import matplotlib.pyplot as plt

from pyomo.common.config import (ConfigDict, ConfigList, ConfigValue, In)

tf = 10
tau = 1
K = 5

# define u(t)
u = lambda t: 0 if t < 5 else 1

# create a model object
model = ConcreteModel()

# define the independent variable
model.t = ContinuousSet(bounds=(0, tf))

# define the dependent variables
model.y = Var(model.t)
model.dydt = DerivativeVar(model.y)

# fix the initial value of y
model.y[0].fix(0)

# define the differential equation as a constraint
model.ode = Constraint(model.t,
    rule=lambda model, t: tau*model.dydt[t] + model.y[t] == K*u(t))

# transform dae model to discrete optimization problem
TransformationFactory('dae.finite_difference').apply_to(model, nfe=50, method='BACKWARD')

# solve the model
SolverFactory('ipopt').solve(model).write()

# access elements of a ContinuousSet object
t = [t for t in model.t]

# access elements of a Var object
y = [model.y[t]() for t in model.y]

plt.plot(t,y)
plt.xlabel('time / sec')
plt.ylabel('response')
plt.title('Response of a linear first-order ODE')
```

```python

```


# SymPy

```python
import sympy as sp
import numpy as np
import matplotlib.pyplot as plt
```

```python
x = sp.symbols('x')
sp.init_printing()
```

---

$$ \Large
{\displaystyle f(x)=3e^{-{\frac {x^{2}}{8}}}} \sin(x/3)
$$

$$\Large
{\displaystyle g(x)=\frac{1}{5}x^{3} + \frac{1}{2}x^{2} + \frac{1}{3}x - \frac{1}{2}}
$$

---

## Show that:

$$ \Large
\left.{\frac {d}{dx}\, f(x)}\ \right|_{\ x\, =\, 0}\ = \ \left.{\frac {d^{2}}{dx^{2}}\, g(x)}\ \right|_{\ x\, =\, 0}
$$

```python

```

```python

```

```python

```

```python

```

## Plot $f(x)$ and $g(x)$ on the same plot over the interval $[-4 \le x \le 2]$

```python

```

```python

```

```python

```

```python

```

## What are the value(s) for $x$ where $f(x)\ = \ g(x)$

```python

```

```python

```

```python

```

---
### Due Wed Mar 2 - 1 pm
- `File -> Download as -> HTML (.html)`
- `upload your .html file to the class Canvas page`

```python

```