## Coordinate System
Coordinate system in 6 Dof

```python
import warnings
#warnings.filterwarnings('ignore')
%matplotlib inline
%load_ext autoreload
%autoreload 2
```

```python
import sympy as sp
import sympy.physics.mechanics as me
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from substitute_dynamic_symbols import substitute_dynamic_symbols, find_name, find_derivative_name, lambdify, find_derivatives

```

```python
phi,theta,psi = sp.symbols('phi theta psi')

p_x = sp.Symbol('p_x')
p_y = sp.Symbol('p_y')
p_z = sp.Symbol('p_z')

```

```python
N = me.ReferenceFrame('N')
```

```python
#M = N.orientnew('M', 'Axis', [psi, N.z])
#O = me.ReferenceFrame('O')
#O.orient(M,'Axis',[phi,M.x])
#
#S = me.ReferenceFrame('S')
#S.orient(O,'Axis',[theta,M.y])

```

```python
S = N.orientnew('S','Body',[psi,theta,phi],'ZYX')
```

```python
S.dcm(N)
```

$\displaystyle \left[\begin{matrix}\cos{\left(\psi \right)} \cos{\left(\theta \right)} & \sin{\left(\psi \right)} \cos{\left(\theta \right)} & - \sin{\left(\theta \right)}\\\sin{\left(\phi \right)} \sin{\left(\theta \right)} \cos{\left(\psi \right)} - \sin{\left(\psi \right)} \cos{\left(\phi \right)} & \sin{\left(\phi \right)} \sin{\left(\psi \right)} \sin{\left(\theta \right)} + \cos{\left(\phi \right)} \cos{\left(\psi \right)} & \sin{\left(\phi \right)} \cos{\left(\theta \right)}\\\sin{\left(\phi \right)} \sin{\left(\psi \right)} + \sin{\left(\theta \right)} \cos{\left(\phi \right)} \cos{\left(\psi \right)} & - \sin{\left(\phi \right)} \cos{\left(\psi \right)} + \sin{\left(\psi \right)} \sin{\left(\theta \right)} \cos{\left(\phi \right)} & \cos{\left(\phi \right)} \cos{\left(\theta \right)}\end{matrix}\right]$

```python
P = p_x*S.x + p_y*S.y + p_z*S.z
```

```python
P.to_matrix(N)
```

$\displaystyle \left[\begin{matrix}p_{x} \cos{\left(\psi \right)} \cos{\left(\theta \right)} + p_{y} \left(\sin{\left(\phi \right)} \sin{\left(\theta \right)} \cos{\left(\psi \right)} - \sin{\left(\psi \right)} \cos{\left(\phi \right)}\right) + p_{z} \left(\sin{\left(\phi \right)} \sin{\left(\psi \right)} + \sin{\left(\theta \right)} \cos{\left(\phi \right)} \cos{\left(\psi \right)}\right)\\p_{x} \sin{\left(\psi \right)} \cos{\left(\theta \right)} + p_{y} \left(\sin{\left(\phi \right)} \sin{\left(\psi \right)} \sin{\left(\theta \right)} + \cos{\left(\phi \right)} \cos{\left(\psi \right)}\right) + p_{z} \left(- \sin{\left(\phi \right)} \cos{\left(\psi \right)} + \sin{\left(\psi \right)} \sin{\left(\theta \right)} \cos{\left(\phi \right)}\right)\\- p_{x} \sin{\left(\theta \right)} + p_{y} \sin{\left(\phi \right)} \cos{\left(\theta \right)} + p_{z} \cos{\left(\phi \right)} \cos{\left(\theta \right)}\end{matrix}\right]$

```python
args = [
    p_x,
    p_y,
    p_z,
    phi,
    theta,
    psi,
]

rotation_lambda = sp.lambdify(args,P.to_matrix(N),modules='numpy')
```

```python
def rotate2(p,phi_=0,theta_=0,psi_=0):

    args = {
        'p_x':p[0],
        'p_y':p[1],
        'p_z':p[2],
        'phi':phi_,
        'theta':theta_,
        'psi':psi_
        }

    result = rotation_lambda(**args)
    return result.reshape(3,result.shape[2]).T

```

```python
import ipyvolume as ipv
import ipywidgets as widgets

class Scene:
    def __init__(self):

        self.figure = ipv.figure(animation=0.,animation_exponent=0,width=800,height=800)

        R = 1.
        ipv.xlim(0, R)
        ipv.ylim(0, R)
        ipv.zlim(0, R)
        ipv.style.box_off()

        self.load_ship_geometry()

        self.port = ipv.plot_mesh(self.X, self.Y_port, self.Z, wireframe=False)
        self.stbd = ipv.plot_mesh(self.X, self.Y_starboard, self.Z, wireframe=False)

        ipv.pylab.view(azimuth=180, elevation=60, distance=1)
        ipv.show()

        self.ui = widgets.interact(self.update, phi=(-180.,180.), theta=(-180., 180.), psi=(-180., 180.))

    def load_ship_geometry(self):

        self.N = 10
        L = 1
        B = 0.1
        T = 0.05

        x = np.linspace(0,L,self.N)
        y = np.linspace(0,B,self.N)
        z = np.linspace(0,T,self.N)

        X,Y = np.meshgrid(x,y)
        Z=T-np.array(Y)
        Y*=(L-X)**(0.4)

        self.X = X-L/2
        self.Z = Z
        self.Y_port = Y
        self.Y_starboard = -Y

    def update(self, phi, theta, psi):
        print("update with", phi, theta, psi)

        angles = np.deg2rad([phi, theta, psi])

        X = self.X.flatten()
        Y = self.Y_starboard.flatten()
        Z = self.Z.flatten()
        result = rotate2(p=[X,Y,Z],phi_=angles[0],theta_=angles[1],psi_=angles[2])
        X_ = result[:,0]
        Y_ = result[:,1]
        Z_ = result[:,2]
        self.stbd.x = X_
        self.stbd.y = Y_
        self.stbd.z = Z_

        X = self.X.flatten()
        Y = self.Y_port.flatten()
        Z = self.Z.flatten()
        result = rotate2(p=[X,Y,Z],phi_=angles[0],theta_=angles[1],psi_=angles[2])
        X_ = result[:,0]
        Y_ = result[:,1]
        Z_ = result[:,2]
        self.port.x = X_
        self.port.y = Y_
        self.port.z = Z_

scene = Scene()
```

    VBox(children=(Figure(animation=0.0, animation_exponent=0.0, camera=PerspectiveCamera(fov=46.0, position=(6.12…

    interactive(children=(FloatSlider(value=0.0, description='phi', max=180.0, min=-180.0), FloatSlider(value=0.0,…

```python

```


# Lecture 11

## Differential Equations III:

### Steady State Solutions and Mass Action

```python
import numpy as np
import sympy as sp
import scipy.integrate
sp.init_printing()
##################################################
##### Matplotlib boilerplate for consistency #####
##################################################
from ipywidgets import interact
from ipywidgets import FloatSlider
from matplotlib import pyplot as plt

%matplotlib inline

from IPython.display import set_matplotlib_formats
set_matplotlib_formats('svg')

global_fig_width = 10
global_fig_height = global_fig_width / 1.61803399
font_size = 12

plt.rcParams['axes.axisbelow'] = True
plt.rcParams['axes.edgecolor'] = '0.8'
plt.rcParams['axes.grid'] = True
plt.rcParams['axes.labelpad'] = 8
plt.rcParams['axes.linewidth'] = 2
plt.rcParams['axes.titlepad'] = 16.0
plt.rcParams['axes.titlesize'] = font_size * 1.4
plt.rcParams['figure.figsize'] = (global_fig_width, global_fig_height)
plt.rcParams['font.sans-serif'] = ['Computer Modern Sans Serif', 'DejaVu Sans', 'sans-serif']
plt.rcParams['font.size'] = font_size
plt.rcParams['grid.color'] = '0.8'
plt.rcParams['grid.linestyle'] = 'dashed'
plt.rcParams['grid.linewidth'] = 2
plt.rcParams['lines.dash_capstyle'] = 'round'
plt.rcParams['lines.dashed_pattern'] = [1, 4]
plt.rcParams['xtick.labelsize'] = font_size
plt.rcParams['xtick.major.pad'] = 4
plt.rcParams['xtick.major.size'] = 0
plt.rcParams['ytick.labelsize'] = font_size
plt.rcParams['ytick.major.pad'] = 4
plt.rcParams['ytick.major.size'] = 0
##################################################
```

## Steady-state solutions

- It is often useful to find the **steady state** of a system described by
  ordinary differential equations

- This occurs when all rates (i.e. derivatives) are zero

- A steady state may be **stable** or **unstable**, depending on whether or not small deviations from the steady state tend to be corrected or amplified

- We can evaluate the stability of a steady state by considering the sign of the derivative nearby

## Example:

Recall the radioactive decay equation $\displaystyle \frac{{\rm d}N}{{\rm d}t} = -\lambda N.$

```python
t = np.linspace(0,6,100)
l = 1.0
N0 = 1.0
def dNdt(N, t):
    return -l * N
N = scipy.integrate.odeint(dNdt, N0, t)

def plot_graph():
    plt.plot(t,N,label='N')
    plt.plot(t,dNdt(N,t),label='dN/dt')
    plt.legend()
    plt.xlabel('t')
```

```python
plot_graph()
```

- Here the steady state occurs when $N=0$, and is 'reached' after infinite time.

- For small positive $N$ the derivative $\;\displaystyle \frac{{\rm d}N}{{\rm d}t}\;$ is negative, moving the
  system towards the steady state.

- Physical considerations show that negative $N$ is impossible, hence the steady
  state is stable as we would expect.

## Example:

Consider a simple model of the production and degradation of a protein, shown by the reaction chain $\;\;\overset{v_1}{\rightarrow}S\overset{v_2}{\rightarrow}\;\;$ where $\;v_1\;$ and $\;v_2\;$ are reaction rates, and let $\;s=[S]$.

We can represent the change in concentration of the protein by

$$ \frac{{\rm d}s}{{\rm d}t} = v_1 - v_2 $$

so the steady state occurs when $\;v_1 = v_2$.

### Mass action

$$\;\;\overset{v_1}{\rightarrow}S\overset{v_2}{\rightarrow}\;\;$$

Suppose the reaction is governed by "mass action" kinetics, so $\;v_1=k_1\;$ (constant) and $\;v_2=k_2 s$.

The equation is then

$$\frac{{\rm d}s}{{\rm d}t} = k_1 - k_2 s.$$

The steady state is given by $s = k_1/k_2$, and it is stable:

```python
def plot_linear_steady_state():
    k_1 = 1.0
    k_2 = 1.0
    s = np.linspace(0,2,100)
    dsdt = k_1 - k_2*s
    plt.plot(s,dsdt)
    plt.xlabel('s')
    plt.ylabel(r'$\frac{ds}{dt}$')
    plt.axvline(x=k_1/k_2,ls='--')
    plt.axhline(y=0,ls='--')
```

```python
plot_linear_steady_state()
```

We can solve the differential equation by separation of variables.

$$ \int {1\over k_1 - k_2 s}~{\rm d}s = \int~{\rm d}t $$

$$ s(t) = Be^{-k_2 t} + {k_1\over k_2} $$

Thus the concentration of $S$ relaxes exponentially to the steady state, no matter the initial condition.

If the initial concentration of $S$ is given by $s_0$ then

$$ s(t) = s_0 e^{-k_2 t} + {k_1\over k_2}\left(1-e^{-k_2 t}\right) $$

Suppose instead that $S$ enhances its own production (positive feedback) and the degradation rate is nonlinear, so $v_1=k_1 s$ and $v_2=k_2 s^2$:

$$\frac{{\rm d}s}{{\rm d}t} = k_1s - k_2 s^2$$

Then the system will be at steady state when $k_1 s = k_2 s^2$, i.e. $s=0$ or
$s=k_1/k_2$.

We can plot $\;\displaystyle \frac{{\rm d}s}{{\rm d}t}\;$ versus $\;s\;$ to see if they are stable or unstable:

```python
def plot_dsdt(k_1,k_2):
    s = np.linspace(-0.1,2,100)
    dsdt = k_1*s - k_2*s**2
    plt.plot(s,dsdt)
    plt.xlabel('s')
    plt.ylabel(r'$\frac{ds}{dt}$')
    plt.axvline(x=k_1/k_2,ls='--')
    plt.axvline(x=0,ls='--')
    plt.axhline(y=0,ls='--')

def plot_quadratic_steady_state():
    interact(plot_dsdt, k_1 = FloatSlider(value=1,min=0.01,max=2.0,step=0.01,continuous_update=False), k_2 = FloatSlider(value=1,min=0.01,max=2.0,step=0.01,continuous_update=False), continuous_update=False)
```

```python
plot_quadratic_steady_state()
```

### Non-graphical method

$$\frac{{\rm d}s}{{\rm d}t} = k_1s - k_2 s^2$$

- When $s$ is small (and positive, since it is a concentration) $s\gg s^2$ so the derivative $k_1 s - k_2 s^2$ will be positive, making $s=0$ an unstable steady state.

- For the other steady state, consider small deviations by $\varepsilon$ from the steady state, e.g. $s=k_1/k_2 + \varepsilon$.

Then $\;\;\displaystyle s^2=\frac{k_1^2}{k_2^2} + 2 \varepsilon\frac{k_1}{k_2} + \varepsilon^2\;$ and the derivative is $\; - k_1\varepsilon - k_2\varepsilon^2$.

It is, since $\varepsilon\gg\varepsilon^2$, negative, pushing $s$ back towards the steady state, hence it is a stable steady state.

## Example:

The growth of a cell colony can be modelled by the *logistic* equation

$$ \frac{{\rm d}N}{{\rm d}t} = rN\left(1 - {N\over K}\right) $$

where $\;N(t)\;$ is the number of cells at time $\;t,\;$ and $\;r\;$ and $\;K\;$ are constant parameters (both positive).

The steady state for the system, or equilibrium population size, occurs when the growth rate is zero, i.e.

$$ \frac{{\rm d}N}{{\rm d}t} = 0 \quad\Rightarrow\quad rN = {r\over K}N^2 \quad\Rightarrow\quad N = 0 {\rm~or~} N = K.$$

```python
def plot_dNdt(r,K):
    N = np.linspace(-0.1,2,100)
    dNdt = r*N*(1 - N/K)
    plt.plot(N,dNdt)
    plt.xlabel('N')
    plt.ylabel(r'$\frac{dN}{dt}$')
    plt.axvline(x=K,ls='--')
    plt.axvline(x=0,ls='--')
    plt.axhline(y=0,ls='--')

def plot_logistic_steady_state():
    interact(plot_dNdt, r = FloatSlider(value=1,min=0.01,max=2.0,step=0.01),
                    K = FloatSlider(value=1,min=0.01,max=2.0,step=0.01))
```

```python
plot_logistic_steady_state()
```

- for small positive $N$, $rN>0$ and $N<K$ so $\displaystyle \frac{{\rm d}N}{{\rm d}t}>0$ and the population
  size will increase, meaning that $N=0$ is an unstable steady state.

- In fact the growth rate is positive for $0<N<K$ and negative for $N>K$, making
  $N=K$ a stable steady state.

We can solve the differential equation to examine the transient behaviour.
This is a separable equation, so

$$ \int {1\over rKN-rN^2}~{\rm d}N = \int {1\over K}~{\rm d}t $$

Solve this, using partial fractions on the left hand side:

- With initial conditions $N(0)=N_0$ you get
$$ N(t) = {K N_0 e^{rt}\over K + N_0(e^{rt}-1)} $$

## Summary of first order differential equations

To solve a differential equation:

1. Calculate the general solution
    1. Try to write it as a separable equation first
    2. Other methods (e.g. integrating factors) not covered in this course
2. This general solution will include an arbitrary constant this may be eliminated using initial conditions (if these are given)
3. Can check your solution numerically using Python

To find steady state (equilibrium) solutions, find points where all (first) derivatives are zero.

To determine stability of these steady states, look at the behaviour of the first derivative in the vicinity of the steady state.
You can sketch the first derivative, or use Python to help with this.

## Chemical reactions to equations: Mass action

- Chemical reactions assume uniform mixing

- Rates proportional to product of masses

- Predator/prey and epidemics have similar rules

1. A and B produce C. The reaction is governed by A meeting B:

$$A+B\underset{^k}{\rightarrow} C$$

\begin{align*}
\frac{{\rm d}[A]}{{\rm d}t} &= -k[A][B] \\
\frac{{\rm d}[B]}{{\rm d}t} &= -k[A][B] \\
\frac{{\rm d}[C]}{{\rm d}t} &= k[A][B] \\
\end{align*}

2. Predation of R by W

$$R+W\underset{^k}{\rightarrow} W$$

$$\frac{{\rm d}R}{{\rm d}t} = -kRW$$

3. Constant production (zeroth order)

$$0 \underset{^k}{\rightarrow} A$$

$$\frac{{\rm d}[A]}{{\rm d}t} = k$$

4. Degradation/death

$$C \underset{^k}{\rightarrow} 0$$

$$\frac{{\rm d}[C]}{{\rm d}t} = -k[C]$$

## Numerically solving differential equations

What if we can't solve the differential equation?

$$ \frac{{\rm d}y}{{\rm d}t} = \sin\left(\frac{y\ln(t\cos(y))}{\sqrt{1+y/e^t}}\right) + 3$$

## Numerically solving differential equations

What if we can't solve the differential equation?

$$ \frac{{\rm d}y}{{\rm d}t} = \sin\left(\frac{y\ln(t\cos(y))}{\sqrt{1+y/e^t}}\right) + 3$$

(or we just don't want to)

## Euler's method

Given a differential equation

$$ \frac{{\rm d}y}{{\rm d}t} = f(y, t)$$

with initial state $\;y(t = t_0) = y_0,\;$ we can *approximate* the state at $t = t_0 + \delta{t}$ as:

$$ y_1 = y(t + \delta{t}) \approx y_0 + f(y, t) \cdot \delta{t}$$

and the next state as
$$ y_2 = y(t + 2\delta{t}) \approx y_1 + f(y, t) \cdot \delta{t}$$

and so on!

## Euler's method

This mean's we can estimate the *entire time course of $y(t)$*, provided:

1. We can calculate $f(y, t)$ (or approximate it with a computer)
2. We're patient enough to take really tiny steps $\delta{t}$

This is really simple with a computer! See the Python helpsheet for details.


### Mutual Information Theory Derivation

Let us begin with the definition of mutual information.

Observations $x$ come from an unknown data generating distribution $x \sim P(X)$. The random variable $Y$ represents the weights of a neuron and its pdf is a Gaussian. Because it will be referenced frequently, we denote it with a $q(y)$. The weights of the neuron are updated after being stimulated by samples from the data generating distribution $P$. Therefore there is a dependence between the two and $p(x,y) \neq p(x)q(y)$. One way to quantify this interdependence with the mutual information

$$I(X;Y) = \int_{X\times Y} p(x,y) \log \frac{p(x,y)}{p(x)q(y)}d(x,y)$$

Fubini-Tonelli's Theorem tell us that as long as $X$ and $Y$ are $\sigma$-finite measure spaces and $f$ is a measurable function then

$${\displaystyle \int _{X\times Y}\big|f(x,y)\big|\,{\text{d}}(x,y)} = \int _{X}\left(\int _{Y}\big|f(x,y)\big|\,{\text{d}}y\right)\,{\text{d}}x=\int _{Y}\left(\int _{X}\big|f(x,y)\big|\,{\text{d}}x\right)\,{\text{d}}y$$

If we break up the integrand using logarithmic properties we can simplify the formula

$$I(X;Y) = \int_X \int_Y p(x,y) \log \frac{p(x,y)}{p(x)}{\text{d}}y \,{\text{d}}x - \int_Y \int_X p(x,y) \log p(y) {\text{d}}x \,{\text{d}}y$$

To simplify things futher wesubstitute $p(x,y) = p(y|x)p(x)$ in the left term

\begin{align}
I(X;Y) &= \int_X \int_Y  p(y|x)p(x) \log p(y|x){\text{d}}y \,{\text{d}}x - \int_Y \bigg( \int_X p(x,y) {\text{d}}x \bigg) \log p(y) {\text{d}}y \\
&= \int_X p(x) \left( \int_Y  p(y|x) \log p(y|x){\text{d}}y\right) \,{\text{d}}x - \int_Y p(y) \log p(y) {\text{d}}y \\
&= \mathbb{E}_X \left[ h(x) \right] + H(Y)
\end{align}

where $h(x)$ is the negative entropy of the conditional distribution $P(Y|X=x )$

### **To be done in the future**: Show that $h(x) \approx q(x)$

Assuming that $h(x) \approx q(x)$...

\begin{align}
q(\boldsymbol{x} | \theta) &= \frac{1}{\sqrt{(2\pi)^{k}
|\Sigma|}} \exp \bigg\{ -\frac{1}{2}(\boldsymbol{x}-\boldsymbol{\mu})^T\Sigma^{-1}
(\boldsymbol{x}-\boldsymbol{\mu}) \bigg \} \\
&= \frac{1}{C}\exp \{ -\frac{1}{2}d^2(\boldsymbol{x}, \boldsymbol{\mu};\Sigma)\}
\end{align}

where $C=\sqrt{(2\pi)^{k} |\Sigma|}$ is the volume under an unnormalized Gaussian and $d(\boldsymbol{x}, \boldsymbol{\mu};\Sigma)$ is the mahalanobis distance with respect to covariance matrix $\Sigma$.

Our objective is to learn a set of parameters $\theta$ that maximize the mutual information $I(X;Y)$, so we represent this as an objective function in terms of the parameters

\begin{align}
f(\mu,\Sigma) &= \mathbb{E}_X [q(\boldsymbol{x} | \mu, \Sigma) ] + H[q(\boldsymbol{x} | \mu, \Sigma)] \\
\end{align}

In addition we can calculate the differential entropy of the multivariate gaussian pdf for $Y$ analytically and substitute it into our objective.

\begin{align}
H[q(\boldsymbol{x} | \mu, \Sigma)] &= \frac{1}{2}\log | 2 \pi e \Sigma | \\
f(\mu,\Sigma) &= \mathbb{E}_X [q(\boldsymbol{x} | \mu, \Sigma) ] + \frac{1}{2}\log | 2 \pi e \Sigma | \\
\end{align}

To maximize the mutual information we maximize the objective using traditional techniques

\begin{align}
\frac{\partial f}{\boldsymbol{\mu}} &= \mathbb{E}_X \big[ \frac{\partial q(x|\boldsymbol{\mu}, \Sigma)}{\partial \boldsymbol{\mu}} \big] \\
\frac{\partial f}{\partial \Sigma} &= \mathbb{E}_X \big[ \frac{\partial q(x|\boldsymbol{\mu}, \Sigma)}{\partial \Sigma} \big] + \frac{1}{2}\Sigma^{-1} \\
\end{align}

We begin by recalling the partials of the multivariate gaussian pdf for both the mean $\boldsymbol{\mu}$ and the covariance $\Sigma$.

\begin{align}
\frac{\partial q}{\boldsymbol{\mu}} &= q(x|\boldsymbol{\mu}, \Sigma) \Sigma^{-1} (\boldsymbol{x}-\boldsymbol{\mu}) \\
\frac{\partial q}{\partial \Sigma} &= -q(x|\boldsymbol{\mu}, \Sigma) \frac{1}{2}(\Sigma^{-1}-\Sigma^{-1}(\boldsymbol{x}-\boldsymbol{\mu})(\boldsymbol{x}-\boldsymbol{\mu})^T\Sigma^{-1}) \\
\end{align}

(see appendix for derivations)

Substituting these back into our gradient terms we arrive at the following for the mean $\mu$

\begin{align}
\frac{\partial f}{\boldsymbol{\mu}} &=  \mathbb{E}_X \big[ q(x|\boldsymbol{\mu}, \Sigma)\Sigma^{-1} (\boldsymbol{x}-\boldsymbol{\mu}) \big] \\
&=  \Sigma^{-1} \mathbb{E}_X \big[ q(x|\boldsymbol{\mu}, \Sigma)(\boldsymbol{x}-\boldsymbol{\mu}) \big] \\
\end{align}

and below for $\Sigma$

\begin{align}
\frac{\partial f}{\partial \Sigma} &= \mathbb{E}_X \big[ -q(x|\boldsymbol{\mu}, \Sigma) \frac{1}{2}(\Sigma^{-1}-\Sigma^{-1}(\boldsymbol{x}-\boldsymbol{\mu})(\boldsymbol{x}-\boldsymbol{\mu})^T\Sigma^{-1}) \big] + \frac{1}{2}\Sigma^{-1} \\
&= \frac{1}{2}\Sigma^{-1} \bigg[ \mathbb{E}_X \big[q(x|\boldsymbol{\mu}, \Sigma) ((\boldsymbol{x}-\boldsymbol{\mu})(\boldsymbol{x}-\boldsymbol{\mu})^T-\Sigma) \big] + \Sigma \bigg] \Sigma^{-1} \\
\end{align}

If the expectation in the above expression equals zero, then then entire partial derivative is zero. Furthermore, if $\Sigma$ is invertible then it is the only way the partial can be zero.

Lastly we substitute the definition of the Gaussian $q$ from above.

\begin{align}
\frac{\partial f}{\partial \mu} &=  \Sigma^{-1} \mathbb{E}_X \big[ q(x|\boldsymbol{\mu}, \Sigma)(\boldsymbol{x}-\boldsymbol{\mu}) \big] \\
&=  \frac{1}{C}\Sigma^{-1} \mathbb{E}_X \big[\exp \{ -\frac{1}{2}d^2(\boldsymbol{x}, \boldsymbol{\mu};\Sigma) \}(\boldsymbol{x}-\boldsymbol{\mu}) \big] \\
\end{align}

Doing the same for the partial with respect to $\Sigma$ we arrive at

\begin{align}
\frac{\partial f}{\partial \Sigma} &= \frac{1}{2}\Sigma^{-1} \bigg[ \mathbb{E}_X \big[q(x|\boldsymbol{\mu}, \Sigma) ((\boldsymbol{x}-\boldsymbol{\mu})(\boldsymbol{x}-\boldsymbol{\mu})^T-\Sigma) \big] - \Sigma \bigg] \Sigma^{-1} \\
&= \frac{1}{2C}\Sigma^{-1} \bigg[ \mathbb{E}_X \big[\exp \{ -\frac{1}{2}d^2(\boldsymbol{x}, \boldsymbol{\mu};\Sigma) \} [(\boldsymbol{x}-\boldsymbol{\mu})(\boldsymbol{x}-\boldsymbol{\mu})^T-\Sigma] \big] + \Sigma\sqrt{(2\pi)^{k} |\Sigma|} \bigg] \Sigma^{-1} \\
\end{align}

Finally we can produce our stochastic approximation update equations...

\begin{align}
\mu_{i+1} &= \mu_i + \exp \{ -\frac{1}{2}d^2(\boldsymbol{x}_i, \boldsymbol{\mu}_i;\Sigma_i) \}(\boldsymbol{x}_i-\boldsymbol{\mu}_i) \\
\Sigma_{i+1} & = \Sigma_i + \exp \{ -\frac{1}{2}d^2(\boldsymbol{x}_i, \boldsymbol{\mu}_i;\Sigma_i) \} [(\boldsymbol{x}_i-\boldsymbol{\mu}_i)(\boldsymbol{x}_i-\boldsymbol{\mu}_i)^T-\Sigma_i] + \Sigma_i\sqrt{(2\pi)^{k} |\Sigma_i|}
\end{align}

```python

```

```python

```


## 9. Show mathematically that the combination of a mirror plane and a translation vector $\mathbf{t}$ perpendicular to the mirror plane implies the existence of a mirror plane at $\mathbf{t}$/2.

Solution:

Without loss of generality, let's define the translation vector be to the c-basis vector (note that we can always redefine basis vectors to make this the case).

The mirror operation is therefore given by:

```
import numpy as np
M = np.eye(4).astype(np.int)
M[2, 2] = -1
print M
```

    [[ 1  0  0  0]
     [ 0  1  0  0]
     [ 0  0 -1  0]
     [ 0  0  0  1]]

The translation operation is given by:

```
from sympy import symbols
t = symbols("t")
T = [[1, 0, 0, 0],
     [0, 1, 0, 0],
     [0, 0, 1, t],
     [0, 0, 0, 1]]
```

Consider the relationship between a reflected point and a translated point. The operation that maps the reflected point to the translated point is given by:

$Op_m = p_t$

$OMp = Tp$

$OM = T$

=> $O = TM^{-1}$

```
O = np.dot(T, np.linalg.inv(M))
print "O = %s" % str(O)
```

    O = [[1.0 0.0 0.0 0.0]
     [0.0 1.0 0.0 0.0]
     [0 0 -1.00000000000000 1.0*t]
     [0.0 0.0 0.0 1.0]]

Now consider a mirror plane at t/2. The symmetry operation is given by:

$O' = T_{t/2}MT_{-t/2}$

```
 T__t2 = [[1, 0, 0, 0],
     [0, 1, 0, 0],
     [0, 0, 1, -t/2],
     [0, 0, 0, 1]]
T_t2 = [[1, 0, 0, 0],
     [0, 1, 0, 0],
     [0, 0, 1, t/2],
     [0, 0, 0, 1]]
O = np.dot(T_t2, np.dot(M, T__t2))
print "O' = %s" % str(O)
```

    O' = [[1 0 0 0]
     [0 1 0 0]
     [0 0 -1 t]
     [0 0 0 1]]

We may observe that $O' = O$, i.e., a reflection with a perpendicular translation **t** implies the existence of a mirror plane at **t/2**.

```

```


## Variational Autoencoder

A variational autoencoder is a generative neural network model consisting of an encoder, a sampling layer and a decoder. Autoencoders are generally used to learn a latent representation of data by creating an information bottleneck through a pyramid-like structure of the encoder where each layers has fewer degrees of freedom compared to the previous layer. This results in dimensionality reduction of the input. The decoder on the other hand takes a compressed latent representation and reconstructs the original input. In order to make an autoencoder generative, we learn a variational approximation of the hidden layer.

Autoencoders are data driven and application specific, for example an autoencoder trained on digits will not perform well on images of faces. Autoencoders result in lossy compression. They are useful in data denoising and dimensionality reduction. Training variational autoencoders consists of optimizing a custom loss function consisting of reconstruction loss and regularization loss:

\begin{equation}
    \min H(p,q) + KL(z||pz)
\end{equation}

where $H(p,q)$ is the cross-entropy reconstruction loss defined as $-\sum_i p_i \log q_i$ where $p$ is the ground truth distribution and $q$ is the predicted distribution and $KL(q||z)$ is the regularization loss that constraints our variational approximation $z$ to be close to our prior $pz \sim N(0,1)$ which is assumed to be a standard normal in our example. The regularization loss forces similar inputs to be close in the latent space. In variational autoencoder, we learn the variational approximation parameters ($\mu$ and $\log \sigma$) using the neural network. Thus, we can construct our approximate distribution $z$ as follows:

\begin{equation}
    z = \mu + \exp(\log \sigma) \times N(0,1)
\end{equation}

As we'll see in the following example, we'll be able to generate an image manifold by sampling from our distribution $z \sim N(\mu, \sigma^2)$ above and feeding the samples through a decoder to generate an image.

```python
%matplotlib inline
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

import math
from scipy.stats import norm

import keras
from keras.layers import Input, Dense, Lambda, Layer
from keras.models import Model
from keras import backend as K
from keras import metrics
from keras import optimizers
from keras.datasets import mnist

from keras.callbacks import ModelCheckpoint
from keras.callbacks import TensorBoard
from keras.callbacks import LearningRateScheduler
from keras.callbacks import EarlyStopping

from keras.utils import plot_model
```

    Using TensorFlow backend.

    Couldn't import dot_parser, loading of dot files will not be possible.

In training variational autoencoder, we'll use a learning rate schedule that divides the initial learning rate by 2 every fixed number of epochs, we'll also use a callback to monitor the learning rate.

```python
DATA_PATH = '/data/vision/fisher/data1/vsmolyakov/vae/'

def step_decay(epoch):
    lr_init = 0.001
    drop = 0.5
    epochs_drop = 16.0
    lr_new = lr_init * math.pow(drop, math.floor((1+epoch)/epochs_drop))
    return lr_new

class LR_hist(keras.callbacks.Callback):
    def on_train_begin(self, logs={}):
        self.losses = []
        self.lr = []
    def on_epoch_end(self, batch, logs={}):
        self.losses.append(logs.get('loss'))
        self.lr.append(step_decay(len(self.losses)))
```

Let's load the data and define training and model parameters:

```python
#load data
print "loading data..."
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.astype('float32') / 255.0
x_test = x_test.astype('float32') / 255.0
x_train = x_train.reshape((len(x_train), np.prod(x_train.shape[1:])))
x_test = x_test.reshape((len(x_test), np.prod(x_test.shape[1:])))

#training params
batch_size = 100
num_epochs = 128

#model params
original_dim = 784
intermediate_dim = 256
latent_dim = 2
epsilon_std = 1.0
```

    loading data...

We'll implement the sampling layer using a Lambda layer in Keras.

```python
#sampling layer
def sampling(args):
    z_mean, z_log_sigma = args
    epsilon = K.random_normal(shape=(K.shape(z_mean)[0], latent_dim), mean=0.0, stddev=epsilon_std)
    return z_mean + K.exp(z_log_sigma) * epsilon
```

Let's define the variational autoencoder architecture:

```python
#VAE architecture
#encoder
x = Input(shape=(original_dim,))
h = Dense(intermediate_dim, activation='relu')(x)
z_mean = Dense(latent_dim)(h)
z_log_sigma = Dense(latent_dim)(h)

#sampling
z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_sigma])

#decoder
decoder_h = Dense(intermediate_dim, activation='relu')
decoder_mean = Dense(original_dim, activation='sigmoid')
h_decoded = decoder_h(z)
x_decoded_mean = decoder_mean(h_decoded)
```

Because our loss function consists of two terms, we have to define a custom loss layer:

```python
#custom loss layer
class CustomVariationalLayer(Layer):
    def __init__(self, **kwargs):
        self.is_placeholder = True
        super(CustomVariationalLayer, self).__init__(**kwargs)

    def vae_loss(self, x, x_decoded_mean):
        xent_loss = original_dim * metrics.binary_crossentropy(x, x_decoded_mean)
        kl_loss = - 0.5 * K.sum(1 + z_log_sigma - K.square(z_mean) - K.exp(z_log_sigma), axis=-1)
        return K.mean(xent_loss + kl_loss)

    def call(self, inputs):
        x = inputs[0]
        x_decoded_mean = inputs[1]
        loss = self.vae_loss(x, x_decoded_mean)
        self.add_loss(loss, inputs=inputs)
        return x

```

We can finally compile our variational autoencoder model using Adam optimizer:

```python
y = CustomVariationalLayer()([x, x_decoded_mean])
vae = Model(x, y)

adam = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
vae.compile(loss=None, optimizer=adam, metrics=['accuracy'])
vae.summary()
```

    ____________________________________________________________________________________________________
    Layer (type)                     Output Shape          Param #     Connected to
    ====================================================================================================
    input_1 (InputLayer)             (None, 784)           0
    ____________________________________________________________________________________________________
    dense_1 (Dense)                  (None, 256)           200960      input_1[0][0]
    ____________________________________________________________________________________________________
    dense_2 (Dense)                  (None, 2)             514         dense_1[0][0]
    ____________________________________________________________________________________________________
    dense_3 (Dense)                  (None, 2)             514         dense_1[0][0]
    ____________________________________________________________________________________________________
    lambda_1 (Lambda)                (None, 2)             0           dense_2[0][0]
                                                                       dense_3[0][0]
    ____________________________________________________________________________________________________
    dense_4 (Dense)                  (None, 256)           768         lambda_1[0][0]
    ____________________________________________________________________________________________________
    dense_5 (Dense)                  (None, 784)           201488      dense_4[0][0]
    ____________________________________________________________________________________________________
    custom_variational_layer_1 (Cust [(None, 784), (None,  0           input_1[0][0]
                                                                       dense_5[0][0]
    ====================================================================================================
    Total params: 404,244
    Trainable params: 404,244
    Non-trainable params: 0
    ____________________________________________________________________________________________________

    /usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:5: UserWarning: Output "custom_variational_layer_1" missing from loss dictionary. We assume this was done on purpose, and we will not be expecting any data to be passed to "custom_variational_layer_1" during training.
      """

```python
#define callbacks
file_name = DATA_PATH + 'weights-checkpoint.hdf5'
checkpoint = ModelCheckpoint(file_name, monitor='loss', verbose=1, save_best_only=True, mode='min')
tensor_board = TensorBoard(log_dir='./logs', write_graph=True)
hist_lr = LR_hist()
reduce_lr = LearningRateScheduler(step_decay)
early_stopping = EarlyStopping(monitor='loss', min_delta=0.01, patience=64, verbose=1)
callbacks_list = [checkpoint, tensor_board, hist_lr, reduce_lr, early_stopping]

```

```python
print "training vae model..."
hist = vae.fit(x_train, shuffle=True, epochs=num_epochs, batch_size=batch_size, callbacks=callbacks_list, validation_data=(x_test, None), verbose=2)
```

    training vae model...
    Train on 60000 samples, validate on 10000 samples
    Epoch 1/128
    Epoch 00000: loss improved from inf to 207.46185, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 207.4619 - val_loss: 181.7194
    Epoch 2/128
    Epoch 00001: loss improved from 207.46185 to 175.27829, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 175.2783 - val_loss: 170.4656
    Epoch 3/128
    Epoch 00002: loss improved from 175.27829 to 168.62863, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 168.6286 - val_loss: 166.0275
    Epoch 4/128
    Epoch 00003: loss improved from 168.62863 to 165.28341, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 165.2834 - val_loss: 163.8910
    Epoch 5/128
    Epoch 00004: loss improved from 165.28341 to 163.33535, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 163.3354 - val_loss: 162.1486
    Epoch 6/128
    Epoch 00005: loss improved from 163.33535 to 161.73204, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 161.7320 - val_loss: 160.7561
    Epoch 7/128
    Epoch 00006: loss improved from 161.73204 to 160.28968, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 160.2897 - val_loss: 159.2922
    Epoch 8/128
    Epoch 00007: loss improved from 160.28968 to 158.86565, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 158.8656 - val_loss: 158.5544
    Epoch 9/128
    Epoch 00008: loss improved from 158.86565 to 157.58157, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 157.5816 - val_loss: 156.9721
    Epoch 10/128
    Epoch 00009: loss improved from 157.58157 to 156.33295, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 156.3329 - val_loss: 155.6949
    Epoch 11/128
    Epoch 00010: loss improved from 156.33295 to 155.37949, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 155.3795 - val_loss: 155.2493
    Epoch 12/128
    Epoch 00011: loss improved from 155.37949 to 154.52507, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 154.5251 - val_loss: 154.0871
    Epoch 13/128
    Epoch 00012: loss improved from 154.52507 to 153.86931, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 153.8693 - val_loss: 153.7534
    Epoch 14/128
    Epoch 00013: loss improved from 153.86931 to 153.30271, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 153.3027 - val_loss: 153.2158
    Epoch 15/128
    Epoch 00014: loss improved from 153.30271 to 152.72257, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 152.7226 - val_loss: 152.9466
    Epoch 16/128
    Epoch 00015: loss improved from 152.72257 to 151.76216, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 151.7622 - val_loss: 152.1998
    Epoch 17/128
    Epoch 00016: loss improved from 151.76216 to 151.46707, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 151.4671 - val_loss: 152.0505
    Epoch 18/128
    Epoch 00017: loss improved from 151.46707 to 151.21640, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 151.2164 - val_loss: 151.9097
    Epoch 19/128
    Epoch 00018: loss improved from 151.21640 to 150.98383, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 150.9838 - val_loss: 151.8249
    Epoch 20/128
    Epoch 00019: loss improved from 150.98383 to 150.73573, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 150.7357 - val_loss: 151.6568
    Epoch 21/128
    Epoch 00020: loss improved from 150.73573 to 150.51922, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 150.5192 - val_loss: 151.6240
    Epoch 22/128
    Epoch 00021: loss improved from 150.51922 to 150.30904, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 150.3090 - val_loss: 151.5045
    Epoch 23/128
    Epoch 00022: loss improved from 150.30904 to 150.11874, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 150.1187 - val_loss: 151.5699
    Epoch 24/128
    Epoch 00023: loss improved from 150.11874 to 149.93569, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 149.9357 - val_loss: 151.2348
    Epoch 25/128
    Epoch 00024: loss improved from 149.93569 to 149.73812, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 149.7381 - val_loss: 151.1765
    Epoch 26/128
    Epoch 00025: loss improved from 149.73812 to 149.57059, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 149.5706 - val_loss: 151.0119
    Epoch 27/128
    Epoch 00026: loss improved from 149.57059 to 149.39235, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 149.3923 - val_loss: 151.0611
    Epoch 28/128
    Epoch 00027: loss improved from 149.39235 to 149.21818, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 149.2182 - val_loss: 150.7763
    Epoch 29/128
    Epoch 00028: loss improved from 149.21818 to 149.08684, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 149.0868 - val_loss: 150.8500
    Epoch 30/128
    Epoch 00029: loss improved from 149.08684 to 148.90517, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 148.9052 - val_loss: 150.8162
    Epoch 31/128
    Epoch 00030: loss improved from 148.90517 to 148.76809, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 148.7681 - val_loss: 150.8209
    Epoch 32/128
    Epoch 00031: loss improved from 148.76809 to 148.26388, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 148.2639 - val_loss: 150.3329
    Epoch 33/128
    Epoch 00032: loss improved from 148.26388 to 148.16450, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 148.1645 - val_loss: 150.4133
    Epoch 34/128
    Epoch 00033: loss improved from 148.16450 to 148.07045, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 148.0705 - val_loss: 150.2414
    Epoch 35/128
    Epoch 00034: loss improved from 148.07045 to 147.99750, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 147.9975 - val_loss: 150.2835
    Epoch 36/128
    Epoch 00035: loss improved from 147.99750 to 147.91649, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 147.9165 - val_loss: 150.2635
    Epoch 37/128
    Epoch 00036: loss improved from 147.91649 to 147.84790, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 147.8479 - val_loss: 150.2183
    Epoch 38/128
    Epoch 00037: loss improved from 147.84790 to 147.75718, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 147.7572 - val_loss: 150.2958
    Epoch 39/128
    Epoch 00038: loss improved from 147.75718 to 147.68742, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 147.6874 - val_loss: 150.3576
    Epoch 40/128
    Epoch 00039: loss improved from 147.68742 to 147.61803, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 147.6180 - val_loss: 150.1109
    Epoch 41/128
    Epoch 00040: loss improved from 147.61803 to 147.53446, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 147.5345 - val_loss: 150.1735
    Epoch 42/128
    Epoch 00041: loss improved from 147.53446 to 147.47079, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 147.4708 - val_loss: 150.1849
    Epoch 43/128
    Epoch 00042: loss improved from 147.47079 to 147.38060, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 147.3806 - val_loss: 150.1782
    Epoch 44/128
    Epoch 00043: loss improved from 147.38060 to 147.32383, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 147.3238 - val_loss: 150.2023
    Epoch 45/128
    Epoch 00044: loss improved from 147.32383 to 147.23673, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 147.2367 - val_loss: 150.0926
    Epoch 46/128
    Epoch 00045: loss improved from 147.23673 to 147.17098, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 147.1710 - val_loss: 150.0273
    Epoch 47/128
    Epoch 00046: loss improved from 147.17098 to 147.11243, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 147.1124 - val_loss: 149.9769
    Epoch 48/128
    Epoch 00047: loss improved from 147.11243 to 146.86245, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.8624 - val_loss: 149.8431
    Epoch 49/128
    Epoch 00048: loss improved from 146.86245 to 146.81145, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.8115 - val_loss: 149.8324
    Epoch 50/128
    Epoch 00049: loss improved from 146.81145 to 146.78653, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.7865 - val_loss: 149.8825
    Epoch 51/128
    Epoch 00050: loss improved from 146.78653 to 146.72616, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 146.7262 - val_loss: 149.8740
    Epoch 52/128
    Epoch 00051: loss improved from 146.72616 to 146.70117, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.7012 - val_loss: 149.8225
    Epoch 53/128
    Epoch 00052: loss improved from 146.70117 to 146.66899, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.6690 - val_loss: 149.8604
    Epoch 54/128
    Epoch 00053: loss improved from 146.66899 to 146.62405, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.6241 - val_loss: 149.8451
    Epoch 55/128
    Epoch 00054: loss improved from 146.62405 to 146.57467, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.5747 - val_loss: 149.8275
    Epoch 56/128
    Epoch 00055: loss improved from 146.57467 to 146.54511, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.5451 - val_loss: 149.8263
    Epoch 57/128
    Epoch 00056: loss improved from 146.54511 to 146.51676, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.5168 - val_loss: 149.8087
    Epoch 58/128
    Epoch 00057: loss improved from 146.51676 to 146.49652, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 146.4965 - val_loss: 149.8314
    Epoch 59/128
    Epoch 00058: loss improved from 146.49652 to 146.44952, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 146.4495 - val_loss: 149.8319
    Epoch 60/128
    Epoch 00059: loss improved from 146.44952 to 146.41819, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.4182 - val_loss: 149.8506
    Epoch 61/128
    Epoch 00060: loss improved from 146.41819 to 146.38260, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.3826 - val_loss: 149.7496
    Epoch 62/128
    Epoch 00061: loss improved from 146.38260 to 146.36248, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.3625 - val_loss: 149.8067
    Epoch 63/128
    Epoch 00062: loss improved from 146.36248 to 146.31845, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 146.3185 - val_loss: 149.7689
    Epoch 64/128
    Epoch 00063: loss improved from 146.31845 to 146.19048, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.1905 - val_loss: 149.7059
    Epoch 65/128
    Epoch 00064: loss improved from 146.19048 to 146.15993, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.1599 - val_loss: 149.6759
    Epoch 66/128
    Epoch 00065: loss improved from 146.15993 to 146.15078, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.1508 - val_loss: 149.7215
    Epoch 67/128
    Epoch 00066: loss improved from 146.15078 to 146.12988, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.1299 - val_loss: 149.6360
    Epoch 68/128
    Epoch 00067: loss improved from 146.12988 to 146.11893, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.1189 - val_loss: 149.6864
    Epoch 69/128
    Epoch 00068: loss improved from 146.11893 to 146.08901, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 146.0890 - val_loss: 149.7056
    Epoch 70/128
    Epoch 00069: loss improved from 146.08901 to 146.08488, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.0849 - val_loss: 149.7462
    Epoch 71/128
    Epoch 00070: loss improved from 146.08488 to 146.05973, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.0597 - val_loss: 149.6856
    Epoch 72/128
    Epoch 00071: loss improved from 146.05973 to 146.04602, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.0460 - val_loss: 149.6931
    Epoch 73/128
    Epoch 00072: loss improved from 146.04602 to 146.02700, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 146.0270 - val_loss: 149.7390
    Epoch 74/128
    Epoch 00073: loss improved from 146.02700 to 145.99736, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.9974 - val_loss: 149.7076
    Epoch 75/128
    Epoch 00074: loss improved from 145.99736 to 145.99558, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.9956 - val_loss: 149.7026
    Epoch 76/128
    Epoch 00075: loss improved from 145.99558 to 145.99270, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.9927 - val_loss: 149.6258
    Epoch 77/128
    Epoch 00076: loss improved from 145.99270 to 145.94872, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.9487 - val_loss: 149.6698
    Epoch 78/128
    Epoch 00077: loss did not improve
    9s - loss: 145.9548 - val_loss: 149.6526
    Epoch 79/128
    Epoch 00078: loss improved from 145.94872 to 145.92872, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.9287 - val_loss: 149.6328
    Epoch 80/128
    Epoch 00079: loss improved from 145.92872 to 145.85481, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.8548 - val_loss: 149.6230
    Epoch 81/128
    Epoch 00080: loss improved from 145.85481 to 145.84402, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.8440 - val_loss: 149.6277
    Epoch 82/128
    Epoch 00081: loss improved from 145.84402 to 145.84301, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 145.8430 - val_loss: 149.6380
    Epoch 83/128
    Epoch 00082: loss improved from 145.84301 to 145.82151, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 145.8215 - val_loss: 149.6593
    Epoch 84/128
    Epoch 00083: loss did not improve
    9s - loss: 145.8283 - val_loss: 149.6599
    Epoch 85/128
    Epoch 00084: loss did not improve
    9s - loss: 145.8226 - val_loss: 149.6392
    Epoch 86/128
    Epoch 00085: loss improved from 145.82151 to 145.79993, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.7999 - val_loss: 149.6925
    Epoch 87/128
    Epoch 00086: loss did not improve
    9s - loss: 145.8020 - val_loss: 149.6411
    Epoch 88/128
    Epoch 00087: loss improved from 145.79993 to 145.78678, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.7868 - val_loss: 149.6104
    Epoch 89/128
    Epoch 00088: loss improved from 145.78678 to 145.77177, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.7718 - val_loss: 149.6495
    Epoch 90/128
    Epoch 00089: loss did not improve
    9s - loss: 145.7738 - val_loss: 149.5923
    Epoch 91/128
    Epoch 00090: loss improved from 145.77177 to 145.76849, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.7685 - val_loss: 149.5719
    Epoch 92/128
    Epoch 00091: loss improved from 145.76849 to 145.74702, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.7470 - val_loss: 149.6446
    Epoch 93/128
    Epoch 00092: loss did not improve
    9s - loss: 145.7517 - val_loss: 149.5809
    Epoch 94/128
    Epoch 00093: loss improved from 145.74702 to 145.72771, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    11s - loss: 145.7277 - val_loss: 149.6039
    Epoch 95/128
    Epoch 00094: loss improved from 145.72771 to 145.72395, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.7240 - val_loss: 149.6169
    Epoch 96/128
    Epoch 00095: loss improved from 145.72395 to 145.68434, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.6843 - val_loss: 149.5976
    Epoch 97/128
    Epoch 00096: loss did not improve
    9s - loss: 145.6873 - val_loss: 149.6068
    Epoch 98/128
    Epoch 00097: loss improved from 145.68434 to 145.67932, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.6793 - val_loss: 149.6142
    Epoch 99/128
    Epoch 00098: loss improved from 145.67932 to 145.66919, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.6692 - val_loss: 149.5717
    Epoch 100/128
    Epoch 00099: loss did not improve
    9s - loss: 145.6815 - val_loss: 149.6050
    Epoch 101/128
    Epoch 00100: loss improved from 145.66919 to 145.65051, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.6505 - val_loss: 149.5857
    Epoch 102/128
    Epoch 00101: loss did not improve
    9s - loss: 145.6612 - val_loss: 149.5564
    Epoch 103/128
    Epoch 00102: loss did not improve
    9s - loss: 145.6711 - val_loss: 149.6020
    Epoch 104/128
    Epoch 00103: loss did not improve
    9s - loss: 145.6557 - val_loss: 149.5986
    Epoch 105/128
    Epoch 00104: loss improved from 145.65051 to 145.64327, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 145.6433 - val_loss: 149.5855
    Epoch 106/128
    Epoch 00105: loss did not improve
    9s - loss: 145.6450 - val_loss: 149.5933
    Epoch 107/128
    Epoch 00106: loss improved from 145.64327 to 145.63598, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 145.6360 - val_loss: 149.5353
    Epoch 108/128
    Epoch 00107: loss improved from 145.63598 to 145.63409, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 145.6341 - val_loss: 149.6438
    Epoch 109/128
    Epoch 00108: loss improved from 145.63409 to 145.63216, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 145.6322 - val_loss: 149.5546
    Epoch 110/128
    Epoch 00109: loss did not improve
    9s - loss: 145.6322 - val_loss: 149.5500
    Epoch 111/128
    Epoch 00110: loss improved from 145.63216 to 145.63096, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.6310 - val_loss: 149.6161
    Epoch 112/128
    Epoch 00111: loss improved from 145.63096 to 145.60660, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    10s - loss: 145.6066 - val_loss: 149.5684
    Epoch 113/128
    Epoch 00112: loss improved from 145.60660 to 145.60015, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 145.6002 - val_loss: 149.5835
    Epoch 114/128
    Epoch 00113: loss improved from 145.60015 to 145.59763, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 145.5976 - val_loss: 149.5670
    Epoch 115/128
    Epoch 00114: loss improved from 145.59763 to 145.59681, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 145.5968 - val_loss: 149.5997
    Epoch 116/128
    Epoch 00115: loss did not improve
    9s - loss: 145.5982 - val_loss: 149.5843
    Epoch 117/128
    Epoch 00116: loss did not improve
    9s - loss: 145.6055 - val_loss: 149.5793
    Epoch 118/128
    Epoch 00117: loss improved from 145.59681 to 145.58814, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 145.5881 - val_loss: 149.6058
    Epoch 119/128
    Epoch 00118: loss did not improve
    9s - loss: 145.5951 - val_loss: 149.6007
    Epoch 120/128
    Epoch 00119: loss did not improve
    9s - loss: 145.5903 - val_loss: 149.5758
    Epoch 121/128
    Epoch 00120: loss did not improve
    9s - loss: 145.5914 - val_loss: 149.5590
    Epoch 122/128
    Epoch 00121: loss improved from 145.58814 to 145.58057, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 145.5806 - val_loss: 149.5911
    Epoch 123/128
    Epoch 00122: loss improved from 145.58057 to 145.58024, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 145.5802 - val_loss: 149.5867
    Epoch 124/128
    Epoch 00123: loss did not improve
    9s - loss: 145.5820 - val_loss: 149.5984
    Epoch 125/128
    Epoch 00124: loss did not improve
    9s - loss: 145.5804 - val_loss: 149.5401
    Epoch 126/128
    Epoch 00125: loss improved from 145.58024 to 145.56877, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 145.5688 - val_loss: 149.5702
    Epoch 127/128
    Epoch 00126: loss did not improve
    9s - loss: 145.5734 - val_loss: 149.6175
    Epoch 128/128
    Epoch 00127: loss improved from 145.56877 to 145.56005, saving model to /data/vision/fisher/data1/vsmolyakov/vae/weights-checkpoint.hdf5
    9s - loss: 145.5601 - val_loss: 149.5803

Let's save the model and final weights, we can then load the saved model as follows:

```python
vae.save(DATA_PATH + 'final_model.h5', overwrite=True)
vae.save_weights(DATA_PATH + 'final_weights.h5', overwrite=True)

#load saved model
#vae = load_model(DATA_PATH + 'final_model.h5')
```

```python
encoder = Model(x, z_mean)

#generative model
decoder_input = Input(shape=(latent_dim,))
_h_decoded = decoder_h(decoder_input)
_x_decoded_mean = decoder_mean(_h_decoded)
generator = Model(decoder_input, _x_decoded_mean)
```

Let's visualize our training loss and the learning rate:

```python
#generate plots
plt.figure()
plt.plot(hist.history['loss'], lw=2.0, label='Adam')
plt.title('VAE model')
plt.xlabel('Epochs')
plt.ylabel('Training Loss')
plt.legend()
plt.savefig('./figures/vae_training_loss.png')

```

```python
plt.figure()
plt.plot(hist_lr.lr, lw=2.0, label='learning rate')
plt.title('VAE model')
plt.xlabel('Epochs')
plt.ylabel('Learning Rate')
plt.legend()
plt.savefig('./figures/vae_learning_rate.png')

```

We see a staircase pattern in the learning rate caused by a division by 2 of the initial learning rate every 16 epochs as desired.

Finally, we can scan the latent space to generate a manifold of images learned by the variational autoencoder. Since we know that our variational approximation $z \sim N(\mu, \sigma^2)$ is close to $N(0,1)$ due to the regularization loss, we can define a grid of dimension $h=2$ centered at zero and for each sample in that grid generate an image by feeding our samples through a decoder.

```python
n = 15 #number of samples
digit_size = 28
figure = np.zeros((digit_size * n, digit_size * n))
grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
grid_y = norm.ppf(np.linspace(0.05, 0.95, n))

for i, yi in enumerate(grid_x):
    for j, xi in enumerate(grid_y):
        z_sample = np.array([[xi, yi]])
        x_decoded = generator.predict(z_sample)
        digit = x_decoded[0].reshape(digit_size, digit_size)
        figure[i * digit_size: (i+1) * digit_size,
               j * digit_size: (j+1) * digit_size] = digit

```

```python
plt.figure()
plt.imshow(figure)
plt.savefig('./figures/vae_manifold.png')
```

## References
[1] D. Kingma et al., "Auto-Encoding Variational Bayes", ICLR 2014
[2] F. Chollet, "Keras Blog", https://blog.keras.io/


# Ordinary Differential Equation Solvers: Runge-Kutta Methods
## Christina Lee

So what's an Ordinary Differential Equation?

Differential Equation means we have some equation (or equations) that have derivatives in them.

The ordinary part differentiates them from partial differential equations (the ones with curly $\partial$ derivatives).  Here, we only have one independent variable (let's call it $t$), and one or more dependent  variables (let's call them $x_1, x_2, ...$).  In partial differential equations, we can have more than one independent variable.

This ODE can either be written as a system of the form
$$
\frac{d x_1}{dt}=f_1(t,x_1,x_2,...,x_n)
$$
$$
\frac{d x_2}{dt}=f_2(t,x_1,x_2,...,x_n)
$$
...
$$
\frac{d x_n}{dt}=f_n(t,x_1,x_n,...,x_n)
$$
or a single n'th order ODE of the form
$$
f_n(t,x) \frac{d^n x}{dt^n}+...+f_1(t,x) \frac{dx}{dt}+f_0(t,x)=0
$$
that can be rewritten in terms of a system of first order equations by performing variable substitutions such as
$$
\frac{d^i x}{dt^i}=x_i
$$

Though STEM students such as I have probably spent thousands of hours pouring of ways to analytically solve both ordinary and partial differential equations, unfortunately, the real world is rarely so kind as to provide us with an exactly solvable differential equation.  At least for interesting problems.

We can sometimes approximate the real world as an exactly solvable situation, but for the situation we are interested in, we have to turn to numerics.  I'm not saying those thousand different analytic methods are for nothing.  We need an idea ahead of time of what the differential equation should be doing, to tell if it's misbehaving or not.  We can't just blindly plug and chug.

Today will be about introducing four different methods based on Taylor expansion to a specific order, also known as Runge-Kutta Methods.  We can improve these methods with adaptive stepsize control, but that is a topic for another time, just like the other modern types of solvers such as Richardson extrapolation and predictor-corrector.

Nonetheless, to work with ANY computational differential equation solver, you need to understand the fundamentals of routines like Euler and Runge-Kutta, their error propagation, and where they can go wrong. Otherwise, you might misinterpret the results of more advanced methods.

WARNING: If you need to solve a troublesome differential equation for a research problem, use a package, like [DifferentialEquations](https://github.com/JuliaDiffEq/DifferentialEquations.jl).  These packages have much better error handling and optimization.

Let's first add our plotting package and colors.

```julia
using Plots
plotlyjs()

```

    Plots.PlotlyJSBackend()

We will be benchmarking our solvers on one of the simplest and most common ODE's,

\begin{equation}
\frac{d}{d t}x=x \;\;\;\;\;\;\; x(t)=C e^t
\end{equation}

Though this only has one dependent variable, we want to structure our code so that we can accommodate a series of dependent variables, $y_1,y_2,...,y_n$, and their associated derivative functions.  Therefore, we create a function for each dependent variable, and then `push` it into an array declared as type `Function`.

```julia
function f1(t::Float64,x::Array{Float64,1})
    return x[1]
end
f=Function[]
push!(f,f1)

```

    1-element Array{Function,1}:
     f1

### Euler's Method

First published in Euler's Instutionum calculi integralis in 1768, this method gets a lot of milage, and if you are to understand anything, this method is it.

We march along with step size $h$, and at each new point, calculate the slope.  The slope gives us our new direction to travel for the next $h$.

We can determine the error from the Taylor expansion of the function
$$
x_{n+1}=x_n+h f(x_n,t) + \mathcal{O}(h^2).
$$
In case you haven't seen it before, the notation $\mathcal{O}(x)$ stands for "errors of the order x".
Summing over the entire interval, we accumuluate error according to
$$N\mathcal{O}(h^2)= \frac{x_f-x_0}{h}\mathcal{O}(h^2)=h $$,
making this a first order method.  Generally, if a technique is $n$th order in the Taylor expansion for one step, its $(n-1)$th order over the interval.

```julia
function Euler(f::Array{Function,1},t0::Float64,x::Array{Float64,1},h::Float64)
    d=length(f)
    xp=copy(x)
    for ii in 1:d
        xp[ii]+=h*f[ii](t0,x)
    end

    return t0+h,xp
end
```

    Euler (generic function with 1 method)

```julia

```

## Implicit Method or Backward Euler

If $f(t,x)$ has a form that is invertible, we can form a specific expression for the next step.  For example, if we use our exponential,
\begin{equation}
x_{n+1}=x_n+ h f(t_{n+1},x_{n+1})
\end{equation}
\begin{equation}
x_{n+1}-h x_{n+1}=x_n
\end{equation}
\begin{equation}
x_{n+1}=\frac{x_n}{1-h}
\end{equation}

This expression varies for each differential equation and only exists if the function is invertible.

```julia

function Implicit(f::Array{Function,1},t0::Float64,x::Array{Float64,1},h::Float64)
    return t0+h,x[1]/(1-h)
end
```

    Implicit (generic function with 1 method)

## 2nd Order Runge-Kutta

So in the Euler Method, we could just make more, tinier steps to achieve more precise results. Here, we make bettter steps.  Each step itself takes more work than a step in the first order methods, but we win by having to perform fewer steps.

This time, we are going to work with the Taylor expansion up to second order,
\begin{equation}
x_{n+1}=x_n+h f(t_n,x_n) + \frac{h^2}{2} f^{\prime}(t_n,x_n)+ \mathcal{O} (h^3).
\end{equation}

Define
\begin{equation}
k_1=f(t_n,x_n),
\end{equation}
so that we can write down the derivative of our $f$ function, and the second derivative (curvature), of our solution,
\begin{equation}
f^{\prime}(t_n,x_n)=\frac{f(t_n+h/2,x_n+h k_1/2)-k_1}{h/2}+\mathcal{O}(h^2).
\end{equation}
Plugging this expression back into our Taylor expanion, we get a new expression for $x_{n+1}$
\begin{equation}
x_{n+1}=x_n+hf(t_n+h/2,x_n+h k_1/2)+\mathcal{O}(h^3)
\end{equation}

We can also interpret this technique as using the slope at the center of the interval, instead of the start.

```julia
function RK2(f::Array{Function,1},t0::Float64,x::Array{Float64,1},h::Float64)
    d=length(f)

    xp=copy(x)
    xk1=copy(x)

    for ii in 1:d
        xk1[ii]+=f[ii](t0,x)*h/2
    end
    for ii in 1:d
        xp[ii]+=f[ii](t0+h/2,xk1)*h
    end

    return t0+h,xp
end
```

    RK2 (generic function with 1 method)

## 4th Order Runge-Kutta
Wait! Where's 3rd order? There exists a 3rd order method, but I only just heard about it while fact-checking for this post.  RK4 is your dependable, multi-purpose workhorse, so we are going to skip right to it.

$$
k_1= f(t_n,x_n)
$$
$$
k_2= f(t_n+h/2,x_n+h k_1/2)
$$
$$
k_3 = f(t_n+h/2, x_n+h k_2/2)
$$
$$
k_4 = f(t_n+h,x_n+h k_3)
$$
$$
x_{n+1}=x_n+\frac{h}{6}\left(k_1+2 k_2+ 2k_3 +k_4 \right)
$$
I'm not going to prove here that the method is fourth order, but we will see numerically that it is.

Note: I premultiply the $h$ in my code to reduce the number of times I have to multiply $h$.

```julia
function RK4(f::Array{Function,1},t0::Float64,x::Array{Float64,1},h::Float64)
    d=length(f)

    hk1=zeros(Float64,length(x))
    hk2=zeros(Float64,length(x))
    hk3=zeros(Float64,length(x))
    hk4=zeros(Float64,length(x))

    for ii in 1:d
        hk1[ii]=h*f[ii](t0,x)
    end
    for ii in 1:d
        hk2[ii]=h*f[ii](t0+h/2,x+hk1/2)
    end
    for ii in 1:d
        hk3[ii]=h*f[ii](t0+h/2,x+hk2/2)
    end
    for ii in 1:d
        hk4[ii]=h*f[ii](t0+h,x+hk3)
    end

    return t0+h,x+(hk1+2*hk2+2*hk3+hk4)/6
end
```

    RK4 (generic function with 1 method)

This next function merely iterates over a certain number of steps for a given method.

```julia
function Solver(f::Array{Function,1},Method::Function,t0::Float64,
        x0::Array{Float64,1},h::Float64,N::Int64)
    d=length(f)
    ts=zeros(Float64,N+1)
    xs=zeros(Float64,d,N+1)

    ts[1]=t0
    xs[:,1]=x0

    for i in 2:(N+1)
        ts[i],xs[:,i]=Method(f,ts[i-1],xs[:,i-1],h)
    end

    return ts,xs
end
```

    Solver (generic function with 1 method)

```julia
N=1000
xf=10
t0=0.
x0=[1.]
dt=(xf-t0)/N

tEU,xEU=Solver(f,Euler,t0,x0,dt,N);
tIm,xIm=Solver(f,Implicit,t0,x0,dt,N);
tRK2,xRK2=Solver(f,RK2,t0,x0,dt,N);
tRK4,xRK4=Solver(f,RK4,t0,x0,dt,N);

xi=tEU
yi=exp(xi);

errEU=reshape(xEU[1,:],N+1)-yi
errIm=reshape(xIm[1,:],N+1)-yi
errRK2=reshape(xRK2[1,:],N+1)-yi;
errRK4=reshape(xRK4[1,:],N+1)-yi;
```

```julia
plot(x=tEU,y=xEU[1,:],Geom.point,
Theme(highlight_width=0pt,default_color=green,
default_point_size=3pt))
```

```julia
lEU=layer(x=tEU,y=xEU[1,:],Geom.point,
Theme(highlight_width=0pt,default_color=green,
default_point_size=3pt))

lIm=layer(x=tIm,y=xIm[1,:],Geom.point,
Theme(highlight_width=0pt,default_color=yellow,
default_point_size=3pt))

lRK2=layer(x=tRK2,y=xRK2[1,:],Geom.point,
Theme(highlight_width=0pt,default_color=cyan,
default_point_size=2pt))

lRK4=layer(x=tRK4,y=xRK4[1,:],Geom.point,
Theme(highlight_width=0pt,default_color=violet,
default_point_size=4pt))

lp=layer(x->e^x,-.1,10,Geom.line,Theme(default_color=red))

plot(lp,lEU,lIm,lRK2,lRK4,
Guide.manual_color_key("Legend",["Euler","Implicit","RK2","RK4","Exact"],
[green,yellow,cyan,violet,red]),
Coord.cartesian(xmin=9.5,xmax=10.1))
```

```julia
lEU=layer(x=xi,y=errEU,Geom.point,
Theme(highlight_width=0pt,default_color=green,
default_point_size=1pt))

lIm=layer(x=xi,y=errIm,Geom.point,
Theme(highlight_width=0pt,default_color=yellow,
default_point_size=1pt))

lRK2=layer(x=xi,y=errRK2,Geom.point,
Theme(highlight_width=0pt,default_color=cyan,
default_point_size=1pt))

lRK4=layer(x=xi,y=errRK4,Geom.point,
Theme(highlight_width=0pt,default_color=violet,
default_point_size=1pt))

plot(lEU,lIm,lRK2,lRK4,Scale.y_asinh,
Guide.manual_color_key("Legend",["Euler","Implicit","RK2","RK4"],
[green,yellow,cyan,violet]))
```

## Scaling of the Error

I talked above about the error scaling either as $h,h^2$, or $h^4$.  I won't just talk but here will numerically demonstrate the relationship as well.  For a variety of different step sizes, the below code calculates the final error for each method.  Then we will plot the error and see how it scales.

```julia
t0=0.
tf=1.
dx=tf-t0
x0=[1.]

dt=collect(.001:.0001:.01)

correctans=exp(tf)
errfEU=zeros(dt)
errfIm=zeros(dt)
errfRK2=zeros(dt)
errfRK4=zeros(dt)

for ii in 1:length(dt)
    N=round(Int,dx/dt[ii])
    dt[ii]=dx/N

    tEU,xEU=Solver(f,Euler,t0,x0,dt[ii],N);
    tIm,xIm=Solver(f,Implicit,t0,x0,dt[ii],N);
    tRK2,xRK2=Solver(f,RK2,t0,x0,dt[ii],N);
    tRK4,xRK4=Solver(f,RK4,t0,x0,dt[ii],N);

    errfEU[ii]=xEU[1,end]-correctans
    errfIm[ii]=xIm[1,end]-correctans
    errfRK2[ii]=xRK2[1,end]-correctans
    errfRK4[ii]=xRK4[1,end]-correctans
end
```

```julia
plot(x=dt,y=errfEU)
```

```julia
lEU=layer(x=dt,y=errfEU,Geom.point,
Theme(highlight_width=0pt,default_color=green,
default_point_size=1pt))

lIm=layer(x=dt,y=errfIm,Geom.point,
Theme(highlight_width=0pt,default_color=yellow,
default_point_size=1pt))

lRK2=layer(x=dt,y=errfRK2*10^5,Geom.point,
Theme(highlight_width=0pt,default_color=cyan,
default_point_size=1pt))

lRK4=layer(x=dt,y=errfRK4*10^10,Geom.point,
Theme(highlight_width=0pt,default_color=violet,
default_point_size=1pt))

tempEU(x)=(errfEU[end]*x/.01)
tempIm(x)=(errfIm[end]*x/.01)
#le=layer([tempEU,tempIm],0,.01,Geom.line,Theme(default_color=base01))
le=layer(tempEU,0,.01,Geom.line,Theme(default_color=base01))
lei=layer(tempIm,0,.01,Geom.line,Theme(default_color=base01))

temp2(x)=(errfRK2[end]*(x/.01)^2*10^5)
l2=layer(temp2,0,.01,Geom.line,Theme(default_color=base00))

temp4(x)=(errfRK4[end]*(x/.01)^4*10^10)
l4=layer(temp4,0,.01,Geom.line,Theme(default_color=base00))

xl=Guide.xlabel("h")
ylrk2=Guide.ylabel("Error e-5")
ylrk4=Guide.ylabel("Error e-10")
yl=Guide.ylabel("Error")

pEU=plot(lEU,lIm,le,lei,xl,yl,Guide.title("Euler and Implicit, linear error"))
p2=plot(lRK2,l2,xl,ylrk2,Guide.title("RK2, error h^2"))
p4=plot(lRK4,l4,xl,ylrk4,Guide.title("RK4, error h^4"))
vstack(hstack(p2,p4),pEU)
```

## Arbitrary Order
While I have presented 4 concrete examples, many more exist. For any choice of variables $a_i, \beta_{i,j},a_i$ that fulfill
$$
x_{n+1}=x_n+h\left(\sum_{i=1}^s a_i k_i \right)+ \mathcal{O}(h^p)
$$
with
$$
k_i=f\left(t_n+\alpha_i h,x_n+h\left(\sum_{j=1}^s \beta_{ij} k_j \right) \right)
$$
we have a Runge-Kutta method of order $p$, where $p\geq s$.  The Butcher tableau provides a set of consistent coefficients.

These routines are also present in the M4 folder in a module named `diffeq.jl`. For later work, you may simply import the module.

Stay tuned for when we tuned these routines to the stiff van der Pol equations!

```julia

```


# Detrending, Stylized Facts and the Business Cycle

In an influential article, Harvey and Jaeger (1993) described the use of unobserved components models (also known as "structural time series models") to derive stylized facts of the business cycle.

Their paper begins:

    "Establishing the 'stylized facts' associated with a set of time series is widely considered a crucial step
    in macroeconomic research ... For such facts to be useful they should (1) be consistent with the stochastic
    properties of the data and (2) present meaningful information."

In particular, they make the argument that these goals are often better met using the unobserved components approach rather than the popular Hodrick-Prescott filter or Box-Jenkins ARIMA modeling techniques.

Statsmodels has the ability to perform all three types of analysis, and below we follow the steps of their paper, using a slightly updated dataset.

```python
%matplotlib inline

import numpy as np
import pandas as pd
import statsmodels.api as sm
import matplotlib.pyplot as plt

from IPython.display import display, Latex
```

## Unobserved Components

The unobserved components model available in Statsmodels can be written as:

$$
y_t = \underbrace{\mu_{t}}_{\text{trend}} + \underbrace{\gamma_{t}}_{\text{seasonal}} + \underbrace{c_{t}}_{\text{cycle}} + \sum_{j=1}^k \underbrace{\beta_j x_{jt}}_{\text{explanatory}} + \underbrace{\varepsilon_t}_{\text{irregular}}
$$

see Durbin and Koopman 2012, Chapter 3 for notation and additional details. Notice that different specifications for the different individual components can support a wide range of models. The specific models considered in the paper and below are specializations of this general equation.

### Trend

The trend component is a dynamic extension of a regression model that includes an intercept and linear time-trend.

$$
\begin{align}
\underbrace{\mu_{t+1}}_{\text{level}} & = \mu_t + \nu_t + \eta_{t+1} \qquad & \eta_{t+1} \sim N(0, \sigma_\eta^2) \\\\
\underbrace{\nu_{t+1}}_{\text{trend}} & = \nu_t + \zeta_{t+1} & \zeta_{t+1} \sim N(0, \sigma_\zeta^2) \\
\end{align}
$$

where the level is a generalization of the intercept term that can dynamically vary across time, and the trend is a generalization of the time-trend such that the slope can dynamically vary across time.

For both elements (level and trend), we can consider models in which:

- The element is included vs excluded (if the trend is included, there must also be a level included).
- The element is deterministic vs stochastic (i.e. whether or not the variance on the error term is confined to be zero or not)

The only additional parameters to be estimated via MLE are the variances of any included stochastic components.

This leads to the following specifications:

|                                                                      | Level | Trend | Stochastic Level | Stochastic Trend |
|----------------------------------------------------------------------|-------|-------|------------------|------------------|
| Constant                                                             | ✓     |       |                  |                  |
| Local Level
 (random walk)                                     | ✓     |       | ✓                |                  |
| Deterministic trend                                                  | ✓     | ✓     |                  |                  |
| Local level with deterministic trend
 (random walk with drift) | ✓     | ✓     | ✓                |                  |
| Local linear trend                                                   | ✓     | ✓     | ✓                | ✓                |
| Smooth trend
 (integrated random walk)                         | ✓     | ✓     |                  | ✓                |

### Seasonal

The seasonal component is written as:

$$
\gamma_t = - \sum_{j=1}^{s-1} \gamma_{t+1-j} + \omega_t \qquad \omega_t \sim N(0, \sigma_\omega^2)
$$

The periodicity (number of seasons) is `s`, and the defining character is that (without the error term), the seasonal components sum to zero across one complete cycle. The inclusion of an error term allows the seasonal effects to vary over time.

The variants of this model are:

- The periodicity `s`
- Whether or not to make the seasonal effects stochastic.

If the seasonal effect is stochastic, then there is one additional parameter to estimate via MLE (the variance of the error term).

### Cycle

The cyclical component is intended to capture cyclical effects at time frames much longer than captured by the seasonal component. For example, in economics the cyclical term is often intended to capture the business cycle, and is then expected to have a period between "1.5 and 12 years" (see Durbin and Koopman).

The cycle is written as:

$$
\begin{align}
c_{t+1} & = c_t \cos \lambda_c + c_t^* \sin \lambda_c + \tilde \omega_t \qquad & \tilde \omega_t \sim N(0, \sigma_{\tilde \omega}^2) \\\\
c_{t+1}^* & = -c_t \sin \lambda_c + c_t^* \cos \lambda_c + \tilde \omega_t^* & \tilde \omega_t^* \sim N(0, \sigma_{\tilde \omega}^2)
\end{align}
$$

The parameter $\lambda_c$ (the frequency of the cycle) is an additional parameter to be estimated by MLE. If the seasonal effect is stochastic, then there is one another parameter to estimate (the variance of the error term - note that both of the error terms here share the same variance, but are assumed to have independent draws).

### Irregular

The irregular component is assumed to be a white noise error term. Its variance is a parameter to be estimated by MLE; i.e.

$$
\varepsilon_t \sim N(0, \sigma_\varepsilon^2)
$$

In some cases, we may want to generalize the irregular component to allow for autoregressive effects:

$$
\varepsilon_t = \rho(L) \varepsilon_{t-1} + \epsilon_t, \qquad \epsilon_t \sim N(0, \sigma_\epsilon^2)
$$

In this case, the autoregressive parameters would also be estimated via MLE.

### Regression effects

We may want to allow for explanatory variables by including additional terms

$$
\sum_{j=1}^k \beta_j x_{jt}
$$

or for intervention effects by including

$$
\begin{align}
\delta w_t \qquad \text{where} \qquad w_t & = 0, \qquad t < \tau, \\\\
& = 1, \qquad t \ge \tau
\end{align}
$$

These additional parameters could be estimated via MLE or by including them as components of the state space formulation.

## Data

Following Harvey and Jaeger, we will consider the following time series:

- US real GNP, "output", ([GNPC96](https://research.stlouisfed.org/fred2/series/GNPC96))
- US GNP implicit price deflator, "prices", ([GNPDEF](https://research.stlouisfed.org/fred2/series/GNPDEF))
- US monetary base, "money", ([AMBSL](https://research.stlouisfed.org/fred2/series/AMBSL))

The time frame in the original paper varied across series, but was broadly 1954-1989. Below we use data from the period 1948-2008 for all series. Although the unobserved components approach allows isolating a seasonal component within the model, the series considered in the paper, and here, are already seasonally adjusted.

All data series considered here are taken from [Federal Reserve Economic Data (FRED)](https://research.stlouisfed.org/fred2/). Conveniently, the Python library [Pandas](https://pandas.pydata.org/) has the ability to download data from FRED directly.

```python
# Datasets
from pandas_datareader.data import DataReader

# Get the raw data
start = '1948-01'
end = '2008-01'
us_gnp = DataReader('GNPC96', 'fred', start=start, end=end)
us_gnp_deflator = DataReader('GNPDEF', 'fred', start=start, end=end)
us_monetary_base = DataReader('AMBSL', 'fred', start=start, end=end).resample('QS').mean()
recessions = DataReader('USRECQ', 'fred', start=start, end=end).resample('QS').last().values[:,0]

# Construct the dataframe
dta = pd.concat(map(np.log, (us_gnp, us_gnp_deflator, us_monetary_base)), axis=1)
dta.columns = ['US GNP','US Prices','US monetary base']
dates = dta.index._mpl_repr()
```

To get a sense of these three variables over the timeframe, we can plot them:

```python
# Plot the data
ax = dta.plot(figsize=(13,3))
ylim = ax.get_ylim()
ax.xaxis.grid()
ax.fill_between(dates, ylim[0]+1e-5, ylim[1]-1e-5, recessions, facecolor='k', alpha=0.1);
```

## Model

Since the data is already seasonally adjusted and there are no obvious explanatory variables, the generic model considered is:

$$
y_t = \underbrace{\mu_{t}}_{\text{trend}} + \underbrace{c_{t}}_{\text{cycle}} + \underbrace{\varepsilon_t}_{\text{irregular}}
$$

The irregular will be assumed to be white noise, and the cycle will be stochastic and damped. The final modeling choice is the specification to use for the trend component. Harvey and Jaeger consider two models:

1. Local linear trend (the "unrestricted" model)
2. Smooth trend (the "restricted" model, since we are forcing $\sigma_\eta = 0$)

Below, we construct `kwargs` dictionaries for each of these model types. Notice that rather that there are two ways to specify the models. One way is to specify components directly, as in the table above. The other way is to use string names which map to various specifications.

```python
# Model specifications

# Unrestricted model, using string specification
unrestricted_model = {
    'level': 'local linear trend', 'cycle': True, 'damped_cycle': True, 'stochastic_cycle': True
}

# Unrestricted model, setting components directly
# This is an equivalent, but less convenient, way to specify a
# local linear trend model with a stochastic damped cycle:
# unrestricted_model = {
#     'irregular': True, 'level': True, 'stochastic_level': True, 'trend': True, 'stochastic_trend': True,
#     'cycle': True, 'damped_cycle': True, 'stochastic_cycle': True
# }

# The restricted model forces a smooth trend
restricted_model = {
    'level': 'smooth trend', 'cycle': True, 'damped_cycle': True, 'stochastic_cycle': True
}

# Restricted model, setting components directly
# This is an equivalent, but less convenient, way to specify a
# smooth trend model with a stochastic damped cycle. Notice
# that the difference from the local linear trend model is that
# `stochastic_level=False` here.
# unrestricted_model = {
#     'irregular': True, 'level': True, 'stochastic_level': False, 'trend': True, 'stochastic_trend': True,
#     'cycle': True, 'damped_cycle': True, 'stochastic_cycle': True
# }
```

We now fit the following models:

1. Output, unrestricted model
2. Prices, unrestricted model
3. Prices, restricted model
4. Money, unrestricted model
5. Money, restricted model

```python
# Output
output_mod = sm.tsa.UnobservedComponents(dta['US GNP'], **unrestricted_model)
output_res = output_mod.fit(method='powell', disp=False)

# Prices
prices_mod = sm.tsa.UnobservedComponents(dta['US Prices'], **unrestricted_model)
prices_res = prices_mod.fit(method='powell', disp=False)

prices_restricted_mod = sm.tsa.UnobservedComponents(dta['US Prices'], **restricted_model)
prices_restricted_res = prices_restricted_mod.fit(method='powell', disp=False)

# Money
money_mod = sm.tsa.UnobservedComponents(dta['US monetary base'], **unrestricted_model)
money_res = money_mod.fit(method='powell', disp=False)

money_restricted_mod = sm.tsa.UnobservedComponents(dta['US monetary base'], **restricted_model)
money_restricted_res = money_restricted_mod.fit(method='powell', disp=False)
```

Once we have fit these models, there are a variety of ways to display the information. Looking at the model of US GNP, we can summarize the fit of the model using the `summary` method on the fit object.

```python
print(output_res.summary())
```

For unobserved components models, and in particular when exploring stylized facts in line with point (2) from the introduction, it is often more instructive to plot the estimated unobserved components (e.g. the level, trend, and cycle) themselves to see if they provide a meaningful description of the data.

The `plot_components` method of the fit object can be used to show plots and confidence intervals of each of the estimated states, as well as a plot of the observed data versus the one-step-ahead predictions of the model to assess fit.

```python
fig = output_res.plot_components(legend_loc='lower right', figsize=(15, 9));
```

Finally, Harvey and Jaeger summarize the models in another way to highlight the relative importances of the trend and cyclical components; below we replicate their Table I. The values we find are broadly consistent with, but different in the particulars from, the values from their table.

```python
# Create Table I
table_i = np.zeros((5,6))

start = dta.index[0]
end = dta.index[-1]
time_range = '%d:%d-%d:%d' % (start.year, start.quarter, end.year, end.quarter)
models = [
    ('US GNP', time_range, 'None'),
    ('US Prices', time_range, 'None'),
    ('US Prices', time_range, r'$\sigma_\eta^2 = 0$'),
    ('US monetary base', time_range, 'None'),
    ('US monetary base', time_range, r'$\sigma_\eta^2 = 0$'),
]
index = pd.MultiIndex.from_tuples(models, names=['Series', 'Time range', 'Restrictions'])
parameter_symbols = [
    r'$\sigma_\zeta^2$', r'$\sigma_\eta^2$', r'$\sigma_\kappa^2$', r'$\rho$',
    r'$2 \pi / \lambda_c$', r'$\sigma_\varepsilon^2$',
]

i = 0
for res in (output_res, prices_res, prices_restricted_res, money_res, money_restricted_res):
    if res.model.stochastic_level:
        (sigma_irregular, sigma_level, sigma_trend,
         sigma_cycle, frequency_cycle, damping_cycle) = res.params
    else:
        (sigma_irregular, sigma_level,
         sigma_cycle, frequency_cycle, damping_cycle) = res.params
        sigma_trend = np.nan
    period_cycle = 2 * np.pi / frequency_cycle

    table_i[i, :] = [
        sigma_level*1e7, sigma_trend*1e7,
        sigma_cycle*1e7, damping_cycle, period_cycle,
        sigma_irregular*1e7
    ]
    i += 1

pd.set_option('float_format', lambda x: '%.4g' % np.round(x, 2) if not np.isnan(x) else '-')
table_i = pd.DataFrame(table_i, index=index, columns=parameter_symbols)
table_i
```


# Solving Optimal Stopping problems using LCP:

## Introduction:

An optimal stopping problem is a problem that requires an agent to choose an optimal stopping time in order to maximize (or minimize) some value.

An example of a simple optimal stopping problem is choosing the optimal shut-down time of a plant. Given a state variable $x_t$, continuing the plant's operations at time $t$ one obtains profit $u(x_t)$ discounted by $e^{-\rho t}$, where $\rho$ is the discount factor. While choosing to shut down the plant at time $t=\tau$ one no longer ooptains any profits from the plant, however, one obtains a one time profit of $S(x_{\tau})$ discounted by $e^{-\rho \tau}$. We can think of this value $S(x_{\tau})$ as the profit from liquidating the assets of the plant.

To make this problem simple, we can assume that $x_t$ is exogenous, that is, the agent has no control over $\{x_{t}\}_{t=0}^{\infty}$.

We could for instance consider the case where $x_t$ follows a Brownian diffusion

$$x_t = \mu(x_t)dt + \sigma(x_t)dW_t$$

To ensure that the plant will eventually shut down, we consider the case where $\mu(x) < 0$ for all $x$.

Thus, we can formulate the optimal stopping problem as:

$$v(x) = \max_{\tau}\mathbb{E}_0\int_{0}^{\tau}e^{-\rho t} u(x_t)dt + e^{-\rho \tau}S(x_{\tau}),\hspace{1cm} dx_t = \mu(x_t)dt+\sigma(x_t)dt$$

## Hamilton-Jacobi Bellman Variational Inequality

For values of $x$ such that the plant does not exit, the following HJB equation holds

$$\rho v(x) = u(x) + \mu(x)v'(x) + \frac{\sigma^2(x)}{2}v''(x)$$

Define $X$ to be the set of $x$ for which there is no exit. Then

- $x\in X: v(x)\geq S(x)\hspace{1cm} $ (i.e the value of not exiting, $v(x)$, is at least as great as the value of exiting $S(x)$)

- $x\notin X: v(x) = S(x)\hspace{1cm}$ (i.e. the maximum possible value, $v(x)$, is the value of choosing to exit, $S(x)$

Which then leads to:

- $x\in X: v(x)\geq S(x), \rho v(x) = u(x)+\mu(x)v'(x) +\frac{\sigma^2(x)}{2}v''(x)$

- $x\notin X$: v(x) = S(x), \rho v(x) \geq u(x)+\mu(x)v'(x) +\frac{\sigma^2(x)}{2}v''(x)$

This can be written compactly as a HJB variational inequality:

$$\min\{\rho v(x) - u(x)-\mu(x)v'(x) -\frac{\sigma^2(x)}{2}v''(x), v(x) - S(x)\} = 0$$

## Solving the HJBVI as a Linear Complementarity Problem

Without the exit option the HJB can be discretized as

$$\rho v_i^n = u_i + \mu_i(v_i^n)' + \frac{\sigma^2_i}{2}(v_i^n)''$$

Denoting the differential operator on $v$ by $A$, we can equivalently write the above as

$$ \rho v = u + Av$$

Hence we can rewrite the earlier HJBVI in discretized form:

$$\min\{\rho v - u - Av, v- S\} = 0$$

Furthermore, this the above HJBVI can be equivalently written as:

- $(v-S)^T(\rho v - u - Av) = 0$

- $v\geq S$

- $\rho v - u - Av\geq 0 $

Noting the form of an LCP in the appendix, we can rewrite the above problem as follows.

Denote the "excess value" $z =  v - S$, $B = \rho I - A$, and $q = -u + BS$.

Then problem can be written as:

- $z^T(Bz + q) = 0$

- $z \geq 0$

- $Bz+q \geq  0$

This is the standard form for LCPs (as denoted in the appendix). We can thus proceed to solve this problem, obtaining a solution $z$, using a LCP solver. From there, we can obtain a solution to our original problem $v = z + S$.

Reference: Ben Moll's notes "Exercising an Option: Numerical Solution"

## Example Problem

We are given an optimal stopping problem:

$$v(x) = \max_{\tau}\mathbb{E}_0\int_{0}^{\tau}e^{-\rho t} u(x_t)dt + e^{-\rho \tau}S(x_{\tau}),\hspace{1cm} dx_t = \mu(x_t)dt+\sigma(x_t)dt$$

where the parameters of the stochastic process, discount factor, utility, and stopping-value are:

- $\mu(x) = -0.01x$ for all $x$
- $\sigma^2(x) = (0.01x)^2$
- $S(x)= 10$ for all $x$
- $\gamma = 0.5$ where $u(x)  = x^\gamma$
- $\rho = 0.05$

We will also discretize the problem on the interval $[0.01, 1.0]$ utilizing $1000$ grid points.

We create the stopping problem below:

```julia
using Parameters, LinearAlgebra

StoppingProblem = @with_kw (μ_bar = -0.01,
                            σ_bar = 0.01,
                            S_bar = 10.0,
                            γ = 0.5, # u(x) = x^γ
                            ρ = 0.05, # discount rate
                            x_min = 0.01,
                            x_max = 1.0,
                            M = 1000) # num of grid points
```

    #4 (generic function with 2 methods)

Let's also copy the discretization method from [PTW](https://github.com/jlperla/PerlaTonettiWaugh.jl/commits/master/src/utils/diffusionoperators.jl). This is for a reflecting barrier at each boundary.

-- in order to create the linear operator for the discretized problem.

```julia
# Diffusion operators with regular grids
function diffusionoperators(x::AbstractRange; ξ = 0.0) # ξ = 0.0 corresponds to the unrescaled case from PTW
    Δ = step(x)
    P = length(x)

    dl_1 = zeros(P-1)
    d_1 = -ones(P)
    du_1 = ones(P-1)
    d_1[end] = d_1[end] + du_1[end] * (1-ξ*Δ)
    L_1_plus = Tridiagonal(dl_1, d_1, du_1)/Δ

    dl_m1 = -ones(P-1)
    d_m1 = ones(P)
    du_m1 = zeros(P-1)
    d_m1[1] = d_m1[1] + dl_m1[1] * (1+ξ*Δ)
    L_1_minus = Tridiagonal(dl_m1, d_m1, du_m1)/Δ

    dl_2 = ones(P-1)
    d_2 = -2 * ones(P)
    d_2[1] = -2 + (1+ξ*Δ)
    d_2[end] = -2 + (1-ξ*Δ)
    du_2 = ones(P-1)
    L_2 = Tridiagonal(dl_2, d_2, du_2)/(Δ^2)

    return (x, L_1_minus, L_1_plus, L_2)
end
```

    diffusionoperators (generic function with 1 method)

### Approach 1: Simplest LCP with PATHSolver.jl
-------

Here, we consider the case with constrant drift $\mu(x) = \bar{\mu}$, stopping value $S(x) \equiv \bar{S}$, and variance $\sigma(x) = \bar{\sigma}$.

First, here's a function to take model parameters and return LCP objects.

```julia
function LCP_objects(sp)
    # setup work
    @unpack μ_bar, σ_bar, S_bar, γ, ρ, x_min, x_max, M = sp
    grid = range(x_min, x_max, length = M)
    tmp, L_1_minus, L_1_plus, L_2 = diffusionoperators(grid)
    μ = μ_bar
    S = x -> S_bar
    u = x -> x^γ # L36 from MATLAB
    σ = σ_bar
    # construct objects
    A = μ * L_1_minus + 1/2 * σ^2 * L_2 # stochastic generator. use L_1_minus because drift is always negative
    B = ρ*I - A
    S_vec = S.(grid)
    -u.(grid)
    q_vec = -u.(grid) + B*S_vec
    # return
    return (A = A, B = B, S = S_vec, q = q_vec)
end
```

    LCP_objects (generic function with 1 method)

Let's generate these objects for the default case.

```julia
sp = StoppingProblem()
@unpack A, B, S, q = LCP_objects(sp);
```

This suggests the following linear complementarity problem:

$$
\begin{align}
z'(\mathbf{B}z + q) &= 0 \\
z &\geq 0 \\
\mathbf{B}z + q &\geq 0
\end{align}
$$

Let's give it to PATHSolver.

```julia
using PATHSolver
```

```julia
f = z -> B*z + q
n = sp.M
lb = zeros(n)
ub = 1000*ones(n) # a reasonable guess?
options(convergence_tolerance = 1e-12, output = :no, time_limit = 600) # 10 minute budget
@time exit_code, sol_z, sol_f = solveLCP(f, lb, ub)
```

      1.484931 seconds (3.40 M allocations: 197.787 MiB, 4.39% gc time)
    Reading options file path.opt
     > convergence_tolerance 1.0e-12
    Read of options file complete.

    1000 row/cols, 2998 non-zeros, 0.30% dense.

    Path 4.7.03 (Thu Jan 24 15:44:12 2013)
    Written by Todd Munson, Steven Dirkse, and Michael Ferris

     ** EXIT - other error.

    (:StationaryPointFound, [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0  …  7.6867, 7.69559, 7.70406, 7.71202, 7.71937, 7.72598, 7.73171, 7.73638, 7.73977, 7.74162], [0.4, 0.395162, 0.390538, 0.386101, 0.381831, 0.37771, 0.373723, 0.369858, 0.366105, 0.362454  …  -3.58602e-14, 1.84297e-14, -7.07212e-14, 2.39808e-14, -3.58602e-14, 5.09592e-14, 8.43769e-14, -1.4877e-14, 7.63833e-14, 5.68434e-14])

Let's generate these objects for the default case.

```julia
exit_code # this lets us handle logic like converged(exit_code) || error("foobar")
```

    :StationaryPointFound

Now, we can unpack the solutions and plot them.

```julia
using Plots
v = sol_z + S
plot(v, title = "Value Function for Vanilla Model", legend = false, xlabel = "x", ylabel = "v(x)")
```

This is the kind of thing we expect --- below some threshold you get the stopping value $S \equiv 10$, and above it you get a smoothly increasing value.

## Appendix

### LCP Problems:

Source Wikipedia