```python
import numpy as np
from keras import optimizers

X = np.array([[0,0],[0,1],[1,0],[1,1]])
y = np.array([0,1,1,0])

from keras import models
from keras import layers

model = models.Sequential()
model.add(layers.Dense(2, activation='tanh'))
model.add(layers.Dense(1, activation='sigmoid'))
sgd = optimizers.SGD(lr=0.1)
model.compile(optimizer=sgd, loss='binary_crossentropy')
model.fit(X, y, epochs=1000, batch_size=1, verbose=0)
model.predict(X)

```

    array([[0.0076664 ],
           [0.9866918 ],
           [0.98638123],
           [0.00661352]], dtype=float32)

\begin{equation}
    \frac{\partial L}{\partial \vec{w}^{\,2}} =
    \begin{bmatrix}
        \frac{\partial L}{\partial {w}^{2}_{11}}\\
        \\
        \frac{\partial L}{\partial {w}^{2}_{21}} \\
    \end{bmatrix}
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {w}^{2}_{11}} = \frac{\partial L}{\partial {\hat{y}}_{1}}
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {w}^{2}_{11}}
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {\hat{y}}_{1}} =  (\frac{1-y}{(1-\hat{y}_{1})} - \frac{y}{\hat{y}_{1}  })
\end{equation}

\begin{equation}
    \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}} = 1
\end{equation}

\begin{equation}
    \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}} =
    \frac{\partial}{\partial {a}^{2}_{1}}
    \cdot \sigma ({a}^{2}_{1}) = \sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))
\end{equation}

\begin{equation}
    \frac{\partial {a}^{2}_{1}}{\partial {w}^{2}_{11}} = \frac{\partial}{\partial {w}^{2}_{11}} \cdot ({h}^{1}_{1} \cdot {w}^{2}_{11} + {h}^{1}_{2} \cdot {w}^{2}_{21} + b^{2}) = {h}^{1}_{1}
\end{equation}

\begin{equation}
   \frac{\partial L}{\partial {w}^{2}_{11}} = (\frac{1-y}{(1-\hat{y}_{1})} - \frac{y}{\hat{y}_{1}  }) \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
   \cdot {h}^{1}_{1})
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {w}^{2}_{21}} = \frac{\partial L}{\partial {\hat{y}}_{1}}
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {w}^{2}_{21}} =
    (\frac{1-y}{(1-\hat{y}_{1})  } - \frac{y}{\hat{y}_{1}  }) \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
   \cdot {h}^{1}_{2})
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {b}^{2}} = \frac{\partial L}{\partial {\hat{y}}_{1}}
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {b}^{2}} =
    (\frac{1-y}{(1-\hat{y}_{1})  } - \frac{y}{\hat{y}_{1}  }) \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
   \cdot 1)
\end{equation}

LAYER 1
\begin{equation}
        \frac{\partial L}{\partial \vec{w}^{\,1}} =
    \begin{bmatrix}
        \frac{\partial L}{\partial {w}^{1}_{11}} && \frac{\partial L}{\partial {w}^{1}_{12}}\\
        \\
        \frac{\partial L}{\partial {w}^{1}_{21}}  && \frac{\partial L}{\partial {w}^{1}_{22}}\\
    \end{bmatrix}
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {w}^{1}_{11}} = \frac{\partial L}{\partial {\hat{y}}_{1}}
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {h}^{1}_{1}}
    \cdot \frac{\partial {h}^{1}_{1}}{\partial {a}^{1}_{1}}
    \cdot \frac{\partial {a}^{1}_{1}}{\partial {w}^{1}_{11}}
    = (\frac{1-y}{(1-\hat{y}_{1})  } - \frac{y}{\hat{y}_{1}  }) \cdot 1
    \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
    \cdot {w}^{2}_{11}
    \cdot [1-\tanh{({a}^{1}_{1})}]
   \cdot {h}^{0}_{1})
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {w}^{1}_{11}} = \frac{\partial L}{\partial {\hat{y}}_{1}}
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {h}^{1}_{2}}
    \cdot \frac{\partial {h}^{1}_{2}}{\partial {a}^{1}_{2}}
    \cdot \frac{\partial {a}^{1}_{2}}{\partial {w}^{1}_{12}}
    = (\frac{1-y}{(1-\hat{y}_{1})  } - \frac{y}{\hat{y}_{1}  }) \cdot 1
    \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
    \cdot {w}^{2}_{21}
    \cdot [1-\tanh{({a}^{1}_{2})}]
   \cdot {h}^{0}_{1})
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {w}^{1}_{11}} = \frac{\partial L}{\partial {\hat{y}}_{1}}
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {h}^{1}_{1}}
    \cdot \frac{\partial {h}^{1}_{1}}{\partial {a}^{1}_{1}}
    \cdot \frac{\partial {a}^{1}_{1}}{\partial {w}^{1}_{21}}
    = (\frac{1-y}{(1-\hat{y}_{1})  } - \frac{y}{\hat{y}_{1}  }) \cdot 1
    \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
    \cdot {w}^{2}_{11}
    \cdot [1-\tanh{({a}^{1}_{1})}]
   \cdot {h}^{0}_{2})
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {w}^{1}_{11}} = \frac{\partial L}{\partial {\hat{y}}_{1}}
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {h}^{1}_{2}}
    \cdot \frac{\partial {h}^{1}_{2}}{\partial {a}^{1}_{2}}
    \cdot \frac{\partial {a}^{1}_{2}}{\partial {w}^{1}_{22}}
    = (\frac{1-y}{(1-\hat{y}_{1})  } - \frac{y}{\hat{y}_{1}  }) \cdot 1
    \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
    \cdot {w}^{2}_{21}
    \cdot [1-\tanh{({a}^{1}_{2})}]
   \cdot {h}^{0}_{2})
\end{equation}

BIASES

\begin{equation}
        \frac{\partial L}{\partial \vec{b}^{\,1}} =
    \begin{bmatrix}
        \frac{\partial L}{\partial {b}^{1}_{1}}\\
        \\
        \frac{\partial L}{\partial {b}^{1}_{2}}\\
    \end{bmatrix}
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {b}^{1}_{1}} = \frac{\partial L}{\partial {\hat{y}}_{1}}
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {h}^{1}_{1}}
    \cdot \frac{\partial {h}^{1}_{1}}{\partial {a}^{1}_{1}}
    \cdot \frac{\partial {a}^{1}_{1}}{\partial {b}^{1}_{1}}
    = (\frac{1-y}{(1-\hat{y}_{1})  } - \frac{y}{\hat{y}_{1}  }) \cdot 1
    \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
    \cdot {w}^{2}_{11}
    \cdot [1-\tanh{({a}^{1}_{1})}]
   \cdot {1})
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {b}^{1}_{2}} = \frac{\partial L}{\partial {\hat{y}}_{1}}
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {h}^{1}_{2}}
    \cdot \frac{\partial {h}^{1}_{2}}{\partial {a}^{1}_{2}}
    \cdot \frac{\partial {a}^{1}_{2}}{\partial {b}^{1}_{2}}
    = (\frac{1-y}{(1-\hat{y}_{1})} - \frac{y}{\hat{y}_{1}}) \cdot 1
    \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
    \cdot {w}^{2}_{21}
    \cdot [1-\tanh{({a}^{1}_{2})}]
   \cdot {1})
\end{equation}

```python
import numpy as np

class MyModel:
  def __init__(self):
    #Initialize weights with Xavier initialization
    self.weight_1 = np.random.normal(loc=0.0, scale = 1, size = (2,2))
    self.weight_2 = np.random.normal(loc=0.0, scale = 1, size = (2,1))

    #Initalize bias to zero
    self.bias_1 = np.zeros((2,1))
    self.bias_2 = np.zeros((1,1))

  def fit(self, X=None, y=None, epochs=1000, learning_rate=0.1):
    for _ in range(epochs):
      for i in range(len(X)):
        #Perform forward propagation
        yHat = self.forward(X[i].reshape((len(X[i]),1)))

        #Perform backward propagation
        gradient_weight_1, gradient_weight_2, gradient_bias_1, gradient_bias_2 = self.backward(y[i],yHat.item())

        #Simultaneously update all parameters
        self.weight_1 =  self.weight_1 - (learning_rate * gradient_weight_1)
        self.weight_2 = self.weight_2 - (learning_rate * gradient_weight_2)
        self.bias_1 = self.bias_1 - (learning_rate * gradient_bias_1)
        self.bias_2 = self.bias_2 - (learning_rate * gradient_bias_2)

  def forward(self, X):
    #Feed forward one sample at a time
    self.h_0 = X
    self.a_1 = np.dot(np.transpose(self.weight_1),X) + self.bias_1
    self.h_1 = np.tanh(self.a_1)
    self.a_2 = np.dot(np.transpose(self.weight_2),self.h_1) + self.bias_2
    self.h_2 = self.sigmoid(self.a_2)
    return self.h_2

  def backward(self, y, yHat):
    #Backward propergation one sample at a time

    #Calcualte partile derivitive
    dL_dy = self.binary_crossentropy_prime(y,yHat)
    dy_da21 = self.sigmoid_prime(self.a_2)

    #Calcualte gradients (Stacks values into matrix forms to simplify calculation)
    gradient_weight_2 =   np.full((2,1),dL_dy) * np.full((2,1),dy_da21) * self.h_1
    gradient_bias_2 =     np.full((1,1),dL_dy) * np.full((1,1),dy_da21)
    gradient_weight_1 =   np.full((2,2),dL_dy) * np.full((2,2),dy_da21) * np.transpose(np.column_stack((self.weight_2,self.weight_2))) * np.column_stack((self.tanh_prime(self.a_1),self.tanh_prime(self.a_1))).T * np.column_stack((self.h_0,self.h_0))
    gradient_bias_1 =     np.full((2,1),dL_dy) * np.full((2,1),dy_da21) * self.weight_2 * self.tanh_prime(self.a_1)

    return gradient_weight_1, gradient_weight_2, gradient_bias_1, gradient_bias_2

  def sigmoid(self,x):
      return 1/(1+np.exp(-x))

  def sigmoid_prime(self, x):
    return self.sigmoid(x) * (1.0 - self.sigmoid(x))

  def tanh(self, x ):
    return np.tanh(x)

  def tanh_prime(self, x):
    return 1 - np.tanh(x)**2

  def binary_crossentropy_prime(self,y, yHat):
    return -(y/yHat)+((1-y)/(1-yHat))

  def predict(self, X):
    results = np.zeros((len(X)))
    for i in range(len(X)):
       results [i] = self.forward(X[i].reshape((len(X[i]),1))).item()
    return results

X = np.array([[0,0],[0,1],[1,0],[1,1]])
y = np.array([0,1,1,0])

model = MyModel()
model.fit(X,y)

model.predict(X)

```

    array([0.01430112, 0.99191275, 0.9918608 , 0.01547358])

```python

```


```python
%matplotlib inline

from pssm.dglm import NormalDLM, PoissonDLM, BinomialDLM, CompositeDLM
from pssm.structure import UnivariateStructure, MultivariateStructure
import numpy as np
from scipy.stats import multivariate_normal as mvn
import matplotlib.pyplot as plt
plt.rcParams["figure.figsize"] =(12,4)
np.random.seed(42)
```

# Dynamic Generalised Linear Models

## Dynamic Linear Models

Dynamic linear models are a specific instance of state-space models where the observation and states are defined by

\begin{align}
Y_t|\theta_t,\Phi_t &\sim \mathcal{N}\left(\theta_t; \mathsf{F}^T\theta_{t}, V\right) \\
\theta_t|\theta_{t-1},\Phi_t &\sim \mathcal{N}\left(\theta_t;\mathsf{G}\theta_{t-1}, \mathsf{W}\right)
\end{align}

### Locally constant

The _locally constant_ DLM corresponds to a simple state random walk, where the structure is defined by

\begin{align}
\mathsf{F} &= \begin{bmatrix}1\end{bmatrix} \\
\mathsf{G} &= \begin{bmatrix}1\end{bmatrix}
\end{align}

This can be specified by using (in this case with a state variance of $\tau^2=3.4$):

```python
lc = {'structure': UnivariateStructure.locally_constant(3.4)}
print("F = {}, G = {}".format(lc['structure'].F, lc['structure'].G))
```

    F = [[1]], G = [[1]]

We can then instantiate a DLM (with observation variance $\sigma^2=1.4$):

```python
ndlm = NormalDLM(structure=lc['structure'], V=1.4)
```

Assuming we have a state prior of

$$\theta_0 \sim \mathcal{N}\left(0, 1\right)$$

we can then generate the states $\theta_{1:T}$ and observation $y_{1:T}$:

```python
# the initial state prior
m0 = np.array([0])
C0 = np.matrix([[1]])
state0 = mvn(m0, C0).rvs()

lc['states'] = [state0]

for t in range(1, 100):
    lc['states'].append(ndlm.state(lc['states'][t-1]))

lc['obs'] = [None]
for t in range(1, 100):
    lc['obs'].append(ndlm.observation(lc['states'][t]))
```

```python
t = range(100)
plt.plot(t, lc['states'])
plt.plot(t, lc['obs'])
plt.legend([r"$\theta_{1:T}$", r"$y_{1:T}$"], loc=1)
plt.title(r"Locally constant NDLM, $\tau^2=3.4, \sigma^2=1.4$")
plt.tight_layout()
```

### Locally linear

A _locally linear_ structure incorporates both an underlying mean and a trend. In this case the structure corresponds to
$$
\mathsf{F}=\begin{bmatrix}1 & 0\end{bmatrix}, \qquad \mathsf{G}=\begin{bmatrix}1 & 1 \\ 1 & 0 \end{bmatrix}.
$$
If we assume a state covariance of
$$
\mathsf{W}=\begin{bmatrix}0.1 & 0 \\ 0 & 0.1 \end{bmatrix}
$$
we can create the structure as:

```python
ll = {'structure': UnivariateStructure.locally_linear(W=np.matrix([[0.1, 0], [0, 0.1]]))}
```

and the Normal DLM structure can be created (with a state variance of $\sigma^2=2.5$) using:

```python
ndlm = NormalDLM(structure=ll['structure'], V=2.5)
```

In this case we will assume a state prior of
$$
\theta_0 \sim \mathcal{N}\left(\begin{bmatrix}0 & -1\end{bmatrix}, \begin{bmatrix}1 & 0 \\ 0 & 1 \end{bmatrix}\right)
$$
and we can generate the states and observations as:

```python
# the initial state prior
m0 = np.array([0, -1])
C0 = np.identity(2)
state0 = mvn(m0, C0).rvs()

ll['states'] = [state0]

for t in range(1, 100):
    ll['states'].append(ndlm.state(ll['states'][t-1]))

ll['obs'] = [None]
for t in range(1, 100):
    ll['obs'].append(ndlm.observation(ll['states'][t]))
```

```python
import matplotlib.pyplot as plt
t=range(100)
plt.subplot(1, 3, 1)
plt.plot(t, [state[0] for state in ll['states']])
plt.legend([r"$\theta_{1,1:T}$"], loc=0)
plt.title(r"LL NDLM $\theta_1$")
plt.subplot(1, 3, 2)
plt.plot(t, [state[1] for state in ll['states']])
plt.legend([r"$\theta_{2,1:T}$"], loc=0)
plt.title(r"LL NDLM $\theta_2$")
plt.subplot(1, 3, 3)
plt.plot(range(100), ll['obs'])
plt.legend([r"$y_{1:T}$"], loc=0)
plt.title(r"LL NDLM y")
plt.tight_layout()
```

### Fourier seasonality

```python
fourier = {'structure': UnivariateStructure.cyclic_fourier(period=10, harmonics=2, W=np.identity(4))}
```

```python
print("F(fourier) = {}".format(fourier['structure'].F))
print("G(fourier) = {}".format(fourier['structure'].G))
```

    F(fourier) = [[1.]
     [0.]
     [1.]
     [0.]]
    G(fourier) = [[ 0.80901699  0.58778525  0.          0.        ]
     [-0.58778525  0.80901699  0.          0.        ]
     [ 0.          0.          0.30901699  0.95105652]
     [ 0.          0.         -0.95105652  0.30901699]]

```python
ndlm_comp = NormalDLM(structure=(lc['structure']+fourier['structure']), V=2.5)
# the initial state prior
m0 = np.array([0, 0, 0, 0, 0])
C0 = np.identity(5)
state0 = np.random.multivariate_normal(m0, C0)

fourier['states'] = [state0]

for t in range(1, 100):
    fourier['states'].append(ndlm_comp.state(fourier['states'][t-1]))

fourier['obs'] = [None]
for t in range(1, 100):
    fourier['obs'].append(ndlm_comp.observation(fourier['states'][t]))

import matplotlib.pyplot as plt
plt.plot(range(100), fourier['states'])
plt.plot(range(100), fourier['obs'])
```

### ARMA

#### ARMA(1)

##### Normal DLM

```python
arma1 = {'structure': UnivariateStructure.arma(p=1, betas=[0.3], W=2.5)}
```

```python
print("F(ARMA(1)) = {}".format(arma1['structure'].F))
print("G(ARMA(1)) = {}".format(arma1['structure'].G))
print("W(ARMA(1)) = {}".format(arma1['structure'].W))
```

    F(ARMA(1)) = [[1.]]
    G(ARMA(1)) = [[0.3]]
    W(ARMA(1)) = [[2.5]]

```python
# the initial state prior
m0 = np.array([0])
C0 = np.identity(1)
state0 = mvn(m0, C0).rvs()

ndlm_arma1 = NormalDLM(structure=(arma1['structure']), V=2.5)

arma1['states'] = [state0]

for t in range(1, 100):
    arma1['states'].append(ndlm_arma1.state(arma1['states'][t-1]))

arma1['obs'] = [None]
for t in range(1, 100):
    arma1['obs'].append(ndlm_arma1.observation(arma1['states'][t]))
```

```python
t = range(100)
plt.plot(t, arma1['states'])
plt.plot(t, arma1['obs'])
plt.legend([r"$\theta_{1:T}$", r"$y_{1:T}$"], loc=1)
plt.title(r"ARMA(1) NDLM, $\tau^2=2.5, \sigma^2=2.5$")
plt.tight_layout()
```

##### Poisson DLM

```python
# the initial state prior
m0 = np.array([0])
C0 = np.identity(1)
state0 = mvn(m0, C0).rvs()

podlm_arma1 = PoissonDLM(structure=(arma1['structure']))

arma1['states'] = [state0]

for t in range(1, 100):
    arma1['states'].append(podlm_arma1.state(arma1['states'][t-1]))

arma1['obs'] = [None]
for t in range(1, 100):
    arma1['obs'].append(podlm_arma1.observation(arma1['states'][t]))
```

```python
t = range(100)
plt.plot(t, arma1['states'])
plt.plot(t, arma1['obs'], 'bo')
plt.legend([r"$\theta_{1:T}$", r"$y_{1:T}$"], loc=1)
plt.title(r"ARMA(1) PoDLM, $\tau^2=2.5")
plt.tight_layout()
```

#### ARMA(2)

##### Normal DLM

```python
arma2 = {'structure': UnivariateStructure.arma(p=2, betas=[0.3, 0.2], W=2.5)}
```

```python
print("F(ARMA(2)) = {}".format(arma2['structure'].F))
print("G(ARMA(2)) = {}".format(arma2['structure'].G))
print("W(ARMA(2)) = {}".format(arma2['structure'].W))
```

    F(ARMA(2)) = [[1.]
     [0.]]
    G(ARMA(2)) = [[0.3 0.2]
     [1.  0. ]]
    W(ARMA(2)) = [[2.5 0. ]
     [0.  0. ]]

```python
# the initial state prior
m0 = np.array([0, 0])
C0 = np.identity(2)
state0 = mvn(m0, C0).rvs()

ndlm_arma2 = NormalDLM(structure=(arma2['structure']), V=2.5)

arma2['states'] = [state0]

for t in range(1, 100):
    arma2['states'].append(ndlm_arma2.state(arma2['states'][t-1]))

arma2['obs'] = [None]
for t in range(1, 100):
    arma2['obs'].append(ndlm_arma2.observation(arma2['states'][t]))
```

```python
import matplotlib.pyplot as plt
t=range(100)
plt.subplot(1, 3, 1)
plt.plot(t, [state[0] for state in arma2['states']])
plt.legend([r"$\theta_{1,1:T}$"], loc=0)
plt.title(r"ARMA(2) NDLM $\theta_1$")
plt.subplot(1, 3, 2)
plt.plot(t, [state[1] for state in arma2['states']])
plt.legend([r"$\theta_{2,1:T}$"], loc=0)
plt.title(r"ARMA(2) NDLM $\theta_2$")
plt.subplot(1, 3, 3)
plt.plot(range(100), arma2['obs'])
plt.legend([r"$y_{1:T}$"], loc=0)
plt.title(r"ARMA(2) NDLM y")
plt.tight_layout()
```

##### Poisson DLM

```python
# the initial state prior
m0 = np.array([0, 0])
C0 = np.identity(2)
state0 = mvn(m0, C0).rvs()

podlm_arma2 = PoissonDLM(structure=(arma2['structure']))

arma2['states'] = [state0]

for t in range(1, 100):
    arma2['states'].append(podlm_arma2.state(arma2['states'][t-1]))

arma2['obs'] = [None]
for t in range(1, 100):
    arma2['obs'].append(podlm_arma2.observation(arma2['states'][t]))
```

```python
import matplotlib.pyplot as plt
t=range(100)
plt.subplot(1, 3, 1)
plt.plot(t, [state[0] for state in arma2['states']])
plt.legend([r"$\theta_{1,1:T}$"], loc=0)
plt.title(r"ARMA(2) PoDLM $\theta_1$")
plt.subplot(1, 3, 2)
plt.plot(t, [state[1] for state in arma2['states']])
plt.legend([r"$\theta_{2,1:T}$"], loc=0)
plt.title(r"ARMA(2) PoDLM $\theta_2$")
plt.subplot(1, 3, 3)
plt.plot(range(100), arma2['obs'], 'bo')
plt.legend([r"$y_{1:T}$"], loc=0)
plt.title(r"ARMA(2) PoDLM y")
plt.tight_layout()
```

### Filtering

#### Locally constant

```python
from pssm.filters import KalmanFilter

kf = KalmanFilter(structure=lc['structure'], V=1.4)

m0 = np.array([0])
C0 = np.matrix([[1e3]])
filtered = {'ms': [m0], 'Cs': [C0]}

for t in range(1, 100):
    m, C = kf.filter(y=lc['obs'][t], m=filtered['ms'][t-1], C=filtered['Cs'][t-1])
    filtered['ms'].append(m)
    filtered['Cs'].append(C)
```

```python
t = range(100)
plt.plot(t, filtered['ms'])
plt.plot(t, lc['states'], '--')
lb = []
ub = []
for m, C in zip(filtered['ms'], filtered['Cs']):
    lb.append(np.asscalar(m[0] - C[0][0]))
    ub.append(np.asscalar(m[0] + C[0][0]))
plt.fill_between(t[1:], lb[1:], ub[1:], alpha=0.3)
plt.legend([r"$\tilde{\theta}_{1:T}$", r"$\theta_{1:T}$"], loc=1)
plt.title(r"Locally constant NDLM, Filtered $\theta$, $\tau^2=1.4, \sigma^2=1.4$")
plt.tight_layout()
```

#### Locally linear

```python
kf = KalmanFilter(structure=ll['structure'], V=2.5)

m0 = np.array([0, 0])
C0 = np.diag([1e3, 1e3])
filtered = {'ms': [m0], 'Cs': [C0]}

for t in range(1, 100):
    m, C = kf.filter(y=ll['obs'][t], m=filtered['ms'][t-1], C=filtered['Cs'][t-1])
    filtered['ms'].append(m)
    filtered['Cs'].append(C)
```

```python
t=range(100)

lb = []
ub = []
for m, C in zip(filtered['ms'], filtered['Cs']):
    lb.append((m[0] - C.item((0,0)), m[1] - C.item((1,1)),))
    ub.append((m[0] + C.item((0,0)), m[1] + C.item((1,1)),))

plt.subplot(1, 2, 1)
plt.plot(t, [state[0] for state in ll['states']], '--')
plt.plot(t, [state[0] for state in filtered['ms']])
plt.fill_between(t[2:], [b[0] for b in lb[2:]], [b[0] for b in ub[2:]], alpha=0.3)
plt.legend([r"$\theta_{1,1:T}$"], loc=0)
plt.title(r"LL NDLM $\theta_1$")
plt.subplot(1, 2, 2)
plt.plot(t, [state[1] for state in ll['states']], '--')
plt.plot(t, [state[1] for state in filtered['ms']])
plt.fill_between(t[2:], [b[1] for b in lb[2:]], [b[1] for b in ub[2:]], alpha=0.3)
plt.legend([r"$\theta_{2,1:T}$"], loc=0)
plt.title(r"LL NDLM $\theta_2$")

plt.tight_layout()
```

### Parameter estimation (FFBS)

#### Locally constant

```python
from pssm.mc import *

m0 = np.array([0])
C0 = np.identity(1)*1000

ffbs = FFBS(lc['structure'], m0, C0, 0.001, [0.001], lc['obs'])

Vs = [1]
Ws = [np.identity(1)]
states = []

for i in range(1, 20):
    W, V, _states = ffbs.run(V=Vs[i-1], W=Ws[i-1], states=True)
    Ws.append(W)
    Vs.append(V)
    states.append(_states)
```

```python
t = range(99)
plt.plot(t, lc['obs'][1:], c='red')
for i in range(19):
    plt.plot(t, states[i], c='black', alpha=0.2)
plt.legend([r"$\theta_{1:T}$", r"$y_{1:T}$"], loc=1)
plt.title(r"Locally constant NDLM, $\tau^2=3.4, \sigma^2=1.4$")
plt.tight_layout()
```

```python
for i in range(20, 1000):
    W, V = ffbs.run(V=Vs[i-1], W=Ws[i-1], states=False)
    Ws.append(W)
    Vs.append(V)
```

```python
plt.subplot(1, 2, 1)
plt.hist(Vs, bins=50)
plt.axvline(x=np.mean(Vs), c='black', ls='--', alpha=0.75)
plt.subplot(1, 2, 2)
_Ws = [W[0][0] for W in Ws]
plt.hist(_Ws, bins=50)
plt.axvline(x=np.mean(_Ws), c='black', ls='--', alpha=0.75)
plt.tight_layout()
```

#### Locally linear

```python
from pssm.mc import *

m0 = np.array([0, 0])
C0 = np.identity(2)*1000

ffbs = FFBS(ll['structure'], m0, C0, 1.0, [1.0, 1.0], ll['obs'])

Vs = [1]
Ws = [np.identity(2)]
states = []

for i in range(1, 20):
    W, V, _states = ffbs.run(V=Vs[i-1], W=Ws[i-1], states=True)
    Ws.append(W)
    Vs.append(V)
    states.append(_states)
```

```python
t = range(99)

plt.subplot(1, 2, 1)
plt.plot(t, [s[0] for s in ll['states'][1:]], c='red')
for i in range(19):
    plt.plot(t, [s[0] for s in states[i]], c='black', alpha=0.2)
plt.legend([r"$\theta_{1:T}$", r"$y_{1:T}$"], loc=1)
plt.title(r"Locally constant NDLM, $\tau^2=3.4, \sigma^2=1.4$")

plt.subplot(1, 2, 2)
plt.plot(t, [s[1] for s in ll['states'][1:]], c='red')
for i in range(19):
    plt.plot(t, [s[1] for s in states[i]], c='black', alpha=0.2)
plt.legend([r"$\theta_{1:T}$", r"$y_{1:T}$"], loc=1)
plt.title(r"Locally constant NDLM, $\tau^2=3.4, \sigma^2=1.4$")

plt.tight_layout()
```

```python
for i in range(20, 1000):
    W, V = ffbs.run(V=Vs[i-1], W=Ws[i-1], states=False)
    Ws.append(W)
    Vs.append(V)
```

```python
print("V (mean) = {}".format(np.mean(Vs)))
print("W1 (mean) = {}".format(np.mean([W[0][0] for W in Ws])))
print("W2 (mean) = {}".format(np.mean([W[1][1] for W in Ws])))
```

    V (mean) = 97.98805501460944
    W1 (mean) = 11.519005567159674
    W2 (mean) = 0.3411164850030208

```python
plt.subplot(1, 3, 1)
plt.hist(Vs, bins=50)
plt.axvline(x=np.mean(Vs), c='black', ls='--', alpha=0.75)
plt.subplot(1, 3, 2)
_Ws = [W[0][0] for W in Ws]
plt.hist(_Ws, bins=50)
plt.axvline(x=np.mean(_Ws), c='black', ls='--', alpha=0.75)
plt.subplot(1, 3, 3)
_Ws = [W[1][1] for W in Ws]
plt.hist(_Ws, bins=50)
plt.axvline(x=np.mean(_Ws), c='black', ls='--', alpha=0.75)
plt.tight_layout()
```

## Non-linear models

### Poisson DLM

```python
plc = {'structure': UnivariateStructure.locally_constant(0.01)}
podlm = PoissonDLM(structure=plc['structure'])
```

```python
# the initial state prior
m0 = np.array([0])
C0 = np.matrix([[1]])
state0 = mvn(m0, C0).rvs()

plc['states'] = [state0]

for t in range(1, 100):
    plc['states'].append(podlm.state(plc['states'][t-1]))

plc['obs'] = [None]
for t in range(1, 100):
    plc['obs'].append(podlm.observation(plc['states'][t]))
```

```python
t = range(100)
plt.plot(t, plc['states'])
plt.plot(t, plc['obs'], 'bo')
plt.legend([r"$\theta_{1:T}$", r"$y_{1:T}$"], loc=1)
plt.title(r"Locally constant PoDLM, $\tau^2=0.01$")
plt.tight_layout()
```

### Binomial DLM

```python
bdlm = BinomialDLM(structure=plc['structure'], categories=1)
```

```python
# the initial state prior
m0 = np.array([0])
C0 = np.matrix([[1]])
state0 = mvn(m0, C0).rvs()

plc['states'] = [state0]

for t in range(1, 100):
    plc['states'].append(bdlm.state(plc['states'][t-1]))

plc['obs'] = [None]
for t in range(1, 100):
    plc['obs'].append(bdlm.observation(plc['states'][t]))
```

```python
t = range(100)
plt.plot(t, plc['states'])
plt.plot(t, plc['obs'], 'bo')
plt.legend([r"$\theta_{1:T}$", r"$y_{1:T}$"], loc=1)
plt.title(r"Locally constant BDLM, $\tau^2=0.01$")
plt.tight_layout()
```

```python
bdlm = BinomialDLM(structure=plc['structure'], categories=16)
```

```python
# the initial state prior
m0 = np.array([0])
C0 = np.matrix([[5]])
state0 = mvn(m0, C0).rvs()

plc['states'] = [state0]

for t in range(1, 100):
    plc['states'].append(bdlm.state(plc['states'][t-1]))

plc['obs'] = [None]
for t in range(1, 100):
    plc['obs'].append(bdlm.observation(plc['states'][t]))
```

```python
t = range(100)
plt.plot(t, plc['states'])
plt.plot(t, plc['obs'], 'bo')
plt.legend([r"$\theta_{1:T}$", r"$y_{1:T}$"], loc=1)
plt.title(r"Locally constant BDLM, $\tau^2=0.01$")
plt.tight_layout()
```

### Composite (multivariate) DGLMs

```python
comp_lc = UnivariateStructure.locally_constant(W=0.3)
comp_seasonal = UnivariateStructure.cyclic_fourier(period=50, harmonics=2, W=np.eye(4))

comp_ndlm = NormalDLM(structure=comp_lc + comp_seasonal, V=2.3)
comp_bdlm = BinomialDLM(structure=comp_lc, categories=4)

composite = CompositeDLM(comp_ndlm, comp_bdlm)
```

```python
# the initial state prior
m0 = np.array([0.0]*6)
C0 = np.eye(6)
state0 = mvn(m0, C0).rvs()

comp_states = [state0]

for t in range(1, 100):
    comp_states.append(composite.state(comp_states[t-1]))

comp_obs = [None]
for t in range(1, 100):
    comp_obs.append(composite.observation(comp_states[t]))
```

```python
t=range(99)
plt.subplot(1, 2, 1)
plt.plot(t, [y[0] for y in comp_obs[1:]])
plt.legend([r"$y{1,1:T}$"], loc=0)
plt.title(r"LC + F(2, 50) NDLM $y_1$")
plt.subplot(1, 2, 2)
plt.plot(t, [y[1] for y in comp_obs[1:]], 'bo')
plt.legend([r"$y{2,1:T}$"], loc=0)
plt.title(r"LC BDLM $y_2$")
plt.tight_layout()
```

```python

```


```python
%matplotlib inline
```

# Probability and Statistics Review

Based on "Machine Learning: A Probabilistic Perspective" by Kevin P. Murphy (Chapter 2).

# What is probability?

* At least two different interpretations:
    * **Frequentist**: probabilities are long-run frequencies of events
    * **Bayesian**: probabilities are used to quantify our **uncertainty**

One advantage of the Bayesian interpretation is that it can be used to model events that do not have long-term frequencies.

# A brief review of probability theory

## Discrete random variables

$p(A)$ denotes the probability that the event $A$ is true

* $0 \leq p(A) \leq 1$

We write $p(\bar{A})$ to denote the probability of the event not $A$

* $p(\bar{A}) = 1 - p(A)$

We can extend the notion of binary events by defining a **discrete random variable** $X$ which can take on any value from a finite or countably infinite set $\mathcal{X}$. We denote the probability of the event that $X = x$ by $p(X = x)$ or just $p(x)$ for short.

* $0 \leq p(x) \leq 1$
* $\sum_{x \in \mathcal{X}} p(x) = 1$

Let's look at some discrete distributions:

```python
import numpy as np
import matplotlib.pyplot as plt

fig, ax = plt.subplots(1, 2)

ax[0].bar([1, 2, 3, 4],[0.25, 0.25, 0.25, 0.25], align='center')
ax[0].set_ylim([0, 1])
_ = ax[0].set_xticks([1, 2, 3, 4])
ax[0].set_title('Uniform distribution')

ax[1].bar([1, 2, 3, 4],[0, 1.0, 0, 0], align='center')
ax[1].set_ylim([0, 1])
_ = ax[1].set_xticks([1, 2, 3, 4])
ax[1].set_title('Degenerate distribution')

```

## Fundamental rules

### Probability of a union of two events

Given two events, $A$ and $B$, we define the probability of $A$ or $B$ as

$$
\begin{align}
p(A \lor B) &= p(A) + p(B) - p(A \land B) \\
&= p(A) + p(B) & \text{if $A$ and $B$ are mutually exclusive}
\end{align}
$$

### Joint probabilities

We define the probability of the joint event $A$ and $B$ as

$$
p(A,B) = p(A \land B) = p(A|B)p(B)
$$

Given a **joint distribution** on two events p(A,B), we define the **marginal distribution** as

$$
p(A) = \sum_b p(A,B) = \sum_b p(A|B)p(B)
$$

### Conditional probability

We define the **conditional probability** of event $A$, given that event $B$ is true, as

$$
\begin{align}
p(A|B) &= \frac{p(A,B)}{p(B)} & \text{if $p(B) > 0$}
\end{align}
$$

## Bayes' rule

Manipulating the basic definition of conditional probability gives us one of the most important formulas in probability theory

$$p(X=x|Y=y) = \frac{p(X=x,Y=y)}{P(Y=y)} = \frac{p(Y=y|X=x)p(X=x)}{\sum_{x'}p(Y=y|X=x')p(X=x')}$$

## Independence and conditional independence

We say $X$ and $Y$ are **unconditionally independent** or **marginally independent**, denoted $X \perp Y$, if we can represent the joint as the product of the two marginals, i.e.,

$$X \perp Y \Longleftrightarrow p(X,Y) = p(X)p(Y)$$

In general, we say a **set** of variables is mutually independent if the joint can be written as a product of marginals.

We say $X$ and $Y$ are **conditionally independent** given $Z$ iff the conditional joint can be written as a product of conditional marginals:

$$X \perp Y|Z \Longleftrightarrow p(X,Y|Z)=p(X|Z)p(Y|Z)$$

CI assumptions allow us to build large probabilistic models from small pieces.

## Continuous random variables

Suppose $X$ is some uncertain continuous quantity. The probability that $X$ lies in any interval $a \leq X \leq b$ can be computed as follows. Define the events $A = (X \leq a), B = (X \leq b)$ and $W = (a < X \leq b)$. We have that $B = A \vee W$, and since $A$ and $W$ are mutually exclusive, the sum rule gives

$$p(B) = p(A) + p(W)$$

and hence

$p(W) = p(B) - p(A)$

Define the function $F(q) \triangleq p(X \leq q)$. This is called the **cumulative distribution function** or **cdf** of $X$. This is a monotonically non-decreasing function.

```python
# CDF of Gaussian N(0,1)
import scipy.stats as stats
f = lambda x : stats.norm.cdf(x, 0, 1)
x = np.arange(-3, 3, 0.1)
y = f(x)

plt.plot(x, y, 'b')
plt.title('CDF')

```

Using the above notation, we have
$$p(a < X \leq b) = F(b) - F(a)$$

Now define $f(x) = \frac{d}{dx} F(x)$ (we assume this derivative exists); this is called a **probability density function** or **pdf**. Given a pdf, we can compute the probability of a continuous variable being in a finite interval as follows:

$$P(a < X \leq b) = \int_a^b f(x) dx$$

```python
# PDF of Gaussian N(0,1)
# shaded area has 0.05 of the mass
# also written mu +/- 2 \sigma
f = lambda x : stats.norm.pdf(x, 0, 1)
x = np.arange(-4, 4, 0.1)
y = f(x)

plt.plot(x, y, 'b')
l_x = np.arange(-4, -1.96, 0.01)
plt.fill_between(l_x, f(l_x))
u_x = np.arange(1.96, 4, 0.01)
plt.fill_between(u_x, f(u_x))

plt.title('PDF')
```

We require $p(x) \geq 0$, but it is possible for $p(x)>1$ for any given $x$, so long as the density integrates to 1.

```python
# Example of p(x) > 1, Uniform distribution on (0, 0.5)
f = lambda x: stats.uniform.pdf(x, 0, 0.5)
x = np.arange(-0.5, 1, 0.01)
y = f(x)

plt.plot(x, y, 'b')
plt.title('Uniform PDF')
```

## Mean and variance

The most familiar property of a distribution is its **mean**, or **expected value**, denoted by $\mu$. For discrete rv's, it is defined as $\mathbb{E}[X] \triangleq \sum_{x \in \mathcal{X}} x p(x)$, and for continuous rv's, it is defined as $\mathbb{E}[X] \triangleq \int_{\mathcal{X}} x p(x) dx$.

The **variance** is a measure of the "spread" of a distribution, denoted by $\sigma^2$. This is defined as follows:

$$
\begin{align}
\text{var}[X] & \triangleq \mathbb{E}\left[ \left( X - \mu\right)^2 \right] = \int \left( x - \mu \right) ^2 p(x) dx \\\
&= \int x^2 p(x)dx + \mu^2 \int p(x) dx - 2 \mu \int x p(x) dx = \mathbb{E}[X^2] - \mu^2
\end{align}
$$

from which we derive the useful result

$$\mathbb{E}[X^2] = \mu^2 + \sigma^2$$

The **standard deviation** is defined as

$$\text{std}[X] \triangleq \sqrt{\text{var}[X]}$$

# Some common discrete distributions

## The binomial and Bernoulli distributions

Suppose we toss a coin $n$ times. Let $X \in {0, \ldots, n}$ be the number of heads. If the probability of heads is $\theta$, then we say $X$ has a **binomial** distribution, written as $X \sim \text{Bin}(n, \theta)$. The probability mass function (pmf) is given by

$$\text{Bin}(k|n,\theta) \triangleq {n\choose k} \theta^k(1 - \theta)^{n-k}$$

where
$$ {n\choose k} \triangleq \frac{n!}{(n-k)!k!}$$

is the number of ways to choose $k$ items from $n$.

This distribution has a mean of $n\theta$ and a variance of $n\theta(1-\theta)$.

```python
fig, ax = plt.subplots(1, 2)

x = np.arange(11)

f = lambda x : stats.binom.pmf(x, 10, 0.25)

ax[0].bar(x, f(x), align='center')
#ax[0].set_ylim([0, 1])
_ = ax[0].set_xticks(x)
ax[0].set_title(r'$\theta$ = 0.25')

f = lambda x : stats.binom.pmf(x, 10, 0.9)

ax[1].bar(x, f(x), align='center')
#ax[1].set_ylim([0, 1])
_ = ax[1].set_xticks(x)
ax[1].set_title(r'$\theta$ = 0.9')
```

Now suppose we toss a coin only once. Let $X \in {0,1}$ be a binary random variable, with probability of "success" or "heads" of $\theta$. We say that $X$ has a **Bernoulli** distribution. This is written as $X \sim \text{Ber}(\theta)$, where the pmf is defined as

$$\text{Ber}(x|\theta) = \theta^{\mathbb{I}(x=1)}(1-\theta)^{\mathbb{I}(x=0)}$$

In other words,

$$ \text{Ber}(x|\theta) = \left\{
\begin{array}{rl}
 \theta &\mbox{ if $x=1$} \\
  1 - \theta &\mbox{ if $x=0$}
\end{array}
\right. $$

This is obviously just a special case of a Binomial distribution with $n=1$.

### The multinomial and multinoulli distribution

To model the outcomes of tossing a $K$-sided die, we can use the **multinomial** distribution. This is defined as follows: let $\mathbf{x}=(x_1, \ldots, x_K)$ be a random vector, where $x_j$ is the number of times side $j$ of the die occurs. Then $\mathbf{x}$ has the following pmf:

$$\text{Mu}(\mathbf{x}|n, \mathbf{\theta}) \triangleq {n \choose x_1,\ldots,x_K} \prod_{j=1}^K \theta_j^{x_j}$$

where $\theta_j$ is the probability that side $j$ shows up, and

$${n \choose x_1,\ldots,x_K} \triangleq \frac{n!}{x_1!x_2! \ldots x_K!}$$

is the **multinomial coefficient** (the number of ways to divide a set of size $n=\sum_{k=1}^K x_k$ into subsets with sizes $x_1$ up to $x_K$).

Now suppose $n=1$. This is like rolling a $K$-sided dice once, so $\mathbf{x}$ will be a vector of 0s and 1s (a bit vector), in which only one bit can be turned on. Specifically, if the dice shows up as face $k$, then the $k$'th bit will be on. In this case, we can think of $x$ as being a scalar categorical random variable with $K$ states (values), and $\mathbf{x}$ is its **dummy encoding**, that is, $\mathbf{x} = \left[\mathbb{I}(x=1),\ldots,\mathbb{I}(x=K)\right]$. For example, if $K=3$, we encode the states 1, 2, and 3 as $(1, 0, 0), (0, 1, 0)$ and $(0, 0, 1)$. This is also called **one-hot encoding**. In this case, the pmf becomes

$$\text{Mu}(\mathbf{x}|1, \mathbf{\theta}) = \prod_{j=1}^K \theta_j^{\mathbb{I}(x_j=1)}$$

This very common special case is known as a **categorical** or **discrete** distribution (Kevin Murphy's text adopts the term **multinoulli distribution** by analogy with the binomial/Bernoulli distinction). We will use the following notation

$$\text{Cat}(x|\mathbf{\theta}) \triangleq \text{Mu}(\mathbf{x}|1, \mathbf{\theta})$$

# Some common continuous distributions

## Gaussian (normal) distribution

The most widely used distribution in statistics and machine learning is the Gaussian or normal distribution. Its pdf is given by

$$\mathcal{N}(x|\mu, \sigma^2) \triangleq \frac{1}{\sqrt{2 \pi \sigma^2}} e^{-\frac{1}{2\sigma^2}(x - \mu)^2}$$

where $\mu = \mathbb{E}[X]$ is the mean (and mode), and $\sigma^2 = \text{var}[X]$ is the variance. $\frac{1}{\sqrt{2 \pi \sigma^2}}$ is the normalization constant needed to ensure the density integrates to 1.

We write $X \sim \mathcal{N}(\mu, \sigma^2)$ to denote that $p(X=x) = \mathcal{N}(x|\mu, \sigma^2)$. If $X \sim \mathcal{N}(0,1)$, we say $X$ follows a **standard normal** distribution.

We will sometimes talk about the **precision** of a Gaussian, by which we mean the inverse variance: $\lambda = 1/\sigma^2$.

The Gaussian distribution is the most widely used distribution in statistics. Why?

* It has two parameters that are easy to interpret
* The central limit theorem tells us that sums of independent random variables have an approximately Gaussian distribution, making it a good fit for modeling residual errors or "noise"
* The Gaussian distribution makes the least number of assumptions (i.e. has maximum entropy) which makes it a good default choice in many cases
* It has a simple mathematical form, which results in easy to implement, but often highly effective methods

## The Student $t$ distribution

One problem with the Gaussian distribution is that it is sensitive to outliers, since the log-probability only decays quadratically with distance from the centre. A more robust distribution is the **Student** $t$ **distribution**. Its pdf is as follows

$$\mathcal{T}(x|\mu, \sigma^2, \nu) \propto \left[ 1 + \frac{1}{\nu} \left( \frac{x-\mu}{\sigma}\right)^2\right]^{-\left(\frac{\nu + 1}{2}\right)}$$

where $\mu$ is the mean, $\sigma^2>0$ is the scale parameter, and $\nu > 0$ is called the **degrees of freedom**.

The distribution has the following properties:

mean = $\mu$, mode = $\mu$, var = $\frac{\nu \sigma^2}{(\nu - 2)}$

The variance is only defined if $\nu > 2$. The mean is only defined if $\nu > 1$. It is common to use $\nu = 4$, which gives good performance in a range of problems. For $\nu \gg 5$, the Student distribution rapidly approaches a Gaussian distribution and loses its robustness properties.

## The Laplace distribution

Another distribution with heavy tails is the **Laplace distribution**, also known as the **double sided exponential** distribution. This has the following pdf:

$$\text{Lap}(x|\mu,b) \triangleq \frac{1}{2b} \exp \left( - \frac{|x - \mu|}{b}\right)$$

Here $\mu$ is a location parameter and $b>0$ is a scale parameter. This distribution has the following properties:

mean = $\mu$, mode = $\mu$, var = $2b^2$

Not only does it have heavier tails, it puts more probability density at 0 than the Gaussian. This property is a useful way to encourage sparsity in a model, as we will see later.

```python
# Show Gaussian, Student, Laplace pdfs and log pdfs
fig, ax = plt.subplots(2, 1, sharex=True)

g = lambda x : stats.norm.pdf(x, loc=0, scale=1)
t = lambda x : stats.t.pdf(x, df=1, loc=0, scale=1)
l = lambda x : stats.laplace.pdf(x, loc=0, scale=1/np.sqrt(2))

x = np.arange(-4, 4, 0.1)

ax[0].plot(x, g(x), 'b-', label='Gaussian')
ax[0].plot(x, t(x), 'r.', label='Student')
ax[0].plot(x, l(x), 'g--', label='Laplace')

ax[0].legend(loc='best')
ax[0].set_title('pdfs')

ax[1].plot(x, np.log(g(x)), 'b-', label='Gaussian')
ax[1].plot(x, np.log(t(x)), 'r.', label='Student')
ax[1].plot(x, np.log(l(x)), 'g--', label='Laplace')
ax[1].set_title('log pdfs')
```

```python
# Demonstrate fitting Gaussian, Student, and Laplace to data
# with and without outliers
n = 30  # n data points
np.random.seed(0)
data = np.random.randn(n)

outliers = np.array([8, 8.75, 9.5])
nn = len(outliers)
nbins = 7

# fit each of the models to the data (no outliers)
model_g = stats.norm.fit(data)
model_t = stats.t.fit(data)
model_l = stats.laplace.fit(data)

fig, ax = plt.subplots(2, 1, sharex=True)

x = np.arange(-10, 10, 0.1)

g = lambda x : stats.norm.pdf(x, loc=model_g[0], scale=model_g[1])
t = lambda x : stats.t.pdf(x, df=model_t[0], loc=model_t[1], scale=model_t[2])
l = lambda x : stats.laplace.pdf(x, loc=model_l[0], scale=model_l[1])

ax[0].hist(data, bins=25, range=(-10, 10),
           normed=True, alpha=0.25, facecolor='gray')
ax[0].plot(x, g(x), 'b-', label='Gaussian')
ax[0].plot(x, t(x), 'r.', label='Student')
ax[0].plot(x, l(x), 'g--', label='Laplace')

ax[0].legend(loc='best')
ax[0].set_title('no outliers')

# fit each of the models to the data (with outliers)
newdata = np.r_[data, outliers]  # row concatenation
model_g = stats.norm.fit(newdata)
model_t = stats.t.fit(newdata)
model_l = stats.laplace.fit(newdata)

g = lambda x : stats.norm.pdf(x, loc=model_g[0], scale=model_g[1])
t = lambda x : stats.t.pdf(x, df=model_t[0], loc=model_t[1], scale=model_t[2])
l = lambda x : stats.laplace.pdf(x, loc=model_l[0], scale=model_l[1])

ax[1].hist(newdata, bins=25, range=(-10, 10),
           normed=True, alpha=0.25, facecolor='gray')
ax[1].plot(x, g(x), 'b-', label='Gaussian')
ax[1].plot(x, t(x), 'r.', label='Student')
ax[1].plot(x, l(x), 'g--', label='Laplace')

ax[1].set_title('with outliers')
```

# Joint probability distributions

A **joint probability distribution** has the form $p(x_1,\ldots,x_D)$ for a set of $D>1$ variables, and models the (stochastic) relationships between the variables. If all the variables are discrete, we can represent the joint distribution as a big multi-dimensional array, with one variable per dimension. However, the number of parameters needed to define such a model is $O(K^D)$, where $K$ is the number of states for each variable.

We can define high dimensional joint distributions using fewer parameters by making conditional independence assumptions. In the case of continuous distributions, an alternative approach is to restrict the form of the pdf to certain functional forms, some of which are examined below.

## Covariance and correlation

The **covariance** between two rv's $X$ and $Y$ measures the degree to which $X$ and $Y$ are (linearly) related. Covariance is defined as

$$\text{cov}[X,Y] \triangleq \mathbb{E}\left[\left(X - \mathbb{E}[X]\right)\left(Y - \mathbb{E}[Y]\right)\right]=\mathbb{E}[XY] - \mathbb{E}[X]\mathbb{E}[Y]$$

If $\mathbf{x}$ is a $d$-dimensional random vector, its **covariance matrix** is defined to be the following symmetric, positive semi-definite matrix:

$$
\begin{align}
\text{cov}[\mathbf{x}] & \triangleq \mathbf{E} \left[\left(\mathbf{x} - \mathbb{E}[\mathbf{x}]\right)\left(\mathbf{x} - \mathbb{E}[\mathbf{x}]\right)^T\right]\\
& = \left( \begin{array}{ccc}
  \text{var}[X_1] & \text{cov}[X_1, X_2] &  \ldots & \text{cov}[X_1, X_d] \\
  \text{cov}[X_2, X_1] & \text{var}[X_2] &  \ldots & \text{cov}[X_2, X_d] \\
  \vdots               & \vdots          & \ddots  & \vdots\\
  \text{cov}[X_d, X_1] & \text{cov}[X_d, X_2] &  \ldots & \text{var}[X_d]
  \end{array} \right)
\end{align}
$$

Covariances can be between $-\infty$ and $\infty$. Sometimes it is more convenient to work with a normalized measure, with finite bounds. The (Pearson) **correlation coefficient** between $X$ and $Y$ is defined as

$$\text{corr}[X,Y] \triangleq \frac{\text{cov}[X,Y]}{\sqrt{\text{var}[X]\text{var}[Y]}}$$

A **correlation matrix** has the form

$$
\mathbf{R} = \left( \begin{array}{ccc}
  \text{corr}[X_1, X_1] & \text{corr}[X_1, X_2] &  \ldots & \text{corr}[X_1, X_d] \\
  \text{corr}[X_2, X_1] & \text{corr}[X_2, X_2] &  \ldots & \text{corr}[X_2, X_d] \\
  \vdots               & \vdots          & \ddots  & \vdots\\
  \text{corr}[X_d, X_1] & \text{corr}[X_d, X_2] &  \ldots & \text{corr}[X_d, X_d]
  \end{array} \right)
$$

One can show that $-1 \leq \text{corr}[X,Y] \leq 1$. Hence, in a correlation matrix, each entry on the diagonal is 1, and the other entries are between -1 and 1. One can also show that $\text{corr}[X,Y]=1$ iff $Y=aX + b$ for some parameters $a$ and $b$, i.e. there is a *linear* relationship between $X$ and $Y$. A good way to think of the correlation coefficient is as a degree of linearity.

If $X$ and $Y$ are independent, meaning $p(X,Y)=p(X)p(Y)$, then $\text{cov}[X,Y]=0$, and hence $\text{corr}[X,Y]=0$ so they are uncorrelated. However, the converse is not true: *uncorrelated does not imply independent*. Some striking examples are shown below.

Source: http://upload.wikimedia.org/wikipedia/commons/0/02/Correlation_examples.png

## The multivariate Gaussian

The **multivariate Gaussian** or **multivariate normal (MVN)** is the most widely used joint probability density function for continuous variables. The pdf of the MVN in $D$ dimensions is defined by the following

$$\mathcal{N}(\mathbf{x}|\boldsymbol\mu,\mathbf{\Sigma}) \triangleq \frac{1}{(2 \pi)^{D/2}|\mathbf{\Sigma}|^{1/2}} \exp \left[ - \frac{1}{2} \left(\mathbf{x} - \boldsymbol\mu \right)^T \mathbf{\Sigma}^{-1} \left(\mathbf{x} - \boldsymbol\mu\right)\right]$$

where $\boldsymbol\mu = \mathbb{E}[\mathbf{x}] \in \mathbb{R}^D$ is the mean vector, and $\Sigma = \text{cov}[\mathbf{x}]$ is the $D \times D$ covariance matrix. Sometimes we will work in terms of the **precision matrix** or **concentration matrix** instead. This is just the inverse covariance matrix, $\Lambda = \Sigma^{-1}$. The normalization constant $(2 \pi)^{-D/2}|\Lambda|^{1/2}$ ensures that the pdf integrates to 1.

The figure below plots some MVN densities in 2d for three different kinds of covariance matrices. A full covariance matrix has $D(D+1)/2$ parameters (we divide by 2 since $\Sigma$ is symmetric). A diagonal covariance matrix has $D$ parameters, and has 0s on the off-diagonal terms. A **spherical** or **isotropic** covariance, $\Sigma = \sigma^2 \mathbf{I}_D$, has one free parameter.

```python
# plot a MVN in 2D and 3D
import matplotlib.mlab as mlab
from scipy.linalg import eig, inv
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm

delta = 0.05
x = np.arange(-10.0, 10.0, delta)
y = np.arange(-10.0, 10.0, delta)
X, Y = np.meshgrid(x, y)

S = np.asarray([[2.0, 1.8],
                [1.8, 2.0]])
mu = np.asarray([0, 0])

Z = mlab.bivariate_normal(X, Y, sigmax=S[0, 0], sigmay=S[1, 1],
                          mux=mu[0], muy=mu[1], sigmaxy=S[0, 1])

#fig, ax = plt.subplots(2, 2, figsize=(10, 10),
#                       subplot_kw={'aspect': 'equal'})

fig = plt.figure(figsize=(10, 10))

ax = fig.add_subplot(2, 2, 1)

CS = ax.contour(X, Y, Z)
plt.clabel(CS, inline=1, fontsize=10)

ax.set_xlim((-6, 6))
ax.set_ylim((-6, 6))
ax.set_title('full')

# Decorrelate
[D, U] = eig(S)

S1 = np.dot(np.dot(U.T, S), U)

Z = mlab.bivariate_normal(X, Y, sigmax=S1[0, 0], sigmay=S1[1, 1],
                          mux=mu[0], muy=mu[0], sigmaxy=S1[0, 1])

ax = fig.add_subplot(2, 2, 2)
CS = ax.contour(X, Y, Z)
plt.clabel(CS, inline=1, fontsize=10)

ax.set_xlim((-10, 10))
ax.set_ylim((-5, 5))
ax.set_title('diagonal')

# Whiten
A = np.dot(np.sqrt(np.linalg.inv(np.diag(np.real(D)))), U.T)
mu2 = np.dot(A, mu)
S2 = np.dot(np.dot(A, S), A.T)  # may not be numerically equal to I

#np.testing.assert_allclose(S2, np.eye(2))  # check
print np.allclose(S2, np.eye(2))

# plot centred on original mu, not shifted mu
Z = mlab.bivariate_normal(X, Y, sigmax=S2[0, 0], sigmay=S2[1, 1],
                          mux=mu[0], muy=mu[0], sigmaxy=S2[0, 1])

ax = fig.add_subplot(2, 2, 3)
CS = ax.contour(X, Y, Z)
plt.clabel(CS, inline=1, fontsize=10)

ax.set_xlim((-6, 6))
ax.set_ylim((-6, 6))
ax.set_title('spherical')

# demonstration of how to do a surface plot
axx = fig.add_subplot(2, 2, 4, projection='3d')
surf = axx.plot_surface(X, Y, Z, rstride=5, cstride=5, cmap=cm.coolwarm,
                        linewidth=0, antialiased=False)
axx.set_title('spherical')
```


# Метод главных компонент

### Алгоритм PCA

Дана матрица данных $D$ и коэффициент $\alpha$:
1. вычисляем среднее значение $\mu = \frac{1}{n} \sum \limits_{i=1}^{n} x_i$;
2. центруем матрицу данных $Z = D - 1 \cdot \mu^T$;
3. вычисляем матрицу ковариаций $K = \frac{1}{n} (Z^T Z)$;
4. вычисляем собственные значения $(\lambda_1, \lambda_2, \ldots, \lambda_d)$;
5. вычисляем собственные векторы $U = (u_1 u_2 \ldots u_d)$;
6. вычисляем долю общей дисперсии $f(r) = \frac{\sum_{i=1}^{r} \lambda_i} {\sum_{i=1}^{n} \lambda_i}$;
7. выбираем наименьший $r$ так, чтобы $f(r) \geq \alpha$;
8. выбираем $r$ первых векторов $U = (u_1 u_2 \ldots u_r)$;
9. уменьшаем признаковое пространство: $A = \{ a_i \: | \: a_i = U_r^T x_i, \: для \: i = 1, \ldots, n \} $

Реализуйте алгоритм для первых двух главных компонент. Найдите $\lambda_1$, $\lambda_2$ и $u_1$, $u_2$. Спроецируйте данные на плоскость образуемую этими векторами, визуализируйте полученные результаты c метками классов. Сравните реузльтаты с методом PCA из библиотеки sklearn.decomposition.

В качестве данных возьмите изображения с рукописными цифрами.

```python
import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_iris
%matplotlib inline

data = load_iris()
x, y = data['data'], data['target']
```

```python
def custom_PCA(D, a):
    Z = (D[:,0] - D[:,0].mean(), D[:,1].mean() - D[:,1], D[:,2] - D[:,2].mean(), D[:,3] - D[:,3].mean())
    K = np.cov(Z)
    L, U = np.linalg.eig(K)
    fr = sum(L[:a])/sum(L)
    print('доля общей дисперсии: %f' %fr)
    U = U[:,:a]
    return [np.dot(U.transpose(), Z), L]
```

```python
custom_PCA_result = custom_PCA(x, 2)
plt.figure()
plt.scatter(custom_PCA_result[0][0], custom_PCA_result[0][1] ,c=y)
plt.gcf().set_size_inches(18.5, 10.5)
```

```python
from sklearn.decomposition import PCA

PCA_result = PCA(n_components=2, random_state=0).fit_transform(x)
plt.figure()
plt.scatter(PCA_result[:,0], PCA_result[:,1], c=y)
plt.gcf().set_size_inches(18.5, 10.5)
```

На основе полученных собственных значений ковариационной матриы $K$ постройте график зависимости покрываемой дисперсии трансформированных данных от количества главных компонент.

```python
list_value = [sum(custom_PCA_result[1][:i]) / sum(custom_PCA_result[1]) for i in range(1,5)]
plt.plot(list_value)
plt.show()
```

# Интерпретация компонент
## Корреляция с исходными признаками
Главные компоненты являются линейными кобинациями исходных признаков. В этой связи необходимо оценить связь каждой компоненты с исходными признаками.

Рассчет взаимосвязи будем проводить с помощью корреляции Пирсона:

\begin{equation}
r_{jk} = \frac{\sum_{i=1}^n (x_{ij} - \bar{x}_j) (x_{ik}' - \bar{x}_k')}{\sqrt{\sum_{i=1}^n (x_{ij} - \bar{x}_j)^2 \sum_{i=1}^n (x_{ik}' - \bar{x}_k')^2}}
\end{equation}

где
$\bar{x}_j$ - среднее значение $j$-го признака,
$\bar{x}_k'$ - среднее значение проекции на $k$-ю главную компоненту,
$n$ - количество объектов.

Отметим, что корреляция Пирсона изменяется от $-1$ до $+1$. Она равна $0$ в случае, когда величины независимы, и $\pm 1$, если они линейно зависимы.

Определите по коэффициентам корреляции, какие признаки в какую компоненту вошли.

```python
def custom_pearson(x, y):
    n = len(x)
    sum_x = float(sum(x))
    sum_y = float(sum(y))
    sum_x_sq = sum(map(lambda x: pow(x, 2), x))
    sum_y_sq = sum(map(lambda x: pow(x, 2), y))
    psum = sum(map(lambda x, y: x * y, x, y))
    num = psum - (sum_x * sum_y/n)
    den = pow((sum_x_sq - pow(sum_x, 2) / n) * (sum_y_sq - pow(sum_y, 2) / n), 0.5)
    if den == 0: return 0
    return num / den
```

```python
for index in range(0,4):
    print(f'Корреляция данных с исходными для стобца {index}: {round(custom_pearson(custom_PCA_result[0][0], x[:,index]), 3)}, {round(custom_pearson(custom_PCA_result[0][1], x[:,index]), 3)}')
```

    Корреляция данных с исходными для стобца 0: 0.897, 0.391
    Корреляция данных с исходными для стобца 1: -0.399, 0.825
    Корреляция данных с исходными для стобца 2: 0.998, -0.048
    Корреляция данных с исходными для стобца 3: 0.967, -0.049

## Работу выполнил
---
Студент группы **РИМ-181226**

Кабанов Евгений Алексеевич


# 4 - Hybdrid Absorbing Boundary Condition (HABC)

# 4.1 - Introduction

In this notebook we describe absorbing boundary conditions and their use combined with the *Hybdrid Absorbing Boundary Condition* (*HABC*). The common points to the previous notebooks Introduction to Acoustic Problem, Damping and  PML will be used here, with brief descriptions.

# 4.2 - Absorbing Boundary Conditions

 We initially describe absorbing boundary conditions, the so called A1 and A2 Clayton's conditions and
 the scheme from Higdon. These methods can be used as pure boundary conditions, designed to reduce reflections,
 or as part of the Hybrid Absorbing Boundary Condition, in which they are combined with an absorption layer in a manner to be described ahead.

 In the presentation of these boundary conditions we initially consider the wave equation to be solved on
 the spatial domain $\Omega=\left[x_{I},x_{F}\right] \times\left[z_{I},z_{F}\right]$ as show in the figure bellow. More details about the equation and domain definition can be found in the Introduction to Acoustic Problem notebook.

## 4.2.1 - Clayton's A1 Boundary Condition

Clayton's A1 boundary condition is based on a one way wave equation (OWWE). This simple condition
  is such that outgoing waves normal to the border would leave without reflection. At the $\partial \Omega_1$ part of the boundary
    we have,

- $\displaystyle\frac{\partial u(x,z,t)}{\partial t}-c(x,z)\displaystyle\frac{\partial u(x,z,t)}{\partial x}=0.$

while at $\partial \Omega_3$ the condition is

- $\displaystyle\frac{\partial u(x,z,t)}{\partial t}+c(x,z)\displaystyle\frac{\partial u(x,z,t)}{\partial x}=0.$

and at $\partial \Omega_2$

- $\displaystyle\frac{\partial u(x,z,t)}{\partial t}+c(x,z)\displaystyle\frac{\partial u(x,z,t)}{\partial z}=0.$

## 4.2.2 - Clayton's A2 Boundary Condition

The A2 boundary condition aims to impose a boundary condition that would make outgoing waves leave the domain without being reflected. This condition is approximated (using a Padé approximation in the wave dispersion relation) by the following equation to be imposed on the boundary part $\partial \Omega_1$

- $\displaystyle\frac{\partial^{2} u(x,z,t)}{\partial t^{2}}+c(x,z)\displaystyle\frac{\partial^{2} u(x,z,t)}{\partial x \partial t}+\frac{c^2(x,z)}{2}\displaystyle\frac{\partial^{2} u(x,z,t)}{\partial z^{2}}=0.$

At $\partial \Omega_3$ we have

- $\displaystyle\frac{\partial^{2} u(x,z,t)}{\partial t^{2}}-c(x,z)\displaystyle\frac{\partial^{2} u(x,z,t)}{\partial z \partial t}+\frac{c^2(x,z)}{2}\displaystyle\frac{\partial^{2} u(x,z,t)}{\partial x^{2}}=0.$

while at $\partial \Omega_2$ the condition is

- $\displaystyle\frac{\partial^{2} u(x,z,t)}{\partial t^{2}}-c(x,z)\displaystyle\frac{\partial^{2} u(x,z,t)}{\partial x \partial t}+\frac{c^2(x,z)}{2}\displaystyle\frac{\partial^{2} u(x,z,t)}{\partial z^{2}}=0.$

At the corner points the condition is

- $\displaystyle\frac{\sqrt{2}\partial u(x,z,t)}{\partial t}+c(x,z)\left(\displaystyle\frac{\partial u(x,z,t)}{\partial x}+\displaystyle\frac{\partial u(x,z,t)}{\partial z}\right)=0.$

## 4.2.3 -  Higdon Boundary Condition

The Higdon Boundary condition of order p is given at $\partial \Omega_1$ and $\partial \Omega_3$n by:

- $\Pi_{j=1}^p(\cos(\alpha_j)\left(\displaystyle\frac{\partial }{\partial t}-c(x,z)\displaystyle\frac{\partial }{\partial x}\right)u(x,z,t)=0.$

and at $\partial \Omega_2$

- $\Pi_{j=1}^p(\cos(\alpha_j)\left(\displaystyle\frac{\partial}{\partial t}-c(x,z)\displaystyle\frac{\partial}{\partial z}\right)u(x,z,t)=0.$

 This method would make that outgoing waves with angle of incidence at the boundary equal to $\alpha_j$ would
 present no reflection. The method we use in this notebook employs order 2 ($p=2$) and angles $0$ and $\pi/4$.

 Observation: There are similarities between Clayton's A2 and the Higdon condition. If one chooses $p=2$ and
 both angles equal to zero in Higdon's method, this leads to the condition:
 $ u_{tt}-2cu_{xt}+c^2u_{xx}=0$. But, using the wave equation, we have that $c^2u_{xx}=u_{tt}-c^2u_{zz}$. Replacing this relation in the previous equation, we get: $2u_{tt}-2cu_{xt}-c^2u_{zz}=0$ which is Clayton's A2
 boundary condition. In this sence, Higdon's method would generalize Clayton's scheme. But the discretization of
 both methods are quite different, since in Higdon's scheme the boundary operators are unidirectional, while
 in Clayton's A2 not.

# 4.3 - Acoustic Problem with HABC

In the hybrid absorption boundary condition (HABC) scheme we will also extend the spatial domain as $\Omega=\left[x_{I}-L,x_{F}+L\right] \times\left[z_{I},z_{F}+L\right]$.

We added to the target domain $\Omega_{0}=\left[x_{I},x_{F}\right]\times\left[z_{I},z_{F}\right]$ an extension zone, of length $L$ in both ends of the direction $x$ and at the end of the domain in the direction $z$, as represented in the figure bellow.

 The difference with respect to previous schemes, is that this extended region will now be considered as the union of several gradual extensions. As represented in the next figure, we define a region $A_M=\Omega_{0}$. The regions $A_k, k=M-1,\cdots,1$ will be defined as the previous region $A_{k+1}$ to which we add one extra grid line to the left,
 right and bottom sides of it, such that the final region $A_1=\Omega$ (we thus have $M=L+1$).

 We now consider the temporal evolution
 of the solution of the HABC method. Suppose that $u(x,z,t-1)$ is the solution at a given instant $t-1$ in all the
 extended $\Omega$ domain. We update it to instant $t$, using one of the absorbing boundary conditions described in the previous section (A1, A2 or Higdon) producing a preliminar new function $u(x,z,t)$. Now, call $u_{1}(x,z,t)$ the solution at instant $t$ constructed in the extended region, by applying the same absorbing boundary condition at the border of each of the domains  $A_k,k=1,..,M$. The HABC solution will be constructed as a convex combination of $u(x,z,t)$ and $u_{1}(x,z,t)$:

- $u(x,z,t) = (1-\omega)u(x,z,t)+\omega u_{1}(x,z,t)$.

The function $u_{1}(x,z,t)$ is defined (and used) only in the extension of the domain. The function $w$ is a
weight function growing from zero at the boundary $\partial\Omega_{0}$ to one at $\partial\Omega$. The particular weight function to be used could vary linearly, as when the scheme was first proposed by Liu and Sen. But HABC produces better results with a non-linear weight function to be described ahead.

The wave equation employed here will be the same as in the previous notebooks, with same velocity model, source term and initial conditions.

## 4.3.1 The weight  function $\omega$

One can choose a *linear* weight function as

\begin{equation}
\omega_{k} = \displaystyle\frac{M-k}{M};
\end{equation}

or preferably a *non linear*

\begin{equation}
\omega_{k}=\left\{ \begin{array}{ll}
1, & \textrm{if $1\leq k \leq P+1$,} \\ \left(\displaystyle\frac{M-k}{M-P}\right)^{\alpha} , & \textrm{if $P+2 \leq k \leq  M-1.$} \\ 0 , & \textrm{if $k=M$.}\end{array}\right.
\label{eq:elo8}
\end{equation}

In general we take $P=2$ and we choose $\alpha$ as follows:

- $\alpha = 1.5 + 0.07(npt-P)$, in the case of A1 and A2;
- $\alpha = 1.0 + 0.15(npt-P)$, in the case of Higdon.

The value *npt* designates the number of discrete points that define the length of the blue band in the direction $x$ and/or $z$.

# 4.4 - Finite Difference Operators and Discretization of Spatial and Temporal Domains

 We employ the same methods as in the previous notebooks.

# 4.5 - Standard Problem

Redeeming the Standard Problem definitions discussed on the notebook Introduction to Acoustic Problem we have that:

- $x_{I}$ =  0.0 Km;
- $x_{F}$ =  1.0 Km =  1000 m;
- $z_{I}$ =  0.0 Km;
- $z_{F}$ =  1.0 Km =  1000 m;

The spatial discretization parameters are given by:
- $\Delta x$ = 0.01 km = 10m;
- $\Delta z$ = 0.01 km = 10m;

Let's consider a $I$ the time domain with the following limitations:

- $t_{I}$ = 0 s = 0 ms;
- $t_{F}$ = 1 s = 1000 ms;

The temporal discretization parameters are given by:

- $\Delta t$ $\approx$ 0.0016 s = 1.6 ms;
- $NT$ = 626.

The source term, velocity model and positioning of receivers will be as in the previous notebooks.

# 4.6 - Numerical Simulations

For the numerical simulations of this notebook we use several of the notebook codes presented in  Damping e PML. The new features will be described in more detail.

So, we import the following Python and Devito packages:

```python
# NBVAL_IGNORE_OUTPUT

import numpy                   as np
import matplotlib.pyplot       as plot
import math                    as mt
import matplotlib.ticker       as mticker
from   mpl_toolkits.axes_grid1 import make_axes_locatable
from   matplotlib              import cm
```

From Devito's library of examples we import the following structures:

```python
# NBVAL_IGNORE_OUTPUT

%matplotlib inline
from   examples.seismic  import TimeAxis
from   examples.seismic  import RickerSource
from   examples.seismic  import Receiver
from   examples.seismic  import plot_velocity
from   devito            import SubDomain, Grid, NODE, TimeFunction, Function, Eq, solve, Operator
```

The mesh parameters that we choose define the domain $\Omega_{0}$ plus the absorption region. For this, we use the following data:

```python
nptx   =  101
nptz   =  101
x0     =  0.
x1     =  1000.
compx  =  x1-x0
z0     =  0.
z1     =  1000.
compz  =  z1-z0;
hxv    =  (x1-x0)/(nptx-1)
hzv    =  (z1-z0)/(nptz-1)
```

As we saw previously, HABC has three approach possibilities (A1, A2 and Higdon) and two types of weights (linear and non-linear). So, we insert two control variables. The variable called *habctype* chooses the type of HABC approach and is such that:

- *habctype=1* is equivalent to choosing A1;
- *habctype=2* is equivalent to choosing A2;
- *habctype=3* is equivalent to choosing Higdon;

Regarding the weights, we will introduce the variable *habcw* that chooses the type of weight and is such that:

- *habcw=1* is equivalent to linear weight;
- *habcw=2* is equivalent to non-linear weights;

In this way, we make the following choices:

```python
habctype  = 3
habcw     = 2
```

The number of points of the absorption layer in the directions $x$ and $z$ are given, respectively, by:

```python
npmlx  = 20
npmlz  = 20
```

The lengths $L_{x}$ and $L_{z}$ are given, respectively, by:

```python
lx = npmlx*hxv
lz = npmlz*hzv
```

For the construction of the *grid* we have:

```python
nptx    =  nptx + 2*npmlx
nptz    =  nptz + 1*npmlz
x0      =  x0 - hxv*npmlx
x1      =  x1 + hxv*npmlx
compx   =  x1-x0
z0      =  z0
z1      =  z1 + hzv*npmlz
compz   =  z1-z0
origin  = (x0,z0)
extent  = (compx,compz)
shape   = (nptx,nptz)
spacing = (hxv,hzv)
```

As in the case of the acoustic equation with Damping and in the acoustic equation with PML, we can define specific regions in our domain, since the solution $u_{1}(x,z,t)$ is only calculated in the blue region. We will soon follow a similar scheme for creating *subdomains* as was done on notebooks Damping and PML.

First, we define a region corresponding to the entire domain, naming this region as *d0*. In the language of *subdomains* *d0* it is written as:

```python
class d0domain(SubDomain):
    name = 'd0'
    def define(self, dimensions):
        x, z = dimensions
        return {x: x, z: z}
d0_domain = d0domain()
```

The blue region will be built with 3 divisions:

- *d1* represents the left range in the direction *x*, where the pairs $(x,z)$ satisfy: $x\in\{0,npmlx\}$ and $z\in\{0,nptz\}$;
- *d2* represents the rigth range in the direction *x*, where the pairs $(x,z)$ satisfy: $x\in\{nptx-npmlx,nptx\}$ and $z\in\{0,nptz\}$;
- *d3* represents the left range in the direction *y*, where the pairs $(x,z)$ satisfy: $x\in\{npmlx,nptx-npmlx\}$ and $z\in\{nptz-npmlz,nptz\}$;

Thus, the regions *d1*, *d2* and *d3* aare described as follows in the language of *subdomains*:

```python
class d1domain(SubDomain):
    name = 'd1'
    def define(self, dimensions):
        x, z = dimensions
        return {x: ('left',npmlx), z: z}
d1_domain = d1domain()

class d2domain(SubDomain):
    name = 'd2'
    def define(self, dimensions):
        x, z = dimensions
        return {x: ('right',npmlx), z: z}
d2_domain = d2domain()

class d3domain(SubDomain):
    name = 'd3'
    def define(self, dimensions):
        x, z = dimensions
        if((habctype==3)&(habcw==1)):
            return {x: x, z: ('right',npmlz)}
        else:
            return {x: ('middle', npmlx, npmlx), z: ('right',npmlz)}
d3_domain = d3domain()
```

The figure below represents the division of domains that we did previously:

After we defining the spatial parameters and constructing the *subdomains*, we then generate the *spatial grid* and set the velocity field:

```python
grid = Grid(origin=origin, extent=extent, shape=shape, subdomains=(d0_domain,d1_domain,d2_domain,d3_domain), dtype=np.float64)
```

```python
v0 = np.zeros((nptx,nptz))
X0 = np.linspace(x0,x1,nptx)
Z0 = np.linspace(z0,z1,nptz)

x10 = x0+lx
x11 = x1-lx

z10 = z0
z11 = z1 - lz

xm = 0.5*(x10+x11)
zm = 0.5*(z10+z11)

pxm = 0
pzm = 0

for i in range(0,nptx):
    if(X0[i]==xm): pxm = i

for j in range(0,nptz):
    if(Z0[j]==zm): pzm = j

p0 = 0
p1 = pzm
p2 = nptz

v0[0:nptx,p0:p1] = 1.5
v0[0:nptx,p1:p2] = 2.5
```

Previously we introduce the local variables *x10,x11,z10,z11,xm,zm,pxm* and *pzm* that help us to create a specific velocity field, where we consider the whole domain (including the absorpion region). Below we include a routine to plot the velocity field.

```python
def graph2dvel(vel):
        plot.figure()
        plot.figure(figsize=(16,8))
        fscale =  1/10**(3)
        scale  = np.amax(vel[npmlx:-npmlx,0:-npmlz])
        extent = [fscale*(x0+lx),fscale*(x1-lx), fscale*(z1-lz), fscale*(z0)]
        fig = plot.imshow(np.transpose(vel[npmlx:-npmlx,0:-npmlz]), vmin=0.,vmax=scale, cmap=cm.seismic, extent=extent)
        plot.gca().xaxis.set_major_formatter(mticker.FormatStrFormatter('%.1f km'))
        plot.gca().yaxis.set_major_formatter(mticker.FormatStrFormatter('%.1f km'))
        plot.title('Velocity Profile')
        plot.grid()
        ax = plot.gca()
        divider = make_axes_locatable(ax)
        cax = divider.append_axes("right", size="5%", pad=0.05)
        cbar = plot.colorbar(fig, cax=cax, format='%.2e')
        cbar.set_label('Velocity [km/s]')
        plot.show()
```

Below we include the plot of velocity field.

```python
# NBVAL_IGNORE_OUTPUT

graph2dvel(v0)
```

Time parameters are defined and constructed by the following sequence of commands:

```python
t0    = 0.
tn    = 1000.
CFL   = 0.4
vmax  = np.amax(v0)
dtmax = np.float64((min(hxv,hzv)*CFL)/(vmax))
ntmax = int((tn-t0)/dtmax)+1
dt0   = np.float64((tn-t0)/ntmax)
```

With the temporal parameters, we generate the time properties with *TimeAxis* as follows:

```python
time_range = TimeAxis(start=t0,stop=tn,num=ntmax+1)
nt         = time_range.num - 1
```

The symbolic values associated with the spatial and temporal grids that are used in the composition of the equations are given by:

```python
(hx,hz) = grid.spacing_map
(x, z)  = grid.dimensions
t       = grid.stepping_dim
dt      = grid.stepping_dim.spacing
```

We set the Ricker source:

```python
f0      = 0.01
nsource = 1
xposf   = 0.5*(compx-2*npmlx*hxv)
zposf   = hzv
```

```python
src = RickerSource(name='src',grid=grid,f0=f0,npoint=nsource,time_range=time_range,staggered=NODE,dtype=np.float64)
src.coordinates.data[:, 0] = xposf
src.coordinates.data[:, 1] = zposf
```

Below we include the plot of Ricker source.

```python
# NBVAL_IGNORE_OUTPUT

src.show()
```

We set the receivers:

```python
nrec   = nptx
nxpos  = np.linspace(x0,x1,nrec)
nzpos  = hzv
```

```python
rec = Receiver(name='rec',grid=grid,npoint=nrec,time_range=time_range,staggered=NODE,dtype=np.float64)
rec.coordinates.data[:, 0] = nxpos
rec.coordinates.data[:, 1] = nzpos
```

The displacement field *u* and the velocity *vel* are allocated:

```python
u = TimeFunction(name="u",grid=grid,time_order=2,space_order=2,staggered=NODE,dtype=np.float64)
```

```python
vel = Function(name="vel",grid=grid,space_order=2,staggered=NODE,dtype=np.float64)
vel.data[:,:] = v0[:,:]
```

We include the source term as *src_term* using the following command:

```python
src_term = src.inject(field=u.forward,expr=src*dt**2*vel**2)
```

The Receivers are again called *rec_term*:

```python
rec_term = rec.interpolate(expr=u)
```

The next step is to generate the $\omega$ weights, which are selected using the *habcw* variable. Our construction approach will be in two steps: in a first step we build local vectors *weightsx* and *weightsz* that represent the weights in the directions $x$ and $z$, respectively. In a second step, with the *weightsx* and *weightsz* vectors, we distribute them in two global arrays called *Mweightsx* and *Mweightsz* that represent the distribution of these weights along the *grid* in the directions $x$ and $z$ respectively. The *generateweights* function below perform the operations listed previously:

```python
def generateweights():

    weightsx   = np.zeros(npmlx)
    weightsz   = np.zeros(npmlz)
    Mweightsx  = np.zeros((nptx,nptz))
    Mweightsz  = np.zeros((nptx,nptz))

    if(habcw==1):

        for i in range(0,npmlx):
            weightsx[i] = (npmlx-i)/(npmlx)

        for i in range(0,npmlz):
            weightsz[i] = (npmlz-i)/(npmlz)

    if(habcw==2):

        mx = 2
        mz = 2

        if(habctype==3):

            alphax  = 1.0 + 0.15*(npmlx-mx)
            alphaz  = 1.0 + 0.15*(npmlz-mz)

        else:

            alphax  = 1.5 + 0.07*(npmlx-mx)
            alphaz  = 1.5 + 0.07*(npmlz-mz)

        for i in range(0,npmlx):

            if(0<=i<=(mx)):
                weightsx[i] = 1
            elif((mx+1)<=i<=npmlx-1):
                weightsx[i] = ((npmlx-i)/(npmlx-mx))**(alphax)
            else:
                weightsx[i] = 0

        for i in range(0,npmlz):

            if(0<=i<=(mz)):
                weightsz[i] = 1
            elif((mz+1)<=i<=npmlz-1):
                weightsz[i] = ((npmlz-i)/(npmlz-mz))**(alphaz)
            else:
                weightsz[i] = 0

    for k in range(0,npmlx):

        ai = k
        af = nptx - k - 1
        bi = 0
        bf = nptz - k
        Mweightsx[ai,bi:bf] = weightsx[k]
        Mweightsx[af,bi:bf] = weightsx[k]

    for k in range(0,npmlz):

        ai = k
        af = nptx - k
        bf = nptz - k - 1
        Mweightsz[ai:af,bf] = weightsz[k]

    return Mweightsx,Mweightsz
```

Once the *generateweights* function has been created, we execute it with the following command:

```python
Mweightsx,Mweightsz = generateweights();
```

Below we include a routine to plot the weight fields.

```python
def graph2dweight(D):
    plot.figure()
    plot.figure(figsize=(16,8))
    fscale = 1/10**(-3)
    fscale = 10**(-3)
    scale  = np.amax(D)
    extent = [fscale*x0,fscale*x1, fscale*z1, fscale*z0]
    fig = plot.imshow(np.transpose(D), vmin=0.,vmax=scale, cmap=cm.seismic, extent=extent)
    plot.gca().xaxis.set_major_formatter(mticker.FormatStrFormatter('%.1f km'))
    plot.gca().yaxis.set_major_formatter(mticker.FormatStrFormatter('%.1f km'))
    plot.title('Weight Function')
    plot.grid()
    ax = plot.gca()
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="5%", pad=0.05)
    cbar = plot.colorbar(fig, cax=cax, format='%.2e')
    cbar.set_label('Weights')
    plot.show()
```

Below we include the plot of weights field in $x$ direction.

```python
# NBVAL_IGNORE_OUTPUT

graph2dweight(Mweightsx)
```

Below we include the plot of weights field in $z$ direction.

```python
# NBVAL_IGNORE_OUTPUT

graph2dweight(Mweightsz)
```

Next we create the fields for the weight arrays  *weightsx* and *weightsz*:

```python
weightsx = Function(name="weightsx",grid=grid,space_order=2,staggered=NODE,dtype=np.float64)
weightsx.data[:,:] = Mweightsx[:,:]

weightsz = Function(name="weightsz",grid=grid,space_order=2,staggered=NODE,dtype=np.float64)
weightsz.data[:,:] = Mweightsz[:,:]
```

For the discretization of the A2 and Higdon's boundary conditions (to calculate $u_{1}(x,z,t)$) we need  information from three time levels, namely $u(x,z,t-1)$, $u (x,z,t)$ and $u(x,z,t+1)$. So it is convenient to create the three fields:

```python
u1  = Function(name="u1"   ,grid=grid,space_order=2,staggered=NODE,dtype=np.float64)
u2  = Function(name="u2"   ,grid=grid,space_order=2,staggered=NODE,dtype=np.float64)
u3  = Function(name="u3"   ,grid=grid,space_order=2,staggered=NODE,dtype=np.float64)
```

We will assign to each of them the three time solutions described previously, that is,

- u1(x,z) = u(x,z,t-1);
- u2(x,z) = u(x,z,t);
- u3(x,z) = u(x,z,t+1);

These three assignments can be represented by the *stencil01* given by:

```python
stencil01 = [Eq(u1,u.backward),Eq(u2,u),Eq(u3,u.forward)]
```

An update of the term *u3(x,z)* will be necessary after updating *u(x,z,t+1)* in the direction $x$, so that we can continue to apply the HABC method. This update is given by *stencil02* defined as:

```python
stencil02 = [Eq(u3,u.forward)]
```

For the acoustic equation with HABC without the source term we need in $\Omega$

- eq1 = u.dt2 - vel0 * vel0 * u.laplace;

So the *pde* that represents this equation is given by:

```python
pde0 = Eq(u.dt2 - u.laplace*vel**2)
```

And the *stencil* for *pde0* is given to:

```python
stencil0  = Eq(u.forward, solve(pde0,u.forward))
```

For the blue region we will divide it into $npmlx$ layers in the $x$ direction and $npmlz$ layers in the $z$ direction. In this case, the representation is a little more complex than shown in the figures that exemplify the regions $A_{k}$ because there are intersections between the layers.

**Observation:** Note that the representation of the $A_{k}$ layers that we present in our text reflects the case where $npmlx=npmlz$. However, our code includes the case illustrated in the figure, as well as situations in which $npmlx\neq npmlz$. The discretizations of the bounadry conditions A1, A2 and Higdon follow in the bibliographic references at the end. They will not be detailled here, but can be seen in the codes below.

In the sequence of codes below we build the *pdes* that represent the *eqs* of the regions $B_{1}$, $B_{2}$ and $B_{3}$ and/or in the corners (red points in the case of *A2*) as represented in the following figure:

In the sequence, we present the *stencils* for each of these *pdes*.

So, for the A1 case we have the following *pdes* and *stencils*:

```python
if(habctype==1):

    # Region B_{1}
    aux1      = ((-vel[x,z]*dt+hx)*u2[x,z] + (vel[x,z]*dt+hx)*u2[x+1,z] + (vel[x,z]*dt-hx)*u3[x+1,z])/(vel[x,z]*dt+hx)
    pde1      = (1-weightsx[x,z])*u3[x,z] + weightsx[x,z]*aux1
    stencil1  = Eq(u.forward,pde1,subdomain = grid.subdomains['d1'])

    # Region B_{3}
    aux2      = ((-vel[x,z]*dt+hx)*u2[x,z] + (vel[x,z]*dt+hx)*u2[x-1,z] + (vel[x,z]*dt-hx)*u3[x-1,z])/(vel[x,z]*dt+hx)
    pde2      = (1-weightsx[x,z])*u3[x,z] + weightsx[x,z]*aux2
    stencil2  = Eq(u.forward,pde2,subdomain = grid.subdomains['d2'])

    # Region B_{2}
    aux3      = ((-vel[x,z]*dt+hz)*u2[x,z] + (vel[x,z]*dt+hz)*u2[x,z-1] + (vel[x,z]*dt-hz)*u3[x,z-1])/(vel[x,z]*dt+hz)
    pde3      = (1-weightsz[x,z])*u3[x,z] + weightsz[x,z]*aux3
    stencil3  = Eq(u.forward,pde3,subdomain = grid.subdomains['d3'])
```

For the A2 case we have the following *pdes* and *stencils*:

```python
if(habctype==2):

    # Region B_{1}
    cte11 =  (1/(2*dt**2)) + (1/(2*dt*hx))*vel[x,z]
    cte21 = -(1/(2*dt**2)) + (1/(2*dt*hx))*vel[x,z] - (1/(2*hz**2))*vel[x,z]*vel[x,z]
    cte31 = -(1/(2*dt**2)) - (1/(2*dt*hx))*vel[x,z]
    cte41 =  (1/(dt**2))
    cte51 =  (1/(4*hz**2))*vel[x,z]**2

    aux1      = (cte21*(u3[x+1,z] + u1[x,z]) + cte31*u1[x+1,z] + cte41*(u2[x,z]+u2[x+1,z]) + cte51*(u3[x+1,z+1] + u3[x+1,z-1] + u1[x,z+1] + u1[x,z-1]))/cte11
    pde1      = (1-weightsx[x,z])*u3[x,z] + weightsx[x,z]*aux1
    stencil1  = Eq(u.forward,pde1,subdomain = grid.subdomains['d1'])

    # Region B_{3}
    cte12 =  (1/(2*dt**2)) + (1/(2*dt*hx))*vel[x,z]
    cte22 = -(1/(2*dt**2)) + (1/(2*dt*hx))*vel[x,z] - (1/(2*hz**2))*vel[x,z]**2
    cte32 = -(1/(2*dt**2)) - (1/(2*dt*hx))*vel[x,z]
    cte42 =  (1/(dt**2))
    cte52 =  (1/(4*hz**2))*vel[x,z]*vel[x,z]

    aux2      = (cte22*(u3[x-1,z] + u1[x,z]) + cte32*u1[x-1,z] + cte42*(u2[x,z]+u2[x-1,z]) + cte52*(u3[x-1,z+1] + u3[x-1,z-1] + u1[x,z+1] + u1[x,z-1]))/cte12
    pde2      = (1-weightsx[x,z])*u3[x,z] + weightsx[x,z]*aux2
    stencil2  = Eq(u.forward,pde2,subdomain = grid.subdomains['d2'])

    # Region B_{2}
    cte13 =  (1/(2*dt**2)) + (1/(2*dt*hz))*vel[x,z]
    cte23 = -(1/(2*dt**2)) + (1/(2*dt*hz))*vel[x,z] - (1/(2*hx**2))*vel[x,z]**2
    cte33 = -(1/(2*dt**2)) - (1/(2*dt*hz))*vel[x,z]
    cte43 =  (1/(dt**2))
    cte53 =  (1/(4*hx**2))*vel[x,z]*vel[x,z]

    aux3      = (cte23*(u3[x,z-1] + u1[x,z]) + cte33*u1[x,z-1] + cte43*(u2[x,z]+u2[x,z-1]) + cte53*(u3[x+1,z-1] + u3[x-1,z-1] + u1[x+1,z] + u1[x-1,z]))/cte13
    pde3      = (1-weightsz[x,z])*u3[x,z] + weightsz[x,z]*aux3
    stencil3  = Eq(u.forward,pde3,subdomain = grid.subdomains['d3'])

    # Red point rigth side
    stencil4  = [Eq(u[t+1,nptx-1-k,nptz-1-k],(1-weightsz[nptx-1-k,nptz-1-k])*u3[nptx-1-k,nptz-1-k] +
                 weightsz[nptx-1-k,nptz-1-k]*(((-(1/(4*hx))   + (1/(4*hz)) - (np.sqrt(2))/(4*vel[nptx-1-k,nptz-1-k]*dt))*u3[nptx-1-k,nptz-2-k]
                 + ((1/(4*hx))  - (1/(4*hz)) - (np.sqrt(2))/(4*vel[nptx-1-k,nptz-1-k]*dt))*u3[nptx-2-k,nptz-1-k]
                 + ((1/(4*hx))  + (1/(4*hz)) - (np.sqrt(2))/(4*vel[nptx-1-k,nptz-1-k]*dt))*u3[nptx-2-k,nptz-2-k]
                 + (-(1/(4*hx)) - (1/(4*hz)) + (np.sqrt(2))/(4*vel[nptx-1-k,nptz-1-k]*dt))*u2[nptx-1-k,nptz-1-k]
                 + (-(1/(4*hx)) + (1/(4*hz)) + (np.sqrt(2))/(4*vel[nptx-1-k,nptz-1-k]*dt))*u2[nptx-1-k,nptz-2-k]
                 + ((1/(4*hx))  - (1/(4*hz)) + (np.sqrt(2))/(4*vel[nptx-1-k,nptz-1-k]*dt))*u2[nptx-2-k,nptz-1-k]
                 + ((1/(4*hx))  + (1/(4*hz)) + (np.sqrt(2))/(4*vel[nptx-1-k,nptz-1-k]*dt))*u2[nptx-2-k,nptz-2-k])
                 / (((1/(4*hx)) + (1/(4*hz)) + (np.sqrt(2))/(4*vel[nptx-1-k,nptz-1-k]*dt))))) for k in range(0,npmlz)]

    # Red point left side
    stencil5  = [Eq(u[t+1,k,nptz-1-k],(1-weightsx[k,nptz-1-k] )*u3[k,nptz-1-k]
                                + weightsx[k,nptz-1-k]*(( (-(1/(4*hx)) + (1/(4*hz)) - (np.sqrt(2))/(4*vel[k,nptz-1-k]*dt))*u3[k,nptz-2-k]
                                + ((1/(4*hx))  - (1/(4*hz)) - (np.sqrt(2))/(4*vel[k,nptz-1-k]*dt))*u3[k+1,nptz-1-k]
                                + ((1/(4*hx))  + (1/(4*hz)) - (np.sqrt(2))/(4*vel[k,nptz-1-k]*dt))*u3[k+1,nptz-2-k]
                                + (-(1/(4*hx)) - (1/(4*hz)) + (np.sqrt(2))/(4*vel[k,nptz-1-k]*dt))*u2[k,nptz-1-k]
                                + (-(1/(4*hx)) + (1/(4*hz)) + (np.sqrt(2))/(4*vel[k,nptz-1-k]*dt))*u2[k,nptz-2-k]
                                + ((1/(4*hx))  - (1/(4*hz)) + (np.sqrt(2))/(4*vel[k,nptz-1-k]*dt))*u2[k+1,nptz-1-k]
                                + ((1/(4*hx))  + (1/(4*hz)) + (np.sqrt(2))/(4*vel[k,nptz-1-k]*dt))*u2[k+1,nptz-2-k])
                                / (((1/(4*hx)) + (1/(4*hz)) + (np.sqrt(2))/(4*vel[k,nptz-1-k]*dt))))) for k in range(0,npmlx)]
```

For the Higdon case we have the following *pdes* and *stencils*:

```python
if(habctype==3):

    alpha1 = 0.0
    alpha2 = np.pi/4
    a1 = 0.5
    b1 = 0.5
    a2 = 0.5
    b2 = 0.5

    # Region B_{1}
    gama111 = np.cos(alpha1)*(1-a1)*(1/dt)
    gama121 = np.cos(alpha1)*(a1)*(1/dt)
    gama131 = np.cos(alpha1)*(1-b1)*(1/hx)*vel[x,z]
    gama141 = np.cos(alpha1)*(b1)*(1/hx)*vel[x,z]

    gama211 = np.cos(alpha2)*(1-a2)*(1/dt)
    gama221 = np.cos(alpha2)*(a2)*(1/dt)
    gama231 = np.cos(alpha2)*(1-b2)*(1/hx)*vel[x,z]
    gama241 = np.cos(alpha2)*(b2)*(1/hx)*vel[x,z]

    c111 =  gama111 + gama131
    c121 = -gama111 + gama141
    c131 =  gama121 - gama131
    c141 = -gama121 - gama141

    c211 =  gama211 + gama231
    c221 = -gama211 + gama241
    c231 =  gama221 - gama231
    c241 = -gama221 - gama241

    aux1      = ( u2[x,z]*(-c111*c221-c121*c211) + u3[x+1,z]*(-c111*c231-c131*c211) + u2[x+1,z]*(-c111*c241-c121*c231-c141*c211-c131*c221)
                + u1[x,z]*(-c121*c221) + u1[x+1,z]*(-c121*c241-c141*c221) + u3[x+2,z]*(-c131*c231) +u2[x+2,z]*(-c131*c241-c141*c231)
                + u1[x+2,z]*(-c141*c241))/(c111*c211)
    pde1      = (1-weightsx[x,z])*u3[x,z] + weightsx[x,z]*aux1
    stencil1  = Eq(u.forward,pde1,subdomain = grid.subdomains['d1'])

    # Region B_{3}
    gama112 = np.cos(alpha1)*(1-a1)*(1/dt)
    gama122 = np.cos(alpha1)*(a1)*(1/dt)
    gama132 = np.cos(alpha1)*(1-b1)*(1/hx)*vel[x,z]
    gama142 = np.cos(alpha1)*(b1)*(1/hx)*vel[x,z]

    gama212 = np.cos(alpha2)*(1-a2)*(1/dt)
    gama222 = np.cos(alpha2)*(a2)*(1/dt)
    gama232 = np.cos(alpha2)*(1-b2)*(1/hx)*vel[x,z]
    gama242 = np.cos(alpha2)*(b2)*(1/hx)*vel[x,z]

    c112 =  gama112 + gama132
    c122 = -gama112 + gama142
    c132 =  gama122 - gama132
    c142 = -gama122 - gama142

    c212 =  gama212 + gama232
    c222 = -gama212 + gama242
    c232 =  gama222 - gama232
    c242 = -gama222 - gama242

    aux2      = ( u2[x,z]*(-c112*c222-c122*c212) + u3[x-1,z]*(-c112*c232-c132*c212) + u2[x-1,z]*(-c112*c242-c122*c232-c142*c212-c132*c222)
                + u1[x,z]*(-c122*c222) + u1[x-1,z]*(-c122*c242-c142*c222) + u3[x-2,z]*(-c132*c232) +u2[x-2,z]*(-c132*c242-c142*c232)
                + u1[x-2,z]*(-c142*c242))/(c112*c212)
    pde2      = (1-weightsx[x,z])*u3[x,z] + weightsx[x,z]*aux2
    stencil2  = Eq(u.forward,pde2,subdomain = grid.subdomains['d2'])

    # Region B_{2}
    gama113 = np.cos(alpha1)*(1-a1)*(1/dt)
    gama123 = np.cos(alpha1)*(a1)*(1/dt)
    gama133 = np.cos(alpha1)*(1-b1)*(1/hz)*vel[x,z]
    gama143 = np.cos(alpha1)*(b1)*(1/hz)*vel[x,z]

    gama213 = np.cos(alpha2)*(1-a2)*(1/dt)
    gama223 = np.cos(alpha2)*(a2)*(1/dt)
    gama233 = np.cos(alpha2)*(1-b2)*(1/hz)*vel[x,z]
    gama243 = np.cos(alpha2)*(b2)*(1/hz)*vel[x,z]

    c113 =  gama113 + gama133
    c123 = -gama113 + gama143
    c133 =  gama123 - gama133
    c143 = -gama123 - gama143

    c213 =  gama213 + gama233
    c223 = -gama213 + gama243
    c233 =  gama223 - gama233
    c243 = -gama223 - gama243

    aux3      = ( u2[x,z]*(-c113*c223-c123*c213) + u3[x,z-1]*(-c113*c233-c133*c213) + u2[x,z-1]*(-c113*c243-c123*c233-c143*c213-c133*c223)
                + u1[x,z]*(-c123*c223) + u1[x,z-1]*(-c123*c243-c143*c223) + u3[x,z-2]*(-c133*c233) +u2[x,z-2]*(-c133*c243-c143*c233)
                + u1[x,z-2]*(-c143*c243))/(c113*c213)
    pde3      = (1-weightsz[x,z])*u3[x,z] + weightsz[x,z]*aux3
    stencil3  = Eq(u.forward,pde3,subdomain = grid.subdomains['d3'])
```

The surface boundary conditions of the problem are the same as in the notebook Introduction to Acoustic Problem. They are placed in the term *bc* and have the following form:

```python
bc  = [Eq(u[t+1,x,0],u[t+1,x,1])]
```

We will then define the operator (*op*) that will join the acoustic equation, source term, boundary conditions and receivers.

- 1. The acoustic wave equation in the *d0* region: *[stencil01];*
- 2. Source term: *src_term;*
- 3. Updating solutions over time: *[stencil01,stencil02];*
- 4. The acoustic wave equation in the *d1*, *d2* e *d3* regions: *[stencil1,stencil2,stencil3];*
- 5. The equation for red points for A2 method: *[stencil5,stencil4];*
- 6. Boundry Conditions: *bc;*
- 7. Receivers: *rec_term;*

We then define two types of *op*:

- The first *op* is for the cases A1 and Higdon;
- The second *op* is for the case A2;

The *ops* are constructed by the following commands:

```python
# NBVAL_IGNORE_OUTPUT

if(habctype!=2):
    op  = Operator([stencil0] + src_term + [stencil01,stencil3,stencil02,stencil2,stencil1] + bc + rec_term,subs=grid.spacing_map)
else:
    op  = Operator([stencil0] + src_term + [stencil01,stencil3,stencil02,stencil2,stencil1,stencil02,stencil4,stencil5] + bc + rec_term,subs=grid.spacing_map)
```

Initially:

```python
u.data[:]  = 0.
u1.data[:] = 0.
u2.data[:] = 0.
u3.data[:] = 0.
```

We assign to *op* the number of time steps it must execute and the size of the time step in the local variables *time* and *dt*, respectively.

```python
# NBVAL_IGNORE_OUTPUT

op(time=nt,dt=dt0)
```

    Operator `Kernel` run in 0.04 s

    PerformanceSummary([(PerfKey(name='section0', rank=None),
                         PerfEntry(time=0.009703000000000015, gflopss=0.0, gpointss=0.0, oi=0.0, ops=0, itershapes=[])),
                        (PerfKey(name='section1', rank=None),
                         PerfEntry(time=3.8999999999999986e-05, gflopss=0.0, gpointss=0.0, oi=0.0, ops=0, itershapes=[])),
                        (PerfKey(name='section2', rank=None),
                         PerfEntry(time=0.028597999999999967, gflopss=0.0, gpointss=0.0, oi=0.0, ops=0, itershapes=[])),
                        (PerfKey(name='section3', rank=None),
                         PerfEntry(time=0.0011050000000000083, gflopss=0.0, gpointss=0.0, oi=0.0, ops=0, itershapes=[]))])

We view the result of the displacement field at the end time using the *graph2d* routine given by:

```python
def graph2d(U,i):
    plot.figure()
    plot.figure(figsize=(16,8))
    fscale =  1/10**(3)
    x0pml  = x0 + npmlx*hxv
    x1pml  = x1 - npmlx*hxv
    z0pml  = z0
    z1pml  = z1 - npmlz*hzv
    scale  = np.amax(U[npmlx:-npmlx,0:-npmlz])/10.
    extent = [fscale*x0pml,fscale*x1pml,fscale*z1pml,fscale*z0pml]
    fig = plot.imshow(np.transpose(U[npmlx:-npmlx,0:-npmlz]),vmin=-scale, vmax=scale, cmap=cm.seismic, extent=extent)
    plot.gca().xaxis.set_major_formatter(mticker.FormatStrFormatter('%.1f km'))
    plot.gca().yaxis.set_major_formatter(mticker.FormatStrFormatter('%.1f km'))
    plot.axis('equal')
    if(i==1): plot.title('Map - Acoustic Problem with Devito - HABC A1')
    if(i==2): plot.title('Map - Acoustic Problem with Devito - HABC A2')
    if(i==3): plot.title('Map - Acoustic Problem with Devito - HABC Higdon')
    plot.grid()
    ax = plot.gca()
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="5%", pad=0.05)
    cbar = plot.colorbar(fig, cax=cax, format='%.2e')
    cbar.set_label('Displacement [km]')
    plot.draw()
    plot.show()
```

```python
# NBVAL_IGNORE_OUTPUT

graph2d(u.data[0,:,:],habctype)
```

We plot the Receivers shot records using the *graph2drec* routine.

```python
def graph2drec(rec,i):
        plot.figure()
        plot.figure(figsize=(16,8))
        fscaled = 1/10**(3)
        fscalet = 1/10**(3)
        x0pml  = x0 + npmlx*hxv
        x1pml  = x1 - npmlx*hxv
        scale   = np.amax(rec[:,npmlx:-npmlx])/10.
        extent  = [fscaled*x0pml,fscaled*x1pml, fscalet*tn, fscalet*t0]
        fig = plot.imshow(rec[:,npmlx:-npmlx], vmin=-scale, vmax=scale, cmap=cm.seismic, extent=extent)
        plot.gca().xaxis.set_major_formatter(mticker.FormatStrFormatter('%.1f km'))
        plot.gca().yaxis.set_major_formatter(mticker.FormatStrFormatter('%.1f s'))
        plot.axis('equal')
        if(i==1): plot.title('Receivers Signal Profile - Devito with HABC A1')
        if(i==2): plot.title('Receivers Signal Profile - Devito with HABC A2')
        if(i==3): plot.title('Receivers Signal Profile - Devito with HABC Higdon')
        ax = plot.gca()
        divider = make_axes_locatable(ax)
        cax = divider.append_axes("right", size="5%", pad=0.05)
        cbar = plot.colorbar(fig, cax=cax, format='%.2e')
        plot.show()
```

```python
# NBVAL_IGNORE_OUTPUT

graph2drec(rec.data,habctype)
```

```python
assert np.isclose(np.linalg.norm(rec.data), 990, rtol=1)
```

# 4.7 - Conclusions

We have presented the HABC method for the acoustic wave equation, which can be used with any of the
absorbing boundary conditions A1, A2 or Higdon. The notebook also include the possibility of using these boundary conditions alone, without being combined with the HABC. The user has the possibilty of testing several combinations of parameters and observe the effects in the absorption of spurious reflections on computational boundaries.

 The relevant references for the boundary conditions are furnished next.

## 4.8 - References

- Clayton, R., & Engquist, B. (1977). "Absorbing boundary conditions for acoustic and elastic wave equations", Bulletin of the seismological society of America, 67(6), 1529-1540. Reference Link.

- Engquist, B., & Majda, A. (1979). "Radiation boundary conditions for acoustic and elastic wave calculations," Communications on pure and applied mathematics, 32(3), 313-357. DOI: 10.1137/0727049.  Reference Link.

- Higdon, R. L. (1987). "Absorbing boundary conditions for difference approximations to the multidimensional wave equation," Mathematics of computation, 47(176), 437-459. DOI: 10.1090/S0025-5718-1986-0856696-4. Reference Link.

- Higdon, Robert L. "Numerical absorbing boundary conditions for the wave equation," Mathematics of computation, v. 49, n. 179, p. 65-90, 1987. DOI: 10.1090/S0025-5718-1987-0890254-1. Reference Link.

- Liu, Y., & Sen, M. K. (2018). "An improved hybrid absorbing boundary condition for wave equation modeling," Journal of Geophysics and Engineering, 15(6), 2602-2613. DOI: 10.1088/1742-2140/aadd31. Reference Link.


## OIQ-Exam-Question-1 (Version 2)
Technical exam question from Ordre des ingénieurs du Québec.  Obviously meant to be done using moment-distribution, but even easier using slope-deflection.  This version use a newer 'sdutil' that also computes end shears.

```python
from sympy import *
init_printing(use_latex='mathjax')
from IPython import display
```

```python
display.SVG('oiq-exam-1.svg')
```

```python
from sdutil2 import SD, FEF
var('EI theta_a theta_b theta_c theta_d')
Mab,Mba,Vab,Vba = SD(6,EI,theta_a,theta_b) + FEF.p(6,180,4)
Mbc,Mcb,Vbc,Vcb = SD(8,2*EI,theta_b,theta_c) + FEF.udl(8,45)
Mcd,Mdc,Vcd,Vdc = SD(6,EI,theta_c,theta_d)
```

```python
Mab
```

$$\frac{EI}{6} \left(4 \theta_{a} + 2 \theta_{b}\right) - 80.0$$

Solve equilbrium equations for rotations:

```python
soln = solve( [Mab,Mba+Mbc,Mcb+Mcd,Mdc],[theta_a,theta_b,theta_c,theta_d] )
soln
```

$$\left \{ \theta_{a} : \frac{75.0}{EI}, \quad \theta_{b} : \frac{90.0}{EI}, \quad \theta_{c} : - \frac{190.0}{EI}, \quad \theta_{d} : \frac{95.0}{EI}\right \}$$

Member end moments:

```python
[m.subs(soln) for m in [Mab,Mba,Mbc,Mcb,Mcd,Mdc]]
```

$$\left [ 0, \quad 245.0, \quad -245.0, \quad 95.0, \quad -95.0, \quad 0\right ]$$

Member end shears:

```python
[v.subs(soln).n(4) for v in [Vab,Vba,Vbc,Vcb,Vcd,Vdc]]
```

$$\left [ 19.17, \quad -160.8, \quad 198.8, \quad -161.3, \quad 15.83, \quad 15.83\right ]$$

Reactions:

```python
Ra = Vab
Rb = Vbc - Vba
Rc = Vcd - Vcb
Rd = -Vdc
[r.subs(soln).n(4) for r in [Ra,Rb,Rc,Rd]]
```

$$\left [ 19.17, \quad 359.6, \quad 177.1, \quad -15.83\right ]$$

#### Check overal equilibrium

```python
# sum forces in vertical dirn.
(Ra+Rb+Rc+Rd - 180 - 45*8).subs(soln)
```

$$0$$

```python
# sum moments about left
(-Rb*6 - Rc*(6+8) -Rd*(6+8+6) + 180*4 + 45*8*(6 + 8/2.)).subs(soln)
```

$$0$$

```python
Ra.expand()
```

$$- \frac{EI \theta_{a}}{6} - \frac{EI \theta_{b}}{6} + 46.6666666666667$$

```python

```