# Nonlinear prediction of chaotic dynamical systems

Assume you observe a time series $(y_1, y_2, \dots, y_T)$ that represents a variable of a high dimensional and possibly chaotic dynamical system.

The method proposed by Sugihara and May (Nature, 1990) entails first choosing an embedding dimension, $n$, and then predicting $y_t$ by using past observations $\vec y_p(t) = (y_{t-1}, y_{t-2}, \dots, y_{t-n}) \in \mathbb{R}^n$.

Intuitively, the prediction is obtained by finding a set of vectors $\{ \vec y^1_p, \dots, \vec y^n_p \}$ in the past, which are nearest neighbors of $\vec y_p$, and basing future predictions on what occurred immediately after these past events.

Let's begin with an example.

```python
# simulate Lorenz attractor
import numpy as np
import matplotlib.pyplot as plt
from scipy.integrate import odeint
from mpl_toolkits.mplot3d import Axes3D

# standard parameters
rho=28;sigma=10;beta=8/3
def f(X, t):
  x, y, z = X
  return sigma * (y - x), x * (rho - z) - y, x * y - beta * z
# initial value
x0 = np.array([-11.40057002, -14.01987468,  27.49928125])
t = np.arange(0.0, 100, 0.01)
lorenz = odeint(f, x0, t)
```

```python
# 3D plot of Lorenz attractor - beautiful!
fig = plt.figure()
ax = fig.gca(projection='3d')
ax.plot(lorenz[:,0], lorenz[:,1], lorenz[:,2])
plt.axis('off')
plt.xlim([-13,14])
plt.ylim([-20,25])
ax.set_zlim([15,38])
plt.show()
```

```python
# plot only first component
plt.plot(lorenz[:,0])
```

#### How can we predict the future of such a chaotic system, given its past??

Many methods have been proposed that are based on Takens (1981) and Sauer (1991) theorems (see: http://www.scholarpedia.org/article/Attractor_reconstruction)

In detail, the method proposed by Sugihara and May (https://www.nature.com/articles/344734a0) works in three steps:

 ( 1 )  divide your time series $y_1,\dots, y_T$ into train and test. Use train time series as a library of past patterns, i.e., by computing past vectors $\vec y_p(t) = (y_{t-1}, y_{t-2}, \dots, y_{t-n}) \in \mathbb{R}^n$ for each point $t>n$, with associated future value $y_t$.

 ( 2 )  for each test time point $y^*$ compute its past vector $\vec y_p^* = (y_{t^*-1}, y_{t^*-2}, \dots, y_{t^*-n})$, and find the $n+1$ nearest neighbors in the library of past patterns: $\{y_p^1, \dots, y_p^{n+1}\}$ and compute their distance to $y_p^*$: $d_i = || y_p^* - y_p^i||$.

 ( 3 )  predict test time point $y^*$ by taking a weighted average of future values of the $n+1$ nearest neighbors found in the library of past patterns:
    \begin{equation}
        \hat y^* = \frac{\sum_i^{n+1} y^i e^{-d_i}}{\sum_i^{n+1} e^{-d_i}}.
    \end{equation}

```python
from sklearn.neighbors import NearestNeighbors
from scipy import stats as stats

# split dataset into train/test
def train_test_split(X, fraction_train=.75):
    split = int(len(X)*fraction_train)
    return X[:split], X[split:]

# exponential weights: w_i = exp(-d_i) / sum_i exp(-d_i)
def weights(distances):
    num = np.exp(-distances) # numerator: e^{-d_i}
    den = np.sum(num,axis=1,keepdims=True) # denominator: sum_i e^{-d_i}
    return num/den

# embed vectors into n-dimensional past values (the last element is the one to be predicted)
def embed_vectors_1d(X, n_embed):
    size = len(X)
    leng = size-n_embed
    out_vects = np.zeros((leng,n_embed + 1))
    for i in range(leng):
        out_vects[i,:] = X[i:i+n_embed+1]
    return out_vects

# implement the Sugihara nonlinear prediction
def nonlinear_prediction(X_train, X_test, n_embed):

    # initialize nearest neighbors from sklearn
    knn = NearestNeighbors(n_neighbors=n_embed+1)

    # Nearest neigbors is fit on the train data (only on the past vectors - i.e. till [:-1])
    knn.fit(X_train[:,:-1])

    # find the nearest neighbors for each test vector input
    dist,ind = knn.kneighbors(X_test[:,:-1])

    # compute exponential weights given distances
    W = weights(dist)

    # predict test using train (weighted average)
    x_pred = np.sum(X_train[ind][:,-1] * W, axis=1)

    return x_pred
```

Find the best embedding dimension by cross-validation - i.e., find best reconstruction

```python
nonlinear_reconstr_cor = []
for n_embed in np.arange(1,5):

  X = embed_vectors_1d(lorenz[:,0],n_embed)

  # split train/test
  X_train, X_test = train_test_split(X,fraction_train=0.7)

  # nonlinear prediction on individual time series
  x_p = nonlinear_prediction(X_train, X_test, n_embed)

  # simply check correlation of real vs predicted
  nonlinear_reconstr_cor.append(np.corrcoef(X_test[:,-1], x_p)[0,1])
```

```python
plt.plot(np.arange(1,5),nonlinear_reconstr_cor[:5])
plt.xticks(np.arange(1,5))
plt.xlabel('Embedding dimension (n_embed)')
plt.ylabel('Pearson correlation')
```

The best embedding dimension is 2!
Now visualize the results and check where the highest errors are

```python
n_embed = 2

# prediction using optimal n_embed
X = embed_vectors_1d(lorenz[:,0],n_embed)

# split train/test
X_train, X_test = train_test_split(X,fraction_train=0.7)

# nonlinear prediction on individual time series
x_p = nonlinear_prediction(X_train, X_test, n_embed)

# simply check correlation of
nonlinear_reconstr_cor.append(np.corrcoef(X_test[:,-1], x_p)[0,1])

# scatter plot real vs predicted
plt.scatter(X_test[:,-1], x_p)
plt.xlabel('Test Data')
plt.ylabel('Prediction Data')
```

Are errors higher where the gradient is higher? Yes!

```python
plt.scatter(np.abs(np.gradient(X_test[:,-1])), np.abs(X_test[:,-1]- x_p))
plt.xlabel('Abs. Gradient')
plt.ylabel('Abs. Prediction Error')
```

Notebook 1 Nonlinear prediction of chaotic dynamical systems.ipynb


### Example 2: Nonlinear convection in 2D

Following the initial convection tutorial with a single state variable $u$, we will now look at non-linear convection (step 6 in the original). This brings one new crucial challenge: computing a pair of coupled equations and thus updating two time-dependent variables $u$ and $v$.

The full set of coupled equations is now

\begin{aligned}
\frac{\partial u}{\partial t} + u \frac{\partial u}{\partial x} + v \frac{\partial u}{\partial y} = 0 \\
\\
\frac{\partial v}{\partial t} + u \frac{\partial v}{\partial x} + v \frac{\partial v}{\partial y} = 0\\
\end{aligned}

and rearranging the discretized version gives us an expression for the update of both variables

\begin{aligned}
u_{i,j}^{n+1} &= u_{i,j}^n - u_{i,j} \frac{\Delta t}{\Delta x} (u_{i,j}^n-u_{i-1,j}^n) - v_{i,j}^n \frac{\Delta t}{\Delta y} (u_{i,j}^n-u_{i,j-1}^n) \\
\\
v_{i,j}^{n+1} &= v_{i,j}^n - u_{i,j} \frac{\Delta t}{\Delta x} (v_{i,j}^n-v_{i-1,j}^n) - v_{i,j}^n \frac{\Delta t}{\Delta y} (v_{i,j}^n-v_{i,j-1}^n)
\end{aligned}

So, for starters we will re-create the original example run in pure NumPy array notation, before demonstrating
the Devito version. Let's start again with some utilities and parameters:

```python
from examples.cfd import plot_field, init_hat
import numpy as np
import sympy
%matplotlib inline

# Some variable declarations
nx = 101
ny = 101
nt = 80
c = 1.
dx = 2. / (nx - 1)
dy = 2. / (ny - 1)
sigma = .2
dt = sigma * dx
```

Let's re-create the initial setup with a 2D "hat function", but this time for two state variables.

```python
#NBVAL_IGNORE_OUTPUT

# Allocate fields and assign initial conditions
u = np.empty((nx, ny))
v = np.empty((nx, ny))

init_hat(field=u, dx=dx, dy=dy, value=2.)
init_hat(field=v, dx=dx, dy=dy, value=2.)

plot_field(u)
```

Now we can create the two stencil expression for our two coupled equations according to the discretized equation above. We again use some simple Dirichlet boundary conditions to keep the values on all sides constant.

```python
#NBVAL_IGNORE_OUTPUT
for n in range(nt + 1): ##loop across number of time steps
    un = u.copy()
    vn = v.copy()
    u[1:, 1:] = (un[1:, 1:] -
                 (un[1:, 1:] * c * dt / dx * (un[1:, 1:] - un[1:, :-1])) -
                  vn[1:, 1:] * c * dt / dy * (un[1:, 1:] - un[:-1, 1:]))
    v[1:, 1:] = (vn[1:, 1:] -
                 (un[1:, 1:] * c * dt / dx * (vn[1:, 1:] - vn[1:, :-1])) -
                 vn[1:, 1:] * c * dt / dy * (vn[1:, 1:] - vn[:-1, 1:]))

    u[0, :] = 1
    u[-1, :] = 1
    u[:, 0] = 1
    u[:, -1] = 1

    v[0, :] = 1
    v[-1, :] = 1
    v[:, 0] = 1
    v[:, -1] = 1

plot_field(u)
```

Excellent, we again get a wave that resembles the one from the oiginal examples.

Now we can set up our coupled problem in Devito. Let's start by creating two initial state variables $u$ and $v$, as before, and intialising them with our "hat function.

```python
#NBVAL_IGNORE_OUTPUT
from devito import Grid, TimeFunction

# First we need two time-dependent data fields, both initialized with the hat function
grid = Grid(shape=(nx, ny), extent=(2., 2.))
u = TimeFunction(name='u', grid=grid)
init_hat(field=u.data[0], dx=dx, dy=dy, value=2.)

v = TimeFunction(name='v', grid=grid)
init_hat(field=v.data[0], dx=dx, dy=dy, value=2.)

plot_field(u.data[0])
```

Using the two `TimeFunction` objects we can again derive our dicretized equation, rearrange for the forward stencil point in time and define our variable update expression - only we have to do everything twice now! We again use foward differences for time via `u.dt` and backward differences in space via `u.dxl` and `u.dyl` to match the original tutorial.

```python
from devito import Eq, solve

eq_u = Eq(u.dt + u*u.dxl + v*u.dyl)
eq_v = Eq(v.dt + u*v.dxl + v*v.dyl)

# We can use the same SymPy trick to generate two
# stencil expressions, one for each field update.
stencil_u = solve(eq_u, u.forward)
stencil_v = solve(eq_v, v.forward)
update_u = Eq(u.forward, stencil_u, subdomain=grid.interior)
update_v = Eq(v.forward, stencil_v, subdomain=grid.interior)

print("U update:\n%s\n" % update_u)
print("V update:\n%s\n" % update_v)
```

    U update:
    Eq(u(t + dt, x, y), dt*(-(u(t, x, y)/h_x - u(t, x - h_x, y)/h_x)*u(t, x, y) - (u(t, x, y)/h_y - u(t, x, y - h_y)/h_y)*v(t, x, y) + u(t, x, y)/dt))

    V update:
    Eq(v(t + dt, x, y), dt*(-(v(t, x, y)/h_x - v(t, x - h_x, y)/h_x)*u(t, x, y) - (v(t, x, y)/h_y - v(t, x, y - h_y)/h_y)*v(t, x, y) + v(t, x, y)/dt))

We then set Dirichlet boundary conditions at all sides of the domain to $1$.

```python
x, y = grid.dimensions
t = grid.stepping_dim
bc_u = [Eq(u[t+1, 0, y], 1.)]  # left
bc_u += [Eq(u[t+1, nx-1, y], 1.)]  # right
bc_u += [Eq(u[t+1, x, ny-1], 1.)]  # top
bc_u += [Eq(u[t+1, x, 0], 1.)]  # bottom
bc_v = [Eq(v[t+1, 0, y], 1.)]  # left
bc_v += [Eq(v[t+1, nx-1, y], 1.)]  # right
bc_v += [Eq(v[t+1, x, ny-1], 1.)]  # top
bc_v += [Eq(v[t+1, x, 0], 1.)]  # bottom
```

And finally we can put it all together to build an operator and solve our coupled problem.

```python
#NBVAL_IGNORE_OUTPUT
from devito import Operator

# Reset our data field and ICs
init_hat(field=u.data[0], dx=dx, dy=dy, value=2.)
init_hat(field=v.data[0], dx=dx, dy=dy, value=2.)

op = Operator([update_u, update_v] + bc_u + bc_v)
op(time=nt, dt=dt)

plot_field(u.data[0])
```

Excellent, we have now a scalar implementation of a convection problem, but this can be writen as a single vectorial equation:

$\frac{d U}{dt} + \nabla(U)U = 0$

Let's now use devito vectorial utilities and implement the vectorial equation

```python
from devito import VectorTimeFunction, grad

U = VectorTimeFunction(name='U', grid=grid)
init_hat(field=U[0].data[0], dx=dx, dy=dy, value=2.)
init_hat(field=U[1].data[0], dx=dx, dy=dy, value=2.)

plot_field(U[1].data[0])

eq_u = Eq(U.dt + grad(U)*U)
```

We now have a vectorial equation. Unlike hte previous case, we do not need to play woth legt/right derivatives
as the automated staggering of vectorial function takes care of it.

```python
eq_u
```

$\displaystyle \left[\begin{matrix}\operatorname{U_{x}}{\left(t,x + \frac{h_{x}}{2},y \right)} \frac{\partial}{\partial x} \operatorname{U_{x}}{\left(t,x + \frac{h_{x}}{2},y \right)} + \operatorname{U_{y}}{\left(t,x,y + \frac{h_{y}}{2} \right)} \frac{\partial}{\partial y} \operatorname{U_{x}}{\left(t,x + \frac{h_{x}}{2},y \right)} + \frac{\partial}{\partial t} \operatorname{U_{x}}{\left(t,x + \frac{h_{x}}{2},y \right)}\\\operatorname{U_{x}}{\left(t,x + \frac{h_{x}}{2},y \right)} \frac{\partial}{\partial x} \operatorname{U_{y}}{\left(t,x,y + \frac{h_{y}}{2} \right)} + \operatorname{U_{y}}{\left(t,x,y + \frac{h_{y}}{2} \right)} \frac{\partial}{\partial y} \operatorname{U_{y}}{\left(t,x,y + \frac{h_{y}}{2} \right)} + \frac{\partial}{\partial t} \operatorname{U_{y}}{\left(t,x,y + \frac{h_{y}}{2} \right)}\end{matrix}\right] = 0$

Then we set the nboundary conditions

```python
x, y = grid.dimensions
t = grid.stepping_dim
bc_u = [Eq(U[0][t+1, 0, y], 1.)]  # left
bc_u += [Eq(U[0][t+1, nx-1, y], 1.)]  # right
bc_u += [Eq(U[0][t+1, x, ny-1], 1.)]  # top
bc_u += [Eq(U[0][t+1, x, 0], 1.)]  # bottom
bc_v = [Eq(U[1][t+1, 0, y], 1.)]  # left
bc_v += [Eq(U[1][t+1, nx-1, y], 1.)]  # right
bc_v += [Eq(U[1][t+1, x, ny-1], 1.)]  # top
bc_v += [Eq(U[1][t+1, x, 0], 1.)]  # bottom
```

```python
# We can use the same SymPy trick to generate two
# stencil expressions, one for each field update.
stencil_U = solve(eq_u, U.forward)
update_U = Eq(U.forward, stencil_U, subdomain=grid.interior)
```

And we have the update (stencil) as a vectorial equation once again

```python
update_U
```

$\displaystyle \left[\begin{matrix}\operatorname{U_{x}}{\left(t + dt,x + \frac{h_{x}}{2},y \right)}\\\operatorname{U_{y}}{\left(t + dt,x,y + \frac{h_{y}}{2} \right)}\end{matrix}\right] = \left[\begin{matrix}dt \left(- \left(- \frac{\operatorname{U_{x}}{\left(t,x - \frac{h_{x}}{2},y \right)}}{h_{x}} + \frac{\operatorname{U_{x}}{\left(t,x + \frac{h_{x}}{2},y \right)}}{h_{x}}\right) \operatorname{U_{x}}{\left(t,x + \frac{h_{x}}{2},y \right)} - \left(\frac{\operatorname{U_{x}}{\left(t,x + \frac{h_{x}}{2},y \right)}}{h_{y}} - \frac{\operatorname{U_{x}}{\left(t,x + \frac{h_{x}}{2},y - h_{y} \right)}}{h_{y}}\right) \operatorname{U_{y}}{\left(t,x,y + \frac{h_{y}}{2} \right)} + \frac{\operatorname{U_{x}}{\left(t,x + \frac{h_{x}}{2},y \right)}}{dt}\right)\\dt \left(- \left(\frac{\operatorname{U_{y}}{\left(t,x,y + \frac{h_{y}}{2} \right)}}{h_{x}} - \frac{\operatorname{U_{y}}{\left(t,x - h_{x},y + \frac{h_{y}}{2} \right)}}{h_{x}}\right) \operatorname{U_{x}}{\left(t,x + \frac{h_{x}}{2},y \right)} - \left(- \frac{\operatorname{U_{y}}{\left(t,x,y - \frac{h_{y}}{2} \right)}}{h_{y}} + \frac{\operatorname{U_{y}}{\left(t,x,y + \frac{h_{y}}{2} \right)}}{h_{y}}\right) \operatorname{U_{y}}{\left(t,x,y + \frac{h_{y}}{2} \right)} + \frac{\operatorname{U_{y}}{\left(t,x,y + \frac{h_{y}}{2} \right)}}{dt}\right)\end{matrix}\right]$

We finally run the operator

```python
#NBVAL_IGNORE_OUTPUT
op = Operator([update_U] + bc_u + bc_v)
op(time=nt, dt=dt)

# The result is indeed the expected one.
plot_field(U[0].data[0])
```

```python
from devito import norm
assert np.isclose(norm(u), norm(U[0]), rtol=1e-5, atol=0)
```


# Übungsblatt 2: Kleinste Quadrate

```python
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
import scipy.optimize
plt.style.use('ggplot')

from functools import partial
```

---
## Aufgabe 1
Betrachten Sie $y_k, k=1,2,\dots,10$ Poisson-verteilte Messwerte um die Erwartungswerte $\mu_k = a\cdot k$. Der Parameter $a$ soll mit der Methode der kleinsten Quadrate geschätzt werden. Betrachten Sie dazu die folgenden $\chi^2$-Funktionen:

$$\chi^2 = \sum_{k=1}^{10} w_k (y_k - ak)^2 \quad\text{mit}\quad w_k\in \left\{1, \frac{1}{y_k}, \frac{1}{ak}, \frac{1}{k}, \frac{1}{a_{n-1}k} \right\}$$

### 1.1
Bestimmen Sie analytisch die Schätzwerte $\hat{a}$ als Funktion der $y_k$, welche die entsprechenden $\chi^2$-Funktion minimieren.

---

Für den Fall $w_k = 1$ ist
\begin{align}
    \chi^2 &= \sum_k\left(y_k - a k\right)^2 \\
           &= \sum_k y_k^2 - 2 y_k a k + a^2 k^2 \\
    \frac{\partial\chi^2}{\partial a} &= -2\sum_k y_k k + 2\sum_k a k^2 \overset{!}{=} 0 \\
    &\Rightarrow a \sum_k k^2 = \sum_k y_k k \\
    &\Rightarrow a = \frac{\sum_k y_k k}{\sum_k k^2}
\end{align}

Für den Fall $w_k = 1 / y_k$ ist
\begin{align}
    \chi^2 &= \sum_k \frac{y_k^2 - 2 y_k a k + a^2 k^2}{y_k} \\
           &= \sum_k y_k - 2 a k + \frac{a^2 k^2}{y_k} \\
    \frac{\partial\chi^2}{\partial a} &= -2\sum_k k + 2a\sum_k \frac{k^2}{y_k} \overset{!}{=} 0 \\
    &\Rightarrow a \sum_k \frac{k^2}{y_k} = \sum_k k \\
    &\Rightarrow a = \frac{\sum_k k}{\sum_k \frac{k^2}{y_k}}
\end{align}

Für den Fall $w_k = 1 / ak$ ist
\begin{align}
    \chi^2 &= \sum_k \frac{y_k^2 - 2 y_k a k + a^2 k^2}{ak} \\
           &= \sum_k \frac{y_k^2}{ak} - 2y_k + ak \\
    \frac{\partial\chi^2}{\partial a} &= -\frac{1}{a^2}\sum_k \frac{y_k^2}{k} + \sum_k k \overset{!}{=} 0 \\
    &\Rightarrow a^2 = \frac{\sum_k \frac{y_k^2}{k}}{\sum_k k} \\
    &\Rightarrow a = \sqrt{\frac{\sum_k \frac{y_k^2}{k}}{\sum_k k}}
\end{align}

Für den Fall $w_k = 1 / k$ ist
\begin{align}
    \chi^2 &= \sum_k \frac{y_k^2 - 2 y_k a k + a^2 k^2}{k} \\
           &= \sum_k \frac{y_k^2}{k} - 2ay_k + a^2k \\
    \frac{\partial\chi^2}{\partial a} &= -2\sum_k y_k + 2a\sum_k k \overset{!}{=} 0 \\
    &\Rightarrow a\sum_k k = \sum_k y_k \\
    &\Rightarrow a = \frac{\sum_k y_k}{\sum_k k}
\end{align}

Für den Fall $w_k = 1 / a_{n-1} k$ ist
\begin{align}
    \chi^2 &= \sum_k \frac{y_k^2 - 2 y_k a k + a^2 k^2}{a_{n-1}k} \\
           &= \sum_k \frac{y_k^2}{a_{n-1}k} - \frac{2ay_k}{a_{n-1}} + \frac{a^2k}{a_{n-1}} \\
    \frac{\partial\chi^2}{\partial a} &= -\frac{2}{a_{n-1}}\sum_k y_k + 2\frac{a}{a_{n-1}}\sum_k k \overset{!}{=} 0 \\
    &\Rightarrow \frac{a}{a_{n-1}}\sum_k k = \frac{1}{a_{n-1}}\sum_k y_k \\
    &\Rightarrow a = \frac{\sum_k y_k}{\sum_k k}
\end{align}

Das bedeutet also, dass die Minimierung in diesem Fall im ersten Schritt konvergiert und eine Iteration keine Verbesserung liefert.

---
### 1.2
Bestimmen Sie mittels Monte Carlo die Verteilungen der verschiedenen Schätzwerte $\hat{a}$.

---

```python
def generate_sample(a, n=10, size=10000):
    ks = np.arange(1, n + 1)
    yss = np.random.poisson(a * ks, size=(size, n))
    return ks, yss

a = 4.2
ks, yss = generate_sample(a)

estimators = [
    ('1', lambda xs, ys: np.sum(ys * xs) / np.sum(xs ** 2)),
    ('1/y_k', lambda xs, ys: np.sum(xs) / np.sum(xs ** 2 / ys)),
    ('1/ak', lambda xs, ys: np.sqrt(np.sum(ys ** 2 / xs) / np.sum(xs))),
    ('1/k', lambda xs, ys: np.sum(ys) / np.sum(xs)),
    ('1/a_{n-1}k', lambda xs, ys: np.sum(ys) / np.sum(xs))
]

for w, wf in estimators:
    as_ = np.array([wf(ks, ys) for ys in yss])
    mean = np.mean(as_)
    var = np.var(as_)
    plt.hist(as_, bins=20)
    plt.title(r'$w_k = {}$, $\mu = {:.3f}$, $\sigma^2 = {:.3f}$'.format(w, mean, var))
    plt.show()
```

---
## Aufgabe 2
Betrachten Sie ein Histogramm mit 10 Bins im Bereich $[0,1]$. Generieren Sie eine Poisson-verteilte Zufallszahl $N$ mit Mittelwert $\langle N\rangle = 400$, und füllen Sie das Histogramm mit $N$ Zufallszahlen die Sie gemäß

$$x = \frac{1}{2}\left(\sqrt{1 + 8z} - 1\right)$$

erzeugen, wobei $z$ gleichverteilt im Intervall $[0, 1]$ ist. Die Theorie sagt voraus, dass $x$ gemäß einer Wahrscheinlichkeitsdichte

$$ \rho(x) = \frac{1 + \alpha x}{1 + \alpha/2}$$

verteilt ist.

---

```python
def transform_x(z):
    return 1 / 2 * (np.sqrt(1 + 8 * z) - 1)

N = np.random.poisson(400)

xs = transform_x(np.random.uniform(size=N))
ns, bins, _ = plt.hist(xs, range=(0, 1), bins=10)
plt.show()
```

---
Bestimmen sie mit der Methode der kleinsten Quadrate einen Schätzwert für $\alpha$. Nehmen Sie dazu an, dass die Anzahl der Einträge in jedem Bin Poisson-verteilt um ihren Erwartungswert streut. Beschreiben Sie die Daten durch eine lineare Funktion $y = a_0 + a1_x$, bestimmen Sie $a_0$ und $a_1$ und deren Kovarianzmatrix, und berechnen Sie daraus $\alpha(a_0, a_1)$ sowie dessen Unsicherheit. Gehen Sie beim Fit folgendermassen vor:

* Berechnen Sie die erwartete Zahl von Einträgen für alle Bins.
* Benutzen Sie diese zur Formulierung der $\chi^2$-Funktion.
* Minimieren Sie diese iterativ. Verwenden Sie im ersten Schritt für alle Bins konstante Gewichte $w_k = 1$, und danach als Gewicht $w_k = 1/V_k$, wobei $V_k$ die mit den Parametern aus dem vorhergehenden Fit abgeschätzte Varianz im Bin $k$ ist.

---

Mit Hilfe der PDF können wir die Anzahl erwarteter Einträge $T_k$ im $k$-ten Bin berechnen. Es ist

\begin{equation}
    T_k(\alpha) = N \int_{x_k}^{x_{k+1}} \mathrm{d}x\, \rho(x) = \frac{N}{1 + \alpha/2}\bigl( x_{k+1} - x_{k} + \frac{1}{2}\alpha(x_{k+1}^2 - x_k^2) \bigr) \,.
\end{equation}

Damit können wir unser $\chi^2$ formulieren.

\begin{equation}
    \chi^2(\alpha) = \sum_k w_k \left(n_k - T_k(\alpha)\right)^2
\end{equation}

wobei $n_k$ die beobachtete Anzahl Einträge im $k$-ten Bin ist. Wir könnten jetzt hergehen und die Funktion händisch ableiten, um unseren Schätzer zu erhalten. Um uns das Leben etwas zu vereinfachen verwenden wir die `minimize`-Funktion aus `scipy.optimize`.

```python
def expected_ns(N, bins, alpha):
    widths = bins[1:] - bins[:-1]
    sqdiff = bins[1:] ** 2 - bins[:-1] ** 2
    return N / (1 + alpha / 2) * (widths + alpha / 2 * sqdiff)

tk = partial(expected_ns, N, bins)

def chi2(N, bins, ns, ws, alpha):
    ts = expected_ns(N, bins, alpha)
    return np.sum(ws * (ns - ts) ** 2)

c2 = partial(chi2, N, bins, ns, 1)

alpha = scipy.optimize.minimize(c2, [1]).x[0]
print(alpha)

for _ in range(5):
    ws = 1 / expected_ns(N, bins, alpha)
    c2 = partial(chi2, N, bins, ns, ws)
    alpha = scipy.optimize.minimize(c2, [alpha]).x[0]
    print(alpha)
```

    2.59531028316
    2.15357731606
    2.18635905531
    2.18368340332
    2.18390013228
    2.18388260744

```python
widths = bins[1:] - bins[:-1]
centres = bins[:-1] + widths / 2
plt.errorbar(centres, ns, xerr=widths / 2, yerr=np.sqrt(ns), fmt='.')

def rho(x):
    return (1 + alpha * x) / (1 + alpha / 2)

xs_ = np.linspace(0, 1, 2)
plt.plot(xs_, N * widths[0] * rho(xs_))
plt.show()
```

---
Wiederholen Sie die Fits mit unabhängigen Datensätzen und bestimmen Sie die Verteilung von $\alpha$. Variieren Sie die Methode, indem Sie den Fit nicht iterieren oder andere Gewichtsfunktionen benutzen.

---

```python
def fit_alpha(N, xs, iterations=5):
    ns, bins = np.histogram(xs, range=(0, 1), bins=10)
    c2 = partial(chi2, N, bins, ns, 1)

    alpha = scipy.optimize.minimize(c2, [1]).x[0]

    for _ in range(iterations):
        ws = 1 / expected_ns(N, bins, alpha)
        c2 = partial(chi2, N, bins, ns, ws)
        alpha = scipy.optimize.minimize(c2, [alpha]).x[0]

    return alpha

def generate_sample(mean_N=400):
    N = np.random.poisson(mean_N)
    xs = transform_x(np.random.uniform(size=N))
    return N, xs

alphas = np.array([fit_alpha(*generate_sample()) for _ in range(100)])
plt.hist(alphas)
plt.show()
print('mean = {}'.format(np.mean(alphas)))
print('variance = {}'.format(np.var(alphas)))

alphas = alphas[np.logical_and(-5 < alphas, alphas < 5)]
plt.hist(alphas)
plt.show()
print('mean = {}'.format(np.mean(alphas)))
print('variance = {}'.format(np.var(alphas)))
```

### Alternative Lösung ohne Verwendung von `scipy.optimize`

Wir können unser $T_k$ auch als lineares Modell zweier Parameter $a_0$ und $a_1$ ausdrücken, indem wir zwei Variablen $p_k$ und $q_k$ einführen die definiert sind als

\begin{align}
    p_k &= x_{k + 1} - x_k \,\text{und} \\
    q_k &= \frac{1}{2}(x_{k + 1}^2 - x_k^2) \,.
\end{align}

Damit ist

\begin{equation}
    T_k(a_0, a_1) = a_0 p_k + a_1 q_k
\end{equation}

Damit können wir zur Minimierung des $\chi^2$ die analytische Lösung für lineare Modelle verwenden. Als Designmatrix $X$ verwenden wir

\begin{equation}
    X = \pmatrix{p_1 & q_1 \\
                 \vdots & \vdots \\
                 p_k & q_k}
\end{equation}

und erhalten

\begin{equation}
    \mathbf{\hat{a}} = (X^T W X)^{-1} X^T W y
\end{equation}

wobei $W$ die Gewichtsmatrix ist. Um $\hat{a}$ iterativ zu berechnen, verwenden wir

\begin{equation}
    \mathbf{\hat{a}^{(n+1)}} = (X^T W^{(n)} X)^{-1} X^T W^{(n)} y
\end{equation}

wobei

\begin{equation}
    W^{(n)} = \frac{1}{{\hat{a}_0^{(n)}p_k + \hat{a}_1^{(n)}q_k}} \mathbb{1} \,.
\end{equation}

```python
def prepare_xs(xs):
    y, bins = np.histogram(xs)
    ps = bins[1:] - bins[:-1]
    qs = (bins[1:] ** 2 - bins[:-1] ** 2) / 2
    X = np.matrix([ps, qs]).T
    return np.matrix(y).T, X

def regression(y, X, W=None):
    if W is None:
        W = np.identity(X.shape[0])
    a = (X.T * W * X).I * X.T * W * y
    W = (np.diag(np.array(1 / (X * a))[:,0]))
    return a, W

N, xs = generate_sample()
y, X = prepare_xs(xs)
I = np.identity(np.shape(X)[0])

a, W = regression(y, X)
print(a)
for _ in range(10):
    a, W = regression(y, X, W)
    print(a)
```

    [[ 192.02643213]
     [ 401.39193664]]
    [[ 187.87434483]
     [ 409.69780569]]
    [[ 187.75813701]
     [ 409.93026876]]
    [[ 187.75485754]
     [ 409.93682905]]
    [[ 187.75476497]
     [ 409.93701423]]
    [[ 187.75476235]
     [ 409.93701945]]
    [[ 187.75476228]
     [ 409.9370196 ]]
    [[ 187.75476228]
     [ 409.9370196 ]]
    [[ 187.75476228]
     [ 409.9370196 ]]
    [[ 187.75476228]
     [ 409.9370196 ]]
    [[ 187.75476228]
     [ 409.9370196 ]]

Jetzt müssen wir nur noch $\alpha$ aus $a_0$ und $a_1$ bestimmen.

```python
a0, a1 = a

alpha = a1 / a0
alpha
```

    matrix([[ 2.18336416]])


## Perceptron algorithm in plain Python

The perceptron is a simple supervised machine learning algorithm and one of the earliest **neural network** architectures. It was introduced by Rosenblatt in the late 1950s. A perceptron represents a **binary linear classifier** that maps a set of training examples (of $d$ dimensional input vectors) onto binary output values using a $d-1$ dimensional hyperplane.

The perceptron as follows.

**Given:**
- dataset $\{(\boldsymbol{x}^{(1)}, y^{(1)}), ..., (\boldsymbol{x}^{(m)}, y^{(m)})\}$
- with $\boldsymbol{x}^{(i)}$ being a $d-$dimensional vector $\boldsymbol{x}^i = (x^{(i)}_1, ..., x^{(i)}_d)$
- $y^{(i)}$ being a binary target variable, $y^{(i)} \in \{0,1\}$

The perceptron is a very simple neural network:
- it has a real-valued weight vector $\boldsymbol{w}= (w^{(1)}, ..., w^{(d)})$
- it has a real-valued bias $b$
- it uses the Heaviside step function as its activation function

* * *
A perceptron is trained using **gradient descent**. The training algorithm has different steps. In the beginning (step 0) the model parameters are initialized. The other steps (see below) are repeated for a specified number of training iterations or until the parameters have converged.

**Step 0: ** Initialize the weight vector and bias with zeros (or small random values).
* * *

**Step 1: ** Compute a linear combination of the input features and weights. This can be done in one step for all training examples, using vectorization and broadcasting:
$\boldsymbol{a} = \boldsymbol{X} \cdot \boldsymbol{w} + b$

where $\boldsymbol{X}$ is a matrix of shape $(n_{samples}, n_{features})$ that holds all training examples, and $\cdot$ denotes the dot product.
* * *

**Step 2: ** Apply the Heaviside function, which returns binary values:

$\hat{y}^{(i)} = 1 \, if \, a^{(i)} \geq 0, \, else \, 0$
* * *

** Step 3: ** Compute the weight updates using the perceptron learning rule

\begin{equation}
\Delta \boldsymbol{w} = \eta \, \boldsymbol{X}^T \cdot \big(\boldsymbol{\hat{y}} - \boldsymbol{y} \big)
\end{equation}
$$ \Delta b = \eta \, \big(\boldsymbol{\hat{y}} - \boldsymbol{y} \big) $$

where $\eta$ is the learning rate.
* * *

** Step 4: ** Update the weights and bias

\begin{equation}
\boldsymbol{w} = \boldsymbol{w} + \Delta \boldsymbol{w}
\end{equation}

$$
b = b  + \Delta b
$$

```python
import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_blobs
from sklearn.model_selection import train_test_split
np.random.seed(123)

% matplotlib inline
```

## Dataset

```python
X, y = make_blobs(n_samples=1000, centers=2)
fig = plt.figure(figsize=(8,6))
plt.scatter(X[:,0], X[:,1], c=y)
plt.title("Dataset")
plt.xlabel("First feature")
plt.ylabel("Second feature")
plt.show()
```

```python
y_true = y[:, np.newaxis]

X_train, X_test, y_train, y_test = train_test_split(X, y_true)
print(f'Shape X_train: {X_train.shape}')
print(f'Shape y_train: {y_train.shape})')
print(f'Shape X_test: {X_test.shape}')
print(f'Shape y_test: {y_test.shape}')
```

    Shape X_train: (750, 2)
    Shape y_train: (750, 1))
    Shape X_test: (250, 2)
    Shape y_test: (250, 1)

## Perceptron model class

```python
class Perceptron():

    def __init__(self):
        pass

    def train(self, X, y, learning_rate=0.05, n_iters=100):
        n_samples, n_features = X.shape

        # Step 0: Initialize the parameters
        self.weights = np.zeros((n_features,1))
        self.bias = 0

        for i in range(n_iters):
            # Step 1: Compute the activation
            a = np.dot(X, self.weights) + self.bias

            # Step 2: Compute the output
            y_predict = self.step_function(a)

            # Step 3: Compute weight updates
            delta_w = learning_rate * np.dot(X.T, (y - y_predict))
            delta_b = learning_rate * np.sum(y - y_predict)

            # Step 4: Update the parameters
            self.weights += delta_w
            self.bias += delta_b

        return self.weights, self.bias

    def step_function(self, x):
        return np.array([1 if elem >= 0 else 0 for elem in x])[:, np.newaxis]

    def predict(self, X):
        a = np.dot(X, self.weights) + self.bias
        return self.step_function(a)
```

## Initialization and training the model

```python
p = Perceptron()
w_trained, b_trained = p.train(X_train, y_train,learning_rate=0.05, n_iters=500)
```

## Testing

```python
y_p_train = p.predict(X_train)
y_p_test = p.predict(X_test)

print(f"training accuracy: {100 - np.mean(np.abs(y_p_train - y_train)) * 100}%")
print(f"test accuracy: {100 - np.mean(np.abs(y_p_test - y_test)) * 100}%")
```

    training accuracy: 100.0%
    test accuracy: 100.0%

## Visualize decision boundary

```python
def plot_hyperplane(X, y, weights, bias):
    """
    Plots the dataset and the estimated decision hyperplane
    """
    slope = - weights[0]/weights[1]
    intercept = - bias/weights[1]
    x_hyperplane = np.linspace(-10,10,10)
    y_hyperplane = slope * x_hyperplane + intercept
    fig = plt.figure(figsize=(8,6))
    plt.scatter(X[:,0], X[:,1], c=y)
    plt.plot(x_hyperplane, y_hyperplane, '-')
    plt.title("Dataset and fitted decision hyperplane")
    plt.xlabel("First feature")
    plt.ylabel("Second feature")
    plt.show()
```

```python
plot_hyperplane(X, y, w_trained, b_trained)
```


# 01 - Introduction to seismic modelling

This notebook is the first in a series of tutorial highlighting various aspects of seismic inversion based on Devito operators. In this first example we aim to highlight the core ideas behind seismic modelling, where we create a numerical model that captures the processes involved in a seismic survey. This forward model will then form the basis for further tutorials on the implementation of inversion processes using Devito operators.

## Modelling workflow

The core process we are aiming to model is a seismic survey, which consists of two main components:

- **Source** - A source is positioned at a single or a few physical location where artificial pressure is injected into the domain we want to model. In the case of land survey, it is usually dynamite blowing up at a given location, or a vibroseis (a vibrating engine generating continuous  sound waves). For a marine survey, the source is an air gun sending a bubble of compressed air into the water that will expand and generate a seismic wave.
- **Receiver** - A set set of microphones or hydrophones are used to measure the resulting wave and create a set of measurements called a *Shot Record*. These measurements are recoded at multiple locations, and usually at the surface of the domain or at the bottom of the ocean in some marine cases.

In order to create a numerical model of a seismic survey, we need to solve the wave equation and implement source and receiver interpolation to inject the  source and record the seismic wave at sparse point locations in the grid.

## The acoustic seismic wave equation
The acoustic wave equation for the square slowness $m$ defined as $m=\frac{1}{c^2}$, where $c$ is the speed of sound in the given physical media, and a source $q$ is given by:

\begin{cases}
 &m \frac{d^2 u(x,t)}{dt^2} - \nabla^2 u(x,t) = q \ \text{in } \Omega \\
 &u(.,t=0) = 0 \\
 &\frac{d u(x,t)}{dt}|_{t=0} = 0
\end{cases}

with the zero initial conditions to guarantee unicity of the solution.
The boundary conditions are Dirichlet conditions :
\begin{equation}
 u(x,t)|_\delta\Omega = 0
\end{equation}

where $\delta\Omega$ is the surface of the boundary of the model $\Omega$.

# Finite domains

The last piece of the puzzle is the computational limitation. In the field, the seismic wave propagates in every directions to an "infinite" distance. However, solving the wave equation in a mathematically/discrete infinite domain is not feasible. In order to compensate, Absorbing Boundary Conditions (ABC) or Perfectly Matched Layers (PML) are required to mimic an infinite domain. These two method allow to approximate an infinite media by damping and absorbing the waves at the limit of the domain to avoid reflections.

The simplest of these methods is the absorbing damping mask. The core idea is to extend the physical domain and to add a Sponge mask in this extension that will absorb the incident waves. The acoustic wave equation with this damping mask can be rewritten as:

\begin{cases}
 &m \frac{d^2 u(x,t)}{dt^2} - \nabla^2 u(x,t) + \eta \frac{d u(x,t)}{dt}=q  \ \text{in } \Omega \\
 &u(.,0) = 0 \\
 &\frac{d u(x,t)}{dt}|_{t=0} = 0
\end{cases}

where $\eta$ is the damping mask equal to $0$ inside the physical domain and increasing inside the sponge layer. Multiple choice of profile can be chosen for $\eta$ from linear to exponential.

# Seismic modelling with devito

We describe here a step by step setup of seismic modelling with Devito in a simple 2D case. We will create a physical model of our domain and define a single source and an according set of receivers to model for the forward model. But first, we initialize some basic utilities.

```python
import numpy as np
%matplotlib inline
```

## Define the physical problem

The first step is to define the physical model:

- What are the physical dimension of interest
- What is the velocity profile of this physical domain

We will create a simple velocity model here by hand for demonstration purposes. This model essentially consists of two layers, each with a different velocity: $1.5km/s$ in the top layer and $2.5km/s$ in the bottom layer. We will use this simple model a lot in the following tutorials, so we will rely on a utility function to create it again later.

```python
#NBVAL_IGNORE_OUTPUT
from examples.seismic import Model, plot_velocity

# Define a physical size
shape = (101, 101)  # Number of grid point (nx, nz)
spacing = (10., 10.)  # Grid spacing in m. The domain size is now 1km by 1km
origin = (0., 0.)  # What is the location of the top left corner. This is necessary to define
# the absolute location of the source and receivers

# Define a velocity profile. The velocity is in km/s
v = np.empty(shape, dtype=np.float32)
v[:, :51] = 1.5
v[:, 51:] = 2.5

# With the velocity and model size defined, we can create the seismic model that
# encapsulate this properties. We also define the size of the absorbing layer as 10 grid points
model = Model(vp=v, origin=origin, shape=shape, spacing=spacing,
              space_order=2, nbpml=10)

plot_velocity(model)
```

# Acquisition geometry

To fully define our problem setup we also need to define the source that injects the wave to model and the set of receiver locations at which to sample the wavefield. The source time signature will be modelled using a Ricker wavelet defined as

\begin{equation}
  q(t) = (1-2\pi^2 f_0^2 (t - \frac{1}{f_0})^2 )e^{- \pi^2 f_0^2 (t - \frac{1}{f_0})}
\end{equation}

To fully define the source signature we first need to define the time duration for our model and the timestep size, which is dictated by the CFL condition and our grid spacing. Luckily, our `Model` utility provides us with the critical timestep size, so we can fully discretize our model time axis as an array:

```python
from examples.seismic import TimeAxis

t0 = 0.  # Simulation starts a t=0
tn = 1000.  # Simulation last 1 second (1000 ms)
dt = model.critical_dt  # Time step from model grid spacing

time_range = TimeAxis(start=t0, stop=tn, step=dt)
```

The source is positioned at a $20m$ depth and at the middle of the $x$ axis ($x_{src}=500m$), with a peak wavelet frequency of $10Hz$.

```python
#NBVAL_IGNORE_OUTPUT
from examples.seismic import RickerSource

f0 = 0.010  # Source peak frequency is 10Hz (0.010 kHz)
src = RickerSource(name='src', grid=model.grid, f0=f0,
                   npoint=1, time_range=time_range)

# First, position source centrally in all dimensions, then set depth
src.coordinates.data[0, :] = np.array(model.domain_size) * .5
src.coordinates.data[0, -1] = 20.  # Depth is 20m

# We can plot the time signature to see the wavelet
src.show()
```

Similarly to our source object, we can now define our receiver geometry as a symbol of type `Receiver`. It is worth noting here that both utility classes, `RickerSource` and `Receiver` are thin wrappers around Devito's `PointData` type, which encapsulates sparse point data and allows us to inject and interpolate values into and out of the computational grid. As we have already seen, both types provide a `point.coordinates` property to define the position within the domain of all points encapsulated by that symbol.

In this example we will position receivers at the same depth as the source, every $10m$ along the x axis. The `rec.data` property will be initialized, but left empty, as we will compute the receiver readings during the simulation.

```python
#NBVAL_IGNORE_OUTPUT
from examples.seismic import Receiver

# Create symbol for 101 receivers
rec = Receiver(name='rec', grid=model.grid, npoint=101, time_range=time_range)

# Prescribe even spacing for receivers along the x-axis
rec.coordinates.data[:, 0] = np.linspace(0, model.domain_size[0], num=101)
rec.coordinates.data[:, 1] = 20.  # Depth is 20m

# We can now show the source and receivers within our domain:
# Red dot: Source location
# Green dots: Receiver locations (every 4th point)
plot_velocity(model, source=src.coordinates.data,
              receiver=rec.coordinates.data[::4, :])
```

# Finite-difference discretization

Devito is a finite-difference DSL that solves the discretized wave-equation on a Cartesian grid. The finite-difference approximation is derived from Taylor expansions of the continuous field after removing the error term.

## Time discretization

We only consider the second order time discretization for now. From the Taylor expansion, the second order discrete approximation of the second order time derivative is:
\begin{equation}
\begin{aligned}
 \frac{d^2 u(x,t)}{dt^2} = \frac{\mathbf{u}(\mathbf{x},\mathbf{t+\Delta t}) - 2 \mathbf{u}(\mathbf{x},\mathbf{t}) + \mathbf{u}(\mathbf{x},\mathbf{t-\Delta t})}{\mathbf{\Delta t}^2} + O(\mathbf{\Delta t}^2).
\end{aligned}
\end{equation}

where $\mathbf{u}$ is the discrete wavefield, $\mathbf{\Delta t}$ is the discrete
time-step (distance between two consecutive discrete times) and $O(\mathbf{\Delta
  t}^2)$ is the discretization error term. The discretized approximation of the
second order time derivative is then given by dropping the error term. This derivative is represented in Devito by `u.dt2` where u is a `TimeFunction` object.

## Spatial discretization

We define the discrete Laplacian as the sum of the second order spatial
derivatives in the three dimensions:
\begin{equation}
\begin{aligned}
\Delta \mathbf{u}(\mathbf{x},\mathbf{y},\mathbf{z},\mathbf{t})= \sum_{j=1}^{j=\frac{k}{2}} \Bigg[\alpha_j \Bigg(&
\mathbf{u}(\mathbf{x+jdx},\mathbf{y},\mathbf{z},\mathbf{t})+\mathbf{u}(\mathbf{x-jdx},\mathbf{y},\mathbf{z},\mathbf{t}) + \\
&\mathbf{u}(\mathbf{x},\mathbf{y+jdy},\mathbf{z},\mathbf{t})+\mathbf{u}(\mathbf{x},\mathbf{y-jdy},\mathbf{z}\mathbf{t}) + \\
&\mathbf{u}(\mathbf{x},\mathbf{y},\mathbf{z+jdz},\mathbf{t})+\mathbf{u}(\mathbf{x},\mathbf{y},\mathbf{z-jdz},\mathbf{t})\Bigg) \Bigg] + \\
&3\alpha_0 \mathbf{u}(\mathbf{x},\mathbf{y},\mathbf{z},\mathbf{t}).
\end{aligned}
\end{equation}

This derivative is represented in Devito by `u.laplace` where u is a `TimeFunction` object.

## Wave equation

With the space and time discretization defined, we can fully discretize the wave-equation with the combination of time and space discretizations and obtain the following second order in time and $k^{th}$ order in space discrete stencil to update one grid point at position $\mathbf{x}, \mathbf{y},\mathbf{z}$ at time $\mathbf{t}$, i.e.
\begin{equation}
\begin{aligned}
\mathbf{u}(\mathbf{x},\mathbf{y},\mathbf{z},\mathbf{t+\Delta t}) = &2\mathbf{u}(\mathbf{x},\mathbf{y},\mathbf{z},\mathbf{t}) - \mathbf{u}(\mathbf{x},\mathbf{y}, \mathbf{z},\mathbf{t-\Delta t}) +\\
& \frac{\mathbf{\Delta t}^2}{\mathbf{m(\mathbf{x},\mathbf{y},\mathbf{z})}} \Big(\Delta \mathbf{u}(\mathbf{x},\mathbf{y},\mathbf{z},\mathbf{t}) + \mathbf{q}(\mathbf{x},\mathbf{y},\mathbf{z},\mathbf{t}) \Big).
\end{aligned}
\end{equation}

```python
# In order to represent the wavefield u and the square slowness we need symbolic object
# corresponding to time-space-varying field (u, TimeFunction) and
# space-varying field (m, Function)
from devito import TimeFunction

# Define the wavefield with the size of the model and the time dimension
u = TimeFunction(name="u", grid=model.grid, time_order=2, space_order=2)

# We can now write the PDE
pde = model.m * u.dt2 - u.laplace + model.damp * u.dt

# This discrete PDE can be solved in a time-marching way updating u(t+dt) from the previous time step
# Devito as a shortcut for u(t+dt) which is u.forward. We can then rewrite the PDE as
# a time marching updating equation known as a stencil using customized SymPy functions
from devito import Eq, solve

stencil = Eq(u.forward, solve(pde, u.forward))
```

# Source injection and receiver interpolation

With a numerical scheme to solve the homogenous wave equation, we need to add the source to introduce seismic waves and to implement the measurement operator, and interpolation operator. This operation are linked to the discrete scheme and need to be done at the proper time step. The semi-discretized in time wave equation with a source reads:

\begin{equation}
\begin{aligned}
\mathbf{u}(\mathbf{x},\mathbf{y},\mathbf{z},\mathbf{t+\Delta t}) = &2\mathbf{u}(\mathbf{x},\mathbf{y},\mathbf{z},\mathbf{t}) - \mathbf{u}(\mathbf{x},\mathbf{y}, \mathbf{z},\mathbf{t-\Delta t}) +\\
& \frac{\mathbf{\Delta t}^2}{\mathbf{m(\mathbf{x},\mathbf{y},\mathbf{z})}} \Big(\Delta \mathbf{u}(\mathbf{x},\mathbf{y},\mathbf{z},\mathbf{t}) + \mathbf{q}(\mathbf{x},\mathbf{y},\mathbf{z},\mathbf{t}) \Big).
\end{aligned}
\end{equation}

It shows that in order to update $\mathbf{u}$ at time $\mathbf{t+\Delta t}$ we have to inject the value of the source term $\mathbf{q}$ of time $\mathbf{t}$. In Devito, it correspond the update of $u$ at index $t+1$ (t = time implicitly) with the source of time $t$.
On the receiver side, the problem is either as it only require to record the data at the given time step $t$ for the in the receiver at time $time=t$.

```python
# Finally we define the source injection and receiver read function to generate the corresponding code
src_term = src.inject(field=u.forward, expr=src * dt**2 / model.m, offset=model.nbpml)

# Create interpolation expression for receivers
rec_term = rec.interpolate(expr=u.forward, offset=model.nbpml)
```

# Devito operator and solve
After constructing all the necessary expressions for updating the wavefield, injecting the source term and interpolating onto the receiver points, we can now create the Devito operator that will generate the C code at runtime. When creating the operator, Devito's two optimization engines will log which performance optimizations have been performed:
* **DSE:** The Devito Symbolics Engine will attempt to reduce the number of operations required by the kernel.
* **DLE:** The Devito Loop Engine will perform various loop-level optimizations to improve runtime performance.

**Note**: The argument `subs=model.spacing_map` causes the operator to substitute values for our current grid spacing into the expressions before code generation. This reduces the number of floating point operations executed by the kernel by pre-evaluating certain coeffecients.

```python
#NBVAL_IGNORE_OUTPUT
from devito import Operator

op = Operator([stencil] + src_term + rec_term, subs=model.spacing_map)
```

Now we can execute the create operator for a number of timesteps. We specify the number of timesteps to compute with the keyword `time` and the timestep size with `dt`.

```python
#NBVAL_IGNORE_OUTPUT
op(time=time_range.num-1, dt=model.critical_dt)
```

    CustomCompiler: compiled /var/folders/qr/4hc1nw7d3hj55jbd1z7kkrvh0000gn/T/devito-4z7_m1bt/f192cfb88f64adbf6d0bda9f31b8aa78d3ba2f0d.c [0.48 s]
    =========================================================================================
    Section section_1<596,1> with OI=0.84 computed in 0.000 s [0.35 GFlops/s]
    Section section_2<596,101> with OI=1.62 computed in 0.001 s [2.33 GFlops/s]
    Section main<596,121,121> with OI=1.88 computed in 0.020 s [12.99 GFlops/s, 0.43 GPts/s]
    =========================================================================================

After running our operator kernel the data associated with the receiver symbol `rec.data` has now been populated due to the interpolation expression we inserted into the operator. This allows us the visualize the shot record:

```python
#NBVAL_IGNORE_OUTPUT
from examples.seismic import plot_shotrecord

plot_shotrecord(rec.data, model, t0, tn)
```

```python
assert np.isclose(np.linalg.norm(rec.data), 450, rtol=10)
```


# the German tank problem

the Germans have a population of $n$ tanks labeled with serial numbers $1,2,...,n$ on the tanks. The number of tanks $n$ is unknown and of interest to the Allied forces. The Allied forces randomly capture $k$ tanks from the Germans with replacement and observe their serial numbers $\{x_1, x_2, ..., x_k\}$. The goal is to, from observing the serial numbers on this random sample of the tanks, estimate $n$.

the *estimator* of $n$ maps an outcome of the experiment to an estimate of $n$, $\hat{n}$.

```julia
using StatsBase
using PyPlot
using Statistics
using Printf

PyPlot.matplotlib.style.use("seaborn-pastel")

rcParams = PyPlot.PyDict(PyPlot.matplotlib."rcParams")
rcParams["font.size"] = 16;
```

## data structure for a tank

for elegance

```julia
struct Tank
    serial_no::Int
end
tank = Tank(3)
```

    Tank(3)

## visualizing the captured tanks and their serial numbers

```julia
function viz_tanks(tanks::Array{Tank}, savename=nothing)
    nb_tanks = length(tanks)

    img = PyPlot.matplotlib.image.imread("tank.png")

    fig, ax = subplots(1, nb_tanks)
    for (t, tank) in enumerate(tanks)
        ax[t].imshow(img)
        ax[t].set_title(tank.serial_no)
        ax[t].axis("off")
    end
    tight_layout()

    if ! isnothing(savename)
        savefig(savename * ".png", format="png", dpi=300)
        # Linux command line tool to trim white space
        run(`convert $savename.png -trim $savename.png`)
    end
end

n = 7
tanks = [Tank(s) for s in 1:n]
viz_tanks(tanks)
```

## simulating tank capture

write a function `capture_tanks` to simulate the random sampling of `nb_tanks_captured` tanks from all `nb_tanks` tanks the Germans have (without replacement). return a random sample of tanks.

```julia
function capture_tanks(num_captured::Int, num_tanks::Int)
    return sample([Tank(i) for i in 1:num_tanks], num_captured, replace=false)
end

tanks = capture_tanks(4, 50)
viz_tanks(tanks)
```

## defining different estimators

an estimator maps an outcome $\{x_1, x_2, ..., x_k\}$ to an estimate for $n$, $\hat{n}$.

### estimator (1): maximum serial number

this is the maximum likelihood estimator.

\begin{equation}
\hat{n} = \max_i x_i
\end{equation}

```julia
function max_serial_no(captured_tanks::Array{Tank})
    return maximum([t.serial_no for t in captured_tanks])
end

max_serial_no(tanks)
```

    44

### estimator (2): maximum serial number plus initial gap

\begin{equation}
\hat{n} = \max_i x_i + \bigl(\min_i x_i -1\bigr)
\end{equation}

```julia
function max_plus_first_gap(captured_tanks::Array{Tank})
    # only need to store this array once
    serials = [t.serial_no for t in captured_tanks]
    return maximum(serials) + minimum(serials) - 1
end

max_plus_first_gap(tanks)
```

    52

### estimator (3): maximum serial number plus gap if samples are evenly spaced

\begin{equation}
\hat{n} = \max_i x_i + \bigl( \max_i x_i / k -1 \bigr)
\end{equation}

```julia
function max_plus_even_gap(captured_tanks::Array{Tank})
    serials = [t.serial_no for t in captured_tanks]
    max_serial = maximum(serials)
    # operations are costly; don't do it this way
    # return maximum(serials) * (1 + 1 / length(serials)) - 1
    return max_serial + max_serial / length(serials) - 1
end

max_plus_even_gap(tanks)
```

    54.0

## assessing the bias and variance of different estimators

say the Germans have `nb_tanks` tanks, and we randomly capture `nb_tanks_captured`. what is the distribution of the estimators (over different outcomes of this random experiment), and how does the distribution compare to the true `nb_tanks`?

```julia
function sim_̂n(num_sims::Int, num_tanks::Int, num_tanks_captured::Int, estimator::Function)
    return [estimator(capture_tanks(num_tanks_captured, num_tanks)) for i in 1:num_sims]
end

num_tanks = 100
num_captured = 5
num_sims = 1000
mean(sim_̂n(num_sims, num_tanks, num_captured, max_serial_no))
```

    84.74

```julia
num_sims = 10000

fig, ax = subplots(1, 1, figsize=(18, 8))
estimators = [max_serial_no, max_plus_first_gap, max_plus_even_gap]
for e in estimators
    results = sim_̂n(num_sims, num_tanks, num_captured, e)
    est_name = replace(string(e), "_" => " ")
    ax.hist(results, label=est_name, alpha=0.7)
    println(est_name)
    println("     (̂n):", mean(results))
    println("  std(̂n):", std(results), "\n")
end
ax.axvline(num_tanks, color="k", ls="--", linewidth=3)
ax.legend()
```

```julia
print(max_plus_even_gap)
```

    max_plus_even_gap

notes:
* **efficiency**: small variance
* **consistency**: as number of samples in the estimate increases, the estimator converges
* **unbiasedness**: average estimator is correct

## what happens as we capture more and more tanks, i.e. increase $k$?

assess estimator (3).

```julia

```

## one-sided confidence interval

how confident are we that the Germans don't have *more* tanks?

significance level: $\alpha$

test statistic = estimator (3) = $\hat{n} = \max_i x_i + \bigl( \max_i x_i / k -1 \bigr)$

**null hypothesis**: the number of tanks is $n=n_0$<br>
**alternative hypothesis**: the number of tanks is less than $n_0$

we reject the null hypothesis (say, "the data does not support the null hypothesis") that the number of tanks is $n=n_0$ if the p-value is less than $\alpha$. the p-value is the probability that, if the null hypothesis is true, we get a test statistic equal to or smaller than we observed.

we want to find the highest $n_0$ such that we have statistical power to reject the null hypothesis in favor of the alternative hypothesis. this is the upper bound on the confidence interval!

then the idea is that, if the null hypothesis is that the number of tanks is absurdly large compared to the largest serial number we saw in our sample, it would be very unlikely that we would see such small serial numbers compared to the number of tanks, so we'd reject the null hypothesis.

```julia

```

say $\alpha=0.05$ and we seek a 95% one-sided confidence interval

```julia

```

```julia

```

In-Class Notes/German Tank Problem/German tank problem_sparse.ipynb


# Librerias

```python
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
```

# Data Set

```python
# Loading pre-defined Boston Dataset
boston_dataset = datasets.load_boston()
print(boston_dataset.DESCR)
```

    .. _boston_dataset:

    Boston house prices dataset
    ---------------------------

    **Data Set Characteristics:**

        :Number of Instances: 506

        :Number of Attributes: 13 numeric/categorical predictive. Median Value (attribute 14) is usually the target.

        :Attribute Information (in order):
            - CRIM     per capita crime rate by town
            - ZN       proportion of residential land zoned for lots over 25,000 sq.ft.
            - INDUS    proportion of non-retail business acres per town
            - CHAS     Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)
            - NOX      nitric oxides concentration (parts per 10 million)
            - RM       average number of rooms per dwelling
            - AGE      proportion of owner-occupied units built prior to 1940
            - DIS      weighted distances to five Boston employment centres
            - RAD      index of accessibility to radial highways
            - TAX      full-value property-tax rate per $10,000
            - PTRATIO  pupil-teacher ratio by town
            - B        1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town
            - LSTAT    % lower status of the population
            - MEDV     Median value of owner-occupied homes in $1000's

        :Missing Attribute Values: None

        :Creator: Harrison, D. and Rubinfeld, D.L.

    This is a copy of UCI ML housing dataset.
    https://archive.ics.uci.edu/ml/machine-learning-databases/housing/

    This dataset was taken from the StatLib library which is maintained at Carnegie Mellon University.

    The Boston house-price data of Harrison, D. and Rubinfeld, D.L. 'Hedonic
    prices and the demand for clean air', J. Environ. Economics & Management,
    vol.5, 81-102, 1978.   Used in Belsley, Kuh & Welsch, 'Regression diagnostics
    ...', Wiley, 1980.   N.B. Various transformations are used in the table on
    pages 244-261 of the latter.

    The Boston house-price data has been used in many machine learning papers that address regression
    problems.

    .. topic:: References

       - Belsley, Kuh & Welsch, 'Regression diagnostics: Identifying Influential Data and Sources of Collinearity', Wiley, 1980. 244-261.
       - Quinlan,R. (1993). Combining Instance-Based and Model-Based Learning. In Proceedings on the Tenth International Conference of Machine Learning, 236-243, University of Massachusetts, Amherst. Morgan Kaufmann.

```python
df = pd.DataFrame(data = boston_dataset.data, columns=boston_dataset.feature_names)
df['House Price'] = boston_dataset.target
df.head()
```

      CRIM
      ZN
      INDUS
      CHAS
      NOX
      RM
      AGE
      DIS
      RAD
      TAX
      PTRATIO
      B
      LSTAT
      House Price

      0
      0.00632
      18.0
      2.31
      0.0
      0.538
      6.575
      65.2
      4.0900
      1.0
      296.0
      15.3
      396.90
      4.98
      24.0

      1
      0.02731
      0.0
      7.07
      0.0
      0.469
      6.421
      78.9
      4.9671
      2.0
      242.0
      17.8
      396.90
      9.14
      21.6

      2
      0.02729
      0.0
      7.07
      0.0
      0.469
      7.185
      61.1
      4.9671
      2.0
      242.0
      17.8
      392.83
      4.03
      34.7

      3
      0.03237
      0.0
      2.18
      0.0
      0.458
      6.998
      45.8
      6.0622
      3.0
      222.0
      18.7
      394.63
      2.94
      33.4

      4
      0.06905
      0.0
      2.18
      0.0
      0.458
      7.147
      54.2
      6.0622
      3.0
      222.0
      18.7
      396.90
      5.33
      36.2

```python
df.info()
```

    <class 'pandas.core.frame.DataFrame'>
    RangeIndex: 506 entries, 0 to 505
    Data columns (total 14 columns):
     #   Column       Non-Null Count  Dtype
    ---  ------       --------------  -----
     0   CRIM         506 non-null    float64
     1   ZN           506 non-null    float64
     2   INDUS        506 non-null    float64
     3   CHAS         506 non-null    float64
     4   NOX          506 non-null    float64
     5   RM           506 non-null    float64
     6   AGE          506 non-null    float64
     7   DIS          506 non-null    float64
     8   RAD          506 non-null    float64
     9   TAX          506 non-null    float64
     10  PTRATIO      506 non-null    float64
     11  B            506 non-null    float64
     12  LSTAT        506 non-null    float64
     13  House Price  506 non-null    float64
    dtypes: float64(14)
    memory usage: 55.5 KB

```python
# Generate scatter plot of independent vs Dependent variable
plt.style.use('ggplot')
fig = plt.figure(figsize = (18, 18))

for index, feature_name in enumerate(boston_dataset.feature_names):
    ax = fig.add_subplot(4, 4, index + 1)
    ax.scatter(boston_dataset.data[:, index], boston_dataset.target)
    ax.set_ylabel('House Price', size = 12)
    ax.set_xlabel(feature_name, size = 12)

plt.show()
```

```python
X = df.iloc[:, :-1]
y = df.iloc[:, -1]

```

```python
X_train, X_test, y_train, y_test = train_test_split(
    X, y,
    test_size = 0.25)

print("Train data shape X = % s and y = % s : "%(
    X_train.shape, y_train.shape))

print("Test data shape X = % s and y = % s : "%(
    X_test.shape, y_test.shape))
```

    Train data shape X = (379, 13) and y = (379,) :
    Test data shape X = (127, 13) and y = (127,) :

```python
# StandarScale
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
X_train_scaled = pd.DataFrame(scaler.fit_transform(X_train), columns = X_train.columns)
X_test_scaled = pd.DataFrame(scaler.transform(X_test), columns = X_test.columns)
```

```python
X_train
```

      CRIM
      ZN
      INDUS
      CHAS
      NOX
      RM
      AGE
      DIS
      RAD
      TAX
      PTRATIO
      B
      LSTAT

      211
      0.37578
      0.0
      10.59
      1.0
      0.489
      5.404
      88.6
      3.6650
      4.0
      277.0
      18.6
      395.24
      23.98

      2
      0.02729
      0.0
      7.07
      0.0
      0.469
      7.185
      61.1
      4.9671
      2.0
      242.0
      17.8
      392.83
      4.03

      195
      0.01381
      80.0
      0.46
      0.0
      0.422
      7.875
      32.0
      5.6484
      4.0
      255.0
      14.4
      394.23
      2.97

      434
      13.91340
      0.0
      18.10
      0.0
      0.713
      6.208
      95.0
      2.2222
      24.0
      666.0
      20.2
      100.63
      15.17

      90
      0.04684
      0.0
      3.41
      0.0
      0.489
      6.417
      66.1
      3.0923
      2.0
      270.0
      17.8
      392.18
      8.81

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      128
      0.32543
      0.0
      21.89
      0.0
      0.624
      6.431
      98.8
      1.8125
      4.0
      437.0
      21.2
      396.90
      15.39

      318
      0.40202
      0.0
      9.90
      0.0
      0.544
      6.382
      67.2
      3.5325
      4.0
      304.0
      18.4
      395.21
      10.36

      372
      8.26725
      0.0
      18.10
      1.0
      0.668
      5.875
      89.6
      1.1296
      24.0
      666.0
      20.2
      347.88
      8.88

      164
      2.24236
      0.0
      19.58
      0.0
      0.605
      5.854
      91.8
      2.4220
      5.0
      403.0
      14.7
      395.11
      11.64

      73
      0.19539
      0.0
      10.81
      0.0
      0.413
      6.245
      6.2
      5.2873
      4.0
      305.0
      19.2
      377.17
      7.54

379 rows × 13 columns

```python
X_train_scaled
```

      CRIM
      ZN
      INDUS
      CHAS
      NOX
      RM
      AGE
      DIS
      RAD
      TAX
      PTRATIO
      B
      LSTAT

      0
      -0.365345
      -0.496544
      -0.032725
      3.610684
      -0.554810
      -1.241634
      0.723924
      -0.089781
      -0.624840
      -0.752319
      0.105433
      0.410116
      1.604117

      1
      -0.403233
      -0.496544
      -0.546098
      -0.276956
      -0.724461
      1.198062
      -0.238694
      0.513705
      -0.857810
      -0.961052
      -0.258658
      0.383643
      -1.179552

      2
      -0.404699
      2.828737
      -1.510133
      -0.276956
      -1.123142
      2.143256
      -1.257319
      0.829468
      -0.624840
      -0.883523
      -1.806046
      0.399022
      -1.327457

      3
      1.106457
      -0.496544
      1.062570
      -0.276956
      1.345287
      -0.140277
      0.947951
      -0.758477
      1.704863
      1.567596
      0.833615
      -2.826040
      0.374838

      4
      -0.401108
      -0.496544
      -1.079890
      -0.276956
      -0.554810
      0.146021
      -0.063673
      -0.355211
      -0.857810
      -0.794066
      -0.258658
      0.376503
      -0.512588

      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...

      374
      -0.370819
      -0.496544
      1.615322
      -0.276956
      0.590338
      0.165198
      1.080967
      -0.948362
      -0.624840
      0.201888
      1.288729
      0.428351
      0.405535

      375
      -0.362493
      -0.496544
      -0.133358
      -0.276956
      -0.088268
      0.098076
      -0.025168
      -0.151191
      -0.624840
      -0.591297
      0.014410
      0.409787
      -0.296313

      376
      0.492611
      -0.496544
      1.062570
      3.610684
      0.963571
      -0.596436
      0.758928
      -1.264866
      1.704863
      1.567596
      0.833615
      -0.110112
      -0.502821

      377
      -0.162412
      -0.496544
      1.278420
      -0.276956
      0.429169
      -0.625203
      0.835937
      -0.665876
      -0.508355
      -0.000881
      -1.669512
      0.408688
      -0.117712

      378
      -0.384957
      -0.496544
      -0.000639
      -0.276956
      -1.199485
      -0.089593
      -2.160429
      0.662108
      -0.624840
      -0.585333
      0.378501
      0.211625
      -0.689794

379 rows × 13 columns

# Plot Function

```python
def plot_model(model_coeffs):
  # plotting the coefficient score
  fig, ax = plt.subplots(figsize =(10, 5))

  color =['tab:gray', 'tab:blue', 'tab:orange',
  'tab:green', 'tab:red', 'tab:purple', 'tab:brown',
  'tab:pink', 'tab:gray', 'tab:olive', 'tab:cyan',
  'tab:orange', 'tab:green', 'tab:blue', 'tab:olive']

  ax.bar(model_coeffs["Columns"],
  model_coeffs['Coefficient Estimate'],
  color = color)

  ax.spines['bottom'].set_position('zero')

  plt.style.use('ggplot')
  plt.show()

```

# Multiple Linear Regression

$$y = w_0 + w_1 x_{1} + w_2 x_{2} + ... + w_n x_{n} + \epsilon$$

$$J(w) = \sum_{i=1}^m r_i^2 = \sum_{i=1}^m (y_i - \tilde{y}_i)^2 = \sum_{i=1}^m (y_i - \tilde{w}_0 - \tilde{w}_1 x_1 - ... - \tilde{w}_n x_n)^2$$

```python
# multiple Linear Regression Model
lreg = LinearRegression()
lreg.fit(X_train_scaled, y_train)

# Prediccion en test set
lreg_y_pred = lreg.predict(X_test_scaled)

# MSE train set
mse_train = np.mean((lreg.predict(X_train_scaled) - y_train)**2)
print("Mean squared Error on train set : ", mse_train)

# calculating Mean Squared Error (mse)
mean_squared_error = np.mean((lreg_y_pred - y_test)**2)
print("Mean squared Error on test set : ", mean_squared_error)

# Putting together the coefficient and their corresponding variable names
lreg_coefficient = pd.DataFrame()
lreg_coefficient["Columns"] = X_train.columns
lreg_coefficient['Coefficient Estimate'] = pd.Series(lreg.coef_)
print(lreg_coefficient)
```

    Mean squared Error on train set :  20.11486173313239
    Mean squared Error on test set :  37.173340973712655
        Columns  Coefficient Estimate
    0      CRIM             -0.981919
    1        ZN              1.103023
    2     INDUS              0.706474
    3      CHAS              0.181104
    4       NOX             -2.054656
    5        RM              3.302312
    6       AGE             -0.067952
    7       DIS             -2.860861
    8       RAD              2.200599
    9       TAX             -2.191210
    10  PTRATIO             -2.275399
    11        B              1.048264
    12    LSTAT             -3.420124

```python
plot_model(lreg_coefficient)
```

# Ridge Regression

\begin{equation}
  J(w)= \sum_i (w^T x_i - y_i)^2  + \lambda w^2
\end{equation}

```python
def ridge(lambd, X_train = X_train_scaled, y_train = y_train, X_test = X_test_scaled, y_test = y_test):
  from sklearn.linear_model import Ridge

  # Train the model
  ridgeR = Ridge(alpha = lambd)
  ridgeR.fit(X_train, y_train)
  y_pred = ridgeR.predict(X_test)

  # mse train set
  mse_train = np.mean((ridgeR.predict(X_train) - y_train)**2)
  #print('MSE RIDGE train set :', mse_train)

  # mean square error
  mean_squared_error_ridge = np.mean((y_pred - y_test)**2)

  #ridge coefficient
  df_ridge_coefficient = pd.DataFrame()
  df_ridge_coefficient["Columns"]= X_train.columns
  df_ridge_coefficient['Coefficient Estimate'] = pd.Series(ridgeR.coef_)
  return mean_squared_error_ridge, df_ridge_coefficient, ridgeR.coef_, mse_train
```

```python
mse_ridge, ridge_coefficient,_,mse_train = ridge(1)
print('MSE RIDGE train set :', mse_train)
print('MSE RIDGE test set :', mse_ridge)
print('-----------------')
print(ridge_coefficient)
```

    MSE RIDGE train set : 21.84825265392536
    MSE RIDGE test set : 27.298676841645808
    -----------------
        Columns  Coefficient Estimate
    0      CRIM             -1.009143
    1        ZN              0.980959
    2     INDUS             -0.194660
    3      CHAS              0.701500
    4       NOX             -1.807307
    5        RM              3.044062
    6       AGE             -0.198334
    7       DIS             -3.205758
    8       RAD              2.488452
    9       TAX             -1.959227
    10  PTRATIO             -2.060459
    11        B              0.594750
    12    LSTAT             -3.844630

```python
plot_model(ridge_coefficient)
```

### Lambda vs MSE

```python
# Veamos como varia el MSE por lambda
lambdas = np.linspace(0,1000,1000)
ridge_test_mses = [ridge(l)[0] for l in lambdas]
ridge_train_mses = [ridge(l)[3] for l in lambdas]

fig = plt.figure(figsize = (10, 5))
fig.add_subplot(1,2,1)
plt.plot(lambdas, ridge_test_mses)
plt.title('Test set')
fig.add_subplot(1,2,2)
plt.plot(lambdas, ridge_train_mses)
plt.title('Train set')
```

### Lambda vs Weights

```python
# Veamos como varia el MSE por lambda
lambdas = 10**np.linspace(10,-2,100)*0.5
ridge_coeffs = [ridge(l)[2] for l in lambdas]
ax = plt.gca()
ax.plot(lambdas, ridge_coeffs)
ax.set_xscale('log')
plt.axis('tight')
plt.xlabel('alpha')
plt.ylabel('weights')
```

# Lasso Regression

\begin{equation}
  J(w)= \sum_i (w^T x_i - y_i)^2  + \lambda ||w||_1
\end{equation}

```python
def lasso(lambd, X_train = X_train_scaled, y_train = y_train, X_test = X_test_scaled, y_test = y_test):
  from sklearn.linear_model import Lasso

  # Train the model
  lasso = Lasso(alpha = lambd)
  lasso.fit(X_train, y_train)
  y_pred1 = lasso.predict(X_test)

  # mse train set
  mse_train = np.mean((lasso.predict(X_train) - y_train)**2)
  #print('MSE RIDGE train set :', mse_train)

  # Calculate Mean Squared Error
  mean_squared_error = np.mean((y_pred1 - y_test)**2)

  df_lasso_coeff = pd.DataFrame()
  df_lasso_coeff["Columns"] = X_train.columns
  df_lasso_coeff['Coefficient Estimate'] = pd.Series(lasso.coef_)

  return mean_squared_error, df_lasso_coeff, lasso.coef_, mse_train

```

```python
mse_lasso, lasso_coefficient,_,mse_train = lasso(1)
print('MSE LASSO train set :', mse_train)
print('MSE LASSO :', mse_lasso)
print('-----------------')
print(lasso_coefficient)
```

    MSE LASSO train set : 28.39264682044312
    MSE LASSO : 29.095045432946005
    -----------------
        Columns  Coefficient Estimate
    0      CRIM             -0.000000
    1        ZN              0.000000
    2     INDUS             -0.000000
    3      CHAS              0.000000
    4       NOX             -0.000000
    5        RM              3.063196
    6       AGE             -0.000000
    7       DIS             -0.000000
    8       RAD             -0.000000
    9       TAX             -0.000000
    10  PTRATIO             -1.555405
    11        B              0.000000
    12    LSTAT             -3.730375

```python
plot_model(lasso_coefficient)
```

### Lambda Vs MSE

```python
# Veamos como varia el MSE por lambda
lambdas = np.linspace(0,30,1000)
lasso_test_mses = [lasso(l)[0] for l in lambdas]
lasso_train_mses = [lasso(l)[3] for l in lambdas]

fig = plt.figure(figsize = (10, 5))
fig.add_subplot(1,2,1)
plt.plot(lambdas, lasso_test_mses)
plt.title('Test set')
fig.add_subplot(1,2,2)
plt.plot(lambdas, lasso_train_mses)
plt.title('Train set')
```

### Lambda vs Weights

```python
# Veamos como varia el MSE por lambda
lambdas = 10**np.linspace(10,-2,100)*0.5
lasso_coeffs = [lasso(l)[2] for l in lambdas]
ax = plt.gca()
ax.plot(lambdas, lasso_coeffs)
ax.set_xscale('log')
plt.axis('tight')
plt.xlabel('alpha')
plt.ylabel('weights')
```

# Conclusiones:
## Ridge:
- Incluye todas (o ninguna) las variables del modelo. Por lo tanto, la principal ventaja de la regresión ridge es el 'shrinkage' del coeficiente y la reducción de la complejidad del modelo.
- Normalmente se utiliza para reducit el sobre ajuste
- En general, funciona bien incluso en presencia de variables altamente correlacionadas, ya que las incluirá todas en el modelo, pero los coeficientes se distribuirán entre ellas según la correlación.

## Lasso:
- Junto con los coeficientes de reducción, Lasso también realiza la selección de variables.Algunos de los coeficientes se vuelven exactamente cero, lo que equivale a que la variable particular se excluya del modelo.

- Normalmente se utiliza para reducir las variables y obtener una'sparse solution' (Ej: cuando hay miles de variables)

- Selecciona arbitrariamente cualquier característica entre las altamente correlacionadas y reduce los coeficientes del resto a cero. Además, la variable elegida cambia aleatoriamente con el cambio en los parámetros del modelo. Por lo general, esto no funciona tan bien en comparación con la regresión ridge

# Referencias:
- https://www.cienciadedatos.net/documentos/py14-ridge-lasso-elastic-net-python.html
- https://www.analyticsvidhya.com/blog/2016/01/ridge-lasso-regression-python-complete-tutorial/
- https://www.geeksforgeeks.org/implementation-of-lasso-ridge-and-elastic-net/
- https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Lasso.html
- https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html

```python

```

```python

```

```python

```


# Sympy - Symbolic algebra in Python

J.R. Johansson (jrjohansson at gmail.com), updated by M. V. dos Santos (marcelo.santos at df.ufcg.edu.br)

The latest version of
this [Jupyter notebook](https://jupyter.org/) lecture is available at [https://github.com/mvsantosdev/scientific-python-lectures.git](https://github.com/mvsantosdev/scientific-python-lectures.git).

The other notebooks in this lecture series are indexed at [http://jrjohansson.github.io](http://jrjohansson.github.io).

```python
%matplotlib inline
import matplotlib.pyplot as plt
```

## Introduction

There are two notable Computer Algebra Systems (CAS) for Python:

* [SymPy](http://sympy.org/en/index.html) - A python module that can be used in any Python program, or in an IPython session, that provides powerful CAS features.
* [Sage](http://www.sagemath.org/) - Sage is a full-featured and very powerful CAS enviroment that aims to provide an open source system that competes with Mathematica and Maple. Sage is not a regular Python module, but rather a CAS environment that uses Python as its programming language.

Sage is in some aspects more powerful than SymPy, but both offer very comprehensive CAS functionality. The advantage of SymPy is that it is a regular Python module and integrates well with the IPython notebook.

In this lecture we will therefore look at how to use SymPy with IPython notebooks. If you are interested in an open source CAS environment I also recommend to read more about Sage.

To get started using SymPy in a Python program or notebook, import the module `sympy`:

```python
import sympy as sp
```

To get nice-looking $\LaTeX$ formatted output run:

```python
sp.init_printing()
```

## Symbolic variables

In SymPy we need to create symbols for the variables we want to work with. We can create a new symbol using the `Symbol` class:

```python
x = sp.Symbol('x')

x
```

```python
(sp.pi + x)**2
```

```python
# alternative way of defining symbols
a, b, c, sig = sp.symbols("a b c \hat{\sigma}")

a, b, c, sig
```

```python
type(a)
```

    sympy.core.symbol.Symbol

We can add assumptions to symbols when we create them:

```python
x = sp.Symbol('x', real=True)
```

```python
x.is_imaginary
```

    False

```python
x = sp.Symbol('x', positive=True)
```

```python
x > 0
```

### Complex numbers

The imaginary unit is denoted `I` in Sympy.

```python
1+sp.I
```

```python
sp.I**2
```

```python
(x * sp.I + 1)**2
```

### Rational numbers

There are three different numerical types in SymPy: `Real`, `Rational`, `Integer`:

```python
r1 = sp.Rational(4,5)
r2 = sp.Rational(5,4)
```

```python
r1
```

```python
r1+r2
```

```python
r1/r2
```

## Numerical evaluation

SymPy uses a library for artitrary precision as numerical backend, and has predefined SymPy expressions for a number of mathematical constants, such as: `pi`, `E`, `oo` for infinity.

To evaluate an expression numerically we can use the `evalf` function (or `N`). It takes an argument `n` which specifies the number of significant digits.

```python
sp.pi.evalf(n=50)
```

```python
y = (x + sp.pi)**2

y
```

```python
sp.N(y, 5) # same as evalf
```

When we numerically evaluate algebraic expressions we often want to substitute a symbol with a numerical value. In SymPy we do that using the `subs` function:

```python
y.subs(x, 1.5)
```

```python
y.subs(x, 1.5).evalf()
```

The `subs` function can of course also be used to substitute Symbols and expressions:

```python
y.subs(x, a+sp.pi)
```

We can also combine numerical evolution of expressions with NumPy arrays:

```python
import numpy as np
```

```python
x_vec = np.arange(0, 10, 0.1)

x_vec
```

    array([0. , 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. , 1.1, 1.2,
           1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2. , 2.1, 2.2, 2.3, 2.4, 2.5,
           2.6, 2.7, 2.8, 2.9, 3. , 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8,
           3.9, 4. , 4.1, 4.2, 4.3, 4.4, 4.5, 4.6, 4.7, 4.8, 4.9, 5. , 5.1,
           5.2, 5.3, 5.4, 5.5, 5.6, 5.7, 5.8, 5.9, 6. , 6.1, 6.2, 6.3, 6.4,
           6.5, 6.6, 6.7, 6.8, 6.9, 7. , 7.1, 7.2, 7.3, 7.4, 7.5, 7.6, 7.7,
           7.8, 7.9, 8. , 8.1, 8.2, 8.3, 8.4, 8.5, 8.6, 8.7, 8.8, 8.9, 9. ,
           9.1, 9.2, 9.3, 9.4, 9.5, 9.6, 9.7, 9.8, 9.9])

```python
y_vec = np.array([
    sp.N(y.subs(x, xx))
    for xx in x_vec
])

y_vec
```

    array([9.86960440108936, 10.5079229318073, 11.1662414625253,
           11.8445599932432, 12.5428785239612, 13.2611970546792,
           13.9995155853971, 14.7578341161151, 15.5361526468330,
           16.3344711775510, 17.1527897082689, 17.9911082389869,
           18.8494267697049, 19.7277453004228, 20.6260638311408,
           21.5443823618587, 22.4827008925767, 23.4410194232947,
           24.4193379540126, 25.4176564847306, 26.4359750154485,
           27.4742935461665, 28.5326120768845, 29.6109306076024,
           30.7092491383204, 31.8275676690383, 32.9658861997563,
           34.1242047304742, 35.3025232611922, 36.5008417919102,
           37.7191603226281, 38.9574788533461, 40.2157973840640,
           41.4941159147820, 42.7924344455000, 44.1107529762179,
           45.4490715069359, 46.8073900376538, 48.1857085683718,
           49.5840270990898, 51.0023456298077, 52.4406641605257,
           53.8989826912436, 55.3773012219616, 56.8756197526795,
           58.3939382833975, 59.9322568141155, 61.4905753448334,
           63.0688938755514, 64.6672124062693, 66.2855309369873,
           67.9238494677053, 69.5821679984232, 71.2604865291412,
           72.9588050598591, 74.6771235905771, 76.4154421212951,
           78.1737606520130, 79.9520791827310, 81.7503977134489,
           83.5687162441669, 85.4070347748848, 87.2653533056028,
           89.1436718363208, 91.0419903670387, 92.9603088977567,
           94.8986274284746, 96.8569459591926, 98.8352644899106,
           100.833583020629, 102.851901551346, 104.890220082064,
           106.948538612782, 109.026857143500, 111.125175674218,
           113.243494204936, 115.381812735654, 117.540131266372,
           119.718449797090, 121.916768327808, 124.135086858526,
           126.373405389244, 128.631723919962, 130.910042450680,
           133.208360981398, 135.526679512116, 137.864998042834,
           140.223316573552, 142.601635104270, 144.999953634988,
           147.418272165706, 149.856590696424, 152.314909227142,
           154.793227757860, 157.291546288577, 159.809864819295,
           162.348183350013, 164.906501880731, 167.484820411449,
           170.083138942167], dtype=object)

```python
fig, ax = plt.subplots()
ax.plot(x_vec, y_vec);
```

However, this kind of numerical evolution can be very slow, and there is a much more efficient way to do it: Use the function `lambdify` to "compile" a Sympy expression into a function that is much more efficient to evaluate numerically:

```python
sp.lambdify?
```

```python
f = sp.lambdify([x], (x + sp.pi)**2, 'numpy')
# the first argument is a list of variables that
# f will be a function of: in this case only x -> f(x)
```

```python
y_vec = f(x_vec)  # now we can directly pass a numpy array and f(x) is efficiently evaluated
```

The speedup when using "lambdified" functions instead of direct numerical evaluation can be significant, often several orders of magnitude. Even in this simple example we get a significant speed up:

```python
%%timeit

y_vec = np.array([sp.N(((x + sp.pi)**2).subs(x, xx)) for xx in x_vec])
```

    16.8 ms ± 217 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)

```python
%%timeit

y_vec = f(x_vec)
```

    1.62 µs ± 24.9 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)

## Algebraic manipulations

One of the main uses of an CAS is to perform algebraic manipulations of expressions. For example, we might want to expand a product, factor an expression, or simply an expression. The functions for doing these basic operations in SymPy are demonstrated in this section.

### Expand and factor

The first steps in an algebraic manipulation

```python
(x+1)*(x+2)*(x+3)
```

```python
sp.expand((x+1)*(x+2)*(x+3))
```

The `expand` function takes a number of keywords arguments which we can tell the functions what kind of expansions we want to have performed. For example, to expand trigonometric expressions, use the `trig=True` keyword argument:

```python
sp.sin(a+b)
```

```python
sp.expand(sp.sin(a+b), trig=True)
```

```python

```

See `help(sp.expand)` for a detailed explanation of the various types of expansions the `expand` functions can perform.

The opposite a product expansion is of course factoring. The factor an expression in SymPy use the `factor` function:

```python
sp.factor(x**3 + 6 * x**2 + 11*x + 6)
```

```python
sp.factor(x**2 + 2*x + 1, gaussian=True)
```

### Simplify

The `simplify` tries to simplify an expression into a nice looking expression, using various techniques. More specific alternatives to the `simplify` functions also exists: `trigsimp`, `powsimp`, `logcombine`, etc.

The basic usages of these functions are as follows:

```python
# simplify expands a product
sp.simplify((x+1)*(x+2)*(x+3))
```

```python
sp.sin(a)**2 + sp.cos(a)**2
```

```python
# simplify uses trigonometric identities
sp.simplify(sp.sin(a)**2 + sp.cos(a)**2)
```

```python
sp.simplify(sp.cos(x)/sp.sin(x))
```

### apart and together

To manipulate symbolic expressions of fractions, we can use the `apart` and `together` functions:

```python
f1 = 1/((a+1)*(a+2))
```

```python
f1
```

```python
sp.apart(f1)
```

```python
f2 = 1/(a+2) + 1/(a+3)
```

```python
f2
```

```python
sp.together(f2)
```

Simplify usually combines fractions but does not factor:

```python
sp.simplify(f2)
```

## Calculus

In addition to algebraic manipulations, the other main use of CAS is to do calculus, like derivatives and integrals of algebraic expressions.

### Differentiation

Differentiation is usually simple. Use the `diff` function. The first argument is the expression to take the derivative of, and the second argument is the symbol by which to take the derivative:

```python
y = (x + a + sp.pi)**3

y
```

```python
sp.diff(y**2, x, a, x)
```

```python
y.diff(x, x)
```

For higher order derivatives we can do:

```python
sp.diff(y**2, x, x)
```

```python
sp.diff(y**2, x, 2) # same as above
```

```python
(y**2).diff(x, 2)
```

To calculate the derivative of a multivariate expression, we can do:

```python
x, y, z = sp.symbols("x y z")
```

```python
f = sp.sin(x*y) + sp.cos(y*z)
```

$\frac{d^3f}{dxdy^2}$

```python
f.diff(x, 1, y, 2)
```

## Integration

Integration is done in a similar fashion:

```python
f = sp.cos(y*z)

f
```

```python
sp.integrate(f, x, y, z)
```

By providing limits for the integration variable we can evaluate definite integrals:

```python
sp.integrate(f, (x, -1, 1))
```

and also improper integrals

```python
sigma = sp.symbols('\sigma', real=True)

sigma.is_imaginary
```

    False

```python
sp.integrate(sp.exp(-x**2/(2*sigma**2)), (x, -sp.oo, sp.oo))
```

Remember, `oo` is the SymPy notation for inifinity.

### Sums and products

We can evaluate sums and products using the functions: 'Sum'

```python
n = sp.Symbol("n")
```

```python
sp.Sum(1/n**2, (n, 1, 10))
```

```python
sp.Sum(1/n**2, (n,1, 10)).evalf()
```

```python
sp.Sum(1/n**2, (n, 1, sp.oo)).evalf()
```

Products work much the same way:

```python
sp.Product(n, (n, 1, 10)) # 10!
```

```python
sp.Product(n, (n, 1, 10)).evalf()
```

## Limits

Limits can be evaluated using the `limit` function. For example,

```python
sp.limit(sp.sin(x)/x, x, 0)
```

We can use 'limit' to check the result of derivation using the `diff` function:

```python
f
```

```python
f.diff(x)
```

$\displaystyle \frac{\mathrm{d}f(x,y)}{\mathrm{d}x} = \frac{f(x+h,y)-f(x,y)}{h}$

```python
h = sp.Symbol("h")
```

```python
sp.limit((f.subs(x, x+h) - f)/h, h, 0)
```

OK!

We can change the direction from which we approach the limiting point using the `dir` keywork argument:

```python
sp.limit(1/x, x, 0, dir="+")
```

```python
sp.limit(1/x, x, 0, dir="-")
```

## Series

Series expansion is also one of the most useful features of a CAS. In SymPy we can perform a series expansion of an expression using the `series` function:

```python
sp.series(sp.exp(x), x)
```

By default it expands the expression around $x=0$, but we can expand around any value of $x$ by explicitly include a value in the function call:

```python
sp.series(sp.exp(x), x, 1)
```

And we can explicitly define to which order the series expansion should be carried out:

```python
sp.series(sp.exp(x), x, 1, 10)
```

The series expansion includes the order of the approximation, which is very useful for keeping track of the order of validity when we do calculations with series expansions of different order:

```python
s1 = sp.cos(x).series(x, 0, 5)
s1
```

```python
s2 = sp.sin(x).series(x, 0, 2)
s2
```

```python
s1.removeO()
```

If we want to get rid of the order information we can use the `removeO` method:

```python
sp.expand(s1.removeO() * s2.removeO())
```

But note that this is not the correct expansion of $\cos(x)\sin(x)$ to $5$th order:

```python
(sp.cos(x)*sp.sin(x)).series(x, 0, 6)
```

## Linear algebra

### Matrices

Matrices are defined using the `Matrix` class:

```python
m11, m12, m21, m22 = sp.symbols("m11, m12, m21, m22")
b1, b2 = sp.symbols("b1, b2")
```

```python
A = sp.Matrix([[m11, m12],[m21, m22]])
A
```

```python
b = sp.Matrix([[b1], [b2]])
b
```

With `Matrix` class instances we can do the usual matrix algebra operations:

```python
A**2
```

```python
A * b
```

And calculate determinants and inverses, and the like:

```python
A.det()
```

```python
A.inv()
```

## Solving equations

For solving equations and systems of equations we can use the `solve` function:

```python
sp.solve(x**2 - 1, x) #the expression is equal to zero
```

```python
sp.solve(x**4 - x**2 - 1, x)
```

System of equations:

```python
sp.solve([x + y - 1, x - y - 1], [x,y])
```

In terms of other symbolic expressions:

```python
sp.solve([x + y - a, x - y - c], [x,y])
```

## Solving differential equations

```python
f = sp.Function('f')
```

```python
f(x).diff(x, 2) + a**2 * f(x)
```

```python
sp.dsolve(f(x).diff(x, 2) + a**2 * f(x), f(x))
```

```python
ics={
    f(0): 1,
    f(x).diff(x).subs(x, 0): 0
}

ics
```

```python
Y = sp.dsolve(f(x).diff(x, 2) + a**2 * f(x), f(x), ics=ics)

Y
```

```python
Y.simplify()
```

```python
sp.Ynm(0, 0, x, y)
```

## Further reading

* http://sympy.org/en/index.html - The SymPy projects web page.
* https://github.com/sympy/sympy - The source code of SymPy.
* http://live.sympy.org - Online version of SymPy for testing and demonstrations.

## Versions

```python
%reload_ext version_information

%version_information numpy, matplotlib, sympy
```

Software	Version

Python	2.7.10 64bit [GCC 4.2.1 (Apple Inc. build 5577)]

IPython	3.2.1

OS	Darwin 14.1.0 x86_64 i386 64bit

numpy	1.9.2

matplotlib	1.4.3

sympy	0.7.6

Sat Aug 15 11:37:37 2015 JST


# Truss Bridge

# Equilibrium equations

## a

$
\begin{align}
-f_1 \cos(45) - f_2 +r_{ax} & = 0 \\
-f_1 \cos(45) + r_{ay} & = 0
\end{align}
$

## b

$
\begin{align}
f_1 \cos(45) - f_4 & = 0 \\
f_1 \sin(45) + f_3 & = 0 \\
\end{align}
$

## c

$
\begin{align}
f_2 - f_5 \cos(45) - f_6 & = 0 \\
-f_3 - f_5 \cos(45) & = -T_c \\
\end{align}
$

## d

$
\begin{align}
f_4 + f_5 \cos(45) - f_8 \cos(45) & = 0 \\
f_8 \sin(45) + f_5 \sin(45) + f_7 & = 0 \\
\end{align}
$

## e

$
\begin{align}
f_6 - f_9 & = 0 \\
- f_7 & = - T_e \\
\end{align}
$

## f

$
\begin{align}
f_8 \cos(45) + f_9 & = 0 \\
-f_8 \cos(45) + r_{fy} & = 0
\end{align}
$

# Matrix equation

$
\begin{align}
f_i &= [f_1,f_2,f_3,f_4,f_5,f_6,f_7,f_8,f_9, r_{ax}, r_{ay}, r_{fy}]^T \\
f_e &= [0,0,0,0,0,T_c,0,0,0, T_e, 0, 0]^T \\
\end{align}
$

$$ A_{12 \times 12} =
\begin{bmatrix}
-\frac{\sqrt(2)}{2} & -1 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & 1 & 0 & 0 \\
-\frac{\sqrt(2)}{2} & 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & 1 & 0 \\
\frac{\sqrt(2)}{2} & 0 & 0 & -1 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0\\
\frac{\sqrt(2)}{2} & 0 & 1 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0\\
0 & 1 & 0 & 0 & -\frac{\sqrt(2)}{2} & -1 & 0 & 0 & 0 & 0 & 0 & 0\\
0 & 0 & -1 & 0 & -\frac{\sqrt(2)}{2} & 0 & 0 & 0 & 0 & 0 & 0 & 0\\
0 & 0 & 0 & 1 & \frac{\sqrt(2)}{2} & 0 & 0 & -\frac{\sqrt(2)}{2} & 0 & 0 & 0 & 0\\
0 & 0 & 0 & 0 & \frac{\sqrt(2)}{2} & 0 & 1 & \frac{\sqrt(2)}{2} & 0 & 0 & 0 & 0\\
0 & 0 & 0 & 0 & 0 & 1 & 0 & 0 & -1 & 0 & 0 & 0\\
0 & 0 & 0 & 0 & 0 & 0 & -1 & 0 & 0 & 0 & 0 & 0\\
0 & 0 & 0 & 0 & 0 & 0 & 0 & \frac{\sqrt(2)}{2} & 1 & 0 & 0 & 0\\
0 & 0 & 0 & 0 & 0 & 0 & 0 & -\frac{\sqrt(2)}{2} & 0 & 0 & 0 & 1
\end{bmatrix}
$$

# Solution

```julia
A = [-sqrt(2)/2  -1  0  0  0  0  0  0  0  1  0  0 ;
-sqrt(2)/2  0  0  0  0  0  0  0  0  0  1  0 ;
sqrt(2)/2  0  0  -1  0  0  0  0  0  0  0  0;
sqrt(2)/2  0  1  0  0  0  0  0  0  0  0  0;
0  1  0  0  -sqrt(2)/2  -1  0  0  0  0  0  0;
0  0  -1  0  -sqrt(2)/2  0  0  0  0  0  0  0;
0  0  0  1  sqrt(2)/2  0  0  -sqrt(2)/2  0  0  0  0;
0  0  0  0  sqrt(2)/2  0  1  sqrt(2)/2  0  0  0  0;
0  0  0  0  0  1  0  0  -1  0  0  0;
0  0  0  0  0  0  -1  0  0  0  0  0;
0  0  0  0  0  0  0  sqrt(2)/2  1  0  0  0;
0  0  0  0  0  0  0  -sqrt(2)/2  0  0  0  1];

fₑ = [0,0,0,0,0,7.5,0,0,0, 7.5, 0, 0]

fᵢ = fₑ \ A;
fᵢ'
```

    12-element Array{Float64,1}:
      0.0
      0.0
     -0.06666666666666667
      0.0
     -0.04714045207910317
      0.0
     -0.06666666666666667
      0.0
      0.0
      0.0
      0.0
      0.0

## Will the bridge collapse?

No, and not even with 20 ton truck.

```julia
fₑ = [0,0,0,0,0,10,0,0,0, 10, 0, 0]
fᵢ = fₑ \ A;
fᵢ'
```

    12-element Array{Float64,1}:
      0.0
      0.0
     -0.05
      0.0
     -0.03535533905932738
      0.0
     -0.05
      0.0
      0.0
      0.0
      0.0
      0.0

## Adjourn

```julia
using Dates
println("mahdiar")
Dates.format(now(), "Y/U/d HH:MM")
```

    mahdiar

    "2021/February/11 14:51"


```
#Exhaustive search method
f = lambda x: x**2+54/x
a=0
b=5
n=10
print(f"a={a},b={b},n={n}")
dx = (b-a)/n
print(f"Δx = b-a/n = {b}-{a}/{n} = {dx}")
print(f"x1 = a, x2 = x1 + Δx,  x3 = x1 + 2Δx")
print("write function")
x1=a
x2=x1+dx
x3=x1+2*dx
i=1
flag=0
while(flag == 0):
  print("Iteration ",i)
  i+=1
  try:
      f1 = f(x1)
  except ZeroDivisionError:
      f1 = 99999999999999
  try:
      f2 = f(x2)
  except ZeroDivisionError:
      f2 = 99999999999999
  try:
      f3 = f(x3)
  except ZeroDivisionError:
      f3 = 99999999999999
  print(f"x1 = {x1}, x2 = {x1} + {dx} = {x2}, x3 = {x1} + 2*{dx} = {x3}")
  print(f"f(x1) = f({x1}) = {f1} ,f(x2) = f({x2}) = {f2},f(x3) = f({x3}) = {f3}")
  p =[x1,x2,x3,f1,f2,f3]
  p = [round(num,3) for num in p]
  print(p)

  if (f1>=f2<=f3):
    flag=1
    print("since f(x1)>=f(x2)<=f(x3)")
    print("Yes")
    print(f"Desired Bracket is ({x1},{x3}) ")
  else:
    print("since f(x1)>=f(x2)<=f(x3)")
    print("NO")
    print("x1 = x2, x2 = x3,x3 = x2 + Δx")
    x1=x2
    x2=x3
    x3=x2+dx
```

```
#Bounding phase method
import math

f = lambda x: math.sin(2*x)+x**2+21
x=1
d=0.5
k=0
xk=x
flag=0
i=1
while flag==0:
  print("Iteration ",i)
  i+=1
  xk1=xk+(2**k)*d
  print(f"k = {k}, Δ = {d}")
  print(f"x({k}) = {xk}, x({k+1}) = x({k}) + 2^({k})*{d}  = {x} + {(2**k)*d} = {xk1}")
  try:
    p = f(xk)
  except ZeroDivisionError:
    p = 99999999999999
  try:
    q = f(xk1)
  except ZeroDivisionError:
    q = 99999999999999
  print(f"f(x({k})) = {p}, f(x({k+1})) = {q}")
  pr =[xk,xk1,p,q]
  pr = [round(num,3) for num in pr]
  print(pr)
  k+=1
  if(q<p):
    xk=xk1
    print("Yes")
  else:
    flag=1
    print("NO")
    print(f"Desired Bracket ({round(xk-(2**(k-2))*d,3)} ,{xk1})")
```

```
#Interval Halving method
f = lambda x: x**3+648*(x**(-2))
a=-1
b=4
e=10**(-3)
l=b-a
print(f"a={a},b={b},l={l},∈={e}")
while l>0.5:
  xm=(a+b)/2
  x1=a+l/4
  x2=b-l/4
  print(f"xm =(a+b)/2 = ({a}+{b})/2 = {xm}, x1 = a+l/4 = {a}+{l}/4 = {x1}, x2 = b-l/4 = {b}-{l}/4 = {x2}")

  try:
    f1 = f(x1)
  except ZeroDivisionError:
    f1 = 99999999999999
  try:
    f2 = f(x2)
  except ZeroDivisionError:
    f2 = 99999999999999
  try:
    fm = f(xm)
  except ZeroDivisionError:
    fm = 99999999999999
  print(f"f(x1) = f({x1}) = {f1} ,f(x2) = f({x2}) = {f2},f(xm) = f({xm}) = {fm}")
  p=[x1,x2,xm,f1,f2,fm]
  p = [round(num,3) for num in p]
  print(p)

  if(f1>fm and f2>fm ):
    a=x1
    b=x2
    l=b-a
    print("Two sides ")
    print(f"f(x1)>f(xm) and f(x2)>f(xm) \n\n,a = {a},b = {b} and l = {l}")
  elif(f1>fm and f2<fm):
    a=xm
    l=b-a
    print("Cut Two left side")
    print(f"f(x1)>f(xm) and f(x2)<f(xm)  \n\n,a = {a},b = {b} and l = {l}")
  elif(f1<fm and f2>fm):
    b=xm
    l=b-a
    print("Cut Two right side ")
    print(f"f(x1)<f(xm) and f(x2)>f(xm) \n\n,a = {a},b = {b} and l = {l}")

```

```
#Fibonacci search method

f = lambda x: x**3-9*x
a=1
b=4
k=2
n=3
l=b-a
print(f"a={a},b={b},k={k},n={n},L=b-a={l}")
def Fibonacci(n):
    if n<0:
        print("Incorrect input")
    # First Fibonacci number is 0
    elif n==0:
        return 1
    # Second Fibonacci number is 1
    elif n==1:
        return 1
    else:
        return Fibonacci(n-1)+Fibonacci(n-2)

while(k<=n):
  lk=(Fibonacci(n-k+1)/Fibonacci(n+1))*l
  x1=a+lk
  x2=b-lk
  print(f"Lk*=(F(n-k+1)/F(n+1))*L")
  print(f"L{k}*=(F({n-k+1})/F({n+1}))*{l} = {lk}")
  print(f"x1=a+L{k}*={a}+{lk}={x1}")
  print(f"x2=b-L{k}*={b}-{lk}={x2}")
  try:
    f1 = f(x1)
  except ZeroDivisionError:
    f1 = 99999999999999
  try:
    f2 = f(x2)
  except ZeroDivisionError:
    f2 = 99999999999999
  print(f"f(x1) = f({x1}) = {f1} ,f(x2) = f({x2}) = {f2}")
  p=[x1,x2,f1,f2]
  p = [round(num,3) for num in p]
  print(p)
  k+=1
  if (f1>f2):
    a=x1
    print("f(x1)> f(x2)")
    print("left side\n")
    print(f"a={a},b={b},k={k},n={n}")
  elif (f1<f2):
    b=x2
    print("f(x1)< f(x2)")
    print("right side\n")
    print(f"a={a},b={b},k={k},n={n}")

```

```
#Golden Section search method
f = lambda x: x**4-14*(x**3)+60*(x**2)-70*x

a=1
b=4

fw = lambda w: f(w*(b-a)+a)

aw=0
bw=1
lw=bw-aw
print(f"w = x-a/b-a , aw =0,bw =1")
print(f"w = (x-{a})/({b}-{a}) , w = (x-{a})/({b-a})")
print(f"x = w*{b-a}+{a}")
print("Calculate f(w)")
while(lw>0.1):
  lw=bw-aw
  w1=aw+0.618*lw
  w2=bw-0.618*lw

  print(f"lw = {bw}-{aw} ={lw}")
  print(f"w1 = aw+0.618*lw ={aw}+0.618*{lw} = {w1}")
  print(f"w2 = bw-0.618*lw ={bw}-0.618*{lw} = {w2}")
  try:
    f1 = fw(w1)
  except ZeroDivisionError:
    f1 = 99999999999999
  try:
    f2 = fw(w2)
  except ZeroDivisionError:
    f2 = 99999999999999
  print(f"f(w1) = f({w1}) = {f1} ,f(w2) = f({w2}) = {f2}")
  p=[aw,bw,w1,w2,f1,f2]
  p = [round(num,3) for num in p]
  print(p)
  if (f1<f2):
    aw=w2
    print("f(x1)< f(x2)")
    print("cut left side \n")
    print(f"aw = {aw},bw = {bw}")
  elif (f1>f2):
    bw=w1
    print("f(x1)> f(x2)")
    print("cut right side \n")
    print(f"aw = {aw},bw = {bw}")
  else:
    aw=w1
    bw=w2
    print("f(x1)=f(x2)")
    print("cut both side \n")
    print(f"aw = {aw},bw = {bw}")

```

```
# Successive Quadratic Estimation Method
f = lambda x: x**2+54/x
x1 = 1
d = 1

x2 =x1+d
fx1 = f(x1)
fx2 =f(x2)
if(fx1>fx2):
  x3 = x1+2*d
  fx3 =f(x3)

fmin = min(fx1,fx2,fx3)
if(fmin==fx1):
  xmin = x1
elif(fmin == fx2):
  xmin = x2
else:
  xmin = x3

a0 = fx1
a1 = (fx2-fx1)/(x2-x1)
a2 = (1/(x3-x2))*((fx3-fx1)/(x3-x1)-a1)
xd = ((x1+x2)/2)-(a1/(2*a2))

print(a0,a1,a2,xd)
```

    55.0 -24.0 10.0 2.7

```
#Newton-Raphson Method
import sympy as sp
from sympy import *
x= sp.Symbol('x')

f = lambda x: x**2+54/x

df = sp.diff(f(x),x)
dff =sp.diff(df,x)
print(f(x))
print(df)
print(dff)
df = lambdify(x, df)
dff = lambdify(x,dff)

x=1
k=1
xk=x

while(k<8):
  print(f"Iteration {k}")
  dfxk =round(df(xk),3)
  dffxk =round(dff(xk),3)
  xk1 = round(xk-dfxk/dffxk,3)
  dfxk1 = round( df(xk1),3)
  print(xk,dfxk,dffxk,xk1,dfxk1)
  k+=1
  xk = xk1
```

    x**2 + 54/x
    2*x - 54/x**2
    2 + 108/x**3
    Iteration 1
    1 -52.0 110.0 1.473 -21.942
    Iteration 2
    1.473 -21.942 35.792 2.086 -8.238
    Iteration 3
    2.086 -8.238 13.898 2.679 -2.166
    Iteration 4
    2.679 -2.166 7.617 2.963 -0.225
    Iteration 5
    2.963 -0.225 6.152 3.0 0.0
    Iteration 6
    3.0 0.0 6.0 3.0 0.0
    Iteration 7
    3.0 0.0 6.0 3.0 0.0

```
# Bisection Method
import sympy as sp
from sympy import *
x= sp.Symbol('x')

f = lambda x: x**2+54/x

df = sp.diff(f(x),x)
print(f(x))
print(df)
df = lambdify(x, df)

a = 2
b = 5
x1 = a
x2 = b
i=0
while(i<3):
  i+=1
  print(f"Iteration {i}")

  z = round((x1+x2)/2,3)
  dfz =round(df(z),3)
  print(x1,x2)
  print(z,dfz)
  if(dfz>0):
    print(f"eliminate the right part (the region ({z},{x2}))")
    x2 = z
  elif(dfz<0):
    print(f"eliminate the left part (the region ({x1},{z}))")
    x1 = z

print(x1,x2)
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
y= np.linspace(a,b)
plt.plot(y,f(y))
#plt.plot(y,df(y))
```

```
# Secant Method
import sympy as sp
from sympy import *
x= sp.Symbol('x')

f = lambda x: x**2+54/x

df = sp.diff(f(x),x)
print(f(x))
print(df)
df = lambdify(x, df)

a = 2
b = 5
x1 = a
x2 = b
i=0
while(i<5):
  i+=1
  print(f"Iteration {i}")
  dfa = round(df(x1),4)
  dfb = round(df(x2),4)
  z = round(x2 - dfb/((dfb-dfa)/(x2-x1)),4)
  dfz =round(df(z),4)
  print(x1,x2)
  print(dfa,dfb,z,dfz)
  if(dfz>0):
    print(f"eliminate the right part (the region ({z},{x2}))")
    x2 = z

%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
y= np.linspace(a,b)
plt.plot(y,f(y))
#plt.plot(y,df(y))
```

```
#cubic search method

import sympy as sp
from sympy import *
x= sp.Symbol('x')

f = lambda x: 2*x**2+16/x

df = sp.diff(f(x),x)
print(f(x))
print(df)
df = lambdify(x, df)

d=0.5
x1=1

k=0
i=0
while(i<5):
  print(f"k = {k}")
  i+=1
  print(f"x{k} = {x1}")
  df1=df(x1)
  print(f"df({k}) = {df1}")
  if(df(x1)>0):
    print("\n")
  elif(df(x1)<0):
    d=0.5
    x2=x1+(2**k)*d
    x1=x2
    k+=1
    print(f"x{k} = {x2}")
    print("\n")

```

    2*x**2 + 16/x
    4*x - 16/x**2
    k = 0
    x0 = 1
    df(0) = -12.0
    x1 = 1.5

    k = 1
    x1 = 1.5
    df(1) = -1.1111111111111107
    x2 = 2.5

    k = 2
    x2 = 2.5
    df(2) = 7.4399999999999995

    k = 2
    x2 = 2.5
    df(2) = 7.4399999999999995

    k = 2
    x2 = 2.5
    df(2) = 7.4399999999999995