```python
import numpy as np
from matplotlib import pyplot as plt

import sympy as smp

from sklearn.base import BaseEstimator, RegressorMixin
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import BaggingRegressor
```

```python
def generate_problem(n, p, q, sigma, dist='identity', dist_params=None):

    if dist_params is None:
        dist_params = {}

    if dist == 'identity':
        Sig = np.eye(p)
        X = np.random.randn(n, p + q)
        X_test = np.random.randn(n, p + q)
    elif dist == 'rho':
        rho = dist_params['rho'] if 'rho' in dist_params else 0.5
        Sig = (1 - rho) * np.eye(p) + rho * np.ones((p, p))
        _, Q = np.linalg.eigh(Sig)
        X = np.random.randn(n, p + q) @ Q.T
        X_test = np.random.randn(n, p + q) @ Q.T
    elif dist == 'logspace':
        lowvar = dist_params['lowvar'] if 'lowvar' in dist_params else 1e-4
        Q = dist_params['Q'] if 'Q' in dist_params else None
        variances = np.random.permutation(np.logspace(0, np.log10(lowvar), p + q))
        stds = np.sqrt(variances)
        if Q is None:
            Sig = np.diag(variances[:p])
            X = np.random.randn(n, p + q) * stds[None, :]
            X_test = np.random.randn(n, p + q) * stds[None, :]
        else:
            if Q == 'random':
                A = np.random.randn(p, p)
                _, Q = np.linalg.eigh(A + A.T)
            Sig = Q @ (variances[:p, None] * Q.T)
            X = (np.random.randn(n, p + q) * stds[None, :]) @ Q.T
            X_test = (np.random.randn(n, p + q) * stds[None, :]) @ Q.T

    beta = np.random.randn(p) / np.sqrt(p)
    y = X[:, :p] @ beta + sigma * np.random.randn(n)

    return X, y, beta, X_test, Sig

def risk(beta, beta_hat, Sig=None, X_test=None):

    if Sig is None:
        if X_test is None:
            raise ValueError('Data covariance or test data must be provided.')
        n, p = X_test.shape
        Sig = X_test.T @ X_test / n

    beta_delta = beta - beta_hat
    return beta_delta @ (Sig @ beta_delta)

def ensemble_risk_ij(intersection_size, complement_intersection_size, gamma, sigma, eta=1):

    return complement_intersection_size * (1 + intersection_size * gamma / (eta - intersection_size * gamma)) + intersection_size * gamma * sigma**2 / (eta - intersection_size * gamma)

def ensemble_risk(alpha, gamma, sigma, k=None, eta=1):

    if k is None:
        return ensemble_risk_ij(alpha**2, (1 - alpha)**2, gamma, sigma, eta=1)
    else:
        return (1 - 1/k) * ensemble_risk_ij(alpha**2, (1 - alpha)**2, gamma, sigma, eta=1) + 1/k * ensemble_risk_ij(alpha, 1-alpha, gamma, sigma, eta=eta)

def ensemble_optimal_alpha(gamma, sigma, k=None, eta=None):

    if k is None:

        a = 1
        b = - (1 / gamma + 1 + sigma**2)
        c = 1 / gamma
        alpha = (-b - np.sqrt(b**2 - 4 * a * c)) / (2 * a)

        return alpha

    else:
        if eta is None:
            eta = 1

        alpha = smp.symbols('a', real=True)
        r = ensemble_risk(alpha, gamma, sigma, k, eta)
        drdalpha = smp.diff(r, alpha)
        numerator, _ = smp.fraction(smp.simplify(drdalpha))
        coeffs = smp.Poly(numerator).coeffs()
        roots = [np.abs(r) for r in np.roots(coeffs) if np.angle(r) < 1e-2 and 0 <= np.abs(r) <= 1/gamma]
        roots += [0, 1/gamma - 1e-6]

        return min(roots, key=lambda x: ensemble_risk(x, gamma, sigma, k, eta))

def ridge_optimal_risk(gamma, sigma):

    dobriban = 1 / 2 * (1 + (gamma - 1) / gamma / sigma**2 + np.sqrt((1 - (gamma - 1) / gamma / sigma**2)**2 + 4 / sigma**2))
    return (dobriban - 1) * sigma**2

class EnsembleOLS(BaggingRegressor):

    def __init__(self, fit_intercept=False, *args, **kwargs):

        self.fit_intercept = fit_intercept

        kwargs.update({
            'base_estimator': LinearRegression(fit_intercept=fit_intercept),
            'bootstrap': False,
            'bootstrap_features': False
        })
        super().__init__(*args, **kwargs)

    def get_coefs(self):

        p = self.n_features_
        beta = np.zeros(p)

        for features, estimator in zip(self.estimators_features_, self.estimators_):

            beta[features] += estimator.coef_ / len(self.estimators_)

        return beta
```

```python
n = 200
gammas = [0.5, 1, 2]
sigma = 1
dist = 'identity'
dist_params = None
n_trials = 50
ks = np.unique(np.logspace(0, 3, dtype=int))

risks_ridge = np.zeros(len(gammas))
risks_ensemble = np.zeros((len(gammas), n_trials, len(ks)))
risks_ensemble_eta = np.zeros((len(gammas), n_trials, len(ks)))

for t, gamma in enumerate(gammas):

    print(gamma, end='')

    p = int(gamma*n)

    alpha = ensemble_optimal_alpha(gamma, sigma)
    risks_ridge[t] = ridge_optimal_risk(gamma, sigma)
    r_ensemble = ensemble_risk(alpha, gamma, sigma)

    eta = min(1.0, 1.1*alpha*gamma)

    for i in range(n_trials):
        print('.', end='')

        X, y, beta, X_test, Sig = generate_problem(n, p, 0, sigma, dist, dist_params)

        ens_ols = EnsembleOLS(n_estimators=1, warm_start=True, max_samples=1.0, max_features=alpha, n_jobs=-1)
        ens_ols_eta = EnsembleOLS(n_estimators=1, warm_start=True, max_samples=eta, max_features=alpha, n_jobs=-1)

        for j, k in enumerate(ks):
            ens_ols.n_estimators = k
            ens_ols.fit(X, y)
            beta_hat = ens_ols.get_coefs()
            risks_ensemble[t, i, j] = risk(beta, beta_hat, Sig)

            ens_ols_eta.n_estimators = k
            ens_ols_eta.fit(X, y)
            beta_hat = ens_ols_eta.get_coefs()
            risks_ensemble_eta[t, i, j] = risk(beta, beta_hat, Sig)

    print('')

np.savez('risk_versus_k.npz', gammas=gammas, ks=ks, risks_ensemble=risks_ensemble, risks_ensemble_eta=risks_ensemble_eta, risks_ridge=risks_ridge)
```

    0.5..................................................
    1..................................................
    2..................................................

```python
colors = plt.rcParams['axes.prop_cycle'].by_key()['color']

plt.figure(dpi=100)

for t, gamma in enumerate(gammas):
    plt.loglog(ks, risks_ensemble[t, :, :].mean(0), color=colors[t], label=r'$\gamma=%g$' % gamma)
    plt.plot(ks, risks_ensemble_eta[t, :, :].mean(0), ':', color=colors[t])
    plt.plot(ks, [risks_ridge[t]] * len(ks), '--', color=colors[t])

plt.legend()
plt.show()
```

```python
n = 200
gamma = 2
p = int(gamma*n)
sigmas = np.logspace(-1, 1, 21)
dist = 'identity'
dist_params = None
ks = [20, 100, 500]
n_trials = 100

risks_ridge = np.zeros(len(sigmas))
risks_ensemble = np.zeros((len(sigmas), n_trials, len(ks)))

for t, sigma in enumerate(sigmas):

    print(sigma, end='')

    alpha = ensemble_optimal_alpha(gamma, sigma)
    risks_ridge[t] = ridge_optimal_risk(gamma, sigma)

    for i in range(n_trials):

        print('.', end='')

        for j, k in enumerate(ks):
            X, y, beta, X_test, Sig = generate_problem(n, p, 0, sigma, dist, dist_params)
            ens_ols = EnsembleOLS(n_estimators=k, warm_start=False, max_samples=1.0, max_features=alpha, n_jobs=-1)
            ens_ols.fit(X, y)
            beta_hat = ens_ols.get_coefs()
            risks_ensemble[t, i, j] = risk(beta, beta_hat, Sig)

    print('')

np.savez('risk_versus_sigma.npz', sigmas=sigmas, ks=ks, risks_ensemble=risks_ensemble, risks_ridge=risks_ridge)
```

    0.1....................................................................................................
    0.12589254117941673....................................................................................................
    0.15848931924611134....................................................................................................
    0.19952623149688797....................................................................................................
    0.251188643150958....................................................................................................
    0.31622776601683794....................................................................................................
    0.3981071705534973....................................................................................................
    0.5011872336272724....................................................................................................
    0.6309573444801934....................................................................................................
    0.7943282347242815....................................................................................................
    1.0....................................................................................................
    1.2589254117941675....................................................................................................
    1.584893192461114....................................................................................................
    1.9952623149688797....................................................................................................
    2.511886431509581....................................................................................................
    3.1622776601683795....................................................................................................
    3.981071705534973....................................................................................................
    5.011872336272725....................................................................................................
    6.309573444801933....................................................................................................
    7.943282347242818....................................................................................................
    10.0....................................................................................................

```python
colors = plt.rcParams['axes.prop_cycle'].by_key()['color']

plt.figure(dpi=100)

for j, k in enumerate(ks):
    plt.loglog(sigmas, risks_ensemble[:, :, j].mean(1), color=colors[j], label='$k=%d$' % k)
plt.plot(sigmas, risks_ridge, '--', color='black')

plt.legend()
plt.show()
```

```python
n = 200
gamma = 2
p = int(gamma*n)
sigmas = np.logspace(-1, 1, 21)
dist = 'identity'
dist_params = None
ks = [20, 100, 500]
n_trials = 100

risks_ridge = np.zeros(len(sigmas))
risks_ensemble = np.zeros((len(sigmas), n_trials, len(ks)))

for j, k in enumerate(ks):
    print(k)

    for t, sigma in enumerate(sigmas):

        print(sigma, end='')

        alpha = ensemble_optimal_alpha(gamma, sigma, k)
        if alpha*p < 1:
            alpha = 1
        risks_ridge[t] = ridge_optimal_risk(gamma, sigma)

        for i in range(n_trials):

            print('.', end='')

            X, y, beta, X_test, Sig = generate_problem(n, p, 0, sigma, dist, dist_params)

            ens_ols = EnsembleOLS(n_estimators=k, warm_start=False, max_samples=1.0, max_features=alpha, n_jobs=-1)

            ens_ols.fit(X, y)
            beta_hat = ens_ols.get_coefs()
            risks_ensemble[t, i, j] = risk(beta, beta_hat, Sig)

        print('')

np.savez('risk_versus_sigma_optimal_alpha.npz', sigmas=sigmas, ks=ks, risks_ensemble=risks_ensemble, risks_ridge=risks_ridge)
```

    20
    0.1....................................................................................................
    0.12589254117941673....................................................................................................
    0.15848931924611134....................................................................................................
    0.19952623149688797....................................................................................................
    0.251188643150958....................................................................................................
    0.31622776601683794....................................................................................................
    0.3981071705534973....................................................................................................
    0.5011872336272724....................................................................................................
    0.6309573444801934....................................................................................................
    0.7943282347242815....................................................................................................
    1.0....................................................................................................
    1.2589254117941675....................................................................................................
    1.584893192461114....................................................................................................
    1.9952623149688797....................................................................................................
    2.511886431509581....................................................................................................
    3.1622776601683795....................................................................................................
    3.981071705534973....................................................................................................
    5.011872336272725....................................................................................................
    6.309573444801933....................................................................................................
    7.943282347242818....................................................................................................
    10.0....................................................................................................
    100
    0.1....................................................................................................
    0.12589254117941673....................................................................................................
    0.15848931924611134....................................................................................................
    0.19952623149688797....................................................................................................
    0.251188643150958....................................................................................................
    0.31622776601683794....................................................................................................
    0.3981071705534973....................................................................................................
    0.5011872336272724....................................................................................................
    0.6309573444801934....................................................................................................
    0.7943282347242815....................................................................................................
    1.0....................................................................................................
    1.2589254117941675....................................................................................................
    1.584893192461114....................................................................................................
    1.9952623149688797....................................................................................................
    2.511886431509581....................................................................................................
    3.1622776601683795....................................................................................................
    3.981071705534973....................................................................................................
    5.011872336272725....................................................................................................
    6.309573444801933....................................................................................................
    7.943282347242818....................................................................................................
    10.0....................................................................................................
    500
    0.1....................................................................................................
    0.12589254117941673....................................................................................................
    0.15848931924611134....................................................................................................
    0.19952623149688797....................................................................................................
    0.251188643150958....................................................................................................
    0.31622776601683794....................................................................................................
    0.3981071705534973....................................................................................................
    0.5011872336272724....................................................................................................
    0.6309573444801934....................................................................................................
    0.7943282347242815....................................................................................................
    1.0....................................................................................................
    1.2589254117941675....................................................................................................
    1.584893192461114....................................................................................................
    1.9952623149688797....................................................................................................
    2.511886431509581....................................................................................................
    3.1622776601683795....................................................................................................
    3.981071705534973....................................................................................................
    5.011872336272725....................................................................................................
    6.309573444801933....................................................................................................
    7.943282347242818....................................................................................................
    10.0....................................................................................................

```python
colors = plt.rcParams['axes.prop_cycle'].by_key()['color']

plt.figure(dpi=100)

for j, k in enumerate(ks):
    plt.loglog(sigmas, risks_ensemble[:, :, j].mean(1), color=colors[j], label='$k=%d$' % k)
plt.plot(sigmas, risks_ridge, '--', color='black')

plt.legend()
plt.show()
```

```python

```


# Modelización de Procesos Cognitivos: Bases de Sympy y Conjuntos

## Conjuntos

Hagamos operaciones de conjuntos

```python
# Definamos primero los dos conjuntos a trabajar
c1 = {3,6,2,7,8}
c2 = {1,2,6,3}
```

```python
#Trabajemos primero con la unión de conjuntos que se denota por el operador |
c1|c2
```

    {1, 2, 3, 6, 7, 8}

```python
# De manera opcional se puede emplear el comando print
print(c1|c2)
```

    {1, 2, 3, 6, 7, 8}

```python
# Para el caso de la intersección de conjuntos se emplea el operador &
c1&c2
```

    {2, 3, 6}

```python
# la diferencia de conjuntos se denota por -, que son los elemetos que están en A pero no están en B
c1-c2
```

    {7, 8}

Se puede también trabajar con caracteres

```python
# por ejemplo los conjuntos de nombres de personas
a1 = {'juan','jenny','sandra','noe'}
a2 = {'jose', 'rita', 'juan', 'ramon', 'sandra'}
```

```python
print(a1|a2)
print(a1&a2)
print(a1-a2)
```

    {'juan', 'jenny', 'jose', 'sandra', 'ramon', 'noe', 'rita'}
    {'juan', 'sandra'}
    {'noe', 'jenny'}

## Calculo con Sympy

```python
from __future__ import division
from sympy import * # Para importar todas las funciones de Sympy
x, y, z, t = symbols('x y z t')
k, m, n = symbols('k m n', integer = True)
f, g, h = symbols('f g h', cls = Function)
```

Podemos ocupar Sympy para resolver ecuaciones como $x^2 + 9 x - 70 = 0$

```python
solve(x**2 + 9*x - 70, x) # se resuelve la ecuación para x
```

    [-14, 5]

Se pueden factorizar ecuaciones como $x^2 - x - 6$

```python
factor(x**2 - x - 6)
```

$\displaystyle \left(x - 3\right) \left(x + 2\right)$

Se pueden expandir los términos

```python
expr1 = (3*x**2 + 5*x)*(x - 2)
expand(expr1)
```

$\displaystyle 3 x^{3} - x^{2} - 10 x$

Se pueden resolver sistemas de ecuaciones: por ejemplo, las ecuaciones $x+3y=4$ y $2x+5y=5$

```python
solve([x + 3*y - 4, 2*x + 5*y -5],[x,y])
```

    {x: -5, y: 3}

Se puede trabajar con polinomios

```python
P = (x - 3)*(x - 4)*(x - 6)
P
```

$\displaystyle \left(x - 6\right) \left(x - 4\right) \left(x - 3\right)$

```python
P.expand()
```

$\displaystyle x^{3} - 13 x^{2} + 54 x - 72$

```python
P.factor()
```

$\displaystyle \left(x - 6\right) \left(x - 4\right) \left(x - 3\right)$

Se pueden extraer las raíces (soluciones) del polinomio

```python
raices = solve(P,x)
raices
```

    [3, 4, 6]

Se pueden definir funciones por ejemplo: $f(x) = x^2$

```python
def f(x):
    return x**2 # definimos la función x^2
```

```python
f(2) # se sustituye el valor de interes
```

    4

```python
f(x)
```

$\displaystyle x^{2}$

Podemos trabajar con límites dentro de Sympy

```python
limit(x**2, x, 0)
```

$\displaystyle 0$

```python
limit(1/x, x, 0)
```

$\displaystyle \infty$

```python
limit(1/x, x, oo)
```

$\displaystyle 0$

Y se pueden evaluar límites lateral

```python
print(limit(x/abs(x), x, 0, dir='+'))
print(limit(x/abs(x), x, 0, dir='-'))
```

    1
    -1

```python
p1 = plot(x/abs(x),show=False)
p1.show()
```

Se pueden obtener las derivadas de la funciones por medio del comando **diff(f(x), x)**. Así por ejemplo, para la función lineal $f(x) = x$

```python
def f(x):
    return x # se define la función f(x) = x
```

```python
diff(f(x), x) # se calcula la respectiva derivada
```

$\displaystyle 1$

```python
diff(x, x) # o simplemente calcular la derivada de la expresión sin necesidad de definir la función
```

$\displaystyle 1$

```python
diff(x**k, x)
```

$\displaystyle \frac{k x^{k}}{x}$

```python

```


# Constrained Optimization

```python
import numpy as np
from scipy.linalg import cholesky, solve_triangular, cho_solve, cho_factor
from scipy.linalg import solve
from scipy.optimize import minimize
```

### Lagrange multipliers and constrained optimization

Recall why Lagrange multipliers are useful for constrained optimization - a stationary point must be where the constraint surface $g$ touches a level set of the function $f$ (since the value of $f$ does not change on a level set). At that point, $f$ and $g$ are parallel, and hence their gradients are also parallel (since the gradient is normal to the level set). So we want to solve

$$\nabla f = -\lambda \nabla g$$

or equivalently,

$$\nabla f + \lambda \nabla g = 0$$

## Example 1

Maximize $f (x, y, z) = xy + yz$ subject to the constraints $x + 2y = 6$ and $x − 3z = 0$.

We set up the equations

$$
F (x, y, z, λ, μ) = xy + yz − λ(x + 2y − 6) − μ(x − 3z)
$$

Now set partial derivatives to zero and solve the following set of equations

\begin{align}
y - \lambda - \mu &= 0 \\
x + z - 2\lambda &= 0 \\
y +3\mu &= 0 \\
x + 2y - 6 &= 0 \\
x - 3z &= 0
\end{align}

which is a linear equation in $x, y, z, \lambda, \mu$

$$
\begin{pmatrix}
0 & 1 & 0 & -1 & -1 \\
1 & 0 & 1 & -2 & 0 \\
0 & 1 & 0 & 0 & 3 \\
1 & 2 & 0 & 0 & 0 \\
1 & 0 & -3 & 0 & 0 \\
\end{pmatrix}\pmatrix{x \\ y \\ z \\ \lambda \\ \mu} = \pmatrix{0 \\ 0 \\ 0 \\ 6 \\ 0}
$$

```python
A = np.array([
    [0, 1, 0, -1, -1],
    [1, 0, 1, -2, 0],
    [0, 1, 0, 0, 3],
    [1, 2, 0, 0, 0],
    [1, 0,-3, 0, 0]])

b = np.array([0,0,0,6,0])

sol = solve(A, b)
```

```python
sol
```

    array([ 3. ,  1.5,  1. ,  2. , -0.5])

```python
def f(x, y, z):
    return x*y + y*z
```

```python
f(*sol[:3])
```

    6.0

### Using `scipy.optimize`

We first need to set this as a minimization problem.

```python
def f(x):
    return -(x[0]*x[1] + x[1]*x[2])
```

```python
cons = ({'type': 'eq',
         'fun' : lambda x: np.array([x[0] + 2*x[1] - 6, x[0] - 3*x[2]])})
```

```python
x0 = np.array([2,2,0.67])
res = minimize(f, x0, constraints=cons)
```

    array([2.99999979, 1.50000011, 0.99999993])

```python
res.fun
```

    -5.999999999999969

```python
res.x
```

    array([2.99999979, 1.50000011, 0.99999993])

## Example 2

Find the minimum of the following quadratic function on $\mathbb{R}^2$

$$f(x) = x^TAx +b^Tx +c$$
where
$$A = \left(\begin{matrix}13&5\\5&7\end{matrix}\right), b = \left(\begin{matrix}1\\1\end{matrix}\right) \textrm {and } c = 2$$

Under the constraints:
$$g(x) = 2x_1-5x_2=2 \;\;\;\;\;\; \textrm{ and } \;\;\;\;\;\; h(x) = x_1+x_2=1$$

1. Use a matrix decomposition method to find the minimum of the *unconstrained* problem without using `scipy.optimize` (Use library functions - no need to code your own). Note: for full credit you should exploit matrix structure.
2. Find the solution using constrained optimization with the `scipy.optimize` package.
2. Use Lagrange multipliers and solving the resulting set of equations directly without using `scipy.optimize`.

### Solve unconstrained problem

To find the minimum, we differentiate $f(x)$ with respect to $x^T$ and set it equal to $0$. We thus need to solve

$$
2Ax + b = 0
$$

or

$$
Ax = -\frac{b}{2}
$$

We see that $A$ is a symmetric positive definite real matrix. Hence we use Cholesky factorization.

Steps

$$
L = \text{cholesky}(A) \\
\text{solve } Ly = b \\
\text{solve } L^Tx = y
$$

```python
A = np.array([
    [13,5],
    [5,7]
])
b = np.array([1,1]).reshape(-1,1)
c = 2
```

```python
L = cholesky(A, lower=True)
y = solve_triangular(L, -b/2, lower=True)
x = solve_triangular(L.T, y, lower=False)
```

```python
x
```

    array([[-0.01515152],
           [-0.06060606]])

#### Short cut

```python
cho_solve(cho_factor(A), -b/2)
```

    array([[-0.01515152],
           [-0.06060606]])

### Solve constrained problem using `scipy.optimize`

```python
f = lambda x, A, b, c: x.T @ A @ x + b.T @ x + c
```

```python
from scipy.optimize import minimize
```

```python
cons = ({'type': 'eq', 'fun': lambda x: 2*x[0] - 5*x[1] - 2},
        {'type': 'eq', 'fun': lambda x: x[0] + x[1] - 1})

res = minimize(f, [0,0], constraints=cons, args=(A, b, c))
res.x
```

    array([1.00000000e+00, 3.41607086e-16])

### Solve constrained problem using Lagrange multipliers

We set up the equations

$$
F(x_1, x_2, \lambda, \mu) = f + \lambda g + \mu h
$$

Sometimes this is written as

$$
F(x_1, x_2, \lambda, \mu) = f - \lambda g - \mu h
$$

All this means is a final sign change in the estimated values of $\lambda$ and $\mu$.

We show the original equations for convenience

$$f(x) = x^TAx +b^Tx +c$$

where

$$A = \left(\begin{matrix}13&5\\5&7\end{matrix}\right), b = \left(\begin{matrix}1\\1\end{matrix}\right) \textrm {and } c = 2$$

Under the constraints:

$$g(x) = 2x_1-5x_2=2 \;\;\;\;\;\; \textrm{ and } \;\;\;\;\;\; h(x) = x_1+x_2=1$$

To make the calculations explicit, we rewrite $F$ as

$$
F = 13x_1^2 + 10xy_1x_2+ 7x_2^2 + x_1 + x_2 +2 + \lambda(2x_1 - 5x_2 -2) + \mu(x_1 + x_2 -1)
$$

Taking partial derivatives, we get

$$
\begin{align}
\frac{\delta F}{\delta x_1} &=& 26 x_1 + 10 x_2 + 1 + 2\lambda + \mu &= 0 \\
\frac{\delta F}{\delta x_2} &=& 10 x_1 + 14 x_2 + 1 - 5\lambda + \mu &= 0 \\
\frac{\delta F}{\delta \lambda} &=& 2 x_1 - 5 x_2 -2 &= 0 \\
\frac{\delta F}{\delta \mu} &=& x_1 + x_2 - 1 &= 0
\end{align}
$$

Plugging in the numbers and expressing in matrix form, we get

$$
\begin{bmatrix}
26 & 10 & 2 & 1 \\
10 & 14 & -5 & 1 \\
2 & -5 & 0 & 0 \\
1 & 1 & 0 & 0
\end{bmatrix} \begin{bmatrix}
x_1 \\
x_2 \\
\lambda \\
\mu
\end{bmatrix} = \begin{bmatrix}
-1 \\
-1 \\
2 \\
1
\end{bmatrix}
$$

With a bit of practice, you can probably just write the matrix directly by inspection.

```python
A = np.array([
    [26, 10, 2, 1],
    [10, 14, -5, 1],
    [2, -5, 0, 0],
    [1, 1, 0, 0]
])
b = np.array([-1, -1, 2, 1]).reshape(-1,1)
```

```python
solve(A, b)
```

    array([[ 1.00000000e+00],
           [-4.37360585e-17],
           [-2.28571429e+00],
           [-2.24285714e+01]])

```python

```


###### Content under Creative Commons Attribution license CC-BY 4.0, code under MIT license (c)2014 L.A. Barba, G.F. Forsyth, C. Cooper. Based on [CFDPython](https://github.com/barbagroup/CFDPython), (c)2013 L.A. Barba, also under CC-BY license.

# Space & Time

## Stability and the CFL condition

Welcome back! This is the second IPython Notebook of the series *Space and Time — Introduction to Finite-difference solutions of PDEs*, the second module of ["Practical Numerical Methods with Python"](http://openedx.seas.gwu.edu/courses/GW/MAE6286/2014_fall/about).

In the first lesson of this series, we studied the numerical solution of the linear and non-linear convection equations, using the finite-difference method. Did you experiment there using different parameter choices? If you did, you probably ran into some unexpected behavior. Did your solution ever blow up (sometimes in a cool way!)?

In this IPython Notebook, we will explore why changing the discretization parameters can affect your solution in such a drastic way.

With the solution parameters we initially suggested, the spatial grid had 41 points and the timestep was 0.25.  Now, we're going to experiment with the number of points in the grid.  The code below corresponds to the linear convection case, but written into a function so that we can easily examine what happens as we adjust just one variable: **the grid size**.

```
import numpy
import matplotlib.pyplot as plt
%matplotlib inline
from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16
```

```
def linearconv(nx):
    """Solve the linear convection equation.

    Solves the equation d_t u + c d_x u = 0 where
    * the wavespeed c is set to 1
    * the domain is x \in [0, 2]
    * 20 timesteps are taken, with \Delta t = 0.025
    * the initial data is the hat function

    Produces a plot of the results

    Parameters
    ----------

    nx : integer
        number of internal grid points

    Returns
    -------

    None : none
    """
    dx = 2./(nx-1)
    nt = 20
    dt = .025
    c = 1

    u = numpy.ones(nx)
    u[.5/dx : 1/dx+1]=2

    un = numpy.ones(nx)

    for n in range(nt):
        un = u.copy()
        u[1:] = un[1:] -c*dt/dx*(un[1:] -un[0:-1])
        u[0] = 1.0

    plt.plot(numpy.linspace(0,2,nx), u, color='#003366', ls='--', lw=3)
    plt.ylim(0,2.5);
```

Now let's examine the results of the linear convection problem with an increasingly fine mesh.  We'll try 41, 61 and 71 points ... then we'll shoot for 85. See what happens:

```
linearconv(41) #convection using 41 grid points
```

```
linearconv(61)
```

```
linearconv(71)
```

So far so good—as we refine the spatial grid, the wave is more square, indicating that the discretization error is getting smaller. But what happens when we refine the grid even further? Let's try 85 grid points.

```
linearconv(85)
```

Oops. This doesn't look anything like our original hat function. Something has gone awry. It's the same code that we ran each time, so it's not a bug!

### What happened?

To answer that question, we have to think a little bit about what we're actually implementing in code when we solve the linear convection equation with the forward-time/backward-space method.

In each iteration of the time loop, we use the existing data about the solution at time $n$ to compute the solution in the subsequent time step, $n+1$.  In the first few cases, the increase in the number of grid points returned more accurate results.  There was less discretization error and the translating wave looked more like a square wave than it did in our first example.

Each iteration of the time loop advances the solution by a time-step of length $\Delta t$, which had the value 0.025 in the examples above. During this iteration, we evaluate the solution $u$ at each of the $x_i$ points on the grid.  But in the last plot, something has clearly gone wrong.

What has happened is that over the time period $\Delta t$, the wave is travelling a distance which is greater than `dx`, and we say that the solution becomes *unstable* in this situation.  The length `dx` of grid spacing is inversely proportional to the number of total points `nx`: we asked for more grid points, so `dx` got smaller. Once `dx` got smaller than the $c\Delta t$—the distance travelled by the numerical solution in one time step—it's no longer possible for the numerical scheme to solve the equation correctly!

####Graphical interpretation of the CFL condition.

Consider the illustration above. When the distance $c\Delta t$, covered by the numerical solution in one time step, is smaller than $\Delta x$, then the information about the solution in the previous step is contained within the green triangle, representing the _domain of dependence_ of the numerical scheme.

If $\Delta x$ is smaller than $c\Delta t$, then the information about the solution needed for $u_i^{n+1}$ is not available in the numerical scheme, because the characteristic line traced from the grid coordinate $i, n+1$ lands _behind_ the point $i-1$ on the grid. But information on the solution there is not available from the numerical scheme!

The following condition ensures that the domain of dependence of the differential equation should be contained in the _numerical_ domain of dependence:

\begin{equation}\sigma = \frac{c \Delta t}{\Delta x} \leq 1
\end{equation}

Stability of the numerical solution can be enforced if the step size `dt` is calculated with respect to the size of `dx` to satisfy the condition above.

The value of $c\Delta t/\Delta x$ is called the **Courant-Friedrichs-Lewy number** (CFL number), often denoted by $\sigma$. The value $\sigma_{\text{max}}$ that will ensure stability depends on the discretization used; for the forward-time/backward-space scheme, the condition for stability is $\sigma<1$.

In a new version of our code—written _defensively_—, we'll use the CFL number to calculate the appropriate time-step `dt` depending on the size of `dx`.

```
def linearconv(nx):
    """Solve the linear convection equation.

    Solves the equation d_t u + c d_x u = 0 where
    * the wavespeed c is set to 1
    * the domain is x \in [0, 2]
    * 20 timesteps are taken, with \Delta t computed using the CFL 0.5
    * the initial data is the hat function

    Produces a plot of the results

    Parameters
    ----------

    nx : integer
        number of internal grid points

    Returns
    -------

    None : none
    """
    dx = 2./(nx-1)
    nt = 20
    c = 1
    sigma = .5

    dt = sigma*dx

    u = numpy.ones(nx)
    u[.5/dx : 1/dx+1]=2

    un = numpy.ones(nx)

    for n in range(nt):
        un = u.copy()
        u[1:] = un[1:] -c*dt/dx*(un[1:] -un[0:-1])
        u[0] = 1.0

    plt.plot(numpy.linspace(0,2,nx), u, color='#003366', ls='--', lw=3)
    plt.ylim(0,2.5);
```

Now, it doesn't matter how many points we use for the spatial grid: the solution will always be stable!

```
linearconv(41)
```

```
linearconv(61)
```

```
linearconv(81)
```

```
linearconv(101)
```

```
linearconv(121)
```

Notice that as the number of points `nx` increases, the wave convects a shorter and shorter distance.  The number of time iterations we have advanced the solution to is held constant at `nt = 20`, but depending on the value of `nx` and the corresponding values of `dx` and `dt`, a shorter time window is being examined overall.

---

######The cell below loads the style of the notebook.

```
from IPython.core.display import HTML
css_file = '../../styles/numericalmoocstyle.css'
HTML(open(css_file, "r").read())
```

<link href='http://fonts.googleapis.com/css?family=Alegreya+Sans:100,300,400,500,700,800,900,100italic,300italic,400italic,500italic,700italic,800italic,900italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Arvo:400,700,400italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=PT+Mono' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Shadows+Into+Light' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Nixie+One' rel='stylesheet' type='text/css'>
<style>

@font-face {
    font-family: "Computer Modern";
    src: url('http://mirrors.ctan.org/fonts/cm-unicode/fonts/otf/cmunss.otf');
}

#notebook_panel { /* main background */
    background: rgb(245,245,245);
}

div.cell { /* set cell width */
    width: 750px;
}

div #notebook { /* centre the content */
    background: #fff; /* white background for content */
    width: 1000px;
    margin: auto;
    padding-left: 0em;
}

#notebook li { /* More space between bullet points */
margin-top:0.8em;
}

/* draw border around running cells */
div.cell.border-box-sizing.code_cell.running {
    border: 1px solid #111;
}

/* Put a solid color box around each cell and its output, visually linking them*/
div.cell.code_cell {
    background-color: rgb(256,256,256);
    border-radius: 0px;
    padding: 0.5em;
    margin-left:1em;
    margin-top: 1em;
}

div.text_cell_render{
    font-family: 'Alegreya Sans' sans-serif;
    line-height: 140%;
    font-size: 125%;
    font-weight: 400;
    width:600px;
    margin-left:auto;
    margin-right:auto;
}

/* Formatting for header cells */
.text_cell_render h1 {
    font-family: 'Nixie One', serif;
    font-style:regular;
    font-weight: 400;
    font-size: 45pt;
    line-height: 100%;
    color: rgb(0,51,102);
    margin-bottom: 0.5em;
    margin-top: 0.5em;
    display: block;
}
.text_cell_render h2 {
    font-family: 'Nixie One', serif;
    font-weight: 400;
    font-size: 30pt;
    line-height: 100%;
    color: rgb(0,51,102);
    margin-bottom: 0.1em;
    margin-top: 0.3em;
    display: block;
}

.text_cell_render h3 {
    font-family: 'Nixie One', serif;
    margin-top:16px;
	font-size: 22pt;
    font-weight: 600;
    margin-bottom: 3px;
    font-style: regular;
    color: rgb(102,102,0);
}

.text_cell_render h4 {    /*Use this for captions*/
    font-family: 'Nixie One', serif;
    font-size: 14pt;
    text-align: center;
    margin-top: 0em;
    margin-bottom: 2em;
    font-style: regular;
}

.text_cell_render h5 {  /*Use this for small titles*/
    font-family: 'Nixie One', sans-serif;
    font-weight: 400;
    font-size: 16pt;
    color: rgb(163,0,0);
    font-style: italic;
    margin-bottom: .1em;
    margin-top: 0.8em;
    display: block;
}

.text_cell_render h6 { /*use this for copyright note*/
    font-family: 'PT Mono', sans-serif;
    font-weight: 300;
    font-size: 9pt;
    line-height: 100%;
    color: grey;
    margin-bottom: 1px;
    margin-top: 1px;
}

.CodeMirror{
        font-family: "PT Mono";
        font-size: 90%;
}

</style>


# Solution {-}

A classical problem in Wiener filter theory in one of separating signal from noise when both the signal and noise have exponential autocorrelation functions.

Autocorrelation:
\begin{equation}
  R_s(\tau)=\sigma_s^2 e^{-\beta_s |\tau|} \qquad
  R_n(\tau)=\sigma_n^2 e^{-\beta_n |\tau|}
\end{equation}

Spectral densities:
\begin{equation}
  S_s(j\omega)=\frac{2\sigma_s^2\beta_s}{\omega^2 + \beta_s^2} \qquad
  S_n(j\omega)=\frac{2\sigma_n^2\beta_n}{\omega^2 + \beta_n^2}
\end{equation}

Shaping filters:
\begin{equation}
  G_s(s)=\frac{\sqrt{2\sigma_s^2\beta_s}}{s + \beta_s} \qquad
  G_n(s)=\frac{\sqrt{2\sigma_n^2\beta_n}}{s + \beta_n}
\end{equation}

Differential equations:
\begin{equation}
  \dot{s} + \beta_s s = \sqrt{2\sigma_s^2\beta_s} u_s(t) \qquad
  \dot{n} + \beta_n n = \sqrt{2\sigma_n^2\beta_n} u_n(t)
\end{equation}

Matrix form:
\begin{equation}
  \begin{bmatrix}
    \dot{x_1}\\
    \dot{x_2}\\
  \end{bmatrix}
  =
  \begin{bmatrix}
    -\beta_s &0\\
    0& -\beta_n\\
  \end{bmatrix}
  \begin{bmatrix}
    x_1\\
    x_2\\
  \end{bmatrix}
  +
  \begin{bmatrix}
    \sqrt{2\sigma_s^2\beta_s}  &0\\
    0 &\sqrt{2\sigma_n^2\beta_n}\\
  \end{bmatrix}
  \begin{bmatrix}
    u_s(t)\\
    u_n(t)\\
  \end{bmatrix}
\end{equation}

```python
from sympy import Matrix, symbols, sqrt, eye, inverse_laplace_transform, integrate

s, t, beta_s, beta_n, sigma_s, sigma_n = symbols('s t beta_s beta_n sigma_s sigma_n', positive=True)

# Dynamic matrix
F = Matrix([[-beta_s, 0],
            [0, -beta_n]])

# Dynamic matrix
G = Matrix([[sqrt(2*sigma_s**2*beta_s), 0],
            [0, sqrt(2*sigma_n**2*beta_n)]])

# Transition matrix
phi = inverse_laplace_transform((s*eye(2)-F).inv(), s, t)
display(phi)

# Process noise matrix
Q = integrate(phi@G@G.T@phi.T, (t, 0, t))
display(Q)
```

$\displaystyle \left[\begin{matrix}e^{- \beta_{s} t} & 0\\0 & e^{- \beta_{n} t}\end{matrix}\right]$

$\displaystyle \left[\begin{matrix}\sigma_{s}^{2} - \sigma_{s}^{2} e^{- 2 \beta_{s} t} & 0\\0 & \sigma_{n}^{2} - \sigma_{n}^{2} e^{- 2 \beta_{n} t}\end{matrix}\right]$

## Simulation

```python
from numpy import array, exp, sqrt, eye, arange
from numpy.linalg import inv
from vanloan import numeval
import matplotlib.pyplot as plt

# System values
samples = 50
sigmas = 3
sigman = 1
betas = 0.1  # 1/second
betan = 1    # 1/second
dt = 1       # second

# Dynamics matrix
F = array([[-betas, 0],
           [0, -betan]])

# Spectral coefficients
G = array([[sqrt(2*sigmas**2*betas), 0],
           [0, sqrt(2*sigman**2*betan)]])

# Numerical evaluation
[phi, Q] = numeval(F, G, dt)

# Initial values
P = array([[sigmas**2, 0],
           [0, sigman**2]])

# Design matrix
H = array([[1, 1]])

# Measurement noise
R = array([[0]])

# Plot vectors
vars = []
varn = []

# Main loop
for i in range(0, samples):

    # Time update
    P = phi@P@phi.T + Q

    # Gain
    K = P@H.T@inv(H@P@H.T + R)

    # State covariance matrix update
    P = (eye(2) - K@H)@P@(eye(2) - K@H).T + K@R@K.T

    # Error variance
    vars.append(P[0, 0])
    varn.append(P[1, 1])

# Time
time = arange(0, samples)

# Plotting process
plt.figure(1)
plt.plot(time, vars, 'b', label='Signal')
plt.plot(time, varn, 'r', label='Noise')
plt.title('Wiener filter ')
plt.xlabel('Time')
plt.ylabel('Variance')
plt.legend(loc='lower right')
plt.grid(True, which='both')
plt.show()

```


# Expectation Maximization for the Gaussian Mixture Model

* Last class we introduced the Gaussian Mixture Model:
     * $p(x) = \sum_{k=1}^K \pi_k N(x | \mu_k, \Sigma_k)$  where $0 \le \pi_k \le 1$ and $\sum_k \pi_k = 1$

* Suppose we are given $X = \left\{ \mathbf{x}_1, \ldots, \mathbf{x}_N\right\}$ where each $\mathbf{x}_i$ is a sample from one of the $K$ Gaussians in our mixture model.  We want to estimate $\pi_k, \mu_k, \Sigma_k$ given $X$.
*  So, we want to maximize the following data likelihood:
\begin{equation}
\hat\Theta = argmax_\Theta \prod_{i=1}^N \sum_{k=1}^K \pi_k N(x | \mu_k, \Sigma_k)
\end{equation}
where $\Theta = \left\{ \pi_k, \mu_k, \Sigma_k \right\}_{k=1}^K$

* It is difficult to maximize! We should try to simpler version in which we add latent variables to simplify the problem (and apply EM).

*  The hidden/latent/missing variable we added was the label of the Gaussian from which $\mathbf{x}_i$ was drawn

\begin{equation}
\mathbf{x}, z \sim f(\mathbf{x},z|\theta)
\end{equation}

* The complete data likelihood is then:
\begin{eqnarray}
L^c &=& \prod_{i=1}^N p(\mathbf{x}_i | z_i, \theta)p(z_i)\\
&=& \prod_{i=1}^N N(\mathbf{x}_i| \mu_{z_i}, \theta_{z_i})\pi_{z_i}
\end{eqnarray}

* Since we do not know the $z_i$ values, we do not just guess one value - we average over all possible values for $z_i$.  In other words, we take the *expectation* of the complete likelihood with respect to $z$

\begin{eqnarray}
Q(\Theta, \Theta^t) &=& \mathbb{E}\left[ \ln p(\mathbf{X}, \mathbf{z}|\Theta) | \mathbf{X}, \Theta^{t} \right]\\
&=& \sum_{\mathbf{z}} p(\mathbf{z}|\mathbf{X},\Theta^t)\ln p(\mathbf{X}, \mathbf{z}|\Theta)\\
&=& \sum_{i=1}^N \sum_{z_i=1}^K p(z_i|\mathbf{x}_i,\Theta^t)\ln p(\mathbf{x}_i, z_i|\Theta)\\
&=& \sum_{i=1}^N \sum_{z_i=1}^K p(z_i|\mathbf{x}_i,\Theta^t)\ln p(\mathbf{x}_i|z_i,\Theta) p(z_i)
\end{eqnarray}

* Thus, to take the expectation, we need $p(z_i|\mathbf{x}_i,\Theta^t)$
\begin{eqnarray}
p(z_i|\mathbf{x}_i,\Theta^t) &=& \frac{\pi_{z_i}^t p_{z_i}(\mathbf{x}_i|\theta_{z_i}^t, z_i)}{p(\mathbf{x}_i|\Theta^t)}\\
&=& \frac{\pi_{z_i}^t p_{z_i}(\mathbf{x}_i|\theta_{z_i}^t, z_i)}{\sum_{k=1}^K \pi_k^t p_k(\mathbf{x}_i|\theta_k^t, k)}
\end{eqnarray}

* This completes the Expectation step in EM.  Now, we must derive the update equations for the Maximization step.  So, we need to maximize for $\pi_k, \Sigma_k, \mu_k$.

## Update equation for mean of the kth Gaussian

* For simplicity, let us assume that $\Sigma_k = \sigma_k^2\mathbf{I}$
\begin{eqnarray}
Q(\Theta, \Theta^t) &=& \sum_{i=1}^N \sum_{z_i=1}^K p(z_i|\mathbf{x}_i,\Theta^t)\ln p(\mathbf{x}_i|z_i,\Theta) p(z_i)\\
 &=& \sum_{i=1}^N \sum_{k=1}^K p(z_i=k|\mathbf{x}_i,\Theta^t)\ln {N}(\mathbf{x}_i|\mu_k, \sigma_k^2) \pi_k\\
 &=& \sum_{i=1}^N \sum_{k=1}^K p(z_i=k|\mathbf{x}_i,\Theta^t)\ln {N}(\mathbf{x}_i|\mu_k, \sigma_k^2) + \ln \pi_k\\
  &=& \sum_{i=1}^N \sum_{k=1}^K p(z_i=k|\mathbf{x}_i,\Theta^t)\left( -\frac{d}{2}\ln\sigma_k^2 -\frac{1}{2\sigma_k^2}\left\| \mathbf{x}_i - \mu_k \right\|_2^2 + \ln \pi_k \right)
\end{eqnarray}

\begin{eqnarray}
\frac{\partial Q(\Theta, \Theta^t)}{\partial \mu_k} &=& \frac{\partial}{\partial \mu_k} \left[\sum_{i=1}^N \sum_{k=1}^K p(z_i=k|\mathbf{x}_i,\Theta^t)\left( -\frac{d}{2}\ln\sigma_k^2 -\frac{1}{2\sigma_k^2}\left\| \mathbf{x}_i - \mu_k \right\|_2^2 + \ln \pi_k \right)\right] = 0\\
&=& \sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t)\left(  \frac{1}{\sigma_k^2}\left(\mathbf{x}_i - \mu_k \right) \right) = 0\\
&=& \sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t)\frac{\mathbf{x}_i}{\sigma_k^2} - \sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t) \frac{\mu_k}{\sigma_k^2} = 0\\
&=& \sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t)\frac{\mathbf{x}_i}{\sigma_k^2} = \sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t) \frac{\mu_k}{\sigma_k^2}\\
&=& \sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t)\mathbf{x}_i = \sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t) \mu_k\\
& &\mu_k = \frac{\sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t)\mathbf{x}_i}{\sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t)}
%
\end{eqnarray}

## Update equation for the variance of the kth Gaussian

\begin{eqnarray}
\frac{\partial Q(\Theta, \Theta^t)}{\partial \sigma_k^2} &=& \frac{\partial}{\partial \sigma_k^2} \left[\sum_{i=1}^N \sum_{k=1}^K p(z_i=k|\mathbf{x}_i,\Theta^t)\left( -\frac{d}{2}\ln\sigma_k^2 -\frac{1}{2\sigma_k^2}\left\| \mathbf{x}_i - \mu_k \right\|_2^2 + \ln \pi_k \right)\right] = 0\\
&=&  \sum_{i=1}^N p(z_i=k|\mathbf{x}_i,\Theta^t)\left[\left( -\frac{1}{2\sigma_k^2} + \frac{2}{2\left(\sigma_k^2\right)^2}\left\| \mathbf{x}_i - \mu_k \right\|_2^2 \right)\right] = 0\\
&=&  \sum_{i=1}^N p(z_i=k|\mathbf{x}_i,\Theta^t)\frac{2}{2\sigma_k^2} = \sum_{i=1}^N p(z_i=k|\mathbf{x}_i,\Theta^t)\frac{1}{\left(\sigma_k^2\right)^2}\left\| \mathbf{x}_i - \mu_k \right\|_2^2  \\
&=&  \sum_{i=1}^N p(z_i=k|\mathbf{x}_i,\Theta^t) = \sum_{i=1}^N p(z_i=k|\mathbf{x}_i,\Theta^t)\frac{1}{\sigma_k^2}\left\| \mathbf{x}_i - \mu_k \right\|_2^2  \\
&=&  \sum_{i=1}^N p(z_i=k|\mathbf{x}_i,\Theta^t) = \frac{1}{\sigma_k^2} \sum_{i=1}^N p(z_i=k|\mathbf{x}_i,\Theta^t)\left\| \mathbf{x}_i - \mu_k \right\|_2^2  \\
&=&   \sigma_k^2 = \frac{\sum_{i=1}^N p(z_i=k|\mathbf{x}_i,\Theta^t)\left\| \mathbf{x}_i - \mu_k \right\|_2^2}{  \sum_{i=1}^N p(z_i=k|\mathbf{x}_i,\Theta^t)}
%
\end{eqnarray}

## Update Equation for mixture weights

\begin{eqnarray}
& & \frac{\partial Q(\Theta, \Theta^t)}{\partial \pi_k} = \nonumber \\
& &\frac{\partial}{\partial \pi_k} \left[\sum_{i=1}^N \sum_{k=1}^K p(z_i=k|\mathbf{x}_i,\Theta^t)\left( -\frac{d}{2}\ln\sigma_k^2 -\frac{1}{2\sigma_k^2}\left\| \mathbf{x}_i - \mu_k \right\|_2^2 + \ln \pi_k \right) - \lambda\left(\sum_{k=1}^K \pi_k - 1\right)\right] = 0 \nonumber\\
&=&\sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t)\left(\frac{1}{\pi_k} \right) - \lambda  = 0 \nonumber\\
&=& \left(\frac{1}{\pi_k} \right)  \sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t)- \lambda  = 0 \nonumber\\
&=&  \sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t)  = \lambda \pi_k \nonumber\\
%
\end{eqnarray}

Since $\sum_k \pi_k = 1$, then:
\begin{eqnarray}
& \sum_{k=1}^K \frac{\sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t)}{\lambda}  =  \sum_{k=1}^K \pi_k = 1\nonumber\\
& \lambda = \sum_{k=1}^K \sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t) \nonumber\\
\end{eqnarray}

So:
\begin{eqnarray}
\pi_k &=&\frac{\sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t)}{\sum_{k=1}^K \sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t)} \\
 &=&\frac{\sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t)}{\sum_{i=1}^N \sum_{k=1}^K  p(z_i=k|\mathbf{x}_i,\Theta^t)}\\
  &=&\frac{\sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t)}{\sum_{i=1}^N 1}\\
    &=&\frac{\sum_{i=1}^N  p(z_i=k|\mathbf{x}_i,\Theta^t)}{N}
\end{eqnarray}

* We now have everything we need to implement the EM algorithm.

* Pseudo-code for the algorithm is:

    * Initialize all parameters
    * t = 1
    * While convergence not yet reached:
        * E-step:  Compute $p(z_i=k|\mathbf{x}_i,\Theta^t)$ for every $\mathbf{x}_i$ and $k$
        * M-step:
            * Update $\mu_k$ for all $k$
            * Update $\sigma_k^2$ for all $k$
            * Update $\pi_k$ for all $k$
        * t = t+1
        * Check convergence criteria

```python

```

Lecture10_EM/Expectation Maximization for the Gaussian Mixture Model.ipynb


# Kalman filter for Nomoto model
A Kalman filter with a Nomoto model as the predictor will be developed.
The filter is run on simulated data as well as real model test data.

```python
%load_ext autoreload
%autoreload 2

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from numpy.linalg import inv
import sympy as sp

import src.visualization.book_format as book_format
book_format.set_style()
from src.substitute_dynamic_symbols import lambdify
from sympy import Matrix
from sympy.physics.mechanics import (dynamicsymbols, ReferenceFrame,
                                      Particle, Point)
from IPython.display import display, Math, Latex
from src.substitute_dynamic_symbols import run, lambdify
from sympy.physics.vector.printing import vpprint, vlatex
from src.data import mdl
from src.kalman_filter import ssa, filter_yaw
```

## First order Nomoto model for ship manoeuvring dynamics
The first order Nomoto model can be written as:

```python
r,r1d,r2d = sp.symbols('r \dot{r} \ddot{r}')
psi,psi1d = sp.symbols('psi \dot{\psi}')
h,u = sp.symbols('h u')
x, x1d = sp.symbols('x \dot{x}')
A,B,C,D,E, Phi = sp.symbols('A B C D E Phi')
w = sp.symbols('w')

K, delta, T_1, T_2 = sp.symbols('K delta T_1 T_2')

eq_nomoto = sp.Eq(K*delta,
                 r + T_1*r1d + T_2*r2d)
Math(vlatex(eq_nomoto))
```

where $r$ is yaw rate with its time derivatives and $\delta$ is the rudder angle. $K$, $T_{1}$
 and $T_{1}$ are the coefficients describing the hydrodynamics of the ship.

For slow manoeuvres this equation can be further simplified by removing the $\ddot{r}$ term:

```python
eq_nomoto_simple = eq_nomoto.subs(r2d,0)
Math(vlatex(eq_nomoto_simple))
```

### Simulation model

```python
eq_system = sp.Eq(x1d, A*x + B*u + E*w)
eq_system
```

Where the state vector $x$:

```python
eq_x = sp.Eq(x, sp.UnevaluatedExpr(Matrix([psi,r])))
eq_x
```

and input vector $u$:

```python
eq_u = sp.Eq(u, sp.UnevaluatedExpr(Matrix([delta])))
eq_u
```

and $w$ is zero mean Gausian process noise

For the nomoto model the time derivatives for the states can be expressed as:

```python
eq_psi1d = sp.Eq(psi1d,r)
eq_psi1d
```

```python
eq_r1d = sp.Eq(r1d,sp.solve(eq_nomoto_simple,r1d)[0])
eq_r1d
```

So that matrix $A$ can be expressed as:

```python
eqs = (eq_psi1d, eq_r1d,)
A_,b_ =sp.linear_eq_to_matrix(eqs, list(eq_x.rhs.doit()))
eq_A = sp.Eq(A, sp.UnevaluatedExpr(-A_))
eq_A
```

and $B$ as:

```python
B_,b_ =sp.linear_eq_to_matrix(eqs, list(eq_u.rhs.doit()))
eq_B = sp.Eq(B,sp.UnevaluatedExpr(B_))
eq_B
```

### Discrete time step model

The first term of a taylor expansion of $A$ can be written as:

```python
eq_Phi = sp.Eq(Phi, sp.UnevaluatedExpr(sp.eye(2)) + sp.UnevaluatedExpr(eq_A.rhs.doit()*h))
display(eq_Phi)
#eq_Phi = sp.Eq(Phi, eq_Phi.rhs.doit())
```

```python
A_d,B_d = sp.symbols('A_d B_d')
```

The discrete state transition matrixes can then be written:

```python
eq_A_d=sp.Eq(A_d,Phi)
eq_A_d
```

```python
eq_B_d = sp.Eq(B_d,B*h)
eq_B_d
```

```python
lambda_Ad = lambdify(eq_Phi.rhs.doit())
lambda_Bd = lambdify(eq_B.rhs.doit()*h)
```

```python
eq_B.rhs.doit()*h
```

```python
eq_Phi.rhs.doit()
```

```python
T_1_ = 1.8962353076056344
K_ = 0.17950970687951323
h_ = 0.02

Ad_=lambda_Ad(T_1=T_1_, h=h_)
Bd_=lambda_Bd(K=K_, T_1=T_1_, h=h_)
```

```python
def simulate(Ad, Bd, E,ws, t, us):

    simdata = []
    x_=np.deg2rad(np.array([[0,0]]).T)
    for u_,w_ in zip(us,ws):

        x_ = (Ad@x_ + Bd*u_) + E * w_

        simdata.append(x_.flatten())

    simdata = np.array(simdata)
    df = pd.DataFrame(simdata, columns=["psi","r"], index=t)
    df['delta'] = us

    return df
```

```python
N_ = 4000

t_ = np.arange(0,N_*h_,h_)

us = np.deg2rad(np.concatenate((-10*np.ones(int(N_/4)),
                                10*np.ones(int(N_/4)),
                                -10*np.ones(int(N_/4)),
                                10*np.ones(int(N_/4)))))

np.random.seed(42)
E = np.array([[0, 1]]).T
process_noise = np.deg2rad(0.01)
ws = process_noise*np.random.normal(size=N_)
df = simulate(Ad=Ad_, Bd=Bd_, E=E, ws=ws, t=t_, us=us)

measurement_noise = np.deg2rad(0.5)
df['epsilon'] = measurement_noise*np.random.normal(size=N_)
df['psi_measure'] = df['psi'] + df['epsilon']
df['psi_deg'] = np.rad2deg(df['psi'])
df['psi_measure_deg'] = np.rad2deg(df['psi_measure'])
df['delta_deg'] = np.rad2deg(df['delta'])

```

```python
fig,ax=plt.subplots()
df.plot(y='psi_deg', ax=ax)
df.plot(y='psi_measure_deg', ax=ax, zorder=-1)
df.plot(y='delta_deg', ax=ax, zorder=-1)
df.plot(y='r')
ax.set_title('Simulation with measurement and process noise')
ax.set_xlabel('Time [s]');

```

```python
x0=np.deg2rad(np.array([[0,0]]).T)
P_prd = np.diag(np.deg2rad([1, 0.1]))

Qd = np.deg2rad(np.diag([0, 0.5]))

Rd = np.deg2rad(1)

ys = df['psi_measure'].values

E_ = np.array(
    [[0,0], [0,1]],
)

C_ = np.array([[1, 0]])

Cd_ = C_
Ed_ = h_ * E_

time_steps = filter_yaw(x0=x0, P_prd=P_prd,h_m=h_,h=h_, us=us, ys=ys, Ad=Ad_, Bd=Bd_, Cd=Cd_, Ed=Ed_, Qd=Qd, Rd=Rd)
x_hats = np.array([time_step["x_hat"] for time_step in time_steps]).T
time = np.array([time_step["time"] for time_step in time_steps]).T
Ks = np.array([time_step["K"] for time_step in time_steps]).T

```

```python
n=len(Ad_)
fig,axes=plt.subplots(nrows=n)

keys = ['psi','r']
for i,key in enumerate(keys):

    ax=axes[i]
    df.plot(y=key, ax=ax, label="True")
    if key=='psi':
        df.plot(y='psi_measure', ax=ax, label="Measured", zorder=-1)

    ax.plot(time, x_hats[i, :], "-", label="kalman")
    ax.set_ylabel(key)
    ax.legend()
```

```python
fig,ax=plt.subplots()
for i,key in enumerate(keys):
    ax.plot(time,Ks[i,:],label=key)
ax.set_title('Kalman gains')
ax.legend();
ax.set_ylim(0,0.1);

```

# Real data
Using the developed Kalman filter on some real model test data

## Load test

```python
id=22773
df, units, meta_data = mdl.load(id=id)
df.index = df.index.total_seconds()
df.index-=df.index[0]
```

```python
from src.visualization.plot import track_plot
fig,ax=plt.subplots()
fig.set_size_inches(10,10)
track_plot(df=df, lpp=meta_data.lpp, x_dataset='x0', y_dataset='y0',  psi_dataset='psi', beam=meta_data.beam, ax=ax);
```

```python
ys = df['psi'].values
h_m=h_ = df.index[1]-df.index[0]
x0=np.deg2rad(np.array([[0,0]]).T)
us = df['delta'].values

P_prd = np.diag(np.deg2rad([1, 0.1]))
Qd = np.deg2rad(np.diag([0, 5]))
Rd = np.deg2rad(0.1)

time_steps = filter_yaw(x0=x0, P_prd=P_prd,h_m=h_,h=h_, us=us, ys=ys, Ad=Ad_, Bd=Bd_, Cd=Cd_, Ed=Ed_, Qd=Qd, Rd=Rd)
x_hats = np.array([time_step["x_hat"] for time_step in time_steps]).T
time = np.array([time_step["time"] for time_step in time_steps]).T
Ks = np.array([time_step["K"] for time_step in time_steps]).T
```

```python
n=len(Ad_)
fig,axes=plt.subplots(nrows=n)

ax=axes[0]
df.plot(y='psi', ax=ax, label="Measured", zorder=-1)
df['-delta']=-df['delta']
df.plot(y='-delta', ax=ax, label='$-\delta$', zorder=-10)
ax.plot(time, x_hats[0, :], "-", label="kalman", zorder=10)
ax.set_ylabel('$\Psi$')
ax.legend()

ax=axes[1]
ax.plot(time, x_hats[1, :], "-", label="kalman")
ax.set_ylabel('$r$')

ax.legend();
```


# Exercise 6
* Due: 18.2.2016 at noon
* Max points: 8

## General rules
* Send the source code or the written notes of your answers as an attachment to markus.hartikainen@jyu.fi before the due time
* You will get feedback about your answers a week from the due date
* From the second week on, the exercises will be given on the previous Wednesday's lecture

## Exercises

1. **2 points** Solve optimization problem
$$
\begin{align}
\min \quad &\log(x_1^2+1)+x_2^4+x_1x_3\\
\text{s.t. }\quad &x_1^3-x_2^2\geq 1\\
& x_1,x_3\geq 0, \ x_2\in\mathbb R
\end{align}
$$
using any method that you have available.
2. **2 points** Study multiobjective optimization problem
$$
\begin{align}
\min & \{\|x-(1,0)\|^2,\|x-(0,1)\|^2\}\\
\text{s.t. }&x\in \mathbb R^2.
\end{align}
$$
Chacterize algebraicly the full set of Pareto optimal solutions.
3. **2 points** Calculate the ideal and nadir vectors for the problem of 2. You can use any methods available.
4. **2 points** Try to generate a representative set of Pareto optimal solutions using the weighting method for the problem of 2. Compare this set to the set of Pareto optimal solutions from 2. What do you notice?


Having heard about the announcement about Theano from Bengio lab , as a Theano user, I am happy and sad to see the fading of the old hero, caused by many raising stars. Sad to see it is too old to compete with its industrial competitors, and happy to have so many excellent deep learning frameworks to choose from. Recently I started translating some of my old codes to Pytorch and have been really impressed by its dynamic nature and clearness. But at the very beginning, I was very confused by the ``backward()`` function when reading the tutorials and documentations. This motivated me to write this post in order for other Pytorch beginners to ease the understanding a bit. And I'll assume that you already know the [``autograd``](http://pytorch.org/docs/master/autograd.html) module and what a [``Variable``](http://pytorch.org/docs/0.1.12/_modules/torch/autograd/variable.html) is, but are a little confused by definition of ``backward()``.

First let's recall the gradient computing under mathematical notions. For an independent variable $x$ (scalar or vector), the whatever operation on $x$ is $y = f(x)$. Then the gradient of $y$ w.r.t $x_i$s is
$$\begin{align}\nabla y&=\begin{bmatrix}
\frac{\partial y}{\partial x_1}\\
\frac{\partial y}{\partial x_2}\\
\vdots
\end{bmatrix}
\end{align}.
$$
Then for a specific point of $x=[X_1, X_2, \dots]$, we'll get the gradient of $y$ on that point as a vector. With these notions in my mind, those things are a bit confusing at the beginning

1. Mathematically, we would say "The gradients of a function w.r.t. the independent variables", whereas the ``.grad`` is attached to the leaf ``Variable``s. In Theano and Tensorflow, the computed gradients are stored separately in a variable. But with a memont of adjustment, it is fairly easy to buy that. In Pytorch it is also possible to get the ``.grad`` for intermediate ``Variable``s with help of ``register_hook`` function

2. The parameter ``grad_variables`` of the function ``torch.autograd.backward(variables, grad_variables=None, retain_graph=None, create_graph=None, retain_variables=None)`` is not straightforward for knowing its functionality.

3. What is ``retain_graph`` doing?

```python
import torch as T
import torch.autograd
from torch.autograd import Variable
import numpy as np
```

### Simplicity of using ``backward()``

```python
'''
Define a scalar variable, set requires_grad to be true to add it to backward path for computing gradients

It is actually very simple to use backward()

first define the computation graph, then call backward()
'''

x = Variable(T.randn(1, 1), requires_grad=True) #x is a leaf created by user, thus grad_fn is none
print('x', x)
#define an operation on x
y = 2 * x
print('y', y)
#define one more operation to check the chain rule
z = y ** 3
print('z', z)

```

    x Variable containing:
     0.6194
    [torch.FloatTensor of size 1x1]

    y Variable containing:
     1.2388
    [torch.FloatTensor of size 1x1]

    z Variable containing:
     1.9013
    [torch.FloatTensor of size 1x1]

The simple operations defined a forward path $z=(2x)^3$, $z$ will be the final output ``Variable`` we would like to compute gradient: $dz=24x^2dx$, which will be passed to the parameter ``Variables`` in ``backward()`` function.

```python
#yes, it is just as simple as this to compute gradients:
z.backward()
```

```python
print('z gradient', z.grad)
print('y gradient', y.grad)
print('x gradient', x.grad) # note that x.grad is also a Variable
```

    z gradient None
    y gradient None
    x gradient Variable containing:
     9.2083
    [torch.FloatTensor of size 1x1]

The gradients of both $y$ and $z$ are None, since the function returns the gradient for the leaves, which is $x$ in this case. At the very beginning, I was assuming something like this:

``x gradient None
y gradient None
z gradient Variable containing:
 128.3257
[torch.FloatTensor of size 1x1]``,
since the gradient is for the final output $z$.

With a blink of thinking, we could figure out it would be practically chaos if $x$ is a multi-dimensional vector. ``x.grad`` should be interpreted as the gradient of $z$ at $x$.

### How do we use ``grad_variables``?

``grad_variables`` should be a list of torch tensors. In default case, the ``backward()`` is applied to scalar-valued function, the default value of ``grad_variables`` is thus ``torch.FloatTensor([1])``. But why is that? What if we put some other values to it?

```python
x = Variable(T.randn(1, 1), requires_grad=True) #x is a leaf created by user, thus grad_fn is none
print('x', x)
#define an operation on x
y = 2 * x
#define one more operation to check the chain rule
z = y ** 3

z.backward(T.FloatTensor([1]), retain_graph=True)
print('Keeping the default value of grad_variables gives')
print('z gradient', z.grad)
print('y gradient', y.grad)
print('x gradient', x.grad)
```

    x Variable containing:
    -0.2782
    [torch.FloatTensor of size 1x1]

    Keeping the default value of grad_variables gives
    z gradient None
    y gradient None
    x gradient Variable containing:
     1.8581
    [torch.FloatTensor of size 1x1]

```python
x.grad.data.zero_()
z.backward(T.FloatTensor([0.1]), retain_graph=True)
print('Modifying the default value of grad_variables to 0.1 gives')
print('z gradient', z.grad)
print('y gradient', y.grad)
print('x gradient', x.grad)
```

    Modifying the default value of grad_variables to 0.1 gives
    z gradient None
    y gradient None
    x gradient Variable containing:
     0.1858
    [torch.FloatTensor of size 1x1]

Now let's set $x$ to be a matrix. Note that $z$ will also be a matrix.

```python
'''
Try to set x to be column vector or row vector! You'll see different behaviors.

'''

x = Variable(T.randn(2, 2), requires_grad=True) #x is a leaf created by user, thus grad_fn is none
print('x', x)
#define an operation on x
y = 2 * x
#define one more operation to check the chain rule
z = y ** 3

print('z shape:', z.size())

z.backward(T.FloatTensor([1, 0]), retain_graph=True)
print('x gradient', x.grad)

x.grad.data.zero_() #the gradient for x will be accumulated, it needs to be cleared.

z.backward(T.FloatTensor([0, 1]), retain_graph=True)
print('x gradient', x.grad)

x.grad.data.zero_()

z.backward(T.FloatTensor([1, 1]), retain_graph=True)
print('x gradient', x.grad)

```

    x Variable containing:
     1.3689 -1.6859
     1.0549 -0.9156
    [torch.FloatTensor of size 2x2]

    z shape: torch.Size([2, 2])
    x gradient Variable containing:
     44.9719   0.0000
     26.7060   0.0000
    [torch.FloatTensor of size 2x2]

    x gradient Variable containing:
      0.0000  68.2102
      0.0000  20.1196
    [torch.FloatTensor of size 2x2]

    x gradient Variable containing:
     44.9719  68.2102
     26.7060  20.1196
    [torch.FloatTensor of size 2x2]

 We can clearly see the gradients of $z$ are computed w.r.t to each dimension of $x$, because the operations are all element-wise. ``T.FloatTensor([1, 0])`` will give the gradients for first column of $x$.

Then what if we render the output one-dimensional (scalar) while $x$ is two-dimensional. This is a real simplified scenario of neural networks.
$$f(x)=\frac{1}{n}\sum_i^n(2x_i)^3$$
$$f'(x)=\frac{1}{n}\sum_i^n24x_i^2$$

```python
x = Variable(T.randn(2, 2), requires_grad=True) #x is a leaf created by user, thus grad_fn is none
print('x', x)
#define an operation on x
y = 2 * x
#print('y', y)
#define one more operation to check the chain rule
z = y ** 3
out = z.mean()
print('out', out)
out.backward(T.FloatTensor([1]), retain_graph=True)
print('x gradient', x.grad)
x.grad.data.zero_()
out.backward(T.FloatTensor([0.1]), retain_graph=True)
print('x gradient', x.grad)
```

    x Variable containing:
     0.0192 -0.1596
    -1.1868 -0.1631
    [torch.FloatTensor of size 2x2]

    out Variable containing:
    -3.3603
    [torch.FloatTensor of size 1]

    x gradient Variable containing:
     0.0022  0.1528
     8.4515  0.1596
    [torch.FloatTensor of size 2x2]

    x gradient Variable containing:
     0.0002  0.0153
     0.8451  0.0160
    [torch.FloatTensor of size 2x2]

### What is ``retain_graph`` doing?

When training a model, the graph will be re-generated for each iteration. Therefore each iteration will consume the graph if the ``retain_graph`` is false, in order to keep the graph, we need to set it be true.

```python
x = Variable(T.randn(2, 2), requires_grad=True) #x is a leaf created by user, thus grad_fn is none
print('x', x)
#define an operation on x
y = 2 * x
#print('y', y)
#define one more operation to check the chain rule
z = y ** 3
out = z.mean()
print('out', out)
out.backward(T.FloatTensor([1]))  #without setting retain_graph to be true, this gives an error.
print('x gradient', x.grad)
x.grad.data.zero_()
out.backward(T.FloatTensor([0.1]))
print('x gradient', x.grad)
```

### Wrap up

1. The ``backward()`` function made differentiation very simple
2. For non-scalar ``Variable``s, we need to specify ``grad_variables``
3. If you need to backward() twice on a graph or subgraph, you will need to set ``retain_graph`` to be true.

```python

```

_drafts/.ipynb_checkpoints/understanding backward() in Pytorch-checkpoint.ipynb


# Lecture 15:  Solution to the Diffusion Equation (Separation of Variables)

## Reading and Reference

* Essential Mathematical Methods for Physicists, H. Weber and G. Arfken, Academic Press, 2003
* Advanced engineering Mathematics, E. Kreyszig, John wiley and Sons, 2010
* S. Farlow, Partial Differential Equations for Scientists and Engineers, Dover, 1993

## What to learn?

* See the solution to a partial differential equation (PDE) by reducing it to two ordinary differential equations (ODE).
* Determine constants by applying the boundary conditions.
* Develop an initial condition that is consistent with the physical problem.
* Visualize the results.

## What to do?

* Solve the diffusion PDE using separation of variables.
* Practice using `dsolve` and applying boundary conditions using substitutions.
* Develop code to help you visualize solutions.

### Introduction
----

Separation of variables is a technique that reduces a partial differential equation to a pair of ordinary differential equations.  Solutions of the resulting ODEs are more straightforward than for the PDE.  This lecture and computational notebook will help you analyze one such solution to the diffusion equation.  To start we will select a problem that is best solved with a Fourier `sin` series.  Making this selection at the start is arbitrary, however the consequences will define the physics of the problem.  This is most easily seen in the restrictions on the boundary conditions.  For this problem our boundary conditions will be chosen to be compatible with a Fourier `sin` series.

```python
%matplotlib notebook
import sympy as sp
# You may have to comment out the init_sesison() line if you have not used my session.py file.
sp.init_session(quiet=True)
sp.init_printing()
dum = sp.symbols('dum')
```

```python
def initial_condition(x):
    return x

def b_m_amplitudes(m, funToProject, center=0.5, width=1.0):
    return (2/width)*sp.integrate(funToProject(dum)*sp.sin(2*m*sp.pi*dum/width), (dum,center-width/2,center+width/2))

def b_m_vectorspace_element(m, var, width=1.0):
    return sp.sin(2*m*sp.pi*var/width)

terms = 10
width = 2
center = 0

sinSeries = sum([b_m_amplitudes(m, initial_condition, center=center, width=width)*b_m_vectorspace_element(m,x,width=width) for m in range(terms)])
```

```python
sp.plot(sinSeries,(x,0,1))
```

The physical problem of interest is one where an extensive quantity (energy, mass, etc.) evolves diffusivly in some medium.  Using mass diffusion as an example, we will identify the concentration of species in our medium at any point and tiime, $c(x,t)$.  We must choose a domain (the geometry of the medium) in which this diffusion process will take place.  In the solution developed below we assume that we have a long bar of material with a constant cross section and this bar is insulated so that none of our extensive quantity can escape through the ends or through the cylindrical surface.  These assumptions permit simplification of the problem to one dimension.

We will assume that our bar of material has a fixed length $L = 1.0$ units long.  The medium will contain an initial amount of the diffusing substance whose distribution will be given by a function that represents the initial condition.  We will represent our initial condition using the symbol $\phi(x,0)$ where the zero indicates time $t=0$.  For this exercise we will choose:

$$
\phi(x,0) = x
$$

The solution to the problem depends in part on our ability to represent the initial condition as a Fourier series.  The choice of $\phi(x)=x$ permits us demonstrate something about the series solution at the boundary.  Let us visualize the solution to get a feel for what we're dealing with first and then we will develop the solution ourselves later in this notebook.  In the `interact` function we can change the time, the number of terms, and the diffusion constant ($\alpha$).

* We define a function called `solutionAtX` that takes a pair of values ($x$, $t$), a diffusion coefficient $\alpha$ and the number of terms to use in the Fourier series.
* We define another function to produce a plot (`make_plot`) that we pass to `interact` so that we have an interactive plot to examine the solution and a set of parameters.

```python
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
from ipywidgets import interact

def solutionAtX(x, t, alpha, terms):
    # This is an already developed solution to the diffusion equation.
    # If you can follow along in the lecture - your solution will look
    # identical.
    return sum([
            (2/(np.pi*n)**2)*
            (np.sin(n*np.pi*x))*
            (-np.pi*n*np.cos(np.pi*n)+np.sin(np.pi*n))*
            np.exp(-t*(alpha*n*np.pi)**2)
         for n in range(1,terms+1,1)
        ])

def make_plot(t=0.01, alpha=0.2, terms=20):
    plt.figure(figsize=(7,5))
    x = np.arange(0, 1, 0.001)
    y = solutionAtX(x, t, alpha, terms)
    plt.plot(x, y, x, x, 'orange')
    plt.fill_between(x, 0, y, alpha=0.2)
    plt.ylim(0.0, 1.0)
    plt.xlabel('Distance')
    plt.ylabel('Concentration')
    plt.show()
```

```python
interact(make_plot, t=(0.001,2,0.01), alpha=(0.2,0.5,0.01), terms=(1,40,1));
```

Using the interactive widgets above, explore different combinations of parameters and connect your physical intuition for diffusion with the solution.  The orange line is the line $\phi(x,0)=x$ and the blue line is the solution to the diffusion equation.  As you advance time try and connect your physical intuition of diffusion with the evolution of the area under the blue curve in the solution.  At the boundaries the diffusing species is allowed to escape the medium.  Over time, we should see a gradual decrease in the amount of substance (shaded area under the blue curve) in the solution to the equation.

### 1D Diffusion Problem - Separating Variables
----

There are many ways to solve a PDE.  Separating variables is one such method to reduce a PDE to a pair (or more) of ODEs.  We start by assuming that the solution has the form of a time dependent piece and a spatially dependent piece.  So we could write:

$$ c(x,t) = X(x) T(t) $$

We substitute the assumed form of the solution into Fick's law:

$$
\frac{\partial c(x,t)}{\partial t} = D \frac{\partial^2 c(x,t)}{\partial x^2}
$$

this gives us:

$$ \frac{\partial X(x) T(t)}{\partial t} = D \frac{\partial^2 X(x) T(t)}{\partial x^2} $$

We will differentate the assumed solution of $c(x,t)$ above according to the derivatives in the differential equation.  It is usual to let D be $\alpha^2$ to guarantee that D is always positive, we can do this in `sympy` with a keyword in the `symbols` function call.

```python
import sympy as sp
sp.init_session(quiet=True)
sp.init_printing()
```

```python
var('X,T, c',cls=Function)
```

```python
var('alpha gamma', positive=True)
```

With our substitutions in place:

$$ \frac{\partial X(x) T(t)}{\partial t} = \alpha^2 \frac{\partial^2 X(x) T(t)}{\partial x^2} $$

After taking the derivatives we have:

$$ X(x)T'(t) = \alpha^2 X''(x)T(t) $$

We can arrange these expressions so that all the functions of "t" are on one side and the functions of "x" are on the other side.  Divide both the LHS and RHS of the above expression by $\alpha^2 X(x) T(t)$ to seperate variables on either sides of the equals sign.  Note also that this means that the product $\alpha^2 X(x) T(t)$ should never be zero.  Add a seperation constant $\gamma$ and integrate.

The rationalization for the seperation constant is that since the LHS and RHS each depend on one variable then the only way the LHS and RHS can be equal is if they are both equal to a constant.  Our ODEs are then written as:

```python
timeODE = sp.Eq(T(t).diff(t,1)/(alpha**2*T(t)),-gamma**2)
timeODE
```

```python
spaceODE = sp.Eq(X(x).diff(x,2)/X(x),-gamma**2)
spaceODE
```

Invoke `dsolve()`

```python
sp.dsolve(timeODE,T(t))
```

This is not the form you are likely to find in a textbook.  It is the correct solution, but it may not be the best solution for us.  We can examine all the possible forms of the solution so that we can inspect the list and pick the one we like.  We create a list of hints and then use a list comprehension to solve the equation for each hint.  The notebook displays all the solutions so that we can find the one that suits our needs.

```python
list_of_hints = sp.classify_ode(timeODE)
list_of_solutions = [sp.dsolve(timeODE,T(t), hint=hint) for hint in list_of_hints]
list_of_solutions
```

 A bit of matching up with the list of hints and we find that the one we are interested in is `1st_linear`:

```python
timePiece = sp.dsolve(timeODE,T(t), hint='1st_linear')
timePiece
```

```python
spacePiece = sp.dsolve(spaceODE,X(x))
spacePiece
```

The solution (with constants) is the product of the two solutions above:

$$
c(x,t) = X(x)T(t) = \Big( C_{1} \sin{\left (\gamma x \right )} + C_{2} \cos{\left (\gamma x \right )} \Big)\Big ( C_{3} e^{- \alpha^{2} \gamma^{2} t} \Big )
$$

We will need to "absorb" the time solution constant ($C_3$) into the space solution constants.  There are an infinity of solutions depending on our choices of $C_1$ and $C_2$.  These will ultimately be determined by the boundary conditions.

I would also note here the following:  the CAS (SymPy in this case) did all the work for us above.  The code below is really to just keep ourselves organized.  I don't intend to do much with `generalSolution` except to make substitutions to find the boundary conditions.  Working with a CAS is equal parts setup, organization, and deciphering the results.

Clean up the solution for use below.  Eliminate one of the constants as the product of C1 and C2 is also a constant.  Sympy does not automatically create the symbols corresponding to the constants for us.  So we need to create those before proceeding:

```python
var('C1 C2')
```

```python
generalSolution = Eq(c(x,t),(timePiece.rhs).subs([(C1,1)])*spacePiece.rhs)
generalSolution
```

### Boundary Conditions
----

All that is left to do is impose the boundary conditions to find the constants $C_1$ and $C_2$ and get the particular solution that satisfies our initial condition.

The following conditions apply to this problem:

$$ c(0,t) = 0 $$
$$ c(1,t) = 0 $$
$$ c(x,0) = x $$

These conditions apply for the left-hand boundary, the right-hand boundary and the initial condition (the time boundary).  The domain of the experiment is $\{\,x\, \mid 0 < x < 1\,\}$

```python
generalSolution
```

Start by substituting the boundary condition $c(0,t) = 0$:

```python
generalSolution.subs({x:0})
```

The only way this can be true is if $C_2$ is zero.  Make that substition and insert the second boundary condition where $c(1,t)$ must also be zero:

```python
generalSolution.subs({x:1, C2:0})
```

_So - wait, what?  Can can we let $C1=0$ too?_

That IS a solution.  It just is not the one we're looking for, so we need to examine how the third factor (the sin term) can help us.

What about the exponential?  At the boundary there is no dependence of $x$ in this factor however we know that the parameters in the exponential will not cause that factor to go to zero.  It starts at one and just gets smaller - so looking at the second factor will not help us much.

This means that we have to let $\sin(\gamma)=0$.  We know this can happen with the right choice of $\gamma$.  What choices can we use?

What if:

$$\gamma = \pm \pi, \pm 2\pi, \pm 3\pi, ... $$

and in general:

$$\gamma = \pm n\pi$$

Let us go back an examine the initial condition where $t=0$.

```python
generalSolution.subs({t:0, C2:0, gamma:n*sp.pi})
```

Now this statement tells us that we need to find a constant $C_1$ such that $c(x,0)=x$.  You may be wondering how we can do that - but you already know how to do that.

We identify the diffusion PDE (Fick's Law) as a homogeneous PDE. Kreyszig defines homogeneous as an equation in $u$ where "each of its terms contains either $u$ or one of its partial derivatives".  Because the equation is a homogeneous PDE we can invoke the principle of superposition that combinations of solutions to the PDE are also solutions to the PDE.

So it is therefore possible to write (without proof) that the initial condition:

$$c(x,0) = \sum_{n=1}^N C_n \sin(n\pi x) = \phi(x) = x$$

and the general solution to the equation is:

$$c(x,t) = \sum_{n=1}^N C_n \sin(n\pi x) e^{- \alpha^2 n^2 \pi^2 t}$$

The summation is the Fourier sine series - so as long as you can compute the coefficients $C_n$ then you can write down a summation that will be an approximate solution to the problem.  All that remains is to find the Fourier coefficient for the function $\phi(x)$ and substitute those into the general solution.  Go back and look at the plotted solution and see if you can write your own function derived from the general solution and a Fourier series.

Your homework assignment is to set up a different initial condition and solve the problem again.

### Homework
----

From Farlow's book entitled "Partial Differential Equations for Scientists and Engineers" he poses the following two problems:

Find the Fourier sine expansion of $\phi(x)=1$ on the domain $0 \le x \le 1$.  Structure your solution/code so that you can investigate the series as a function of the number of terms it contains.  It will be helpful to pay attention to the values of the wavenumber and coefficients.  (Maybe symbolic computation will help?)  By choosing hte `sin` series you will also be requiring the values of the function to be zero at the boundaries.  Hint:  The domain of the series may be larger than the domain of the physical problem.

Using the results of the previous problem, what is the solution to the following initial boundary value problem (PDE):

$$
\begin{array}{lll}
\mathrm{PDE} & u_t = u_{xx} & 0 \le x \le 1 \\
\mathrm{BCs} & u(0,t) = 0 & 0 \le t \le \infty \\
 & u(1,t) = 0 & \\
\mathrm{IC}  & u(x,0) = 1 & 0 \le x \le 1
\end{array}
$$

Note that there is a physical impossibility in this problem.  Can you identify it and discuss as part of your solution?

```python

```


```python
# import the library

import tensorflow as tf
```

### Some Basic terms in Tensorflow
* `Variable`: Something that was able to __vary__ data, and that can __updated__ in any point of time

* `constant`: Something that can __not__ be change after initialization

* `placeholder`: A __promise__ to provide value later

```python
# parameters

w = tf.Variable(0.3, tf.float32, name = "weight")
b = tf.Variable(-0.3, tf.float32, name = "bias")

# inputs and outputs

x = tf.placeholder(tf.float32, name="input")
y = tf.placeholder(tf.float32, name="output")

```

```python
# Basic linear Model

linear_model = (w * x) + b
```

### Build the computational Graph and Run it

In tensorflow `Variables`, `constant`, `placeholder` doesn't hold anything except the structure of it, until it `run` in a `Session`. When we initialize a `Variable`, `constant` or a `placeholder` it's just creates a computational graph. Nothing happens until a `Session` being Start. The acutal works done with a Tensorflow `Session`.

```python
# initialize the variable with global_variables_initializer()

init = tf.global_variables_initializer()
```

#### The mathematical formula behind loss function:
\begin{equation}
J(\theta_0, \theta_1) = \frac{1}{2m} \sum(h_\theta x^i - y^i)^2
\end{equation}

here,
* $h_\theta x^i$ is the hypotheis or in other words is the prediction we made
* $ y^i $ is the actual output
* m is the number of trainning examples

```python
# Calculating loss/ loss function

squared_delta = tf.square(linear_model - y)
loss = tf.reduce_sum(squared_delta)
```

```python
# x and y are placeholder and we promise to provide value later, Now this was the time when we provide the values.

with tf.Session() as sess:
    sess.run(init)
    print(sess.run(loss, {x:[1,2,3,4], y:[0,-1,-2,-3]}))
```

    23.66

__So here we see that we have a loss of: `23.66`__

### Now we need to reduce the loss using a Optimizer

An `Optimizer` is used for reduce the `loss`. `Optimizier` looks the changes of loss by updating/changing the Variable in any (forward/postive or backward/negative) direction. If it see that changes in postive direction will increase the loss then it will try to update the variable in negative direction. The main goal of a `Optimizer` is to reduce the `loss`.

```python
# Reducing the Loss using Gradient Descent Optimizer with a 0.01 learning rate
# learning rate is nothing but the steps, number of steps the Variable will update by the Optimizer

optimizer = tf.train.GradientDescentOptimizer(0.01)
train = optimizer.minimize(loss)

```

```python
# After all structure are defined its time to run the computation graph into a Session

with tf.Session() as sess:
    sess.run(init)
    for i in range(1000):
        sess.run(train, {x:[1,2,3,4], y:[0,-1,-2,-3]})

    print("Updated weight and bias to reduce the loss upto 100%\n",sess.run([w,b]))
```

    Updated weight and bias to reduce the loss upto 100%
     [-0.9999969, 0.9999908]

* updated weight ~= -1  and
* updated bias ~= 1

according to the result, if we update the `weight` from `0.3` to `-1` and `bias` from `-0.3` to `1` the approximate `error/loss` will be `0`

### Lets Validating the optimizer result:

```
inputs  =>  x:[1,2,3,4]
outputs =>  y:[0,-1,-2,-3]
weight  => -1
bias    =>  1
```

for `x = 1`,

\begin{equation}
linearModel = (w * x) +b\\
= (-1 * 1 + 1)\\
= 0
\end{equation}
so, for x = 1 the y = 0 is correct.

for `x = 2`,

\begin{equation}
linearModel = (-1 * 2 + 1)\\
= -1
\end{equation}

for `x = 3`,

\begin{equation}
linearModel = (-1 * 3 + 1)\\
= -2
\end{equation}
for `x = 4`,

\begin{equation}
linearModel = (-1 * 4 + 1)\\
= -3
\end{equation}

### Hence, we can say that the `Optimizer` result is correct.

__For Visualizing a Tensorflow Graph we have tensorboard: Let us see how to use it:__

```python
with tf.Session() as sess:
    sess.run(init)
    for i in range(1000):
        sess.run(train, {x:[1,2,3,4], y:[0,-1,-2,-3]})

    File_Writer = tf.summary.FileWriter("tensorboard_output", sess.graph)
    print("Updated weight and bias to reduce the loss upto 100%\n",sess.run([w,b]))
```

    Updated weight and bias to reduce the loss upto 100%
     [-0.9999969, 0.9999908]

__In `FileWriter` we need to provide a directory or we can simply put a folder name. The folder will be created by the `FileWriter` to your default working directory__

In my case i created a folder named `tensorboard_output`, and it was created in my working directory.

To visualize the graph in a browser we need to open up the `Command Prompt`,

and need to write `tensorboard --logdir="path of your tensorboard output folder directory"`

and then you need to browse `http:6006`

```python

```

Introduction to Tensorflow/A very first introduction with Tensorflow.ipynb