```python
import sympy as sp
from sympy import *
import numpy as np
import matplotlib.pyplot as plt
from scipy.interpolate import interp1d
from scipy.integrate import quad
from scipy.optimize import fmin
import scipy.integrate as integrate
import scipy.special as special
import scipy.stats as st
import sys
font1 = {'size'   : 20, 'family':'STIXGeneral'}

from platform import python_version

print(python_version())
```

    3.7.7

```python
#LCDM fractions

mptkm = 3.086*10**(19)
H0 = 67.32/mptkm
Oc = 0.265
Ob = 0.0494
Om = Oc + Ob
Orad = 0.000093
ai = 0.000001
arad=0.0002264 #radiation -DM equality
acmb = 0.0009
Gnewton = 6.67*10**(-11)

def Hub(Om, Orad, a):
    return H0*np.sqrt(Om/a**3 + Orad/a**4 + (1-Om-Orad))
def rhoc(a):
    return 3*Hub(Om, Orad, a)**2/(8*np.pi/Gnewton)
def Omegac(a):
    return Oc/a**3*(H0/Hub(Om,Orad,a))**2
def Omegarad(a):
    return Orad/a**4*(H0/Hub(Om,Orad,a))**2
def Omegab(a):
    return Ob/a**3*(H0/Hub(Om,Orad,a))**2
```

```python
fig0 = plt.figure()

plt.figure(figsize=(10,10))

#Load Omega_pbh data at early and late times
dat1 = np.loadtxt("data/8gev+9orad/peakm_5e-7.dat")
dat2 = np.loadtxt("data/8gev+9orad/peakm_2e11.dat")
dat3 = np.loadtxt("data/8gev+9orad/peakm_5e11.dat")
dat4 = np.loadtxt("data/8gev+9orad/peakm_5e13.dat")
dat5 = np.loadtxt("data/8gev+9orad/peakm_5e33.dat")

avals = np.logspace(-6, 0, num=1000)

ax = plt.subplot(2, 1, 1)
plt.xscale('log')

plt.plot(avals, Omegac(avals),linestyle='dashed', color='b', label= '$\Omega_{\\rm cdm}$',alpha=1.)

plt.plot(dat1[:,0], dat1[:,1], label='$M_{pk} = 5\\times 10^{-7}$kg', alpha=0.6)
plt.plot(dat2[:,0], dat2[:,1], label='$M_{pk} = 2\\times 10^{11}$kg', alpha=0.6)
plt.plot(dat3[:,0], dat3[:,1], label='$M_{pk} = 5\\times 10^{11}$kg',alpha=0.6)
plt.plot(dat4[:,0], dat4[:,1], label='$M_{pk} = 5\\times 10^{13}$kg',alpha=0.6)
plt.plot(dat5[:,0], dat5[:,1], label='$M_{pk} = 5\\times 10^{33}$kg',alpha=0.6)

plt.axhline(y=1., xmin=0., xmax=10,color='k',linestyle='dashed')
plt.axvline(acmb,0.,10, color='k', linestyle='dotted')

#plt.text(2e-6, 0.3 , '$T_{\\rm RH} = 10^{8}{\\rm GeV}$',  **font1)

ax.tick_params(axis='both', which='major', labelsize=15)
plt.ylim(-0.,1.45)
plt.xlim(ai,1)
plt.ylabel('Density fraction of PBH ($\Omega_{\\rm PBH}$) ',**font1)
plt.xlabel('scale factor (a)', **font1)
plt.legend(loc='best',prop={'size': 16})
plt.tight_layout(pad=3.0)

ax = plt.subplot(2, 1, 2)
plt.xscale('log')

plt.plot(avals, Omegab(avals),linestyle='dashed', color='r', label= '$\Omega_b$',alpha=1.)

plt.plot(dat1[:,0], dat1[:,2], label='$\lambda = 3.4\\times 10^{96}$', alpha=0.6)
plt.plot(dat2[:,0], dat2[:,2], label='$\lambda = 1.9\\times 10^{98}$', alpha=0.6)
plt.plot(dat3[:,0], dat3[:,2], label='$\lambda = 4.8\\times 10^{98}$', alpha=0.6)
plt.plot(dat4[:,0], dat4[:,2], label='$\lambda = 4.7\\times 10^{100}$', alpha=0.6)
plt.plot(dat5[:,0], dat5[:,2], label='$\lambda = 3.5\\times 10^{120}$', alpha=0.6)

plt.axvline(acmb,0.,10, color='k', linestyle='dotted')

ax.tick_params(axis='both', which='major', labelsize=15)
plt.ylim(0,0.2)
plt.xlim(ai,1)
plt.ylabel('Density fraction of baryons ($\Omega_{\\rm b}$) ',**font1)
plt.xlabel('scale factor (a)', **font1)
plt.legend(loc='best',prop={'size': 16})

#plt.setp(plt.subplot(2,1,1).get_xticklabels(), visible=False)
plt.subplots_adjust(hspace=0.2)
plt.subplots_adjust(wspace=0.)

plt.savefig('plots/omega_all.png', format="png", bbox_inches = 'tight')

```

```python
# Plotting LCDM fractions
fig0 = plt.figure()
plt.figure(figsize=(10,5))

avals = np.logspace(-6, 0, num=1000)

ax = plt.subplot(1, 1, 1)
plt.xscale('log')

plt.plot(avals, Omegac(avals),linestyle='dashed', color='b', label= '$\Omega_c$',alpha=1.)
plt.plot(avals, Omegab(avals),linestyle='dashed', color='g', label= '$\Omega_b$',alpha=1.)
plt.plot(avals, Omegarad(avals),linestyle='dashed', color='r', label= '$\Omega_\gamma$',alpha=1.)

ax.axvspan(ai, 0.000215, alpha=0.5, color='orange')
plt.axhline(y=0.6856, xmin=0., xmax=10,color='k')

ax.tick_params(axis='both', which='major', labelsize=15)
plt.ylim(-0.1,1.5)
plt.xlim(ai,1)
plt.xlabel('scale factor (a)', **font1)
plt.ylabel('Density fraction ($\Omega$) ',**font1)
plt.legend(loc='best',prop={'size': 14})
plt.tight_layout(pad=3.0)

plt.savefig('plots/lcdm_epochs.png', format="png", bbox_inches = 'tight')
```

```python
fig0 = plt.figure()

plt.figure(figsize=(10,10))

#Load Omega_pbh data at early and late times
dat1 = np.loadtxt("data/8gev+9orad+rem/peakm_5e-7_rem.dat")
dat2 = np.loadtxt("data/8gev+9orad+rem/peakm_2e11_rem.dat")
dat4 = np.loadtxt("data/8gev+9orad+rem/peakm_5e13_rem.dat")

dat1a = np.loadtxt("data/8gev+9orad/peakm_5e-7.dat")
dat2b = np.loadtxt("data/8gev+9orad/peakm_2e11.dat")
dat4c = np.loadtxt("data/8gev+9orad/peakm_5e13.dat")

avals = np.logspace(-6, 0, num=1000)

ax = plt.subplot(2, 1, 1)
plt.xscale('log')

#plt.plot(avals, Omegac(avals),linestyle='dashed', color='b', label= '$\Omega_c$',alpha=1.)

plt.plot(dat1[:,0], dat1[:,1]/dat1a[:,1] , label='$M_{pk} = 5\\times 10^{-7}$kg', alpha=0.6)
plt.plot(dat2[:,0], dat2[:,1]/dat2b[:,1], label='$M_{pk} = 2\\times 10^{11}$kg', alpha=0.6)
#plt.plot(dat3[:,0], dat3[:,1], label='$M_{pk} = 5\\times 10^{11}$kg',alpha=0.6)
#plt.plot(dat4[:,0], dat4[:,1]/dat4c [:,1], label='$M_{pk} = 5\\times 10^{13}$kg',alpha=0.6)
#plt.plot(dat5[:,0], dat5[:,1], label='$M_{pk} = 5\\times 10^{33}$kg',alpha=0.6)

plt.axhline(y=1., xmin=0., xmax=10,color='k',linestyle='dashed')
plt.axvline(acmb,0.,10, color='k', linestyle='dotted')

plt.text(2e-6, 0.3 , '$T_{\\rm RH} = 10^{8}{\\rm GeV}$',  **font1)

ax.tick_params(axis='both', which='major', labelsize=15)
plt.ylim(0.,2)
plt.xlim(ai,1)
plt.ylabel('Density fraction of PBH ($\Omega_{\\rm PBH}$) ',**font1)
plt.xlabel('scale factor (a)', **font1)
plt.legend(loc='best',prop={'size': 16})
plt.tight_layout(pad=3.0)

ax = plt.subplot(2, 1, 2)
plt.xscale('log')

#plt.plot(avals, Omegab(avals),linestyle='dashed', color='r', label= '$\Omega_b$',alpha=1.)

plt.plot(dat1[:,0], dat1[:,2]/dat1a[:,2], label='$\lambda = 3.4\\times 10^{96}$', alpha=0.6)
plt.plot(dat2[:,0], dat2[:,2]/dat2b[:,2], label='$\lambda = 1.9\\times 10^{98}$', alpha=0.6)
#plt.plot(dat3[:,0], dat3[:,2], label='$\lambda = 4.8\\times 10^{98}$', alpha=0.6)
#plt.plot(dat4[:,0], dat4[:,2]/dat4c[:,2], label='$\lambda = 4.7\\times 10^{100}$', alpha=0.6)
#plt.plot(dat5[:,0], dat5[:,2], label='$\lambda = 3.5\\times 10^{120}$', alpha=0.6)

plt.axvline(acmb,0.,10, color='k', linestyle='dotted')

ax.tick_params(axis='both', which='major', labelsize=15)
plt.ylim(0.6,1.4)
plt.xlim(ai,1)
plt.ylabel('Density fraction of baryons ($\Omega_{\\rm b}$) ',**font1)
plt.xlabel('scale factor (a)', **font1)
plt.legend(loc='best',prop={'size': 16})

#plt.setp(plt.subplot(2,1,1).get_xticklabels(), visible=False)
plt.subplots_adjust(hspace=0.2)
plt.subplots_adjust(wspace=0.)

plt.savefig('plots/remnants.png', format="png", bbox_inches = 'tight')

```

```python

```


# Ridge Rigression

This notebook implements an implementation of the Ridge Rigression. The implementation of the Ridge Regression has two different forms. Once with `normal equation` and once with `Gradient Descent`.

### Theory

The cost-function for Ridge Regression looks as follows. This term looks alike the cost-function of simple Linear Regression, but has one additional term, which is related to the model paramters $\beta$. if we optimize this cost objective the parameters will automatically penalized by this term $\alpha\sum_{j=1}^{p} \beta_j^2$. Our model will shrink large coefficients depeding on the weight $\alpha$ which has to be defined at the initialization of the model.

\begin{equation}
J(\beta) = \sum_{i=1}^{N} (y^{(i)}-\beta_0 - \sum_{j=1}^{p} x^{(i)}_j\beta_j)^2 + \alpha\sum_{j=1}^{p} \beta_j^2
\end{equation}

### Derivation of Cost-Function

#### Derivation of the cost-function w.r.t. $\beta_0$:

$$\frac{\partial J(\beta)}{\beta_0}=  \sum_{i=1}^{N} -1*(y^{(i)} - \beta_0 - \sum_{j=1}^{p} x^{(i)}_j\beta_j)$$

Important to mention here is that $\beta_0$ will not be regularized, so the L2-Regularization terms falls off the equation. This formula can also be rewritten in matrix-notation which will be much more helpful for the implementation:

$$\nabla J(\beta) =
\frac{\partial J(\beta)}{\partial \beta_0} =
-e^T(y-X\beta)$$

Here the vector $e \in R^{nx1}$ is a vector , which consists of 1's only.

#### Derivation of the cost-function w.r.t. $\beta_{1...n}$:

$$\nabla J(\beta) =
\frac{\partial J(\beta)}{\partial \beta_0} =
 \sum_{i=1}^{N}-x^{(i)}(y^{(i)} - \beta_0 - \sum_{j=1}^{p} x^{(i)}_j\beta_j) + 2\alpha\sum_{j=1}^{p} \beta_j$$

Here again we can rewrite the gradient in matrix notation. We end-up with following equation:

$$\nabla J(\beta) =
\frac{\partial J(\beta)}{\partial \beta} =
-X^T(y-X\beta)+2\alpha\beta$$

#### Normal Equation

if the term above without the normalization on a sample ($\frac{1}{N}$) is considered to be zero we have an equation which we can be solved. First we make the term a little bit simpler solving the term in brackets and remove the scalar of 2:

$$-2X^T(y-X\beta)+2\alpha\beta = 0 \longrightarrow
-2X^Ty + 2X^TX\beta + 2\alpha\beta = 0  \longrightarrow
-X^Ty + X^TX\beta + \alpha\beta = 0
$$

Next we solve the equation to $\beta$:

$$
X^TX\beta + \alpha\beta = X^Ty \longrightarrow
(X^TX+\alpha)\beta = X^Ty \longrightarrow
(X^TX+\alpha)^{-1}(X^TX+\alpha)\beta = (X^TX+\alpha)^{-1}X^Ty
$$

Finally we end-up with this equation. As $\alpha$ can not multiplied like this we have to multiply it first with Identity-Matrix $I$ with same shaoe as parameters, where the first index is zero as we do not want to penalize $\beta_0$.

$$\beta = (X^TX+\alpha I)^{-1}X^Ty
$$

This equatio represents the `normal equation` of our Regression Model.

### Implementation

```python
import numpy as np
import pandas as pd
import seaborn as sns
import sys
from sklearn.base import BaseEstimator
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_boston
from matplotlib import pyplot as plt
```

```python
X, y = load_boston(return_X_y=True)
```

```python
from sklearn.base import BaseEstimator

class RidgeRegression(BaseEstimator):
    """Implementation of Ridge Regression with L2-Regularizaion"""

    def __init__(self, opt_method='gd', alpha=1., eta=0.0001, maxsteps=100, eps=0.00001):
        '''Implements a Ridge Regression estimator.

        Arguments
        ---------
        alpha:      Regularization proportionality factor. Larger values
                    correspond with stronger regularization.

        opt_method: Optimization method to choose for the cost function.
                    Can be either 'gd' (Gradient Descent) or 'neq'.

        maxsteps:   Maximum number of Gradient Descent steps to take.

        eps:        Epsilon, length of gradient to be reached with Gradient
                    Descent.

        eta:        Fixed step lenght to take at each gradient descent
                    iteration.
        '''
        # parameters
        self.alpha = alpha
        self.opt_method = opt_method
        self.maxsteps = maxsteps
        self.eps = eps
        self.eta = eta
        # attributes
        # model coefficients
        self.beta_ = None
        # values of cost function along gradient descent iterations
        self.costs_ = []

    def fit(self, X, y):
        '''
        Fits the model to the given data predictors X to a target y

        arguments:
        --------------
        X:         Training data X
        y:         Training Data y
        '''
        if self.opt_method == 'neq':
            self.normalequation(X, y)
        elif self.opt_method == 'gd':
            self.gradient_descent(X, y)
        else:
            raise Exception(f"Choose opt_method 'neq' or 'gd'. You've chosen {self.opt_method}")

        return self

    def gradient_descent(self, X, y):
        '''
        Computes the coefficients of the ridge regression cost function
        using gradient descent.

        arguments:
        --------------
        X:         Training data X
        y:         Training Data y
        '''
        X = np.hstack((np.ones(X.shape[0]).reshape(-1,1), X))

        # Set start parameter beta
        self.beta_ = np.random.uniform(size=(X.shape[1]))

        # Set start parameter to pass first iteration of while loop
        grad = np.ones(shape=(X.shape[1]))
        steps = 0
        while steps < self.maxsteps and np.linalg.norm(self.eta*grad) > self.eps:

            grad = self.gradient(beta=self.beta_, X=X, y=y, alpha=self.alpha)
            self.beta_ = self.beta_ - self.eta * grad
            self.costs_.append(self.costfunction(beta=self.beta_, X=X, y=y, alpha=self.alpha))

            steps += 1

    def normalequation(self , X, y):
        '''
        Computes the coefficients of the ridge regression cost function
        using the normalequation.

        arguments:
        --------------
        X:         Training data X
        y:         Training Data y

        '''
        # Add additional Axis for beta 0 with ones only
        X = np.hstack((np.ones(X.shape[0]).reshape(-1,1), X))

        # Create Matrix for regulation term with first line zeros
        I = np.identity(X.shape[1])
        I[0, 0] = 0

        # Normalequation with regulation term
        self.beta_ = np.linalg.inv(X.T @ X + self.alpha * I) @ X.T @ y

    @staticmethod
    def gradient(beta, X, y, alpha):
        '''
        (array, array, array, float) --> array

        Computes and returns the gradient of the ridge regression cost function.

        arguments:
        --------------
        X:         Training data X
        y:         Training Data y
        alpha:     scalar factor for L2-regularization

        returns:
        ---------------
        gradient:  Calculated gradient of derived cost-objective

        gradient([1,2,3], X, y, 0.1)
        >>> [1.11,1.2]
        '''
        y_pred = X @ beta
        gradient = (-2 * X.T) @ (y - y_pred)

        # Make beta0 to 0 to not regulize
        beta_ = beta.copy()
        beta_[0] = 0

        # Add regulation term
        gradient = gradient + 2*alpha*beta_

        return gradient

    @staticmethod
    def costfunction(beta,X,y,alpha):
        '''
        (array, array, array, float) --> float

        Computes and returns the value of the ridge regression cost function.
        cost = (1/(2*n)) * sum((y - pred)**2)

        arguments:
        --------------
        beta:      Model parameters
        X:         Training data X
        y:         Training Data y
        alpha:     scalar factor for L2-regularization

        costfunction(beta, X, y, 1)
        >>> 34.31
        '''
        pred = X @ beta
        cost = (1/X.shape[0]) * np.sum((y - pred)**2) + alpha * np.sum(beta[1:]**2)

        return cost

    def predict(self, X):
        '''
        (array) --> array

        Computes the predictions of the current model.
        pred = X @ b

        arguments:
        --------------
        X:         Training data X

        model.predict(X)
        >>> [[1],[2],[3] ... [n]]
        '''
        X = np.hstack((np.ones(X.shape[0]).reshape(-1,1), X))

        return X @ self.beta_

    def score(self, X, y):
        '''
        (array, array) --> float

        Returns R^2 for given input/output data given the model
        coefficients.

        Formula: 1 - SST/SSR

        arguments:
        --------------
        X:         Training data X

        y:         Train data y

        model.score(X,y)
        >>> 0.91
        '''
        SSR = np.sum((y - self.predict(X))**2)
        SST = np.sum((y - np.mean(y))**2)

        return 1 - SSR/SST
```

### Check

#### Convergence of Coefficients

Here I check the convergence of coefficients by calculating the error of normal equation and the gradient descent optimization method. This is to see how much iterations were needed until the coefficients converge.

```python
coeff_error = {}

for i in np.linspace(100, 50000, num=20, dtype=int):
    neq_ridge = RidgeRegression(opt_method='neq', alpha=0)
    neq_ridge.fit(X[:,:6], y)

    gd_ridge = RidgeRegression(opt_method='gd', alpha=0, eps=1e-7, eta=.0000025, maxsteps=i)
    gd_ridge.fit(X[:,:6],y)

    coeff_error[i] = np.abs(neq_ridge.beta_ - gd_ridge.beta_)
```

```python
fig = plt.subplots(figsize=(12, 6))
tmp = pd.DataFrame(coeff_error)

for i in tmp.index.to_list():
    p = sns.lineplot(x=tmp.columns.to_list(), y=tmp.loc[i], label=r'$\beta$'+str(i+1))

sns.despine()
plt.ylabel('MAE of Coefficients')
plt.xlabel(r'GD Iterations')
plt.title('Deviations of Beta Coefficients over GD Iterations', loc='left', fontsize=13)

plt.show()
```

#### Regularization

```python
coefficients = {}
for a in range(0, 500, 5):
    ridge = RidgeRegression(opt_method='neq', alpha=a)
    ridge.fit(X[:,:5], y)
    coefficients[a] = ridge.beta_[1:]

fig = plt.subplots(figsize=(12, 6))
tmp = pd.DataFrame(coefficients)

for i in tmp.index.to_list():
    p = sns.lineplot(x=tmp.columns.to_list(), y=tmp.loc[i], label=r'$\beta$'+str(i+1))

sns.despine()
plt.ylabel(r'$\beta$')
plt.xlabel(r'Regularization Strength $\alpha$')
plt.title('L2-Regularization strength on Modell Coefficients with Normal Equation', loc='left', fontsize=13)

plt.show()
```

```python
coefficients = {}
for a in range(0, 500, 5):
    ridge = RidgeRegression(opt_method='gd', alpha=a, eps=1e-7, eta=.000002, maxsteps=5000)
    ridge.fit(X[:,:5], y)
    coefficients[a] = ridge.beta_[1:]

fig = plt.subplots(figsize=(12, 6))
tmp = pd.DataFrame(coefficients)

for i in tmp.index.to_list():
    p = sns.lineplot(x=tmp.columns.to_list(), y=tmp.loc[i], label=r'$\beta$'+str(i+1))

sns.despine()
plt.ylabel(r'$\beta$')
plt.xlabel(r'Regularization Strength $\alpha$')
plt.title('L2-Regularization strength on Modell Coefficients with GD', loc='left', fontsize=13)

plt.show()
```

### Sources

pthesling (https://stats.stackexchange.com/users/64162/pthesling), How to derive the ridge regression solution?, URL (version: 2017-07-17): https://stats.stackexchange.com/q/164540

Murphy, K. P. (2012). Machine learning: a probabilistic perspective. MIT press.


# Messtechnik HS2021 - Tutorial 3

```python
import numpy as np
import matplotlib.pyplot as plt

# Fourier transformation
def compute_fft(t,X):
    """ Manually compute the FT of a simple signal"""
    t = np.atleast_1d(t)
    X = np.atleast_1d(X)
    # Construct grid of points for spectrum
    N = np.size(t)
    grid = np.arange(N)
    dt = (max(t) - min(t))/(N - 1)
    # Frequency axis
    omega = 4*np.pi/N*grid
    # Shift zero-frequency component to center of spectrum
    omega = omega - 0.5*max(omega)
    # Scale by Nyquist criterion
    omega = omega/(2*dt)
    # Calculate Fourier transform operator
    FToperator = np.exp(-1j*np.outer(omega.T,t))
    # Transform signal
    Y = FToperator@X
    return omega,Y

# Inverse Fourier transformation
def compute_ifft(t,omega,X):
    """ Manually compute the FT of a simple signal"""
    t = np.atleast_1d(t)
    omega = np.atleast_1d(omega)
    X = np.atleast_1d(X)
    # Calculate Fourier transform operator
    FToperator = np.exp(+1j*np.outer(t,omega.T))
    # Transform signal
    Y = FToperator@X
    return Y

```

# Aufgabe 1: Lineare Antwort
----------------------

Gegeben ist einen Input EM-Puls $ x(t) $ der durch einen Kabel zum Spektrometer geschickt wird. Am Ende des Kabels wird der Output Puls $ y(t) $ neu gemessen.
Die Input und Output Pulsen sind in der Numpy-formattierte Datei `pulses.npz` gespeichert.

1) Plotten Sie die gemessene Pulsen. Welche Unterschieden sehen Sie?
2) Berechnen und vergleichen Sie die Pulsen im Frequenzbereich. Benutzen Sie dafür die vorgegebene Funktion `compute_fft`.
3) Berechnen Sie die Antwortsfunktion des Kabels. Benutzen Sie dafür die vorgegebene Funktion `compute_ifft`.
4) Wie würden Sie die gewünschte Puls-Form im Spektrometer erreichen?

```python
# Load external data file
data = np.load('pulses.npz')
# Extract the variables
t = data['t']
pulse_input = data['input']
pulse_output = data['output']
```

## Aufgabe 2: Faltung
--------------------

Berechnen Sie numerisch die Faltung einer Rechteckfunktion (box function) mit einer identischen Rechteckfunktion (a) sowie die Faltung einer Rechteckfunktion mit einer Gauss-Funktion (b):

Benutzen Sie dazu die Numpy-Funktion ``convolve``. Definieren Sie alle Ausgangfunktionen zwischen $ t = -10 $ und $ t=10 $ mit 2001 Punkten. Konstruieren Sie eine Rechteckfunktion die eine Breite von 18.0 hat und eine Gauss-Funktion mit einer Standardabweichung $ \sigma = 1 $. Die Zeitachse der Faltung soll explizit berechnet werden.\\
\textit{Hinweis:} Um den Vektor $ t $ zu kreieren können Sie den Befehl ``t = np.linspace(-10,10,2001)`` verwenden. Für die Gaussfunktion brauchen sie die foglende Definition:

\begin{equation}
f(x,\mu,\sigma) = \frac{1}{\sigma \sqrt{2\pi}}\exp\left(-\frac{(x - \mu)^2}{2\sigma^2} \right)
\end{equation}

```python

```

## Aufgabe 3: Fourier-Transformation
--------------------

Berechnen Sie numerisch die Fourier Transformierten der folgenden Funktionen:

(a) Rechteckfunktion mit der Breite $ \Delta t = 25.0 $, zentriert um $ t=0 $.

(b) Rechteckfunktion mit der Breite $ \Delta t = 5.0 $, zentriert um $ t = 0 $.

(c) Rechteckfunktion mit der Breite $ \Delta t = 25.0 $, zentriert um $ t=0 $, multipliziert mit einer harmonischen Funktion $ \cos(2\pi t) $, mit $ \nu = 1.0 $.

(d) Rechteckfunktion mit der Breite $ \Delta t = 5.0 $, zentriert um $ t=25 $.

*Hinweis:* Verwenden Sie für Ihre Berechnungen die folgende Zeitachse:
``t = np.linspace(-50,50,4096); # Zeitachse der Fourier Transformation``

```python

```


# Elementary frequency domain filtering

**Author: Uzhva Denis Romanovich**

**Lecturer: Soloviev Igor Pavlovich**

## The basics of filtering in the frequency domain
### Theory

A concept of frequency is very important in the study of signal processing.
Frequency is the number of occurrences of a repeating event per unit of time.
In order to obtain a spectrum (and phases) of a signal, e.g. a set of frequencies, we usually use the discrete fourier transformation.
Since an image has two spatial dimensions, the transformation is also two-dimensional.
The corresponding equation is as follows:
$$
\begin{equation}
F(u, v) = \sum_{x=0}^{H-1} \sum_{y=0}^{W-1} f(x, y) e^{-j2\pi (ux/H + vy/W)},
\tag{1}
\end{equation}
$$
where $f(x, y)$ is a digital image of size $H \times W$.

### Code

#### Discrete fourier transformation

```python
import numpy as np

def get_dft(img, shift=False):
    h = img.shape[0]
    w = img.shape[1]
    img_dft = np.zeros_like(img, dtype=np.complex)
    img = img.astype(np.complex)

    x_arr = np.arange(h)
    y_arr = np.arange(w)

    x_mat = np.repeat(x_arr.reshape((h, 1)), w, 1)
    y_mat = np.repeat(y_arr.reshape((1, w)), h, 0)

    if shift:
        exp_val = (x_mat + y_mat) * np.pi * 1j
        img = np.multiply(img, np.exp(exp_val))

    if len(img.shape) == 3:
        channels = img.shape[-1]
        for ch in range(channels):
            for u in range(h):
                for v in range(w):
                    exp_val = -(u * x_mat / h + v * y_mat / w) * np.pi * 2j
                    img_dft[u, v] = np.sum(np.multiply(img[:, :, ch], np.exp(exp_val)))
    else:
         for u in range(h):
            for v in range(w):
                exp_val = (u * x_mat / h + v * y_mat / w) * np.pi * 2j
                img_dft[u, v] = np.sum(np.multiply(img, np.exp(exp_val)))

    return img_dft
```

#### Inverse DFT

```python
import numpy as np

def get_idft(img_dft, shift=False):
    h = img_dft.shape[0]
    w = img_dft.shape[1]
    img_idft = np.zeros_like(img_dft, dtype=np.complex)
    img_dft = img_dft.astype(np.complex)

    u_arr = np.arange(h)
    v_arr = np.arange(w)

    u_mat = np.repeat(u_arr.reshape((h, 1)), w, 1)
    v_mat = np.repeat(v_arr.reshape((1, w)), h, 0)

    if shift:
        exp_val = (u_mat + v_mat) * np.pi * 1j
        img_dft = np.multiply(img_dft, np.exp(exp_val))

    if len(img_dft.shape) == 3:
        channels = img_dft.shape[-1]
        for ch in range(channels):
            for x in range(h):
                for y in range(w):
                    exp_val = (x * u_mat / h + y * v_mat / w) * np.pi * 2j
                    img_idft[x, y] = np.sum(np.multiply(img_dft[:, :, ch], np.exp(exp_val)))
    else:
        for x in range(h):
            for y in range(w):
                exp_val = (x * u_mat / h + y * v_mat / w) * np.pi * 2j
                img_idft[x, y] = np.sum(np.multiply(img_dft, np.exp(exp_val)))

    img_idft /= (h * w)

    return img_idft
```

### Results

#### Processing + visualization

First of all, we need to load images for the further processing.
We load images with quite simple patterns in order to describe the relations between the frequency and spatial domains.

```python
import numpy as np
import matplotlib.pyplot as plt
from PIL import Image

img_1 = Image.open('./pattern1.png')
img_2 = Image.open('./pattern2.png')
```

```python
# represent the images as tensors
np_1 = np.array(img_1)
np_2 = np.array(img_2)
print('Picture dimensions:')
print(np_1.shape)
print(np_2.shape)
```

    Picture dimensions:
    (64, 64, 3)
    (64, 64, 3)

```python
# simplify the images by summing the channels
np_1_sum = np.sum(np_1, axis=2) // 3
np_2_sum = np.sum(np_2, axis=2) // 3
```

Apply DFT to the images:

```python
np_1_dft = get_dft(np_1_sum)
```

```python
np_2_dft = get_dft(np_2_sum)
```

```python
# log of abs

np_1_dft_logabs = np.log(np.abs(np_1_dft))
np_2_dft_logabs = np.log(np.abs(np_2_dft))
```

```python
fig, axs = plt.subplots(2, 2, figsize=(15, 15), dpi=120)
axs[0, 0].imshow(np_1_sum, cmap='gray')
axs[0, 0].set_title('Original of the 1st image')
axs[0, 1].matshow(np_1_dft_logabs, cmap='gray')
axs[0, 1].set_title('Abs. of DFT of the 1st image')
axs[1, 0].imshow(np_2_sum, cmap='gray')
axs[1, 0].set_title('Original of the 2nd image')
axs[1, 1].matshow(np_2_dft_logabs, cmap='gray')
axs[1, 1].set_title('Abs. of DFT of the 2nd image')
plt.show()
```

It can be seen that patterns emerge in the frequency domain, which correspond to the repetative nature of the patterns on the original images.
The 1st image is espetially interesting, since the pattern looks like a "sawtooth" signal, so that is depicted on the DFT.

## Simple band-rejection filter
### Theory

The idea is to filter an image in the frequency domain, so that we adjust certain frequencies of an image.
The band-rejection (or band-stop) filter is a very simple filter that nullifies certain frequencies or just makes them very low.
In order to apply such a filter, we use an operation of convolution in the spatial domain, which corresponds to multiplication in the frequency domain:
$$
\begin{equation}
f(x, y) * h(x, y) \iff F(u, v) H(u, v),
\tag{1}
\end{equation}
$$
where $*$ is a convolution, $f(x,y)$ and $F(u, v)$ are an image and its fourier transform, while $h(x, y)$ and $H(u, v)$ stand for a filter and its representation in the frequency domain obtained by DFT.
With that in mind, the simplest band-rejection filter can be defined as follows:
$$
\begin{equation}
H(u, v) =
\begin{cases}
      0, & \text{if}\ u = H/2, v = W/2 \\
      1, & \text{otherwise}
    \end{cases}
\tag{1}
\end{equation}
$$

### Code

#### Center frequency rejection

```python
import numpy as np

def apply_cr_H(img_dft, size=3):
    h2 = img_dft.shape[0] // 2
    w2 = img_dft.shape[1] // 2

    img_dft[h2-size:h2+size+1, w2-size:w2+size+1] = 0.j

    return(img_dft)
```

### Results

#### Processing + visualization

```python
import numpy as np
import matplotlib.pyplot as plt
from PIL import Image

img_21 = Image.open('./int.jpg')
img_22 = Image.open('./flux.jpg')
```

```python
# represent the images as tensors
np_21 = np.array(img_21)
np_22 = np.array(img_22)
print('Picture dimensions:')
print(np_21.shape)
print(np_22.shape)
```

    Picture dimensions:
    (128, 128, 3)
    (128, 128, 3)

```python
# simplify the images by summing the channels
np_21_sum = np.sum(np_21, axis=2)
np_22_sum = np.sum(np_22, axis=2)
```

Apply DFT and filter to the images:

```python
np_21_dft = get_dft(np_21_sum, True)
```

```python
np_22_dft = get_dft(np_22_sum, True)
```

```python
np_21_dft_h = apply_cr_H(np_21_dft, 3)
np_22_dft_h = apply_cr_H(np_22_dft, 3)
```

Obtain IDFT:

```python
np_21_idft = get_idft(np_21_dft_h)
```

```python
np_22_idft = get_idft(np_22_dft_h)
```

```python
np_21_idft_abs = np.abs(np_21_idft)
np_22_idft_abs = np.abs(np_22_idft)
```

```python
np_21_idft_abs /= np.max(np_21_idft_abs)
np_22_idft_abs /= np.max(np_22_idft_abs)
```

```python
np_21_idft_abs[np_21_idft_abs < 0] = 0.
np_22_idft_abs[np_22_idft_abs < 0] = 0.
```

```python
np_21_idft_abs = np.rot90(np.rot90(np_21_idft_abs))
np_22_idft_abs = np.rot90(np.rot90(np_22_idft_abs))
```

```python
fig, axs = plt.subplots(2, 2, figsize=(15, 15), dpi=120)
axs[0, 0].imshow(np_21_sum, cmap='gray')
axs[0, 0].set_title('Original of the 1st image')
axs[0, 1].imshow(np_21_idft_abs, cmap='gray')
axs[0, 1].set_title('1st image after band reject filtering')
axs[1, 0].imshow(np_22_sum, cmap='gray')
axs[1, 0].set_title('Original of the 2nd image')
axs[1, 1].imshow(np_22_idft_abs, cmap='gray')
axs[1, 1].set_title('2nd image after band reject filtering')
plt.show()
```

It is clear that band reject filtering allowed to better recognize the defects on the circuits: it highlighted them by removing low frequencies, so that it is easier to detect anomalies.

```python

```

Task5 - Elementary frequency domain filtering/Elementary frequency domain filtering.ipynb


# Linear Algebra

## What is Linear Algebra?
Linear algebra is a branch of mathematics, but the truth of it is that linear algebra is the mathematics of data. Matrices and vectors are the language of data [Learn More](https://machinelearningmastery.com/gentle-introduction-linear-algebra/).

## Applications of Linear Algebra in Data Science
- Loss Functions
- Regularization
- Covariance Matrix
- Support Vector Machine Classification
- Principal Component Analysis (PCA)
- Singular Value Decomposition
- Word Embeddings
- Latent Semantic Analysis (LSA)
- Image Representation as Tensors
- Convolution and Image Processing

__SEE DETAILS:__
https://www.analyticsvidhya.com/blog/2019/07/10-applications-linear-algebra-data-science/

## Linear Algebra Operations

```python
# import numpy
import numpy as np
```

### Determinant of matrix
- The determinant of a matrix is a special number that can be calculated from a **square matrix**
\begin{equation}
A = \begin{bmatrix} a & b \\ c & d \end{bmatrix} \\
det(A) = ad - bc
\end{equation}
Whre, A is a $2 \times 2$ matrix.

- `np.linalg.det()` - performs determinant of the matrix
- **Syntax:** `np.linalg.det(matrix)`

__Source:__
https://www.mathsisfun.com/algebra/matrix-determinant.html

```python
# create matrix A
A = np.matrix("4, 5, 16, 7; 2,-3,2,3; 3,4,5,6; 4,7,8,9")
print(A)
```

    [[ 4  5 16  7]
     [ 2 -3  2  3]
     [ 3  4  5  6]
     [ 4  7  8  9]]

```python
# create matrix B
B = np.matrix("4,5,6,7;2,-3,3,3; 3,4,5,6; 4, 7,8,9")
print(B)
```

    [[ 4  5  6  7]
     [ 2 -3  3  3]
     [ 3  4  5  6]
     [ 4  7  8  9]]

```python
# determinant of A
np.linalg.det(A)
```

    128.00000000000009

```python
# determinant of B
np.linalg.det(B)
```

    12.000000000000005

### Rank of matrix
- `np.linalg.matrix_rank()` - returns rank of the matrix
- **Syntax:** `np.linalg.matrix_rank(matrix)`

```python
A.shape
```

    (4, 4)

```python
# rank of matrix A
np.linalg.matrix_rank(A)
```

    4

```python
# rank of matrix B
np.linalg.matrix_rank(B)
```

    4

### Inverse of a Matrix
- `np.linalg.inv()` - returns the multiplicative inverse of a matrix.
- **Syntax:** `np.linalg.inv(matrix)`

```python
# inverse of matrix A
np.linalg.inv(A)
```

    matrix([[ 9.37500000e-02, -4.68750000e-01,  3.68750000e+00,
             -2.37500000e+00],
            [ 3.53252781e-17, -2.50000000e-01,  5.00000000e-01,
             -2.50000000e-01],
            [ 9.37500000e-02,  3.12500000e-02, -3.12500000e-01,
              1.25000000e-01],
            [-1.25000000e-01,  3.75000000e-01, -1.75000000e+00,
              1.25000000e+00]])

```python
# inverse of matrix B
np.linalg.inv(B)
```

    matrix([[ 1.50000000e+00, -2.08166817e-17, -1.00000000e+00,
             -5.00000000e-01],
            [ 2.50000000e-01, -1.66666667e-01, -3.33333333e-01,
              8.33333333e-02],
            [ 1.00000000e+00,  3.33333333e-01, -3.33333333e+00,
              1.33333333e+00],
            [-1.75000000e+00, -1.66666667e-01,  3.66666667e+00,
             -9.16666667e-01]])

03 - Working with NumPy/notebooks/09-Linear Algebra.ipynb


# Linear Regression

In this notebook, we will show you how to implement linear regression from scratch.

```python
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from util import print_result

sns.set()
%matplotlib inline
```

## Introduction

In this chapter, we introduce linear models for regression applications. Linear models are relatively simple models but it is well worth studying linear models due to two main reasons. If the relationship between input features (denoted by $\mathbf{x}$) and response variable (denoted by $\mathbf{y}$) linear models perform better than more advanced machine learning models. Secondly, a firm understanding of linear models will help us to easily grasp more advanced concepts we will study under the non-linear models.

## Let’s Predict Car Price

Let’s start our discussion with a simple real-world example. Assuming you are running a secondhand car showroom. The main object of your business is to sell used cars to potential customers. In order to prevent losing customers and hence, maximizing your profile, it is necessary to assign correct price tag for each car you are selling in your showroom.
Since you are somewhat familiar with machine learning you are are going to address this problem using linear regression. From your previous experience, you are well aware a price tag of a car depends on several parameters such as mileage, manufacturing year, and engine capacity. For simplicity initially, we assume that price depends only on mileage, manufacturing year. So you collect those data from your previous successful deals and presented as shown in **Table 1**.

|Mileage ( in 10,000 km )|Manufacturing Year| Price in ( 1000 USD)|
| :------------- |:-------------| :-----|
| 5      | 2010     |5.5  |
| 10     | 2000     |3.1  |
| 12     | 1999     |2.1  |
| 4      | 2012     |5.1  |
| 8      | 2001     |2.1  |
| 7      | 2002     |2.2  |
| 15     | 1997     |0.8  |
| 3      | 2008     |3.5  |
| 6.5    | 2003     |4.0  |
| 3.4    | 2013     |5.1  |

In order to better understand the relationship between car price and manufacturing year, let’s draw a scatter plot.

Figure 1: Distribution of car prices against the manufacturing year.

According to our scatter plot, it looks like we can use a straight line to represent the relationship between the manufacturing year and the car price. So if I use $x$ to represent manufacturing year and $y$ is to represent car price we can use $y = w_1*x + w_0$ equation to represent the relationship between the manufacturing year and the car price. However, from your high-school mathematics, you know just two points are enough for drawing a straight line. Hence, we can draw so many lines. Two of those lines are shown in **Figure 2**.

Figure 2: Three possible straight lines

Now the question is how we can select the best straight line which correctly describes the relationship. In Figure 3 we show one possible straight line. Also, distances between line and the points which are not on the line are denoted by $\Delta$. So the basic idea of the linear regression is to find out a straight line which minimizes the error terms.

Figure 3: Error distances

So if we consider one particular prediction (denoted by $f(x_n; w_0, w_1)$  and coresponsing actual target value (denoted by $t_n$), the error between actual and predicted values can be contified by:

$$\Delta_n = (t_n - f(x_n; w_0, w_1))^2$$

Considering all N data points this can be written as:

$$L(\mathbf{w}) = \frac{1}{N}\sum_{i=1}^N(t_n - f(x_n; w_0, w_1)^2$$

The same thing can be written in matrix notation as given below.

$$L(\mathbf{w}) = \frac{1}{N}(\mathbf{t} - \mathbf{Xw})^T(\mathbf{t} - \mathbf{Xw})$$

So now we want to find $\mathbf{w}$  in order to minimize $L(\mathbf{w})$.

$$
\begin{align}
  L(\mathbf{w}) & = \frac{1}{N}(\mathbf{t} - \mathbf{Xw})^T (\mathbf{t} - \mathbf{Xw})\\
                & = \frac{1}{N}(\mathbf{t}^T - \mathbf{w^TX^T})(\mathbf{t} - \mathbf{Xw}) \\
                & = \frac{1}{N}(\mathbf{tt^T} - \mathbf{t^TXw} - \mathbf{w^TX^Tt} + \mathbf{w^TX^TXw}) \\
                & = \frac{1}{N}(\mathbf{tt^T} -2\mathbf{w^TX^Tt} + \mathbf{w^TX^TXw}) \\
\end{align}
$$

Please note that, both $ \mathbf{t^TXw}$ and $\mathbf{w^TX^Tt}$ are scalars and transpose of one another. Hece, we use $2\mathbf{w^TX^Tt}$. Next, we take the derivative of $L(\mathbf{w})$ w. r. t. $\mathbf{w}$. Also, it is worth to mention the following useful identities when differentiating w. r. t. a vector.

| $f(\mathbf{w})$                | $\frac{{\partial f}}{ {\partial \mathbf{w}}}$ |
|:-------------------------------:|:---------------:|
| $\mathbf{w^T}x$                | $\mathbf{x}$             |
| $\mathbf{x^Tw}$                | $\mathbf{w^T}$ |
| $\mathbf{w^Tw}$ | $\mathbf{2\mathbf{w}}$ |
| $\mathbf{w^TCw}$ | $\mathbf{2\mathbf{Cw}}$ |
Table 2: Useful identities when differentiating w. r. t. a vector

$$
\begin{align}
  \Delta_{\mathbf{w}}L & = \frac{1}{N}\Big[\mathbf{0} - 2\mathbf{X^Tt} + 2\mathbf{X^TXw}\Big] \\
                       & = \frac{2}{N}\Big[\mathbf{X^TXw} - \mathbf{X^Tt}\Big] \\
                       & = \frac{2\mathbf{X^T}}{N}\Big[\mathbf{Xw} - \mathbf{t}\Big] \\
\end{align}
$$

Great, now we have calculated the gradient of the loss function w.r.t. w. Next, we are going to use Gradient Descent to implement our very own linear regression algorithm in Python.

```python
class LinearRegression:
    def __init__(self, learning_rate=1e-3, max_iters=250):
        self.learning_rate = learning_rate
        self.max_iters = max_iters
        self.weights = None

    def fit(self, X_train, y_train):
        X_train = self._fit_intercept(X_train)
        self.weights = np.random.rand(X_train.shape[1])

        for iter in range(self.max_iters):
            self.weights = self.weights - self.learning_rate * \
                self._grad(X_train, y_train, self.weights)
            cost = self._cost(X_train, y_train, self.weights)
            print_result(iter, cost)

    def predict(self, X_predict):
        """ This method predicts responses for new data.

        This method will be used for end-users to predict responses
        for future data.

        Arguments:
            X_predict: new data

        Returns: a vector of responses
        """
        return X_predict @ self.weights

    @staticmethod
    def _cost(X_train, y_train, weights):
        """Calculates the cost w.r.t. weights

        This helper method calculates the cost w.r.t. weights

        Arguments:
            X_train: training features
            y_train: training labels.
            weights: weights vector of the model

        Returns: scalar value which represents the cost
        """
        dif = (y_train - X_train @ weights)
        return (dif.T @ dif) / X_train.shape[0]

    @staticmethod
    def _fit_intercept(X_train):
        """Add the intercepting term to the training dataset

        This simple helper method adds the intercepting term to the
        training dataset.

        Arguments:
            X_train: training dataset

        Returns: training dataset with the interceptor
        """
        return np.column_stack((np.ones(X_train.shape[0]), X_train))

    @staticmethod
    def _grad(X_train, y_train, weights):
        """Calculates the gradient of the cost function w.r.t. weight vector

        This methods calculated the gradient of the cost function w.r.t.
        weight vector.

        Arguments:
            X_train: training features
            y_train: training labels.
            weights: weights vector of the model

        Returns: scalar value which represents the cost
        """
        return 2 * (X_train.T @ (X_train @ weights - y_train)) / X_train.shape[0]
```

```python
X_train = np.array([2010, 2000, 1999, 2012, 2001, 2002,
                        1997, 2008, 2003, 2013]).reshape((10, 1))
X_train = X_train - X_train.min()
y_train = np.array([5.5, 3.1, 2.0, 6.1, 2.1, 2.2, 0.8, 3.5, 4.0, 5.1])
model = LinearRegression()
model.fit(X_train, y_train)

predicted_points = np.linspace(X_train.min(), X_train.max(), 100) * \
        model.weights[1] + model.weights[0]

fig, axes = plt.subplots(1, 2, figsize=(14, 6))

axes[0].scatter(X_train, y_train, s=75, color='#dd1c77', alpha=0.5, edgecolors='none')
axes[0].set_title('Car Price Dataset')
axes[0].set_xlabel('Car Price in 1000 USD')
axes[0].set_ylabel('Manufacturing Year - 1996')

axes[1].scatter(X_train, y_train, s=75, color='#dd1c77', alpha=0.5, edgecolors='none')
axes[1].plot(np.linspace(X_train.min(), X_train.max(), 100),
                 predicted_points, color='#c994c7', linewidth=3, alpha=0.5)
axes[1].set_title('Car Price Dataset with Fitted Line')
axes[1].set_xlabel('Car Price in 1000 USD')
axes[1].set_ylabel('Manufacturing Year - 1996')

fig.suptitle('Training a simple Linear Regression to car prices dataset')
plt.show()
```

## Summary

In this notebook, we implemented linear regression algorithms form the scratch. We started with necessary assumptions and derived all necessary mathematical formula. Next, we converted those formulae into python code.

## References and Further Reading


# Relax and hold steady

## Stokes flow

This notebook presents the coding assignment for **Module 5** of the course [*"Practical Numerical Methods with Python.*](https://github.com/numerical-mooc/numerical-mooc) Your mission is to solve Stokes flow in a square cavity, using the vorticity-streamfunction formulation.
Stokes flow, also known as *creeping flow*, refers to flows that are dominated by viscous forces and not by the advective/convective forces.  The Stokes-flow assumption works well for flows that have very low Reynolds number, much smaller than 1: very slow, highly viscous or flows at microscopic length scales.

Stokes flow allows us to simplify the Navier-Stokes equations, eliminating the non-linearity.  Let's run through a quick derivation of the vorticity-transport equation with Stokes-flow assumptions.

### Vorticity

We start with the Navier-Stokes equations for incompressible flow:

\begin{equation}
\frac{\partial u}{\partial t} + u \cdot \nabla u = -\frac{1}{\rho}\nabla p + \nu\nabla^2 u
\end{equation}

If we scale Equation $(1)$ to make it non-dimensional, we can rewrite it as

\begin{equation}
Re \left(\frac{\partial u^*}{\partial t} + u^* \cdot \nabla u^* \right) = -\nabla p^* + \nabla^2 u^*
\end{equation}

Where $u^*$ and $p^*$ are the non-dimensional velocity and pressure, respectively.

To obtain Stokes flow, we assume that the Reynolds number approaches zero.  Applying that assumption to Equation $(2)$ and dropping the stars, yields

\begin{equation}
0 = - \nabla p + \nabla^2 u
\end{equation}

That simplified things!  Now, we apply the curl operator on both sides of the equation:

\begin{equation}
\nabla \times 0 = \nabla \times \left( - \nabla p + \nabla^2 u\right)
\end{equation}

The left-hand side remains zero, while the first term on the right-hand side is

\begin{equation}
\nabla \times - \nabla p = 0
\end{equation}

because $\nabla \times \nabla \phi = 0$ where $\phi$ is a scalar.

Finally,

\begin{equation}
\nabla \times \nabla^2 u =\nabla^2\omega
\end{equation}

where $\nabla \times u = \omega$ is the vorticity.

Combining all of these equations, we arrive at the simplified vorticity transport equation for Stokes flow:

\begin{equation}
\nabla ^2 \omega = 0
\end{equation}

Look familiar?

### Stream function

Define the stream function $\psi$, such that

\begin{equation}
u = \frac{\partial \psi}{\partial y} \text{   and   } v = - \frac{\partial \psi}{\partial x}
\end{equation}

In 2D, we can write out the vorticity as

\begin{equation}
\omega = \frac{\partial v}{\partial x} - \frac{\partial u}{\partial y}
\end{equation}

which, combined with the previous equation yields another familiar looking equation:

\begin{equation}
\nabla^2 \psi = -\omega
\end{equation}

We have a system of two coupled equations that can describe the fluid flow in a lid-driven cavity at very low Reynolds numbers.

\begin{equation}
\nabla^2 \omega = 0
\end{equation}

\begin{equation}
\nabla^2 \psi = -\omega
\end{equation}

Note that by substituting Equation $(12)$ into $(11)$, we arrive at a new equation: the *biharmonic equation*:

\begin{equation}
\nabla^4 \psi= 0
\end{equation}

### Solving the biharmonic equation

Is it possible to discretize a 4th-order partial differential equation?  Of course!  Are we going to?  No!

There's nothing wrong with a 4th-order equation, but in this course module we learned about the Poisson equation and that's what we're going to use.

## Cavity flow

You will solve a problem called *lid-driven cavity flow*.  This is a common test problem for Navier-Stokes solvers—we'll be using it to examine Stokes flow.

Assume that the lid of a square cavity moves at a constant velocity of $u=1$, with no fluid leaking out past the moving lid. We we want to visualize what the flow field inside the cavity looks like at steady state.

All of the surfaces, including the lid, are assumed to have no-slip boundary conditions.  The boundary conditions are all specified in terms of the streamfunction $\psi$, as shown below in Figure $(1)$.

#### Figure 1. Lid-driven Cavity Flow

### Boundary conditions

One of the major hurdles with the vorticity-streamfunction formulation is the treatment of boundary conditions.

The boundary conditions are all specified in terms of $\psi$ and its derivatives, but the Laplace equation

$$\nabla \omega^2 = 0$$

has no $\psi$ value.  Instead, we need a way to represent the boundary conditions for $\omega$ in terms of $\psi$.

Consider the equation $\nabla ^2 \psi = -\omega$ along the top surface of the cavity (the moving surface).  There can't be any velocity in the $y$-direction because the surface is solid, so $\frac{\partial ^2 \psi}{\partial x^2}$ goes to zero and the equation simplifies to

\begin{equation}
\frac{\partial ^2 \psi}{\partial y^2} = -\omega
\end{equation}

A 2nd-order central difference discretization gives

\begin{equation}
\omega_j = - \left(\frac{\psi_{j+1} - 2\psi_j + \psi_{j-1}}{\Delta y^2}\right)
\end{equation}

but the value $\psi_{j+1}$ is outside of the domain.  Now take a 3rd-order discretization of $\frac{\partial \psi}{\partial y}$ evaluated along the top edge.

\begin{equation}
\left.\frac{\partial \psi}{\partial y}\right|_j = \frac{2\psi_{j+1} + 3\psi_j - 6\psi_{j-1} + \psi_{j-2}}{6 \Delta y}
\end{equation}

$\frac{\partial \psi}{\partial y}$ is a given boundary value in the problem along the top edge

\begin{equation}
\left.\frac{\partial \psi}{\partial y}\right|_j = u_j
\end{equation}

which leaves us with a value for $\psi_{j+1}$ that consists only of points within the domain.

\begin{equation}
\psi_{j+1} = \frac{6\Delta y u_j - 3\psi_j + 6 \psi_{j-1} - \psi_{j-2}}{2}
\end{equation}

Plug in that result into the initial discretization from Equation $(16)$ and we have a boundary condition for $\omega$ along the top surface in terms of $\psi$:

\begin{equation}
\omega_{i,j} = -\frac{1}{2 \Delta y^2} (8\psi_{i, j-1} - \psi_{i, j-2}) - \frac{3u_j}{\Delta y} + \mathcal{O}(\Delta y^2)
\end{equation}

### Coding assignment

Solve Stokes flow in a lid-driven cavity using the parameters given below.

You should iteratively solve for both $\omega$ and $\psi$ until the L1 norm of the difference between successive iterations is less than $1$$\tt{E}$$^-6$ for **both** quantities.

```python
nx = 41
ny = 41

l = 1.
h = 1.

dx = l/(nx-1)
dy = h/(ny-1)

l1_target = 1e-6
```

```python
def L1norm(new, old):
    norm = numpy.sum(numpy.abs(new-old))
    return norm
```

The final result should resemble the plot shown in Figure $(2)$.

##### Hint

The boundary conditions for $\omega$ depend upon the current value of $\psi$.  The two equations are *coupled*.  If you try to solve them in a *uncoupled* way, things will go poorly.

#### Figure 2.  Contour plot of streamfunction at steady state

### References

*  Fletcher, C. A. (1988). Computational Techniques for Fluid Dynamics: Volume 2: Specific Techniques for Different Flow Categories.

*  Ghia, U. K. N. G., Ghia, K. N., & Shin, C. T. (1982). High-Re solutions for incompressible flow using the Navier-Stokes equations and a multigrid method. Journal of computational physics, 48(3), 387-411.

* Greenspan, D. (1974). Discrete numerical methods in physics and engineering (Vol. 312). New York: Academic Press.

* Heil, Matthias (Year unknown).  [Viscous Fluid Flow Vorticity Handout (pdf)](http://www.maths.manchester.ac.uk/~mheil/Lectures/Fluids/Material/Vorticity.pdf)

* Non-dimensionalization and scaling of the Navier Stokes equations.  (n.d.). In *Wikipedia*.  Retrieved January 30, 2015 [http://en.wikipedia.org/w/index.php?title=Non-dimensionalization_and_scaling_of_the_Navier-Stokes_equations](http://en.wikipedia.org/w/index.php?title=Non-dimensionalization_and_scaling_of_the_Navier%E2%80%93Stokes_equations&oldid=641860920)

```python
from IPython.core.display import HTML
css_file = '../../styles/numericalmoocstyle.css'
HTML(open(css_file, "r").read())
```

<link href='http://fonts.googleapis.com/css?family=Alegreya+Sans:100,300,400,500,700,800,900,100italic,300italic,400italic,500italic,700italic,800italic,900italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Arvo:400,700,400italic' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=PT+Mono' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Shadows+Into+Light' rel='stylesheet' type='text/css'>
<link href='http://fonts.googleapis.com/css?family=Nixie+One' rel='stylesheet' type='text/css'>
<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro' rel='stylesheet' type='text/css'>
<style>

@font-face {
    font-family: "Computer Modern";
    src: url('http://mirrors.ctan.org/fonts/cm-unicode/fonts/otf/cmunss.otf');
}

#notebook_panel { /* main background */
    background: rgb(245,245,245);
}

div.cell { /* set cell width */
    width: 750px;
}

div #notebook { /* centre the content */
    background: #fff; /* white background for content */
    width: 1000px;
    margin: auto;
    padding-left: 0em;
}

#notebook li { /* More space between bullet points */
    margin-top:0.8em;
}

/* draw border around running cells */
div.cell.border-box-sizing.code_cell.running {
    border: 1px solid #111;
}

/* Put a solid color box around each cell and its output, visually linking them*/
div.cell.code_cell {
    background-color: rgb(256,256,256);
    border-radius: 0px;
    padding: 0.5em;
    margin-left:1em;
    margin-top: 1em;
}

div.text_cell_render{
    font-family: 'Alegreya Sans' sans-serif;
    line-height: 140%;
    font-size: 125%;
    font-weight: 400;
    width:600px;
    margin-left:auto;
    margin-right:auto;
}

/* Formatting for header cells */
.text_cell_render h1 {
    font-family: 'Nixie One', serif;
    font-style:regular;
    font-weight: 400;
    font-size: 45pt;
    line-height: 100%;
    color: rgb(0,51,102);
    margin-bottom: 0.5em;
    margin-top: 0.5em;
    display: block;
}

.text_cell_render h2 {
    font-family: 'Nixie One', serif;
    font-weight: 400;
    font-size: 30pt;
    line-height: 100%;
    color: rgb(0,51,102);
    margin-bottom: 0.1em;
    margin-top: 0.3em;
    display: block;
}

.text_cell_render h3 {
    font-family: 'Nixie One', serif;
    margin-top:16px;
    font-size: 22pt;
    font-weight: 600;
    margin-bottom: 3px;
    font-style: regular;
    color: rgb(102,102,0);
}

.text_cell_render h4 {    /*Use this for captions*/
    font-family: 'Nixie One', serif;
    font-size: 14pt;
    text-align: center;
    margin-top: 0em;
    margin-bottom: 2em;
    font-style: regular;
}

.text_cell_render h5 {  /*Use this for small titles*/
    font-family: 'Nixie One', sans-serif;
    font-weight: 400;
    font-size: 16pt;
    color: rgb(163,0,0);
    font-style: italic;
    margin-bottom: .1em;
    margin-top: 0.8em;
    display: block;
}

.text_cell_render h6 { /*use this for copyright note*/
    font-family: 'PT Mono', sans-serif;
    font-weight: 300;
    font-size: 9pt;
    line-height: 100%;
    color: grey;
    margin-bottom: 1px;
    margin-top: 1px;
}

.CodeMirror{
    font-family: "Source Code Pro";
    font-size: 90%;
}

.alert-box {
    padding:10px 10px 10px 36px;
    margin:5px;
}

.success {
    color:#666600;
    background:rgb(240,242,229);
}
</style>


```python
import numpy as np
import scipy as sp
import scipy.stats
import matplotlib.pyplot as plt
import math as mt
import scipy.special
import seaborn as sns
plt.style.use('fivethirtyeight')
from statsmodels.graphics.tsaplots import plot_acf
import pandas as pd
```

#  Gibbs Sampling Algorithm

The **Gibbs sampler** is a special case of the Metropolis sampler in which the proposal distributions exactly match the posterior conditional distributions and naturally proposals are accepted 100% of the time.

However a specialty of Gibbs Sampler is that can allow one to estimate multiple parameters.

In this section, we will use Normal-Normal and Gamma-Normal conjugate priors to demonstrate the algorithm of Gibbs sampler.

Suppose you want to know the average height of female in your city, in the current setting, we assume $\mu$ and $\tau$ are our parameters of interest for estimation. Note that in conjugate prior section we assumed $\tau$ to be known, however in Gibbs sampling, both can be estimated.

A prior of _normal distribution_ will be assumed for $\mu$ with hyperparameters
$$
\text{inverse of }\sigma_0:\tau_0 = .35\\
\text{mean}:\mu_0 = 170
$$

A prior of _gamma distribution_ will be assumed for $\tau$ since it can't be negative.
$$
\text{shape}: \alpha_0 = 2\\
\text{rate}:\beta_0 = 1
$$

The priors graphically are

```python
mu_0, tau_0 = 170, .35
x_mu = np.linspace(150, 190, 100)
y_mu = sp.stats.norm(loc=mu_0, scale=1/tau_0).pdf(x_mu)

alpha_0, beta_0 = 2, 1
x_tau = np.linspace(0, 8, 100)
y_tau = sp.stats.gamma(a=alpha_0, scale=1/beta_0).pdf(x_tau)

fig, ax = plt.subplots(figsize=(15,5), nrows=1, ncols=2)
ax[0].plot(x_mu, y_mu, label =r'$\mu_0={}, \tau_0={}$'.format(mu_0, tau_0))
ax[0].set_title(r'Prior of $\mu$')
ax[0].legend()
ax[1].plot(x_tau, y_tau, label = r'$\alpha_0={}, \beta_0={}$'.format(alpha_0, beta_0))
ax[1].set_title(r'Prior of $\tau$')
ax[1].legend()
plt.show()
```

Here is the joint frequency distribution of $\mu$ and $\tau$.

```python
n = 30000
mu_height = sp.stats.norm(loc=mu_0, scale=1/tau_0).rvs(n)
tau_height = sp.stats.gamma(a=alpha_0, scale=1/beta_0).rvs(n)
fig, ax = plt.subplots(figsize=(8,5))
ax.scatter(mu_height, tau_height, s = 3)
plt.show()
```

```python

```

Choose an initial value of proposal of $\tau$, denoted as $\tau_{\text{proposal},0}$, the $0$ subscript represents the time period, since this is the initial value.

Say
$$
\tau_{\text{proposal},0} = 7
$$

Next step is to obtain
$$
\mu_{\text{proposal},0}|\tau_{\text{proposal},0}
$$
where $\mu_{\text{proposal},0}$ is the first value of proposal $\mu$ conditional on $\tau_{\text{proposal},0}$.

Now go collect some data, for instance you measured $10$ random women's heights, here's the data.

```python
heights = np.array([156, 167, 178, 182, 169, 174, 175, 164, 181, 170])
np.sum(heights)
```

    1716

Recall we have a sets of analytical solutions on Normal-Normal conjugate derived in chapter 2

\begin{align}
\mu_{\text {posterior }} &=\frac{\tau_{0} \mu_{0}+\tau \sum x_{i}}{\tau_{0}+n \tau}\\
\tau_{\text {posterior }} &=\tau_{0}+n \tau
\end{align}

Substitute $\tau_{\text{proposal},0}$ into both formula.

$$
\mu_{\text {posterior},1} =\frac{\tau_{0} \mu_{0}+\tau_{\text{proposal},0} \sum_{i=0}^{100} x_{i}}{\tau_{0}+n \tau_{\text{proposal},0}}=\frac{.15\times170+7\times 1716}{.15+10\times7}\\
\tau_{\text {posterior}, 1} =\tau_{0}+n \tau_{\text{proposal},0} = .15 + 10\times 7
$$

```python
mu_post = [0]
tau_post = [0] # 0 is placeholder, there isn't 0th eletment, according to algo

tau_proposal = [7]
mu_proposal = [0] # 0 is placeholder

mu_post.append((.15*170+tau_proposal[0]*1716)/(.15+10*tau_proposal[0]))
tau_post.append(.15+10*tau_proposal[0])
```

Draw a proposal from updated distribution for $\mu$, that is $\mu_{\text{posterior}, 1}$ and $\tau_{\text{posterior}, 1}$

```python
mu_proposal_draw = sp.stats.norm(loc=mu_post[1], scale=1/tau_post[1]).rvs()
mu_proposal.append(mu_proposal_draw)
```

Now turn to $\tau$ for a proposal, there is also Gamma-Normal conjugate prior which we didn't derive before. But here are the posterior
\begin{align}
\alpha_{\text{posterior}}&=\alpha_0+\frac{n}{2}\\
\beta_{\text{posterior}}&=\beta_0+\frac{\sum_{i=1}^{n}\left(x_{i}-\mu\right)^{2}}{2}
\end{align}

```python
alpha_post = [0]
beta_post = [0]

alpha_post.append(alpha_0+10/2)
beta_post.append(beta_0+np.sum((heights-mu_post[-1])**2)/2)

tau_proposal_draw = sp.stats.gamma(a=alpha_post[-1], scale=1/beta_post[-1]).rvs()
tau_proposal.append(tau_proposal_draw)
```

```python
tau_proposal
```

    [7, 0.01714209509457282]

#  Gibbs Sampling in a Loop

```python
mu_post, tau_post, alpha_post, beta_post, mu_proposal, chain_size = [], [], [], [], [], 10000

def gibbs_norm_gam_joint(mu_0=170, tau_0=.35, tau_proposal_init=7, alpha_0=2, beta_0=1, chain_size = chain_size, data=heights):
    tau_proposal = [tau_proposal_init]
    n = len(data)
    for i in range(chain_size):

        mu_post.append((tau_0*mu_0+tau_proposal[-1]*np.sum(data))/(tau_0+n*tau_proposal[-1]))
        tau_post.append(tau_0+n*tau_proposal[-1])

        mu_proposal_draw = sp.stats.norm(loc=mu_post[-1], scale=1/tau_post[-1]).rvs()
        mu_proposal.append(mu_proposal_draw)

        alpha_post.append(alpha_0+n/2)
        beta_post.append(beta_0+np.sum((heights-mu_post[-1])**2)/2)

        tau_proposal_draw = sp.stats.gamma(a=alpha_post[-1], scale=1/beta_post[-1]).rvs()
        tau_proposal.append(tau_proposal_draw)
    return mu_post, tau_post
```

```python
mu_post, tau_post = gibbs_norm_gam_joint()
```

Calculate the moment matching PDF for both distributions.

```python
x_mu = np.linspace(170, 172.2, 100)
mu_dist = sp.stats.norm(loc=np.mean(mu_post), scale=np.std(mu_post)).pdf(x_mu)
```

To acquire $\alpha$ and $\beta$, we need solve a system of equations
$$
\mu=\frac{\alpha}{\beta}\\
\sigma^2=\frac{\alpha}{\beta^2}
$$

```python
from scipy.optimize import fsolve

def solve_equ(x, mu=np.mean(tau_post), sigma=np.std(tau_post)):
    a, b = x[0], x[1]
    F = np.empty(2)
    F[0] = mu-x[0]/x[1]
    F[1] = sigma**2-x[0]/(x[1]**2)
    return F
xGuess = np.array([2, 2])
z = fsolve(solve_equ, xGuess)
print('alpha: {}'.format(z[0]))
print('beta: {}'.format(z[1]))
```

    alpha: 0.7043480496517439
    beta: 1.1937535586319403

```python
x_tau = np.linspace(.37, 1, 100)
tau_dist = sp.stats.gamma(a=z[0],scale=1/z[1]).pdf(x_tau)
```

```python
fig, ax = plt.subplots(figsize = (18, 12), nrows = 2, ncols = 2)
ax[0,0].hist(mu_post, bins=100, density=True)
ax[0,0].plot(x_mu, mu_dist, alpha=.7)
ax[0,0].set_ylabel('Posterior distribution of $\mu$')
ax[1,0].hist(tau_post[1:], bins=100, density=True)
ax[1,0].plot(x_tau, tau_dist, alpha=.7)
ax[1,0].set_ylabel(r'Posterior distribution of $\tau$')
ax[0,1].plot(np.arange(chain_size-1),mu_post[1:],lw=.5)
ax[1,1].plot(np.arange(chain_size-1),tau_post[1:], lw=.5)
fig.suptitle('Gibbs Sampling Example of Female Height', y = .93, size=26)
plt.show()
```

```python

```

```python

```


Let $D = \{ X_i:i \in \mathbb N \}$ be a data set of i.i.d. sequence from a random generator of distribution $\mathcal{N}(b, \sigma ^2)$ for some unknown parameters $b$ and $\sigma$. Our goal is to estimate $b$ using so called stochastic approximation ($\mathrm{SA}$) with a given learning rate $\alpha \in (0,1)$.

1. Write pseudocode for $\mathrm{SA}$.

The pseudocode for $\mathrm{SA}$ is shown as follows:

[pseudocode](https://github.com/Jun-629/20MA573/blob/master/src/Hw10_pseudocode.pdf)

2. Generate a data set $D = \{ x_i:1 \leq i \leq 10000 \}$ with $\mathcal{N}(1,4)$.

```
import numpy as np
import matplotlib.pyplot as plt

sampleNo = 10000
mu = 1
sigma = 4
#np.random.seed(0)
D = np.random.normal(mu, sigma, sampleNo)
#plt.plot(range(sampleNo), D)
```

3. Write a code to implement $\mathrm {SA}$ on $D$, and demonstrate $b_n \to b$ as $n \to \infty$.

```
def SA(b0,alpha,N=10000):                       # N is the number of the sequence of random variable Xi
  b = [b0]
  for k in range(N):
    b_k = b[k] + alpha * (D[k] - b[k])
    b.append(b_k)
  return b[N-1]
SA(2,0.01)
```

    1.0483728903289875

4. Prove that $\lim_n \mathbb E b_n = b$.

__Pf:__

Due to the fact in the lecture, we know that
$$\mu_k = \frac{1}{k} \sum_{i=1}^k x_i$$
is equivalent to
$$\mu_k = \mu_{k-1} + \frac{1}{k}(x_k - \mu_{k-1}).$$
Here, we already have that
$$b_{k+1} = b_k + \alpha(x_k - \mu_k), k \in [0,9999], k \in \mathbb N,$$
Thus, we deduce that
$$b_n = \frac{1}{n} \sum_{i=1}^n x_i, x_i \sim \mathcal{N}(b,\sigma^2).$$
Therefore, by LCT, we have
$$\lim_n \mathbb E b_n = \lim_n \mathbb E [\frac{1}{n} \sum_{i=1}^n x_i] = \lim_n \frac{1}{n}  \mathbb E [\sum_{i=1}^n x_i] = b.$$

5. Can you prove or disprove that $b_n \to b$ in $L^2$?

__Pf:__

If $b_n \to b$ in $L^2$, then we need to show that
$$\lim_n \mathbb E[|b_n - b|^2] = 0$$

\begin{equation}
\begin{aligned}
L.H.S &= \lim_n \mathbb E [b_n^2 - 2\cdot b\cdot b_n + b^2] \\
&= \lim_n \left( \mathbb E [ (\frac{1}{n} \sum_{i=1}^n x_i)^2 ] - 2b \mathbb E [b_n]\right) + b^2 \\
&= \lim_n \frac{1}{n^2} \left( \sum_{i=1}^n \mathbb E [x_i^2] + \sum_{i \neq j} \mathbb E [x_i] \cdot \mathbb E[x_j] \right) - b^2 \\
&= \lim_n \frac{1}{n^2} \left[ n(b^2 + \sigma^2) + (n^2 - n) b^2 \right] - b^2 \\
&= \lim_n \frac{\sigma^2}{n} = 0.\\
\end{aligned}
\end{equation}

Therefore, $b_n \to b$ in $L^2$.


# Section 8.5 $\quad$ Dynamic System

Consider a $2\times 2$  homogeneous system of differential equations
\begin{eqnarray*}
% \nonumber to remove numbering (before each equation)
  \frac{dx}{dt} &=& ax+by \\
  \frac{dy}{dt} &=& cx+dy \\
\end{eqnarray*}
If the independent variable $t$ denotes time, then the above system is a **dynamic system**.
- The solution $x = x(t)$ and $y=y(t)$ represents

- The $xy$-plane is called

- A sketch of trajectory in the phase plane is called a

- $\frac{dx}{dt}$, $\frac{dy}{dt}$ represent velocity in $x$ and $y$ direction. A point in the phase plane at which both $\frac{dx}{dt}$ and $\frac{dy}{dt}$ are zero is called

**Different Types of Equilibrium Points:**

**Stable** $\hspace{2in}$ **Unstable** $\hspace{2in}$ **Saddle Point**

The coefficient matrix $A$ will determine features of the phase portraits of the system.

**Case 1:** $\lambda_1 < 0$, $\lambda_2 < 0$

**Case 2:** $\lambda_1 > 0$, $\lambda_2 > 0$

**Case 3:** $\lambda_1 < 0$, $\lambda_2 > 0$

**Case 4:** $\lambda_1,\lambda_2$ are Complex Numbers

### Example 1

Determine the phase portrait of the dynamic system:
\begin{equation*}
  \mathbf{x}'(t) =
  \left[
        \begin{array}{rr}
          -2 & -2 \\
          1 & -5 \\
        \end{array}
      \right]
  \mathbf{x}(t)
\end{equation*}

```python
from sympy import *

t = symbols('t');
x1 = Function('x1');
x2 = Function('x2');
eq1 = x1(t).diff(t) + 2*x1(t) + 2*x2(t);
eq2 = x2(t).diff(t) - x1(t) + 5*x2(t);

dsolve([eq1, eq2])
```

    [x1(t) == -2*C1*exp(-4*t) - 2*C2*exp(-3*t),
     x2(t) == -2*C1*exp(-4*t) - C2*exp(-3*t)]

```python
from scipy.integrate import odeint
import numpy as np
import matplotlib.pyplot as plt

def F(x, t):
    x1, x2 = x;
    return np.array([-2*x1-2*x2, x1-5*x2]);

for i in np.arange(100):
    t = np.arange(0, 30, 0.01);
    init = (np.cos(i), np.sin(i));
    track = odeint(F, init, t);
    plt.plot(track[:,0], track[:,1]);

plt.show()
```

### Example 2

Determine the phase portrait of the dynamic system:
\begin{equation*}
  \mathbf{x}'(t) =
  \left[
        \begin{array}{rr}
          1 & -1 \\
          -2 & 0 \\
        \end{array}
      \right]
  \mathbf{x}(t)
\end{equation*}

```python
from sympy import *

t = symbols('t');
x1 = Function('x1');
x2 = Function('x2');
eq1 = x1(t).diff(t) - x1(t) + x2(t);
eq2 = x2(t).diff(t) + 2*x1(t);

dsolve([eq1, eq2])
```

    [x1(t) == -C1*exp(-t) - C2*exp(2*t), x2(t) == -2*C1*exp(-t) + C2*exp(2*t)]

```python
from scipy.integrate import odeint
import numpy as np
import matplotlib.pyplot as plt

def F(x, t):
    x1, x2 = x;
    return np.array([x1-x2, -2*x1]);

for i in np.arange(100):
    t = np.arange(0, 1, 0.01);
    init = (np.cos(i), np.sin(i));
    track = odeint(F, init, t);
    plt.plot(track[:,0], track[:,1]);

plt.show()
```


```python
%pylab inline
```

    Populating the interactive namespace from numpy and matplotlib

```python
import sympy as S
from sympy import stats
p = S.symbols('p',positive=True,real=True)
x=stats.Binomial('x',5,p)
t = S.symbols('t',real=True,positive=True)
mgf = stats.E(S.exp(t*x))
```

```python
from IPython.display import Math
```

```python
Math(S.latex(S.simplify(mgf)))
```

$$p^{5} e^{5 t} + 5 p^{4} \left(- p + 1\right) e^{4 t} + 10 p^{3} \left(p - 1\right)^{2} e^{3 t} - 10 p^{2} \left(p - 1\right)^{3} e^{2 t} + 5 p \left(p - 1\right)^{4} e^{t} - \left(p - 1\right)^{5}$$

```python
import scipy.stats
```

```python
def gen_sample(ns=100,n=10):
    out =[]
    for i in range(ns):
        p=scipy.stats.uniform(0,1).rvs()
        out.append(scipy.stats.binom(n,p).rvs())
    return out
```

```python
from collections import Counter
xs = gen_sample(1000)
hist(xs,range=(1,10),align='mid')
Counter(xs)
```

## sum of IID normally distributed random variables

```python
S.var('x:2',real=True)
S.var('mu:2',real=True)
S.var('sigma:2',positive=True)
S.var('t',positive=True)
x0=stats.Normal(x0,mu0,sigma0)
x1=stats.Normal(x1,mu1,sigma1)
```

```python
mgf0=S.simplify(stats.E(S.exp(t*x0)))
mgf1=S.simplify(stats.E(S.exp(t*x1)))
mgfY=S.simplify(mgf0*mgf1)
```

```python
Math(S.latex(mgf0))
```

$$e^{\mu_{0} t + \frac{\sigma_{0}^{2} t^{2}}{2}}$$

```python
Math(S.latex(S.powsimp(mgfY)))
```

$$e^{\frac{t}{2} \left(2 \mu_{0} + 2 \mu_{1} + \sigma_{0}^{2} t + \sigma_{1}^{2} t\right)}$$

```python
S.collect(S.expand(S.log(mgfY)),t)
```

    t**2*(sigma0**2/2 + sigma1**2/2) + t*(mu0 + mu1)

```python

```


```python
import matplotlib.pyplot as plt
import numpy as np

%matplotlib inline
```

```python
plt.rcParams['figure.dpi'] = 120
```

# Overview of the Konno-Ohmachi Smoothing Window

This is an overview of the Konno-Ohmachi smoothing window and the implemenation within `pyko`. The document is based on
the summary prepared by Jamie Steidl and Paul Hegarty.

The Konno-Ohmachi smoothing window is defined as:

$$
\begin{align}
W_B(f, f_c) &  = a \cdot \left( \frac{\sin(b \log_{10}(f / f_c))}{b \log_{10}(f / f_c)} \right)^4 \\
            &  = a \cdot \text{sinc} (b \log_{10}(f / f_c) ) ^ 4
\end{align}
$$
where $a$ is the the scaling parameter and $b$ controls the bandwidth or "fatness" of the lobes. This function is symmetric and uniform in log frequency space about the the given center frequency $f_c$. As the value of $b$ increases, the bandwidth decreases.

```python
freq_ratio = np.logspace(-1, 1, num=256)
```

```python
fig, ax = plt.subplots()

for b in [1, 10, 100]:
    window = np.sinc(b * np.log10(freq_ratio)) ** 4
    window /= window.max()
    ax.plot(freq_ratio, window, label=f'{b:.0f}')

ax.legend(title='$b$')
ax.set(xlabel='$f/f_c$', xscale='log', ylabel='Magnitude')

fig.tight_layout();
```

```python
fig, ax = plt.subplots()

ko_bw = 30.
freq_ratio = np.logspace(-1, 1, num=256)
sinc = np.sinc(ko_bw * np.log10(freq_ratio))
sinc /= sinc.max()

ax.plot(freq_ratio, sinc, label='Sinc')
ax.plot(freq_ratio, sinc ** 4, label='Sinc$^4$')

ax.legend()
ax.set(xlabel='$f/f_c$', xscale='log', )

fig.tight_layout();
```

# Limits

In order to improve the performance of the filter, it is necessary to compute the range over which the window is non-zero. The filter is non-zero between $f_c/\alpha$ and $\alpha \cdot f_c$, where $\alpha = 10^{\pi / b} \approx 10^{3 / b}$. Consider when $f = f_c / \alpha = f_c / 10^{3 / b}$:
$$
\begin{align}
W_B(f, f_c) & = a \cdot \left( \frac{\sin \left( b \log_{10} \left( \frac{f_c}{10^{3 / b} / f_c} \right) \right) }{b \log_{10} \left( \frac{f_c}{10^{3 / b} / f_c} \right)} \right) ^4 \\
            & = a \cdot \left( \frac{\sin( b \cdot -3 / b)}{b \cdot -3 / b} \right)^4 \\
            & = a \cdot \left( \frac{\sin(-3)}{3} \right)^4 \\
            &  \approx 0
\end{align}
$$

Similarly, for an upper frequency of $f = f_c \cdot \alpha = f_c \cdot 10^{3 / b}$:
$$
\begin{align}
W_B(f, f_c) & = a \cdot \left( \frac{\sin(3)}{3} \right)^4 \\
            &  \approx 0
\end{align}
$$

The window is computed over this frequency range, which corrresponds (approximately) to the $\pm\pi$ zero-crossing of the sinc function regardless of the selected bandwidth.

# Recommended $b$ Values

The PEER Report (Selection of Random Vibration
Procedures for the NGA-East Project)[https://peer.berkeley.edu/sites/default/files/2018_05_goulet_final_0.pdf] provides a recommends the Konno-Ohmachi filter with bandwidth ($b_w$) of 1/30 be applied to compute the effective amplitude spectrum. The bandwidth is related to the filter parameter $b$ by:
$$
b = \frac{2\pi}{b_w}
$$
Thus, a bandwidth of 1/30 corresponds to a $b$ value of 188.5.

```python

```


## effective increment

### created by Yuying Liu, 04/29/2020

For autonomous system
\begin{equation}
    \bf{\dot{x} = f(x)}
\end{equation}

we visualize
\begin{equation}
    F_{\Delta}(x(t)):= x(t+\Delta)-x(t)
\end{equation}

This script can be used to visualize Figure 10.

```python
%matplotlib inline

import numpy as np
from scipy import integrate

from tqdm.notebook import tqdm
from matplotlib import pyplot as plt
from matplotlib import gridspec
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.colors import cnames
from matplotlib import animation
from ipywidgets import interact, interactive
from IPython.display import clear_output, display, HTML
```

```python
# shared parameters
max_time = 10.24
dt = 0.16
t = np.linspace(0, max_time, int(max_time/dt)+1)
```

### Hyperbolic Fixed Point

\begin{split}
    \dot{x} &= \mu x \\
    \dot{y} &= \lambda(y-x^2)
\end{split}

```python
mu = -0.05
lam = -1.0
def hyperbolic_deriv(x_y):
        """Compute the time-derivative of the hyperbolic example."""
        x, y = x_y
        return [mu*x, lam*(y-x**2)]

# simulate
xvalues, yvalues = np.meshgrid(np.arange(-1.0,1.0,0.02), np.arange(-1.0,1.0,0.02))
inits = np.stack([xvalues, yvalues], 2)
sols = np.zeros((inits.shape[0], inits.shape[1], len(t), 2))
for i in tqdm(range(inits.shape[0])):
    for j in range(inits.shape[1]):
        init = inits[i, j]
        sol = integrate.solve_ivp(lambda _, x: hyperbolic_deriv(x), [0, max_time], init, t_eval=t)
        sols[i, j, :, :] = sol.y.T

# compute increments
vmin0 = [float('inf')]*2
vmax0 = [float('-inf')]*2
incre0 = np.zeros((inits.shape[0], inits.shape[1], len(t)-1, 2))
for i in tqdm(range(1, len(t))):
    for j in range(2):
        vmin0[j] = min(np.min(sols[:, :, i, j] - sols[:, :, 0, j]), vmin0[j])
        vmax0[j] = max(np.max(sols[:, :, i, j] - sols[:, :, 0, j]), vmax0[j])
    incre0[:, :, i-1, :] = sols[:, :, i, :] - sols[:, :, 0, :]

def viz_cubic_increment(time=25):
    fig = plt.figure(figsize=(10, 16))
    gs = gridspec.GridSpec(nrows=2, ncols=1, hspace=0.2)
    ax0 = fig.add_subplot(gs[0, :])
    ax1 = fig.add_subplot(gs[1, :])

    # prepare the axes limits
    ax0.set_xlim((-2, 2))
    ax0.set_ylim((-2, 2))
    ax1.set_xlim((-2, 2))
    ax1.set_ylim((-2, 2))

    # plot
    idx = int(time/dt)-1
    im0 = ax0.imshow(incre0[:, :, idx, 0], extent=[-2,2,-2,2], vmin=vmin0[0], vmax=vmax0[1])
    im1 = ax1.imshow(incre0[:, :, idx, 1], extent=[-2,2,-2,2], vmin=vmin0[0], vmax=vmax0[1])

    # colorbar
    cbar0 = fig.colorbar(im0, ax=ax0, aspect=5)
    cbar1 = fig.colorbar(im1, ax=ax1, aspect=5)
    cbar0.ax.tick_params(labelsize=60)
    cbar1.ax.tick_params(labelsize=60)

    # title
    ax0.set_title('x', fontsize=50)
    ax1.set_title('y', fontsize=50)

    # ticks
    ax0.tick_params(axis='both', which='major', labelsize=60)
    ax1.tick_params(axis='both', which='major', labelsize=60)
    ax0.axis('off')
    ax1.axis('off')

    return t, fig

# viz
w0 = interactive(viz_cubic_increment, time=(dt, max_time, dt))
display(w0)
```

    HBox(children=(FloatProgress(value=0.0), HTML(value='')))

    HBox(children=(FloatProgress(value=0.0, max=64.0), HTML(value='')))

    interactive(children=(FloatSlider(value=10.24, description='time', max=10.24, min=0.16, step=0.16), Output()),…

### Cubic Oscillator

\begin{split}
    \dot{x} &= -0.1x^3 + 2y^3 \\
    \dot{y} &= -2x^3 - 0.1y^3
\end{split}

```python
def cubic_deriv(x_y):
        """Compute the time-derivative of the cubic oscillator."""
        x, y = x_y
        return [-0.1*x**3+2*y**3, -2*x**3-0.1*y**3]

# simulate
xvalues, yvalues = np.meshgrid(np.arange(-1.0,1.0,0.02), np.arange(-1.0,1.0,0.02))
inits = np.stack([xvalues, yvalues], 2)
sols = np.zeros((inits.shape[0], inits.shape[1], len(t), 2))
for i in tqdm(range(inits.shape[0])):
    for j in range(inits.shape[1]):
        init = inits[i, j]
        sol = integrate.solve_ivp(lambda _, x: cubic_deriv(x), [0, max_time], init, t_eval=t)
        sols[i, j, :, :] = sol.y.T

# compute increments
vmin1 = [float('inf')]*2
vmax1 = [float('-inf')]*2
incre1 = np.zeros((inits.shape[0], inits.shape[1], len(t)-1, 2))
for i in tqdm(range(1, len(t))):
    for j in range(2):
        vmin1[j] = min(np.min(sols[:, :, i, j] - sols[:, :, 0, j]), vmin1[j])
        vmax1[j] = max(np.max(sols[:, :, i, j] - sols[:, :, 0, j]), vmax1[j])
    incre1[:, :, i-1, :] = sols[:, :, i, :] - sols[:, :, 0, :]

def viz_cubic_increment(time=25):
    fig = plt.figure(figsize=(10, 16))
    gs = gridspec.GridSpec(nrows=2, ncols=1, hspace=0.2)
    ax0 = fig.add_subplot(gs[0, :])
    ax1 = fig.add_subplot(gs[1, :])

    # prepare the axes limits
    ax0.set_xlim((-1, 1))
    ax0.set_ylim((-1, 1))
    ax1.set_xlim((-1, 1))
    ax1.set_ylim((-1, 1))

    # plot
    idx = int(time/dt)-1
    im0 = ax0.imshow(incre1[:, :, idx, 0], extent=[-1,1,-1,1], vmin=vmin1[0], vmax=vmax1[1])
    im1 = ax1.imshow(incre1[:, :, idx, 1], extent=[-1,1,-1,1], vmin=vmin1[0], vmax=vmax1[1])

    # colorbar
    cbar0 = fig.colorbar(im0, ax=ax0, aspect=5)
    cbar1 = fig.colorbar(im1, ax=ax1, aspect=5)
    cbar0.ax.tick_params(labelsize=60)
    cbar1.ax.tick_params(labelsize=60)

    # title
    ax0.set_title('x', fontsize=50)
    ax1.set_title('y', fontsize=50)

    # ticks
    ax0.tick_params(axis='both', which='major', labelsize=60)
    ax1.tick_params(axis='both', which='major', labelsize=60)
    ax0.axis('off')
    ax1.axis('off')

    return t, fig

# viz
w1 = interactive(viz_cubic_increment, time=(dt, max_time, dt))
display(w1)
```

### Van der Pol Oscillator

\begin{split}
    \dot{x} &= y \\
    \dot{y} &= \mu(1-x^2)y - x
\end{split}

where $\mu=2.0$

```python
mu = 2.0
def vdp_deriv(x_y):
        """Compute the time-derivative of the vdp."""
        x, y = x_y
        return [y, mu*(1-x**2)*y-x]

# simulate
xvalues, yvalues = np.meshgrid(np.arange(-2.0,2.0,0.02), np.arange(-1.0,1.0,0.01))
inits = np.stack([xvalues, yvalues], 2)
sols = np.zeros((inits.shape[0], inits.shape[1], len(t), 2))
for i in tqdm(range(inits.shape[0])):
    for j in range(inits.shape[1]):
        init = inits[i, j]
        sol = integrate.solve_ivp(lambda _, x: vdp_deriv(x), [0, max_time], init, t_eval=t)
        sols[i, j, :, :] = sol.y.T

# compute increments
vmin2 = [float('inf')]*2
vmax2 = [float('-inf')]*2
incre2 = np.zeros((inits.shape[0], inits.shape[1], len(t)-1, 2))
for i in tqdm(range(1, len(t))):
    for j in range(2):
        vmin2[j] = min(np.min(sols[:, :, i, j] - sols[:, :, 0, j]), vmin2[j])
        vmax2[j] = max(np.max(sols[:, :, i, j] - sols[:, :, 0, j]), vmax2[j])
    incre2[:, :, i-1, :] = sols[:, :, i, :] - sols[:, :, 0, :]

def viz_vdp_increment(time=25):
    fig = plt.figure(figsize=(10, 16))
    gs = gridspec.GridSpec(nrows=2, ncols=1, hspace=0.2)
    ax0 = fig.add_subplot(gs[0, :])
    ax1 = fig.add_subplot(gs[1, :])

    # prepare the axes limits
    ax0.set_xlim((-2, 2))
    ax0.set_ylim((-1, 1))
    ax1.set_xlim((-2, 2))
    ax1.set_ylim((-1, 1))

    # plot
    idx = int(time/dt)-1
    im0 = ax0.imshow(incre2[:, :, idx, 0], extent=[-2,2,-1,1], vmin=vmin2[0], vmax=vmax2[0], aspect=2)
    im1 = ax1.imshow(incre2[:, :, idx, 1], extent=[-2,2,-1,1], vmin=vmin2[1], vmax=vmax2[1], aspect=2)

    # colorbar
    cbar0 = fig.colorbar(im0, ax=ax0, aspect=5)
    cbar1 = fig.colorbar(im1, ax=ax1, aspect=5)
    cbar0.ax.tick_params(labelsize=60)
    cbar1.ax.tick_params(labelsize=60)

    # title
    ax0.set_title('x', fontsize=50)
    ax1.set_title('y', fontsize=50)

    # ticks
    ax0.tick_params(axis='both', which='major', labelsize=60)
    ax1.tick_params(axis='both', which='major', labelsize=60)
    ax0.axis('off')
    ax1.axis('off')

    return t, fig

# viz
w2 = interactive(viz_vdp_increment, time=(dt, max_time, dt))
display(w2)
```

### Hopf bifurcation

\begin{split}
    \dot{\mu} &= 0 \\
    \dot{x} &= \mu x + y -x(x^2+y^2) \\
    \dot{y} &= \mu y - x -y(x^2+y^2)
\end{split}

```python
def hopf_deriv(mu_x_y):
    """Compute the time-derivative of the hopf"""
    mu, x, y = mu_x_y
    return np.array([0, mu*x+y-x*(x**2+y**2), -x+mu*y-y*(x**2+y**2)])

# simulate
muvalues, xvalues, yvalues = np.meshgrid(np.arange(-0.2,0.6,0.1), np.arange(-1,2,0.02), np.arange(-1,1,0.02), indexing='ij')
inits = np.stack([muvalues, xvalues, yvalues], 3)
sols = np.zeros((inits.shape[0], inits.shape[1], inits.shape[2], len(t), 3))
for i in tqdm(range(inits.shape[0])):
    for j in range(inits.shape[1]):
        for k in range(inits.shape[2]):
            init = inits[i, j, k, :]
            sol = integrate.solve_ivp(lambda _, x: hopf_deriv(x), [0, max_time], init, t_eval=t)
            sols[i, j, k, :, :] = sol.y.T

# compute increments
vmin3 = [float('inf')]*3
vmax3 = [float('-inf')]*3
incre3 = np.zeros((inits.shape[0], inits.shape[1], inits.shape[2], len(t)-1, 3))
for i in tqdm(range(1, len(t))):
    for j in range(3):
        vmin3[j] = min(np.min(sols[:, :, :, i, j] - sols[:, :, :, 0, j]), vmin3[j])
        vmax3[j] = max(np.max(sols[:, :, :, i, j] - sols[:, :, :, 0, j]), vmax3[j])
    incre3[:, :, :, i-1, :] = sols[:, :, :, i, :] - sols[:, :, :, 0, :]

def viz_hopf_increment(time=25, mu=0):
    fig = plt.figure(figsize=(10, 24))
    gs = gridspec.GridSpec(nrows=3, ncols=1, hspace=0.2)
    ax0 = fig.add_subplot(gs[0, :])
    ax1 = fig.add_subplot(gs[1, :])
    ax2 = fig.add_subplot(gs[2, :])

    # prepare the axes limits
    ax0.set_xlim((-1, 2))
    ax0.set_ylim((-1, 1))
    ax1.set_xlim((-1, 2))
    ax1.set_ylim((-1, 1))
    ax2.set_xlim((-1, 2))
    ax2.set_ylim((-1, 1))

    # plot
    idx_t = int(time/dt)-1
    idx_mu = min(int((mu + 0.2) / 0.1), len(np.arange(-0.2,0.6,0.1))-1)
    im0 = ax0.imshow(incre3[:, :, idx_mu, idx_t, 0], extent=[-1,2,-1,1], vmin=vmin3[0], vmax=vmax3[0], aspect=1.5)
    im1 = ax1.imshow(incre3[:, :, idx_mu, idx_t, 1], extent=[-1,2,-1,1], vmin=vmin3[1], vmax=vmax3[1], aspect=1.5)
    im2 = ax2.imshow(incre3[:, :, idx_mu, idx_t, 2], extent=[-1,2,-1,1], vmin=vmin3[2], vmax=vmax3[2], aspect=1.5)

    # colorbar
    cbar0 = fig.colorbar(im0, ax=ax0, aspect=5)
    cbar1 = fig.colorbar(im1, ax=ax1, aspect=5)
    cbar2 = fig.colorbar(im2, ax=ax2, aspect=5)
    cbar0.ax.tick_params(labelsize=60)
    cbar1.ax.tick_params(labelsize=60)
    cbar2.ax.tick_params(labelsize=60)

    # title
    ax0.set_title('$\mu$', fontsize=50)
    ax1.set_title('x', fontsize=50)
    ax2.set_title('y', fontsize=50)

    # ticks
    ax0.tick_params(axis='both', which='major', labelsize=60)
    ax1.tick_params(axis='both', which='major', labelsize=60)
    ax2.tick_params(axis='both', which='major', labelsize=60)
    ax0.axis('off')
    ax1.axis('off')
    ax2.axis('off')

    return t, fig

# viz
w3 = interactive(viz_hopf_increment, time=(dt, max_time, dt), mu=(-0.2,0.6,0.1))
display(w3)
```

### Lorenz

\begin{split}
    \dot{x} &= \sigma(y-x) \\
    \dot{y} &= x(\rho-z)-y \\
    \dot{z} &= xy - \beta z
\end{split}

where $\sigma=10, \rho=28, \beta=8/3$

```python
sigma = 10
rho = 28
beta = 8/3
def lorenz_deriv(x_y_z):
        """Compute the time-derivative of the lorenz system."""
        x, y, z = x_y_z
        return [sigma*(y-x), x*(rho-z)-y, x*y-beta*z]

# simulate
xvalues, yvalues, zvalues = np.meshgrid(np.arange(-9,-7,0.02), np.arange(6,8,0.02), np.arange(26,28,0.5), indexing='ij')
inits = np.stack([xvalues, yvalues, zvalues], 3)
sols = np.zeros((inits.shape[0], inits.shape[1], inits.shape[2], len(t), 3))
for i in tqdm(range(inits.shape[0])):
    for j in range(inits.shape[1]):
        for k in range(inits.shape[2]):
            init = inits[i, j, k, :]
            sol = integrate.solve_ivp(lambda _, x: lorenz_deriv(x), [0, max_time], init, t_eval=t)
            sols[i, j, k, :, :] = sol.y.T

# compute increments
vmin4 = [float('inf')]*3
vmax4 = [float('-inf')]*3
incre4 = np.zeros((inits.shape[0], inits.shape[1], inits.shape[2], len(t)-1, 3))
for i in tqdm(range(1, len(t))):
    for j in range(3):
        vmin4[j] = min(np.min(sols[:, :, :, i, j] - sols[:, :, :, 0, j]), vmin4[j])
        vmax4[j] = max(np.max(sols[:, :, :, i, j] - sols[:, :, :, 0, j]), vmax4[j])
    incre4[:, :, :, i-1, :] = sols[:, :, :, i, :] - sols[:, :, :, 0, :]

def viz_lorenz_increment(time=5, z=0):
    fig = plt.figure(figsize=(10, 24))
    gs = gridspec.GridSpec(nrows=3, ncols=1, hspace=0.2)
    ax0 = fig.add_subplot(gs[0, :])
    ax1 = fig.add_subplot(gs[1, :])
    ax2 = fig.add_subplot(gs[2, :])

    # prepare the axes limits
    ax0.set_xlim((-9, -7))
    ax0.set_ylim((6, 8))
    ax1.set_xlim((-9, -7))
    ax1.set_ylim((6, 8))
    ax2.set_xlim((-9, -7))
    ax2.set_ylim((6, 8))

    # plot
    idx_t = int(time/dt)-1
    idx_z = min(int((z - 26)/0.5), len(np.arange(26,28,0.1))-1)
    im0 = ax0.imshow(incre4[:, :, idx_z, idx_t, 0], extent=[-9,-7,6,8], vmin=vmin4[0], vmax=vmax4[0])
    im1 = ax1.imshow(incre4[:, :, idx_z, idx_t, 1], extent=[-9,-7,6,8], vmin=vmin4[1], vmax=vmax4[1])
    im2 = ax2.imshow(incre4[:, :, idx_z, idx_t, 2], extent=[-9,-7,6,8], vmin=vmin4[2], vmax=vmax4[2])

    # colorbar
    cbar0 = fig.colorbar(im0, ax=ax0, aspect=5)
    cbar1 = fig.colorbar(im1, ax=ax1, aspect=5)
    cbar2 = fig.colorbar(im2, ax=ax2, aspect=5)
    cbar0.ax.tick_params(labelsize=60)
    cbar1.ax.tick_params(labelsize=60)
    cbar2.ax.tick_params(labelsize=60)

    # title
    ax0.set_title('x', fontsize=50)
    ax1.set_title('y', fontsize=50)
    ax2.set_title('z', fontsize=50)

    # ticks
    ax0.tick_params(axis='both', which='major', labelsize=60)
    ax1.tick_params(axis='both', which='major', labelsize=60)
    ax2.tick_params(axis='both', which='major', labelsize=60)
    ax0.axis('off')
    ax1.axis('off')
    ax2.axis('off')

    return t, fig

# viz
w4 = interactive(viz_lorenz_increment, time=(dt, max_time, dt), z=(26, 28, 0.1))
display(w4)
```

```python

```

```python

```

```python

```

```python

```

```python

```

```python

```

```python

```

```python

```


## ニューラルネットワーク
### 損失関数
ニューラルネットワークは「一つの指標」を手がかりに最適なパラメータを模索する
この関数には二乗和誤差や交差エントロピー誤差などが用いられる
ニューラルネットワークの性能の悪さ
損失関数をマイナスすると「どれだけ性能が悪くないか」を示すことになる

### 二乗和誤差
\begin{equation}
E = \frac {1}{2}\sum_{k} (y_k - t_k)^2
\end{equation}

```python
import numpy as np
y = [0.1,0.05,0.6]
t = [0,0,1]
def mean_squared_error(y, t):
    return 0.5 * np.sum((y - t)**2)
a = mean_squared_error(np.array(y), np.array(t))
print(a)

# y,tの各要素の差→それぞれを2乗、結果を合計し、1/2する
b = 0.1**2
c = 0.05**2
d = (-0.4)**2
e = (b + c + d )*0.5
print(e)
```

    0.08625000000000002
    [ 0.01    0.0025  0.16  ]
    0.1725
    0.08625

### 交差エントロピー誤差
\begin{equation}
E = - \sum_{k} t_k log y_k
\end{equation}

```python
import numpy as np
y = [0.1,0.05,0.6]
t = [0,1,0]
def cross_entropy_error(y, t):
    # np.log(0) は　-inf となってしまうため、微小な値を追加
    delta = 1e-7
    return - np.sum(t* np.log(y + delta))

a = cross_entropy_error(np.array(y), np.array((t)))
print(a)
```

    2.99573027356

### 交差エントロピー誤差 ミニバッチ化
\begin{equation}
E = -\frac {1}{N} \sum_{n} \sum_{k} t_{nk} \log y_{nk}
\end{equation}

```python
import sys, os
sys.path.append(os.pardir)  # 親ディレクトリのファイルをインポートするための設定
import numpy as np
from dataset.mnist import load_mnist
from PIL import Image

# (訓練画像、訓練ラベル)、（テスト画像、テストラベル）
(x_train, t_train), (x_test, t_test) = load_mnist(normalize=False, one_hot_label=True)

train_size = x_train.shape[0]
batch_size = 10
batch_mask = np.random.choice(train_size, batch_size)
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]

#教師データがラベルとして与えられる one-hot表現ではなく、2や7が来る場合
# t* np.log(y + delta) →　np.log(y[np.arange(batch_size), t] + delta)とできる

def cross_entropy_error(y, t):
    if y.ndmi == 1:
        t = t.reshape(1, t.size)
        y = y.reshape(1, y.size)
    batch_size = y.shape[0]
    # np.log(0) は　-inf となってしまうため、微小な値を追加
    delta = 1e-7
    return - np.sum(np.log(y[np.arange(batch_size), t] + delta)) /batch_size

# np.arange
# 0 から batch_size -1 までの配列を生成
```

    (10, 784)

なぜ損失関数を設定するのか？
認識精度を指標にしてはいけない
パラメータの微分が0になってしまうから
ステップ関数の傾きは殆どの場所で0
シグモイド関数の傾き（接線）は0にならない

### 数値微分

#### 微分
\begin{equation}
\frac {df(x)}{dx} = \lim_{h \to 0} \frac{f(x+h)-f(x)}{h} \\
\end{equation}

```python
# BAD
def numerical_diff(f, x):
    h = 1e-50
    return (f(x + h) - f(x)) / h

# 10e-50 は小さすぎ　10e-4 くらいで良い
# 真の接線と近似による接線は誤差が生じる
# 誤差をへらす工夫として中心差分を用いる
def numerical_diff(f, x):
    h = 1e-4
    return (f(x + h) - f(x -h)) / (2* h)

# 数値微分の例
def func_1(x):
    return 0.01*x**2 + 0.1*x

def func_2(x):
    return 0.01*x**2

import numpy as np
import matplotlib.pylab as plt
x = np.arange(0.0, 20.0, 0.1)
y1 = func_1(x)
y2 = func_2(x)
plt.plot(x, y1, label="step")
plt.plot(x, y2, linestyle="--", label="sigmoid")
def tangent_line(f, x):
    d = numerical_diff(f, x)
    print(d)
    y = f(x) - d*x
    return lambda t: d*t + y

tf1 = tangent_line(func_1, 5)
tf2 = tangent_line(func_1, 10)
tf3 = tangent_line(func_2, 5)
tf4 = tangent_line(func_2, 10)
y3= tf1(x)
plt.plot(x, y3)
y4= tf2(x)
plt.plot(x, y4)
y5= tf3(x)
plt.plot(x, y5)
y6= tf4(x)
plt.plot(x, y6)
plt.show()
```

### 偏微分
複数変数からなる関数の微分を偏微分という
また、すべての変数の偏微分をベクトルとしてまとめたものを勾配という

```python
from mpl_toolkits.mplot3d import Axes3D
def func1(x, y):
    return x**2 + y**2

x = np.arange(-2.5, 2.5, 0.25)
y = np.arange(-2.5, 2.5, 0.25)
hoge = np.array([x,y])
X, Y = np.meshgrid(x, y)
Z = func1(X, Y)
fig = plt.figure()
ax = Axes3D(fig)
ax.set_xlabel("x")
ax.set_ylabel("y")
ax.set_zlabel("f(x, y)")
ax.plot_wireframe(X, Y, Z)
plt.show()

```

```python
# 勾配
x0 = np.arange(-2.5, 2.5, 0.25)
x1 = np.arange(-2.5, 2.5, 0.25)
X, Y = np.meshgrid(x0, x1)

X = X.flatten()
Y = Y.flatten()

def _numerical_gradient_no_batch(f, x):
    h = 1e-4  # 0.0001
    grad = np.zeros_like(x)

    for idx in range(x.size):
        tmp_val = x[idx]
        x[idx] = float(tmp_val) + h
        fxh1 = f(x)  # f(x+h)

        x[idx] = tmp_val - h
        fxh2 = f(x)  # f(x-h)
        grad[idx] = (fxh1 - fxh2) / (2*h)

        x[idx] = tmp_val  # 値を元に戻す

    return grad

def function_2(x):
    if x.ndim == 1:
        return np.sum(x**2)
    else:
        return np.sum(x**2, axis=1)

def numerical_gradient(f, X):
    if X.ndim == 1:
        return _numerical_gradient_no_batch(f, X)
    else:
        grad = np.zeros_like(X)

        for idx, x in enumerate(X):
            grad[idx] = _numerical_gradient_no_batch(f, x)

        return grad

grad = numerical_gradient(function_2, np.array([X, Y]).T).T

plt.figure()
plt.quiver(X, Y, -grad[0], -grad[1],  angles="xy",color="#666666")
plt.xlim([-2, 2])

plt.ylim([-2, 2])
plt.xlabel('x0')
plt.ylabel('x1')
plt.grid()
plt.draw()
plt.show()
```

```python

```

```python

```


# Iterative Solvers 4 - Preconditioning

## The basic idea

For both the GMRES method and CG we have seen that the eigenvalue distribution is crucial for fast convergence. In both cases we would like the eigenvalues of the matrix be clustered close together and be well separated from zero. Unfortunately, in many applications the matrices that arise naturally are badly behaved in this respect. Preconditioning is a strategy to try to modify a linear system so that it becomes more ameniable to iterative solvers. Preconditioning is one of the most active research areas and crucial for the solution of linear systems of equations with millions or even billions of unknowns.

Consider the linear system of equations

$$
Ax = b
$$

In its basic form the idea of preconditioning is to multiply the system with a matrix $P^{-1}$ that is some kind of approximation to $A^{-1}$, that is $P^{-1}\approx A^{-1}$ in some sense (making this precise is the difficult bit).

We obtain either the left-preconditioned system

$$
P^{-1}Ax = P^{-1}b
$$

or the right-preconditioned system

$$
AP^{-1}y = b,
$$

where in the latter case we then additionally need to solve $Px=y$.

Classes of preconditioners include

* SPAI (Sparse Approximate Inverses)
* Incomplete LU Decomposition
* Incomplete Cholesky Decomposition
* Splitting Preconditioners
* Algebraic Multigrid Methods

These are also known as **algebraic preconditioners**. They consider the matrix $A$ and try to find an approximation to $A$ that is easily invertible.

A different class of preconditioners are **analytic preconditioners**. These are preconditioners that are often constructed as problems to PDEs that are easier to solve than the original PDE but still approximate to some sense the underlying physics of the problem.

## Sparse Approximate Inverse

As example of the class of algebraic preconditioners we consider here the Sparse Approximate Inverse (SPAI). Incomplete LU decompositions will be discussed later on. We note that SPAI is a technique that works well in certain cases, but is not suitable in others. **There is no general preconditioning technique that always works well.**

We denote by $\|A\|_F$ the Frobenious norm of a matrix $A$ defined by

$$
\|A\|_F^2 := \sum_{i, j}|a_{ij}|^2.
$$

The idea of SPAI is now to try to find a matrix $M := P^{-1}$ such that

$$
F(M) := \|I - AM\|_F
$$

is small. The minimum of this function is obviously reached for $M = A^{-1}$. But this usually not practical. Instead, we try to find a successive sequence of matrices $M_k$ that are approaching the minimum of the function $F$.

There are many ways to define an approximate minimization procedure to minimize $F$. The following is a global minimum residual algorithm, described by Saad in [Iterative Methods for Sparse Linear Systems](https://www-users.cs.umn.edu/~saad/IterMethBook_2ndEd.pdf).

$$
\begin{align}
C_k &= A M_k\nonumber\\
G_k &= I - C_k\nonumber\\
\alpha_k &=\text{tr}(G_k^TAG_k) / \|AG_k\|_F^2\nonumber\\
M_{k+1} &= M_k + \alpha_k G_k\nonumber\\
\end{align}
$$

In each step of the algorithm the matrix $M_k$ becomes slightly denser. Hence, in practice this is often combined with a numerical drop strategy for entries of $M$.

The following code implements this method. As starting matrix $M_0$ we choose $M_0 = \frac{2}{\|AA^T\|_1}A$, which was recommended by Chow and Saad in [Approximate Inverse Preconditioners via Sparse-Sparse Iterations](https://dl.acm.org/doi/10.1137/S1064827594270415). Note that in the following code we did not implement dropping of values. For practical purposes this is essential and strategies are discussed in the paper by Chow and Saad.

```python
def spai(A, m):
    """Perform m step of the SPAI iteration."""
    from scipy.sparse import identity
    from scipy.sparse import diags
    from scipy.sparse.linalg import onenormest

    n = A.shape[0]

    ident = identity(n, format='csr')
    alpha = 2 / onenormest(A @ A.T)
    M = alpha * A

    for index in range(m):
        C = A @ M
        G = ident - C
        AG = A @ G
        trace = (G.T @ AG).diagonal().sum()
        alpha = trace / np.linalg.norm(AG.data)**2
        M = M + alpha * G

    return M
```

We run the code with the following matrix, which is a slightly shifted version of the discrete 3-point second order differential operator.

```python
from scipy.sparse import diags

n = 1000

data = [2.001 * np.ones(n),
        -1. * np.ones(n - 1),
        -1. * np.ones(n - 1)]

offsets = [0, 1, -1]

A = diags(data, offsets=offsets, shape=(n, n), format='csr')
```

The condition number without the preconditioner is

```python
np.linalg.cond(A.todense())
```

    3961.9652414689454

Let us now generate the preconditioner.

```python
M = spai(A, 50)
```

Let us check the condition number of the right-preconditioned system.

```python
np.linalg.cond(A.todense() @ M.todense())
```

    40.18659718436073

It has been reduced by a factor 100. This is a good sign. We are now running CG for the preconditioned and non-preconditioned system.

```python
%matplotlib inline
from matplotlib import pyplot as plt
from scipy.sparse.linalg import cg

n = A.shape[0]
b = np.ones(n)

residuals = []
callback = lambda x: residuals.append(np.linalg.norm(A @ x - b))
x, _ = cg(A, b, callback=callback)

residuals_preconditioned = []
callback = lambda x: residuals_preconditioned.append(np.linalg.norm(A @ x - b) / np.linalg.norm(b))
x, _ = cg(A, b, M=M, callback=callback)

fig = plt.figure(figsize=(8, 8))
ax = fig.add_subplot(111)
ax.semilogy(residuals, 'k--')
ax.semilogy(residuals_preconditioned, 'b--')
ax.set_ylabel('relative residual')
fig.legend(['no preconditioning', 'SPAI'], loc='lower center', fancybox=True, shadow=True)
```

We can see a significant improvement in the number of iterations. However, the comparison is not quite fair. Setting up the preconditioner took time, which should be taken into account. Just considering number of iterations is not sufficient. In practice, the overall computation time is the much more important measure. We have implemented here a very primitive variant of SPAI that should not be used in practice. For practical alternatives see the cited literature. For the type of matrix that $A$ is there are also much better precondtioners available, some of which we will encounter later.

## A note on preconditioned Conjugate Gradient

We have passed here the preconditioner as a matrix into the CG algorithm. This is possible if the preconditioner is also symmetric, positive definite. This is the case in our example. We are not going to discuss the details of preconditioned conjugate gradients in more details here and refer to the book by Saad.