```python
import ambulance_game as abg

```

```python
import networkx as nx
import matplotlib.pyplot as plt
import numpy as np
import sympy as sym
import itertools
import scipy as sci
import scipy.integrate

def build_states(threshold, system_capacity, buffer_capacity):
    if threshold > system_capacity:
        states_1 = [(0, v) for v in range(0, system_capacity + 1)]
        states_2 = [(1, system_capacity)]
        return states_1 + states_2

    states_1 = [(0, v) for v in range(0, threshold)]
    states_2 = [
        (u, v)
        for v in range(threshold, system_capacity + 1)
        for u in range(buffer_capacity + 1)
    ]
    all_states = states_1 + states_2

    return all_states

def visualise_ambulance_markov_chain(
    num_of_servers, threshold, system_capacity, buffer_capacity
):
    all_states = build_states(threshold, system_capacity, buffer_capacity)
    G = nx.MultiDiGraph()
    for _, origin_state in enumerate(all_states):
        for _, destination_state in enumerate(all_states):
            column_adjacent = (
                destination_state[0] - origin_state[0] == 1
                and destination_state[1] - origin_state[1] == 0
            )
            row_adjacent = (
                destination_state[1] - origin_state[1] == 1
                and destination_state[0] - origin_state[0] == 0
            )
            if row_adjacent or column_adjacent:
                G.add_edge(origin_state, destination_state, color="blue")

    plt.figure(figsize=(1.5 * (buffer_capacity + 1), 1.5 * system_capacity))
    pos = {state: list(state) for state in all_states}
    nx.draw_networkx_nodes(
        G,
        pos,
        node_size=2000,
        nodelist=[state for state in all_states if state[1] < num_of_servers],
    )
    nx.draw_networkx_nodes(
        G,
        pos,
        node_size=2000,
        nodelist=[state for state in all_states if state[1] >= num_of_servers],
        node_color="red",
    )
    nx.draw_networkx_edges(G, pos)
    nx.draw_networkx_labels(G, pos)

    plt.axis("off")

    return G

def get_transition_matrix_entry(
    origin, destination, threshold, lambda_2, lambda_1, Lambda, mu, num_of_servers
):
    row_diff = origin[0] - destination[0]
    column_diff = origin[1] - destination[1]

    if row_diff == 0 and column_diff == -1:
        if origin[1] < threshold:
            return Lambda
        return lambda_1
    elif row_diff == -1 and column_diff == 0:
        return lambda_2
    elif (row_diff == 0 and column_diff == 1) or (
        row_diff == 1 and column_diff == 0 and origin[1] == threshold
    ):
        if origin[1] <= num_of_servers:
            return origin[1] * mu
        else:
            return num_of_servers * mu
    else:
        return 0

def get_symbolic_transition_matrix(
    num_of_servers, threshold, system_capacity, buffer_capacity
):
    Lambda = sym.symbols("Lambda")
    lambda_1 = sym.symbols("lambda") ** sym.symbols("o")
    lambda_2 = sym.symbols("lambda") ** sym.symbols("A")
    mu = sym.symbols("mu")

    all_states = build_states(threshold, system_capacity, buffer_capacity)
    Q = sym.zeros(len(all_states))

    if threshold > system_capacity:
        threshold = system_capacity

    for (i, origin_state), (j, destination_state) in itertools.product(
        enumerate(all_states), repeat=2
    ):
        Q[i, j] = get_transition_matrix_entry(
            origin=origin_state,
            destination=destination_state,
            threshold=threshold,
            lambda_2=lambda_2,
            lambda_1=lambda_1,
            Lambda=Lambda,
            mu=mu,
            num_of_servers=num_of_servers,
        )

    sum_of_rates = -np.sum(Q, axis=1)
    Q = Q + sym.Matrix(np.diag(sum_of_rates))

    return Q

def get_transition_matrix(
    lambda_2, lambda_1, mu, num_of_servers, threshold, system_capacity, buffer_capacity
):
    all_states = build_states(threshold, system_capacity, buffer_capacity)
    size = len(all_states)
    Q = np.zeros((size, size))

    if threshold > system_capacity:
        threshold = system_capacity

    for (i, origin_state), (j, destination_state) in itertools.product(
        enumerate(all_states), repeat=2
    ):
        Q[i, j] = get_transition_matrix_entry(
            origin=origin_state,
            destination=destination_state,
            threshold=threshold,
            lambda_2=lambda_2,
            lambda_1=lambda_1,
            Lambda=lambda_2 + lambda_1,
            mu=mu,
            num_of_servers=num_of_servers,
        )
    sum_of_rates = np.sum(Q, axis=1)
    np.fill_diagonal(Q, -sum_of_rates)
    return Q

def convert_symbolic_transition_matrix(Q_sym, lambda_2, lambda_1, mu):
    sym_Lambda = sym.symbols("Lambda")
    sym_lambda_1 = sym.symbols("lambda") ** sym.symbols("o")
    sym_lambda_2 = sym.symbols("lambda") ** sym.symbols("A")
    sym_mu = sym.symbols("mu")

    Q = np.array(
        Q_sym.subs(
            {
                sym_Lambda: lambda_2 + lambda_1,
                sym_lambda_1: lambda_1,
                sym_lambda_2: lambda_2,
                sym_mu: mu,
            }
        )
    ).astype(np.float64)
    return Q

def is_steady_state(state, Q):
    return np.allclose(np.dot(state, Q), 0)

def get_steady_state_numerically(
    Q, max_t=100, number_of_timepoints=1000, integration_function=sci.integrate.odeint
):
    def derivative_odeint(x, t):
        return np.dot(x, Q)

    def derivative_solve_ivp(t, x):
        return np.dot(x, Q)

    dimension = Q.shape[0]
    state = np.ones(dimension) / dimension
    while not is_steady_state(state=state, Q=Q):
        t_span = np.linspace(0, max_t, number_of_timepoints)
        if integration_function == sci.integrate.odeint:
            sol = integration_function(func=derivative_odeint, y0=state, t=t_span)
            state = sol[-1]
        elif integration_function == sci.integrate.solve_ivp:
            sol = integration_function(
                fun=derivative_solve_ivp, y0=state, t_span=t_span
            )
            state = sol.y[:, -1]
    return state

def augment_Q(Q):
    dimension = Q.shape[0]
    M = np.vstack((Q.transpose()[:-1], np.ones(dimension)))
    b = np.vstack((np.zeros((dimension - 1, 1)), [1]))
    return M, b

def get_steady_state_algebraically(Q, algebraic_function=np.linalg.solve):
    M, b = augment_Q(Q)
    if algebraic_function == np.linalg.solve:
        state = algebraic_function(M, b).transpose()[0]
    elif algebraic_function == np.linalg.lstsq:
        state = algebraic_function(M, b, rcond=None)[0][:, 0]
    return state

def get_markov_state_probabilities(
    pi, all_states, output=np.ndarray, system_capacity=None, buffer_capacity=None
):
    if output == dict:
        states_probabilities_dictionary = {}
        for i in range(len(all_states)):
            states_probabilities_dictionary[all_states[i]] = pi[i]
        return states_probabilities_dictionary
    elif output == np.ndarray:
        if buffer_capacity == None:
            buffer_capacity = max([state[0] for state in all_states])
        if system_capacity == None:
            system_capacity = max([state[1] for state in all_states])
        states_probabilities_array = np.full(
            (buffer_capacity + 1, system_capacity + 1), np.NaN
        )
        for index in range(len(all_states)):
            states_probabilities_array[all_states[index]] = pi[index]
        return states_probabilities_array

def get_mean_number_of_patients_in_system(pi, states):
    states = np.array(states)
    mean_patients_in_system = np.sum((states[:, 0] + states[:, 1]) * pi)
    return mean_patients_in_system

def get_mean_number_of_patients_in_hospital(pi, states):
    states = np.array(states)
    mean_patients_in_hospital = np.sum(states[:, 1] * pi)
    return mean_patients_in_hospital

def get_mean_ambulances_blocked(pi, states):
    states = np.array(states)
    mean_ambulances_blocked = np.sum(states[:, 0] * pi)
    return mean_ambulances_blocked

```

```python
lambda_2 = 0.05
lambda_1 = 0.1
mu = 0.1

num_of_servers = 2
threshold = 4
system_capacity = 3
buffer_capacity = 2

```

```python
all_states = abg.markov.build_states(threshold, system_capacity, buffer_capacity)
all_states

```

    [(0, 0), (0, 1), (0, 2), (0, 3)]

```python
Q_1 = abg.markov.get_transition_matrix(
    lambda_2, lambda_1, mu, num_of_servers, threshold, system_capacity, buffer_capacity
)
Q_1

```

    array([[-0.15,  0.15,  0.  ,  0.  ],
           [ 0.1 , -0.25,  0.15,  0.  ],
           [ 0.  ,  0.2 , -0.35,  0.15],
           [ 0.  ,  0.  ,  0.2 , -0.2 ]])

```python
Q_sym = abg.markov.get_symbolic_transition_matrix(
    num_of_servers, threshold, system_capacity, buffer_capacity
)
Q_sym

```

$\displaystyle \left[\begin{matrix}- \Lambda & \Lambda & 0 & 0\\\mu & - \Lambda - \mu & \Lambda & 0\\0 & 2 \mu & - \Lambda - 2 \mu & \Lambda\\0 & 0 & 2 \mu & - 2 \mu\end{matrix}\right]$

```python
Q_2 = abg.markov.convert_symbolic_transition_matrix(Q_sym, lambda_2, lambda_1, mu)
Q_2

```

    array([[-0.15,  0.15,  0.  ,  0.  ],
           [ 0.1 , -0.25,  0.15,  0.  ],
           [ 0.  ,  0.2 , -0.35,  0.15],
           [ 0.  ,  0.  ,  0.2 , -0.2 ]])

```python
Q_1 == Q_2

```

    array([[ True,  True,  True,  True],
           [ True,  True,  True,  True],
           [ True,  True,  True,  True],
           [ True,  True,  True,  True]])

```python
pi = abg.markov.get_steady_state_algebraically(Q_1)
pi

```

    array([0.22377622, 0.33566434, 0.25174825, 0.18881119])

```python
abg.markov.get_markov_state_probabilities(pi, all_states)

```

    array([[0.22377622, 0.33566434, 0.25174825, 0.18881119]])

```python
abg.markov.get_markov_state_probabilities(
    pi, all_states, buffer_capacity=buffer_capacity
)

```

    array([[0.22377622, 0.33566434, 0.25174825, 0.18881119],
           [       nan,        nan,        nan,        nan],
           [       nan,        nan,        nan,        nan]])

```python

```

```python

```

```python

```

```python

```

```python

```


# Using physics informed neural networks (PINNs) to solve parabolic PDEs

In this notebook, we illustrate physics informed neural networks (PINNs) to solve partial differential equations (PDEs) as proposed in

- Maziar Raissi, Paris Perdikaris, George Em Karniadakis. *Physics Informed Deep Learning (Part I): Data-driven Solutions of Nonlinear Partial Differential Equations*. [arXiv 1711.10561](https://arxiv.org/abs/1711.10561)
- Maziar Raissi, Paris Perdikaris, George Em Karniadakis. *Physics Informed Deep Learning (Part II): Data-driven Discovery of Nonlinear Partial Differential Equations*. [arXiv 1711.10566](https://arxiv.org/abs/1711.10566)
- Maziar Raissi, Paris Perdikaris, George Em Karniadakis. *Physics-informed neural networks: A deep learning framework for solving forward and inverse problems involving nonlinear partial differential equations*. J. Comp. Phys. 378 pp. 686-707 [DOI: 10.1016/j.jcp.2018.10.045](https://www.sciencedirect.com/science/article/pii/S0021999118307125)

This notebook is partially based on another implementation of the PINN approach published on [GitHub by pierremtb](https://github.com/pierremtb/PINNs-TF2.0) as well as the original code, see [Maziar Raissi on GitHub](https://github.com/maziarraissi/PINNs).

<a href="https://colab.research.google.com/github/janblechschmidt/PDEsByNNs/blob/main/PINN_Solver.ipynb" target="_parent">

</a>

## Introduction

We describe the PINN approach for approximating the solution $u:[0,T] \times \mathcal{D} \to \mathbb{R}$ of an evolution equation

$$
\begin{align}
    \partial_t u (t,x) + \mathcal{N}[u](t,x) &= 0, && (t,x) \in (0,T] \times \mathcal{D},\\
    u(0,x) &= u_0(x) \quad && x \in \mathcal{D},
\end{align}
$$

where $\mathcal{N}$ is a nonlinear differential operator acting on $u$,
$\mathcal{D} \subset \mathbb{R}^d$ a bounded domain,
$T$ denotes the final time and
$u_0: \mathcal{D} \to \mathbb{R}$ the prescribed initial data.
Although the methodology allows for different types of boundary conditions, we restrict our discussion to the inhomogeneous Dirichlet case and prescribe

$$
\begin{align}
        \hspace{7em} u(t,x) &= u_b(t,x)  && \quad (t,x) \in (0,T] \times \partial \mathcal{D},
\end{align}
$$

where $\partial \mathcal{D}$ denotes the boundary of the domain $\mathcal{D}$ and $u_b: (0,T] \times \partial \mathcal{D} \to \mathbb{R}$ the given boundary data.

## Methodology

The method constructs a neural network approximation

$$
u_\theta(t,x) \approx u(t,x)
$$

of the solution of nonlinear PDE, where $u_\theta :[0,T] \times \mathcal{D} \to \mathbb{R}$ denotes a function realized by a neural network with parameters $\theta$.

The continuous time approach for the parabolic PDE as described in ([Raissi et al., 2017 (Part I)](https://arxiv.org/abs/1711.10561)) is based on the (strong) residual of a given neural network approximation $u_\theta \colon [0,T] \times \mathcal{D} \to \mathbb{R} $ of the solution $u$, i.e.,

$$
\begin{align}
    r_\theta (t,x) := \partial_t u_\theta (t,x) + \mathcal{N}[u_\theta] (t,x).
\end{align}
$$

To incorporate this PDE residual $r_\theta$ into a loss function to be minimized, PINNs require a further differentiation to evaluate the differential operators $\partial_t u_\theta$ and $\mathcal{N}[u_\theta]$.
Thus the PINN term $r_\theta$ shares the same parameters as the original network $u_\theta(t,x)$, but respects the underlying "physics" of the nonlinear PDE.
Both types of derivatives can be easily determined through automatic differentiation with current state-of-the-art machine learning libraries, e.g., TensorFlow or PyTorch.

The PINN approach for the solution of the initial and boundary value problem now proceeds by minimization of the loss functional

$$
\begin{align}
    \phi_\theta(X) := \phi_\theta^r(X^r) + \phi_\theta^0(X^0) + \phi_\theta^b(X^b),
\end{align}
$$

where $X$ denotes the collection of training data and the loss function $\phi_\theta$ contains the following terms:

  - the mean squared residual
$$
  \begin{align*}
   \phi_\theta^r(X^r) := \frac{1}{N_r}\sum_{i=1}^{N_r} \left|r_\theta\left(t_i^r, x_i^r\right)\right|^2
\end{align*}
$$
in a number of collocation points $X^r:=\{(t_i^r, x_i^r)\}_{i=1}^{N_r} \subset (0,T] \times \mathcal{D}$, where $r_\theta$ is the physics-informed neural network,
  - the mean squared misfit with respect to the initial and boundary conditions
$$
    \begin{align*}
   \phi_\theta^0(X^0)
   :=
   \frac{1}{N_0}
   \sum_{i=1}^{N_0} \left|u_\theta\left(t_i^0, x_i^0\right) - u_0\left(x_i^0\right)\right|^2
   \quad \text{ and } \quad
   \phi_\theta^b(X^b)
   :=
   \frac{1}{N_b}
   \sum_{i=1}^{N_b} \left|u_\theta\left(t_i^b, x_i^b\right) - u_b\left(t_i^b, x_i^b\right)\right|^2
    \end{align*}
$$
in a number of points $X^0:=\{(t^0_i,x^0_i)\}_{i=1}^{N_0} \subset \{0\} \times \mathcal{D}$ and $X^b:=\{(t^b_i,x^b_i)\}_{i=1}^{N_b} \subset (0,T] \times \partial \mathcal{D}$, where $u_\theta$ is the neural network approximation of the solution $u\colon[0,T] \times \mathcal{D} \to \mathbb{R}$.

Note that the training data $X$ consists entirely of time-space coordinates.

## Example: Burgers equation

To illustrate the PINN approach we consider the one-dimensional Burgers equation on the spatial domain $\mathcal{D} = [-1,1]$

$$
\begin{aligned}
    \partial_t u + u \, \partial_x u - (0.01/\pi) \, \partial_{xx} u &= 0, \quad &&\quad (t,x) \in (0,1] \times (-1,1),\\
   u(0,x) &= - \sin(\pi \, x),                 \quad &&\quad x \in [-1,1],\\
   u(t,-1) = u(t,1) &= 0,                      \quad &&\quad t \in (0,1].
\end{aligned}
$$

This PDE arises in various disciplines such as traffic flow, fluid mechanics and gas dynamics, and can be derived from the Navier-Stokes equations, see
([Basdevant et al., 1986](https://www.researchgate.net/publication/222935980_Spectral_and_finite_difference_solutions_of_Burgers_equation)).

### 1. Import necessary packages and set problem specific data
This code runs with TensorFlow version `2.3.0`.
The implementation relies mainly on the scientific computing library [NumPy](https://numpy.org/doc/stable/user/whatisnumpy.html) and the machine learning library [TensorFlow](https://www.tensorflow.org/).

All computations were performed on an Intel i7 CPU (8th Gen) with 16 GByte DDR3 RAM (2133 MHz) within a couple of minutes.

```python
# Import TensorFlow and NumPy
import tensorflow as tf
import numpy as np

# Set data type
DTYPE='float32'
tf.keras.backend.set_floatx(DTYPE)

# Set constants
pi = tf.constant(np.pi, dtype=DTYPE)
viscosity = .01/pi

# Define initial condition
def fun_u_0(x):
    return -tf.sin(pi * x)

# Define boundary condition
def fun_u_b(t, x):
    n = x.shape[0]
    return tf.zeros((n,1), dtype=DTYPE)

# Define residual of the PDE
def fun_r(t, x, u, u_t, u_x, u_xx):
    return u_t + u * u_x - viscosity * u_xx
```

### 2. Generate a set of collocation points

We assume that the collocation points $X_r$ as well as the points for the initial time and boundary data $X_0$ and $X_b$ are generated by random sampling from a uniform distribution.
Although uniformly distributed data are sufficient in our experiments, the authors of
([Raissi et al., 2017 (Part I)](https://arxiv.org/abs/1711.10561))
employed a space-filling Latin hypercube sampling strategy ([Stein, 1987](https://www.tandfonline.com/doi/abs/10.1080/00401706.1987.10488205)).
Our numerical experiments indicate that this strategy slightly improves the observed convergence rate, but for simplicity the code examples accompanying this paper employ uniform sampling throughout.
We choose training data of size $N_0 = N_b =50$ and $N_f=10000$.

```python
# Set number of data points
N_0 = 50
N_b = 50
N_r = 10000

# Set boundary
tmin = 0.
tmax = 1.
xmin = -1.
xmax = 1.

# Lower bounds
lb = tf.constant([tmin, xmin], dtype=DTYPE)
# Upper bounds
ub = tf.constant([tmax, xmax], dtype=DTYPE)

# Set random seed for reproducible results
tf.random.set_seed(0)

# Draw uniform sample points for initial boundary data
t_0 = tf.ones((N_0,1), dtype=DTYPE)*lb[0]
x_0 = tf.random.uniform((N_0,1), lb[1], ub[1], dtype=DTYPE)
X_0 = tf.concat([t_0, x_0], axis=1)

# Evaluate intitial condition at x_0
u_0 = fun_u_0(x_0)

# Boundary data
t_b = tf.random.uniform((N_b,1), lb[0], ub[0], dtype=DTYPE)
x_b = lb[1] + (ub[1] - lb[1]) * tf.keras.backend.random_bernoulli((N_b,1), 0.5, dtype=DTYPE)
X_b = tf.concat([t_b, x_b], axis=1)

# Evaluate boundary condition at (t_b,x_b)
u_b = fun_u_b(t_b, x_b)

# Draw uniformly sampled collocation points
t_r = tf.random.uniform((N_r,1), lb[0], ub[0], dtype=DTYPE)
x_r = tf.random.uniform((N_r,1), lb[1], ub[1], dtype=DTYPE)
X_r = tf.concat([t_r, x_r], axis=1)

# Collect boundary and inital data in lists
X_data = [X_0, X_b]
u_data = [u_0, u_b]
```

Next, we illustrate the collocation points (red circles) and the positions where the boundary and initial conditions will be enforced (cross marks, color indicates value).

```python
import matplotlib.pyplot as plt

fig = plt.figure(figsize=(9,6))
plt.scatter(t_0, x_0, c=u_0, marker='X', vmin=-1, vmax=1)
plt.scatter(t_b, x_b, c=u_b, marker='X', vmin=-1, vmax=1)
plt.scatter(t_r, x_r, c='r', marker='.', alpha=0.1)
plt.xlabel('$t$')
plt.ylabel('$x$')

plt.title('Positions of collocation points and boundary data');
#plt.savefig('Xdata_Burgers.pdf', bbox_inches='tight', dpi=300)
```

### 3. Set up network architecture

In this example, adopted from
([Raissi et al., 2017 (Part I)](https://arxiv.org/abs/1711.10561)), we assume a feedforward neural network of the following structure:
- the input is scaled elementwise to lie in the interval $[-1,1]$,
- followed by 8 fully connected layers each containing 20 neurons and each followed by a hyperbolic tangent activation function,
- one fully connected output layer.

This setting results in a network with $3021$ trainable parameters (first hidden layer: $2 \cdot 20 + 20 = 60$; seven intermediate layers: each $20 \cdot 20 + 20 = 420$; output layer: $20 \cdot 1 + 1 = 21$).

```python
def init_model(num_hidden_layers=8, num_neurons_per_layer=20):
    # Initialize a feedforward neural network
    model = tf.keras.Sequential()

    # Input is two-dimensional (time + one spatial dimension)
    model.add(tf.keras.Input(2))

    # Introduce a scaling layer to map input to [lb, ub]
    scaling_layer = tf.keras.layers.Lambda(
                lambda x: 2.0*(x - lb)/(ub - lb) - 1.0)
    model.add(scaling_layer)

    # Append hidden layers
    for _ in range(num_hidden_layers):
        model.add(tf.keras.layers.Dense(num_neurons_per_layer,
            activation=tf.keras.activations.get('tanh'),
            kernel_initializer='glorot_normal'))

    # Output is one-dimensional
    model.add(tf.keras.layers.Dense(1))

    return model
```

### 4. Define routines to determine loss and gradient

In the following code cell, we define a function which evaluates the residual

$$
\begin{align}
    r_\theta (t,x) := \partial_t u_\theta (t,x) + \mathcal{N}[u_\theta] (t,x).
\end{align}
$$

of the nonlinear PDE in the points $X_r = \{(t^r_i,x^r_i)\}_{i=1}^{N_r}$.
To compute the necessary partial derivatives we use the automatic differentiation capabilities of TensorFlow.

For the Burgers equation, this entails computing $\partial_t u_\theta$, $\partial_x u_\theta$ and $\partial_{xx} u_\theta$.
In TensorFlow, this is done via a `GradientTape`, see also the [documentation](https://www.tensorflow.org/api_docs/python/tf/GradientTape), which keeps track of the `watched` variables, in our case `t` and `x`, in order to compute the derivatives.

```python
def get_r(model, X_r):

    # A tf.GradientTape is used to compute derivatives in TensorFlow
    with tf.GradientTape(persistent=True) as tape:
        # Split t and x to compute partial derivatives
        t, x = X_r[:, 0:1], X_r[:,1:2]

        # Variables t and x are watched during tape
        # to compute derivatives u_t and u_x
        tape.watch(t)
        tape.watch(x)

        # Determine residual
        u = model(tf.stack([t[:,0], x[:,0]], axis=1))

        # Compute gradient u_x within the GradientTape
        # since we need second derivatives
        u_x = tape.gradient(u, x)

    u_t = tape.gradient(u, t)
    u_xx = tape.gradient(u_x, x)

    del tape

    return fun_r(t, x, u, u_t, u_x, u_xx)
```

The next function computes the loss for our model

$$
\begin{align}
    \phi_\theta(X) := \phi_\theta^r(X^r) + \phi_\theta^0(X^0) + \phi_\theta^b(X^b),
\end{align}
$$

as a function of our the training data.
The collocation points are given by `X_r`, the initial and boundary data is contained in `X_data = [X_0, X_b]` and `u_data = [u_0, u_b]`.

```python
def compute_loss(model, X_r, X_data, u_data):

    # Compute phi^r
    r = get_r(model, X_r)
    phi_r = tf.reduce_mean(tf.square(r))

    # Initialize loss
    loss = phi_r

    # Add phi^0 and phi^b to the loss
    for i in range(len(X_data)):
        u_pred = model(X_data[i])
        loss += tf.reduce_mean(tf.square(u_data[i] - u_pred))

    return loss
```

The next function computes the gradient of the loss function $\phi_\theta$ with respect to the unknown variables in the model, also called `trainable_variables` in TensorFlow, i.e. $\nabla_\theta \phi_\theta$.
This is also done via a `GradientTape`, but now it keeps track of the parameters $\theta$ in our model, which can be accessed by `model.trainable_variables`.

```python
def get_grad(model, X_r, X_data, u_data):

    with tf.GradientTape(persistent=True) as tape:
        # This tape is for derivatives with
        # respect to trainable variables
        tape.watch(model.trainable_variables)
        loss = compute_loss(model, X_r, X_data, u_data)

    g = tape.gradient(loss, model.trainable_variables)
    del tape

    return loss, g
```

### 5. Set up optimizer and train model

Next we initialize the model, set the learning rate to the step function

$$
\delta(n) = 0.01 \, \textbf{1}_{\{n < 1000\}} + 0.001 \, \textbf{1}_{\{1000 \le n < 3000\}} + 0.0005 \, \textbf{1}_{\{3000 \le n\}}
$$

which decays in a piecewise constant fashion, and set up a `tf.keras.optimizer` to train the model.

```python
# Initialize model aka u_\theta
model = init_model()

# We choose a piecewise decay of the learning rate, i.e., the
# step size in the gradient descent type algorithm
# the first 1000 steps use a learning rate of 0.01
# from 1000 - 3000: learning rate = 0.001
# from 3000 onwards: learning rate = 0.0005

lr = tf.keras.optimizers.schedules.PiecewiseConstantDecay([1000,3000],[1e-2,1e-3,5e-4])

# Choose the optimizer
optim = tf.keras.optimizers.Adam(learning_rate=lr)
```

Train the model for $N=5000$ epochs (takes approximately 3 minutes).
Here, we set up a function `train_step()` which performs one training step.

*Note*: The `@tf.function` is a so-called `Decorator` within Python. This particular decorator redefines the function that follows, in our case `train_step`, as a TensorFlow graph which may speed up the training significantly.

```python
from time import time

# Define one training step as a TensorFlow function to increase speed of training
@tf.function
def train_step():
    # Compute current loss and gradient w.r.t. parameters
    loss, grad_theta = get_grad(model, X_r, X_data, u_data)

    # Perform gradient descent step
    optim.apply_gradients(zip(grad_theta, model.trainable_variables))

    return loss

# Number of training epochs
N = 5000
hist = []

# Start timer
t0 = time()

for i in range(N+1):

    loss = train_step()

    # Append current loss to hist
    hist.append(loss.numpy())

    # Output current loss after 50 iterates
    if i%50 == 0:
        print('It {:05d}: loss = {:10.8e}'.format(i,loss))

# Print computation time
print('\nComputation time: {} seconds'.format(time()-t0))
```

### Plot solution

```python
from mpl_toolkits.mplot3d import Axes3D

# Set up meshgrid
N = 600
tspace = np.linspace(lb[0], ub[0], N + 1)
xspace = np.linspace(lb[1], ub[1], N + 1)
T, X = np.meshgrid(tspace, xspace)
Xgrid = np.vstack([T.flatten(),X.flatten()]).T

# Determine predictions of u(t, x)
upred = model(tf.cast(Xgrid,DTYPE))

# Reshape upred
U = upred.numpy().reshape(N+1,N+1)

# Surface plot of solution u(t,x)
fig = plt.figure(figsize=(9,6))
ax = fig.add_subplot(111, projection='3d')
ax.plot_surface(T, X, U, cmap='viridis');
ax.view_init(35,35)
ax.set_xlabel('$t$')
ax.set_ylabel('$x$')
ax.set_zlabel('$u_\\theta(t,x)$')
ax.set_title('Solution of Burgers equation');
#plt.savefig('Burgers_Solution.pdf', bbox_inches='tight', dpi=300);
```

### Plot the evolution of loss

```python
fig = plt.figure(figsize=(9,6))
ax = fig.add_subplot(111)
ax.semilogy(range(len(hist)), hist,'k-')
ax.set_xlabel('$n_{epoch}$')
ax.set_ylabel('$\\phi_{n_{epoch}}$');
```

## Class implementation of PINNs

In this section, we implement PINNs as a class which can be used for further testing. Here, we derive the class `PINN_NeuralNet` from `tf.keras.Model`.

Required arguments are the lower bound `lb` and upper bound `ub`.

```python
# Define model architecture
class PINN_NeuralNet(tf.keras.Model):
    """ Set basic architecture of the PINN model."""

    def __init__(self, lb, ub,
            output_dim=1,
            num_hidden_layers=8,
            num_neurons_per_layer=20,
            activation='tanh',
            kernel_initializer='glorot_normal',
            **kwargs):
        super().__init__(**kwargs)

        self.num_hidden_layers = num_hidden_layers
        self.output_dim = output_dim
        self.lb = lb
        self.ub = ub

        # Define NN architecture
        self.scale = tf.keras.layers.Lambda(
            lambda x: 2.0*(x - lb)/(ub - lb) - 1.0)
        self.hidden = [tf.keras.layers.Dense(num_neurons_per_layer,
                             activation=tf.keras.activations.get(activation),
                             kernel_initializer=kernel_initializer)
                           for _ in range(self.num_hidden_layers)]
        self.out = tf.keras.layers.Dense(output_dim)

    def call(self, X):
        """Forward-pass through neural network."""
        Z = self.scale(X)
        for i in range(self.num_hidden_layers):
            Z = self.hidden[i](Z)
        return self.out(Z)

```

Next, we derive a class `PINNSolver` which can be used as a base class.
It possesses two methods to solve the PDE:
  1. the method `solve_with_TFoptimizer` uses a `TensorFlow` optimizer object as input, e.g., the `AdamOptimizer` above;
  2. the method `solve_with_LBFGS` resembles the LBFGS method proposed in the original paper using an LBFGS method provided by [`SciPy`](https://www.scipy.org/).

```python
import scipy.optimize

class PINNSolver():
    def __init__(self, model, X_r):
        self.model = model

        # Store collocation points
        self.t = X_r[:,0:1]
        self.x = X_r[:,1:2]

        # Initialize history of losses and global iteration counter
        self.hist = []
        self.iter = 0

    def get_r(self):

        with tf.GradientTape(persistent=True) as tape:
            # Watch variables representing t and x during this GradientTape
            tape.watch(self.t)
            tape.watch(self.x)

            # Compute current values u(t,x)
            u = self.model(tf.stack([self.t[:,0], self.x[:,0]], axis=1))

            u_x = tape.gradient(u, self.x)

        u_t = tape.gradient(u, self.t)
        u_xx = tape.gradient(u_x, self.x)

        del tape

        return self.fun_r(self.t, self.x, u, u_t, u_x, u_xx)

    def loss_fn(self, X, u):

        # Compute phi_r
        r = self.get_r()
        phi_r = tf.reduce_mean(tf.square(r))

        # Initialize loss
        loss = phi_r

        # Add phi_0 and phi_b to the loss
        for i in range(len(X)):
            u_pred = self.model(X[i])
            loss += tf.reduce_mean(tf.square(u[i] - u_pred))

        return loss

    def get_grad(self, X, u):
        with tf.GradientTape(persistent=True) as tape:
            # This tape is for derivatives with
            # respect to trainable variables
            tape.watch(self.model.trainable_variables)
            loss = self.loss_fn(X, u)

        g = tape.gradient(loss, self.model.trainable_variables)
        del tape

        return loss, g

    def fun_r(self, t, x, u, u_t, u_x, u_xx):
        """Residual of the PDE"""
        return u_t + u * u_x - viscosity * u_xx

    def solve_with_TFoptimizer(self, optimizer, X, u, N=1001):
        """This method performs a gradient descent type optimization."""

        @tf.function
        def train_step():
            loss, grad_theta = self.get_grad(X, u)

            # Perform gradient descent step
            optimizer.apply_gradients(zip(grad_theta, self.model.trainable_variables))
            return loss

        for i in range(N):

            loss = train_step()

            self.current_loss = loss.numpy()
            self.callback()

    def solve_with_ScipyOptimizer(self, X, u, method='L-BFGS-B', **kwargs):
        """This method provides an interface to solve the learning problem
        using a routine from scipy.optimize.minimize.
        (Tensorflow 1.xx had an interface implemented, which is not longer
        supported in Tensorflow 2.xx.)
        Type conversion is necessary since scipy-routines are written in Fortran
        which requires 64-bit floats instead of 32-bit floats."""

        def get_weight_tensor():
            """Function to return current variables of the model
            as 1d tensor as well as corresponding shapes as lists."""

            weight_list = []
            shape_list = []

            # Loop over all variables, i.e. weight matrices, bias vectors and unknown parameters
            for v in self.model.variables:
                shape_list.append(v.shape)
                weight_list.extend(v.numpy().flatten())

            weight_list = tf.convert_to_tensor(weight_list)
            return weight_list, shape_list

        x0, shape_list = get_weight_tensor()

        def set_weight_tensor(weight_list):
            """Function which sets list of weights
            to variables in the model."""
            idx = 0
            for v in self.model.variables:
                vs = v.shape

                # Weight matrices
                if len(vs) == 2:
                    sw = vs[0]*vs[1]
                    new_val = tf.reshape(weight_list[idx:idx+sw],(vs[0],vs[1]))
                    idx += sw

                # Bias vectors
                elif len(vs) == 1:
                    new_val = weight_list[idx:idx+vs[0]]
                    idx += vs[0]

                # Variables (in case of parameter identification setting)
                elif len(vs) == 0:
                    new_val = weight_list[idx]
                    idx += 1

                # Assign variables (Casting necessary since scipy requires float64 type)
                v.assign(tf.cast(new_val, DTYPE))

        def get_loss_and_grad(w):
            """Function that provides current loss and gradient
            w.r.t the trainable variables as vector. This is mandatory
            for the LBFGS minimizer from scipy."""

            # Update weights in model
            set_weight_tensor(w)
            # Determine value of \phi and gradient w.r.t. \theta at w
            loss, grad = self.get_grad(X, u)

            # Store current loss for callback function
            loss = loss.numpy().astype(np.float64)
            self.current_loss = loss

            # Flatten gradient
            grad_flat = []
            for g in grad:
                grad_flat.extend(g.numpy().flatten())

            # Gradient list to array
            grad_flat = np.array(grad_flat,dtype=np.float64)

            # Return value and gradient of \phi as tuple
            return loss, grad_flat

        return scipy.optimize.minimize(fun=get_loss_and_grad,
                                       x0=x0,
                                       jac=True,
                                       method=method,
                                       callback=self.callback,
                                       **kwargs)

    def callback(self, xr=None):
        if self.iter % 50 == 0:
            print('It {:05d}: loss = {:10.8e}'.format(self.iter,self.current_loss))
        self.hist.append(self.current_loss)
        self.iter+=1

    def plot_solution(self, **kwargs):
        N = 600
        tspace = np.linspace(self.model.lb[0], self.model.ub[0], N+1)
        xspace = np.linspace(self.model.lb[1], self.model.ub[1], N+1)
        T, X = np.meshgrid(tspace, xspace)
        Xgrid = np.vstack([T.flatten(),X.flatten()]).T
        upred = self.model(tf.cast(Xgrid,DTYPE))
        U = upred.numpy().reshape(N+1,N+1)
        fig = plt.figure(figsize=(9,6))
        ax = fig.add_subplot(111, projection='3d')
        ax.plot_surface(T, X, U, cmap='viridis', **kwargs)
        ax.set_xlabel('$t$')
        ax.set_ylabel('$x$')
        ax.set_zlabel('$u_\\theta(t,x)$')
        ax.view_init(35,35)
        return ax

    def plot_loss_history(self, ax=None):
        if not ax:
            fig = plt.figure(figsize=(7,5))
            ax = fig.add_subplot(111)
        ax.semilogy(range(len(self.hist)), self.hist,'k-')
        ax.set_xlabel('$n_{epoch}$')
        ax.set_ylabel('$\\phi^{n_{epoch}}$')
        return ax
```

### Burgers equation with L-BFGS

The following code cell shows how the new classes `PINN_NeuralNet` and `PINNSolver` can be used to solve the Burgers equation, this time using the `SciPy` implementation of L-BFGS (takes around 3 minutes).

```python
# Initialize model
model = PINN_NeuralNet(lb, ub)
model.build(input_shape=(None,2))

# Initilize PINN solver
solver = PINNSolver(model, X_r)

# Decide which optimizer should be used
#mode = 'TFoptimizer'
mode = 'ScipyOptimizer'

# Start timer
t0 = time()

if mode == 'TFoptimizer':
    # Choose optimizer
    lr = tf.keras.optimizers.schedules.PiecewiseConstantDecay([1000,3000],[1e-2,1e-3,5e-4])
    optim = tf.keras.optimizers.Adam(learning_rate=lr)
    solver.solve_with_TFoptimizer(optim, X_data, u_data, N=4001)

elif mode == 'ScipyOptimizer':
    solver.solve_with_ScipyOptimizer(X_data, u_data,
                            method='L-BFGS-B',
                            options={'maxiter': 50000,
                                     'maxfun': 50000,
                                     'maxcor': 50,
                                     'maxls': 50,
                                     'ftol': 1.0*np.finfo(float).eps})

# Print computation time
print('\nComputation time: {} seconds'.format(time()-t0))
```

Plot solution and loss history.

```python
solver.plot_solution();
solver.plot_loss_history();
```

### Solution of a time-dependent Eikonal equation

As a second example we consider the one-dimensional Eikonal equation backward in time on the domain $\mathcal{D}=[-1,1]$

$$
\begin{align}
   -\partial_t u(t,x) + |\nabla u|(t,x) &= 1,
   		\quad & &(t,x) \in [0,T) \times [-1,1],\\
   u(T,x) &= 0, \quad & &x \in [-1,1],\\
   u(t,-1) = u(t, 1) &= 0, \quad & & t \in [0,T).
\end{align}
$$

Note that the partial differential equation can be equally written as a Hamilton-Jacobi-Bellman equation, viz

$$
    -\partial_t u(t,x) + \sup_{|c| \le 1} \{c \, \nabla u(t,x)\} = 1 \quad (t,x) \in [0,T) \times [-1,1],
$$

which characterizes the solution of an optimal control problem seeking to minimize the distance from a point $(t,x)$ to the boundary $[0,T] \times \partial \mathcal{D} \cup \{T\} \times \mathcal{D}$.
As is easily verified, the solution is given by $u(t,x) = \min\{ 1 - t, 1 - |x| \}$.
The fact that the Eikonal equation runs backward in time is in accordance with its interpretation as the optimality condition of a control problem.
Note that this is equation can be transformed into a forward evolution problem by the change of variables $\hat t = T - t$.

#### Problem specific definitions

```python
N_0 = 50
N_b = 50
N_r = 10000

# Specify boundaries
lb = tf.constant([0., -1.], dtype=DTYPE)
ub = tf.constant([1., 1.], dtype=DTYPE)

def Eikonal_u_0(x):
    n = x.shape[0]
    return tf.zeros((n,1), dtype=DTYPE)

def Eikonal_u_b(t, x):
    n = x.shape[0]
    return tf.zeros((n,1), dtype=DTYPE)
```

#### Generate data
This code snippet is almost identical to the code from above.
We choose $N_b = 50$ and $N_0 = 50$ uniformly distributed initial value and boundary points and sample $N_r = 10000$ collocation points uniformly within the domain boundaries.
We derive a new solver with `PINNSolver` as base class.

```python
tf.random.set_seed(0)

# Final time data
t_0 = tf.ones((N_0,1), dtype=DTYPE) * ub[0]
x_0 = tf.random.uniform((N_0,1), lb[1], ub[1], dtype=DTYPE)
X_0 = tf.concat([t_0, x_0], axis=1)
u_0 = Eikonal_u_0(x_0)

# Boundary data
t_b = tf.random.uniform((N_b,1), lb[0], ub[0], dtype=DTYPE)
x_b = lb[1] + (ub[1] - lb[1]) * tf.keras.backend.random_bernoulli((N_b,1), 0.5, dtype=DTYPE)
X_b = tf.concat([t_b, x_b], axis=1)
u_b = Eikonal_u_b(t_b, x_b)

# Collocation points
t_r = tf.random.uniform((N_r,1), lb[0], ub[0], dtype=DTYPE)
x_r = tf.random.uniform((N_r,1), lb[1], ub[1], dtype=DTYPE)
X_r = tf.concat([t_r, x_r], axis=1)

# Collect boundary and inital data in lists
X_data = [X_0,X_b]
u_data = [u_0,u_b]
```

#### Derive Eikonal solver class
Now, we derive a solver for the Eikonal equation from the `PINNSolver` class. Since the Eikonal equation does not depend on second-order derivatives, we implement a new method `get_r` which avoids the computation of second derivatives.

```python
class EikonalPINNSolver(PINNSolver):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def fun_r(self, t, x, u, u_t, u_x, u_xx):
        """Residual of the PDE"""
        return -u_t + tf.abs(u_x) - 1.

    def get_r(self):
        """We update get_r since the Eikonal equation is a first-order equation.
        Therefore, it is not necessary to compute second derivatives."""

        with tf.GradientTape(persistent=True) as tape:
            # Watch variables representing t and x during this GradientTape
            tape.watch(self.t)
            tape.watch(self.x)

            # Compute current values u(t,x)
            u = self.model(tf.stack([self.t[:,0], self.x[:,0]], axis=1))

        u_x = tape.gradient(u, self.x)

        u_t = tape.gradient(u, self.t)

        del tape

        return self.fun_r(self.t, self.x, u, u_t, u_x, None)
```

#### Setting up the neural network architecture

The neural network model chosen for this particular problem can be simpler.
We decided to use only two hidden layers with 20 neurons in each, resulting in $501$ unknown parameters (first hidden layer: $2 \cdot 20 + 20 = 60$; one intermediate layer: $20 \cdot 20 + 20 = 420$; output layer: $20 \cdot 1 + 1 = 21$).
To account for the lack of smoothness of the solution, we choose a non-differentiable activation function, although the hyperbolic tangent function seems to be able to approximate the kinks in the solution sufficiently well.
Here, we decided to use the \emph{leaky rectified linear unit (leaky ReLU)} activation function

$$
\begin{align*}
    \sigma(z) = \begin{cases}
        z &\text{ if } z \ge 0,\\
        0.1 \, z &\text{ otherwise},
    \end{cases}
\end{align*}
$$

which displays a non-vanishing gradient when the unit is not active, i.e., when $z < 0$.

```python
# Initialize model
model = PINN_NeuralNet(lb, ub, num_hidden_layers=2,
                       activation=tf.keras.layers.LeakyReLU(alpha=0.1),
                                       kernel_initializer='he_normal')
model.build(input_shape=(None,2))

# Initilize PINN solver
eikonalSolver = EikonalPINNSolver(model, X_r)
```

Start training (take approximately 40 seconds).

```python
# Choose step sizes aka learning rate
lr = tf.keras.optimizers.schedules.PiecewiseConstantDecay([3000,7000],[1e-1,1e-2,1e-3])

# Solve with Adam optimizer
optim = tf.keras.optimizers.Adam(learning_rate=lr)

# Start timer
t0 = time()
eikonalSolver.solve_with_TFoptimizer(optim, X_data, u_data, N=10001)

# Print computation time
print('\nComputation time: {} seconds'.format(time()-t0))
```

Plot the results.

```python
eikonalSolver.plot_solution();
#plt.savefig('Eikonal_Solution.pdf', bbox_inches='tight', dpi=300)
eikonalSolver.plot_loss_history();
```

## Parameter identification setting

In this section, we want to demonstrate how the PINN approach could be used to solve partial differential equations with unknown parameters $\lambda$.
To be more precise, we consider the parabolic Eikonal equation

$$
\begin{aligned}
- \partial_t u + \sup_{|c|\le1} c \cdot \nabla u = -\partial_t u + |\nabla u| &= \lambda^{-1}\\
u(T,x) &= 0\\
u(t,-1) = u(t,1) &= 0
\end{aligned}
$$

with unknown parameter $\lambda$.
The explicit solution is $u^*(t,x) = \lambda^{-1} \min\{1-t, 1-|x|\}$.

```python
lambd_star = 3.

def u_expl(t, x, lambd_star):
    """Explicit solution of the parametric Eikonal equation."""
    y = 1./lambd_star
    return y * tf.math.minimum(1-t, 1-tf.abs(x))
```

Next, we draw $N_d = 500$ uniformly distributed measurements of the exact solution.

```python
N_d = 500
noise = 0.0

# Draw points with measurements randomly
t_d = tf.random.uniform((N_d,1), lb[0], ub[0], dtype=DTYPE)
x_d = tf.random.uniform((N_d,1), lb[1], ub[1], dtype=DTYPE)
X_d = tf.concat([t_d, x_d], axis=1)
u_d = u_expl(t_d, x_d, lambd_star)
u_d += noise * tf.random.normal(u_d.shape, dtype=DTYPE)

# Copy original data
X_param = X_data
u_param = u_data
```

Since both the boundary and initial time data are of Dirichlet type, we may handle the measured data exactly like $X_0$ and $X_b$.
Thus, we can simply append $X_d$ and $u_d$ to `Xdata` and `udata`.

Note that the approach illustrated here is slightly different from the one introduced in ([Raissi et al., 2017 (Part II)](https://arxiv.org/abs/1711.10566)) which takes only measurement data into account.

```python
X_param.append(X_d)
u_param.append(u_d)
```

Next, we derive a new network class which takes the additional parameter $\lambda$ into account.
Note that this parameter has to be part of the model in order to be learnt during training.

```python
class PINNIdentificationNet(PINN_NeuralNet):
    def __init__(self, *args, **kwargs):

        # Call init of base class
        super().__init__(*args,**kwargs)

        # Initialize variable for lambda
        self.lambd = tf.Variable(1.0, trainable=True, dtype=DTYPE)
        self.lambd_list = []
```

Now, we derive a new solver class which only updates the evaluation of the residual `fun_r` which now incorporates the $\lambda$-dependency.
In addition, we modify the `callback` function to store the iterates of $\lambda$ in a list `lambd_list` as well.

```python
class EikonalPINNIdentification(EikonalPINNSolver):

    def fun_r(self, t, x, u, u_t, u_x, u_xx):
        """Residual of the PDE"""
        return -u_t + tf.abs(u_x) - 1./self.model.lambd

    def callback(self, xr=None):
        lambd = self.model.lambd.numpy()
        self.model.lambd_list.append(lambd)

        if self.iter % 50 == 0:
            print('It {:05d}: loss = {:10.8e} lambda = {:10.8e}'.format(self.iter, self.current_loss, lambd))

        self.hist.append(self.current_loss)
        self.iter += 1

    def plot_loss_and_param(self, axs=None):
        if axs:
            ax1, ax2 = axs
            self.plot_loss_history(ax1)
        else:
            ax1 = self.plot_loss_history()
            ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis

        color = 'tab:blue'
        ax2.tick_params(axis='y', labelcolor=color)
        ax2.plot(range(len(self.hist)), self.model.lambd_list,'-',color=color)
        ax2.set_ylabel('$\\lambda^{n_{epoch}}$', color=color)
        return (ax1,ax2)
```

Finally, we set up the model consisting of only two hidden layers employing the Leaky ReLU function with slope parameter $\alpha = 0.1$, i.e.,

$$
\sigma(z) = \begin{cases} x & \text{ if } x \ge 0\\ \alpha \, x & \text{ otherwise.} \end{cases}
$$

The training for $n_{epochs} = 10000$ epochs takes around 45 seconds.

```python
# Initialize model
model = PINNIdentificationNet(lb, ub, num_hidden_layers=2,
                                        activation=tf.keras.layers.LeakyReLU(alpha=0.1),
                                        kernel_initializer='he_normal')

model.build(input_shape=(None,2))

# Initilize solver
eikonalIdentification = EikonalPINNIdentification(model, X_r)

# Choose step sizes aka learning rate
lr = tf.keras.optimizers.schedules.PiecewiseConstantDecay([3000,7000],[1e-1,1e-2,1e-3])

# Solve with Adam optimizer
optim = tf.keras.optimizers.Adam(learning_rate=lr)

# Start timer
t0 = time()
eikonalIdentification.solve_with_TFoptimizer(optim, X_param, u_param, N=10001)

# Print computation time
print('\nComputation time: {} seconds'.format(time()-t0))
```

Plot solution $u_\theta(t,x)$ and evolution of loss values $\phi^{n_\text{epoch}}$ and estimated parameters $\lambda^{n_\text{epoch}}$..

```python
ax = eikonalIdentification.plot_solution()
#plt.savefig('Eikonal_PI_Solution.pdf', bbox_inches='tight', dpi=300)
axs = eikonalIdentification.plot_loss_and_param()
#plt.savefig('Eikonal_PI_LossEvolution.pdf', bbox_inches='tight', dpi=300)
```

Finally, we compute the relative error of the identified parameter $\lambda$.

```python
lambd_rel_error = np.abs((eikonalIdentification.model.lambd.numpy()-lambd_star)/lambd_star)
print('Relative error of lambda ', lambd_rel_error)
```

The next code cell performs the previous training 5 times in order to give a more reliable picture of the convergence since the weight matrices are initialized randomly at each run (takes about 4 minutes).

```python
lambd_hist = []
loss_hist = []
for i in range(5):
    print('{:s}\nStart of iteration {:d}\n{:s}'.format(50*'-',i,50*'-'))
    # Initialize model
    model = PINNIdentificationNet(lb, ub, num_hidden_layers=2,
                                        activation=tf.keras.layers.LeakyReLU(alpha=0.1),
                                        kernel_initializer='he_normal')

    model.build(input_shape=(None,2))

    # Initilize solver
    eikonalIdentification = EikonalPINNIdentification(model, X_r)

    # Choose step sizes aka learning rate
    lr = tf.keras.optimizers.schedules.PiecewiseConstantDecay([3000,7000],[1e-1,1e-2,1e-3])
    N=10001

    # Solve with Adam optimizer
    optim = tf.keras.optimizers.Adam(learning_rate=lr)

    eikonalIdentification.solve_with_TFoptimizer(optim, X_param, u_param, N=N)

    # Store evolution of lambdas
    lambd_hist.append(model.lambd_list)

    # Store evolution of losses
    loss_hist.append(eikonalIdentification.hist)
```

Next, we generate a table printing the mean and standard deviations of the identified parameter $\lambda$ obtained for the previous runs.

```python
print('  i     Mean of lambda   Std. of lambda')
for i in [1000,2000,3000,4000,5000,6000,7000,8000,9000,10000]:
    xi = np.array([ x[i] for x in lambd_hist])
    print('{:05d}     {:6.4e}       {:6.4e}'.format(i, xi.mean(), xi.std()))
```

Next, we plot the five evolutions of $\lambda$ (dark gray), its mean (solid blue) and one standard deviation (shaded blue) together with the true value of $\lambda$ (dashed blue).

```python
fig = plt.figure()
ax = fig.add_subplot(111)
color = 'tab:blue'

Lambd = np.stack(lambd_hist)
lmean = Lambd.mean(axis=0)
lstd = Lambd.std(axis=0)
Lambd_RelError = np.abs((Lambd-lambd_star)/lambd_star)
lrange=range(len(lmean))

for i in range(len(lambd_hist)):
    ax.plot(lrange, lambd_hist[i],'-',color='black', alpha=0.5)
ax.plot(lrange, lmean,'-',color=color)
ax.plot(lrange, lambd_star*np.ones_like(lmean),'--',color=color)
ax.fill_between(lrange,lmean-lstd,lmean+lstd, alpha=0.2)
ax.set_ylabel('$\\lambda^{n_{epoch}}$')
ax.set_xlabel('$n_{epoch}$')
ax.set_ylim([2.8,3.2])
#plt.savefig('Eikonal_PI_Evolution.pdf', bbox_inches='tight', dpi=300)
```

Finally, we plot the mean relative error of $\lambda$.

```python
fig = plt.figure()
ax = fig.add_subplot(111)
ax.semilogy(lrange,Lambd_RelError.mean(axis=0))
ax.fill_between(lrange,Lambd_RelError.mean(axis=0)-Lambd_RelError.std(axis=0),
                Lambd_RelError.mean(axis=0)+Lambd_RelError.std(axis=0), alpha=0.2)
ax.set_xlabel('$n_{epoch}$')
ax.set_ylabel('$e_{\\lambda}^{rel}$')
ax.set_title('Mean relative error of $\\lambda$');
```


# Solving a Laplace problem with Dirichlet boundary conditions

### Background

- Laplace problem inside the unit sphere with Dirichlet boundary conditions. Let $\Omega$ be the unit sphere with boundary $\Gamma$. Let $\nu$ be the outward pointing normal on $\Gamma$. The PDE and boundary conditions are given by

\begin{align}
\Delta u &= 0&&\text{in }\Omega,\\
u &= f&&\text{on }\Gamma.
\end{align}

The boundary data is a source $\hat{u}$ located at the point $(0.9,0,0)$.
$$
\hat{u}(\mathbf x)=\frac{1}{4\pi\sqrt{(x-0.9)^2+y^2+z^2}}.
$$

For this example, we will use a direct integral equation of the first kind. Let
$$
g(\mathbf x,\mathbf y) = \frac{1}{4\pi |\mathbf x-\mathbf y|}
$$
be the Green's function for Laplace in three dimensions. From Green's representation theorem, it follows that every harmonic function $u$ in $\Omega$ satisfies

$$
u(\mathbf x) = \int_{\Gamma} g(\mathbf x,\mathbf y)\frac{\partial u(\mathbf y)}{\partial \nu(\mathbf{y})}\mathrm{d}\mathbf y-\int_{\Gamma}\frac{\partial g(\mathbf x,\mathbf y)}{\partial \nu(\mathbf{y})}u(\mathbf y)\mathrm{d}\mathbf y,~\mathbf x\in\Omega\setminus\Gamma
$$

or equivalantly

$$
u(\mathbf x) = \left[\mathcal{V}\frac{\partial u(\mathbf y)}{\partial \nu(\mathbf{y})}\right] (\mathbf{x}) - \left[\mathcal{K}u\right] (\mathbf{x}),~\mathbf x\in\Omega\setminus\Gamma,
$$

where $\mathcal{V}$ and $\mathcal{K}$ are the single and double layer potential operators.

Taking the limit $\mathbf x\rightarrow \Gamma$ we obtain the boundary integral equation

$$
\left[\mathsf{V}\frac{\partial u}{\partial n}\right] (\mathbf x)=\left[(\tfrac12\mathsf{Id}+\mathsf{K})u\right] (\mathbf x),~\mathbf x\in\Gamma.
$$

Here, $\mathsf{V}$ and $\mathsf{K}$ are the single and double layer boundary operators.

This equation is the target equation that we will solve with respect to unknown norrmal derivative on the boundary $\frac{\partial u}{\partial n}$.

When we get normal derivative on the boundary
$$
\frac{\partial u}{\partial n}
$$
we can use Green representation theorem to compute $u(x)$ in any points inside the domain $\Omega$.

### Implementation

We now demonstrate how to solve this problem with Bempp. We begin by importing Bempp and NumPy.

```python
import bempp.api
import numpy as np
```

We next define a mesh or grid. For this problem, we will use the built-in function `sphere` that defines a simple spherical grid. Details of other available grids can be found in the grids tutorial.

```python
grid = bempp.api.shapes.sphere(h=0.1)
```

We now define the spaces.
We will use two spaces:
- the space of continuous, piecewise linear functions
- the space of piecewise constant functions.

The space of piecewise constant functions has the right smoothness for the unknown Neumann data.
We will use continuous, piecewise linear functions to represent the known Dirichlet data.

```python
dp0_space = bempp.api.function_space(grid, "DP", 0)
p1_space = bempp.api.function_space(grid, "P", 1)
```

We can now define the operators. We need the identity, single layer, and double layer boundary operator.

```python
identity = bempp.api.operators.boundary.sparse.identity(
    p1_space, p1_space, dp0_space)
dlp = bempp.api.operators.boundary.laplace.double_layer(
    p1_space, p1_space, dp0_space)
slp = bempp.api.operators.boundary.laplace.single_layer(
    dp0_space, p1_space, dp0_space)
```

We now define the GridFunction object on the sphere grid that represents the Dirichlet data.

```python
def dirichlet_data(x, n, domain_index, result):
    result[0] = 1./(4 * np.pi * ((x[0] - .9)**2 + x[1]**2 + x[2]**2)**(0.5))

dirichlet_fun = bempp.api.GridFunction(p1_space, fun=dirichlet_data)
```

We next assemble the right-hand side of the boundary integral equation, given by
$$
(\tfrac12\mathsf{Id}+\mathsf{K})u.
$$

We use implemented operators algebra to define right-hand side.

```python
rhs = (.5 * identity + dlp) * dirichlet_fun
```

We now solve the linear system using a conjugate gradient (CG) method.

```python
neumann_fun, info = bempp.api.linalg.cg(slp, rhs, tol=1E-3)
```

We now want to provide a simple plot of the solution in the $(x,y)$ plane for $z=0$. We first define points at which to plot the solution.

```python
n_grid_points = 150
plot_grid = np.mgrid[-1:1:n_grid_points*1j, -1:1:n_grid_points*1j]
points = np.vstack((plot_grid[0].ravel(),
                    plot_grid[1].ravel(),
                    np.zeros(plot_grid[0].size)))
```

The variable `points` now contains in its columns the coordinates of the evaluation points. We can now use Green's representation theorem to evaluate the solution on these points. Note in particular the last line of the following code. It is a direct implementation of Green's representation theorem.

```python
slp_pot = bempp.api.operators.potential.laplace.single_layer(
    dp0_space, points)
dlp_pot = bempp.api.operators.potential.laplace.double_layer(
    p1_space, points)
u_evaluated = slp_pot * neumann_fun - dlp_pot * dirichlet_fun
```

We now plot the 2D slice of the solution. For a full three dimensional visualization, Bempp can export the data to Gmsh. Since the solution decays quickly, we use a logarithmic plot.

```python
%matplotlib inline
import matplotlib
matplotlib.rcParams['figure.figsize'] = (5.0, 4.0)
from matplotlib import pylab as plt

# Filter out solution values that are associated with points outside the unit circle.
u_evaluated = u_evaluated.reshape((n_grid_points, n_grid_points))
radius = np.sqrt(plot_grid[0]**2 + plot_grid[1]**2)
u_evaluated[radius>1] = np.nan

plt.imshow(np.log(np.abs(u_evaluated.T)), extent=(-1,1,-1,1))
plt.title('Computed solution')
plt.colorbar()
```


## Naive Bayes and Bayes Classifiers

author: Jacob Schreiber

contact: jmschreiber91@gmail.com

Bayes classifiers are some of the simplest machine learning models that exist, due to their intuitive probabilistic interpretation and simple fitting step. Each class is modeled as a probability distribution, and the data is interpreted as samples drawn from these underlying distributions. Fitting the model to data is as simple as calculating maximum likelihood parameters for the data that falls under each class, and making predictions is as simple as using Bayes' rule to determine which class is most likely given the distributions. Bayes' Rule is the following:

\begin{equation}
P(M|D) = \frac{P(D|M)P(M)}{P(D)}
\end{equation}

where M stands for the model and D stands for the data. $P(M)$ is known as the prior, because it is the probability that a sample is of a certain class before you even know what the sample is. This is generally just the frequency of each class. Intuitively, it makes sense that you would want to model this, because if one class occurs 10x more than another class, it is more likely that a given sample will belong to that distribution. $P(D|M)$ is the likelihood, or the probability, of the data under a given model. Lastly, $P(M|D)$ is the posterior, which is the probability of each component of the model, or class, being the component which generated the data. It is called the posterior because the prior corresponds to probabilities before seeing data, and the posterior corresponds to probabilities after observing the data. In cases where the prior is uniform, the posterior is just equal to the normalized likelihoods. This equation forms the basis of most probabilistic modeling, with interesting priors allowing the user to inject sophisticated expert knowledge into the problem directly.

pomegranate implements two distinct models of this format with the simpler being the naive Bayes classifier. The naive Bayes classifier assumes that each feature is independent from each other feature, and so breaks down $P(D|M)$ to be $\prod\limits_{i=1}^{d} P(D_{i}|M_{i})$ where $i$ is a specific feature in a data set that has $d$ features in it. This typically means faster calculations because covariance across features doesn't need to be considered, but it also is a natural way to model each feature as a different distribution because it ignores the complexities of modeling the covariance between, say, an exponential distribution and a normal distribution.

The more general model is the Bayes classifier which does not assume that each feature is independent of the others. This allows you to plug in anything for the likelihood function, whether it be a multivariate Gaussian distribution or a whole other compositional model. For instance, one could create a classifier whose components are each large mixture models. This enables much more complex models to be learned within the simple framework of Bayes' rule.

```python
%matplotlib inline
import matplotlib.pyplot as plt
import seaborn; seaborn.set_style('whitegrid')
import numpy

from pomegranate import *

numpy.random.seed(0)
numpy.set_printoptions(suppress=True)

%load_ext watermark
%watermark -m -n -p numpy,scipy,pomegranate
```

    Sun Oct 28 2018

    numpy 1.15.1
    scipy 1.1.0
    pomegranate 0.10.0

    compiler   : GCC 7.2.0
    system     : Linux
    release    : 4.15.0-36-generic
    machine    : x86_64
    processor  : x86_64
    CPU cores  : 4
    interpreter: 64bit

### Simple Gaussian Example

```python
X = numpy.concatenate((numpy.random.normal(3, 1, 200), numpy.random.normal(10, 2, 1000)))
y = numpy.concatenate((numpy.zeros(200), numpy.ones(1000)))

x1 = X[:200]
x2 = X[200:]

plt.figure(figsize=(16, 5))
plt.hist(x1, bins=25, color='m', edgecolor='m', label="Class A")
plt.hist(x2, bins=25, color='c', edgecolor='c', label="Class B")
plt.xlabel("Value", fontsize=14)
plt.ylabel("Count", fontsize=14)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.show()
```

The data seems like it comes from two normal distributions, with the cyan class being more prevalent than the magenta class. A natural way to model this data would be to create a normal distribution for the cyan data, and another for the magenta distribution.

Let's take a look at doing that. All we need to do is use the `from_samples` class method of the `NormalDistribution` class.

```python
d1 = NormalDistribution.from_samples(x1)
d2 = NormalDistribution.from_samples(x2)
idxs = numpy.arange(0, 15, 0.1)

p1 = map(d1.probability, idxs)
p2 = map(d2.probability, idxs)

plt.figure(figsize=(16, 5))
plt.plot(idxs, p1, color='m'); plt.fill_between(idxs, 0, p1, facecolor='m', alpha=0.2)
plt.plot(idxs, p2, color='c'); plt.fill_between(idxs, 0, p2, facecolor='c', alpha=0.2)
plt.xlabel("Value", fontsize=14)
plt.ylabel("Probability", fontsize=14)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.show()
```

It looks like some aspects of the data are captured well by doing things this way-- specifically the mean and variance of the normal distributions. This allows us to easily calculate $P(D|M)$ as the probability of a sample under either the cyan or magenta distributions using the normal (or Gaussian) probability density equation:

\begin{align}
P(D|M) &= P(x|\mu, \sigma) \\
       &= \frac{1}{\sqrt{2\pi\sigma^{2}}} exp \left(-\frac{(x-u)^{2}}{2\sigma^{2}} \right)
\end{align}

However, if we look at the original data, we see that the cyan distributions is both much wider than the purple distribution and much taller, as there were more samples from that class in general. If we reduce that data down to these two distributions, we lose the class imbalance. We want our prior to model this class imbalance, with the reasoning being that if we randomly draw a sample from the samples observed thus far, it is far more likely to be a cyan than a magenta sample. Let's take a look at this class imbalance exactly.

```python
magenta_prior = 1. * len(x1) / len(X)
cyan_prior = 1. * len(x2) / len(X)

plt.figure(figsize=(4, 6))
plt.title("Prior Probabilities P(M)", fontsize=14)
plt.bar(0, magenta_prior, facecolor='m', edgecolor='m')
plt.bar(1, cyan_prior, facecolor='c', edgecolor='c')
plt.xticks([0, 1], ['P(Magenta)', 'P(Cyan)'], fontsize=14)
plt.yticks(fontsize=14)
plt.show()
```

The prior $P(M)$ is a vector of probabilities over the classes that the model can predict, also known as components. In this case, if we draw a sample randomly from the data that we have, there is a ~83% chance that it will come from the cyan class and a ~17% chance that it will come from the magenta class.

Let's multiply the probability densities we got before by this imbalance.

```python
d1 = NormalDistribution.from_samples(x1)
d2 = NormalDistribution.from_samples(x2)
idxs = numpy.arange(0, 15, 0.1)

p_magenta = numpy.array(map(d1.probability, idxs)) * magenta_prior
p_cyan = numpy.array(map(d2.probability, idxs)) * cyan_prior

plt.figure(figsize=(16, 5))
plt.plot(idxs, p_magenta, color='m'); plt.fill_between(idxs, 0, p_magenta, facecolor='m', alpha=0.2)
plt.plot(idxs, p_cyan, color='c'); plt.fill_between(idxs, 0, p_cyan, facecolor='c', alpha=0.2)
plt.xlabel("Value", fontsize=14)
plt.ylabel("P(M)P(D|M)", fontsize=14)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.show()
```

This looks a lot more faithful to the original data, and actually corresponds to $P(M)P(D|M)$, the prior multiplied by the likelihood. However, these aren't actually probability distributions anymore, as they no longer integrate to 1. This is why the $P(M)P(D|M)$ term has to be normalized by the $P(D)$ term in Bayes' rule in order to get a probability distribution over the components. However, $P(D)$ is difficult to determine exactly-- what is the probability of the data? Well, we can sum over the classes to get that value, since $P(D) = \sum_{i=1}^{c} P(D|M)P(M)$ for a problem with c classes. This translates into $P(D) = P(M=Cyan)P(D|M=Cyan) + P(M=Magenta)P(D|M=Magenta)$ for this specific problem, and those values can just be pulled from the unnormalized plots above.

This gives us the full Bayes' rule, with the posterior $P(M|D)$ being the proportion of density of the above plot coming from each of the two distributions at any point on the line. Let's take a look at the posterior probabilities of the two classes on the same line.

```python
magenta_posterior = p_magenta / (p_magenta + p_cyan)
cyan_posterior = p_cyan / (p_magenta + p_cyan)

plt.figure(figsize=(16, 5))
plt.subplot(211)
plt.plot(idxs, p_magenta, color='m'); plt.fill_between(idxs, 0, p_magenta, facecolor='m', alpha=0.2)
plt.plot(idxs, p_cyan, color='c'); plt.fill_between(idxs, 0, p_cyan, facecolor='c', alpha=0.2)
plt.xlabel("Value", fontsize=14)
plt.ylabel("P(M)P(D|M)", fontsize=14)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)

plt.subplot(212)
plt.plot(idxs, magenta_posterior, color='m')
plt.plot(idxs, cyan_posterior, color='c')
plt.xlabel("Value", fontsize=14)
plt.ylabel("P(M|D)", fontsize=14)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.show()
```

The top plot shows the same densities as before, while the bottom plot shows the proportion of the density belonging to either class at that point. This proportion is known as the posterior $P(M|D)$, and can be interpreted as the probability of that point belonging to each class. This is one of the native benefits of probabilistic models, that instead of providing a hard class label for each sample, they can provide a soft label in the form of the probability of belonging to each class.

We can implement all of this simply in pomegranate using the `NaiveBayes` class.

```python
idxs = idxs.reshape(idxs.shape[0], 1)
X = X.reshape(X.shape[0], 1)

model = NaiveBayes.from_samples(NormalDistribution, X, y)
posteriors = model.predict_proba(idxs)

plt.figure(figsize=(14, 4))
plt.plot(idxs, posteriors[:,0], color='m')
plt.plot(idxs, posteriors[:,1], color='c')
plt.xlabel("Value", fontsize=14)
plt.ylabel("P(M|D)", fontsize=14)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.show()
```

Looks like we're getting the same plots for the posteriors just through fitting the naive Bayes model directly to data. The predictions made will come directly from the posteriors in this plot, with cyan predictions happening whenever the cyan posterior is greater than the magenta posterior, and vice-versa.

## Naive Bayes

In the univariate setting, naive Bayes is identical to a general Bayes classifier. The divergence occurs in the multivariate setting, the naive Bayes model assumes independence of all features, while a Bayes classifier is more general and can support more complicated interactions or covariances between features. Let's take a look at what this means in terms of Bayes' rule.

\begin{align}
P(M|D) &= \frac{P(M)P(D|M)}{P(D)} \\
       &= \frac{P(M)\prod_{i=1}^{d}P(D_{i}|M_{i})}{P(D)}
\end{align}

This looks fairly simple to compute, as we just need to pass each dimension into the appropriate distribution and then multiply the returned probabilities together. This simplicity is one of the reasons why naive Bayes is so widely used. Let's look closer at using this in pomegranate, starting off by generating two blobs of data that overlap a bit and inspecting them.

```python
X = numpy.concatenate([numpy.random.normal(3, 2, size=(150, 2)), numpy.random.normal(7, 1, size=(250, 2))])
y = numpy.concatenate([numpy.zeros(150), numpy.ones(250)])

plt.figure(figsize=(8, 8))
plt.scatter(X[y == 0, 0], X[y == 0, 1], color='c')
plt.scatter(X[y == 1, 0], X[y == 1, 1], color='m')
plt.xlim(-2, 10)
plt.ylim(-4, 12)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.show()
```

Now, let's fit our naive Bayes model to this data using pomegranate. We can use the `from_samples` class method, pass in the distribution that we want to model each dimension, and then the data. We choose to use `NormalDistribution` in this particular case, but any supported distribution would work equally well, such as `BernoulliDistribution` or `ExponentialDistribution`. To ensure we get the correct decision boundary, let's also plot the boundary recovered by sklearn.

```python
from sklearn.naive_bayes import GaussianNB

model = NaiveBayes.from_samples(NormalDistribution, X, y)
clf = GaussianNB().fit(X, y)

xx, yy = numpy.meshgrid(numpy.arange(-2, 10, 0.02), numpy.arange(-4, 12, 0.02))
Z1 = model.predict(numpy.c_[xx.ravel(), yy.ravel()]).reshape(xx.shape)
Z2 = clf.predict(numpy.c_[xx.ravel(), yy.ravel()]).reshape(xx.shape)

plt.figure(figsize=(16, 8))
plt.subplot(121)
plt.title("pomegranate naive Bayes", fontsize=16)
plt.scatter(X[y == 0, 0], X[y == 0, 1], color='c')
plt.scatter(X[y == 1, 0], X[y == 1, 1], color='m')
plt.contour(xx, yy, Z1, cmap='Blues')
plt.xlim(-2, 10)
plt.ylim(-4, 12)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)

plt.subplot(122)
plt.title("sklearn naive Bayes", fontsize=16)
plt.scatter(X[y == 0, 0], X[y == 0, 1], color='c')
plt.scatter(X[y == 1, 0], X[y == 1, 1], color='m')
plt.contour(xx, yy, Z2, cmap='Blues')
plt.xlim(-2, 10)
plt.ylim(-4, 12)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.show()
```

Drawing the decision boundary helps to verify that we've produced a good result by cleanly splitting the two blobs from each other.

Bayes' rule provides a great deal of flexibility in terms of what the actually likelihood functions are. For example, when considering a multivariate distribution, there is no need for each dimension to be modeled by the same distribution. In fact, each dimension can be modeled by a different distribution, as long as we can multiply the $P(D|M)$ terms together.

### Classifying Signal with Different Distributions on Different Features

Let's consider the example of some noisy signals that have been segmented. We know that they come from two underlying phenomena, the cyan phenomena and the magenta phenomena, and want to classify future segments. To do this, we have three features-- the mean signal of the segment, the standard deviation, and the duration.

```python
def plot_signal(X, n):
    plt.figure(figsize=(16, 6))
    t_current = 0
    for i in range(n):
        mu, std, t = X[i]
        chunk = numpy.random.normal(mu, std, int(t))
        plt.plot(numpy.arange(t_current, t_current+t), chunk, c='cm'[i % 2])
        t_current += t

    plt.xticks(fontsize=14)
    plt.yticks(fontsize=14)
    plt.xlabel("Time (s)", fontsize=14)
    plt.ylabel("Signal", fontsize=14)
    plt.ylim(20, 40)
    plt.show()

def create_signal(n):
    X, y = [], []
    for i in range(n):
        mu = numpy.random.normal(30.0, 0.4)
        std = numpy.random.lognormal(-0.1, 0.4)
        t = int(numpy.random.exponential(50)) + 1
        X.append([mu, std, int(t)])
        y.append(0)

        mu = numpy.random.normal(30.5, 0.8)
        std = numpy.random.lognormal(-0.3, 0.6)
        t = int(numpy.random.exponential(200)) + 1
        X.append([mu, std, int(t)])
        y.append(1)

    return numpy.array(X), numpy.array(y)

X_train, y_train = create_signal(1000)
X_test, y_test = create_signal(250)
plot_signal(X_train, 20)
```

We can start by modeling each variable as Gaussians, like before, and see what accuracy we get.

```python
model = NaiveBayes.from_samples(NormalDistribution, X_train, y_train)
print "Gaussian Naive Bayes: ", (model.predict(X_test) == y_test).mean()

clf = GaussianNB().fit(X_train, y_train)
print "sklearn Gaussian Naive Bayes: ", (clf.predict(X_test) == y_test).mean()
```

    Gaussian Naive Bayes:  0.794
    sklearn Gaussian Naive Bayes:  0.794

We get identical values for sklearn and for pomegranate, which is good. However, let's take a look at the data itself to see whether a Gaussian distribution is the appropriate distribution for the data.

```python
plt.figure(figsize=(14, 4))
plt.subplot(131)
plt.title("Mean")
plt.hist(X_train[y_train == 0, 0], color='c', alpha=0.5, bins=25)
plt.hist(X_train[y_train == 1, 0], color='m', alpha=0.5, bins=25)

plt.subplot(132)
plt.title("Standard Deviation")
plt.hist(X_train[y_train == 0, 1], color='c', alpha=0.5, bins=25)
plt.hist(X_train[y_train == 1, 1], color='m', alpha=0.5, bins=25)

plt.subplot(133)
plt.title("Duration")
plt.hist(X_train[y_train == 0, 2], color='c', alpha=0.5, bins=25)
plt.hist(X_train[y_train == 1, 2], color='m', alpha=0.5, bins=25)
plt.show()
```

So, unsurprisingly (since you can see that I used non-Gaussian distributions to generate the data originally), it looks like only the mean follows a normal distribution, whereas the standard deviation seems to follow either a gamma or a log-normal distribution. We can take advantage of that by explicitly using these distributions instead of approximating them as normal distributions. pomegranate is flexible enough to allow for this, whereas sklearn currently is not.

```python
model = NaiveBayes.from_samples(NormalDistribution, X_train, y_train)
print "Gaussian Naive Bayes: ", (model.predict(X_test) == y_test).mean()

clf = GaussianNB().fit(X_train, y_train)
print "sklearn Gaussian Naive Bayes: ", (clf.predict(X_test) == y_test).mean()

model = NaiveBayes.from_samples([NormalDistribution, LogNormalDistribution, ExponentialDistribution], X_train, y_train)
print "Heterogeneous Naive Bayes: ", (model.predict(X_test) == y_test).mean()
```

    Gaussian Naive Bayes:  0.794
    sklearn Gaussian Naive Bayes:  0.794
    Heterogeneous Naive Bayes:  0.822

It looks like we're able to get a small improvement in accuracy just by using appropriate distributions for the features, without any type of data transformation or filtering. This certainly seems worthwhile if you can determine what the appropriate underlying distribution is.

Next, there's obviously the issue of speed. Let's compare the speed of the pomegranate implementation and the sklearn implementation.

```python
%timeit GaussianNB().fit(X_train, y_train)
%timeit NaiveBayes.from_samples(NormalDistribution, X_train, y_train)
%timeit NaiveBayes.from_samples([NormalDistribution, LogNormalDistribution, ExponentialDistribution], X_train, y_train)
```

    1000 loops, best of 3: 475 µs per loop
    1000 loops, best of 3: 318 µs per loop
    1000 loops, best of 3: 363 µs per loop

Looks as if on this small dataset they're all taking approximately the same time. This is pretty much expected, as the fitting step is fairly simple and both implementations use C-level numerics for the calculations. We can give a more thorough treatment of the speed comparison on larger datasets. Let's look at the average time it takes to fit a model to data of increasing dimensionality across 25 runs.

```python
pom_time, skl_time = [], []

n1, n2 = 15000, 60000,
for d in range(1, 101, 5):
    X = numpy.concatenate([numpy.random.normal(3, 2, size=(n1, d)), numpy.random.normal(7, 1, size=(n2, d))])
    y = numpy.concatenate([numpy.zeros(n1), numpy.ones(n2)])

    tic = time.time()
    for i in range(25):
        GaussianNB().fit(X, y)
    skl_time.append((time.time() - tic) / 25)

    tic = time.time()
    for i in range(25):
        NaiveBayes.from_samples(NormalDistribution, X, y)
    pom_time.append((time.time() - tic) / 25)
```

```python
plt.figure(figsize=(14, 6))
plt.plot(range(1, 101, 5), pom_time, color='c', label="pomegranate")
plt.plot(range(1, 101, 5), skl_time, color='m', label="sklearn")
plt.xticks(fontsize=14)
plt.xlabel("Number of Dimensions", fontsize=14)
plt.yticks(fontsize=14)
plt.ylabel("Time (s)")
plt.legend(fontsize=14)
plt.show()
```

It appears as if the two implementations are basically the same speed. This is unsurprising given the simplicity of the calculations, and as mentioned before, the low level implementation.

### Multivariate Gaussian Bayes Classifiers

The natural generalization of the naive Bayes classifier is to allow any multivariate function take the place of $P(D|M)$ instead of it being the product of several univariate probability distributions. One immediate difference is that now instead of creating a Gaussian model with effectively a diagonal covariance matrix, you can now create one with a full covariance matrix. Let's see an example of that at work.

```python
tilt_a = [[-2, 0.5], [5, 2]]
tilt_b = [[-1, 1.5], [3, 3]]

X = numpy.concatenate((numpy.random.normal(4, 1, size=(250, 2)).dot(tilt_a), numpy.random.normal(3, 1, size=(800, 2)).dot(tilt_b)))
y = numpy.concatenate((numpy.zeros(250), numpy.ones(800)))

model_a = NaiveBayes.from_samples(NormalDistribution, X, y)
model_b = BayesClassifier.from_samples(MultivariateGaussianDistribution, X, y)

xx, yy = numpy.meshgrid(numpy.arange(-5, 30, 0.02), numpy.arange(0, 25, 0.02))
Z1 = model_a.predict(numpy.c_[xx.ravel(), yy.ravel()]).reshape(xx.shape)
Z2 = model_b.predict(numpy.c_[xx.ravel(), yy.ravel()]).reshape(xx.shape)

plt.figure(figsize=(18, 8))
plt.subplot(121)
plt.contour(xx, yy, Z1, cmap='Blues')
plt.scatter(X[y == 0, 0], X[y == 0, 1], color='c', alpha=0.3)
plt.scatter(X[y == 1, 0], X[y == 1, 1], color='m', alpha=0.3)
plt.xlim(-5, 30)
plt.ylim(0, 25)

plt.subplot(122)
plt.contour(xx, yy, Z2, cmap='Blues')
plt.scatter(X[y == 0, 0], X[y == 0, 1], color='c', alpha=0.3)
plt.scatter(X[y == 1, 0], X[y == 1, 1], color='m', alpha=0.3)
plt.xlim(-5, 30)
plt.ylim(0, 25)
plt.show()
```

It looks like we are able to get a better boundary between the two blobs of data. The primary for this is because the data don't form spherical clusters, like you assume when you force a diagonal covariance matrix, but are tilted ellipsoids, that can be better modeled by a full covariance matrix. We can quantify this quickly by looking at performance on the training data.

```python
print "naive training accuracy: {:4.4}".format((model_a.predict(X) == y).mean())
print "bayes classifier training accuracy: {:4.4}".format((model_b.predict(X) == y).mean())
```

    naive training accuracy: 0.9219
    bayes classifier training accuracy: 0.9695

Looks like there is a significant boost. Naturally you'd want to evaluate the performance of the model on separate validation data, but for the purposes of demonstrating the effect of a full covariance matrix this should be sufficient.

### Gaussian Mixture Model Bayes Classifier

While using a full covariance matrix is certainly more complicated than using only the diagonal, there is no reason that the $P(D|M)$ has to even be a single simple distribution versus a full probabilistic model. After all, all probabilistic models, including general mixtures, hidden Markov models, and Bayesian networks, can calculate $P(D|M)$. Let's take a look at an example of using a mixture model instead of a single gaussian distribution.

```python
X = numpy.empty(shape=(0, 2))
X = numpy.concatenate((X, numpy.random.normal(4, 1, size=(200, 2)).dot([[-2, 0.5], [2, 0.5]])))
X = numpy.concatenate((X, numpy.random.normal(3, 1, size=(350, 2)).dot([[-1, 2], [1, 0.8]])))
X = numpy.concatenate((X, numpy.random.normal(7, 1, size=(700, 2)).dot([[-0.75, 0.8], [0.9, 1.5]])))
X = numpy.concatenate((X, numpy.random.normal(6, 1, size=(120, 2)).dot([[-1.5, 1.2], [0.6, 1.2]])))
y = numpy.concatenate((numpy.zeros(550), numpy.ones(820)))

model_a = BayesClassifier.from_samples(MultivariateGaussianDistribution, X, y)

gmm_a = GeneralMixtureModel.from_samples(MultivariateGaussianDistribution, 2, X[y == 0])
gmm_b = GeneralMixtureModel.from_samples(MultivariateGaussianDistribution, 2, X[y == 1])
model_b = BayesClassifier([gmm_a, gmm_b], weights=numpy.array([1-y.mean(), y.mean()]))

xx, yy = numpy.meshgrid(numpy.arange(-10, 10, 0.02), numpy.arange(0, 25, 0.02))
Z1 = model_a.predict(numpy.c_[xx.ravel(), yy.ravel()]).reshape(xx.shape)
Z2 = model_b.predict(numpy.c_[xx.ravel(), yy.ravel()]).reshape(xx.shape)
centroids1 = numpy.array([distribution.mu for distribution in model_a.distributions])
centroids2 = numpy.concatenate([[distribution.mu for distribution in component.distributions] for component in model_b.distributions])

plt.figure(figsize=(18, 8))
plt.subplot(121)
plt.contour(xx, yy, Z1, cmap='Blues')
plt.scatter(X[y == 0, 0], X[y == 0, 1], color='c', alpha=0.3)
plt.scatter(X[y == 1, 0], X[y == 1, 1], color='m', alpha=0.3)
plt.scatter(centroids1[:,0], centroids1[:,1], color='k', s=100)

plt.subplot(122)
plt.contour(xx, yy, Z2, cmap='Blues')
plt.scatter(X[y == 0, 0], X[y == 0, 1], color='c', alpha=0.3)
plt.scatter(X[y == 1, 0], X[y == 1, 1], color='m', alpha=0.3)
plt.scatter(centroids2[:,0], centroids2[:,1], color='k', s=100)
plt.show()
```

Frequently in the real world you will end up with data that looks like the ones in the above plots, not neatly falling into any single simple distribution. Using a mixture here allowed us to model the various components separately and get a more sophisticated decision boundary that doesn't seem to be extremely overfit.

If one wanted to use hidden Markov models to model sequences, all that needs to change is passing in `HiddenMarkovModel` objects instead of `GeneralMixtureModel` ones. Likewise, Bayesian networks are similarly supported and can be passed in just as easily. In fact, the most stacking one could reasonably do is to create a Bayes classifier that distinguishes mixtures of hidden Markov models with mixture emissions (GMM-HMM-GMMs) from each other!


## Solutions to auto-diagnostic test 3
Exercises By L.C. Félix-Herrán, Tec.

## 1)
Zero-order-hold sampling with sampling period $h=0.1$ and process model
$$ G(s) = \frac{s(s+0.3}{(s-0.2)(s+0.7)}. $$
### Solution
#### Step-response of the system
$$ Y(s) = \frac{G(s)}{s} = \frac{(s+0.3}{(s-0.2)(s+0.7)} = \frac{0.556}{s-0.2} + \frac{0.444}{s+0.7} $$
$$ y(t) = \mathcal{L}^{-1} \{ Y(s) \} = 0.556\mathrm{e}^{0.2t} + 0.444 \mathrm{e}^{-0.7t}.$$
#### Sample and apply z-transform
$$ Y(z) = 0.556 \mathcal{Z}\{(\mathrm{e}^{0.2h})^k\} + 0.444 \mathcal{Z}\{(\mathrm{e}^{-0.7h})^k\} = \frac{0.556 z}{z - \mathrm{e}^{0.2h}} + \frac{0.444z}{z - \mathrm{e}^{-0.7h}}. $$
#### Divide by z-transform of step input
\begin{align}
H(z) &= \frac{Y(z)}{U(z)} = \frac{z-1}{z} \left(\frac{0.556 z}{z - \mathrm{e}^{0.2h}} + \frac{0.444z}{z - \mathrm{e}^{-0.7h}} \right)\\
&= (z-1) \frac{ 0.556(z - \mathrm{e}^{-0.7h}) + 0.444(z - \mathrm{e}^{0.2h})}{(z - \mathrm{e}^{0.2h})(z - \mathrm{e}^{-0.7h})}\\
&= \frac{\big(z-1\big)\big( z - \big(0.556 \mathrm{e}^{-0.7h} - 0.444 \mathrm{e}^{0.2h}\big)\big)}{(z - \mathrm{e}^{0.2h})(z - \mathrm{e}^{-0.7h})}\\
\end{align}

## 2)
Closed-loop system with plant model $$G(s) = \frac{27}{s(s+27)}. $$
### a) Do ZOH-sampling
#### Step-response of the  system
$$Y(z) = \frac{G(s)}{s} = \frac{27}{s^2(s+27)} = -\frac{1/27}{s} + \frac{1}{s^2} + \frac{1/27}{s+27}. $$
$$ y(t) = \mathcal{L}^{-1} \{ Y(s) \} = t - \frac{1}{27} + \frac{1}{27} \mathrm{e}^{-27t}.$$
#### Sample and apply z-transform
$$ Y(z) = \frac{zh}{(z-1)^2} - \frac{1/27 z}{z-1} + \frac{1/27 z}{ z - \mathrm{e}^{-27h}}. $$
#### Divide by z-transform of step input
\begin{align}
H(z) &= \frac{Y(z)}{U(z)} = \frac{z-1}{z} \left(\frac{zh}{(z-1)^2} - \frac{1/27 z}{z-1} + \frac{1/27 z}{ z - \mathrm{e}^{-27h}} \right)\\
&= \frac{h}{z-1} - \frac{1}{27} + \frac{ 1/27(z-1)}{ z - \mathrm{e}^{-27h}}\\
&= \frac{h(z - \mathrm{e}^{-27h}) - \frac{1}{27}(z-1)(z - \mathrm{e}^{-27h}) + \frac{1}{27}(z-1)^2}{(z-1)(z - \mathrm{e}^{-27h})}\\
&= \frac{h (z - \mathrm{e}^{-27h}) - \frac{1}{27}(z^2 - (1+\mathrm{e}^{-27h})z + \mathrm{e}^{-27h}) + \frac{1}{27}(z^2 - 2z +1)}{(z-1)(z - \mathrm{e}^{-27h})}\\
&= \frac{\big(h - \frac{1}{27}(1-\mathrm{e}^{-27h})\big) z  - \big(h\mathrm{e}^{-27h} + \frac{1}{27}\mathrm{e}^{-27h} - \frac{1}{27}\big)}{(z-1)(z - \mathrm{e}^{-27h})}
\end{align}

```python
import numpy as np
h = 0.1
e27h = np.exp(-27*h)
b0 = h - 1.0/27*(1 - e27h)
b1 = -(h*e27h + 1.0/27*(e27h - 1))
print("exp(-27h) = %f, b0=%f, b1=%f" %(e27h, b0, b1))
```

    exp(-27h) = 0.067206, b0=0.065452, b1=0.027827

Pulse-transfer function for the process:
$$ H(z) = \frac{0.0065 z + 0.028}{(z-1)(z-0.067)}. $$

### b) Find difference equation corresponding to plant model
Using the shift operator $\text{q}$ we have
$$ c(k) = H(\text{q}) u(k) = \frac{0.0065\text{q} + 0.028}{(\text{q}-1)(\text{q} - 0.067)}u(k) $$
$$ (\text{q}-1)(\text{q} - 0.067) c(k) = (\frac{0.0065\text{q} + 0.028})u(k)$$
$$ (\text{q}^2 - 1.067\text{q} + 0.067) c(k) = (\frac{0.0065\text{q} + 0.028})u(k) $$
$$ c(k+2) - 1.067 c(k+1) + 0.067 c(k) = 0.0065 u(k+1) + 0.028 u(k) $$

## 3)
Given $$ G(s) = \frac{3s\mathrm{e}^{-2s}}{s+1}, $$
Determine the discretization of the system assuming that it is preceeded by a hold and proceeded by an ideal sampler for the following cases

- **a)** ZOH and sampling period $h=0.5$
- **b)** ZOH and sampling period $h=0.3$
- **c)** FOH and sampling  period $h=0.5$

### Solutions
#### b)
The delay $\tau$ in the process is 2, so we have $$ \tau = 6h + mh$$ with $h=0.3$ and $ m = 2/3$. Let $y_0(t)$ be the step-response of the system without the delay, and $y_1(t)$ be the step-response of the system for a delay of $6h$. We get
\begin{align}
y_0(t) &= \mathcal{L}^{-1} \{ \frac{3s}{s+1} \cdot \frac{1}{s} \} = 3\mathrm{e}^{-t}u_H(t)\\
y_1(t) &= y_0(t-6h)\\
y(t) &= y_1(t-mh) = y_0(t-6h-mh)
\end{align}
Applying the z-transform to the sampled signal $y_1(kh)$ gives
$$ Y_1(z) = z^{-6} Y_0(z) = z^{-6} \frac{3z}{z-\mathrm{e}^{-h}}. $$
In order to obtain the z-transform of the actual step-response $y(t)$ we need to use the modified z-transform (See the textbook Fernández y Ramírez, ch 4.6).
\begin{align}
 Y(z, m) &= \sum_{k=0}^\infty y_1(kh+mh)z^{-k} = \sum_{k=0}^\infty 3\mathrm{e}^{-kh-mh}z^{-k}\\
 &= \mathrm{e}^{-mh} \sum_{k=0}^\infty 3\mathrm{e}^{-kh}z^{-k} = \mathrm{e}^{-mh} Y_1(z)\\
 &= \mathrm{e}^{-mh} \frac{3}{z^5(z-\mathrm{e}^{-h})}
 \end{align}

```python

```

```python
import numpy as np
h = 0.3
m = 2.0/3.0
emh = np.exp(-m*h)
eh = np.exp(-h)
print('e^{-mh}=%s' %(emh))
print('e^{-h}=%s' %(eh))
3*emh
np.exp(-0.33*0.3)
```

    e^{-mh}=0.8187307530779818
    e^{-h}=0.7408182206817179

    0.9057427080235485

With numerical values
$$ Y(z,m=2/3) = 0.82 \frac{3}{z^5(z-0.74)}$$
All that is left is to divide by the z-transform of the step signal
\begin{align}
H(z) &= \frac{Y(z)}{U(z)} = \frac{z-1}{z}\frac{3\cdot 0.82}{z^5(z-0.74)}\\
&= \frac{2.46(z-1)}{z^6(z-0.74)}
\end{align}

```python
0.44444444 + 0.555555555555
```

    0.9999999955550001

```python

```


```julia
using ModelingToolkit
using DifferentialEquations
using Plots
```

# Simplified Example

```julia
@variables t x(t) RHS(t)
@parameters τ
```

\begin{equation}
\left[
\begin{array}{c}
\tau \\
\end{array}
\right]
\end{equation}

```julia
D = Differential(t) # define differentiation operator w.r.t. time
```

    (::Differential) (generic function with 2 methods)

```julia
@named fol_separate = ODESystem([
        RHS ~ (1-x)/τ,
        D(x) ~ RHS
        ])
```

\begin{align}
\frac{dx(t)}{dt} =& \mathrm{RHS}\left( t \right) \\
\mathrm{RHS}\left( t \right) =& \frac{1 - x\left( t \right)}{\tau}
\end{align}

```julia
prob = ODEProblem(
    structural_simplify(fol_separate),
    [x => 0.0],
    (0.0, 10.0),
    [τ => 3.0]
)

sol = solve(prob)

plot(sol, vars=[x, RHS])
```

# Registering Functions

What if we want to embed data as a forcing function? MTK allows us to register functions that are excluded from symbolic transformations. That way you could interpolate a time series for example.

```julia
value_vector = randn(10)
f_fun(t) = t >= 10 ? value_vector[end] : value_vector[Int(floor(t))+1]

@register_symbolic f_fun(t)
```

```julia
@variables f(t)

@named fol_external_f = ODESystem([
        f ~ f_fun(t),
        D(x) ~ (f-x)/τ
        ])
```

\begin{align}
\frac{dx(t)}{dt} =& \frac{ - x\left( t \right) + f\left( t \right)}{\tau} \\
f\left( t \right) =& \mathrm{f\_fun}\left( t \right)
\end{align}

```julia
prob = ODEProblem(structural_simplify(fol_external_f), [x=>0.0], (0.0,10.0), [τ=>0.75])
sol = solve(prob)

plot(sol, vars=[x, f])
```

```julia

```


# University of Applied Sciences Munich
## Kalman Filter Tutorial

---
(c) Lukas Köstler (lkskstlr@gmail.com)

```python
import ipywidgets as widgets
from ipywidgets import interact_manual
from IPython.display import display

import matplotlib.pyplot as plt
plt.rcParams['figure.figsize'] = (8, 4)

import numpy as np
%matplotlib notebook
```

    /home/nbuser/anaconda3_420/lib/python3.5/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.
      warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')
    /home/nbuser/anaconda3_420/lib/python3.5/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.
      warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')

## Random Variables
---
* Random variable (r.v.): $X$
* Probability Density Function (pdf): $p(X=x)$
* Independence: $X$, $Y$ r.v. $p(X=x, Y=y) = p(X=x)*p(Y=y)$
 + This means X holds no information about Y and vice versa.
 + $X$, $Y$ could be **samples** from the same distribution $p(.)$

## Random Variables
---
* Mean Value / "Expectation": $\mu$, $E[X]$

 $E[X] = \int_{-\infty}^{\infty} \, x \cdot p(X=x) \,dx$

* Variance / Expected squared deviation: $Var(X)$:

 $Var(X) = \int_{-\infty}^{\infty} \, \left(x-\mu\right)^2 \cdot p(X=x) \,dx = E[X^2] - E[X]^2$

* Standard deviation (std): $\sigma$

 $\sigma = \sqrt{Var(X)}$

## Gaussian Distribution (Normalverteilung)
---
### PDF
If $X$ is a gaussian r.v.:
\begin{equation}
p(X=x) = \frac{1}{\sqrt{2 \pi \sigma^2}} exp\left( -\frac{1}{2 \sigma^2}(x-\mu)^2\right)
\end{equation}
where $\mu \in \mathcal{R}$ is the mean and $\sigma \in [0, \infty)$ is the standard deviation.

---
Sources: The very good [matrix cookbook](https://www.math.uwaterloo.ca/~hwolkowi/matrixcookbook.pdf) (eqn. 355, watch out $\Sigma = \sigma^2$) and surely [wikipedia](https://en.wikipedia.org/wiki/Normal_distribution#Standard_normal_distribution).

```python
%%capture

# Capture captures the output

def normal_pdf(x, mu=0.0, sigma=1.0):
    return 1.0 / np.sqrt(2*np.pi*sigma**2) * np.exp(-0.5/sigma**2 * (x-mu)**2)

xx01 = np.linspace(-5,5,1000)
yy01 = normal_pdf(xx01, 0.0, 1.0)

fig01 = plt.figure();
ax01 = fig01.add_subplot(1,1,1);
line01, = ax01.plot(xx01, yy01);

ax01.set_xlim(-5,5)
ax01.set_ylim(0.0, np.amax(yy01)*1.2)

def update01(mu, sigma):
    global xx01, ax01, line01
    yy01 = normal_pdf(xx01, mu, sigma)
    line01.set_ydata(yy01)
    ax01.set_ylim(0.0, np.amax(yy01)*1.2)
    fig01.canvas.draw()

w01_mu = widgets.FloatSlider(value=0.0, min=-2.0, max=2.0, step = 0.2)
w01_sigma = widgets.FloatSlider(value=1.0, min=0.1, max=10.0, step = 0.1)
```

```python
display(interact_manual(update01, mu=w01_mu, sigma=w01_sigma));
display(fig01);
```

    A Jupyter Widget

    <function __main__.update01>

    <IPython.core.display.Javascript object>

## Gaussian Distribution (Normalverteilung)
---
### Addition
If $X$, $Y$ are **independent** gaussian r.v.s the Sum $Z = X+Y$ is again a gaussian r.v. with:
\begin{align*}
\mu_Z &= \mu_X + \mu_Y \\
\sigma^2_Z &= \sigma^2_X + \sigma^2_Y
\end{align*}

If they are dependent the variance is different.

```python
%%capture
mu02_Y = 0.0
sigma02_Y = 0.5

# assume mu_X = 0.0, sigma_X = 1.0
mu02_Z = mu02_Y
sigma02_Z = np.sqrt(1.0**2 + sigma02_Y**2)

xx02 = np.linspace(-5,5,1000)
yy02_X = normal_pdf(xx02, 0.0, 1.0)
yy02_Y = normal_pdf(xx02, mu02_Y, sigma02_Y)
yy02_Z = normal_pdf(xx02, mu02_Y, np.sqrt(1.0**2 + sigma02_Y**2))

fig02 = plt.figure();
ax02 = fig02.add_subplot(1,1,1);
line02_X, = ax02.plot(xx02, yy02_X, '--', label="p(X=x)", alpha=0.3);
line02_Y, = ax02.plot(xx02, yy02_Y, '--', label="p(Y=y)");
line02_Z, = ax02.plot(xx02, yy02_Z, label="p(Z=z)");

ax02.set_xlim(-5,5)
ax02.set_ylim(0.0, 1.2*max(np.amax(yy02_X), np.amax(yy02_Y), np.amax(yy02_Z)))
ax02.legend()

ax02.set_xlabel("$\\mu_Y={:.2f}$, $\\sigma_Y={:.2}$ $\\rightarrow$ $\\mu_Z={:.2}$, $\\sigma_Z={:.2}$".format(mu02_Y, sigma02_Y, mu02_Z, sigma02_Z))
ax02.set_title(r"$Z = X+Y$"
               "\n"
               r"$\mu_X=0$, $\sigma_X=1$")

def update02(mu_Y, sigma_Y):
    global xx02, ax02, yy02_X, yy02_Y, yy02_Z, line02_Y, line02_Z

    mu_Z = 0.0 + mu_Y
    sigma_Z = np.sqrt(1.0**2 + sigma_Y**2)

    yy02_Y = normal_pdf(xx02, mu_Y, sigma_Y)
    yy02_Z = normal_pdf(xx02, mu_Z, sigma_Z)

    line02_Y.set_ydata(yy02_Y)
    line02_Z.set_ydata(yy02_Z)

    ax02.set_ylim(0.0, 1.2*max(np.amax(yy02_X), np.amax(yy02_Y), np.amax(yy02_Z)))

    ax02.set_xlabel("$\\mu_Y={:.2f}$, $\\sigma_Y={:.2}$ $\\rightarrow$ $\\mu_Z={:.2}$, $\\sigma_Z={:.2}$".format(mu_Y, sigma_Y, mu_Z, sigma_Z))
    fig02.canvas.draw()

w02_mu_Y = widgets.FloatSlider(value=0.0, min=-2.0, max=2.0, step = 0.2)
w02_sigma_Y = widgets.FloatSlider(value=1.0, min=0.1, max=10.0, step = 0.1)
```

```python
display(interact_manual(update02, mu_Y=w02_mu_Y, sigma_Y=w02_sigma_Y));
display(fig02);
```

    A Jupyter Widget

    <function __main__.update02>

    <IPython.core.display.Javascript object>

## Gaussian Distribution (Normalverteilung)
---
### Measurement, i.e. Multiplication of the pdfs
+ **Not**: $Z = X\cdot Y$
+ $X$, $Y$ are **independent** gaussian r.v.s
+ $X$, $Y$ are measurements of the same quantity $Z$
+ Example: Two different people measure the height of the Eiffel tower
+ For $Z$: $p(Z=z) = p(X=z \land Y=z) = c \cdot p(X=z)\cdot p(Y=z)$, $c>0$ constant

## Gaussian Distribution (Normalverteilung)
---
### Measurement, i.e. Multiplication of the pdfs
+ $Z$ is gaussian again with:
\begin{align*}
\sigma_Z^2 &= \left( \frac{1}{\sigma_X^2} + \frac{1}{\sigma_Y^2} \right)^{-1} \\
\mu_Z &= \sigma_Z^2 \left( \frac{\mu_X}{\sigma_X^2} + \frac{\mu_Y}{\sigma_Y^2} \right)\\
\end{align*}

## Gaussian Distribution (Normalverteilung)
---
### Measurement, i.e. Multiplication of the pdfs
Sources: matrix cookbook (eqn. 371, watch out $\Sigma = \sigma^2$) and more easily [here](https://www.johndcook.com/blog/2012/10/29/product-of-normal-pdfs/) (some rearrangement is needed).

```python
%%capture
def mu_sigma_pdf_product(mu_X, sigma_X, mu_Y, sigma_Y):
    sigma_Z = np.sqrt( (1.0/sigma_X**2 + 1.0/sigma_Y**2)**(-1) )
    mu_Z = sigma_Z**2 * (mu_X/sigma_X**2 + mu_Y/sigma_Y**2)
    return mu_Z, sigma_Z

mu03_Y = 0.0
sigma03_Y = 0.5

# assume mu_X = 0.0, sigma_X = 1.0
mu03_Z, sigma03_Z = mu_sigma_pdf_product(0.0, 1.0, mu03_Y, sigma03_Y)

xx03 = np.linspace(-5,5,1000)
yy03_X = normal_pdf(xx03, 0.0, 1.0)
yy03_Y = normal_pdf(xx03, mu03_Y, sigma03_Y)
yy03_Z = normal_pdf(xx03, mu03_Z, sigma03_Z)

fig03 = plt.figure();
ax03 = fig03.add_subplot(1,1,1);
line03_X, = ax03.plot(xx03, yy03_X, '--', label="p(X=x)", alpha=0.3);
line03_Y, = ax03.plot(xx03, yy03_Y, '--', label="p(Y=y)");
line03_Z, = ax03.plot(xx03, yy03_Z, label="p(Z=z) = c p(X=z) p(Y=z)");
# line03_XY, = ax03.plot(xx03, yy03_X*yy03_Y) only to check manual calculation

ax03.set_xlim(-5,5)
ax03.set_ylim(0.0, 1.2*max(np.amax(yy03_X), np.amax(yy03_Y), np.amax(yy03_Z)))
ax03.legend()

ax03.set_xlabel("$\\mu_Y={:.2f}$, $\\sigma_Y={:.2}$ $\\rightarrow$ $\\mu_Z={:.2}$, $\\sigma_Z={:.2}$".format(mu03_Y, sigma03_Y, mu03_Z, sigma03_Z))
ax03.set_title(r"$\mu_X=0$, $\sigma_X=1$")

def update03(mu_Y, sigma_Y):
    global xx03, ax03, yy03_X, yy03_Y, yy03_Z, line03_Y, line03_Z

    mu_Z, sigma_Z = mu_sigma_pdf_product(0.0, 1.0, mu_Y, sigma_Y)

    yy03_Y = normal_pdf(xx03, mu_Y, sigma_Y)
    yy03_Z = normal_pdf(xx03, mu_Z, sigma_Z)

    line03_Y.set_ydata(yy03_Y)
    line03_Z.set_ydata(yy03_Z)
    # line03_XY.set_ydata(yy03_X * yy03_Y) only to check

    ax03.set_ylim(0.0, 1.2*max(np.amax(yy03_X), np.amax(yy03_Y), np.amax(yy03_Z)))

    ax03.set_xlabel("$\\mu_Y={:.2f}$, $\\sigma_Y={:.2}$ $\\rightarrow$ $\\mu_Z={:.2}$, $\\sigma_Z={:.2}$".format(mu_Y, sigma_Y, mu_Z, sigma_Z))
    fig03.canvas.draw()

w03_mu_Y = widgets.FloatSlider(value=0.0, min=-2.0, max=2.0, step = 0.2)
w03_sigma_Y = widgets.FloatSlider(value=1.0, min=0.1, max=10.0, step = 0.1)
```

```python
display(interact_manual(update03, mu_Y=w03_mu_Y, sigma_Y=w03_sigma_Y));
display(fig03);
```

    A Jupyter Widget

    <function __main__.update03>

    <IPython.core.display.Javascript object>

## Gaussian Distribution (Normalverteilung)
---
### Linear function
+ $X$ is a gaussian r.v. with $\mu$, $\sigma$
+ $a$, $b$ are real numbers
+ $Y = aX + b$ is a gaussian r.v. with:
\begin{align*}
\mu_Y &= a \mu_X + b \\
\sigma_Y &= |a|\,\sigma_X
\end{align*}

---
Sources: The very good [matrix cookbook](https://www.math.uwaterloo.ca/~hwolkowi/matrixcookbook.pdf) (eqn. 355, watch out $\Sigma = \sigma^2$)