```python
import cirq
import sympy
import numpy as np
import tensorflow as tf
import tensorflow_quantum as tfq

# visualization tools
%matplotlib inline
import matplotlib.pyplot as plt
from cirq.contrib.svg import SVGCircuit
np.random.seed(1234)
```

```python
(x_train, y_train), (x_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()

# Rescale the images from [0,255] to the [0.0,1.0] range.
x_train, x_test = x_train/255.0, x_test/255.0

print("Number of original training examples:", len(x_train))
print("Number of original test examples:", len(x_test))
```

    Number of original training examples: 60000
    Number of original test examples: 10000

```python
def filter_03(x, y):
    keep = (y == 0) | (y == 3)
    x, y = x[keep], y[keep]
    y = y == 0
    return x,y

x_train, y_train = filter_03(x_train, y_train)
x_test, y_test = filter_03(x_test, y_test)

print("Number of filtered training examples:", len(x_train))
print("Number of filtered test examples:", len(x_test))

print(y_train[0])

plt.imshow(x_train[0, :, :])
plt.colorbar()
```

```python
# Downscaling

def truncate_x(x_train, x_test, n_components=10):
  """Perform PCA on image dataset keeping the top `n_components` components."""
  n_points_train = tf.gather(tf.shape(x_train), 0)
  n_points_test = tf.gather(tf.shape(x_test), 0)

  # Flatten to 1D
  x_train = tf.reshape(x_train, [n_points_train, -1])
  x_test = tf.reshape(x_test, [n_points_test, -1])

  # Normalize.
  feature_mean = tf.reduce_mean(x_train, axis=0)
  x_train_normalized = x_train - feature_mean
  x_test_normalized = x_test - feature_mean

  # Truncate.
  e_values, e_vectors = tf.linalg.eigh(
      tf.einsum('ji,jk->ik', x_train_normalized, x_train_normalized))
  return tf.einsum('ij,jk->ik', x_train_normalized, e_vectors[:,-n_components:]), \
    tf.einsum('ij,jk->ik', x_test_normalized, e_vectors[:, -n_components:])

DATASET_DIM = 10
x_train, x_test = truncate_x(x_train, x_test, n_components=DATASET_DIM)
print(f'New datapoint dimension:', len(x_train[0]))

N_TRAIN = 1000
N_TEST = 200
x_train, x_test = x_train[:N_TRAIN], x_test[:N_TEST]
y_train, y_test = y_train[:N_TRAIN], y_test[:N_TEST]

print("New number of training examples:", len(x_train))
print("New number of test examples:", len(x_test))
```

    New datapoint dimension: 10
    New number of training examples: 1000
    New number of test examples: 200

```python
def single_qubit_wall(qubits, rotations):
  """Prepare a single qubit X,Y,Z rotation wall on `qubits`."""
  wall_circuit = cirq.Circuit()
  for i, qubit in enumerate(qubits):
    for j, gate in enumerate([cirq.X, cirq.Y, cirq.Z]):
      wall_circuit.append(gate(qubit) ** rotations[i][j])

  return wall_circuit

SVGCircuit(single_qubit_wall(
    cirq.GridQubit.rect(1,4), np.random.uniform(size=(4, 3))))
```

```python
def v_theta(qubits):
  """Prepares a circuit that generates V(\theta)."""
  ref_paulis = [
      cirq.X(q0) * cirq.X(q1) + \
      cirq.Y(q0) * cirq.Y(q1) + \
      cirq.Z(q0) * cirq.Z(q1) for q0, q1 in zip(qubits, qubits[1:])
  ]
  exp_symbols = list(sympy.symbols('ref_0:'+str(len(ref_paulis))))
  return tfq.util.exponential(ref_paulis, exp_symbols), exp_symbols

test_circuit, test_symbols = v_theta(cirq.GridQubit.rect(1, 2))
print(f'Symbols found in circuit:{test_symbols}')
SVGCircuit(test_circuit)

```

    Symbols found in circuit:[ref_0]

```python
def v_theta(qubits):
  """Prepares a circuit that generates V(\theta)."""
  ref_paulis = [
      cirq.X(q0) * cirq.X(q1) + \
      cirq.Y(q0) * cirq.Y(q1) + \
      cirq.Z(q0) * cirq.Z(q1) for q0, q1 in zip(qubits, qubits[1:])
  ]
  exp_symbols = list(sympy.symbols('ref_0:'+str(len(ref_paulis))))
  return tfq.util.exponential(ref_paulis, exp_symbols), exp_symbols

test_circuit, test_symbols = v_theta(cirq.GridQubit.rect(1, 2))
print(f'Symbols found in circuit:{test_symbols}')
SVGCircuit(test_circuit)
```

    Symbols found in circuit:[ref_0]

```python
def prepare_pqk_circuits(qubits, classical_source, n_trotter=10):
  """Prepare the pqk feature circuits around a dataset."""
  n_qubits = len(qubits)
  n_points = len(classical_source)

  # Prepare random single qubit rotation wall.
  random_rots = np.random.uniform(-2, 2, size=(n_qubits, 3))
  initial_U = single_qubit_wall(qubits, random_rots)

  # Prepare parametrized V
  V_circuit, symbols = v_theta(qubits)
  exp_circuit = cirq.Circuit(V_circuit for t in range(n_trotter))

  # Convert to `tf.Tensor`
  initial_U_tensor = tfq.convert_to_tensor([initial_U])
  initial_U_splat = tf.tile(initial_U_tensor, [n_points])

  full_circuits = tfq.layers.AddCircuit()(
      initial_U_splat, append=exp_circuit)
  # Replace placeholders in circuits with values from `classical_source`.
  return tfq.resolve_parameters(
      full_circuits, tf.convert_to_tensor([str(x) for x in symbols]),
      tf.convert_to_tensor(classical_source*(n_qubits/3)/n_trotter))

qubits = cirq.GridQubit.rect(1, DATASET_DIM + 1)
q_x_train_circuits = prepare_pqk_circuits(qubits, x_train)
q_x_test_circuits = prepare_pqk_circuits(qubits, x_test)

def get_pqk_features(qubits, data_batch):
  """Get PQK features based on above construction."""
  ops = [[cirq.X(q), cirq.Y(q), cirq.Z(q)] for q in qubits]
  ops_tensor = tf.expand_dims(tf.reshape(tfq.convert_to_tensor(ops), -1), 0)
  batch_dim = tf.gather(tf.shape(data_batch), 0)
  ops_splat = tf.tile(ops_tensor, [batch_dim, 1])
  exp_vals = tfq.layers.Expectation()(data_batch, operators=ops_splat)
  rdm = tf.reshape(exp_vals, [batch_dim, len(qubits), -1])
  return rdm

x_train_pqk = get_pqk_features(qubits, q_x_train_circuits)
x_test_pqk = get_pqk_features(qubits, q_x_test_circuits)
print('New PQK training dataset has shape:', x_train_pqk.shape)
print('New PQK testing dataset has shape:', x_test_pqk.shape)
```

    New PQK training dataset has shape: (1000, 11, 3)
    New PQK testing dataset has shape: (200, 11, 3)

```python
# 2.2 Re-labeling based on PQK features

def compute_kernel_matrix(vecs, gamma):
  """Computes d[i][j] = e^ -gamma * (vecs[i] - vecs[j]) ** 2 """
  scaled_gamma = gamma / (
      tf.cast(tf.gather(tf.shape(vecs), 1), tf.float32) * tf.math.reduce_std(vecs))
  return scaled_gamma * tf.einsum('ijk->ij',(vecs[:,None,:] - vecs) ** 2)

def get_spectrum(datapoints, gamma=1.0):
  """Compute the eigenvalues and eigenvectors of the kernel of datapoints."""
  KC_qs = compute_kernel_matrix(datapoints, gamma)
  S, V = tf.linalg.eigh(KC_qs)
  S = tf.math.abs(S)
  return S, V

S_pqk, V_pqk = get_spectrum(
    tf.reshape(tf.concat([x_train_pqk, x_test_pqk], 0), [-1, len(qubits) * 3]))

S_original, V_original = get_spectrum(
    tf.cast(tf.concat([x_train, x_test], 0), tf.float32), gamma=0.005)

print('Eigenvectors of pqk kernel matrix:', V_pqk)
print('Eigenvectors of original kernel matrix:', V_original)

def get_stilted_dataset(S, V, S_2, V_2, lambdav=1.1):
  """Prepare new labels that maximize geometric distance between kernels."""
  S_diag = tf.linalg.diag(S ** 0.5)
  S_2_diag = tf.linalg.diag(S_2 / (S_2 + lambdav) ** 2)
  scaling = S_diag @ tf.transpose(V) @ \
            V_2 @ S_2_diag @ tf.transpose(V_2) @ \
            V @ S_diag

  # Generate new lables using the largest eigenvector.
  _, vecs = tf.linalg.eig(scaling)
  new_labels = tf.math.real(
      tf.einsum('ij,j->i', tf.cast(V @ S_diag, tf.complex64), vecs[-1])).numpy()
  # Create new labels and add some small amount of noise.
  final_y = new_labels > np.median(new_labels)
  noisy_y = (final_y ^ (np.random.uniform(size=final_y.shape) > 0.95))
  return noisy_y

y_relabel = get_stilted_dataset(S_pqk, V_pqk, S_original, V_original)
y_train_new, y_test_new = y_relabel[:N_TRAIN], y_relabel[N_TRAIN:]
```

    Eigenvectors of pqk kernel matrix: tf.Tensor(
    [[ 0.01717428 -0.04324833  0.01184165 ... -0.00401873  0.02299593
      -0.0287683 ]
     [ 0.01924741 -0.02657823 -0.01638532 ... -0.0135068  -0.03280367
      -0.03066265]
     [-0.00711343  0.01911326 -0.03363318 ... -0.02050909 -0.01803411
      -0.03346212]
     ...
     [-0.0604893  -0.00763153 -0.01099809 ... -0.10575265  0.06482253
      -0.03477421]
     [-0.06658153  0.03661023 -0.00116436 ... -0.18624476  0.43948793
      -0.0352779 ]
     [-0.05538762  0.00191167  0.00148042 ... -0.          0.
      -0.03332098]], shape=(1200, 1200), dtype=float32)
    Eigenvectors of original kernel matrix: tf.Tensor(
    [[ 0.03835681  0.02834728 -0.01169795 ... -0.03899286  0.00329909
       0.03206972]
     [-0.04018165  0.00888094 -0.01388247 ...  0.00456356  0.01613913
       0.0288196 ]
     [-0.01667187  0.0135037  -0.03663851 ... -0.01058048 -0.00294398
       0.02195407]
     ...
     [-0.03015643 -0.01671633 -0.01603395 ... -0.0710168   0.10677747
       0.02365685]
     [ 0.00397754 -0.04998878 -0.00528359 ...  0.10660887  0.744923
       0.02782017]
     [-0.01665729 -0.00818617 -0.04323404 ... -0.00559646 -0.00439418
       0.01875087]], shape=(1200, 1200), dtype=float32)

```python
def create_pqk_model():
    model = tf.keras.Sequential()
    model.add(tf.keras.layers.Dense(32, activation='sigmoid', input_shape=[len(qubits) * 3,]))
    model.add(tf.keras.layers.Dense(16, activation='sigmoid'))
    model.add(tf.keras.layers.Dense(1))
    return model

pqk_model = create_pqk_model()
pqk_model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
              optimizer=tf.keras.optimizers.Adam(learning_rate=0.003),
              metrics=['accuracy'])

pqk_model.summary()

pqk_history = pqk_model.fit(tf.reshape(x_train_pqk, [N_TRAIN, -1]),
          y_train_new,
          batch_size=32,
          epochs=1000,
          verbose=0,
          validation_data=(tf.reshape(x_test_pqk, [N_TEST, -1]), y_test_new))
```

    Model: "sequential"
    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    dense (Dense)                (None, 32)                1088
    _________________________________________________________________
    dense_1 (Dense)              (None, 16)                528
    _________________________________________________________________
    dense_2 (Dense)              (None, 1)                 17
    =================================================================
    Total params: 1,633
    Trainable params: 1,633
    Non-trainable params: 0
    _________________________________________________________________

```python
# 3.2 Create a classical model

def create_fair_classical_model():
    model = tf.keras.Sequential()
    model.add(tf.keras.layers.Dense(32, activation='sigmoid', input_shape=[DATASET_DIM,]))
    model.add(tf.keras.layers.Dense(16, activation='sigmoid'))
    model.add(tf.keras.layers.Dense(1))
    return model

model = create_fair_classical_model()
model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
              optimizer=tf.keras.optimizers.Adam(learning_rate=0.03),
              metrics=['accuracy'])

model.summary()

classical_history = model.fit(x_train,
          y_train_new,
          batch_size=32,
          epochs=1000,
          verbose=0,
          validation_data=(x_test, y_test_new))
```

    Model: "sequential_1"
    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    dense_3 (Dense)              (None, 32)                352
    _________________________________________________________________
    dense_4 (Dense)              (None, 16)                528
    _________________________________________________________________
    dense_5 (Dense)              (None, 1)                 17
    =================================================================
    Total params: 897
    Trainable params: 897
    Non-trainable params: 0
    _________________________________________________________________

```python
# 3.3 Compare performance

plt.figure(figsize=(10,5))
plt.plot(classical_history.history['accuracy'], label='accuracy_classical')
plt.plot(classical_history.history['val_accuracy'], label='val_accuracy_classical')
plt.plot(pqk_history.history['accuracy'], label='accuracy_quantum')
plt.plot(pqk_history.history['val_accuracy'], label='val_accuracy_quantum')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
```


Table of Contents

1   Discrete Distributions

1.1   Joint PMF

1.2   Marginal PMF

1.3   Joint and Marginal CDF

1.4   Conditional PMF and CDF

1.5   Conditional Expectation

1.5.1   Conditional Expectation as A Function

1.5.2   Expectation for Independent Variables

2   Continuous Distributions

2.1   Joint PDF

2.1.1   An Example of Joint PDF

2.2   Marginal PDF

2.3   Joint CDF

2.4   Conditional PDF and CDF

2.5   Independence

```python
import matplotlib.pyplot as plt
import mpl_toolkits.mplot3d.art3d as art3d
import numpy as np
```

In this chapter, we will only be dealing with joint distributions, which are the most important section of the whole course. Joint distributions are used for formulating all kinds of probability model.

#  Discrete Distributions

##  Joint PMF

The joint probability mass function of two discrete random variable is defined as

$$
P_{XY}(x, y) = P(X = x, Y=y)
$$

It is convenient to define a finite range for $X$ and $Y$, $R_X = \{x_1, x_2, ...\}$ and $R_Y = \{y_1, y_2, ...\}$ and its cartesian product

$$
R_{XY}\subset R_X \times R_Y = \{(x_i, y_j)|x_i\in R_X, y_j \in R_Y\}
$$

 is the range for joint distribution.

The most common property for probability distribution is

$$
\sum_{(x_i,y_j)\in R_{XY}}P_{XY}(x_i,y_j)=1
$$

##  Marginal PMF

Let's consider a probability mass function table.

\begin{array}{|c|c|}
\hline & Y = 0 & Y = 1 & Y= 2 \\ \hline
  X = 0 & 1/6  & 1/4 & 1/8  \\ \hline
  X = 1 & 1/8  & 1/6 & 1/6  \\ \hline
\end{array}

1. Find $P(X=0, Y = 1)$. It is easy, just eyeball the table.
$$P(X=0, Y = 1) = 1/4$$

2. Find $P(X=0| Y = 1)$ and $P(Y = 1| X =0)$
$$
P(X=0| Y = 1) = \frac{1/4}{1/4+1/6}=3/5\\
P(Y = 1| X =0) = \frac{1/4}{1/6+1/4+1/8}= 6/13
$$

3. Find marginal PMFs of $X$ and $Y$

```python
from fractions import Fraction as frac
pY_0 = frac(1,6) + frac(1,8)
pY_1 = frac(1,4) + frac(1,6)
pY_2 = frac(1,8) + frac(1,6)

pX_0 = frac(1,6) + frac(1,4) + frac(1,8)
pX_1 = frac(1,8) + frac(1,6) + frac(1,6)

print('Marginal PMF of pY are {0}, {1}, {2}.'.format(pY_0,pY_1,pY_2))
print('Marginal PMF of pX are {0}, {1}.'.format(pX_0,pX_1))
```

    Marginal PMF of pY are 7/24, 5/12, 7/24.
    Marginal PMF of pX are 13/24, 11/24.

The reason we call them marginal is because they are written at the margin of the table.

\begin{array}{|c|c|}
\hline  & Y = 0 & Y = 1 & Y= 2 & P_X(x)\\ \hline
  X = 0 & 1/6  & 1/4 & 1/8 & 13/24 \\ \hline
  X = 1 & 1/8  & 1/6 & 1/6 & 11/24\\ \hline
 P_Y(y) & 7/24  & 5/12 & 7/24  &  \\ \hline
\end{array}

4. Are $X$ and $Y$ independent?

If independent, a conditional probability should equal to marginal probability, for instance

$$
P(X=0| Y = 1)= \frac{1/4}{1/4+1/6} =3/5\\
P_X(X=0)=13/24
$$

They are not equal, which means they are not independent.

The relationship of marginal PMF and conditional PMF is

$$
P(X|Y) = \frac{P(X,Y)}{P_Y(Y)}
$$

i.e.

$$
\text{Conditional PMF} = \frac{\text{Joint PMF}}{\text{Marginal PMF}}
$$

##  Joint and Marginal CDF

The joint CDF of two random variables $X$ and $Y$ is defined as

$$
F_{XY}(x,y)=P(X\leq x, Y\leq y)
$$

where $0\leq F_{XY}(x,y) \leq 1$.

For instance, the joint CDF of $P(X\leq 2, Y\leq 1)$ in range $(-6,\ 6)$ is the probability of the shaded area.

```python
x = np.linspace(-6, 1)
y = 2*np.ones(len(x))
fig, ax = plt.subplots(figsize = (8, 8))

ax.plot([1, -5], [2, 2], color = 'b')
ax.scatter(1, 2, s = 80, zorder = 3, color = 'red')
ax.plot([1, 1], [2, -5], color = 'b')
ax.axis([-5, 6, -5, 6])
ax.scatter(np.random.uniform(low = -5, high = 6, size = 50),
           np.random.uniform(low = -5, high = 6, size = 50))
ax.fill_between(x, y, -5, color = 'red', alpha =.2)
ax.text(1, 2.1, '$(1, 2)$', size = 15)
ax.grid()
```

Marginal CDF $F_X(x)$ and $F_Y(y)$ are denoted

$$
F_X(x) = P(X\leq x, Y\leq \infty)\\
F_Y(y) = P(X\leq \infty, Y\leq y)
$$

##  Conditional PMF and CDF

If $A$ is a random event, the conditional PMF of $X$ given $A$ is denoted as

$$
P_{X|A}(X = x_i) = \frac{P(X=x_i,A)}{P(A)}
$$

Consider a PMF as below

\begin{array}{|c|c|}
\hline & X = -2 & X = -1 & X = 0 & X = 1 & X = 2 \\ \hline
  Y = 2 & 0  & 0 & 1/13 & 0 & 0  \\ \hline
  Y = 1 & 0  & 1/13 & 1/13 & 1/13 & 0  \\ \hline
  Y = 0 & 1/13  & 1/13 & 1/13 & 1/13 & 1/13  \\ \hline
  Y = -1 & 0  & 1/13 & 1/13 & 1/13 & 0  \\ \hline
  Y = -2 & 0  & 0 & 1/13 & 0 & 0  \\ \hline
\end{array}

Mathematically, it is defined as $G=\{(x, y)|x, y \in \mathbb{Z},| x|+| y | \leq 2\}$.

1. Find the marginal PMFs of $X$ and $Y$.

```python
pY_2 = frac(1,13)
pY_1 = frac(1,13)*3
pY_0 = frac(1,13)*5
pY_m1 = frac(1,13)*3
pY_m2 = frac(1,13)

pX_2 = frac(1,13)
pX_1 = frac(1,13)*3
pX_0 = frac(1,13)*5
pX_m1 = frac(1,13)*3
pX_m2 = frac(1,13)

print('Marginal PMF of pY are {0}, {1}, {2}, {3}, {4}.'.format(pY_2,pY_1,pY_0,pY_m1,pY_m2))
print('Marginal PMF of pX are {0}, {1}, {2}, {3}, {4}.'.format(pX_2,pX_1,pX_0,pX_m1,pX_m2))
```

    Marginal PMF of pY are 1/13, 3/13, 5/13, 3/13, 1/13.
    Marginal PMF of pX are 1/13, 3/13, 5/13, 3/13, 1/13.

We add marginals to the table

\begin{array}{|c|c|}
\hline & X = -2 & X = -1 & X = 0 & X = 1 & X = 2 & P_Y(y) \\ \hline
  Y = 2 & 0  & 0 & 1/13 & 0 & 0  & 1/13\\ \hline
  Y = 1 & 0  & 1/13 & 1/13 & 1/13 & 0 & 3/13 \\ \hline
  Y = 0 & 1/13  & 1/13 & 1/13 & 1/13 & 1/13 & 5/13 \\ \hline
  Y = -1 & 0  & 1/13 & 1/13 & 1/13 & 0 & 3/13 \\ \hline
  Y = -2 & 0  & 0 & 1/13 & 0 & 0 &  1/13\\ \hline
  P_X(x) &1/13 &3/13 & 5/13 & 3/13 & 1/13 \\ \hline
\end{array}

2. Find the conditional PMF of $X$ given $Y = 1$, i.e. $P(X|Y=1)$

\begin{array}{|c|c|}
\hline & X = -2 & X = -1 & X = 0 & X = 1 & X = 2\\ \hline
  Y = 1 & 0  & 1/3 & 1/3 & 1/3 & 0  \\ \hline
\end{array}

It shows that given $Y=1$, $X$ is uniformly distributed over $\{-1,0,1\}$.

3. Are X and Y independent?

No, for instance $P(X=0|Y=1) \neq P_X(X = 0)$

If random event $A$ is replaced by a discrete random variable $Y$, the conditional density PMFs are defined as

$$
\begin{array}{l}
P_{X | Y}\left(x_{i} | y_{j}\right)=\frac{P_{X Y}\left(x_{i}, y_{j}\right)}{P_{Y}\left(y_{j}\right)} \\
P_{Y | X}\left(y_{j} | x_{i}\right)=\frac{P_{X Y}\left(x_{i}, y_{j}\right)}{P_{X}\left(x_{i}\right)}
\end{array}
$$

where $x_i$ and $y_j$ are realizations of $X$ and $Y$.

##  Conditional Expectation

The expectation can be conditional on a random event or a realization of random variable.

$$\begin{align}
E[X | A]&=\sum_{x_{i}\in R_{X}}x_{i} P_{X | A}\left(x_{i}|A\right) \\
E[X | Y=y_{j}]&=\sum_{x_{i} \in R_{X}} x_{i} P_{X | Y}\left(x_{i} | Y=y_{j}\right)
\end{align}$$

Use the PMF example in last section, let's try to answer questions below.

1. Find $E[X|Y = 1]$.

To calculate the conditional expectation, we must use conditional probability as weight:

$$
E[X|Y = 1]= -1 \left(\frac{1}{3}\right)+ 0  \left(\frac{1}{3}\right)+1 \left(\frac{1}{3}\right)=0
$$

2. Find $E[X|−1 < Y < 2]$.

First, calculate the conditional PMF

$$
P_{X|-1<Y<2}(x_i |-1<Y<2) = -2\frac{1/13}{8/13}-\frac{2/13}{8/13}+0\frac{3/13}{8/13}+ \frac{2/13}{8/13} + 2\frac{1/13}{8/13}=0
$$

###  Conditional Expectation as A Function

If you paid attention to the conditional expection expression

$$
E[X | Y=y_{j}]=\sum_{x_{i} \in R_{X}} x_{i} P_{X | Y}\left(x_{i} | Y=y_{j}\right)
$$

you would find that it is actually a function of $Y$.

Consider a joint PMF below

$$
\begin{array}{|c|c|}
\hline       &  X = 0 & X = 1 & P_Y(y) \\ \hline
       Y = 0 & 1/5  & 2/5   & 3/5\\ \hline
       Y = 1 & 2/5  & 0 & 2/5 \\ \hline
       P_X(x) &3/5  & 2/5 \\ \hline
\end{array}
$$

1. What are the conditional PMF $P_{X|Y}(x|0)$ and $P_{X|Y}(x|1)$?

$$
P_{X|Y}(X= 0|Y = 0) = \frac{1/5}{3/5}=1/3\\
P_{X|Y}(X= 1|Y = 0) = \frac{2/5}{3/5}=2/3\\
P_{X|Y}(X= 0|Y = 1) = \frac{2/5}{2/5}=1\\
P_{X|Y}(X= 1|Y = 1) = 0\\
$$

2. Let $Z = E[X|Y]$, find PMF of $Z$.

Remember that $Z$ is a function of $Y$. To calculate conditional expectation, we need to use conditional probability as well.

$$
E[X|Y = 0] = 0 \left(\frac{\frac{1}{5}}{\frac{1}{5}+\frac{2}{5}}\right)+1\left(\frac{\frac{2}{5}}{\frac{1}{5}+\frac{2}{5}}\right) =\frac{2}{3}\\
E[X|Y = 1]  = 0
$$

3. Find $E[Z]$, and check that if $E[Z] = E[X]$.

Because $E[X|Y]$ itself is a variable, it must have an expectation as well

$$
E[Z] = E[E[X|Y]] = P_Y(Y = 0)E[X|Y = 0]+ P_Y(Y = 1)E[X|Y = 1] = \frac{3}{5}\cdot\frac{2}{3}+\frac{2}{5}\cdot0=\frac{2}{5}
$$

Actually, $E[Z] = E[E[X|Y]] = E[X]$ must hold, it is the law of iterated expectation.

###  Expectation for Independent Variables

All the rules of expectation for independent variables are here, they are fairly straightforward, because conditioning on $Y$ does not provide any extra information

1. $E[X | Y]=E[ X]$
2. $E[g(X) | Y]=E[g(X)]$
3. $E[X Y]=E[ X] E [Y]$
4. $E[g(X) h(Y)]=E[g(X)] E[h(Y)]$

#  Continuous Distributions

##  Joint PDF

Joint PDF of $X$ and $Y$ is defined as

$$
P((X, Y) \in A)=\iint_{A} f_{X Y}(x, y) d x d y =1
$$

where $f_{XY}(x, y)$ is a non-negative function, mapping $\mathbb{R}^2$ to $\mathbb{R}$.

However, we are particularly interested in the case that $A$ is a rectangular,

$$
P(a\geq X \geq b, c\geq Y \geq d) =\int_c^d\int_a^b f_{X Y}(x, y) d x d y
$$

And within $A$, there are infinite amount of small rectangles

$$
P(a\geq X \geq a+\delta, c\geq Y \geq c+\delta )\approx f_{XY}(a,c)\delta^2
$$

###  An Example of Joint PDF

Let's consider an example other than normal distribution.

$$
f_{X Y}(x, y)=\left\{\begin{array}{ll}
x+c y^{2} & 0 \leq x \leq 1,\quad 0 \leq y \leq 1 \\
0 & \text { otherwise }
\end{array}\right.
$$

1. Find out constant $c$.

Use the property $\iint_{A} f_{X Y}(x, y) d x d y =1$

\begin{align}
\int^1_0\int^1_0(x+cy^2)dxdy &= 1\\
\int^1_0\left[\frac{x^2}{2}+cxy^2\right]^1_0dy &= 1\\
\int^1_0\left[\frac{1}{2}+cy^2\right]dy &= 1\\
\left[\frac{y}{2}+c\frac{y^3}{3}\right]^1_0&=1\\
\frac{1}{2}+\frac{c}{3}&=1\\
c&=\frac{3}{2}\\
\end{align}

2. Find out $P(0 ≤ X ≤ 1/2,0 ≤ Y ≤ 1/2)$

Plug in $c$, perform double integration

\begin{align}
\int^{1/2}_{0}\int^{1/2}_0\left(x+\frac{3}{2}y^2\right)dxdy &=
\int_0^{1/2}\left[\frac{x^2}{2}+\frac{3}{2}y^2x\right]_0^{1/2}dy \\
&=\int_0^{1/2}\left[\frac{1}{8}+\frac{3}{4}y^2\right]dy\\
&=\left[\frac{1}{8}+\frac{y^3}{4}\right]_0^{1/2}\\
&=\frac{3}{32}
\end{align}

The joint distribution is depicted as below, the volume between the curved plane and $xy$ plane is $1$.

```python
x, y = np.linspace(0, 1), np.linspace(0, 1)
X, Y = np.meshgrid(x, y)
Z = X + 3/2*Y**2

fig = plt.figure(figsize = (8, 8))
ax = fig.gca(projection='3d')
ax.plot_surface(X, Y, Z, cmap = 'coolwarm')
ax.contourf(X, Y, Z, zdir='z', offset=0, cmap='coolwarm')
plt.show()
```

##  Marginal PDF

Maringal PDF of $X$ and $Y$ are

\begin{equation}
f_{X}(x)=\int_{-\infty}^{\infty} f_{X Y}(x, y) d y,\quad \text { for all } x \\
f_{Y}(y)=\int_{-\infty}^{\infty} f_{X Y}(x, y) d x,\quad \text { for all } y
\end{equation}

Let's use the same example as in last section to find out $f_X(x)$ and $f_Y(y)$.

$$
f_{X}(x)=\int_{0}^{1}\left(x+\frac{3}{2}y^2\right)  d y =x+\frac{1}{2}\\
f_{Y}(y)=\int_{0}^{1}\left(x+\frac{3}{2}y^2\right)  d x =\frac{3}{2} y^{2}+\frac{1}{2}
$$

##  Joint CDF

Joint CDF and joint PDF has relationship as follows:

$$
F_{X Y}(x, y)=\int_{-\infty}^{y} \int_{-\infty}^{x} f_{X Y}(u, v) d u d v \\
f_{X Y}(x, y)=\frac{\partial^{2}}{\partial x \partial y} F_{X Y}(x, y)
$$

The same PDF as above, find the CDF.

$$
f_{X Y}(x, y)=\left\{\begin{array}{ll}
x+\frac{3}{2} y^{2} & 0 \leq x \leq 1,\quad 0 \leq y \leq 1 \\
0 & \text { otherwise }
\end{array}\right.
$$

\begin{align}
F_{XY}(x,y)=\int_{0}^{y} \int_{0}^{x} f_{X Y}(u, v) d u d v&=\int_{0}^{y} \int_{0}^{x} \left(u+\frac{3}{2}v^2\right) d u d v\\
& = \int_0^y\left[\frac{u^2}{2}+\frac{3}{2}v^2u\right]^x_0dv\\
& =  \int_0^y\left(\frac{x^2}{2}+\frac{3}{2}v^2x\right)dv\\
& = \left[\frac{x^2}{2}v+\frac{3}{2}\frac{v^3}{3}x\right]^y_0\\
& =\frac{x^2y}{2}+\frac{y^3x}{2}
\end{align}

##  Conditional PDF and CDF

Consider the conditional PDF of $X$ given that $X\in A$

\begin{align}
P(x\leq X \leq x+\delta|X \in A)\approx f_{X|X\in A}(x)\cdot \delta &= \frac{P(x\leq X \leq x+\delta,X \in A)}{P(A)}\\
&=\frac{P(x\leq X \leq x+\delta)}{P(A)}\\
&\approx\frac{f_X(x)\delta}{P(A)}
\end{align}

We have shown that

$$
f_{X|X\in A}(x) = \frac{f_X(x)}{P(A)}
$$

You can imagin $P(A)$ as a scaling factor that normalize the conditional PDF into an area of $1$.

For two jointly continuous random variables $X$ and $Y$, we can
define the following conditional concepts:
1. The conditional PDF of $X$ given $Y=y$ :
$$
f_{X | Y}(x | y)=\frac{f_{X Y}(x, y)}{f_{Y}(y)}
$$
2. The conditional probability that $X \in A$ given $Y=y:$
$$
P(X \in A | Y=y)=\int_{A} f_{X | Y}(x | y) d x
$$
3. The conditional CDF of $X$ given $Y=y$
$$
F_{X | Y}(x | y)=P(X \leq x | Y=y)=\int_{-\infty}^{x} f_{X | Y}(x | y) d x
$$

The intuition of the first expression, i.e. conditional PDF is

$$
P(x\leq X \leq x+\delta| y\leq Y\leq y+\epsilon)\approx \frac{f_{XY}(xy)\delta\epsilon}{f_Y(y)\epsilon}=f_{X|Y}(x|y)\delta
$$

Conditional probability must satisfy the basic rule of probability as well,

$$
\int_{-\infty}^\infty f_{X|Y}(x|y)dx = 1
$$

because

$$
\frac{\int_{-\infty}^\infty f_{XY}(xy)dx}{f_Y(y)} = 1
$$

Rearrange the conditional PDF, we obtain the multiplication rule

$$
f_{XY}(xy)=f_{X|Y}(x|y)f_Y(y)
$$

##  Independence

If continuous variables $X$ and $Y$ are independent, then knowing either of them does not provide information for the other. That is

$$
f_{X|Y}(x|y) = f_X(x),\qquad \text{or} \qquad f_{Y|X}(y|x) = f_Y(y)
$$

Thus the multiplication rule for independent distribution

$$
f_{XY}(xy)=f_X(x)f_Y(y)
$$

Other rules derived from this are

\begin{align}
E[XY]&= E[X]E[Y]\\
\text{Var}(X+Y)&=\text{Var}(X)+\text{Var}(Y)\\
E[g(X)h(Y)]&=E[g(X)]E[h(Y)]
\end{align}


```python
import colour
from colour.plotting import *

import itertools

from sympy.ntheory import factorint

class Color():
    def __init__(self,rgb=None,xyy=None,xyz=None):

        if xyz:
            self.XYZ = xyz
            self.RGB = colour.XYZ_to_sRGB(xyz)
        elif xyy:
            self.XYZ = colour.xyY_to_XYZ(xyy)
            self.RGB = colour.XYZ_to_sRGB(self.XYZ)
        elif rgb:
            self.XYZ = colour.sRGB_to_XYZ(rgb)
            self.RGB = rgb

    def __repr__(self):
        return '({:3.0f}, {:3.0f}, {:3.0f})'.format(*[n*255 for n in self.RGB])

custom_colors = [
]

highs = [.6,.8,1]
lows = [0,.2,.4]

total = [0,.5,1]

for h, l in itertools.product(highs,lows):

    for i in range(2**3):
        #l = 0.1
        #h = 0.9
        r = h if ((i & 2*2) >> 2) else l
        g = h if ((i & 2) >> 1) else l
        b = h if i & 1 else l
        custom_colors.append(Color(rgb=[r,g,b]))
```

```python
custom_swatches = []

settings = {
        'width': 100,
        'height': 100,
        'spacing': 20,
        'columns': 8,
        'text_parameters': {
            'size': 8
        },
        'background_colour': (.5,.5,.5)
    }

for c in custom_colors:
    custom_swatches.append(ColourSwatch(RGB=c.RGB))

plot_multi_colour_swatches(custom_swatches, **settings)
```

```python
import math

def ez_palette(swatches, filename=None, columns=None, background=(0,0,0), border=0, aspect=1):
    if not columns:
        columns = round(math.sqrt(len(swatches)*aspect))

    _figure, axes = artist()
    settings = {
        'axes': axes,
        'width': 1,
        'height': 1,
        'spacing': .3,
        'columns': columns,
        'text_parameters': {
            'size': 0
        },
        'background_colour': background,
        'standalone': True
    }
    axes.get_xaxis().set_visible(False)
    axes.get_yaxis().set_visible(False)
    spines = [i[1] for i in axes.spines.items()]
    for s in spines:
        s.set_linewidth(border)
        s.set_edgecolor(background)

    result = plot_multi_colour_swatches(swatches, **settings)

    if filename:
        result[0].savefig(filename, bbox_inches='tight', pad_inches=0.0)

    #print(len(swatches),factorint(len(swatches)))
```

```python
hsv_swatches = []

n = 6
values = [x/n for x in range(n+1)]

for hsv in itertools.product(values, values, values):
    if hsv[2] == 0 or hsv[0] == 1:
        continue # exclude black; exclude phase rollover
    rgb = colour.HSV_to_RGB(hsv)
    ingamut = all(1 >= v >= 0 for v in rgb)
    if ingamut:
        hsv_swatches.append(ColourSwatch(RGB=rgb))

ez_palette(hsv_swatches, filename='hsv.png', columns=21, background = (0,)*3, border=10)
```

```python
xyz_swatches = []

n = 5
values = [x/n for x in range(n+1)]

for xyz in itertools.product(values, values, values):
    rgb = colour.XYZ_to_sRGB(xyz)
    ingamut = all(1 >= v >= 0 for v in rgb)
    if ingamut:
        xyz_swatches.append(ColourSwatch(RGB=rgb))

def gray_swatches(qty):
    # in rgb space
    return[ColourSwatch(RGB=[v/(qty-1)]*3) for v in range(qty)]

xyz_swatches = gray_swatches(3) + xyz_swatches[1:]

ez_palette(xyz_swatches, filename='xyz{}.png'.format(n),columns=6)

def swatches_to_csv(swatches, filename):
    outstr = 'r,g,b'
    for s in swatches:
        rgb = s.RGB
        outstr += '\n{},{},{}'.format(*rgb)
    with open(filename, 'w') as outfile:
        outfile.write(outstr)

swatches_to_csv(xyz_swatches, 'xyz.swatch')
```

```python
rgb_swatches = [ColourSwatch(RGB=(0,0,0))]

n = 4
values = [x/n for x in range(n+1)]

for rgb in itertools.product(values, values, values):
    if all(v==0 for v in rgb):
        continue # exclude black
    rgb_swatches.append(ColourSwatch(RGB=rgb))

ez_palette(rgb_swatches, filename='rgb.png', columns=25, background = (0,)*3)
```

```python
combo_swatches = rgb_swatches#hsv_swatches + xyz_swatches + rgb_swatches

combo_rgb = [swatch.RGB for swatch in combo_swatches]
combo_xyz = [colour.sRGB_to_XYZ(rgb) for rgb in combo_rgb]

start_qty = len(combo_xyz)
target_qty = 25

print('Starting with {} and ending with {}: {} iterations'.format(start_qty, target_qty, start_qty-target_qty))

def dist(a, b):
    #Pythagorean/Ecludian distance (in XYZ space) squared
    return sum([(an-bn)**2 for an, bn in zip(a,b)])

removed_colors = []

for i in range(start_qty - target_qty):
    # remove colors which are most similar to their neighbors
    ds = [2]*len(combo_xyz)
    for ai, bi in itertools.combinations(range(len(combo_xyz)), 2):
        #sim = 1/dist(a,b)
        #ds[ai] += sim
        #ds[bi] += sim
        sim = dist(combo_xyz[ai], combo_xyz[bi])
        ds[ai] = min(sim, ds[ai])
        ds[bi] = min(sim, ds[bi])

    maxi = ds.index(min(ds))
    removed_colors.append(combo_xyz.pop(maxi))
    print(len(combo_xyz), end='  ')

pruned_swatches = []
for xyz in combo_xyz:
    rgb = colour.XYZ_to_sRGB(xyz)
    rgb = [1 if v>1 else v for v in rgb] # clip in-gamut to correct for floating point errors
    pruned_swatches.append(ColourSwatch(RGB=rgb))

ez_palette(pruned_swatches, filename='pruned.png', columns=12)
```

```python
removed_swatches = []
for xyz in removed_colors:
    rgb = colour.XYZ_to_sRGB(xyz)
    rgb = [1 if v>1 else v if v>0 else 0 for v in rgb] # clip in-gamut to correct for floating point errors
    removed_swatches.append(ColourSwatch(RGB=rgb))

ez_palette(removed_swatches, filename='removed.png',)
```

```python
import numpy as np

def average_sRGB(rgbs):
    XYZs = []
    for rgb in rgbs:
        XYZs.append(colour.sRGB_to_XYZ(rgb))
    XYZs = np.array(XYZs)
    avg_XYZ = np.average(XYZs, axis=0)
    return colour.XYZ_to_sRGB(avg_XYZ)

def invavg_sRGB(a, b):
    # returns c such that (a+c)/2 = b
    # (b is the average of a and c)
    A = colour.sRGB_to_XYZ(a)
    B = colour.sRGB_to_XYZ(b)
    C = 2*B - A
    c = colour.XYZ_to_sRGB(C)
    print(c)
    ez_palette([ColourSwatch(RGB=v) for v in [a,b,c]], columns=3)
    return c

invavg_sRGB([.5,.5,.5],[.4,.4,.4])
```

```python
scalergbs = [
    [255, 0, 210],
    [242, 122, 227]
]

rgbs = []
for rgb in scalergbs:
    rgbs.append([v/255 for v in rgb])

try:
    print([round(v*255) for v in invavg_sRGB(*rgbs)])
except ValueError:
    print("Mix impossible!")
```

```python
from scipy.optimize import minimize, Bounds

def clip_vector(vec):
    return [1 if v>1 else v if v>0 else 0 for v in vec]

cv = clip_vector

def vec_to_hex(vec):
    # input domain [0,1]
    outstr = ''
    for v in vec:
        outstr += hex(int(v*255))[2:].zfill(2)

    return outstr

def create_pair(target_rgb8, algo = 1, verbose = 0):
    if algo == -1: # do all the algos
        return [create_pair(target_rgb8, algo=i, verbose=verbose) for i in range(2)]

    t = [v/255 for v in target_rgb8]

    T = colour.sRGB_to_XYZ(t)

    if algo == 0:
        # scale all components uniformly. invert and reduce scaling each iteration.
        scalar = 10
        iter_scalar = -1 - 1e-2
        iters = 0 # -1 when done
        while iters != -1 and iters < 1e3:
            A = cv([v*scalar for v in T])
            a = colour.XYZ_to_sRGB(A)

            b = colour.XYZ_to_sRGB(2*T - A)

            if np.array_equal(b, cv(b)) and np.array_equal(a, cv(a)):

                ez_palette([ColourSwatch(RGB=v) for v in [a,t,b]], columns=3)

                return vec_to_hex(a), vec_to_hex(b)

            iters += 1
            scalar /= iter_scalar

        return -1

    if algo == 1:
        # find most extreme color using optimization techniques

        def loss(A):

            a = colour.XYZ_to_sRGB(A)
            B = 2*T - A
            b = colour.XYZ_to_sRGB(B)

            if np.array_equal(a, cv(a)) and np.array_equal(b, cv(b)):
                # valid colors!
                d = np.sqrt(sum([(an-bn)**2 for an, bn in zip(T, A)]))
                return -d

            else:
                return 100

        res = minimize(loss, T, method='Nelder-Mead',
                       options={'disp':verbose, 'maxfev': 3e3})

        A = res.x
        a = colour.XYZ_to_sRGB(A)
        b = colour.XYZ_to_sRGB(2*T - A)

        if np.array_equal(b, cv(b)) and np.array_equal(a, cv(a)):

                ez_palette([ColourSwatch(RGB=v) for v in [a,t,b]], columns=3)

                return vec_to_hex(a), vec_to_hex(b)
        else:
            print("Optimization failure")
            return -1

```

```python
create_pair((57,125,204), algo=-1)
```

```python
create_pair((166,167,200), algo=-1)
```

```python
create_pair((247,124,231), 1)
```

```python
create_pair((206,247,81), 1)
```

```python
create_pair((239, 227, 248), 0)
```

```python
create_pair((128,)*3, -1)
```

```python
create_pair((230,13,170), -1)
```

```python
create_pair((145,169,229), -1)
```

```python
create_pair((246,196,98), -1)
```

```python
create_pair((190,248,147), -1)
```

```python

```


# Le-Net 1 based architecture

We start with 41X41 (I) after first convolution (9x9)we have 33X33 (L1). The next pooling layer reduces dimension with 3 to an output image of 11X11 with 4x4 pooling kernels (L2). Then we apply different types of convolution 4x4 kernels on the L2 layer resulting in 8x8 (L3) . Then followed by pooling 2X2 resulting in 4x4 output map (L4). So we have 16 connection for each element in layer L4 (which depend on the amount of different Covolutions in L3)

\begin{equation}
f(x)=\frac{1}{1+e^{-x}} \\
F_{k}= f( \sum_{i} \mathbf{W^{k}_{i} \cdot y_{i}}-b_{k})
\end{equation}

\begin{equation}
E=\sum_{k} \frac{1}{2}|t_k-F_{k}|^{2} \\
\Delta W_{ij}= - \eta \frac{dE}{d W_{ij}}
\end{equation}

\begin{equation}
\Delta W_{ij}= \sum_{k} - \eta \frac{dE}{d F_{k}} \frac{dF_{k}}{dx_{k}} \frac{dx_{k}}{dW_{ij}}=\sum_{k} \eta (t_{k}-F_{k})\frac{e^{-x_{k}}}{(1+e^{-x_{k}})^{2}} \frac{dx_{k}}{dW_{ij}} \\
= \eta (t_{k}-F_{k})\frac{e^{-x_{k}}}{(1+e^{-x_{k}})^{2}} y_{ij}
\end{equation}

\begin{equation}
\Delta b_{k}= - \eta \frac{dE}{d F_{k}} \frac{dF_{k}}{dx_{k}} \frac{dx_{k}}{b_{k}}=\eta (t_{k}-F_{k})\frac{e^{-x_{k}}}{(1+e^{-x_{k}})^{2}} \cdot-1
\end{equation}

Since $\frac{e^{-x_{k}}}{(1+e^{-x_{k}})^{2}}$ is always positive we can neglect this term in our programme

\begin{equation}
x_{k}=\sum_{ij} W^{k}[i,j] \; y^{4rb}[i,j] - b_{k}
\end{equation}

\begin{equation}
y^{4rb}[i,j]= \sum_{u,v} W^{3rb}[u,v] \; y^{3rb} [2i+u,2j+v]
\end{equation}

\begin{equation}
y^{3rb} [2i+u,2j+v]= f\left (x^{3rb}[2i+u,2j+v] \right)
\end{equation}

\begin{equation}
x^{3rb}[2i+u,2j+v]=\sum_{nm} W^{2rb}[n,m] \; y^{2rb}[n+(2i+u),m+(2j+v)] -b^{3rb}[2i+u,2j+v]
\end{equation}

\begin{equation}
\begin{split}
\Delta W^{2rb}[n,m] =\sum_{k} - \eta  \frac{dE}{dF_{k}}
\frac{dF_{k}}{dx_{k}}
\sum_{ij} \frac{dx_{k}}{dy^{4rb}[i,j]}
 \sum_{uv}\frac{dy^{4rb}[i,j]}{d y^{3rb} [2i+u,2j+v]}
\frac{d y^{3rb} [2i+u,2j+v]}{d x^{3rb}[2i+u,2j+v]}
\sum_{nm}\frac{d x^{3rb}[2i+u,2j+v]}{d W^{2rb}[n,m]}
\end{split}
\end{equation}

\begin{equation}
\begin{split}
\Delta b^{3rb}[2i+u,2j+v] =\sum_{k} - \eta  \frac{dE}{dF_{k}}
\frac{dF_{k}}{dx_{k}}
\sum_{ij} \frac{dx_{k}}{dy^{4rb}[i,j]}
 \sum_{uv}\frac{dy^{4rb}[i,j]}{d y^{3rb} [2i+u,2j+v]}
\frac{d y^{3rb} [2i+u,2j+v]}{d x^{3rb}[2i+u,2j+v]}
\frac{d x^{3rb}[2i+u,2j+v]}{d b^{3rb}[2i+u,2j+v]}
\end{split}
\end{equation}

\begin{equation}
    \frac{dx_{k}}{dy^{4rb}[i,j]} = W^{4rbk}[i,j]\\
\end{equation}

\begin{equation}
    \frac{dy^{4rb}[i,j]}{d y^{3rb} [2i+u,2j+v]} = W^{3rb}[u,v] \\
 \end{equation}

\begin{equation}
\frac{d y^{3rb} [2i+u,2j+v]}{d x^{3rb}[2i+u,2j+v]}=\frac{e^{-x^{3rb}[2i+u,2j+v]}}{(1+e^{-x^{3rb}[2i+u,2j+v]})^2}
\end{equation}
This term is first not included since it is always positive. If the training will not converge it might be possible to include this term

 \begin{equation}
    \frac{d y^{3rb} [2i+u,2j+v]}{d W^{2rb}[n,m]}= y^{2rb} [n+(2i+u),m+(2j+v)] \\
\end{equation}

\begin{equation}
\frac{d x^{3rb}[2i+u,2j+v]}{d b^{3rb}[2i+u,2j+v]}=-1
\end{equation}

```python
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
from numpy import linalg as lin
import scipy.signal as sig
from PIL import Image
import glob
import matplotlib.cm as cm
import itertools
```

```python
########### Load Input ############################################################################################################################
# In this script I used the brightness to determine structures, instead of one RGB color:
# this is determined by: 0.2126*R + 0.7152*G + 0.0722*B
# Source: https://en.wikipedia.org/wiki/Relative_luminance

patchSize=41 # patchsize this must be 48 since our network can only handle this value

# Open forest
Amount_data= len(glob.glob('Forest/F*'))
dataPatchedF=[]
for k in range (0, Amount_data):
    name="Forest/F%d.png" % (k+1)
    img = Image.open(name)
    data=img.convert('RGB')
    data= np.asarray( data, dtype="int32" )
    data=0.2126*data[:,:,0]+0.7152*data[:,:,1]+0.0722*data[:,:,2]
    Yamount=data.shape[0]/patchSize # Counts how many times the windowsize fits in the picture
    Xamount=data.shape[1]/patchSize # Counts how many times the windowsize fits in the picture
    dataPatchedF.append(np.array([[data[j*patchSize:(j+1)*patchSize,i*patchSize:(i+1)*patchSize] for i in range(0,Xamount)] for j in range(0,Yamount)]))

# Open city
Amount_data= len(glob.glob('City/C*'))
dataPatchedC=[]
for k in range (0, Amount_data):
    name="City/C%d.png" % (k+1)
    img = Image.open(name)
    data=img.convert('RGB')
    data = np.asarray( data, dtype="int32" )
    data=0.2126*data[:,:,0]+0.7152*data[:,:,1]+0.0722*data[:,:,2]
    Yamount=data.shape[0]/patchSize # Counts how many times the windowsize fits in the picture
    Xamount=data.shape[1]/patchSize # Counts how many times the windowsize fits in the picture
    dataPatchedC.append(np.array([[data[j*patchSize:(j+1)*patchSize,i*patchSize:(i+1)*patchSize] for i in range(0,Xamount)] for j in range(0,Yamount)]))

# Open water
dataPatchedW=[]
Amount_data= len(glob.glob('Water/W*'))
for k in range (0, Amount_data):
    name="Water/W%d.png" % (k+1)
    img = Image.open(name)
    data=img.convert('RGB')
    data = np.asarray( data, dtype="int32" )
    data=0.2126*data[:,:,0]+0.7152*data[:,:,1]+0.0722*data[:,:,2]
    Yamount=data.shape[0]/patchSize # Counts how many times the windowsize fits in the picture
    Xamount=data.shape[1]/patchSize # Counts how many times the windowsize fits in the picture
    dataPatchedW.append(np.array([[data[j*patchSize:(j+1)*patchSize,i*patchSize:(i+1)*patchSize] for i in range(0,Xamount)] for j in range(0,Yamount)]))

```

```python
########### Functions ############################################################################################################################

# Define Activitation functions, pooling and convolution functions (the rules)

def Sigmoid(x):
    return (1/(1+np.exp(-x)))

def TanH(x):
    return (1-np.exp(-x))/(1+np.exp(-x))

def Pool(I,W):
    PoolImg=np.zeros((len(I)/len(W),len(I)/len(W))) # W must fit an integer times into I.
    for i in range(0,len(PoolImg)):
        for j in range(0,len(PoolImg)):
            SelAr=I[i*len(W):(i+1)*len(W),j*len(W):(j+1)*len(W)]
            PoolImg[i,j]=np.inner(SelAr.flatten(),W.flatten()) # Now this is just an inner product since we have vectors
    return PoolImg

# To automatically make Gaussian kernels
def makeGaussian(size, fwhm = 3, center=None):
    x = np.arange(0, size, 1, float)
    y = x[:,np.newaxis]

    if center is None:
        x0 = y0 = size // 2
    else:
        x0 = center[0]
        y0 = center[1]

    return np.exp(-4*np.log(2) * ((x-x0)**2 + (y-y0)**2) / fwhm**2)

# To automatically define pooling nodes
def Pool_node(N):
    s=(N,N)
    a=float(N)*float(N)
    return (1.0/a)*np.ones(s)

```

```python
#################### Define pooling layers ###########################################################################
P12=Pool_node(3)*(1.0/100.0) #factor 1000 added to lower values more
P34=Pool_node(2)

#################### Define Convolution layers #######################################################################

######### First C layer #########
C1=[]

## First Kernel

# Inspiration: http://en.wikipedia.org/wiki/Sobel_operator
# http://stackoverflow.com/questions/9567882/sobel-filter-kernel-of-large-size

Kernel=np.array([[4,3,2,1,0,-1,-2,-3,-4],
                 [5,4,3,2,0,-2,-3,-4,-5],
                 [6,5,4,3,0,-3,-4,-5,-6],
                 [7,6,5,4,0,-4,-5,-6,-7],
                 [8,7,6,5,0,-5,-6,-7,-8],
                 [7,6,5,4,0,-4,-5,-6,-7],
                 [6,5,4,3,0,-3,-4,-5,-6],
                 [5,4,3,2,0,-2,-3,-4,-5],
                 [4,3,2,1,0,-1,-2,-3,-4]])

C1.append(Kernel)

## Second Kernel
# Kernel=np.matrix.transpose(Kernel)
Kernel= [[np.sin(x) for x in np.linspace(0,15*np.pi,9)] for i in range(0,9)]
C1.append(Kernel)

## Third Kernel
Kernel=makeGaussian(9,5)
Kernel=(1/np.sum(Kernel))*Kernel
C1.append(Kernel)

## Fourth Kernel
Kernel=np.zeros((9,9))
Kernel[4:5,:]=1
Kernel[5:9,:]=2
Kernel=(1/np.sum(Kernel))*Kernel
C1.append(Kernel)

## Fifth kernel
Kernel=np.zeros((9,9))
for i in range(0,9):
    for j in range(0,9):
        if(i==j):Kernel[i,j]=1
        if(i>j): Kernel[i,j]=2
Kernel=(1/np.sum(Kernel))*Kernel
C1.append(Kernel)

#Sixth Kernel
Kernel=makeGaussian(9,2)
Kernel=(1/np.sum(Kernel))*Kernel
C1.append(Kernel)

# ## Seventh Kernel
# Kernel= np.transpose([[np.sin(x) for x in np.linspace(0,5*np.pi,9)] for i in range(0,9)])+[[np.sin(x) for x in np.linspace(0,5*np.pi,9)] for i in range(0,9)]
# Kernel=(1/np.sum(Kernel))*Kernel
# C1.append(Kernel)

# ## Eigth Kernel
# Kernel= np.transpose([[np.sin(x) for x in np.linspace(0,5*np.pi,9)] for i in range(0,9)])
# Kernel=(1/np.sum(Kernel))*Kernel
# C1.append(Kernel)

# ## Ninth Kernel
# Kernel= np.fliplr(C1[4])
# Kernel=(1/np.sum(Kernel))*Kernel
# C1.append(Kernel)

######### Initialize output weights and biases #########

# Initialisation, since this layer should be trained!
C2=[] # second convolutio weigths
H3_bias=[]
W=[] # end-weight from output to classifier-neurons
# Define the number of branches in one row
N_branches=2
ClassAmount=3 # Forest, City, Water

for i in range (0, len(C1)):
    C2.append(np.random.rand(N_branches,4,4)*(1.0/100.0))
    H3_bias.append(np.random.rand(N_branches,8,8)*(1.0/10.0))
    W.append(np.random.rand(N_branches,ClassAmount,4,4)*(1.0/100.0)) # W[r][b,k,:,:]

bias=np.zeros((ClassAmount))# bias on the three classes
```

```python
N_plts=len(C1)
for i in range(0,N_plts):
    plt.subplot(4,3,i+1)
    plt.imshow(C1[i])
```

```python
#################### Start training  #######################################################################

# Determines which patch is chosen when
N_F=len(dataPatchedF)
Patches_F=[]
j=0;
for i in range(0,N_F):
    N_ff=dataPatchedF[i].shape[0]*dataPatchedF[i].shape[1]
    Patches_F.append(np.reshape(dataPatchedF[i], (N_ff, patchSize, patchSize)))
    j=j+N_ff
N_F=j

N_C=len(dataPatchedC)
Patches_C=[]
j=0;
for i in range(0,N_C):
    N_cc=dataPatchedC[i].shape[0]*dataPatchedC[i].shape[1]
    j=j+N_cc
    Patches_C.append(np.reshape(dataPatchedC[i], (N_cc, patchSize, patchSize)))
N_C=j

N_W=len(dataPatchedW)
Patches_W=[]
j=0;
for i in range(0,N_W):
    N_ww=dataPatchedW[i].shape[0]*dataPatchedW[i].shape[1]
    Patches_W.append(np.reshape(dataPatchedW[i], (N_ww, patchSize, patchSize)))
    j=j+N_ww
N_W=j

N_total=N_F+N_C+N_W
Sequence = np.arange(N_total)
Sequence = np.random.permutation(Sequence)

# Make patches tractable
Patches_F = list(itertools.chain.from_iterable(Patches_F))
Patches_F=np.asarray(Patches_F)

Patches_C = list(itertools.chain.from_iterable(Patches_C))
Patches_C=np.asarray(Patches_C)

Patches_W = list(itertools.chain.from_iterable(Patches_W))
Patches_W=np.asarray(Patches_W)

```

```python
len(Sequence)
```

    15288

```python
from itertools import product
###### Chooses patch and defines label #####
for PP in range(0,len(Sequence)):
       # 1 has to be changed to the number of training data
    SS=Sequence[PP]
    if SS<N_F:
        Class_label=np.array([1,0,0])
        inputPatch=Patches_F[SS]
    elif(SS>=N_F) and (SS<(N_F+N_C)):
        Class_label=np.array([0,1,0])
        inputPatch=Patches_C[SS-N_F]
    else:
        Class_label=np.array([0,0,1])
        inputPatch=Patches_W[SS-N_F-N_C]

    ### Layer 1 ###
    H1=[]
    H2=[]
    H3=[[np.zeros((8,8)) for b in range(0,N_branches)] for r in range(0,len(C1))]
    H4=[[np.zeros((8,8)) for b in range(0,N_branches)] for r in range(0,len(C1))]
    x=np.zeros(ClassAmount)
    f=np.zeros(ClassAmount)
    #learning rates
    n_bias=1*10**-3
    n_W=2*10**-6
    n_C2=1*10**-7
    n_H3_bias=1*10**-3

    for r in range (0, len(C1)):
        H1.append(sig.convolve(inputPatch, C1[r], 'valid'))
        H2.append(Pool(H1[r], P12))
        for b in range(0,N_branches):
            H3[r][b]=Sigmoid(sig.convolve(H2[r], C2[r][b],'valid')-H3_bias[r][b])
            H4[r][b]=Pool(H3[r][b],P34)

    for k in range(0,ClassAmount):
        for r in range (0, len(C1)):
            for b in range(0,N_branches):
                x[k]=x[k]+np.inner(H4[r][b].flatten(),W[r][b,k].flatten())
        f[k]=Sigmoid(x[k]-bias[k])

    ###### Back-propagation #####
    for k in range(0, ClassAmount):
        bias[k]=bias[k]-n_bias*(Class_label[k]-f[k])
        for r in range (0, len(C1)):
            for b in range(0,N_branches):
                W[r][b,k]=W[r][b,k]+n_W*(Class_label[k]-f[k])*H4[r][b]
                for i, j, u, v, n, m in product(range(0,4), range(0,4), range(0,2), range(0,2), range(0,4), range(0,4)):
                    C2[r][b,n,m]=C2[r][b,n,m]+n_C2*(Class_label[k]-f[k])*W[r][b,k,i,j]*H2[r][n+(2*j+u),m+(2*j+v)]
                    H3_bias[r][b,2*i+u,2*j+v]=H3_bias[r][b,2*i+u,2*j+v]-n_H3_bias*(Class_label[k]-f[k])*W[r][b,k,i,j]

    print(Class_label,f)
    if (PP%100==0):print(PP)

```

```python
sig.convolve(H2[2], C2[2][0],'valid')-H3_bias[2][0]
```

    array([[ 0.14935737,  0.12830971,  0.13919184,  0.08549448,  0.05531732,
             0.08943441,  0.14147535,  0.04195647],
           [ 0.1177075 ,  0.14432772,  0.04929092,  0.07709603,  0.09281097,
             0.0759421 ,  0.13508429,  0.10892261],
           [ 0.10693692,  0.0988523 ,  0.14349409,  0.04832444,  0.0929083 ,
             0.04729404,  0.0489684 ,  0.11595924],
           [ 0.0846642 ,  0.14347017,  0.06497644,  0.07102788,  0.05048184,
             0.07993234,  0.12112274,  0.12286457],
           [ 0.05573536,  0.10623589,  0.0564848 ,  0.04061568,  0.09617326,
             0.13263721,  0.0962134 ,  0.11654601],
           [ 0.10117204,  0.07636668,  0.06025513,  0.04890922,  0.07955253,
             0.10313259,  0.07032342,  0.13281523],
           [ 0.14213857,  0.15597651,  0.12649275,  0.14172842,  0.15214657,
             0.10808586,  0.11773363,  0.12505101],
           [ 0.0795919 ,  0.08989868,  0.11933094,  0.11283356,  0.12444131,
             0.10666211,  0.15559137,  0.09718626]])

```python
x
```

    array([-0.69966697, -0.3274634 , -1.10306853])

```python
bias
```

    array([ 0.01459842,  0.07111939, -0.09767391])

```python
W
```

    [array([[[[-0.09451344,  0.01764522,  0.11949737,  0.05031588],
              [-0.01004807,  0.04527503,  0.04744747, -0.03971535],
              [ 0.03838153,  0.01504355, -0.01052378,  0.02718381],
              [-0.06472421,  0.02633871,  0.07561101,  0.10256035]],

             [[ 0.0925761 , -0.06997852, -0.20426371, -0.13304848],
              [ 0.0159448 , -0.13523851, -0.10588554, -0.02101891],
              [-0.04890296, -0.05134114,  0.01119788, -0.07687243],
              [ 0.04130907, -0.08961312, -0.08614837, -0.13360783]],

             [[-0.01342397,  0.04167937,  0.06528107,  0.0829884 ],
              [-0.01932079,  0.08675308,  0.0557435 ,  0.0578841 ],
              [ 0.0004018 ,  0.03261402, -0.00740806,  0.04229913],
              [ 0.00575538,  0.05788358, -0.00819038,  0.01844118]]],

            [[[-0.12770824,  0.02129023,  0.16041599,  0.03354945],
              [-0.0322927 ,  0.06073887,  0.02692608, -0.05585178],
              [ 0.03283028,  0.02612535, -0.03732709,  0.04066974],
              [-0.08803913,  0.03126276,  0.07075652,  0.12170169]],

             [[ 0.1021058 , -0.07622915, -0.27054583, -0.10141596],
              [ 0.02273793, -0.14892472, -0.11073684,  0.01431501],
              [-0.0410166 , -0.0902408 ,  0.05105482, -0.12269707],
              [ 0.05265558, -0.09315412, -0.05679834, -0.1798109 ]],

             [[-0.00237541,  0.03659404,  0.09769915,  0.06018537],
              [ 0.00292207,  0.08215126,  0.06911516,  0.03064045],
              [ 0.00088982,  0.06282902, -0.01923765,  0.06313309],
              [ 0.01569652,  0.066199  , -0.035213  ,  0.04712716]]]]),
     array([[[[ 0.00507463,  0.00780472,  0.01423729,  0.00708448],
              [ 0.01118629,  0.008867  ,  0.0096952 ,  0.01370163],
              [ 0.01454006,  0.0130855 ,  0.00646826,  0.0101494 ],
              [ 0.0155926 ,  0.01411818,  0.00930188,  0.01068359]],

             [[-0.1082048 , -0.1102268 , -0.11065844, -0.10943519],
              [-0.10328994, -0.10692809, -0.10784327, -0.10921619],
              [-0.11081796, -0.10883589, -0.11030501, -0.10478726],
              [-0.11154781, -0.11226336, -0.10497134, -0.10697832]],

             [[ 0.10185704,  0.10010298,  0.10523647,  0.09903463],
              [ 0.10708423,  0.10682162,  0.098414  ,  0.09756669],
              [ 0.10270087,  0.09718347,  0.10754883,  0.10100794],
              [ 0.10416391,  0.10506627,  0.09907078,  0.10051775]]],

            [[[ 0.01076353,  0.01270998,  0.01340022,  0.01804303],
              [ 0.01584464,  0.0103628 ,  0.01861409,  0.01956825],
              [ 0.02275934,  0.01355188,  0.01901942,  0.01263357],
              [ 0.02023453,  0.02031081,  0.01618124,  0.00851546]],

             [[-0.10712697, -0.11646827, -0.11255464, -0.11445601],
              [-0.10481143, -0.11187318, -0.11669296, -0.11546197],
              [-0.11741785, -0.11178083, -0.11803485, -0.11917166],
              [-0.12932737, -0.12064556, -0.1202086 , -0.1107643 ]],

             [[ 0.09479954,  0.0985417 ,  0.10408309,  0.09876059],
              [ 0.10163988,  0.10557031,  0.10305073,  0.10632401],
              [ 0.10715528,  0.1067924 ,  0.10803594,  0.1043488 ],
              [ 0.10221259,  0.10800618,  0.09960432,  0.10292631]]]]),
     array([[[[-0.0054972 , -0.0082966 , -0.01328345, -0.00879509],
              [-0.00880757, -0.00767575, -0.0136435 , -0.00784989],
              [-0.01431509, -0.00919104, -0.00637415, -0.00662353],
              [-0.00873981, -0.00890231, -0.00641546, -0.00985337]],

             [[-0.00022715,  0.00658654,  0.00576399,  0.00480096],
              [ 0.00334558, -0.00164606,  0.00197955,  0.00421732],
              [ 0.0047014 ,  0.0049387 ,  0.00364979,  0.00658337],
              [-0.00117798,  0.0070661 ,  0.00311406,  0.00674847]],

             [[-0.022072  , -0.02104761, -0.02501629, -0.01880138],
              [-0.0179703 , -0.02406015, -0.02173449, -0.0189958 ],
              [-0.01839924, -0.02411311, -0.0249694 , -0.02610005],
              [-0.02354458, -0.02176279, -0.022072  , -0.02282002]]],

            [[[-0.0126394 , -0.01063864, -0.00881648, -0.0102493 ],
              [-0.01492824, -0.01209899, -0.01096503, -0.01090598],
              [-0.01486324, -0.01367285, -0.01184989, -0.01070078],
              [-0.01493207, -0.01651045, -0.0147819 , -0.01323788]],

             [[ 0.00497507,  0.00484086,  0.00858956,  0.00506814],
              [ 0.00530387,  0.00550936,  0.00541287,  0.00467607],
              [ 0.00470221,  0.00602529,  0.00404152,  0.01223231],
              [ 0.00850403,  0.00747606,  0.00354806,  0.00260821]],

             [[-0.02649017, -0.02777472, -0.02104019, -0.02641525],
              [-0.02625522, -0.0288568 , -0.01987536, -0.02293694],
              [-0.02412214, -0.0195542 , -0.02103418, -0.02466444],
              [-0.02296524, -0.02824865, -0.02391112, -0.02136729]]]]),
     array([[[[-0.02600758, -0.02098845, -0.02720445, -0.02563542],
              [-0.02624214, -0.02783755, -0.02335442, -0.01932625],
              [-0.02074395, -0.0182136 , -0.02529795, -0.02308949],
              [-0.02286851, -0.0249688 , -0.02439315, -0.01987909]],

             [[ 0.0271061 ,  0.0230231 ,  0.02898912,  0.02418719],
              [ 0.03013817,  0.02272867,  0.03080398,  0.02395689],
              [ 0.02977066,  0.02537782,  0.0264635 ,  0.02835363],
              [ 0.02558571,  0.02233914,  0.03215606,  0.03234368]],

             [[-0.03214974, -0.03642393, -0.03469447, -0.02947703],
              [-0.02984901, -0.03812227, -0.03559948, -0.03617257],
              [-0.0320278 , -0.03370177, -0.02928969, -0.03072791],
              [-0.03294015, -0.03559171, -0.03166382, -0.02887461]]],

            [[[-0.01713669, -0.02195003, -0.01483689, -0.01385306],
              [-0.01897211, -0.02115993, -0.01235729, -0.02104276],
              [-0.01631379, -0.01453867, -0.0155817 , -0.01441539],
              [-0.01971403, -0.01645545, -0.02245664, -0.01729963]],

             [[ 0.00951442,  0.01466517,  0.01272022,  0.0126009 ],
              [ 0.01480938,  0.01153742,  0.01317437,  0.0176338 ],
              [ 0.01758471,  0.01363715,  0.011302  ,  0.0127394 ],
              [ 0.0143223 ,  0.01192003,  0.01592481,  0.01162502]],

             [[-0.02746733, -0.03234926, -0.02419078, -0.02462123],
              [-0.02654163, -0.03058019, -0.02535117, -0.02376415],
              [-0.02910458, -0.03249412, -0.03008852, -0.02456848],
              [-0.03175768, -0.02989268, -0.02883216, -0.02567991]]]]),
     array([[[[-0.01327824, -0.01845856, -0.01458134, -0.01451495],
              [-0.01187649, -0.01638512, -0.01278855, -0.01828005],
              [-0.01576957, -0.01824181, -0.0181902 , -0.01297498],
              [-0.00987788, -0.01201314, -0.0108885 , -0.01121735]],

             [[ 0.00878301,  0.00971753,  0.01014207,  0.00955929],
              [ 0.00880295,  0.01309003,  0.0106729 ,  0.00877988],
              [ 0.00577223,  0.00848328,  0.00553509,  0.00797128],
              [ 0.01046513,  0.0130631 ,  0.01488298,  0.01075419]],

             [[-0.02915698, -0.02014944, -0.02256722, -0.01908289],
              [-0.02487542, -0.02383461, -0.02648781, -0.02469826],
              [-0.03017265, -0.02874712, -0.02205944, -0.02473355],
              [-0.02383857, -0.02710588, -0.02248241, -0.02338207]]],

            [[[-0.01674851, -0.00881909, -0.01534091, -0.01493336],
              [-0.01257245, -0.01474794, -0.00807357, -0.00853992],
              [-0.00977093, -0.00901168, -0.01075006, -0.01693607],
              [-0.0107176 , -0.01289755, -0.01433068, -0.01517628]],

             [[ 0.00168033,  0.00527259,  0.00537219,  0.00452417],
              [ 0.01030732,  0.00166708,  0.00708773,  0.00976649],
              [ 0.00728993,  0.00442624,  0.00504767,  0.0108352 ],
              [ 0.0083675 ,  0.01038495,  0.00286869,  0.00932652]],

             [[-0.01842837, -0.01939398, -0.02077519, -0.02331738],
              [-0.02125706, -0.01701772, -0.01958394, -0.02443172],
              [-0.0255015 , -0.02154481, -0.02047066, -0.02095854],
              [-0.01857916, -0.02421521, -0.01677902, -0.01927327]]]]),
     array([[[[ -1.05763017e-02,  -1.64628020e-02,  -1.32598434e-02,
                -1.74181420e-02],
              [ -1.27182740e-02,  -1.69228173e-02,  -9.26356257e-03,
                -1.45864132e-02],
              [ -1.58234460e-02,  -1.31484322e-02,  -1.47196395e-02,
                -1.63473519e-02],
              [ -1.15115060e-02,  -1.63223376e-02,  -1.67912125e-02,
                -1.79354610e-02]],

             [[  1.07817765e-02,   1.01040666e-02,   9.79064697e-03,
                 1.14720375e-02],
              [  8.17980355e-03,   1.13028624e-02,   7.14133631e-03,
                 5.48142933e-03],
              [  9.20566183e-03,   1.07531523e-02,   1.56971359e-02,
                 8.26775261e-03],
              [  1.58930636e-02,   7.92911363e-03,   7.92536106e-03,
                 8.60416137e-03]],

             [[ -1.75539835e-02,  -1.84257317e-02,  -2.25025562e-02,
                -2.43005574e-02],
              [ -2.80276161e-02,  -1.85360524e-02,  -2.61736189e-02,
                -2.40671527e-02],
              [ -2.49452205e-02,  -2.48344833e-02,  -2.44247051e-02,
                -2.17676484e-02],
              [ -2.10527332e-02,  -2.03732528e-02,  -1.79319762e-02,
                -2.37514638e-02]]],

            [[[ -1.49802478e-02,  -1.57968853e-02,  -1.51850306e-02,
                -1.20864573e-02],
              [ -9.23611049e-03,  -1.39008568e-02,  -7.59908523e-03,
                -1.39823158e-02],
              [ -1.46107061e-02,  -9.53562755e-03,  -1.25245284e-02,
                -9.09859078e-03],
              [ -6.20554810e-03,  -8.90730422e-03,  -6.10895828e-03,
                -8.02258717e-03]],

             [[  6.07125858e-03,  -7.69681219e-04,  -1.43731041e-03,
                 4.69043216e-03],
              [  3.44628735e-03,  -1.94786649e-03,   3.33231696e-03,
                 5.20270260e-03],
              [  4.67308383e-03,   2.07053690e-03,   4.58131918e-06,
                 5.26006707e-03],
              [  2.22734384e-03,   4.80120548e-03,   9.32709985e-04,
                 3.07333771e-03]],

             [[ -1.54463229e-02,  -2.20104455e-02,  -2.29134842e-02,
                -2.11574039e-02],
              [ -1.73094297e-02,  -2.29961269e-02,  -1.49056918e-02,
                -1.82977576e-02],
              [ -1.57194376e-02,  -2.45553227e-02,  -2.02019997e-02,
                -1.72671494e-02],
              [ -2.25375815e-02,  -1.85155066e-02,  -2.28037177e-02,
                -1.67973113e-02]]]])]

```python
plt.figure(figsize=(10,10))
for i in range(0,len(C1)):
    for j in range(0,N_branches):
        plt.subplot(len(C1),N_branches,(i*N_branches)+j)
        plt.imshow(H4[i][j])
```

```python
plt.figure(figsize=(10,10))
for i in range(0,len(C1)):
    for j in range(0,N_branches):
        plt.subplot(len(C1),N_branches,(i*N_branches)+j)
        plt.imshow(C2[i][j])
```

# save training parameters

```python
with open("C2.txt", 'w') as f:
    f.write(str(C2))
with open("W.txt", 'w') as f:
    f.write(str(W))
with open("H3_bias.txt", 'w') as f:
    f.write(str(H3_bias))
with open("Bias.txt", 'w') as f:
    f.write(str(bias))

```

```python

```


# A Gentle Introduction to HARK: Buffer Stock Saving

[](https://econ-ark.org/materials/gentle-intro-to-hark-buffer-stock-model#launch)

This notebook explores the behavior of a consumer identical to the perfect foresight consumer described in [Gentle-Intro-To-HARK-PerfForesightCRRA](https://econ-ark.org/materials/Gentle-Intro-To-HARK-PerfForesightCRRA) except that now the model incorporates income uncertainty.

```python
# This cell has a bit of initial setup.
# Click the "Run" button immediately above the notebook in order to execute the contents of any cell
# WARNING: Each cell in the notebook relies upon results generated by previous cells
#   The most common problem beginners have is to execute a cell before all its predecessors
#   If you do this, you can restart the kernel (see the "Kernel" menu above) and start over
import matplotlib.pyplot as plt

import numpy as np
import HARK
from copy import deepcopy
mystr = lambda number : "{:.4f}".format(number)
from HARK.utilities import plot_funcs
```

## The Consumer's Problem with Transitory and Permanent Shocks
### Mathematical Description

Our new type of consumer receives two income shocks at the beginning of each period.  Permanent income would grow by a factor $\Gamma$ in the absence of any shock , but its growth is modified by a shock, $\psi_{t+1}$:
\begin{align}
 P_{t+1} & = \Gamma P_{t}\psi_{t+1}
\end{align}
whose expected (mean) value is $\mathbb{E}_{t}[\psi_{t+1}]=1$.  Actual income received $Y$ is equal to permanent income $P$ multiplied by a transitory shock $\theta$:
\begin{align}
 Y_{t+1} & = \Gamma P_{t+1}\theta_{t+1}
\end{align}
where again $\mathbb{E}_{t}[\theta_{t+1}] = 1$.

As with the perfect foresight problem, this model can be rewritten in terms of _normalized_ variables, e.g. the ratio of 'market resources' $M_{t}$ (wealth plus current income) to permanent income is $m_t \equiv M_t/P_t$.  (See [here](http://econ.jhu.edu/people/ccarroll/papers/BufferStockTheory/) for the theory).  In addition, lenders may set a limit on borrowing: The ratio $a_{t}$ of end-of-period assets to permanent income $A_t/P_t$ must be greater than $\underline{a} \leq 0$. (So, if $\underline{a}=-0.3$, the consumer cannot borrow more than 30 percent of their permanent income).

The consumer's (normalized) problem turns out to be:
\begin{eqnarray*}
v_t(m_t) &=& \max_{c_t} ~~u(c_t) + \beta  \mathbb{E} [(\Gamma_{t+1}\psi_{t+1})^{1-\rho} v_{t+1}(m_{t+1}) ], \\
 & \text{s.t.} & \\
a_t &=& m_t - c_t, \\
a_t &\geq& \underline{a}, \\
m_{t+1} &=& a_t  R/(\Gamma_{t+1} \psi_{t+1}) + \theta_{t+1}.
\end{eqnarray*}

For present purposes, we assume that the transitory and permanent shocks are independent.  The permanent shock is assumed to be (approximately) lognormal, while the transitory shock has two components: A probability $\wp$ that the consumer is unemployed, in which case $\theta^{u}=\underline{\theta}$, and a probability $(1-\wp)$ of a shock that is a lognormal with a mean chosen so that $\mathbb{E}_{t}[\theta_{t+n}]=1$.

### Representing the Income Shocks

Computers are discrete devices; even if somehow we knew with certainty that the transitory and permanent shocks were, say, continuously lognormally distributed, in order to be represented on a computer those distributions would need to be approximated by a finite set of points.  A large literature in numerical computation explores ways to construct such approximations; probably the easiest discretization to understand is the equiprobable approximation, in which the continuous distribution is represented by a set of $N$ outcomes that are equally likely to occur.

In the case of a single variable (say, the permanent shock $\psi$), and when the number of equiprobable points is, say, 5, the procedure is to construct a list: $\psi^{0}$ is the mean value of the continuous $\psi$ given that the draw of $\psi$ is in the bottom 20 percent of the distribution of the continuous $\psi$.  $\psi^{1}$ is the mean value of $\psi$ given that the draw is between the 20th and 40th percentiles, and so on.  Having constructed these, the approximation to the expectation of some expression $g(\psi)$ can be very quickly calculated by:

$$
\mathbb{E}_{t}[g(\psi)] \equiv \int_{0}^{\infty} g(\psi) dF_{\psi} \approx (1/N) \sum_{i=0}^{N-1} g(\psi^{i}).
$$

(For a graphical depiction of a particular instance of this, see [SolvingMicroDSOPs/#discreteApprox](http://www.econ2.jhu.edu/people/ccarroll/SolvingMicroDSOPs/#discreteApprox).)

## The New Parameters

In addition to the parameters required for the perfect foresight model (like the time preference factor $\beta$), under the assumptions above, we need to choose values for the following extra parameters that describe the income shock distribution and the artificial borrowing constraint.

| Param | Description | Code | Value |
| :---: | ---         | ---  | :---: |
| $\underline{a}$ | Artificial borrowing constraint | $\texttt{BoroCnstArt}$ | 0.0 |
| $\sigma_\psi$ | Underlying stdev of permanent income shocks | $\texttt{PermShkStd}$ | 0.1 |
| $\sigma_\theta^{e}$ | Underlying stdev of transitory income shocks | $\texttt{TranShkStd}$ | 0.1 |
| $N_\psi$ | Number of discrete permanent income shocks | $\texttt{PermShkCount}$ | 7 |
| $N_\theta$ | Number of discrete transitory income shocks | $\texttt{TranShkCount}$ | 7 |
| $\wp$ | Unemployment probability | $\texttt{UnempPrb}$ | 0.05 |
| $\underline{\theta}$ | Transitory shock when unemployed | $\texttt{IncUnemp}$ | 0.3 |

## Representation in HARK

HARK agents with this kind of problem are instances of the class $\texttt{IndShockConsumerType}$, which is constructed by "inheriting" the properties of the $\texttt{PerfForesightConsumerType}$ and then adding only the _new_ information required:

```python
# This cell defines a parameter dictionary for making an instance of IndShockConsumerType.

IndShockDictionary = {
    'PermShkStd': [0.1],  # ... by specifying the new parameters for constructing the income process.
    'PermShkCount': 7,
    'TranShkStd': [0.1],
    'TranShkCount': 7,
    'UnempPrb': 0.05,
    'IncUnemp': 0.3, # ... and income for unemployed people (30 percent of "permanent" income)
    'BoroCnstArt': 0.0, # ... and specifying the location of the borrowing constraint (0 means no borrowing is allowed)
    'cycles': 0         # signifies an infinite horizon solution (see below)
}
```

## Other Attributes are Inherited from PerfForesightConsumerType

You can see all the **attributes** of an object in Python by using the `dir()` command. From the output of that command below, you can see that many of the model variables are now attributes of this object, along with many other attributes that are outside the scope of this tutorial.

```python
from HARK.ConsumptionSaving.ConsIndShockModel import PerfForesightConsumerType

pfc = PerfForesightConsumerType()
dir(pfc)
```

In python terminology, `IndShockConsumerType` is a **superclass** of `PerfForesightConsumerType`. This means that it builds on the functionality of its parent type (including, for example, the definition of the utility function). You can find the superclasses of a type in Python using the `__bases__` attribute:

```python
from HARK.ConsumptionSaving.ConsIndShockModel import IndShockConsumerType

IndShockConsumerType.__bases__
```

```python
# So, let's create an instance of the IndShockConsumerType
IndShockExample = IndShockConsumerType(**IndShockDictionary)
```

As before, we need to import the relevant subclass of $\texttt{AgentType}$ into our workspace, then create an instance by passing the dictionary to the class as if the class were a function.

## The Discretized Probability Distribution

The scatterplot below shows how the discretized probability distribution is represented in HARK: The lognormal distribution is represented by a set of equiprobable point masses.

```python
# Plot values for equiprobable distribution of permanent shocks

plt.scatter(IndShockExample.PermShkDstn[0].X,
            IndShockExample.PermShkDstn[0].pmf)
plt.xlabel("Value")
plt.ylabel("Probability Mass")
plt.show()
```

This distribution was created, using the parameters in the dictionary above, when the `IndShockConsumerType` object was initialized.

## Solution by Backwards Induction

HARK solves this problem using _backwards induction_: It will derive a solution for each period ($t$) by finding a mapping between specific values of market resources $\{m[0],m[1],...\}$ and the corresponding optimal consumption $\{c[0],c[1],...\}$. The function that "connects the dots" will be stored in a variable named `cFunc`.

Backwards induction requires a "terminal" (last; final) period to work backwards from.  `IndShockExample` constructed above did not specify a terminal consumption function, and consequently it uses the default terminal function in which all resources are consumed: $c_{T} = m_{t}$.

```python
IndShockExample.solution_terminal
```

The consumption function `cFunc` is defined by _piecewise linear interpolation_.

It is defined by a series of $(m,c)$ points on a grid; the value of the function for any $m$ is the $c$ determined by the line connecting the nearest defined gridpoints.

You can see below that in the terminal period, $c = m$; the agent consumes all available resources.

```python
# Plot terminal consumption function
plt.plot(IndShockExample.solution_terminal.cFunc.x_list,
         IndShockExample.solution_terminal.cFunc.y_list,
         color='k')
plt.scatter(IndShockExample.solution_terminal.cFunc.x_list,
            IndShockExample.solution_terminal.cFunc.y_list)
```

The solution also has a representation of a `value function`, the value `v(m)` as a function of available market resources. Because the agent consumes all their resources in the last period, the value function for the terminal solution looks just like the CRRA utility function: $v_{T}(m) = u(m)$.

```python
# Final consumption function c=m
m = np.linspace(0.1,1,100)
plt.plot(m,IndShockExample.solution_terminal.vFunc(m))
```

## Solving the problem

This solution is generated by invoking `solve()` which is a **method** that is an **attribute** of the `IndShockExample` object. **Methods** in Python are supposed to have **documentation** that tell you what they do. You can read the documentation for methods and other attributes in HARK with the built-in Python `help()` function:

```python
help(IndShockExample.solve)
```

### Finite or Infinite Horizon?

$\texttt{ConsIndShockType}$ can solve either finite-horizon (e.g., life-cycle) problems, or infinite-horizon problems (where the problem is the same in every period).  Elsewhere you can find documentation about the finite horizon solution; here we are interested in the infinite-horizon solution which is obtained (by definition) when iterating one more period yields a solution that is essentially the same.  In the dictionary above we signaled to HARK that we want the infinite horizon solution by setting the "cycles" paramter to zero:

```python
IndShockExample.cycles # Infinite horizon solution is computed when cycles = 0
```

```python
# Solve It
IndShockExample.solve(verbose=True) # Verbose prints progress as solution proceeds
```

```python
# plot_funcs([list],min,max) takes a [list] of functions and plots their values over a range from min to max
plot_funcs([IndShockExample.solution[0].cFunc,IndShockExample.solution_terminal.cFunc],0.,10.)
```

## Changing Constructed Attributes

In the parameter dictionary above, we chose values for HARK to use when constructing its numerical representation of $F_t$, the joint distribution of permanent and transitory income shocks. When $\texttt{IndShockExample}$ was created, those parameters ($\texttt{TranShkStd}$, etc) were used by the **constructor** or **initialization** method of $\texttt{IndShockConsumerType}$ to construct an attribute called $\texttt{IncomeDstn}$.

Suppose you were interested in changing (say) the amount of permanent income risk.  From the section above, you might think that you could simply change the attribute $\texttt{TranShkStd}$, solve the model again, and it would work.

That's _almost_ true-- there's one extra step. $\texttt{TranShkStd}$ is a primitive input, but it's not the thing you _actually_ want to change. Changing $\texttt{TranShkStd}$ doesn't actually update the income distribution... unless you tell it to (just like changing an agent's preferences does not change the consumption function that was stored for the old set of parameters -- until you invoke the $\texttt{solve}$ method again).  In the cell below, we invoke the method $\texttt{update_income_process}$ so HARK knows to reconstruct the attribute $\texttt{IncomeDstn}$.

```python
OtherExample = deepcopy(IndShockExample)  # Make a copy so we can compare consumption functions
OtherExample.PermShkStd = [0.2]           # Double permanent income risk (note that it's a one element list)
OtherExample.update_income_process()        # Call the method to reconstruct the representation of F_t
OtherExample.solve()
```

In the cell below, use your blossoming HARK skills to plot the consumption function for $\texttt{IndShockExample}$ and $\texttt{OtherExample}$ on the same figure.

```python
# Use the remainder of this cell to plot the IndShockExample and OtherExample consumption functions against each other
```

## Buffer Stock Saving?

There are some combinations of parameter values under which problems of the kind specified above have "degenerate" solutions; for example, if consumers are so patient that they always prefer deferring consumption to the future, the limiting consumption rule can be $c(m)=0$.

The toolkit has built-in tests for a number of parametric conditions that can be shown to result in various characteristics in the optimal solution.

Perhaps the most interesting such condition is the ["Growth Impatience Condition"](http://econ.jhu.edu/people/ccarroll/Papers/BufferStockTheory/#GIC): If this condition is satisfied, the consumer's optimal behavior is to aim to achieve a "target" value of $m$, to serve as a precautionary buffer against income shocks.

The tests can be invoked using the `checkConditions()` method:

```python
IndShockExample.check_conditions(verbose=True)
```


# Shenfun - High-Performance Computing platform for the Spectral Galerkin method

Professor Mikael Mortensen

Department of Mathematics, University of Oslo

Presented at the International Conference on Scientific Computing and Applications (ICSCA), Xiamen, China, 29/5 - 2019

# Shenfun - facts

1. Shenfun is named in honour of Professor Jie Shen for his seminal work on the spectral Galerkin method:-)
2. Shenfun is a high performance computing platform for solving partial differential equations (PDEs) with the spectral Galerkin method (with numerical integration).
3. Shenfun has been run with 65,000 processors on a Cray XC40.
4. Shenfun is a high-level Python package originally developed for large-scale pseudo-spectral turbulence simulations.

# Python is a scripting language

No compilation - just execute. Much like MATLAB. High-level coding very popular in the scientific computing community.

In this presentation this is a Python terminal:

```python
print('hello world icsca')
```

Inside the terminal any Python code can be executed and if something is printed it is shown below.

```python
print(2+2)
```

# When in presentation mode

like now, the terminal is not alive. However, this presentation is written with [jupyter](https://jupyter.org/)

and if opened in active mode, then all boxes like the one below would be live and active and ready to execute any Python code:

```python

```

If interested (it is really not necessary), then

Open https://github.com/spectralDNS/shenfun/ and press the launch-binder button.

[](https://mybinder.org/v2/gh/spectralDNS/shenfun/master?filepath=binder)

Wait for binder to launch and choose the `Shenfun presentation.ipynb` file to get a live document. There are also some other demos there written in jupyter.

Meanwhile (may take a few minutes and really not necessary) I'll go through some background material required for understanding how shenfun works 😀

# The Spectral Galerkin method (in a nutshell)

approximates solutions $u(x)$ using global trial functions $\phi_k(x)$ and unknown expansion coefficients $\hat{u}_k$

$$
u(x) = \sum_{k=0}^{N-1}\hat{u}_k \phi_k(x)
$$

Multidimensional solutions are formed from outer products of 1D bases

$$
u(x, y) = \sum_{k=0}^{N_0-1}\sum_{l=0}^{N_1-1}\hat{u}_{kl} \phi_{kl}(x, y)\quad \text{ or }\quad
u(x, y, z) = \sum_{k=0}^{N_0-1}\sum_{l=0}^{N_1-1} \sum_{m=0}^{N_2-1}\hat{u}_{klm} \phi_{klm}(x, y, z)
$$

where, for example

$$
\begin{align}
\phi_{kl}(x, y) &= T_k(x) L_l(y)\\
\phi_{klm}(x, y, z) &= T_k(x) L_l(y) \exp(\text{i}mz)
\end{align}
$$

$T_k$ and $L_k$ are Chebyshev and Legendre polynomials.

# The Spectral Galerkin method

solves PDEs, like Poisson's equation

\begin{align}
\nabla^2 u(x) &= f(x), \quad x \in [-1, 1] \\
u(\pm 1) &= 0
\end{align}

using variational forms by the method of weighted residuals. I.e., multiply PDE by a test function $v$ and integrate over the domain. For Poisson this leads to the problem:

Find $u \in H^1_0$ such that

$$(\nabla u, \nabla v)_w^N = -(f, v)_w^N \quad \forall v \in H^1_0$$

Here $(u, v)_w^{N}$ is a weighted inner product and $v(=\phi_j)$ is a test function. Note that test and trial functions are the same for the Galerkin method.

# Weighted inner products

The weighted inner product is defined as

$$
 (u, v)_w = \int_{\Omega} u \overline{v} w \, d\Omega,
$$

where $w(\mathbf{x})$ is a weight associated with the chosen basis (different bases have different weights). The overline represents a complex conjugate (for Fourier).

$\Omega$ is a tensor product domain spanned by the chosen 1D bases.

# In Shenfun quadrature is used for the integrals

1D with Chebyshev basis:

$$
(u, v)_w ^N = \sum_{i=0}^{N-1} u(x_i) v(x_i) \omega_i \approx \int_{-1}^1 \frac{u v}{\sqrt{1-x^2}}  \, {dx},
$$

where $\{\omega_i\}_{i=0}^{N-1}$ are the quadrature weights associated with the chosen basis and quadrature rule. The associated quadrature points are denoted as $\{x_i\}_{i=0}^{N-1}$.

2D with mixed Chebyshev-Fourier:

$$
(u, v)_w^N = \int_{-1}^1\int_{0}^{2\pi} \frac{u \overline{v}}{2\pi\sqrt{1-x^2}} \, {dxdy} \approx \sum_{i=0}^{N_0-1}\sum_{j=0}^{N_1-1} u(x_i, y_j) \overline{v}(x_i, y_j) \omega^{(x)}_i \omega_j^{(y)} ,
$$

# Spectral Galerkin solution procedure

1. Choose function space(s) satisfying correct boundary conditions
2. Transform PDEs to variational forms with inner products
3. Assemble variational forms and solve resulting linear algebra systems

# Orthogonal bases

| Family    | Basis                                    | Domain    |
|  :---:    |         :---:                            |   :---:   |
| Chebyshev | $$\{T_k\}_{k=0}^{N-1}$$                  | $$[-1, 1]$$ |
| Legendre  | $$\{L_k\}_{k=0}^{N-1}$$                  | $$[-1, 1]$$ |
| Fourier   | $$\{\exp(\text{i}kx)\}_{k=-N/2}^{N/2-1}$$| $$[0, 2\pi]$$ |
| Hermite   | $$\{H_k\}_{k=0}^{N-1}$$                  | $$[-\infty, \infty]$$|
| Laguerre  | $$\{La_k\}_{k=0}^{N-1}$$                 | $$[0, \infty]$$ |

```python
from shenfun import *
N = 8
C = FunctionSpace(N, 'Chebyshev')
L = FunctionSpace(N, 'Legendre')
x, w = C.points_and_weights()
print(np.vstack((x, w)).T)
```

# Shen's bases with Dirichlet bcs

| family    | Basis                 | Boundary condition |
|-----------|-----------------------|----------|
| Chebyshev | $$\{T_k-T_{k+2}\}_{k=0}^{N-3}$$ | $$u(\pm 1) = 0$$ |
| Legendre  | $$\{L_k-L_{k+2}\}_{k=0}^{N-3}$$ | $$u(\pm 1) = 0$$ |
| Hermite   | $$\exp(-x^2)\{H_k\}_{k=0}^{N-1}$$ | $$u(\pm \infty) = 0$$ |
| Laguerre  | $$\exp(-x/2)\{La_k-La_{k+1}\}_{k=0}^{N-2}$$| $$u(0) = u(\infty) = 0$$ |

```python
C0 = FunctionSpace(N, 'Chebyshev', bc=(0, 0))
L0 = FunctionSpace(N, 'Legendre', bc=(0, 0))
H0 = FunctionSpace(N, 'Hermite')
La = FunctionSpace(N, 'Laguerre', bc=(0, 0))
```

# Shen's bases with Neumann $u'(\pm 1) = 0$

| family    | Basis                 |
|-----------|-----------------------|
| Chebyshev | $$\left\{T_k-\frac{k^2}{(k+2)^2}T_{k+2}\right\}_{k=0}^{N-3}$$ |
| Legendre  | $$\left\{L_k-\frac{k(k+1)}{(k+2)(k+3)}L_{k+2}\right\}_{k=0}^{N-3}$$ |

```python
CN = FunctionSpace(N, 'Chebyshev', bc={'left': ('N', 0), 'right': ('N', 0)})
LN = FunctionSpace(N, 'Legendre', bc={'left': ('N', 0), 'right': ('N', 0)})
```

# Shen's biharmonic bases $u(\pm 1) = u'(\pm 1) = 0$

| family    | Basis                 |
|-----------| :-----------------:   |
| Chebyshev | $$\left\{T_k-\frac{2(k+2)}{k+3}T_{k+2}+\frac{k+1}{k+3} T_{k+4}\right\}_{k=0}^{N-5}$$ |
| Legendre  | $$\left\{L_k-\frac{2(2k+5)}{(2k+7)}L_{k+2}+\frac{2k+3}{2k+7}L_{k+4}\right\}_{k=0}^{N-5}$$ |

```python
CB = FunctionSpace(N, 'Chebyshev', bc=(0, 0, 0, 0))
LB = FunctionSpace(N, 'Legendre', bc=(0, 0, 0, 0))
```

# Multidimensional tensor product spaces

$$
\begin{align}
L_0 &= \{L_k(x)-L_{k+2}(x)\}_{k=0}^{N-3} \\
C_0 &= \{T_k(x)-T_{k+2}(x)\}_{k=0}^{N-3} \\
L_1 &= \{L_l(y)\}_{l=0}^{N-1} \\
LL(x, y) &= L_0(x) \times L_1(y) \\
CL(x, y) &= C_0(x) \times L_1(y)
\end{align}
$$

```python
L0 = FunctionSpace(N, 'Legendre', bc=(0, 0))
C0 = FunctionSpace(N, 'Chebyshev', bc=(0, 0))
L1 = FunctionSpace(N, 'Legendre')
LL = TensorProductSpace(comm, (L0, L1)) # comm is MPI.COMM_WORLD
CL = TensorProductSpace(comm, (C0, L1))
```

# Basis functions can be created for all bases

```python
L0 = FunctionSpace(N, 'Legendre', bc=(0, 0))
L1 = FunctionSpace(N, 'Legendre')

# 1D
u = TrialFunction(L0)
v = TestFunction(L0)
uh = Function(L0)
uj = Array(L0)

# 2D
LL = TensorProductSpace(comm, (L0, L1)) # comm is MPI.COMM_WORLD

u = TrialFunction(LL)
v = TestFunction(LL)
uh = Function(LL)
uj = Array(LL)

```

# The shenfun `Function` represents the solution

`uh = Function(L0)`

$$
u(x) = \sum_{k=0}^{N-1} \hat{u}_k \phi_{k}(x)
$$

The function evaluated for all quadrature points, $\{u(x_j)\}_{j=0}^{N-1}$, is an `Array`

`uj = Array(L0)`

There is a (fast) `backward` transform for moving from `Function` to `Array`, and a `forward` transform to go the other way. Note that the `Array` is not a basis function!

```python
L0 = FunctionSpace(N, 'Legendre', bc=(0, 0))
uh = Function(L0)
uj = Array(L0)

# Move back and forth
uj = uh.backward(uj)
uh = uj.forward(uh)
```

# Operators in shenfun act on basis functions

`u` is an instance of either `TestFunction`, `TrialFunction` or `Function`

- `div(u)`
- `grad(u)`
- `curl(u)`
- `Dx(u, 0, 1)` (partial derivative in x-direction)

# Assembly
- `project`
- `inner`

```python
L0 = FunctionSpace(N, 'Legendre', bc=(0, 0))
L1 = FunctionSpace(N, 'Legendre')
u = TrialFunction(L0)
v = TestFunction(L0)
uh = Function(L0)
du = grad(u)  # vector valued expression
g = div(du)   # scalar valued expression
c = project(Dx(uh, 0, 1), L1) # project expressions with Functions
```

# Implementation closely matches mathematics

$$
A = (\nabla u, \nabla v)_w^N
$$

```python
A = inner(grad(u), grad(v))
```

```python
dict(A)
```

```python
print(A.diags().todense())
```

A diagonal stiffness matrix!

# Complete Poisson solver with error verification in 1D

```python
# Solve Poisson's equation
from sympy import symbols, sin, lambdify
from shenfun import *

# Use sympy to compute manufactured solution
x = symbols("x")
ue = sin(4*np.pi*x)*(1-x**2) # `ue` is the manufactured solution
fe = ue.diff(x, 2) # `fe` is Poisson's right hand side for `ue`

SD = FunctionSpace(2000, 'L', bc=(0, 0))
u = TrialFunction(SD)
v = TestFunction(SD)

b = inner(v, Array(SD, buffer=fe)) # Array is initialized with `fe`
A = inner(v, div(grad(u)))

uh = Function(SD)
uh = A.solve(b, uh)  # Very fast O(N) solver
print(uh.backward()-Array(SD, buffer=ue))
```

# 2D - Implementation still closely matching mathematics

```python
L0 = FunctionSpace(N, 'Legendre', bc=(0, 0))
F1 = FunctionSpace(N, 'Fourier', dtype='d')
TP = TensorProductSpace(comm, (L0, F1))
u = TrialFunction(TP)
v = TestFunction(TP)
A = inner(grad(u), grad(v))
```

```python
print(A)
```

# ?

A is a list of two TPMatrix objects???

# `TPMatrix` is a Tensor Product matrix

A `TPMatrix` is the outer product of smaller matrices (2 in 2D, 3 in 3D etc).

Consider the inner product:

$$
\begin{align}
(\nabla u, \nabla v)_w &=\int_{-1}^{1}\int_{0}^{2\pi} \left(\frac{\partial u}{\partial x}, \frac{\partial u}{\partial y}\right) \cdot \left(\frac{\partial \overline{v}}{\partial x}, \frac{\partial \overline{v}}{\partial y}\right)  \frac{dxdy}{2\pi} \\
(\nabla u, \nabla v)_w &= \int_{-1}^1 \int_{0}^{2\pi} \frac{\partial u}{\partial x}\frac{\partial \overline{v}}{\partial x} \frac{dxdy}{2\pi} + \int_{-1}^1 \int_{0}^{2\pi} \frac{\partial u}{\partial y}\frac{\partial \overline{v}}{\partial y} \frac{dxdy}{2\pi}
\end{align}
$$

which, like `A`, is a sum of two terms. These two terms are the two `TPMatrix`es returned by `inner` above.

Now each one of these two terms can be written as the outer product of two smaller matrices.

Consider the first, inserting for test and trial functions

$$
\begin{align}
v &= \phi_{kl} = (L_k(x)-L_{k+2}(x))\exp(\text{i}ly) \\
u &= \phi_{mn}
\end{align}
$$

The first term becomes

$$
\small
\begin{align}
\int_{-1}^1 \int_{0}^{2\pi} \frac{\partial u}{\partial x}\frac{\partial \overline{v}}{\partial x} \frac{dxdy}{2\pi} &= \underbrace{\int_{-1}^1 \frac{\partial (L_m-L_{m+2})}{\partial x}\frac{\partial (L_k-L_{k+2})}{\partial x} {dx}}_{a_{km}} \underbrace{\int_{0}^{2\pi}  \exp(iny) \exp(-ily) \frac{dy}{2\pi}}_{\delta_{ln}} \\
   &= a_{km} \delta_{ln}
\end{align}
$$

and the second

$$
\small
\begin{align}
\int_{-1}^1 \int_{0}^{2\pi} \frac{\partial u}{\partial y}\frac{\partial \overline{v}}{\partial y} \frac{dxdy}{2\pi} &= \underbrace{\int_{-1}^1 (L_m-L_{m+2})(L_k-L_{k+2}) {dx}}_{b_{km}} \underbrace{\int_{0}^{2\pi} ln \exp(iny) \exp(-ily)\frac{dy}{2\pi}}_{l^2\delta_{ln}} \\
   &= l^2 b_{km} \delta_{ln}
\end{align}
$$

All in all:

$$
(\nabla u, \nabla v)_w = \left(a_{km} \delta_{ln} + l^2 b_{km} \delta_{ln}\right)
$$

$$
(\nabla u, \nabla v)_w = \left(a_{km} \delta_{ln} + l^2 b_{km} \delta_{ln}\right)
$$

```python
A = inner(grad(u), grad(v)) # <- list of two TPMatrices
print(A[0].mats)
print('Or as dense matrices:')
for mat in A[0].mats:
    print(mat.diags().todense())
```

```python
print(A[1].mats)
print(A[1].scale) # l^2
```

# 3D Poisson (with MPI and Fourier x 2)

```python
import matplotlib.pyplot as plt
from sympy import symbols, sin, cos, lambdify
from shenfun import *

# Use sympy to compute manufactured solution
x, y, z = symbols("x,y,z")
ue = (cos(4*x) + sin(2*y) + sin(4*z))*(1-x**2)
fe = ue.diff(x, 2) + ue.diff(y, 2) + ue.diff(z, 2)

C0 = FunctionSpace(32, 'Chebyshev', bc=(0, 0))
F1 = FunctionSpace(32, 'Fourier', dtype='D')
F2 = FunctionSpace(32, 'Fourier', dtype='d')
T = TensorProductSpace(comm, (C0, F1, F2))
u = TrialFunction(T)
v = TestFunction(T)

# Assemble left and right hand
f_hat = inner(v, Array(T, buffer=fe))
A = inner(v, div(grad(u)))

# Solve
solver = chebyshev.la.Helmholtz(*A) # Very fast O(N) solver
u_hat = Function(T)
u_hat = solver(f_hat, u_hat)
assert np.linalg.norm(u_hat.backward()-Array(T, buffer=ue)) < 1e-12
print(u_hat.shape)
```

# Contour plot of slice with constant y

```python
X = T.local_mesh()
ua = u_hat.backward()
plt.contourf(X[2][0, 0, :], X[0][:, 0, 0], ua[:, 2], 100)
plt.colorbar()
```

# Run with MPI distribution of arrays

Here we would normally run from a bash shell

 [bash shell] mpirun -np 4 python poisson3D.py

But since we are in a Jupyter notebook lets actually do this from python in a live cell:-)

```python
import subprocess
subprocess.check_output('mpirun -np 4 python poisson3D.py', shell=True)
```

Note that Fourier bases are especially attractive because of features easily handled with MPI:

    - diagonal matrices
    - fast transforms

# mpi4py-fft

by Mikael Mortensen and Lisandro Dalcin

Highly configurable Python package for distributing multidimensional arrays and for computing fast Fourier Transforms (FFTs) in parallel. Wraps [FFTW](http://www.fftw.org/) and lies at the core of `shenfun` and distributes large arrays.

[](https://anaconda.org/conda-forge/mpi4py-fft)

# Nonlinearities and convolutions
All treated with pseudo-spectral techniques. For example

$$
\begin{align}
\hat{w}_k &= \widehat{\mathbf{u} \cdot \mathbf{u}}_k \\
          &\text{or} \\
\hat{w}_k &= \widehat{|\nabla f|^2}_k
\end{align}
$$

Nonlinear terms are computed in real space and then forward transformed to spectral space.

3/2-rule or 2/3-rule possible for dealiasing of Fourier.

```python
V = VectorSpace(T)
u = Array(V)
u[:] = np.random.random(u.shape)
w = np.sum(u*u, axis=0)
wh = Function(T)
wh = T.forward(w, wh)
```

# Mixed tensor product spaces

Solve several equations simultaneously

- Coupled equations
- Block matrices and vectors
- Tensor spaces of vectors, like velocity $u \in [\mathbb{R}^3]^3$

# Stokes equations
### lid-driven cavity - coupled solver

$$
\begin{align*}
\nabla^2 \mathbf{u} - \nabla p &= \mathbf{f} \quad \text{in }  \Omega, \quad \quad \Omega = [-1, 1]\times[-1, 1]\\
\nabla \cdot \mathbf{u} &= h \quad \text{in } \Omega  \\
\int_{\Omega} p dx &= 0 \\
\mathbf{u}(\pm 1, y) = \mathbf{u}(x, -1) = (0, 0) &\text{ and }\mathbf{u}(x, 1) = (1, 0) \text{ or } ((1-x^2)(1+x^2), 0)
\end{align*}
$$

Given appropriate spaces $V$ and $Q$ a variational form reads: find $(\mathbf{u}, p) \in V \times Q$ such that

$$
\begin{equation}
a((\mathbf{u}, p), (\mathbf{v}, q)) = L((\mathbf{v}, q)) \quad \forall (\mathbf{v}, q) \in V \times Q
\end{equation}
$$

where bilinear and linear forms are, respectively

$$
\begin{equation}
    a((\mathbf{u}, p), (\mathbf{v}, q)) = \int_{\Omega} (\nabla^2 \mathbf{u} - \nabla p) \cdot {\mathbf{v}} \, dx_w + \int_{\Omega} \nabla \cdot \mathbf{u} \, {q} \, dx_w,
\end{equation}
$$
$$
\begin{equation}
    L((\mathbf{v}, q)) = \int_{\Omega} \mathbf{f} \cdot {\mathbf{v}}\, dx_w + \int_{\Omega} h {q} \, dx_w
\end{equation}
$$

Using integration by parts for Legendre (not really necessary, but looks nicer and more familiar:-)

$$
\begin{equation}
    a((\mathbf{u}, p), (\mathbf{v}, q)) = -\int_{\Omega} \nabla \mathbf{u} \cdot \nabla{\mathbf{v}} \, dx_w + \int_{\Omega} \nabla \cdot \mathbf{v} \, {p} \, dx_w + \int_{\Omega} \nabla \cdot \mathbf{u} \, {q} \, dx_w,
\end{equation}
$$

# Implementation of spaces, basis functions

```python
N = (40, 40)
D0X = FunctionSpace(N[0], 'Legendre', bc=(0, 0))  # For velocity components 0, 1
#D1Y = FunctionSpace(N[1], 'Legendre', bc=(0, 1)) # For velocity component 0
D1Y = FunctionSpace(N[1], 'Legendre', bc=(0, (1-x)**2*(1+x)**2)) # Regularized lid
D0Y = FunctionSpace(N[1], 'Legendre', bc=(0, 0))  # For velocity component 1
PX = FunctionSpace(N[0], 'Legendre')
PY = FunctionSpace(N[1], 'Legendre')

# All required spaces
V0 = TensorProductSpace(comm, (D0X, D1Y)) # velocity conponent 0
V1 = TensorProductSpace(comm, (D0X, D0Y)) # velocity component 1
Q = TensorProductSpace(comm, (PX, PY), modify_spaces_inplace=True)    # pressure
V = VectorSpace([V0, V1])     # Velocity vector (V0 x V1)
VQ = CompositeSpace([V, Q])      # V x Q

PX.slice = lambda: slice(0, PX.N-2) # For inf-sup
PY.slice = lambda: slice(0, PY.N-2) # For inf-sup

# All required test and trial functions
up = TrialFunction(VQ)
vq = TestFunction(VQ)
u, p = up
v, q = vq
```

# Implementation Stokes - matrices and solve

```python
# Assemble matrices
A = inner(grad(v), -grad(u))
G = inner(div(v), p)
D = inner(q, div(u))

# Create Block matrix solver
sol = la.BlockMatrixSolver(A+G+D)

# Add Functions to hold solution and rhs
up_hat = Function(VQ).set_boundary_dofs()
fh_hat = Function(VQ)

# Solve Stokes problem. Note constraint for pressure
up_hat = sol(fh_hat, u=up_hat, constraints=((2, 0, 0),))

# Move solution to Array in real space
up = up_hat.backward()
u_, p_ = up
```

```python
X = Q.local_mesh(True)
plt.quiver(X[0], X[1], u_[0], u_[1])
```

# Block matrix M
$$
M =
  \begin{bmatrix}
      A[0]+A[1] & 0 & G[0] \\
      0 & A[2]+A[3] & G[1] \\
      D[0] & D[1] & 0
  \end{bmatrix}
$$

where $D = G^T$ for the Legendre basis, making $M$ symmetric. For Chebyshev $M$ will not be symmetric.

Solver through [scipy.sparse.linalg](https://docs.scipy.org/doc/scipy/reference/sparse.linalg.html)

For Navier-Stokes of the lid-driven cavity, see https://github.com/spectralDNS/shenfun/blob/master/demo/NavierStokesDrivenCavity.py

# Sparsity pattern

$$
M =
  \begin{bmatrix}
      A[0]+A[1] & 0 & G[0] \\
      0 & A[2]+A[3] & G[1] \\
      D[0] & D[1] & 0
  \end{bmatrix}
$$

```python
%matplotlib notebook
plt.figure(figsize=(6,4))
plt.spy(sol.mat.diags(), markersize=0.5)
```

# Future work

- More linear algebra solvers. Most 1D bases are very good, but coupling two non-periodic directions demands more of
    - Block preconditioners
    - Uzawa solvers
    - matrix decomposition (some implemented)
    - MPI for all solvers
- Fast Legendre transforms
- Chebyshev-Legendre transforms
- Generalized Jacobi polynomials
- Cylindrical/circular domains
- Unbounded domains - Laguerre/Hermite recently added
    - Fast transforms
- Solvers for more than 2 non-periodic bases
- Domain decomposition
- Better documentation😃

# Thank you for your time😃