# Week 2: Linear Algebra II

# Tutorial 2

# [insert your name]

**Important reminders**: Before starting, click "File -> Save a copy in Drive". Produce a pdf for submission by "File -> Print" and then choose "Save to PDF".

To complete this tutorial, you should have watched Videos 2.1 through 2.6.

**Credits:**
The videos you watched for this week were from 3Blue1Brown. Some elements of this problem set are from or inspired by https://openedx.seas.gwu.edu/courses/course-v1:GW+EngComp4+2019/about. In particular, we are using their `plot_linear_transformation` and `plot_linear_transformations` functions, and the demonstration of the additional transformation of a matrix inverse (end of Exercise 2)

```
# Imports
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import scipy.linalg

# Plotting parameters
matplotlib.rcParams.update({'font.size': 22})

```

```
# @title Plotting functions
import numpy
from numpy.linalg import inv, eig
from math import ceil
from matplotlib import pyplot, ticker, get_backend, rc
from mpl_toolkits.mplot3d import Axes3D
from itertools import cycle

def plot_range_3_by_2_matrix(matrix):

    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    # Make data

    x = []
    y = []
    z = []
    for a1 in range(-10, 10, 1):
      for a2 in range(-10, 10, 1):
          vec = a1*matrix[:,0]+a2*matrix[:,1]
          x1, y1, z1 = vec
          x.append(x1)
          y.append(y1)
          z.append(z1)

    ax.scatter(np.array(x), np.array(y), np.array(z), color='b')

def plot_null_2_by_3_matrix(matrix):

    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    # Make data

    basis = scipy.linalg.null_space(matrix)
    x = []
    y = []
    z = []
    for a1 in range(-10, 10, 1):
      for a2 in range(-10, 10, 1):

          vec = a1*basis[:,0]
          if basis.shape[1]==2:
            vec += a2*basis[:,1]
          x1, y1, z1 = vec
          x.append(x1)
          y.append(y1)
          z.append(z1)

    ax.scatter(np.array(x), np.array(y), np.array(z), color='b')

_int_backends = ['GTK3Agg', 'GTK3Cairo', 'MacOSX', 'nbAgg',
                 'Qt4Agg', 'Qt4Cairo', 'Qt5Agg', 'Qt5Cairo',
                 'TkAgg', 'TkCairo', 'WebAgg', 'WX', 'WXAgg', 'WXCairo']
_backend = get_backend()   # get current backend name

# shrink figsize and fontsize when using %matplotlib notebook
if _backend in _int_backends:
    fontsize = 4
    fig_scale = 0.75
else:
    fontsize = 5
    fig_scale = 1

grey = '#808080'
gold = '#cab18c'   # x-axis grid
lightblue = '#0096d6'  # y-axis grid
green = '#008367'  # x-axis basis vector
red = '#E31937'    # y-axis basis vector
darkblue = '#004065'

pink, yellow, orange, purple, brown = '#ef7b9d', '#fbd349', '#ffa500', '#a35cff', '#731d1d'

quiver_params = {'angles': 'xy',
                 'scale_units': 'xy',
                 'scale': 1,
                 'width': 0.012}

grid_params = {'linewidth': 0.5,
               'alpha': 0.8}

def set_rc(func):
    def wrapper(*args, **kwargs):
        rc('font', family='serif', size=fontsize)
        rc('figure', dpi=200)
        rc('axes', axisbelow=True, titlesize=5)
        rc('lines', linewidth=1)
        func(*args, **kwargs)
    return wrapper

@set_rc
def plot_vector(vectors, tails=None):
    ''' Draw 2d vectors based on the values of the vectors and the position of their tails.

    Parameters
    ----------
    vectors : list.
        List of 2-element array-like structures, each represents a 2d vector.

    tails : list, optional.
        List of 2-element array-like structures, each represents the coordinates of the tail
        of the corresponding vector in vectors. If None (default), all tails are set at the
        origin (0,0). If len(tails) is 1, all tails are set at the same position. Otherwise,
        vectors and tails must have the same length.

    Examples
    --------
    >>> v = [(1, 3), (3, 3), (4, 6)]
    >>> plot_vector(v)      # draw 3 vectors with their tails at origin
    >>> t = [numpy.array((2, 2))]
    >>> plot_vector(v, t)   # draw 3 vectors with their tails at (2,2)
    >>> t = [[3, 2], [-1, -2], [3, 5]]
    >>> plot_vector(v, t)   # draw 3 vectors with 3 different tails

    '''
    vectors = numpy.array(vectors)
    assert vectors.shape[1] == 2, "Each vector should have 2 elements."
    if tails is not None:
        tails = numpy.array(tails)
        assert tails.shape[1] == 2, "Each tail should have 2 elements."
    else:
        tails = numpy.zeros_like(vectors)

    # tile vectors or tails array if needed
    nvectors = vectors.shape[0]
    ntails = tails.shape[0]
    if nvectors == 1 and ntails > 1:
        vectors = numpy.tile(vectors, (ntails, 1))
    elif ntails == 1 and nvectors > 1:
        tails = numpy.tile(tails, (nvectors, 1))
    else:
        assert tails.shape == vectors.shape, "vectors and tail must have a same shape"

    # calculate xlimit & ylimit
    heads = tails + vectors
    limit = numpy.max(numpy.abs(numpy.hstack((tails, heads))))
    limit = numpy.ceil(limit * 1.2)   # add some margins

    figsize = numpy.array([2,2]) * fig_scale
    figure, axis = pyplot.subplots(figsize=figsize)
    axis.quiver(tails[:,0], tails[:,1], vectors[:,0], vectors[:,1], color=darkblue,
                  angles='xy', scale_units='xy', scale=1)
    axis.set_xlim([-limit, limit])
    axis.set_ylim([-limit, limit])
    axis.set_aspect('equal')

    # if xticks and yticks of grid do not match, choose the finer one
    xticks = axis.get_xticks()
    yticks = axis.get_yticks()
    dx = xticks[1] - xticks[0]
    dy = yticks[1] - yticks[0]
    base = max(int(min(dx, dy)), 1)   # grid interval is always an integer
    loc = ticker.MultipleLocator(base=base)
    axis.xaxis.set_major_locator(loc)
    axis.yaxis.set_major_locator(loc)
    axis.grid(True, **grid_params)

    # show x-y axis in the center, hide frames
    axis.spines['left'].set_position('center')
    axis.spines['bottom'].set_position('center')
    axis.spines['right'].set_color('none')
    axis.spines['top'].set_color('none')

@set_rc
def plot_transformation_helper(axis, matrix, *vectors, unit_vector=True, unit_circle=False, title=None):
    """ A helper function to plot the linear transformation defined by a 2x2 matrix.

    Parameters
    ----------
    axis : class matplotlib.axes.Axes.
        The axes to plot on.

    matrix : class numpy.ndarray.
        The 2x2 matrix to visualize.

    *vectors : class numpy.ndarray.
        The vector(s) to plot along with the linear transformation. Each array denotes a vector's
        coordinates before the transformation and must have a shape of (2,). Accept any number of vectors.

    unit_vector : bool, optional.
        Whether to plot unit vectors of the standard basis, default to True.

    unit_circle: bool, optional.
        Whether to plot unit circle, default to False.

    title: str, optional.
        Title of the plot.

    """
    assert matrix.shape == (2,2), "the input matrix must have a shape of (2,2)"
    grid_range = 20
    x = numpy.arange(-grid_range, grid_range+1)
    X_, Y_ = numpy.meshgrid(x,x)
    I = matrix[:,0]
    J = matrix[:,1]
    X = I[0]*X_ + J[0]*Y_
    Y = I[1]*X_ + J[1]*Y_
    origin = numpy.zeros(1)

    # draw grid lines
    for i in range(x.size):
        axis.plot(X[i,:], Y[i,:], c=gold, **grid_params)
        axis.plot(X[:,i], Y[:,i], c=lightblue, **grid_params)

    # draw (transformed) unit vectors
    if unit_vector:
        axis.quiver(origin, origin, [I[0]], [I[1]], color=green, **quiver_params)
        axis.quiver(origin, origin, [J[0]], [J[1]], color=red, **quiver_params)

    # draw optional vectors
    color_cycle = cycle([pink, darkblue, orange, purple, brown])
    if vectors:
        for vector in vectors:
            color = next(color_cycle)
            vector_ = matrix @ vector.reshape(-1,1)
            axis.quiver(origin, origin, [vector_[0]], [vector_[1]], color=color, **quiver_params)

    # draw optional unit circle
    if unit_circle:
        alpha =  numpy.linspace(0, 2*numpy.pi, 41)
        circle = numpy.vstack((numpy.cos(alpha), numpy.sin(alpha)))
        circle_trans = matrix @ circle
        axis.plot(circle_trans[0], circle_trans[1], color=red, lw=0.8)

    # hide frames, set xlimit & ylimit, set title
    limit = 4
    axis.spines['left'].set_position('center')
    axis.spines['bottom'].set_position('center')
    axis.spines['left'].set_linewidth(0.3)
    axis.spines['bottom'].set_linewidth(0.3)
    axis.spines['right'].set_color('none')
    axis.spines['top'].set_color('none')
    axis.set_xlim([-limit, limit])
    axis.set_ylim([-limit, limit])
    if title is not None:
        axis.set_title(title)

@set_rc
def plot_linear_transformation(matrix, *vectors, unit_vector=True, unit_circle=False):
    """ Plot the linear transformation defined by a 2x2 matrix using the helper
    function plot_transformation_helper(). It will create 2 subplots to visualize some
    vectors before and after the transformation.

    Parameters
    ----------
    matrix : class numpy.ndarray.
        The 2x2 matrix to visualize.

    *vectors : class numpy.ndarray.
        The vector(s) to plot along with the linear transformation. Each array denotes a vector's
        coordinates before the transformation and must have a shape of (2,). Accept any number of vectors.

    unit_vector : bool, optional.
        Whether to plot unit vectors of the standard basis, default to True.

    unit_circle: bool, optional.
        Whether to plot unit circle, default to False.

    """
    figsize = numpy.array([4,2]) * fig_scale
    figure, (axis1, axis2) = pyplot.subplots(1, 2, figsize=figsize)
    plot_transformation_helper(axis1, numpy.identity(2), *vectors, unit_vector=unit_vector, unit_circle=unit_circle, title='Before transformation')
    plot_transformation_helper(axis2, matrix, *vectors, unit_vector=unit_vector, unit_circle=unit_circle, title='After transformation')

@set_rc
def plot_linear_transformations(*matrices, unit_vector=True, unit_circle=False):
    """ Plot the linear transformation defined by a sequence of n 2x2 matrices using the helper
    function plot_transformation_helper(). It will create n+1 subplots to visualize some
    vectors before and after each transformation.

    Parameters
    ----------
    *matrices : class numpy.ndarray.
        The 2x2 matrices to visualize. Accept any number of matrices.

    unit_vector : bool, optional.
        Whether to plot unit vectors of the standard basis, default to True.

    unit_circle: bool, optional.
        Whether to plot unit circle, default to False.

    """
    nplots = len(matrices) + 1
    nx = 2
    ny = ceil(nplots/nx)
    figsize = numpy.array([2*nx, 2*ny]) * fig_scale
    figure, axes = pyplot.subplots(nx, ny, figsize=figsize)

    for i in range(nplots):  # fig_idx
        if i == 0:
            matrix_trans = numpy.identity(2)
            title = 'Before transformation'
        else:
            matrix_trans = matrices[i-1] @ matrix_trans
            if i == 1:
                title = 'After {} transformation'.format(i)
            else:
                title = 'After {} transformations'.format(i)
        plot_transformation_helper(axes[i//nx, i%nx], matrix_trans, unit_vector=unit_vector, unit_circle=unit_circle, title=title)
    # hide axes of the extra subplot (only when nplots is an odd number)
    if nx*ny > nplots:
        axes[-1,-1].axis('off')

```

# Exercise 1: Don't be a square

In linear algebra, square matrices get a lot of the attention. We will often be dealing with m x n non-square matrices though, especially in neuroscience contexts! In this problem set, we'll explore nonsquare matrices a bit more.

## A) Rank & range if n < m

i) If we have a 4 x 2 matrix A, what is the maximum possible rank of this matrix? (Hint: think about the columns of A)

ii) What does this mean for the range of the associated linear transformation? What is the maximum dimensionality subspace is can be?

iii) If we have an m x n matrix where n < m, can we ever reach every vector in $R^m$?

**Your text answer**

Play around below with inputting a 3 x 2 matrix and seeing the resulting range/column space in $R^3$. In this plot, we're taking all vectors with elements between -10 and 10 in $R^2$ and plotting the transformed vector in $R^3$.

First test a random matrix. Then create a matrix that results in a 1D line.

```
matrix = ...

plot_range_3_by_2_matrix(matrix)
```

## B) Rank & range if m < n

i) If we have a 2 x 4 matrix A, what is the maximum possible rank of this matrix?

ii) What is the minimum possible dimension of the null space for this matrix? (Hint: think about how space squishes)

**Your text answer**

Below, we'll visualize the null space of a 2 x 3 matrix. Try out a random matrix and see what happens! Can you figure out how to make the null space 2D (Hint: take a look at Extra info: The Rank Theorem)?

```
matrix = ...
plot_null_2_by_3_matrix(matrix)
```

## Extra info: Inverses of nonsquare matrices

Nonsquare matrices do not have inverses. By definition, for matrix A, an inverse of A exists only if $A^{-1}A = AA^{-1} = I_n$. If you have a nonsquare matrix A, you can never get the dimensions of $A^{-1}$ to work.

Left and right inverses of nonsquare matrices do exist. If $A$ is m x n and has rank equal to n, $A$ has a left inverse $B$ such that $BA = I_n$, where $B$ is n x m. If $A$ has rank m, then is has a right inverse $C$ such that $AC = I_m$, where C is n x m.

Thinking through why the rank conditions in this definition are what they are is difficult but will help consolidate your knowledge if you want to try! Think about going from larger to smaller spaces vs smaller to larger spaces

## Extra info: Determinants of nonsquare matrices

Nonsquare matrices do not have determinants. Intuitively, determinants denote how much the associated linear transformation changes the area or volume of space covered. Because nonsquare matrices are transforming between dimensions, you cannot really compute the area changes since it changes dimensions (how do you compute the difference between an area and a volume).

## Extra info: The Rank Theorem

There is a theorem that says for an m x n matrix $A$:

$$rank A + dim Nul A = n$$

Basically the rank of a matrix plus the dimensionality of the null space must add up to the number of columns. We will see why this is true next week once we learn briefly how to solve matrix-vector equations by hand.

# (Optional) Exercise 2: Changing basis

In Video 1.3: Vector Spaces, we talked about changing basis (or changing coordinate systems). We looked at transforming a vector from a basis $\beta = \{\bar{b}_1, \bar{b}_2\}$ where $\bar{b}_1 = \begin{bmatrix}
1 \\
1 \\
\end{bmatrix}$ and $\bar{b}_2 = \begin{bmatrix}
-1 \\
1 \\
\end{bmatrix}$ to the standard basis. In particular, we had the $\beta$ coordinates of vector $\bar{a}$ (coordinates of a vector with respect to basis $\beta$) as $[\bar{a}]_{\beta} = \begin{bmatrix} 2 \\ 1 \\ \end{bmatrix}$.  We computed the standard coordinates as:

$$ \begin{align} \bar{a} &= 2 \bar{b}_1 + \bar{b}_2 \\
&= \begin{bmatrix}
1 \\
3\\
\end{bmatrix}
 \end{align}$$

## A) Changing to the standard basis
 Now that we have learned about matrix multiplication, we can figure out how to change basis using it!  We want to write a matrix P so that:

 $$ \bar{x} = P_{\beta}[\bar{x}]_{\beta}$$

 In other words, a matrix that transforms from the vector in coordinates relative to $\beta$ to one with the coordinates relative to the standard basis. $P_{\beta}$ is called the change-of-coordinates matrix from $\beta$ to the standard basis.

What is $P_{\beta}$ for the basis given above?

 Hints:
 For the specific vector example,

 $$\begin{bmatrix}
1 \\
3\\
\end{bmatrix} = P_{\beta}\begin{bmatrix}
2 \\
1\\
\end{bmatrix} $$

You could find it by looking at the computation we did above and thinking how that could have been accomplished by a matrix multiplication or by thinking about what we learned this week: that matrices represent linear transformations and each column corresponds to where the corresponding basis vector ends up after the transformation.

**Your math answer**

Fill in $P_{\beta}$ below and plot the resulting linear transformation. This visualization highlights something essential: when we are in the basis $\beta$  the $\beta$ basis vectors seem like standard basis vectors. The Before transformation is when we are in "$\beta-land$" - it looks like when we are in standard basis vector land. It is only after we transform to standard coordinates, that we see the $\beta$ vectors defined with respect to them (After transformation).

```
p_beta = ...

plot_linear_transformation(p_beta)
```

## B) Changing from the standard basis

Luckily for us, the matrix $P_{\beta}$ is always invertible. We know this from the Invertible Matrix Theorem presented in Part 1 because the columns of $P_{\beta}$ span $R^n$ (since they are a basis for $R^n$). So we can compute the coordinates of a vector with respect to $\beta$ as:

 $$ [\bar{x}]_{\beta} = P^{-1}_{\beta}\bar{x}$$

 We will compute $P^{-1}_{\beta}$ using code and find the $\beta$ coordinates of $\bar{x} = \begin{bmatrix}
4 \\
2\\
\end{bmatrix}$

```
p_beta_inverse = ...

beta_coordinates = ...

print(beta_coordinates)
```

## C) Changing basis criterion

Not every matrix-vector multiplication can be considered a change of basis. What has to be true of the matrix for it to be a change of basis?

**Your text answer**

## Extra info: Changing between two non-standard bases

We can also change from basis $\beta$ to another basis $C$ without first moving to the standard basis. We will not get into this here due to time constraints but you should have all the tools to understand the explanation here if you wish: https://math.berkeley.edu/~arash/54/notes/04_07.pdf


#  CS-109A Introduction to Data Science

## Lab 11: Neural Network Basics - Introduction to `tf.keras`

**Harvard University**

**Fall 2019**

**Instructors:** Pavlos Protopapas, Kevin Rader, Chris Tanner

**Lab Instructors:** Chris Tanner and Eleni Kaxiras.

**Authors:** Eleni Kaxiras, David Sondak, and Pavlos Protopapas.

```python
## RUN THIS CELL TO PROPERLY HIGHLIGHT THE EXERCISES
import requests
from IPython.core.display import HTML
styles = requests.get("https://raw.githubusercontent.com/Harvard-IACS/2018-CS109A/master/content/styles/cs109.css").text
HTML(styles)
```

```python
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import numpy as np
import pandas as pd
%matplotlib inline

from PIL import Image
```

```python
from __future__ import absolute_import, division, print_function, unicode_literals

# TensorFlow and tf.keras
import tensorflow as tf

tf.keras.backend.clear_session()  # For easy reset of notebook state.

print(tf.__version__)  # You should see a 2.0.0 here!
```

    2.0.0

#### Instructions for running `tf.keras` with Tensorflow 2.0:

1. Create a `conda` virtual environment by cloning an existing one that you know works
```
conda create --name myclone --clone myenv
```

2. Go to [https://www.tensorflow.org/install/pip](https://www.tensorflow.org/install/pip) and follow instructions for your machine.

3. In a nutshell:
```
pip install --upgrade pip
pip install tensorflow==2.0.0
```
All references to Keras should be written as `tf.keras`.  For example:

```
model = tf.keras.models.Sequential([
  tf.keras.layers.Flatten(input_shape=(28, 28)),
  tf.keras.layers.Dense(128, activation='relu'),
  tf.keras.layers.Dropout(0.2),
  tf.keras.layers.Dense(10, activation='softmax')
])

model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

tf.keras.models.Sequential
tf.keras.layers.Dense, tf.keras.layers.Activation,
tf.keras.layers.Dropout, tf.keras.layers.Flatten, tf.keras.layers.Reshape
tf.keras.optimizers.SGD
tf.keras.preprocessing.image.ImageDataGenerator
tf.keras.regularizers
tf.keras.datasets.mnist
```

You could avoid the long names by using
```
from tensorflow import keras
from tensorflow.keras import layers
```
These imports do not work on some systems, however, because they pick up previous versions of `keras` and `tensorflow`. That is why I avoid them in this lab.

## Learning Goals
In this lab we will understand the basics of neural networks and how to start using a deep learning library called `keras`. By the end of this lab, you should:

- Understand how a simple neural network works and code some of its functionality from scratch.
- Be able to think and do calculations in matrix notation. Also think of vectors and arrays as tensors.
- Know how to install and run `tf.keras`.
- Implement a simple real world example using a neural network.

## Part 1: Neural Networks 101

Suppose we have an input vector $X=${$x_1, x_2, ... x_L$} to a $k$-layered network.

Each layer has its own number of nodes. For the first layer in our drawing that number is $J$. We can store the weights for each node in a vector $\mathbf{W} \in \mathbb{R}^{JxL+1}$ (accounting for bias).  Similarly, we can store the biases from each node in a vector $\mathbf{b} \in \mathbb{R}^{I}$.  The affine transformation is then written as $$\mathbf{a} = \mathbf{W^T}X + \mathbf{b}$$
  What we then do is "absorb" $\mathbf{b}$ into $X$ by adding a column of ones to $X$. Our $X$ matrix than becomes $\mathbf{X} \in \mathbb{R}^{JxL+1}$ and our equation:

$$\mathbf{a} = \mathbf{W^T}_{plusones}X$$
We have that $\mathbf{a} \in \mathbb{R}^{J}$ as well.  Next we evaluate the output from each node.  We write $$\mathbf{u} = \sigma\left(\mathbf{a}\right)$$ where $\mathbf{u}\in\mathbb{R}^{J}$.  We can think of $\sigma$ operating on each individual element of $\mathbf{a}$ separately or in matrix notation. If we denote each component of $\mathbf{a}$ by $a_{j}$ then we can write $$u_{j} = \sigma\left(a_{j}\right), \quad j = 1, ... J.$$
 In our code we will implement all these equations in matrix notation.
`tf.keras` (Tensorflow) and `numpy` perform the calculations in matrix format.

*Image source: "Modern Mathematical Methods for Computational Science and Engineering" Efthimios Kaxiras and Athanassios Fokas.*

Let's assume that we have 3 input points (L = 3), two hidden layers ($k=2$), and 2 nodes in each layer ($J=2$)

### Input Layer

$𝑋$={$𝑥_1,𝑥_2,x_3$}

### First Hidden Layer

\begin{equation}
 \begin{aligned}
a^{(1)}_1 = w^{(1)}_{10} + w^{(1)}_{11}x_1 + w^{(1)}_{12}x_2 + w^{(1)}_{13}x_3 \\
a^{(1)}_2 = w^{(1)}_{20} + w^{(1)}_{21}x_1 + w^{(1)}_{22}x_2 + w^{(1)}_{23}x_3 \\
\end{aligned}
\end{equation}

 All this in matrix notation: $$\mathbf{a} = \mathbf{W^T}X$$

 NOTE: in $X$ we have added a column of ones to account for the bias

**Then the sigmoid is applied**:
\begin{equation}
 \begin{aligned}
u^{(1)}_1 = \sigma(a^{(1)}_1) \\
u^{(1)}_2 = \sigma(a^{(1)}_2) \\
\end{aligned}
\end{equation}

or in matrix notation: $$\mathbf{u} = \sigma\left(\mathbf{a}\right)$$

### Second Hidden Layer

\begin{equation}
 \begin{aligned}
a^{(2)}_1 = w^{(2)}_{10} + w^{(2)}_{11}u^{(1)}_1 + w^{(2)}_{12}u^{(1)}_2 + w^{(2)}_{13}u^{(1)}_3 \\
a^{(2)}_2 = w^{(2)}_{20} + w^{(2)}_{21}u^{(1)}_1 + w^{(2)}_{22}u^{(1)}_2 + w^{(2)}_{23}u^{(1)}_3 \\
\end{aligned}
\end{equation}

**Then the sigmoid is applied**:

\begin{equation}
 \begin{aligned}
u^{(2)}_1 = \sigma(a^{(2)}_1) \\
u^{(2)}_2 = \sigma(a^{(2)}_2) \\
\end{aligned}
\end{equation}

### Output Layer

#### If the output is categorical:

For example with four classes ($M=4$): $Y$={$y_1, y_2, y_3, y_4$}, we have the affine and then the sigmoid is lastly applied:

\begin{equation}
 \begin{aligned}
a^{(3)}_1 = w^{(3)}_{10} + w^{(3)}_{11}u^{(2)}_1 + w^{(3)}_{12}u^{(2)}_2 \\
a^{(3)}_2 = w^{(3)}_{20} + w^{(3)}_{21}u^{(2)}_1 + w^{(3)}_{22}u^{(2)}_2 \\
a^{(3)}_3 = w^{(3)}_{30} + w^{(3)}_{31}u^{(2)}_1 + w^{(3)}_{32}u^{(2)}_2 \\
a^{(3)}_4 = w^{(3)}_{40} + w^{(3)}_{41}u^{(2)}_1 + w^{(3)}_{42}u^{(2)}_2 \\
\end{aligned}
\end{equation}

\begin{equation}
 \begin{aligned}
y_1 = \sigma(a^{(3)}_1) \\
y_2 = \sigma(a^{(3)}_2) \\
y_3 = \sigma(a^{(3)}_3) \\
y_3 = \sigma(a^{(3)}_4) \\
\end{aligned}
\end{equation}
$\sigma$ will be softmax in the case of multiple classes and sigmoid for binary.

#### If the output is a number (regression):

We have a single y as output:

\begin{equation}
 \begin{aligned}
y = w^{(3)}_{10}+ w^{(3)}_{11}u^{(2)}_1 + w^{(3)}_{12}u^{(2)}_2 + w^{(3)}_{13}u^{(2)}_3 \\
\end{aligned}
\end{equation}

#### Matrix Multiplication and constant addition

```python
# make two arrays and multiply them using np.dot(a, b) or tf.matmul(a, b)

a = np.array([[2,3],[4,5],[6,7]])
b = np.array([[8,9,10],[11,12,13]])

print(a)
print(b)
```

    [[2 3]
     [4 5]
     [6 7]]
    [[ 8  9 10]
     [11 12 13]]

```python
# both Tensorflow and numpy take care of transposing.
print(np.dot(a,b))
print(tf.matmul(a,b))
```

    [[ 49  54  59]
     [ 87  96 105]
     [125 138 151]]
    tf.Tensor(
    [[ 49  54  59]
     [ 87  96 105]
     [125 138 151]], shape=(3, 3), dtype=int64)

```python
# how do we add the constant in the matrix - append column of ones
a = [[1,0],[0,1]]
ones = np.ones((len(a),1))
a = np.append(a, ones, axis=1)
a
```

    array([[1., 0., 1.],
           [0., 1., 1.]])

In class exercise : Plot the sigmoid

Define the `sigmoid` and the `tanh`. For `tanh` you may use `np.tanh` and for the `sigmoid` use the general equation:
\begin{align}
\sigma = \dfrac{1}{1+e^{-2(x-c)/a}} \qquad\text{(1.1)}
\textrm{}
\end{align}

Generate a list of 500 $x$ points from -5 to 5 and plot both functions. What do you observe? What do variables $c$ and $a$ do?

```python
# your code here
x = np.linspace(-5, 5, 500)

def sigmoid(x, c, a):
    z = ((x-c)/a)
    return 1/(1+np.exp(-z))

c = 1
a = 0.5

plt.plot(x, sigmoid(x,c,a), label = 'sigmoid')
plt.plot(x, np.tanh(x), label = 'tanh')
plt.legend();
plt.grid();
```

```python
# %load solutions/sigmoid.py
```

2. In class exercise: Approximate a Gaussian function using a node and manually adjusting the weights. Start with one layer with one node and move to two nodes.

The task is to approximate (learn) a function $f\left(x\right)$ given some input $x$.  For demonstration purposes, the function we will try to learn is a Gaussian function:
\begin{align}
f\left(x\right) = e^{-x^{2}}
\textrm{}
\end{align}

Even though we represent the input $x$ as a vector on the computer, you should think of it as a single input.

#### 2.1 Start by plotting the above function using the $x$ dataset

```python
x = np.linspace(-5.0, 5.0, 500) # input points
def gaussian(x):
    return np.exp(-x*x)

f = gaussian(x)
plt.plot(x, f, label='gaussian')
plt.legend()
```

```python
f.shape
```

    (500,)

#### 2.2 Now, let's code the single node as per the image above.

Write a function named `affine` that does the transformation. The definition is provided below. Then create a simpler sigmoid with just one variable. We choose a **sigmoid** activation function and specifically the **logistic** function.  Sigmoids are a family of functions and the logistic function is just one member in that family. $$\sigma\left(z\right) = \dfrac{1}{1 + e^{-z}}.$$

Define both functions in code.

```python
def affine(x, w, b):
    """Return affine transformation of x

    INPUTS
    ======
    x: A numpy array of points in x
    w: An array representing the weight of the perceptron
    b: An array representing the biases of the perceptron

    RETURN
    ======
    z: A numpy array of points after the affine transformation
       z = wx + b
    """

    # Code goes here
    z = w*x + b
    return z
```

```python
# your code here

```

```python
# %load solutions/affine-sigmoid.py
def affine(x, w, b):
    return w * x + b

def sigmoid(z):
    return 1.0 / (1.0 + np.exp(-z))
```

And now we plot the activation function and the true function. What do you think will happen if you change $w$ and $b$?

```python
w = [2., 0.5, -2.] # Create a list of weights to try
b = [2., 0.1, 2.] # Create a list of biases

fig, ax = plt.subplots(1,1, figsize=(9,5))
SIZE = 16

# plot our true function, the gaussian
ax.plot(x, f, lw=4, ls='-.', label='True function')

# plot 3 "networks"
for wi, bi in zip(w, b):
    h = sigmoid(affine(x, wi, bi))
    ax.plot(x, h, lw=4, label=r'$w = {0}$, $b = {1}$'.format(wi,bi))

ax.set_title('Single neuron network', fontsize=SIZE)

# Create labels (very important!)
ax.set_xlabel('$x$', fontsize=SIZE) # Notice we make the labels big enough to read
ax.set_ylabel('$y$', fontsize=SIZE)

ax.tick_params(labelsize=SIZE) # Make the tick labels big enough to read

ax.legend(fontsize=SIZE, loc='best') # Create a legend and make it big enough to read
```

We didn't do an exhaustive search of the weights and biases, but it sure looks like this single perceptron is never going to match the actual function.  Again, we shouldn't be suprised about this.  The output layer of the network is simple the logistic function, which can only have so much flexibility.

Let's try to make our network more flexible by using **more nodes**!

### Multiple Perceptrons in a Single Layer
It appears that a single neuron is somewhat limited in what it can accomplish.  What if we expand the number of nodes/neurons in our network?  We have two obvious choices here.  One option is to add depth to the network by putting layers next to each other.  The other option is to stack neurons on top of each other in the same layer.  Now the network has some width, but is still only one layer deep.

```python
x = np.linspace(-5.0, 5.0, 500) # input points
f = np.exp(-x*x) # data

w = np.array([2.5, -2.5]) # ENTER TWO VALUES HERE
b = np.array([2, 2]) # HERE TOO

# Affine transformations
z1 = w[0] * x + b[0]
z2 = w[1] * x + b[1]

# Node outputs
h1 = sigmoid(z1)
h2 = sigmoid(z2)
```

Now let's plot things and see what they look like.

```python
fig, ax = plt.subplots(1,1, figsize=(9,5))

ax.plot(x, f, lw=4, ls = '-.', label='True function')
ax.plot(x, h1, lw=4, label='First neuron')
ax.plot(x, h2, lw=4, label='Second neuron')

# Set title
ax.set_title('Comparison of Neuron Outputs', fontsize=SIZE)

# Create labels (very important!)
ax.set_xlabel('$x$', fontsize=SIZE) # Notice we make the labels big enough to read
ax.set_ylabel('$y$', fontsize=SIZE)

ax.tick_params(labelsize=SIZE) # Make the tick labels big enough to read

ax.legend(fontsize=SIZE, loc='best') # Create a legend and make it big enough to read
```

Just as we expected.  Some sigmoids.  Of course, to get the network prediction we must combine these two sigmoid curves somehow.  First we'll just add $h_{1}$ and $h_{2}$ without any weights to see what happens.

#### Note
We are **not** doing classification here.  We are trying to predict an actual function.  The sigmoid activation is convenient when doing classification because you need to go from $0$ to $1$.  However, when learning a function, we don't have as good of a reason to choose a sigmoid.

```python
# Network output
wout = np.ones(2) # Set the output weights to unity to begin
bout = -1 # bias
yout = wout[0] * h1 + wout[1] * h2 + bout
```

And plot.

```python
fig, ax = plt.subplots(1,1, figsize=(9,5))

ax.plot(x, f, ls='-.', lw=4, label=r'True function')
ax.plot(x, yout, lw=4, label=r'$y_{out} = h_{1} + h_{2}$')

# Create labels (very important!)
ax.set_xlabel('$x$', fontsize=SIZE) # Notice we make the labels big enough to read
ax.set_ylabel('$y$', fontsize=SIZE)

ax.tick_params(labelsize=SIZE) # Make the tick labels big enough to read

ax.legend(fontsize=SIZE, loc='best') # Create a legend and make it big enough to read
```

Very cool!  The two nodes interact with each other to produce a pretty complicated-looking function.  It still doesn't match the true function, but now we have some hope.  In fact, it's starting to look a little bit like a Gaussian!

We can do better.  There are three obvious options at this point:
1. Change the number of nodes
2. Change the activation functions
3. Change the weights

#### We will leave this simple example for some other time! Let's move on to fashion items!

## Part 2: Tensors, Fashion, and Reese Witherspoon

We can think of tensors as multidimensional arrays of real numerical values; their job is to generalize matrices to multiple dimensions. While tensors first emerged in the 20th century, they have since been applied to numerous other disciplines, including machine learning. Tensor decomposition/factorization can solve, among other, problems in unsupervised learning settings, temporal and multirelational data. For those of you that will get to handle images for Convolutional Neural Networks, it's a good idea to have the understanding of tensors of rank 3.

We will use the following naming conventions:

- scalar = just a number = rank 0 tensor  ($a$ ∈ $F$,)

- vector = 1D array = rank 1 tensor ( $x = (\;x_1,...,x_i\;)⊤$ ∈ $F^n$ )

- matrix = 2D array = rank 2 tensor ( $\textbf{X} = [a_{ij}] ∈ F^{m×n}$ )

- 3D array = rank 3 tensor ( $\mathscr{X} =[t_{i,j,k}]∈F^{m×n×l}$ )

- $\mathscr{N}$D array = rank $\mathscr{N}$ tensor ( $\mathscr{T} =[t_{i1},...,t_{i\mathscr{N}}]∈F^{n_1×...×n_\mathscr{N}}$ ) <-- Things start to get complicated here...

#### Tensor indexing
We can create subarrays by fixing some of the given tensor’s indices. We can create a vector by fixing all but one index. A 2D matrix is created when fixing all but two indices. For example, for a third order tensor the vectors are

$\mathscr{X}[:,j,k]$ = $\mathscr{X}[j,k]$ (column),

$\mathscr{X}[i,:,k]$ = $\mathscr{X}[i,k]$ (row), and

$\mathscr{X}[i,j,:]$ = $\mathscr{X}[i,j]$ (tube)

#### Tensor multiplication
We can multiply one matrix with another as long as the sizes are compatible ((n × m) × (m × p) = n × p), and also multiply an entire matrix by a constant. Numpy `numpy.dot` performs a matrix multiplication which is straightforward when we have 2D or 1D arrays. But what about > 3D arrays? The function will choose according to the matching dimentions but if we want to choose we should use `tensordot`, but, again, we **do not need tensordot** for this class.

### Reese Witherspoon

This image is from the dataset [Labeled Faces in the Wild](http://vis-www.cs.umass.edu/lfw/person/Reese_Witherspoon.html) used for machine learning training. Images are 24-bit RGB images (height, width, channels) with 8 bits for each of R, G, B channel. Explore and print the array.

```python
# load and show the image
FILE = '../fig/Reese_Witherspoon.jpg'
#img = np.array(Image.open(FILE)) # this works as well
img = mpimg.imread(FILE)
imgplot = plt.imshow(img)
```

```python
print(f'The image is a: {type(img)} of shape {img.shape}')
img[3:5, 3:5, :]
```

    The image is a: <class 'numpy.ndarray'> of shape (150, 150, 3)

    array([[[241, 241, 241],
            [242, 242, 242]],

           [[241, 241, 241],
            [242, 242, 242]]], dtype=uint8)

#### Slicing tensors: slice along each axis

```python
# we want to show each color channel
fig, axes = plt.subplots(1, 3, figsize=(10,10))
for i, subplot in zip(range(3), axes):
    temp = np.zeros(img.shape, dtype='uint8')
    temp[:,:,i] = img[:,:,i]
    subplot.imshow(temp)
    subplot.set_axis_off()
plt.show()
```

#### Multiplying Images with a scalar (just for fun, does not really help us in any way)

```python
temp = img
temp = temp * 2
plt.imshow(temp)
```

For more on image manipulation by `matplotlib` see: [matplotlib-images](https://matplotlib.org/3.1.1/tutorials/introductory/images.html)

### Anatomy of an Artificial Neural Network

In Part 1 we hand-made a neural network by writing some simple python functions.  We focused on a regression problem where we tried to learn a function. We practiced using the logistic activation function in a network with multiple nodes, but a single or two hidden layers.  Some of the key observations were:
* Increasing the number of nodes allows us to represent more complicated functions
* The weights and biases have a very big impact on the solution
* Finding the "correct" weights and biases is really hard to do manually
* There must be a better method for determining the weights and biases automatically

We also didn't assess the effects of different activation functions or different network depths.

###
https://www.tensorflow.org/guide/keras

`tf.keras` is TensorFlow's high-level API for building and training deep learning models. It's used for fast prototyping, state-of-the-art research, and production. `Keras` is a library created by François Chollet. After Google released Tensorflow 2.0, the creators of `keras` recommend that "Keras users who use multi-backend Keras with the TensorFlow backend switch to `tf.keras` in TensorFlow 2.0. `tf.keras` is better maintained and has better integration with TensorFlow features".

#### IMPORTANT:  In `Keras` everything starts with a Tensor of N samples as input and ends with a Tensor of N samples as output.

### The 3 parts of an ANN

- **Part 1: the input layer** (our dataset)
- **Part 2: the internal architecture or hidden layers** (the number of layers, the activation functions, the learnable parameters and other hyperparameters)
- **Part 3: the output layer** (what we want from the network)

In the rest of the lab we will practice with end-to-end neural network training

1. Load the data
2. Define the layers of the model.
3. Compile the model.
4. Fit the model to the train set (also using a validation set).
5. Evaluate the model on the test set.
6. Plot metrics such as accuracy.
7. Predict on random images from test set.
8. Predict on  a random image from the web!

```python
seed = 7
np.random.seed(seed)
```

### Fashion MNIST

MNIST, the set of handwritten digits is considered the Drosophila of Machine Learning. It has been overused, though, so we will try a slight modification to it.

**Fashion-MNIST** is a dataset of clothing article images (created by [Zalando](https://github.com/zalandoresearch/fashion-mnist)), consisting of a training set of 60,000 examples and a test set of 10,000 examples. Each example is a **28 x 28** grayscale image, associated with a label from **10 classes**. The creators intend Fashion-MNIST to serve as a direct drop-in replacement for the original MNIST dataset for benchmarking machine learning algorithms. It shares the same image size and structure of training and testing splits. Each pixel is 8 bits so its value ranges from 0 to 255.

Let's load and look at it!

N.B. https://medium.com/tensorflow/hello-deep-learning-fashion-mnist-with-keras-50fcff8cd74a

#### 1. Load the data

```python
%%time
# get the data from keras

# load the data splitted in train and test! how nice!
(x_train, y_train), (x_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()

# normalize the data by dividing with pixel intensity
# (each pixel is 8 bits so its value ranges from 0 to 255)
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255

# classes are named 0-9 so define names for plotting clarity
class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']

# plot
plt.figure(figsize=(10,10))
for i in range(25):
    plt.subplot(5,5,i+1)
    plt.xticks([])
    plt.yticks([])
    plt.grid(False)
    plt.imshow(x_train[i], cmap=plt.cm.binary)
    plt.xlabel(class_names[y_train[i]])
plt.show()
```

```python
plt.imshow(x_train[3], cmap=plt.cm.binary)
```

```python
x_train.shape, x_test.shape
```

    ((60000, 28, 28), (10000, 28, 28))

```python
y_train.shape
```

    (60000,)

#### 2. Define the layers of the model.

```python

```

#### 3. Compile the model

```python

```

```python
model.summary()
```

    Model: "sequential"
    _________________________________________________________________
    Layer (type)                 Output Shape              Param #
    =================================================================
    flatten (Flatten)            (None, 784)               0
    _________________________________________________________________
    dense (Dense)                (None, 154)               120890
    _________________________________________________________________
    dense_1 (Dense)              (None, 64)                9920
    _________________________________________________________________
    dropout (Dropout)            (None, 64)                0
    _________________________________________________________________
    dense_2 (Dense)              (None, 10)                650
    =================================================================
    Total params: 131,460
    Trainable params: 131,460
    Non-trainable params: 0
    _________________________________________________________________

```python
tf.keras.utils.plot_model(
    model,
    #to_file='model.png', # if you want to save the image
    show_shapes=True, # True for more details than you need
    show_layer_names=True,
    rankdir='TB',
    expand_nested=False,
    dpi=96
)
```

[Everything you wanted to know about a Keras Model and were afraid to ask](https://www.tensorflow.org/api_docs/python/tf/keras/Model)

#### 4. Fit the model to the train set (also using a validation set)

This is the part that takes the longest.

-----------------------------------------------------------
**ep·och**

noun: epoch; plural noun: epochs. A period of time in history or a person's life, typically one marked by notable events or particular characteristics. Examples: "the Victorian epoch", "my Neural Netwok's epochs".

-----------------------------------------------------------

```python
%%time

# the core of the network training

```

#### Save the model

You can save the model so you do not have `.fit` everytime you reset the kernel in the notebook. Network training is expensive!

For more details on this see [https://www.tensorflow.org/guide/keras/save_and_serialize](https://www.tensorflow.org/guide/keras/save_and_serialize)

```python
# save the model so you do not have to run the code everytime
model.save('fashion_model.h5')

# Recreate the exact same model purely from the file
#model = tf.keras.models.load_model('fashion_model.h5')
```

#### 5. Evaluate the model on the test set.

```python
test_loss, test_accuracy = model.evaluate(x_test, y_test, verbose=0)
print(f'Test accuracy={test_accuracy}')
```

    Test accuracy=0.8866999745368958

#### 6. We learn a lot by studying History! Plot metrics such as accuracy.

You can learn a lot about neural networks by observing how they perform while training. You can issue `kallbacks` in `keras`. The networks's performance is stored in a `keras` callback aptly named `history` which can be plotted.

```python
print(history.history.keys())
```

    dict_keys(['loss', 'accuracy', 'val_loss', 'val_accuracy'])

```python
# plot accuracy and loss for the test set
fig, ax = plt.subplots(1,2, figsize=(20,6))

ax[0].plot(history.history['accuracy'])
ax[0].plot(history.history['val_accuracy'])
ax[0].set_title('Model accuracy')
ax[0].set_ylabel('accuracy')
ax[0].set_xlabel('epoch')
ax[0].legend(['train', 'val'], loc='best')

ax[1].plot(history.history['loss'])
ax[1].plot(history.history['val_loss'])
ax[1].set_title('Model loss')
ax[1].set_ylabel('loss')
ax[1].set_xlabel('epoch')
ax[1].legend(['train', 'val'], loc='best')
```

#### 7. Now let's use the Network for what it was meant to do: Predict!

```python
predictions = model.predict(x_test)
```

```python
predictions[0]
```

```python
np.argmax(predictions[0]), class_names[np.argmax(predictions[0])]
```

Let's see if our network predicted right! Is the first item what was predicted?

```python
plt.figure()
plt.imshow(x_test[0], cmap=plt.cm.binary)
plt.xlabel(class_names[y_test[0]])
plt.colorbar()
```

**Correct!!** Now let's see how confident our model is by plotting the probability values:

```python
# code source: https://www.tensorflow.org/tutorials/keras/classification
def plot_image(i, predictions_array, true_label, img):
    predictions_array, true_label, img = predictions_array, true_label[i], img[i]
    plt.grid(False)
    plt.xticks([])
    plt.yticks([])

    plt.imshow(img, cmap=plt.cm.binary)

    predicted_label = np.argmax(predictions_array)
    if predicted_label == true_label:
        color = 'blue'
    else:
        color = 'red'

    plt.xlabel("{} {:2.0f}% ({})".format(class_names[predicted_label],
                                100*np.max(predictions_array),
                                class_names[true_label]),
                                color=color)

def plot_value_array(i, predictions_array, true_label):
    predictions_array, true_label = predictions_array, true_label[i]
    plt.grid(False)
    plt.xticks(range(10))
    plt.yticks([])
    thisplot = plt.bar(range(10), predictions_array, color="#777777")
    plt.ylim([0, 1])
    predicted_label = np.argmax(predictions_array)

    thisplot[predicted_label].set_color('red')
    thisplot[true_label].set_color('blue')
```

```python
i = 406
plt.figure(figsize=(6,3))
plt.subplot(1,2,1)
plot_image(i, predictions[i], y_test, x_test)
plt.subplot(1,2,2)
plot_value_array(i, predictions[i],  y_test)
plt.show()
```

#### 8. Predicting in the real world

Let's see if our network can generalize beyond the MNIST fashion dataset. Let's give it an random googled image of a boot. Does it have to be a clothing item resembling the MNIST fashion dataset? Can it be a puppy?

**Download an image from the internet and resize it to 28x28.**

```python
# your code here.
# Resize it to 28 x 28 and one channel (you could do this outside the notebook)
# make into one channel and see .shape

```

`tf.keras` models are optimized to make predictions on a batch, or collection, of examples at once. Accordingly, even though you're using a single image, you need to add it to a list:

```python
# your code here
# Add the image to a batch where it's the only member.

```

```python
# your code here
# print the prediction

```


# Computational Inquiry: How to Trust a Machine

In CMSE 202 (and 201 before it), you've been building a foundation of experience with fundamental tools and techniques essential to computational scientists.  Even if your interests don't end up taking you to an explicitly computational field of work or study, we still live in a computational world.  Whatever your path, the experience you're gaining better equips you to succeed in such a world.

This notebook introduces an activity that adds another brick to the foundation you're building.  You'll likely find it to be a bit different from your other CMSE classroom experiences.  That's because you'll be engaging in an _inquiry-based activity_.  The goal is to learn through inquiry and investigation, similar to the way science is done.  While it may feel a bit different, some of the scholars and scientists that study how people learn find it to be powerfully effective.  You'll soon be able to judge for yourself, but we hope you'll approach this activity with an open mind and a willingness to try something a bit different.

Alright, alright, so what are we actually trying to learn here? What tools and techniques will you get to play with?  I'm glad you asked.

This notebook focuses on introducing two concepts: **compartmental models** (which are a framework for describing a set of **ordinary differential equations (ODEs)**) and **finite differences**.  Now, you could easily spend up to an entire course on these.  The point here is to give the class just enough of a shared understanding of these concepts such that you can deploy them in your inquiry.  Some of you may already have some experience with these or related concepts, in which case this notebook can refresh and reinforce.  In either case, the plan for this activity is not to go into all the details and nuances, but to learn a bit and get some practical experience.  If your calculus is rusty or non-existent, there's no need to worry.  While we'll be playing with calculus concepts like differential equations and derivatives, neither past knowledge of them nor a calculus background is required for this activity.

That's because the focus of this activity is to develop practical, technical experience with the computational treatment of these concepts.  Even more, you're going to wrestle with the fundamental question of **how can we trust a machine?**.  What are some of the steps we can take, some of the analysis we can do, to convince ourselves and others that the results of a program running on a complex machine are meaningful?  In our everyday use of computers, they tend to act as mysterious "black boxes."  They do all sorts of things for us and we trust them to even if we don't understand the details, the same way we may trust and use a vehicle even if we don't understand all the details of how it works.  As a student of computational methods, however, you'll be writing your own computer code and using that of others.  In this activity, you'll get to look under the hood a bit and learn some of the ways we can evaluate the quality of a code.

#  Compartmental models, an ODE framework

In this section, we introduce the notion of a *compartmental model*.  While this modeling framework is often seen in the medical field (e.g. epidemiology, physiology, related medical imaging), it can be used as a general framework for building basic models of many systems of interest in a variety of scientific disciplines.  Compartmental models give us a way of describing how a system evolves over time, leading to something called an **ordinary differential equation (or ODE)**.  These look something like

$
\begin{equation}
    \frac{df}{dx} = g(x).
\end{equation}
$

In a calculus course, you learn the formal mathematical definition of equations like this.  Informally, for the purposes of this activity, this differential equation is a way of describing the slope of some function $f(x)$ for any valid value of the independent variable $x$.  The right-hand-side, $g(x)$, gives us this slope.  So $g(0)$ is the slope at $x=0$.  Sometimes we describe a line's slope as "rise over run," or in fraction form: $\frac{\mathrm{rise}}{\mathrm{run}}$.  This is, roughly, what $\frac{df}{dx}$ is: it tells you the "rise" ($df$, change in $f(x)$) over the "run" ($dx$, change in $x$).  Lines like $f(x) = ax + b$ have the same slope (in this case, $a$) for _all_ $x$.  Generally, for some unknown, well-behaved function $f(x)$, the slope may be different at different values of $x$.  We'll often call a function "well-behaved" or "differentiable" if a well-defined slope exists for all $x$.  This is a detail we won't worry about too much here.

A differential equation is said to be an _ordinary_ differential equation if it describes functions of _one_ independent variable (like $f(x)$).  Differential equations of functions that depend on multiple independent variables, such as functions of three spatial coordinates like $f(x, y, z)$, are _partial_ differential equations.  We won't worry about those in this activity.

The slope of a line is something you've had experience with and have some intuition for, so it's useful to introduce a differential equation this way.  But in the sciences and engineering, it can be useful to understand a differential equation as describing change, describing evolution with respect to some independent variable (often, but not always, either a spatial variable describing lengths like $x$ or a variable describing time like $t$).  These equations give us a powerful, mathematically robust way of modeling a system's behavior over time.  If we can develop a set of equations that describe this change, we can then use them to make predictions about a system's behavior, about what it will look like tomorrow based on what it looks like today.  In this activity, you will see concrete examples of this.

To help make things a bit more manageable and concrete, instead of drawing from the massive wealth of ODE-based models out there, we restrict ourselves to compartmental models.  A compartmental model consists of some number $N$ of compartments and a set of coefficients that describe how the compartments interact.  The evolution of each compartment $C_j$ in an $N$-compartment model is expressed as

$
\begin{equation}
    \frac{dC_j(t)}{dt} = f_j(u_j(t), C_1(t), ... , C_N(t)),
\end{equation}
$
where $u_j(t)$ is an external function.  There may be one for each compartment ($u_j(t)$ for $j=1,2,...,N$), for only some compartments, or there may only be one specified for the whole model ($u(t)$).  Note that $j$ is an index variable that enumerates the compartments, going from 1 to $N$.

First, let's consider a 1-compartment model composed of an external funtion and a single compartment $C_1$:

$
\begin{equation}
    \frac{dC_1(t)}{dt} = K_1 u(t) - k_2 C_1(t).
\end{equation}
$

Loosely, this can be read as

$
\begin{equation}
    (\text{change in "amount" of }C_1\text{ over time})  =  \\
        (\text{rate of flow from } u \text{ to } C_1) (\text{"amount" in }u(t)) -
        (\text{rate of flow from } C_1 \text{ out of the system}) (\text{"amount" of }C_1).
\end{equation}
$

A diagram of this model can be given as:

Though some real systems are modeled with a 1-compartment model, we can learn a bit more about using compartmental models by adding another compartment that can be interacted with.  You may be able to determine how we'd express such a system by analogy with the 1-compartment model.

We now have two compartments, so we'll want a $C_2(t)$. The quantity being tracked by a particular model can now flow in from the external function into $C_1$, which can in turn exchange with $C_2$, so we'll need some new coefficients for the rates describing such flows.  This could look like

$
\begin{equation}
    \frac{dC_1(t)}{dt} = K_1 u(t) - (k_2 + k_3) C_1(t) + k_4 C_2(t),\\
    \frac{dC_2(t)}{dt} = k_3 C_1(t) - k_4 C_2(t) - k_5 C_2(t).
\end{equation}
$

We can represent this with a diagram as

You may realize this isn't the only way we could write a 2-compartment model.  Any given compartment may have (1) input from or output to an external function (like $u(t)$ into $C_1$ proportional to $K_1$), (2) flow out into another compartment (like from $C_1$ to $C_2$ proportional to $k_3$), (3) flow in from another compartment (like from $C_2$ to $C_1$ proportional to $k_4$), or (4) output that leaves the system (like that marked with constants $k_2$ and $k_5$). For a given compartmental model, each compartment will only have whichever of these makes sense for the model.  Thus, with this basic framework you can model different systems, even if both models are 2-compartment.

So far, we've been pretty generic.  We wanted to first get introduced to the basic concepts and terminology before getting our hands dirty.  But now we're ready for a real 2-compartment model that models the spread of diseases like the common cold that do not confer immunity after recovery.  This means people are either susceptible (S) to infection or are infectious (I) themselves.  Highly original scientists calls this the SI model of disease spread.  Let's look at the equations and diagram, then discuss it:

$
\begin{equation}
    \frac{dS(t)}{dt} = \mu N - \frac{\beta S I}{N} - \nu S  \\
    \frac{dI(t)}{dt} = \frac{\beta S I}{N} - \nu I
\end{equation}
$

$S(t)$ is the fraction of the population that's currently susceptible to infection, $I(t)$ is the fraction already infected, and $N=S+I$ is the total population.  The $\frac{dS(t)}{dt}$ equation is saying that
* (term 1: $\mu N$) the $S$ population is increased over time by new births, with $\mu$ being the rate at which people are born for a population of the size $N$,
* (term 2: $- \frac{\beta S I}{N}$) the $S$ population is decreased over time according to the rate ($\beta/N$) at which susceptible individuals ($S$) in contact with infectious individuals ($I$) become infected, leaving $S$ and joining $I$, and finally
* (term 3: $- \nu S$) the $S$ population is decreased over time acocrding to the mortality rate $\nu$.

Consider the $\frac{dI(t)}{dt}$ equation and see if you can express in words what each term represents, similar to what we've just done for $\frac{dS(t)}{dt}$.

At this point, you should be equipped with enough understanding of compartmental models to be able to carry out an inquiry into what ODE models like this can tell us and what constraints there may be when solving them computationally.  In summary, compartmental models are a framework for modeling systems that can be decomposed into interacting compartments through which some quantity (information, infection, nuclear matter) flows.

So, we have these things called ODEs (oridnary differential equations), which are at the heart of compartmental models (as well as many other modeling techniques).  What can we do with them?  In particular for this course, how can a computer be used to understand the properties and predictions of an ODE model?  And again, what can we do to build confidence that the predictions of a machine we've programmed are trustworthy?  In this inquiry activity, you'll end up investigating these questions.  But before getting there, we need to introduce one more essential computational tool you'll need to work with mathematical equations on a computer: **finite differences**, in particular we'll see one of the most fundamental such differences: the **forward Euler method**.

#  A basic finite difference: forward Euler

In the previous section, we explored a framework for specifying a set of mathematical equations (ODEs).  How do we go from our abstract mathematics to something a computer understands?  Doing so is at the core of the computational sciences, and you're about to learn (or perhaps refresh your memory of) one of the most ubiquitious and important tools for it: the finite difference.

Finite differences are one technique for approximating a derivative (in other words, the left-hand-side of an ODE).  In calculus and most math we may use day-to-day, mathematical functions tend to be continuous.  This means that even in the interval $x \in [0,1]$ there are an infinite number of possible values $x$ can take on when you calculate $f(x)$.  On a computer, however, we have a finite amount of memory available to represent numbersand in practice there's a limit to the smallest rational number you can represent.  A finite difference addresses this by breaking the numberline up into discrete chunks and approximates a derivative using data points from this discrete set of possible $x$ values.

The technique greatly predates the advent of computers, but is now widely used in research and production codes.  For the purposes of this activity, we'll just introduce one of the most basic finite difference methods: **Forward Euler**.  But before all that, let's step back a bit and consider the following continuous function $f(x)$ which has been overlayed on a numberline with some discrete points marked:

In the above plot, the continuous variable $x$ is *discretized* into an equidistant series of values $x_0, x_1, x_2, ... , x_i , ...$.  We use the subscript index variable $i$ to represent a given value of $x$.  Things depending on $x$ will inherit the subscript, such as $f_i \equiv f(x_i)$.

The continuous math is in red.  This is what a computer has trouble handling.  A few discrete points have been marked in blue: $f_i = f(x_i)$ and $f_{i+1} = f(x_{i+1})$.  In calculus, there's a technique we can use to represent _any_ function near a given value of $x$ in terms of the function's derivatives.  This technique is called Taylor expansion.  We won't go into the details, but using this technique we can find that

$
\begin{equation}
    f(x_{i+1}) = f(x_i) + \Delta x \frac{df}{dx} |_{x_i} + \mathcal{O}(\Delta x^2).
\end{equation}
$
Some of these terms are already labeled in the figure above.  The $|_{x_i}$ is a notation that means the derivative $\frac{df}{dx}$ is evaluated at $x=x_{i+1}$. The $\mathcal{O}(\Delta x^2)$ is a notation that means there are other terms in this equation, but they are all proporational to $\Delta x^2$ or higher powers of $\Delta x$.  For example $... + a \Delta x^2 + b \Delta x^3 + c \Delta x^4 + ...$.

For the forward Euler method, we simply neglect all terms that are proportional to $\Delta x^2$ or higher.  That's why it is an approximation (and you may hear people say things like this method is _2nd-order accurate_, because errors are proportional to the second power of $\Delta x$).  This leaves us with

$
\begin{equation}
    f(x_{i+1}) \simeq f(x_i) + \Delta x \frac{df}{dx} |_{x_i}, \text{or:} \\
    f_{i+1} \simeq f_i + \Delta x \frac{df}{dx} |_{x_i}
\end{equation}
$
Thus, we have an approximate equation for the _forward_ value $f_{i+1}$ in terms of $\Delta x$, $f_i$, and $\frac{df}{dx} |_{x_i}$.

The final equation, $f_{i+1} = f_i + \Delta x \frac{df}{dx} |_{x_i} $, is an example of a **finite difference equation**.  It allows us to calculate $f_{i+1}$ if we know $f_i$, the stepsize $\Delta x$, and if we have an expression describing $\frac{df}{dx}$.  Now, can you think of a time you've encountered expressions of the form $\frac{df}{dx}$?  What if I write it like $\frac{dC_j(t)}{dt}$?  Indeed, the ordinary differential equations we explored in the section on compartmental models give us expressions for things like $\frac{dC_j(t)}{dt}$.  We can use these expressions in a finite difference equation to calculate a computer approximation of as many values of $C_j$ as we want.

As we saw, a measure of the accuracy is the order of the terms neglected by our approximation. So if we choose a good step size, perhaps we can get good approximations of the functions in a compartmental model.  For our 1-compartment model, we had

$
\begin{equation}
    \frac{dC_1(t)}{dt} = K_1 u(t) - k_2 C_1(t),
\end{equation}
$

By using this in our finite difference equation, we get

$
\begin{equation}
    C_{1,i+1} = C_{1,i} + \Delta t \left( K_1 u_i - k_2 C_{1,i} \right).
\end{equation}
$

Thus, if we know some initial conditions ($C_{1,i}$, $u_i$), we can approximate the value of $C_1(t=t_i + \Delta t)$.  This value could then be plugged back into the equations to get the *next* value, and so on.  This process of successive approximations is sometimes called *integrating* an equation.  Recall that our analysis approximated the "forward" value of $f_{i+1}$ using a known $f_i$.  Thus, this difference equation is sometimes called a **forward difference**.

For the purposes of this activity, we will not derive the other difference equations, but we will introduce them.  An analysis that approximates the "backward" value ($i-1$) can eventually yield the **backward difference** equation:

$
\begin{equation*}
    f_{i+1} = f_i + \Delta x \frac{df}{dx}|_{x_i}
\end{equation*}
$

An analysis that combines forward and backward difference equations can yield a **centered difference** equation:

$
\begin{equation*}
    f_{i+1} = f_{i-1} + 2 \Delta x \frac{df}{dx}|_{x_i}
\end{equation*}
$

Before ending this section, it's good to note that we can rearrange our equation to write a discrete approximation of a derivative, which can be useful if instead of having an ODE that gives you $\frac{df}{dx} |_{x_i}$ you have a function but want to calculate the derivative:

$
\begin{equation*}
    \frac{df}{dx} |_{x_i} \simeq \frac{\Delta f}{\Delta x} = \frac{f_{i+1} - f{i}}{x_{i+1} - x_i}
\end{equation*}
$

#  Putting it all together: Spreading Rumors

```python

```

# Pre-Class Assignment


# Motivation

In [Novikov eta al. 2016](http://papers.nips.cc/paper/5787-tensorizing-neural-networks.pdf) they use the tensor-train representation to construct a weight matrix. However, the tensor-train constructs a high dimensional teensor and they simply reshape it into a matrix. I thought this was interesting/weird and want to investigate.

Specifically, I am interested in how parameters are shared across the constructed weight matrix. Weight tying is an important part of designing neural networks, and I am interested in the relationship between parameter tying schemes and tensor (networks) and reshaping.

The motivating example would be that a convolution can be written as a parameter sharing scheme in matrix form. Constructed using a circulant, ...?!

***

Secondly, when looking at the [algol for HOSVD](https://lirias.kuleuven.be/bitstream/123456789/72517/1/94-31.pdf) there is an unfolding (aka reshape) operation that is used to matricse the tensors so the left singular vectors of each dimension can be calculated.

### The tensor train format (aka MPS)

```python
import functools

import sympy as sym
sym.init_printing(use_latex='mathjax')
import numpy as np
import matplotlib.pyplot as plt

# NEED a way to visualise!

# dont want to just focus on TT format.
# what other interesting ones are there?
# Have a look at MPS and PEPS?
# something more exotic?
```

```python
idx = 'abcdefghijkl'  # names for indexes

def construct_core(idx, n):
    # construct a 3-tensor
    return sym.tensor.Array([[[sym.Symbol('{}_{}{}{}'.format(idx,i,j,k))
                               for i in range(n)]
                              for j in range(n)]
                             for k in range(n)])

def construct_cores(N, n):
    return [construct_core(idx[i], n) for i in range(N)]

def construct_tensor(cores):
    t = cores[0]
    for i in range(len(cores)-1):
        t = sym.tensorproduct(t, cores[i+1])
        t = sym.tensorcontraction(t, (3,4))  # not sure if this is right...
    return t
```

```python
n_cores = 2
cores = construct_cores(n_cores, 2)
t = construct_tensor(cores)
print(t.shape)

t = sym.simplify(t)
t = sym.reshape(t, [8])
```

    (2, 2, 2, 2)

```python
t
```

$$\left[\begin{matrix}a_{000} \left(b_{000} + b_{011}\right) & a_{000} \left(b_{100} + b_{111}\right) & a_{100} \left(b_{000} + b_{011}\right) & a_{100} \left(b_{100} + b_{111}\right) & a_{010} \left(b_{000} + b_{011}\right) & a_{010} \left(b_{100} + b_{111}\right) & a_{110} \left(b_{000} + b_{011}\right) & a_{110} \left(b_{100} + b_{111}\right)\\a_{001} \left(b_{000} + b_{011}\right) & a_{001} \left(b_{100} + b_{111}\right) & a_{101} \left(b_{000} + b_{011}\right) & a_{101} \left(b_{100} + b_{111}\right) & a_{011} \left(b_{000} + b_{011}\right) & a_{011} \left(b_{100} + b_{111}\right) & a_{111} \left(b_{000} + b_{011}\right) & a_{111} \left(b_{100} + b_{111}\right)\end{matrix}\right]$$

```python
# can even cosntruct tensors where we have the same
# or more parameters than elements, but they are shared
# in interesting ways
```

```python
s = str(t)
print(s.count('a_000'))
print(s.count('a_001'))

# so each parameter is shared over eight spaces?
# want a general formula for this

# also there is an a_{}{}{} parameter in every element.

# kind of locality prior?
# each parameter is shared over some local set (a row or colum).
```

    8
    8

### The various forms of tensor SVD

So, what about all the reshaping funny business going on in HSVD and HOSVD?

```python
def unfold(tensor, axis): # aka Mode, matricization, fibre
    return np.reshape(tensor, (tensor.shape[axis], -1))
```

I do not have any good intuition for why/how taking the
left eigenvectors of a reshaped tensor ...?
so somehow, under reshaping, the left singular vectors are preserved?
S and V are unncessary (which seems rather unusual...)

***
The way the core singular value tensor is calucluated seems like cheating.
$$
\mathcal A = S \times_1 U_1 ... \times_n U_n \\
S = \mathcal A \times_1 U_1^T ... \times_n U_n^T \\
$$
this doesnt seem right, S should be diagonal!?

***

Hierarchical SVD also uses the same trick.
Should I bother coding it?
Seems interesting as now you have multiple core tensors and they need to be reconstructed using the right graph.

```python
class HOSVD():
    def decompose(tensor):
        U = []
        # for each arm of the tensor
        for i, s in enumerate(tensor.shape):
            u, _, _ = np.linalg.svd(unfold(tensor, i))
            U.append(u)

        S = tensor
        for i, leg in enumerate(U):
            S = np.tensordot(leg.T, S, axes=[1, i])
        return U, S

    def construct(legs, core):
        c = core
        # or could outerproduct the legs first and then elementwise mul!?
        for i, leg in enumerate(legs):
            c = np.tensordot(leg, c, axes=[1, i])
        return c

    def test():
        A = np.random.random((5,5,5))
        u, s = HOSVD.decompose(A)
        B = HOSVD.construct(u ,s)
        d = np.sum(np.abs(A - B))
        if d > 1e-8:
            raise ValueError('A and B are not equal. Difference = {}'.format(d))

HOSVD.test()
```

```python

```

Ok, so that is the motivation out of the way... phew. Now lets take a closer look at reshaping.

Main questions;

...

## Reshaping algols

* Row first, outer vs inner.
* Can just be done at read time with different striding patterns (see views in numpy?)

Only real requirements are that is must have an inverse? It is consistent? It ...?
What about more 'random' permutations on the indexes?

What if we thought about it as a function? What does a reshape do?
Have some $f: x->y$ but we change the f while preserving XXX?!? What is preserved? What are its symmetries?

```python
def reshape(tensor):
    pass
```

Is reshaing a linear op!?
Does it commute, associate, distribute, ...
Firstly, its a unary operation?! So not sure what to do with that...

### Associativity

$\varrho(u) + (v + w) = (\varrho(u) + v) + w$

### Commutativity

$\varrho(a) + b = b + \varrho(a)$

$a(\mathring u + v) = \mathring{au} + av$

Reshaping is a permutation of the bases?

Reshape.
Want;
- some properties that I can measure!?!
- some visualisations! (what happens when I reshape?)
- better intuition... need a concrete example to play with
-

#### Neighborhoods

Picture I already have. Neighbors and where they go to.

#### Connectedness (the dual of neighborhoods?)

What about the graph POV?

#### How is reshape like a convolution?

For example, this is what we do when we want to do a convolution. Construct a tensor of patches (examples, X, Y, kernel, kernel) and then reshape it into a (examples x X x Y, kernel x kernel ) matrix.

```python

```

```python

```

```python

```

```python

```

## Parameter sharing

What is it, why do we do it, some examples.
Can represent large(r) spaces with fewer parameters (that is the usual argument for TNs on parameter sharing...)

Sharing over;

* space,
* time,
* relations,
* ?

Nice way to build priors about invariance. (how does this related to the structure of tensor networks!?)

Aka, parameter sharing schemes. If we write the reshaped, constructed tensor, and show the receptive field of original parameters.
- are the receptive fields local, which tensor-nets/reshapings give local receptive fields?
- ?
-

This idea is orthogonal to reshaping, reshaping is just a nice way to visualise it?

$$\begin{aligned}
&= \begin{bmatrix}
a_{11} & a_{12} & a_{13} & a_{14} & a_{15} & a_{16} \\
a_{21} & a_{22} & a_{23} & a_{24} & a_{25} & a_{26} \\
a_{31} & a_{32} & a_{33} & a_{34} & a_{35} & a_{36} \\
a_{41} & a_{42} & a_{43} & a_{44} & a_{45} & a_{46} \\
a_{51} & a_{52} & a_{53} & a_{54} & a_{55} & a_{56} \\
a_{61} & a_{62} & a_{63} & a_{64} & a_{65} & a_{66} \\
\end{bmatrix} \\
&\text{(stack by columns. reshape by first indices fastest)}\\
&= \begin{bmatrix}
\begin{bmatrix}
a_{11} &  a_{31} & a_{51}\\
a_{21} & a_{41} & a_{61}\\
\end{bmatrix} &
\begin{bmatrix}
a_{12} &  a_{32} & a_{52}\\
a_{22} & a_{42} & a_{62}\\
\end{bmatrix}\\
\begin{bmatrix}
a_{13} &  a_{33} & a_{53}\\
a_{23} & a_{43} & a_{63}\\
\end{bmatrix} &
\begin{bmatrix}
a_{14} &  a_{34} & a_{54}\\
a_{24} & a_{44} & a_{64}\\
\end{bmatrix} \\
\begin{bmatrix}
a_{15} &  a_{35} & a_{55}\\
a_{25} & a_{45} & a_{65}\\
\end{bmatrix} &
\begin{bmatrix}
a_{16} &  a_{36} & a_{56}\\
a_{26} & a_{46} & a_{66}\\
\end{bmatrix} \\
\end{bmatrix}\end{aligned}$$

Distances are not preserved. Originally $a_{33}$ is one index away from
$a_{32},a_{34},a_{23},a_{43}$. But after the reshaping, the set of
elements that d=1 are $a_{13},a_{53},a_{43},a_{31},a_{35},a_{34}$.
If we map these back into the original matrix, we can see that the
‘range’ of the indicies is speading. More are in each elements
neighbourhood. What does this mean?

```python

```


# Stability and Stiffness

In the past when you've talked about **stability**, it has likely been regarding the stability of a *system*. Stable systems are those will well-behaved exact solutions, meaning they do not grow unbounded.
In engineering we mostly focus (or want!) stable systems, although there are some interesting unstable systems such as those involving resonance, nonlinear dynamics, or chaos—generally we want to know when that happens so we can prevent it.

We can also define the stability of a *numerical scheme*, which is when the numerical solution exhibits unphysical behavior. In other words, it blows up.

For example, let's consider the relatively simple 1st-order ODE
\begin{equation}
\frac{dy}{dt} = -3 y
\end{equation}
with the initial condition $y(0) = 1$. As we will see, this ODE can cause explicit numerical schemes to become unstable, and thus it is a **stiff** ODE. (Note that we can easily obtain the exact solution for this problem, which is $y(t) = e^{-3 t}$.)

Let's try solving this with the Forward Euler method, integrating over $0 \leq t \leq 10$, for a range of time-step size values: $\Delta t = 0.1, 0.25, 0.5, 0.75$:

```matlab
clear
f = @(t,y) -3*y;

dt = 0.1;
t = 0 : dt : 20;
y = zeros(length(t), 1);
y(1) = 1;
for i = 1 : length(t) - 1
    y(i+1) = y(i) + dt*f(t(i), y(i));
end
subplot(4,1,1);
plot(t, y); title(sprintf('dt = %4.2f', dt));

dt = 0.25;
t = 0 : dt : 20;
y = zeros(length(t), 1);
y(1) = 1;
for i = 1 : length(t) - 1
    y(i+1) = y(i) + dt*f(t(i), y(i));
end
subplot(4,1,2);
plot(t, y); title(sprintf('dt = %4.2f', dt));

dt = 0.5;
t = 0 : dt : 20;
y = zeros(length(t), 1);
y(1) = 1;
for i = 1 : length(t) - 1
    y(i+1) = y(i) + dt*f(t(i), y(i));
end
subplot(4,1,3);
plot(t, y); title(sprintf('dt = %4.2f', dt));

dt = 0.75;
t = 0 : dt : 20;
y = zeros(length(t), 1);
y(1) = 1;
for i = 1 : length(t) - 1
    y(i+1) = y(i) + dt*f(t(i), y(i));
end
subplot(4,1,4);
plot(t, y); title(sprintf('dt = %4.2f', dt));
```

At the smaller step sizes, $\Delta t = 0.1$ and $\Delta t = 0.25$, we see that the solution is well-behaved. But, when we increase $\Delta t$ to 0.5, we see some instability that goes away with time. Then, when we increase $\Delta t$ to 0.75, the solution eventually blows up, leading to error **much** larger than what we should expect based on the method's order of accuracy (first) and the step size value.

Compare this behavior to that for the ODE
\begin{equation}
\frac{dy}{dt} = e^{-t}
\end{equation}
which is **non-stiff**:

```matlab
clear
f = @(t,y) exp(-t);

dt = 0.1;
t = 0 : dt : 10;
y = zeros(length(t), 1);
y(1) = 1;
for i = 1 : length(t) - 1
    y(i+1) = y(i) + dt*f(t(i), y(i));
end
subplot(4,1,1);
plot(t, y); title(sprintf('dt = %4.2f', dt));

dt = 0.25;
t = 0 : dt : 10;
y = zeros(length(t), 1);
y(1) = 1;
for i = 1 : length(t) - 1
    y(i+1) = y(i) + dt*f(t(i), y(i));
end
subplot(4,1,2);
plot(t, y); title(sprintf('dt = %4.2f', dt));

dt = 0.5;
t = 0 : dt : 10;
y = zeros(length(t), 1);
y(1) = 1;
for i = 1 : length(t) - 1
    y(i+1) = y(i) + dt*f(t(i), y(i));
end
subplot(4,1,3);
plot(t, y); title(sprintf('dt = %4.2f', dt));

dt = 0.75;
t = 0 : dt : 10;
y = zeros(length(t), 1);
y(1) = 1;
for i = 1 : length(t) - 1
    y(i+1) = y(i) + dt*f(t(i), y(i));
end
subplot(4,1,4);
plot(t, y); title(sprintf('dt = %4.2f', dt));
```

In this case, we see that the solution remains well-behaved even for larger time-step sizes, and the error matches the expected order based on the method and step-size value.

In general numerical schemes can be:

- **unstable**: the scheme blows up for any choice of parameters
- **conditionally stable**: the scheme is stable for a particular choice of parameters (for example, $\Delta t$ is less than some threshold
- **unconditionally stable**: the scheme is always stable

Schemes may be stable for some problem/system and not for another, and vice versa.

Stability is related to robustness of a method, which is generally a tradeoff between complexity and computational cost. The choice of method and solution strategy depends on what you want, and how long you can wait for it. In general, we almost always want to use the largest $\Delta t$ allowable.

Rather than reducing $\Delta t$ to avoid stability problems, we can also use a method that is unconditionally stable, such as the **Backward Euler** method.

## Backward Euler method

The Backward Euler method is very similar to the Forward Euler method, except in one way: it uses the slope at the *next* time step:
\begin{equation}
    \left(\frac{dy}{dx}\right)_{i+1} \approx \frac{y_{i+1} - y_i}{\Delta x}
\end{equation}
Then, the resulting recursion formula is
\begin{equation}
y_{i+1} = y_i + \Delta x \left(\frac{dy}{dx}\right)_{i+1}, \text{ or} \\
y_{i+1} = y_i + \Delta x \, f(x_{i+1}, y_{i+1})
\end{equation}
where $f(x,y) = dy/dx$.

Notice that this recursion formula cannot be directly solved, because $y_{i+1}$ shows up on both sides. This is an **implicit** method, where all the other methods we have covered (Forward Euler, Heun's, Midpoint, and 4th-order Runge-Kutta) are **explicit**. Implicit methods require more work to actually implement.

### Backward Euler example
For example, consider the problem
\begin{equation}
\frac{dy}{dx} = f(x,y) = 8 e^{-x} (1+x) - 2y
\end{equation}
To actually solve this problem with the Backward Euler method, we need to incorporate the derivative function $f(x,y)$ into the recursion formula and solve for $y_{i+1}$:
\begin{align}
y_{i+1} &= y_i + \Delta x \, f(x_{i+1}, y_{i+1}) \\
y_{i+1} &= y_i + \Delta x \left[ 8 e^{-x_{i+1}} (1 + x_{i+1}) - 2 y_{i+1} \right] \\
y_{i+1} &= y_i + 8 e^{-x_{i+1}} (1 + x_{i+1}) \Delta x - 2 \Delta x \, y_{i+1} \\
y_{i+1} + 2 \Delta x \, y_{i+1} &= y_i + 8 e^{-x_{i+1}} (1 + x_{i+1}) \Delta x \\
y_{i+1} &= \frac{ y_i + 8 e^{-x_{i+1}} (1 + x_{i+1}) \Delta x }{ 1 + 2 \Delta x }
\end{align}
Now we have a useable recursion formula that we can use to solve this problem. Let's use the initial condition $y(0) = 1$, the domain $0 \leq x \leq 7$, and $\Delta x = 0.2$.

```matlab
clear

dx = 0.2;
x = 0 : dx : 7;
y = zeros(length(x), 1);
y(1) = 1;

% Backward Euler loop
for i = 1 : length(x) - 1
    y(i+1) = (y(i) + 8*exp(-x(i+1))*(1 + x(i+1))*dx) / (1 + 2*dx);
end

x_exact = linspace(0, 7);
y_exact = exp(-2.*x_exact).*(8*x_exact.*exp(x_exact) + 1);
plot(x_exact, y_exact); hold on
plot(x, y, 'o--')
legend('Exact solution', 'Backward Euler solution')
```

This matches nearly what we saw with the Forward Euler method before—Backward Euler is also a **first-order** method, so the global error should be proportional to $\Delta x$.

Let's now return to the stiff ODE $y^{\prime} = -3 y$, and see how the Backward Euler method does. First, we need to obtain our useable recursion formula:
\begin{align}
y_{i+1} &= y_i + \Delta t \, f(t_{i+1}, y_{i+1}) \\
y_{i+1} &= y_i + \Delta t \, \left( -3 y_{i+1} \right) \\
y_{i+1} + 3 y_{i+1} \Delta t &= y_i \\
y_{i+1} &= \frac{y_i}{1 + 3 \Delta t}
\end{align}

```matlab
clear

dt = 0.1;
t = 0 : dt : 10;
y = zeros(length(t), 1);
y(1) = 1;
for i = 1 : length(t) - 1
    y(i+1) = y(i) / (1 + 3*dt);
end
subplot(4,1,1);
plot(t, y); title(sprintf('dt = %4.2f', dt));

dt = 0.25;
t = 0 : dt : 10;
y = zeros(length(t), 1);
y(1) = 1;
for i = 1 : length(t) - 1
    y(i+1) = y(i) / (1 + 3*dt);
end
subplot(4,1,2);
plot(t, y); title(sprintf('dt = %4.2f', dt));

dt = 0.5;
t = 0 : dt : 10;
y = zeros(length(t), 1);
y(1) = 1;
for i = 1 : length(t) - 1
    y(i+1) = y(i) / (1 + 3*dt);
end
subplot(4,1,3);
plot(t, y); title(sprintf('dt = %4.2f', dt));

dt = 0.75;
t = 0 : dt : 10;
y = zeros(length(t), 1);
y(1) = 1;
for i = 1 : length(t) - 1
    y(i+1) = y(i) / (1 + 3*dt);
end
subplot(4,1,4);
plot(t, y); title(sprintf('dt = %4.2f', dt));
```

In this case, we see that the solution remains well-behaved for all the step sizes, not showing any of the instability we saw with the Forward Euler method. This is because the Backward Euler method is **unconditionally stable**.

## Stability analysis

### Stability analysis of Forward Euler

We can perform a stability analysis of the stiff problem to identify when the Forward Euler method becomes unstable. Let's apply the method to the ODE at hand:
\begin{align}
\frac{dy}{dt} &= -3 y \\
y_{i+1} &= y_i + \Delta t f(t_i, y_i) \\
y_{i+1} &= y_i + \Delta t (-3 y_i) \\
 &= y_i (1 - 3 \Delta t) \\
\frac{y_{i+1}}{y_i} &= \sigma = 1 - 3 \Delta t
\end{align}
where $\sigma$ is the **amplification factor**. This defines whether the solution grows or decays each step—for a stable physical system, we expect the solution to get smaller or remain contant with each step.

Therefore, for the method to remain stable, we must have $\sigma | \leq 1$. We can use this stability criterion to find conditions on $\Delta t$ for stability:
\begin{gather}
| \sigma | = | 1 - 3 \Delta t | \leq 1 \\
-1 \leq 1 - 3 \Delta t \leq 1 \\
-1 \leq 1 - 3 \Delta t \quad \text{or} \quad 1 - 3 \Delta t \leq 1 \\
\frac{-2}{3} \leq -\Delta t \quad \quad -\Delta t \leq 0 \\
\rightarrow \Delta t \leq \frac{2}{3} \quad \text{and} \quad \Delta t \geq 0 \\
\therefore 0 \leq \Delta t \leq \frac{2}{3}
\end{gather}
for stability. (For safety, we might use $\Delta t < 1/2$ for safety, to stay away from the absolute stability limit.)

The Forward Euler method is then *conditionally stable*.

As a general rule of thumb, all **explicit** methods are conditionally stable; these are methods where the recursion formula for $y_{i+1}$ can be written and calculated explicitly in terms of known quantities.

### Stability analysis of Backward Euler

We can also perform a stability analysis on the Backward Euler method to show that its stability does not depend on the step size:
\begin{align}
\frac{dy}{dt} &= -3 y \\
y_{i+1} &= y_i + \Delta t f(t_{i+1}, y_{i+1}) \\
y_{i+1} &= y_i + \Delta t (-3 y_{i+1}) \\
y_{i+1} (1 + 3 \Delta t) &= y_i \\
\sigma &= \frac{y_{i+1}}{y_i} = \frac{1}{1 + 3 \Delta t}
\end{align}
For stability, we need $| \sigma | \leq 1$:
\begin{align}
| \sigma | &= \left| \frac{1}{1 + 3 \Delta t} \right| \leq 1 \\
\rightarrow \Delta t &> 0
\end{align}
Therefore the Backward Euler method is *unconditionally stable*.


# Matrix generation

## Init symbols for *sympy*

```python
from sympy import *
from geom_util import *
from sympy.vector import CoordSys3D
N = CoordSys3D('N')
alpha1, alpha2, alpha3 = symbols("alpha_1 alpha_2 alpha_3", real = True, positive=True)
init_printing()

%matplotlib inline

%reload_ext autoreload
%autoreload 2
%aimport geom_util
```

### Lame params

```python
h1 = Function("H1")
h2 = Function("H2")
h3 = Function("H3")

H1 = h1(alpha1, alpha2, alpha3)
H2 = S(1)
H3 = h3(alpha1, alpha2, alpha3)
```

### Metric tensor

${\displaystyle \hat{G}=\sum_{i,j} g^{ij}\vec{R}_i\vec{R}_j}$

```python
G_up = getMetricTensorUpLame(H1, H2, H3)
```

${\displaystyle \hat{G}=\sum_{i,j} g_{ij}\vec{R}^i\vec{R}^j}$

```python
G_down = getMetricTensorDownLame(H1, H2, H3)
```

### Christoffel symbols

```python
GK = getChristoffelSymbols2(G_up, G_down, (alpha1, alpha2, alpha3))
```

### Gradient of vector

$
\left(
\begin{array}{c}
\nabla_1 u_1 \\ \nabla_2 u_1 \\ \nabla_3 u_1 \\
\nabla_1 u_2 \\ \nabla_2 u_2 \\ \nabla_3 u_2 \\
\nabla_1 u_3 \\ \nabla_2 u_3 \\ \nabla_3 u_3 \\
\end{array}
\right)
=
B \cdot
\left(
\begin{array}{c}
u_1 \\
\frac { \partial u_1 } { \partial \alpha_1} \\
\frac { \partial u_1 } { \partial \alpha_2} \\
\frac { \partial u_1 } { \partial \alpha_3} \\
u_2 \\
\frac { \partial u_2 } { \partial \alpha_1} \\
\frac { \partial u_2 } { \partial \alpha_2} \\
\frac { \partial u_2 } { \partial \alpha_3} \\
u_3 \\
\frac { \partial u_3 } { \partial \alpha_1} \\
\frac { \partial u_3 } { \partial \alpha_2} \\
\frac { \partial u_3 } { \partial \alpha_3} \\
\end{array}
\right)
= B \cdot D \cdot
\left(
\begin{array}{c}
u^1 \\
\frac { \partial u^1 } { \partial \alpha_1} \\
\frac { \partial u^1 } { \partial \alpha_2} \\
\frac { \partial u^1 } { \partial \alpha_3} \\
u^2 \\
\frac { \partial u^2 } { \partial \alpha_1} \\
\frac { \partial u^2 } { \partial \alpha_2} \\
\frac { \partial u^2 } { \partial \alpha_3} \\
u^3 \\
\frac { \partial u^3 } { \partial \alpha_1} \\
\frac { \partial u^3 } { \partial \alpha_2} \\
\frac { \partial u^3 } { \partial \alpha_3} \\
\end{array}
\right)
$

```python
def row_index_to_i_j_grad(i_row):
    return i_row // 3, i_row % 3

B = zeros(9, 12)
B[0,1] = S(1)
B[1,2] = S(1)

B[2,3] = S(1)

B[3,5] = S(1)
B[4,6] = S(1)
B[5,7] = S(1)

B[6,9] = S(1)
B[7,10] = S(1)
B[8,11] = S(1)

for row_index in range(9):
    i,j=row_index_to_i_j_grad(row_index)
    B[row_index, 0] = -GK[i,j,0]
    B[row_index, 4] = -GK[i,j,1]
    B[row_index, 8] = -GK[i,j,2]

B
```

### Strain tensor

$
\left(
\begin{array}{c}
\varepsilon_{11} \\
\varepsilon_{22} \\
\varepsilon_{33} \\
2\varepsilon_{12} \\
2\varepsilon_{13} \\
2\varepsilon_{23} \\
\end{array}
\right)
=
\left(E + E_{NL} \left( \nabla \vec{u} \right) \right) \cdot
\left(
\begin{array}{c}
\nabla_1 u_1 \\ \nabla_2 u_1 \\ \nabla_3 u_1 \\
\nabla_1 u_2 \\ \nabla_2 u_2 \\ \nabla_3 u_2 \\
\nabla_1 u_3 \\ \nabla_2 u_3 \\ \nabla_3 u_3 \\
\end{array}
\right)$

```python
E=zeros(6,9)
E[0,0]=1
E[1,4]=1
E[2,8]=1
E[3,1]=1
E[3,3]=1
E[4,2]=1
E[4,6]=1
E[5,5]=1
E[5,7]=1
E

```

```python
def E_NonLinear(grad_u):
    N = 3

    du = zeros(N, N)

    #    print("===Deformations===")

    for i in range(N):
        for j in range(N):
            index = i*N+j
            du[j,i] = grad_u[index]

    #    print("========")

    a_values = S(1)/S(2) * du * G_up

    E_NL = zeros(6,9)
    E_NL[0,0] = a_values[0,0]
    E_NL[0,3] = a_values[0,1]
    E_NL[0,6] = a_values[0,2]

    E_NL[1,1] = a_values[1,0]
    E_NL[1,4] = a_values[1,1]
    E_NL[1,7] = a_values[1,2]

    E_NL[2,2] = a_values[2,0]
    E_NL[2,5] = a_values[2,1]
    E_NL[2,8] = a_values[2,2]

    E_NL[3,1] = 2*a_values[0,0]
    E_NL[3,4] = 2*a_values[0,1]
    E_NL[3,7] = 2*a_values[0,2]

    E_NL[4,0] = 2*a_values[2,0]
    E_NL[4,3] = 2*a_values[2,1]
    E_NL[4,6] = 2*a_values[2,2]

    E_NL[5,2] = 2*a_values[1,0]
    E_NL[5,5] = 2*a_values[1,1]
    E_NL[5,8] = 2*a_values[1,2]

    return E_NL

%aimport geom_util
#u=getUHat3D(alpha1, alpha2, alpha3)

u=getUHatU3Main(alpha1, alpha2, alpha3)

gradu=B*u

E_NL = E_NonLinear(gradu)*B

```

### Physical coordinates

$u_i=u_{[i]} H_i$

```python
P=zeros(12,12)
P[0,0]=H1
P[1,0]=(H1).diff(alpha1)
P[1,1]=H1
P[2,0]=(H1).diff(alpha2)
P[2,2]=H1
P[3,0]=(H1).diff(alpha3)
P[3,3]=H1

P[4,4]=H2
P[5,4]=(H2).diff(alpha1)
P[5,5]=H2
P[6,4]=(H2).diff(alpha2)
P[6,6]=H2
P[7,4]=(H2).diff(alpha3)
P[7,7]=H2

P[8,8]=H3
P[9,8]=(H3).diff(alpha1)
P[9,9]=H3
P[10,8]=(H3).diff(alpha2)
P[10,10]=H3
P[11,8]=(H3).diff(alpha3)
P[11,11]=H3
P=simplify(P)
P
```

```python
B_P = zeros(9,9)

for i in range(3):
    for j in range(3):
        ratio=1
        if (i==0):
            ratio = ratio*H1
        elif (i==1):
            ratio = ratio*H2
        elif (i==2):
            ratio = ratio*H3

        if (j==0):
            ratio = ratio*H1
        elif (j==1):
            ratio = ratio*H2
        elif (j==2):
            ratio = ratio*H3

        row_index = i*3+j

        B_P[row_index, row_index] = 1/ratio

Grad_U_P = simplify(B_P*B*P)
Grad_U_P
```

```python
StrainL=simplify(E*Grad_U_P)
StrainL
```

```python
%aimport geom_util

u=getUHatU3Main(alpha1, alpha2, alpha3)

gradup=B_P*B*P*u

E_NLp = E_NonLinear(gradup)*B*P*u

simplify(E_NLp)
```

### Tymoshenko theory

$u^1 \left( \alpha_1, \alpha_2, \alpha_3 \right)=u\left( \alpha_1 \right)+\alpha_3\gamma \left( \alpha_1 \right) $

$u^2 \left( \alpha_1, \alpha_2, \alpha_3 \right)=0 $

$u^3 \left( \alpha_1, \alpha_2, \alpha_3 \right)=w\left( \alpha_1 \right) $

$ \left(
\begin{array}{c}
u^1 \\
\frac { \partial u^1 } { \partial \alpha_1} \\
\frac { \partial u^1 } { \partial \alpha_2} \\
\frac { \partial u^1 } { \partial \alpha_3} \\
u^2 \\
\frac { \partial u^2 } { \partial \alpha_1} \\
\frac { \partial u^2 } { \partial \alpha_2} \\
\frac { \partial u^2 } { \partial \alpha_3} \\
u^3 \\
\frac { \partial u^3 } { \partial \alpha_1} \\
\frac { \partial u^3 } { \partial \alpha_2} \\
\frac { \partial u^3 } { \partial \alpha_3} \\
\end{array}
\right) = T \cdot
\left(
\begin{array}{c}
u \\
\frac { \partial u } { \partial \alpha_1} \\
\gamma \\
\frac { \partial \gamma } { \partial \alpha_1} \\
w \\
\frac { \partial w } { \partial \alpha_1} \\
\end{array}
\right) $

```python
T=zeros(12,6)
T[0,0]=1
T[0,2]=alpha3
T[1,1]=1
T[1,3]=alpha3
T[3,2]=1

T[8,4]=1
T[9,5]=1
T
```

```python
D_p_T = StrainL*T
simplify(D_p_T)
```

```python
u = Function("u")
t = Function("theta")
w = Function("w")

u1=u(alpha1)+alpha3*t(alpha1)
u3=w(alpha1)

gu = zeros(12,1)
gu[0] = u1
gu[1] = u1.diff(alpha1)
gu[3] = u1.diff(alpha3)

gu[8] = u3
gu[9] = u3.diff(alpha1)

gradup=Grad_U_P*gu

# o20=(K*u(alpha1)-w(alpha1).diff(alpha1)+t(alpha1))/2
# o21=K*t(alpha1)
# O=1/2*o20*o20+alpha3*o20*o21-alpha3*K/2*o20*o20
# O=expand(O)
# O=collect(O,alpha3)
# simplify(O)

StrainNL = E_NonLinear(gradup)*gradup
simplify(StrainNL)
```

### Square theory

$u^1 \left( \alpha_1, \alpha_2, \alpha_3 \right)=u_{10}\left( \alpha_1 \right)p_0\left( \alpha_3 \right)+u_{11}\left( \alpha_1 \right)p_1\left( \alpha_3 \right)+u_{12}\left( \alpha_1 \right)p_2\left( \alpha_3 \right) $

$u^2 \left( \alpha_1, \alpha_2, \alpha_3 \right)=0 $

$u^3 \left( \alpha_1, \alpha_2, \alpha_3 \right)=u_{30}\left( \alpha_1 \right)p_0\left( \alpha_3 \right)+u_{31}\left( \alpha_1 \right)p_1\left( \alpha_3 \right)+u_{32}\left( \alpha_1 \right)p_2\left( \alpha_3 \right) $

$ \left(
\begin{array}{c}
u^1 \\
\frac { \partial u^1 } { \partial \alpha_1} \\
\frac { \partial u^1 } { \partial \alpha_2} \\
\frac { \partial u^1 } { \partial \alpha_3} \\
u^2 \\
\frac { \partial u^2 } { \partial \alpha_1} \\
\frac { \partial u^2 } { \partial \alpha_2} \\
\frac { \partial u^2 } { \partial \alpha_3} \\
u^3 \\
\frac { \partial u^3 } { \partial \alpha_1} \\
\frac { \partial u^3 } { \partial \alpha_2} \\
\frac { \partial u^3 } { \partial \alpha_3} \\
\end{array}
\right) = L \cdot
\left(
\begin{array}{c}
u_{10} \\
\frac { \partial u_{10} } { \partial \alpha_1} \\
u_{11} \\
\frac { \partial u_{11} } { \partial \alpha_1} \\
u_{12} \\
\frac { \partial u_{12} } { \partial \alpha_1} \\
u_{30} \\
\frac { \partial u_{30} } { \partial \alpha_1} \\
u_{31} \\
\frac { \partial u_{31} } { \partial \alpha_1} \\
u_{32} \\
\frac { \partial u_{32} } { \partial \alpha_1} \\
\end{array}
\right) $

```python
L=zeros(12,12)
h=Symbol('h')
p0=1/2-alpha3/h
p1=1/2+alpha3/h
p2=1-(2*alpha3/h)**2

L[0,0]=p0
L[0,2]=p1
L[0,4]=p2

L[1,1]=p0
L[1,3]=p1
L[1,5]=p2

L[3,0]=p0.diff(alpha3)
L[3,2]=p1.diff(alpha3)
L[3,4]=p2.diff(alpha3)

L[8,6]=p0
L[8,8]=p1
L[8,10]=p2

L[9,7]=p0
L[9,9]=p1
L[9,11]=p2

L[11,6]=p0.diff(alpha3)
L[11,8]=p1.diff(alpha3)
L[11,10]=p2.diff(alpha3)

L
```

```python
D_p_L = StrainL*L
simplify(D_p_L)
```

```python
h = 0.5
exp=(0.5-alpha3/h)*(1-(2*alpha3/h)**2)#/(1+alpha3*0.8)
p02=integrate(exp, (alpha3, -h/2, h/2))
integral = expand(simplify(p02))
integral
```

## Mass matrix

```python
rho=Symbol('rho')
B_h=zeros(3,12)
B_h[0,0]=1
B_h[1,4]=1
B_h[2,8]=1

M=simplify(rho*P.T*B_h.T*G_up*B_h*P)
M
```