```python
"""Purely numerical 1D finite element program."""
import sys, os, time
sys.path.insert(
    0, os.path.join(os.pardir, os.pardir, 'approx', 'src-approx'))
from numint import GaussLegendre, NewtonCotes
#from fe_approx1D_numint import u_glob

import sympy as sym
import numpy as np

def Lagrange_polynomial(x, i, points):
    """
    Return the Lagrange polynomial no. i.
    points are the interpolation points, and x can be a number or
    a sympy.Symbol object (for symbolic representation of the
    polynomial). When x is a sympy.Symbol object, it is
    normally desirable (for nice output of polynomial expressions)
    to let points consist of integers or rational numbers in sympy.
    """
    p = 1
    for k in range(len(points)):
        if k != i:
            p *= (x - points[k])/(points[i] - points[k])
    return p

def Chebyshev_nodes(a, b, N):
    """Return N+1 Chebyshev nodes (for interpolation) on [a, b]."""
    from math import cos, pi
    half = 0.5
    nodes = [0.5*(a+b) + 0.5*(b-a)*cos(float(2*i+1)/(2*(N+1))*pi)
             for i in range(N+1)]
    return nodes

def basis(d, point_distribution='uniform', symbolic=False):
    """
    Return all local basis function phi and their derivatives,
    in physical coordinates, as functions of the local point
    X in a 1D element with d+1 nodes.
    If symbolic=True, return symbolic expressions, else
    return Python functions of X.
    point_distribution can be 'uniform' or 'Chebyshev'.

    >>> phi = basis(d=1, symbolic=False)
    >>> phi[0][0](0)  # basis func 0 at X=0
    0.5
    >>> phi[1][0](0, h=0.5)  # 1st x-derivative at X=0
    -2
    """
    X, h = sym.symbols('X h')
    phi_sym = {}
    phi_num = {}
    if d == 0:
        phi_sym[0] = [1]
        phi_sym[1] = [0]
    else:
        if point_distribution == 'uniform':
            nodes = np.linspace(-1, 1, d+1)
        elif point_distribution == 'Chebyshev':
            nodes = Chebyshev_nodes(-1, 1, d)

        phi_sym[0] = [Lagrange_polynomial(X, r, nodes)
                      for r in range(d+1)]
        phi_sym[1] = [sym.simplify(sym.diff(phi_sym[0][r], X)*2/h)
                      for r in range(d+1)]
    # Transform to Python functions
    phi_num[0] = [sym.lambdify([X], phi_sym[0][r])
                  for r in range(d+1)]
    phi_num[1] = [sym.lambdify([X, h], phi_sym[1][r])
                  for r in range(d+1)]
    return phi_sym if symbolic else phi_num

def affine_mapping(X, Omega_e):
    x_L, x_R = Omega_e
    return 0.5*(x_L + x_R) + 0.5*(x_R - x_L)*X

def finite_element1D_naive(
    vertices, cells, dof_map,     # mesh
    essbc,                        # essbc[globdof]=value
    ilhs,
    irhs,
    blhs=lambda e, phi, r, s, X, x, h: 0,
    brhs=lambda e, phi, r, X, x, h: 0,
    intrule='GaussLegendre',
    verbose=False,
    ):
    N_e = len(cells)
    N_n = np.array(dof_map).max() + 1

    A = np.zeros((N_n, N_n))
    b = np.zeros(N_n)

    timing = {}
    t0 = time.clock()

    for e in range(N_e):
        Omega_e = [vertices[cells[e][0]], vertices[cells[e][1]]]
        h = Omega_e[1] - Omega_e[0]

        d = len(dof_map[e]) - 1  # Polynomial degree
        # Compute all element basis functions and their derivatives
        phi = basis(d)

        if verbose:
            print(('e=%2d: [%g,%g] h=%g d=%d' % \
                  (e, Omega_e[0], Omega_e[1], h, d)))

        # Element matrix and vector
        n = d+1  # No of dofs per element
        A_e = np.zeros((n, n))
        b_e = np.zeros(n)

        # Integrate over the reference cell
        if intrule == 'GaussLegendre':
            points, weights = GaussLegendre(d+1)
        elif intrule == 'NewtonCotes':
            points, weights = NewtonCotes(d+1)

        for X, w in zip(points, weights):
            detJ = h/2
            x = affine_mapping(X, Omega_e)
            dX = detJ*w

            # Compute contribution to element matrix and vector
            for r in range(n):
                for s in range(n):
                    A_e[r,s] += ilhs(e, phi, r, s, X, x, h)*dX
                b_e[r] += irhs(e, phi, r, X, x, h)*dX

        # Add boundary terms
        for r in range(n):
            for s in range(n):
                A_e[r,s] += blhs(e, phi, r, s, X, x, h)
            b_e[r] += brhs(e, phi, r, X, x, h)

        if verbose:
            print(('A^(%d):\n' % e, A_e));  print(('b^(%d):' % e, b_e))

        # Incorporate essential boundary conditions
        modified = False
        for r in range(n):
            global_dof = dof_map[e][r]
            if global_dof in essbc:
                # dof r is subject to an essential condition
                value = essbc[global_dof]
                # Symmetric modification
                b_e -= value*A_e[:,r]
                A_e[r,:] = 0
                A_e[:,r] = 0
                A_e[r,r] = 1
                b_e[r] = value
                modified = True

        if verbose and modified:
            print('after essential boundary conditions:')
            print(('A^(%d):\n' % e, A_e));  print(('b^(%d):' % e, b_e))

        # Assemble
        for r in range(n):
            for s in range(n):
                A[dof_map[e][r], dof_map[e][s]] += A_e[r,s]
            b[dof_map[e][r]] += b_e[r]

    timing['assemble'] = time.clock() - t0
    t1 = time.clock()
    c = np.linalg.solve(A, b)
    timing['solve'] = time.clock() - t1
    if verbose:
        print(('Global A:\n', A)); print(('Global b:\n', b))
        print(('Solution c:\n', c))
    return c, A, b, timing

def finite_element1D(
    vertices, cells, dof_map,     # mesh
    essbc,                        # essbc[globdof]=value
    ilhs,
    irhs,
    blhs=lambda e, phi, r, s, X, x, h: 0,
    brhs=lambda e, phi, r, X, x, h: 0,
    intrule='GaussLegendre',
    verbose=False,
    ):
    N_e = len(cells)
    N_n = np.array(dof_map).max() + 1

    import scipy.sparse
    A = scipy.sparse.dok_matrix((N_n, N_n))
    b = np.zeros(N_n)

    timing = {}
    t0 = time.clock()

    for e in range(N_e):
        Omega_e = [vertices[cells[e][0]], vertices[cells[e][1]]]
        h = Omega_e[1] - Omega_e[0]

        d = len(dof_map[e]) - 1  # Polynomial degree
        # Compute all element basis functions and their derivatives
        phi = basis(d)

        if verbose:
            print(('e=%2d: [%g,%g] h=%g d=%d' % \
                  (e, Omega_e[0], Omega_e[1], h, d)))

        # Element matrix and vector
        n = d+1  # No of dofs per element
        A_e = np.zeros((n, n))
        b_e = np.zeros(n)

        # Integrate over the reference cell
        if intrule == 'GaussLegendre':
            points, weights = GaussLegendre(d+1)
        elif intrule == 'NewtonCotes':
            points, weights = NewtonCotes(d+1)

        for X, w in zip(points, weights):
            detJ = h/2
            x = affine_mapping(X, Omega_e)
            dX = detJ*w

            # Compute contribution to element matrix and vector
            for r in range(n):
                for s in range(n):
                    A_e[r,s] += ilhs(e, phi, r, s, X, x, h)*dX
                b_e[r] += irhs(e, phi, r, X, x, h)*dX

        # Add boundary terms
        for r in range(n):
            for s in range(n):
                A_e[r,s] += blhs(e, phi, r, s, X, x, h)
            b_e[r] += brhs(e, phi, r, X, x, h)

        if verbose:
            print(('A^(%d):\n' % e, A_e));  print(('b^(%d):' % e, b_e))

        # Incorporate essential boundary conditions
        modified = False
        for r in range(n):
            global_dof = dof_map[e][r]
            if global_dof in essbc:
                # local dof r is subject to an essential condition
                value = essbc[global_dof]
                # Symmetric modification
                b_e -= value*A_e[:,r]
                A_e[r,:] = 0
                A_e[:,r] = 0
                A_e[r,r] = 1
                b_e[r] = value
                modified = True

        if verbose and modified:
            print('after essential boundary conditions:')
            print(('A^(%d):\n' % e, A_e));  print(('b^(%d):' % e, b_e))

        # Assemble
        for r in range(n):
            for s in range(n):
                A[dof_map[e][r], dof_map[e][s]] += A_e[r,s]
            b[dof_map[e][r]] += b_e[r]

    import scipy.sparse.linalg
    t1 = time.clock()
    timing['assemble'] = t1 - t0
    c = scipy.sparse.linalg.spsolve(A.tocsr(), b, use_umfpack=True)
    timing['solve'] = time.clock() - t1
    if verbose:
        print(('Global A:\n', A))
        print(('Nonzero (i,j) in A:', list(A.keys())))
        print(('Global b:\n', b));  print(('Solution c:\n', c))
    return c, A, b, timing

#print basis(d=1, symbolic=True)

def mesh_uniform(N_e, d, Omega=[0,1], symbolic=False):
    """
    Return a 1D finite element mesh on Omega with N_e elements of
    the polynomial degree d. The elements have uniform length.
    Return vertices (vertices), local vertex to global
    vertex mapping (cells), and local to global degree of freedom
    mapping (dof_map).
    If symbolic is True, the vertices are expressed as rational
    sympy expressions with the symbol h as element length.
    """
    vertices = np.linspace(Omega[0], Omega[1], N_e + 1).tolist()
    if d == 0:
        dof_map = [[e] for e in range(N_e)]
    else:
        dof_map = [[e*d + i for i in range(d+1)] for e in range(N_e)]
    cells = [[e, e+1] for e in range(N_e)]
    return vertices, cells, dof_map

def define_cases(name=None):
    C = 0.5;  D = 2;  L = 4  # constants for case 'cubic'
    cases = {
        # u''=0 on (0,1), u(0)=0, u(1)=1 => u(x)=x
        'linear': {
            'Omega': [0,1],
            'ilhs': lambda e, phi, r, s, X, x, h:
            phi[1][r](X, h)*phi[1][s](X, h),
            'irhs': lambda e, phi, r, X, x, h: 0,
            'blhs': lambda e, phi, r, s, X, x, h: 0,
            'brhs': lambda e, phi, r, X, x, h: 0,
            'u_L': 0,
            'u_R': 1,
            'u_exact': lambda x: x,
            },
        # -u''=2 on (0,1), u(0)=0, u(1)=0 => u(x)=x(1-x)
        'quadratic': {
            'Omega': [0,1],
            'ilhs': lambda e, phi, r, s, X, x, h:
            phi[1][r](X, h)*phi[1][s](X, h),
            'irhs': lambda e, phi, r, X, x, h:
            2*phi[0][r](X),
            'blhs': lambda e, phi, r, s, X, x, h: 0,
            'brhs': lambda e, phi, r, X, x, h: 0,
            'u_L': 0,
            'u_R': 0,
            'u_exact': lambda x: x*(1-x),
            },
        # -u''=f(x) on (0,L), u'(0)=C, u(L)=D
        # f(x)=x: u = D + C*(x-L) + (1./6)*(L**3 - x**3)
        'cubic': {
            'Omega': [0,L],
            'ilhs': lambda e, phi, r, s, X, x, h:
            phi[1][r](X, h)*phi[1][s](X, h),
            'irhs': lambda e, phi, r, X, x, h:
            x*phi[0][r](X),
            'blhs': lambda e, phi, r, s, X, x, h: 0,
            'brhs': lambda e, phi, r, X, x, h:
            -C*phi[0][r](-1) if e == 0 else 0,
            'u_R': D,
            'u_exact': lambda x: D + C*(x-L) + (1./6)*(L**3 - x**3),
            'min_d': 1,  # min d for exact finite element solution
            },
        }
    if name is None:
        return cases
    else:
        return {name: cases[name]}

def test_finite_element1D():
    """Solve 1D test problems."""
    cases = define_cases()
    verbose = False

    for name in cases:
        case = cases[name]
        for N_e in [3]:
            for d in [1, 2, 3, 4]:
                # Do we need a minimum d to get exact
                # numerical solution?
                if d < case.get('min_d', 0):
                    continue
                vertices, cells, dof_map = \
                    mesh_uniform(N_e=N_e, d=d, Omega=case['Omega'],
                                 symbolic=False)
                N_n = np.array(dof_map).max() + 1
                # Assume uniform mesh
                x = np.linspace(
                    case['Omega'][0], case['Omega'][1], N_n)

                essbc = {}
                if 'u_L' in case:
                    essbc[0] = case['u_L']
                if 'u_R' in case:
                    essbc[dof_map[-1][-1]] = case['u_R']

                c, A, b, timing = finite_element1D_naive(
                    vertices, cells, dof_map, essbc,
                    case['ilhs'], case['irhs'],
                    case['blhs'], case['brhs'],
                intrule='GaussLegendre',
                verbose=verbose)
                # Compare with exact solution
                tol = 1E-12
                diff = (case['u_exact'](x) - c).max()
                msg = 'naive:  case "%s", N_e=%d, d=%d, diff=%g' % \
                      (name, N_e, d, diff)
                print((msg, 'assemble: %.2f' % timing['assemble'], \
                      'solve: %.2f' % timing['solve']))
                assert diff < tol, msg

                c, A, b, timing = finite_element1D(
                    vertices, cells, dof_map, essbc,
                    case['ilhs'], case['irhs'],
                    case['blhs'], case['brhs'],
                intrule='GaussLegendre',
                verbose=verbose)

                # Compare with exact solution
                diff = (case['u_exact'](x) - c).max()
                msg = 'sparse: case "%s", N_e=%d, d=%d, diff=%g' % \
                      (name, N_e, d, diff)
                print((msg, 'assemble: %.2f' % timing['assemble'], \
                      'solve: %.2f' % timing['solve']))
                assert diff < tol, msg

def investigate_efficiency():
    """Compare sparse and dense matrix versions of the FE algorithm."""
    case = define_cases('linear')

    for N_e in [300000, 1000000]:
        for d in [1, 2, 3]:
            vertices, cells, dof_map = \
                mesh_uniform(N_e=3, d=3, Omega=[0,1],
                             symbolic=False)
            N_n = np.array(dof_map).max() + 1
            x = np.linspace(0, 1, N_n)

            essbc = {}
            essbc[0] = case['u_L']
            essbc[dof_map[-1][-1]] = case['u_R']

            c, A, b, timing = finite_element1D_naive(
                vertices, cells, dof_map, essbc,
                case['ilhs'], case['irhs'],
                case['blhs'], case['brhs'],
            intrule='GaussLegendre',
            verbose=False)
            msg = 'naive:  N_e=%d, d=%d: assemble=%.2e solve=%.2e' % \
                  (N_e, d, timing['assemble'], timing['solve'])
            print(msg)

            c, A, b, timing = finite_element1D(
                vertices, cells, dof_map, essbc,
                case['ilhs'], case['irhs'],
                case['blhs'], case['brhs'],
            intrule='GaussLegendre',
            verbose=False)
            msg = 'sparse: N_e=%d, d=%d: assemble=%.2e solve=%.2e' % \
                  (N_e, d, timing['assemble'], timing['solve'])
            print(msg)

if __name__ == '__main__':
    test_finite_element1D()
    #investigate_efficiency()

```

```python

```

Data Science and Machine Learning/Machine-Learning-In-Python-THOROUGH/EXAMPLES/FINITE_ELEMENTS/INTRO/SRC/19_FE1D.ipynb


*Images generated by a deep neural network that interprets text to generate images (DALL·E 2)*

# PC lab: intro to Neural networks & PyTorch

Deep learning is the subfield of machine learning that concerns neural networks with representation learning capabilities. As of recent years, it is arguably the most quickly growing field within machine learning, enjoying major breakthroughs every year (Listing a couple ones from last year(s): GPT-3, AlphaFold v2, DALL·E 2, AlphaZero). Although the popularity of neural nets is a recent phenomenon, they were first described by Warren McCulloch and Walter Pitts in 1943. Early progress in training competitive neural networks was stalled by a multitude of reasons, such as the limited computer resources, sub-optimal network architectures and the use of smaller datasets. In this PC-lab we will introduce you to the basics of implementing a neural network using contemporary practices.

# 1.1 Background

### From linear models to neurons to neural networks

The core unit of every (artificial) neural network is considered the neuron. Every neuron can be observed as a linear combination of **one or more inputs** $\mathbf{x}$ with weights $\mathbf{w}$ (and optionally adding a **bias** $b$), outputting a **single output** $z$:

$z = \sum\limits_{i=1}^{n}(w_ix_i) + b$

equivalently written as dot product:

$z = \mathbf{x} \cdot \mathbf{w} + b$

When multiple neurons are placed next to eachother, we get multiple output neurons, this is called a **layer**. Mathematically, our weights and biases (intercepts) now become a matrix and vector respectively, and we obtain as output a vector $\mathbf{z}$:

$\mathbf{z} = \mathbf{x} \cdot \mathbf{W} + \mathbf{b}$

Multiple **layers** can be stacked sequentially to make up a deep neural network. Performing this gives us a big linear model, because multiple matrix multiplications performed consecutively can also be written as just one: (i.e. $\mathbf{x} \cdot \mathbf{W_1} \cdot \mathbf{W_2} ...$ could also be written as just $\mathbf{x} \cdot \mathbf{W_{all}}$). To obtain the *deep learning magic*, we need to make the whole thing **non-linear**. We do this by adding **activation functions** after every (hidden) layer. The most classical activation is the sigmoid activation $\sigma()$, used also in logistic regression. Nowadays, we usually opt for a more simple activation function: the **ReLU** $ReLU(z) = max\{0,z\}$. This function has the favorable property that its derivative is very efficient to compute (1 when $z$ is positive, 0 when it is negative). It also acts as a switch: a neuron will have a "dead" ($0$) activation whenever $z$ is negative.

For the output layer of a neural network: our activation depends on the task at hand. For binary classification, we use sigmoid to constrain our output between 0 and 1. For multi-class, we use a softmax operation so the output of all neurons sums to 1. For regression, we simply do not use an activation (or a custom one depending on your data: if you already know that your outputs can't be negative but can take all positive numbers ($\mathbb{R}^+$), then maybe a ReLU activation in the output nodes makes sense)

In order to build more intuition for neural networks: consider the following figure where we "visually build up" a neural network starting from Linear regression with four input features (**a**), to Logistic regression (**b**), to an archetypical output neuron with ReLU activation (**c**). For multi-output settings, we visualize multi-output regression (**d**), multi-label classification (more than one class can be 1 in a sample) (**e**), and multi-class classification via softmax (**f**). Finally, a simple neural network with two hidden layers for binary classification (sigmoid output head) is shown under **g**.

**This figure makes it crystal clear that the most simple neural network is just a bunch of linear regressions stacked on top of eachother with non-linearities inbetween.** More advanced neural network architectures exist that modify how we make information flow between inputs. In this example, everything is just connected to everything with linear weights. This type of neural network is what we call an **MLP** or a **multi layer perceptron**.

Keep in mind that all of the above methods usually fit a bias/intercept in addition to weights fitted on the input features.
For an MLP, visually it would look like this:

THOUGHT EXERCISE:

 How much weights does the model pictured above have (including biases)?

Keep in mind that every layer consists of (1) a matrix multiplication of an input $X \in \mathbb{R}^{N, D}$ with a set of weights $W \in \mathbb{R}^{D, M}$ and (2) a non-linearity. With $D$ the number of input features/dimensions/nodes (including one node/feature for the intercept, see the following equation) and $M$ the number of output dimensions/nodes.
\begin{equation}
XW =
\begin{bmatrix}
1 & x_{0,1} & ...  & x_{0,D-1} & x_{0,D} \\
1 & x_{1,1} & ... & x_{1,D-1} & x_{1,D} \\
... & ... & ... & ... & ...\\
1 & x_{N-1,1} & ... & x_{N-1,D-1} & x_{N-1,D} \\
1 & x_{N,1} & ...  & x_{N,D-1} & x_{N,D} \\
\end{bmatrix}
\begin{bmatrix}
W_{0,0} & W_{0,1} & ...  & W_{0,M-1} & W_{0,M} \\
W_{1,0} & W_{1,1} & ... & W_{1,M-1} & W_{1,M} \\
... & ... & ... & ... & ...\\
W_{D-1,0} & W_{D-1,1} & ... & W_{D-1,M-1} & W_{D-1,M} \\
W_{D,0} & W_{D,1} & ...  & W_{D,M-1} & W_{D,M} \\
\end{bmatrix}
\end{equation}

The whole network is trained by first performing a forward pass to get predictions and compute a loss w.r.t. ground truth known labels, then doing backpropagation (essentially applying the chain rule of derivatives) to obtain the gradient of all weights w.r.t. the loss. These gradients are then used by gradient descent (or more modern variants such as Adam) to optimize the neural network.

Training neural networks is (typically) more computationally demanding than more traditional machine learning methods, and we usually use neural networks when we have large datasets. For these two reasons, it is not a good idea to process the whole dataset at once in every training step/iteration. Therefore, to train the network, we process samples in batches, called **(mini-batch) stochastic gradient descent**. This allows for faster training and improved convergence of the loss during gradient descent. Advantages of stochastic gradient descent or other optimization algorithms for loss calculation are not discussed in this PC-lab, but have been [extensively discussed](https://ruder.io/optimizing-gradient-descent/) before.

### Dropout and Normalization layers

Dropout is a popular addition to neural networks. It is a form of regularization during which we stochastically deactivate a percentage of neurons in every training step by putting their activation to zero. This regularization only happens during training, as during testing we (usually) want deterministic outputs. Conceptually, it is similar to other regularization techniques such as ridge regression and subsampling of features in random forest training, in the sense that it will force our model to look at all features, because sometimes one single feature will not be available during training. The difference here is that we do it by stochastically putting nodes to zero during training, and that we can perform it not only on our input features, but also on our hidden nodes.

Mathematically, it can be performed by simply sampling a boolean vector and doing element-wise multiplication.

Visually, it would look a bit like this, where nodes in cyan are dropped out, and the associated cyan weights do not have any influence on training anymore (in that training step):

Normalization is another operation that we usually put between layers in order to more easily train our networks. In essence, this ensure that our hidden outputs are numerically well behaved, which speeds up training and helps with better convergence. The most popular ones are batch normalization and layer normalization.
More info here: [link](https://www.baeldung.com/cs/normalizing-inputs-artificial-neural-network)

With current techniques, most people often put the order of operations like this:
`Layer -> Activation -> Dropout -> Normalization -> and_repeat ...`

## 1.2 PyTorch

To implement neural networks with more ease, a few high-level python libraries are available: (PyTorch, TensorFlow/keras, JAX, ...). These libraries provide functionality in terms of automatic differentiation (backprop), ready-to-use implementations for various layers, loss functions ...

In this lab, we will use [PyTorch](https://pytorch.org). PyTorch is the most popular library for deep learning in academia as of today. For this course it offers the advantage that it has the most 'pythonic' syntax, to the point where almost all NumPy functions have a PyTorch counterpart.

If you want to run this notebook locally, you can find the installation instructions for PyTorch [here](https://pytorch.org/get-started/locally/). Make sure to select the right installation options depending on your system (if you have a GPU or not).

```python
import torch
import numpy as np
```

### Tensors

Tensors are the fundamental data structures in PyTorch. They are analogous to NumPy arrays. The difference is that tensors can also run on GPU hardware. GPU hardware is optimized for many small computations. Matrix multiplications, the building blocks of all deep learning, run orders-of-magnitude faster on GPU than on CPU. Let's see how tensors are constructed and what we can do with them:

```python
x = [[5,8],[9,8]]
print(torch.tensor(x))
print(np.array(x))
```

```python
x_numpy = np.array(x)
print(torch.from_numpy(x_numpy))

x_torch = torch.tensor(x)
print(x_torch.numpy())
```

```python
print(np.random.randn(8).shape)
print(np.random.randn(8,50).shape)

print(torch.randn(8).shape) # an alternative for .shape in PyTorch is .size()
print(torch.randn(8,50).shape)
```

```python
print(np.zeros((8,50)).shape)
print(torch.zeros(8,50).shape) # works with 'ones' as well
```

In PyTorch, the standard data type for floats is `float32`, which is synonymous to `float` within its framework. `float64` is synonymous to `double`.
This is different from the NumPy defaults and naming conventions: NumPy default data type for float is `float64`. Keep this in mind when converting numpy arrays to tensors and back!

```python
print(np.zeros(8).dtype)
print(torch.zeros(8).dtype)
```

Conversion of data types:

```python
x = torch.randn(8)
print(x.dtype)
x = x.to(torch.float64)
print(x.dtype)
```

`torch.long` is synonymous to `torch.int64`. The only difference between int32 and int64 is the amount of bytes with which you will store every integer. If you go up to very high numbers, you will get numerical overflow faster with more compressed data types. We recommend you to always use the defaults: `torch.long` and `torch.float`

```python
x = torch.randint(low=0, high=8, size=(8,), dtype=torch.int32)
print(x)
print(x.dtype)
x = x.to(torch.long)
print(x.dtype)
```

Indexing and other operations work as in NumPy arrays

```python
x = torch.randn(8,50,60)
print(x.shape)
print(x[:4,10:-10].shape)
x[0,0,:10] = 0
print(x[0,0,:16])

print(torch.min(x), torch.max(x), torch.min(torch.abs(x)))
# most of these functions are also tensor methods:
print(x.min(), x.max(), x.abs().min())
```

Joining tensors via concatenation:

```python
print(x.shape)
x_cat0 = torch.cat([x, x], dim=0)
print(x_cat0.shape)
x_cat1 = torch.cat([x, x, x], dim=1)
print(x_cat1.shape)
```

Matrix multiplication: let's say we have an input `x`, consisting of 8 samples with 26 features, that we linearly combine with weights `w` to get a single output for every sample:

```python
x = torch.randn(8,26)
w = torch.randn(26,1)

y_hat = torch.matmul(x, w) # an alternative and equivalent syntax is x @ w
print(y_hat)
print(y_hat.shape)
```

Note that matrix multiplication is different from element-wise multiplication. For element-wise, `*` is used.

```python
x = torch.ones(8)
print(x)
x = x - 1.5
print(x)
x -= 1.5
print(x)
x += torch.randn(1)
print(x)
x += torch.randn(8)
print(x)
```

Broadcasting works as in NumPy: [link](https://pytorch.org/docs/stable/notes/broadcasting.html)

Keep in mind, just like in NumPy, whatever you want to do with a tensor, there's probably an elegant operation for it implemented somewhere, you just have to look for it (on google and in the documentation)

# 2 Building a neural network in PyTorch

## 2.1 The building blocks

Neural networks are initialized through the use of class objects. You have encountered class objects already during this course: sklearn models are all class objects. The difference here is that we will code our own class first, before using it.

Many of the functionalities necessary to create [**all types of neural networks**](http://www.asimovinstitute.org/neural-network-zoo/) have [**already been implemented**](http://pytorch.org/docs/master/nn.html).

Let's inspect the most basic building blocks first: the [linear layer](https://pytorch.org/docs/master/generated/torch.nn.Linear.html#torch.nn.Linear) and the [ReLU](https://pytorch.org/docs/master/generated/torch.nn.ReLU.html#torch.nn.ReLU)

A linear layer is an object that will perform a matrix multiplication once called. Here, we instantiate such a layer with 20 input nodes and 40 output nodes:

```python
import torch.nn as nn
nn.Linear(20, 40)
```

Let's simulate some random data for this layer: A data set (or batch) with 16 samples and 20 features:

```python
x = torch.randn(16, 20)
```

Now let's use our linear layer on this data:

```python
layer = nn.Linear(20, 40)
print(x.shape)
z = layer(x)
print(z.shape)
```

What happens when we try to feed our layer an input with a different number of features?

```python
x = torch.randn(16, 30)
layer(x)
```

Let's see the ReLU in action:

```python
relu = nn.ReLU()

x = torch.randn(2, 4)
print(x)
z = relu(x)
print(z)
```

As you may have noticed, `nn.Module`s are class objects, a bit like scikit-learn models, that you instantiate and then call.

You can chain `nn.Module`s with the use of `nn.Sequential`:

```python
linear_and_relu = nn.Sequential(nn.Linear(20, 40), nn.ReLU())

x = torch.randn(16, 20)
z = linear_and_relu(x)
```

Or even longer constructs:

Always keep in mind what happens with the dimensions of your input and outputs with every layer, if your first layer outputs 40 features/nodes/hidden dimensions, then logically the next will have to take 40 as input.

```python
a_whole_damn_network = nn.Sequential(
    nn.Linear(128, 64),
    nn.ReLU(),
    nn.Linear(64, 16),
    nn.ReLU(),
    nn.Linear(16, 4),
    nn.ReLU(),
    nn.Linear(4, 1)
    )

x = torch.randn(16, 128)
z = a_whole_damn_network(x)
print(z.shape)
print(z)
```

The output $z$ that we now obtain after this whole network are called **logits**. They are the real-numbered $\mathbb{R}$ outputs that we obtain at the end of the network before our last activation function. This last activation function will be a, depending on the task at hand, sigmoid, softmax, or nothing at all for regression

Similar implementations exist for Dropout and Normalization layers in PyTorch, we invite you to look them up.

## 2.2 Class object neural networks and hyperparameters

We've seen how to implement a neural network using PyTorch `nn.Sequential`. It is more flexible however to write our own model class. This allows us to have more control over which operations we use and define our own hyperparameters. To make a PyTorch model, we specify our class object to be a submodule of `nn.Module` and inherit its methods via `super().__init__()`. Further, we specify all necessary attributes (such as layers) in our `__init__` function (executed upon initialization) and implement a `forward` function which will be executed when we call the object after being initialized.

The following code shows two examples, the first one of a very basic construction of a neural network without hyperparameters. The other one shows the same network, but where we set up our `__init__` function to process hyperparameters as input arguments. We can for example specify a hyperparameter whether we want to use dropout or not. (We could go even further to have an extra hyperparameter specifying the probability of dropout, ...).

```python
class BasicModel(nn.Module):
    def __init__(self):
        super().__init__()

        self.layer1 = nn.Linear(50, 40)
        self.relu1 = nn.ReLU()
        self.dropout1 = nn.Dropout(0.2)

        self.layer2 = nn.Linear(40, 20)
        self.relu2 = nn.ReLU()
        self.dropout2 = nn.Dropout(0.2)

        self.layer3 = nn.Linear(20, 10)
        self.relu3 = nn.ReLU()
        self.dropout3 = nn.Dropout(0.2)

        self.layer4 = nn.Linear(10, 5)
        # Think again: why do we not want a relu and dropout after our last layer again?

    def forward(self, x):
        # call them in separate lines:
        x = self.layer1(x)
        x = self.relu1(x)
        x = self.dropout1(x)

        # or together:
        x = self.dropout2(self.relu2(self.layer2(x)))
        x = self.dropout3(self.relu3(self.layer3(x)))

        # we could've also wrapped everything in a nn.Sequential ..

        x = self.layer4(x)
        return x

class HyperparameterModel(nn.Module):
    def __init__(self, dimensions_from_input_to_output = [50, 40, 20, 10, 5], dropout = True):
        super().__init__()

        layers = []
        # iterate through all layers:
        for i in range(len(dimensions_from_input_to_output) - 2):
            layer = nn.Linear(dimensions_from_input_to_output[i], dimensions_from_input_to_output[i + 1])
            layers.append(layer)
            layers.append(nn.ReLU())
            if dropout == True:
                layers.append(nn.Dropout(0.2))

        # the last layer separate from the loop because we don't want a ReLU and dropout after the last layer
        layer = nn.Linear(dimensions_from_input_to_output[i+1], dimensions_from_input_to_output[i + 2])
        layers.append(layer)

        # wrap the layers in a sequential
        self.net = nn.Sequential(*layers)

    def forward(self, x):
        return self.net(x)

```

Testing out the basic model:

```python
net = BasicModel()
net
```

```python
x = torch.randn(4, 50)
y = net(x)
print(y)
print(y.shape)

```

Testing out the hyperparameter model, when no arguments are specified the default ones are chosen:

```python
net = HyperparameterModel()
net
```

```python
x = torch.randn(4, 50)
y = net(x)
print(y)
print(y.shape)
```

Or by specifying hyperparameters, the following code shows a bit of a deeper model:

```python
net = HyperparameterModel(dimensions_from_input_to_output = [50, 160, 80, 40, 20, 10, 5], dropout = False)
net
```

```python
x = torch.randn(4, 50)
y = net(x)
print(y)
print(y.shape)
```

EXERCISE:

 Copy over any of the two networks above (preferably the one you understand best), and modify it to take as input 784 features and as output 10 nodes. We will use this network later in the PC lab. You can test your network for bugs using some randomly generated data (as above).

```python
######## YOUR CODE HERE #########

#################################
```

## 2.3 Data and training

You may have noticed that the outputs of your model have a `grad_fn` attribute. This grad function will be used by PyTorch automatic differentation engine to perform backward passes and compute gradients for every parameter with respect to the loss/cost function.

Now that we have our model and know that PyTorch will automatically compute gradients for us, we can move on to train a model. To do this, we need a couple more things:

The most basic blueprint of PyTorch model training consists of
- Get your data
- Wrap your data splits in a [data loader](https://pytorch.org/docs/stable/data.html)
- Instantiate the model
- Instantiate a [loss function](https://pytorch.org/docs/stable/nn.html#loss-functions)
- Instantiate an [optimizer object](https://pytorch.org/docs/stable/optim.html), to which you pass the parameters you want to optimize
- Iterate through your training data, for every batch:
    - reset the gradients
    - do forward pass
    - compute loss
    - backward pass
    - update parameters

(Optionally):
- After every full iteration through all training data samples (called an epoch), loop through all batches of validation data:
    - forward pass
    - compute loss and validation scores

In this way, we can monitor how good our model is performing on left-out validation data during training, this is used in early stopping. Notably, we do not compute and reset gradients and update parameters during our validation iterations, because we do not want to fit our model on this data.

Let's start with step one: get your data and wrap them in data loaders. We will first illustrate how we can convert our usual pandas or numpy datasets to be compatible with PyTorch with some random data:

```python
X_train = np.random.randn(100, 784)
y_train = np.random.randint(low = 0, high = 10, size = (100))
```

```python
X_train = torch.from_numpy(X_train)
print(X_train.dtype)
```

Remember to look at your data types: by default NumPy is `float64`, but if you instantiate a model, by default it will have weights in `float32`. It is therefore advised to convert your data to `float32`. In PyTorch, simply `float` is shorthand for `float32`.

```python
X_train = X_train.float()
# Equivalent: X_train.to(torch.float) or X_train.to(torch.float32)
print(X_train.dtype)
```

```python
y_train = torch.tensor(y_train)
print(y_train.dtype)
```

Now that we have our X_train and y_train as tensors, we can make them PyTorch-ready by wrapping them in a PyTorch dataset, and then by wrapping them in a DataLoader, which will create batches for us:

```python
train_dataset = torch.utils.data.TensorDataset(X_train, y_train)
train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=16, pin_memory=True, shuffle=True)
```

Now we can use our train_dataloader as such:

```python
for batch in train_dataloader:
    X, y = batch
    print(X.shape, y.shape)
```

As you can see, we can iterate through our batches by use of a for loop, and it will spit out a training batch consisting of a list of X and y tensors. We can also test out code by isolating one training batch like this (only necessary for testing out code):

```python
batch = next(iter(train_dataloader))
batch
```

For the remainder of the PC lab, we will work with the MNIST dataset included in `torchvision`. (If you're running this code locally, you may have to pip install torchvision).

```python
from torchvision import datasets
from torchvision.transforms import ToTensor
train_data = datasets.MNIST(
    root = 'data',
    train = True,
    transform = ToTensor(),
    download = True,
)
test_data = datasets.MNIST(
    root = 'data',
    train = False,
    transform = ToTensor()
)

X_train = train_data.data
y_train = train_data.targets

X_test = test_data.data
y_test = test_data.targets
```

```python
print('shapes:')
print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)
print('first training image tensor:')
print(X_train[0])
print('first five labels:')
print(y_train[:5])
```

Our data is images, each data sample has $28 \times 28$ input features, signifying the pixels. In order to feed this data to our model, we will need to flatten these features:

```python
X_train = X_train.reshape(-1, 28 * 28)
X_test = X_test.reshape(-1, 28 * 28)
```

In addition, the grayscale values of our images go from 0 to 255. It is perhaps good practice to min-max standardize these numbers by dividing through 255:

```python
X_train = X_train / 255
X_test = X_test / 255
```

Finally, let's check our datatypes to see if everything is looking good to go:

```python
X_train.dtype, X_test.dtype, y_train.dtype, y_test.dtype
```

Let's split up our training set in a training and validation set and finally wrap our data in a data loader:

```python
np.random.seed(42)
train_indices, val_indices = np.split(np.random.permutation(len(X_train)), [int(len(X_train)*0.8)])
X_val = X_train[val_indices]
y_val = y_train[val_indices]
X_train = X_train[train_indices]
y_train = y_train[train_indices]

train_dataset = torch.utils.data.TensorDataset(X_train, y_train)
train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=16, pin_memory=True, shuffle=True)

val_dataset = torch.utils.data.TensorDataset(X_val, y_val)
val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=16, pin_memory=True, shuffle=True)

test_dataset = torch.utils.data.TensorDataset(X_test, y_test)
test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=16, pin_memory=True, shuffle=True)
```

Let's visualize a random batch:

```python
batch = next(iter(train_dataloader))
X_batch, y_batch = batch
import matplotlib.pyplot as plt

figure = plt.figure(figsize=(10, 8))
cols, rows = 4, 4
for i in range(cols * rows):
    img, label = X_batch[i], y_batch[i]
    figure.add_subplot(rows, cols, i+1)
    plt.title(label.item())
    plt.axis("off")
    plt.imshow(img.reshape(-1, 28, 28).squeeze(), cmap="gray")
plt.show()
```

Now that we have our data ready, let's reiterate our PyTorch model blueprint:

The most basic blueprint of PyTorch model training consists of
- Get your data
- Wrap your data splits in a [data loader](https://pytorch.org/docs/stable/data.html)
- Instantiate the model
- Instantiate a [loss function](https://pytorch.org/docs/stable/nn.html#loss-functions)
- Instantiate an [optimizer object](https://pytorch.org/docs/stable/optim.html), to which you pass the parameters you want to optimize
- Iterate through your training data, for every batch:
    - reset the gradients
    - do forward pass
    - compute loss
    - backward pass
    - update parameters

(Optionally):
- After every full iteration through all training data samples (called an epoch), loop through all batches of validation data:
    - forward pass
    - compute loss and validation scores

In one of the previous exercises, we already implemented a model compatible with MNIST: 784 input features and 10 output nodes (one for each class). Hence, we can move on to loss function and optimizers. For multi-class classification of the digits, we will need to use the [Cross Entropy Loss](https://pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html#torch.nn.CrossEntropyLoss). Take a look at what kind of inputs this loss function expects. According to the documentation: `The input is expected to contain raw, unnormalized scores for each class.` Meaning that we can pass logits directly to this loss function, and we do not have to apply a softmax operation ourselves.

For optimizer choice, we can choose [vanilla gradient descent](https://pytorch.org/docs/stable/generated/torch.optim.SGD.html#torch.optim.SGD) or the nowadays-industry-standard [Adam optimizer](https://pytorch.org/docs/stable/generated/torch.optim.Adam.html#torch.optim.Adam), which is itself a pimped version of stochastic gradient descent (with momentum).

Take note that we can also specify our desired learning rate in our model. This learning should almost always be tuned as it will influence how fast our model trains but also convergence.

```python
model = # your model from previous exercises here.

loss_function = nn.CrossEntropyLoss()

optimizer = torch.optim.SGD(model.parameters(), lr=0.005) # SGD = stochastic gradient descent
```

Now we're ready to perform training. We'll build up the training loop step-wise in the following codeblocks. Let's first start with passing one batch to the model, computing the loss, performing backpropagation and updating the weights:

```python
batch = next(iter(train_dataloader))
X_batch, y_batch = batch

y_hat_batch = model(X_batch)

loss = loss_function(y_hat_batch, y_batch) # Compute loss

loss.backward()   # Calculate gradients
optimizer.step()   # Update weights using defined optimizer

print(X_batch.shape, y_batch.shape)
print(y_hat_batch.shape)
print("Outputs as logits, first two samples:")
print(y_hat_batch[:2])
print('loss:', loss)
```

Everytime we perform a training step, we should reset the gradients so that the gradients computed on the previous batch do not influence the next. We can do this by calling `.zero_grad()` on our optimizer. In practice, it is most safe to call this before every forward pass. So if we train a complete epoch:

```python
all_losses = []

for batch in train_dataloader:
    optimizer.zero_grad()
    X_batch, y_batch = batch

    y_hat_batch = model(X_batch)

    loss = loss_function(y_hat_batch, y_batch) # Compute loss

    loss.backward()   # Calculate gradients
    optimizer.step()   # Update weights using defined optimizer

    all_losses.append(loss.item())
```

Plotting the loss function of every batch during one epoch:

```python
plt.plot(np.arange(len(all_losses)), all_losses)
smoothed_losses = np.convolve(all_losses, np.ones(50)/50, mode = "valid")
plt.plot(np.arange(len(smoothed_losses)), smoothed_losses)
```

We have evaluated the training progress during one epoch on our training set. What if we want to see the performance of the validation set during training, so that we can see if/when the model starts overfitting? It is common practice to perform a pass through the whole validation set after every training epoch. However, we should not compute gradients now, we can block automatic gradient computation using `torch.no_grad()`. Also, we need to tell PyTorch to put our model in evaluation mode (so it does not perform dropout anymore, etc...). After the validation epoch, we should put the model back to training mode again. The code should look something like this:

```python
predictions = []
true_labels = []
losses = []

model.eval()
with torch.no_grad():
    for batch in val_dataloader:
        X_batch, y_batch = batch

        y_hat_batch = model(X_batch)

        loss = loss_function(y_hat_batch, y_batch)

        losses.append(loss.item())
        predictions.append(y_hat_batch)
        true_labels.append(y_batch)

model.train()

predictions = torch.cat(predictions)
true_labels = torch.cat(true_labels)
accuracy = (true_labels == predictions.argmax(-1)).sum().item() / len(predictions)

print(accuracy)
print(np.mean(losses))
```

EXERCISE:

 Using the code above, put it all together to train a model for multiple epochs. After every epoch, print or save some training and validation statistics.

Monitor how good your model is training. What things could you change? In particular, try the Adam optimizer or try using gradient descent with the momentum argument. How does this influence training speed? What is it doing? Try tweaking the learning rate.

You should be able to obtain an accuracy of +- 96%

```python
N_EPOCHS = 20

model =  # your model from previous exercises here.

# loss function & optimizer

for i in range(1, N_EPOCHS + 1):

    # train loop

    # eval loop

    # record or print some variables that you want to keep track of during training
```

EXERCISE:

 After training, what is your performance on the test dataset?

```python
### YOUR CODE HERE ###

######################
```

## 2.4 Extra: The autoencoder

An autoencoder tries to reconstruct its own input. By constraining the dimensionality of our hidden layers (the bottleneck), the model is forced to learn a low-dimensional manifold of the data, much like PCA does (in fact, *architecturally* there are only two differences between PCA and an autoencoder: (1) PCA is linear whereas an autoencoder can be arbitrarily complex and (2) the bottleneck in PCA is constrained to be orthogonal, whereas in an autoencoder no such constraints are in place. Other than that, they are also optimized differently.)

Terminology-wise, we can split up our neural network into two parts: the encoder, which encodes the data to a lower-dimensional space, and the decoder, which decodes that encoding back to the original sample space. During training, samples go through both encoder and decoder. After training, we can opt to use different parts of the autoencoder: if we are interested in the lower-dimensional space of samples (for e.g. clustering or visualization), we only use the encoder. If we want to denoise samples (look up denoising autoencoder), we are interested in both encoder and decoder. Finally, if we want to generate samples, we can start from randomly generated vectors in the bottleneck space (e.g. random 10 dimensional vectors) and put them through the decoder to generate new samples (look up variational autoencoders).

It is very easy to implement a simple autoencoder in PyTorch, especially with the code that we have already written. The only things we need to change are:
- The architecture of the network should be changed so that it represents an hourglass with a bottleneck (most easy for visualization is if you have a bottleneck with 2 hidden dimensions, so that you can plot it directly, otherwise if you have e.g. a bottleneck with 10 hidden dimensions, then you need to do an additional PCA or t-SNE first on the samples). A symmetrical network in terms of hidden dimensions is most conventional.
- The loss function should be adjusted to a regression based loss (such as MSE), since our input/output pixels are continuous as well.
- The loss function should be called on the original input, instead of the target classes
- Accuracy can not be computed on reconstruction, only loss.

The following shows how you can easily make an autoencoder:

```python
model = nn.Sequential(
    HyperparameterModel(dimensions_from_input_to_output= [784, 256, 128, 64]),
    HyperparameterModel(dimensions_from_input_to_output= [64, 128, 256, 784])
)

# Alternatively, we could have made our new class for autoencoders
# or we could have just specified one HyperparameterModel passing [784, 128, 2, 128, 784] as argument
# The construct with the Sequential has the advantage that it allows us to call the encoder and decoder separately
# as model[0](input_data) and model[1](encoded_data)
```

EXTRA EXERCISE:

 Train this autoencoder model by reusing the code used in the supervised learning task and adjusting code where necessary, for a to-do list of necessary changes, consult the list above.

```python
N_EPOCHS = 20

model =  # your model from previous exercises here.

# loss function & optimizer

for i in range(1, N_EPOCHS + 1):

    # train loop

    # eval loop

    # record or print some variables that you want to keep track of during training
```

We can now visualize the manifold our model has learned. In order to do so, we encode samples to our $x$-dimensional bottleneck. After which we can use t-SNE to give us a final 2-dimensional space for visualization. Does our model encode digit identity well in its "latent space"?
What about if we fit t-SNE directly on the pixel values? Does our autoencoder noticeably add value? **Note**: t-SNE may take a couple minutes to run.

```python
inputs = []
encoded_inputs = []
classes = []
with torch.no_grad():
    model.eval()
    for batch in val_dataloader:
        X_batch, y_batch = batch
        inputs.append(X_batch)
        encoded_inputs.append(model[0](X_batch))
        classes.append(y_batch)

classes = torch.cat(classes).numpy()
inputs = torch.cat(inputs).numpy()
encoded_inputs = torch.cat(encoded_inputs).numpy()
```

```python
from sklearn.manifold import TSNE
import seaborn as sns

encoded_tsne = TSNE(verbose = 10, init = "pca", learning_rate = "auto").fit_transform(encoded_inputs)
plt.figure(figsize = (10, 10))
sns.scatterplot(x = encoded_tsne[:,0], y = encoded_tsne[:, 1], hue = classes, alpha = .75, palette="deep")
plt.show()
```

```python
encoded_tsne = TSNE(verbose = 10, init = "pca", learning_rate = "auto").fit_transform(inputs)
plt.figure(figsize = (10, 10))
sns.scatterplot(x = encoded_tsne[:,0], y = encoded_tsne[:, 1], hue = classes, alpha = .75, palette="deep")
```

Our autoencoder may not add much value visually on top of just fitting t-SNE directly on pixel values, but the advantage of an autoencoder do not only come from visualization. For example, we can reconstruct samples from the latent space back to the original space with the decoder. As a last, let's visualize some reconstructions:

```python
with torch.no_grad():
    batch = next(iter(val_dataloader))
    X_batch, y_batch = batch
    model.eval()
    y_hat_batch = model(X_batch)

figure = plt.figure(figsize=(10, 8))
cols, rows = 4, 4
for i in range(cols * rows):
    img, label = X_batch[i], y_batch[i]
    figure.add_subplot(rows, cols, i+1)
    plt.title(label.item())
    plt.axis("off")
    plt.imshow(img.reshape(-1, 28, 28).squeeze(), cmap="gray")
plt.show()

figure = plt.figure(figsize=(10, 8))
cols, rows = 4, 4
for i in range(cols * rows):
    img, label = y_hat_batch[i], y_batch[i]
    figure.add_subplot(rows, cols, i+1)
    plt.title(label.item())
    plt.axis("off")
    plt.imshow(img.reshape(-1, 28, 28).squeeze(), cmap="gray")
plt.show()
```

These are just samples generated from an encoding of a true original sample. What if we want to generate completely new synthetic digits?
For this we need to sample vectors in our latent space.
Naively we could just sample vectors from a random gaussian.

```python
samples = torch.randn(16, 64)

with torch.no_grad():
    model.eval()
    decoded_samples = model[1](samples)

figure = plt.figure(figsize=(10, 8))
cols, rows = 4, 4
for i in range(cols * rows):
    img = decoded_samples[i]
    figure.add_subplot(rows, cols, i+1)
    plt.axis("off")
    plt.imshow(img.reshape(-1, 28, 28).squeeze(), cmap="gray")
plt.show()
```

The images vaguely look like something that could be digits, but are not convincing to say the least.
One way to make these generations better is to not sample random vectors, but try to inform our sampling. For this dataset, we have class labels, so we can try to see what distribution samples from a certain class (e.g. 7) generally follow in the latent space. We can then sample from that distribution, instead of from a standard gaussian.

To do this: we (1) first encode samples in the latent space, (2) take the samples belonging to the cluster we're interested in, (3) fit a distribution to the latent space: in this case we will simply compute the mean and standard deviation of all hidden dimensions, and (4) finally, sample some vectors from that distribution.

```python
class_ = 7

encoded_inputs = []
classes = []
with torch.no_grad():
    model.eval()
    for batch in val_dataloader:
        X_batch, y_batch = batch
        encoded_inputs.append(model[0](X_batch))
        classes.append(y_batch)

classes = torch.cat(classes)
encoded_inputs = torch.cat(encoded_inputs)

encoded_inputs_class = encoded_inputs[classes == class_]

mu = encoded_inputs_class.mean(0)
std = encoded_inputs_class.std(0)
print(mu)
print(std)

figure = plt.figure(figsize=(10, 8))
cols, rows = 4, 4
for i in range(cols * rows):
    sampled_vector = torch.normal(mu, std).unsqueeze(0)
    with torch.no_grad():
        model.eval()
        decoded_sample = model[1](sampled_vector)
    img = decoded_sample
    figure.add_subplot(rows, cols, i+1)
    plt.axis("off")
    plt.imshow(img.reshape(-1, 28, 28).squeeze(), cmap="gray")
plt.show()

```

Looks like conditioning on a class-specific distribution helps the model along a bit, but it is still clear that not every sampled vector taken from the latent space gives us a convincing decoded image. An advanced technique based on autoencoders to give us better image generation quality is the Variational Autoencoder, which is out of scope for this course. Another (non-autoencoder) based technique is GANs.

### Extra: Using GPUs

Matrix multiplication run orders of magnitude faster on GPU hardware. If you have a local GPU and have installed a PyTorch version with GPU, you should be able to run this code locally. Otherwise, In Google Colab, you can request access to a GPU via `Runtime > Change runtime type > Hardware accelerator = GPU`.

Briefly, the steps needed to train on GPUs consist of
1. Putting your model on the GPU
2. During your training loop, putting every batch on the GPU before the forward pass
3. If you have a validation loop, doing the same there for every batch.
4. If you have variables that you will use after training (e.g. predictions on the validation set), remember to return this back to the CPU, as GPUs have limited memory.

In PyTorch, we put variables and models on the GPU by specifying their 'device' to be 'cuda' (the parallel computing platform for nvidia GPUs that PyTorch uses).

The following code illustrates how to train your models on GPU hardware:

```python
X, y = next(iter(train_dataloader))

model = model.to('cuda')
print(X.device)
X = X.to('cuda')
print(X.device)

y_hat = model(X)

y_hat
```

We encourage you to try out training on GPUs during the next PC lab(s)


# Extended Kalman Filter initilal Parameters:

```python
import numpy as np
%matplotlib inline
import matplotlib.dates as mdates
import matplotlib.pyplot as plt
from scipy.stats import norm
from sympy import Symbol, symbols, Matrix, sin, cos
from sympy import init_printing
init_printing(use_latex=True)
```

```python
#Total state variables
numstates=4
```

```python
vs, psis, dts, xs, ys, lats, lons = symbols('v \phi a_t x y lat lon')

gs = Matrix([[xs+vs*dts*cos(psis)],
             [ys+vs*dts*sin(psis)],
             [psis],
             [vs]])
state = Matrix([xs,ys,psis,vs])
```

```python
#dynamic fuction:
gs
```

$\displaystyle \left[\begin{matrix}a_{t} v \cos{\left(\phi \right)} + x\\a_{t} v \sin{\left(\phi \right)} + y\\\phi\\v\end{matrix}\right]$

```python
#State vector Sk = [x,y,phi,v]
state
```

$\displaystyle \left[\begin{matrix}x\\y\\\phi\\v\end{matrix}\right]$

```python
# Calculating the Jacoian Jg
gs.jacobian(state)
```

$\displaystyle \left[\begin{matrix}1 & 0 & - a_{t} v \sin{\left(\phi \right)} & a_{t} \cos{\left(\phi \right)}\\0 & 1 & a_{t} v \cos{\left(\phi \right)} & a_{t} \sin{\left(\phi \right)}\\0 & 0 & 1 & 0\\0 & 0 & 0 & 1\end{matrix}\right]$

It has to be computed on every filter step because it consists of state variables!

```python
P = np.eye(numstates)*1000.0
print(P, P.shape)
```

    [[1000.    0.    0.    0.]
     [   0. 1000.    0.    0.]
     [   0.    0. 1000.    0.]
     [   0.    0.    0. 1000.]] (4, 4)

## Measurements:

```python
data = 'drone_data.csv'

millis, \
latitude, \
longitude, \
altitude, \
velocity, \
heading = np.loadtxt(data, delimiter=',', unpack= True, skiprows = 1)

print('Read \'%s\' successfully.' % data)

```

    Read 'drone_data.csv' successfully.

```python
heading = (-heading)
```

### Calculate `dt` (time-steps) for Measurements

```python
dt = np.hstack([0.02, np.diff(millis)])/1000.0 # in s
```

```python
hs = Matrix([[xs],
             [ys]])
hs
```

$\displaystyle \left[\begin{matrix}x\\y\end{matrix}\right]$

```python
JHs=hs.jacobian(state)
JHs
```

$\displaystyle \left[\begin{matrix}1 & 0 & 0 & 0\\0 & 1 & 0 & 0\end{matrix}\right]$

If no GPS measurement is available, simply set the corresponding values in $J_h$ to zero.

```python
varGPS = 6.0 # Standard Deviation of GPS Measurement
Rk = np.diag([varGPS**2.0, varGPS**2.0])

print(Rk, Rk.shape)
```

    [[36.  0.]
     [ 0. 36.]] (2, 2)

## Identity Matrix

```python
I = np.eye(numstates)
print(I, I.shape)
```

    [[1. 0. 0. 0.]
     [0. 1. 0. 0.]
     [0. 0. 1. 0.]
     [0. 0. 0. 1.]] (4, 4)

## Approx. Lat/Lon to Meters to check Location

```python
RadiusEarth = 6378388.0 # m
arc= 2.0*np.pi*(RadiusEarth+altitude)/360.0 # m/°

dx = arc * np.cos(latitude*np.pi/180.0) * np.hstack((0.0, np.diff(longitude))) # in m
dy = arc * np.hstack((0.0, np.diff(latitude))) # in m

mx = np.cumsum(dx)
my = np.cumsum(dy)

ds = np.sqrt(dx**2+dy**2)

GPS=(ds!=0.0).astype('bool') # GPS Trigger for Kalman Filter
```

## Initial State

```python
x = np.matrix([[mx[5], my[10], 0.5*np.pi, 0.0]]).T
print(x, x.shape)
```

    [[-0.00380552]
     [-0.00890594]
     [ 1.57079633]
     [ 0.        ]] (4, 1)

###  Measurement vector

```python
measurements = np.vstack((mx, my))
# Lenth of the measurement
m = measurements.shape[1]
print(measurements.shape)
```

    (2, 12017)

```python
# Preallocation for Plotting
x0, x1, x2, x3 = [], [], [], []
Zx, Zy = [], []
Px, Py, Pdx, Pdy = [], [], [], []
Kx, Ky, Kdx, Kdy = [], [], [], []

def savestates(x, Z, P, K):
    x0.append(float(x[0]))
    x1.append(float(x[1]))
    x2.append(float(x[2]))
    x3.append(float(x[3]))
    Zx.append(float(Z[0]))
    Zy.append(float(Z[1]))
    Px.append(float(P[0,0]))
    Py.append(float(P[1,1]))
    Pdx.append(float(P[2,2]))
    Pdy.append(float(P[3,3]))
    Kx.append(float(K[0,0]))
    Ky.append(float(K[1,0]))
    Kdx.append(float(K[2,0]))
    Kdy.append(float(K[3,0]))
```

# Extended Kalman Filter solution:

```python
for filterstep in range(m):

    # Time Update (Prediction)
    # ========================
    # Project the state ahead
    # see "Dynamic Matrix"
    x[0] = x[0] + dt[filterstep]*x[3]*np.cos(x[2])
    x[1] = x[1] + dt[filterstep]*x[3]*np.sin(x[2])
    x[2] = (x[2]+ np.pi) % (2.0*np.pi) - np.pi
    x[3] = x[3]

    # Calculate the Jacobian of the Dynamic Matrix A
    # see "Calculate the Jacobian of the Dynamic Matrix with respect to the state vector"
    a13 = -dt[filterstep]*x[3]*np.sin(x[2])
    a14 = dt[filterstep]*np.cos(x[2])
    a23 = dt[filterstep]*x[3]*np.cos(x[2])
    a24 = dt[filterstep]*np.sin(x[2])
    JA = np.matrix([[1.0, 0.0, a13, a14],
                    [0.0, 1.0, a23, a24],
                    [0.0, 0.0, 1.0, 0.0],
                    [0.0, 0.0, 0.0, 1.0]])

    # Calculate the Process Noise Covariance Matrix
    sGPS     = 0.5*8.8*dt[filterstep]**2  # assume 8.8m/s2 as maximum acceleration
    sCourse  = 2.0*dt[filterstep] # assume 0.5rad/s as maximum turn rate
    sVelocity= 35.0*dt[filterstep] # assume 8.8m/s2 as maximum acceleration

    Qk = np.diag([sGPS**2, sGPS**2, sCourse**2, sVelocity**2])

    # Project the error covariance ahead
    P = JA*P*JA.T + Qk
    P = P.astype(np.float64)
    # Measurement Update (Correction)
    # ===============================
    # Measurement Function
    hx = np.matrix([[float(x[0])],
                    [float(x[1])]])

    if GPS[filterstep]: # with 10Hz, every 5th step
        JH = np.matrix([[1.0, 0.0, 0.0, 0.0],
                        [0.0, 1.0, 0.0, 0.0]])
    else: # every other step
        JH = np.matrix([[0.0, 0.0, 0.0, 0.0],
                        [0.0, 0.0, 0.0, 0.0]])

    Ck = JH*P*JH.T + Rk
    Ck = Ck.astype(np.float64)

    K = (P*JH.T) * np.linalg.inv(Ck)
    K = K.astype(np.float64)

    # Update the estimate via
    Z = measurements[:,filterstep].reshape(JH.shape[0],1)
    y = Z - (hx)                         # Innovation or Residual
    x = x + (K*y)

    # Update the error covariance
    P = (I - (K*JH))*P

    # Save states for Plotting
    savestates(x, Z, P, K)
```

    C:\Users\singh\anaconda3\lib\site-packages\numpy\matrixlib\defmatrix.py:145: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray
      arr = N.array(data, dtype=dtype, copy=copy)

```python
def plotx():
    fig = plt.figure(figsize=(16,16))

    plt.subplot(311)
    plt.step(range(len(measurements[0])),x0-mx[0], label='$x$')
    plt.step(range(len(measurements[0])),x1-my[0], label='$y$')

    plt.title('Extended Kalman Filter State Estimates (State Vector $x$)')
    plt.legend(loc='best',prop={'size':22})
    plt.ylabel('Position (relative to start) [m]')

    plt.subplot(312)
    plt.step(range(len(measurements[0])),x2, label='$\psi$')
    plt.step(range(len(measurements[0])),
             (heading/180.0*np.pi+np.pi)%(2.0*np.pi) - np.pi,
             label='$\psi$ (from GPS as reference)')
    plt.ylabel('Heading')
    plt.legend(loc='best',prop={'size':16})

    plt.subplot(313)
    plt.step(range(len(measurements[0])),x3, label='$v$')
    plt.step(range(len(measurements[0])),velocity/3.6,
             label='$v$ (from GPS as reference)')
    plt.ylabel('Velocity')
    #plt.ylim([0, 30])
    plt.legend(loc='best',prop={'size':16})
    plt.xlabel('Filter Step')

    plt.savefig('Extended-Kalman-Filter-CHCV-State-Estimates.png',
                dpi=72, transparent=True, bbox_inches='tight')
```

```python
plotx()
```

```python
def plotxy():

    fig = plt.figure(figsize=(16,9))

    # EKF State
    plt.quiver(x0,x1,np.cos(x2), np.sin(x2), color='#94C600',
               units='xy', width=0.05, scale=0.5)
    plt.scatter(x0,x1, label='EKF Position', c='g', lw=5)

    # Measurements
    plt.scatter(mx[::5],my[::5], s=50, label='GPS Measurements', marker='+')
    #cbar=plt.colorbar(ticks=np.arange(20))
    #cbar.ax.set_ylabel(u'EPE', rotation=270)
    #cbar.ax.set_xlabel(u'm')

    plt.xlabel('X [m]')
    plt.ylabel('Y [m]')
    plt.title('Position')
    plt.legend(loc='best')
    plt.axis('equal')
    #plt.tight_layout()
    plt.grid()
    plt.savefig('Extended-Kalman-Filter-CHCV-Position.png',
                dpi=72, transparent=True, bbox_inches='tight')
```

```python
plotxy()
```

```python
def plotxydetails():
    fig = plt.figure(figsize=(12,9))

    plt.subplot(221)
    # EKF State
    #plt.quiver(x0,x1,np.cos(x2), np.sin(x2), color='#94C600', units='xy', width=0.05, scale=0.5)
    plt.plot(x0,x1, label='EKF Position', c='g', lw=5)

    # Measurements
    plt.scatter(mx[::5],my[::5], s=100, label='GPS Measurements', alpha=0.5, marker='+')
    #cbar=plt.colorbar(ticks=np.arange(20))
    #cbar.ax.set_ylabel(u'EPE', rotation=270)
    #cbar.ax.set_xlabel(u'm')

    plt.xlabel('X [m]')
    plt.xlim(70, 130)
    plt.ylabel('Y [m]')
    plt.ylim(140, 200)
    plt.title('Position')
    plt.legend(loc='best')

    plt.subplot(222)

    # EKF State
    #plt.quiver(x0,x1,np.cos(x2), np.sin(x2), color='#94C600', units='xy', width=0.05, scale=0.5)
    plt.plot(x0,x1, label='EKF Position', c='g', lw=1, marker = '*')

    # Measurements
    plt.scatter(mx[::5],my[::5], s=100, label='GPS Measurements', alpha=0.5, marker='+')
    #cbar=plt.colorbar(ticks=np.arange(20))
    #cbar.ax.set_ylabel(u'EPE', rotation=270)
    #cbar.ax.set_xlabel(u'm')

    plt.xlabel('X [m]')
    plt.xlim(0, 260)
    plt.ylabel('Y [m]')
    plt.ylim(0, 160)
    plt.title('Position')
    plt.grid()
    plt.legend(loc='best')
```

```python
plotxydetails()
```

```python
def plot_K():
    fig = plt.figure(figsize=(16,9))
    plt.plot(range(len(measurements[0])),Kx, label='Kalman Gain for $x$')
    plt.plot(range(len(measurements[0])),Ky, label='Kalman Gain for $y$')
    plt.plot(range(len(measurements[0])),Kdx, label='Kalman Gain for $\dot x$')
    plt.plot(range(len(measurements[0])),Kdy, label='Kalman Gain for $\dot y$')

    plt.xlabel('Filter Step')
    plt.ylabel('')
    plt.title('Kalman Gain (the lower, the more the measurement fullfill the prediction)')
    plt.legend(loc='best',prop={'size':22})

plot_K()
```

```python

```


# Introduction

Understanding the behavior of neural networks and why they generalize has been a central pursuit of the theoretical deep learning community.
Our paper [*A Fine-Grained Spectral Perspective on Neural Networks*](https://arxiv.org/abs/1907.10599) attacks this problem by looking at eigenvalues and eigenfunctions, as the name suggests.
We will study the spectra of the *Conjugate Kernel* [[Daniely et al. 2017](http://papers.nips.cc/paper/6427-toward-deeper-understanding-of-neural-networks-the-power-of-initialization-and-a-dual-view-on-expressivity.pdf)], or CK (also called the *Neural Network-Gaussian Process Kernel* [[Lee et al. 2018](http://arxiv.org/abs/1711.00165)]), and the *Neural Tangent Kernel*, or NTK [[Jacot et al. 2018](http://arxiv.org/abs/1806.07572)].
Roughly, the CK and the NTK tell us respectively "what a network looks like at initialization" and "what a network looks like during and after training."
Their spectra then encode valuable information about the initial distribution and the training and generalization properties of neural networks.

## Intuition for the utility of the spectral perspective

Let's take the example of the CK.
We know from [Lee et al. (2018)](http://arxiv.org/abs/1711.00165) that a randomly initialized network is distributed as a Gaussian process $\mathcal N(0, K)$, where $K$ is the corresponding CK, in the infinite-width limit.
If we have the eigendecomposition
\begin{equation}
K = \sum_{i \ge 1} \lambda_i u_i\otimes u_i
\label{eqn:eigendecomposition}
\end{equation}
with eigenvalues $\lambda_i$ in decreasing order and corresponding eigenfunctions $u_i$, then each sample from this GP can be obtained as
$$
\sum_{i \ge 1} \sqrt{\lambda_i} \omega_i u_i,\quad
\omega_i \sim \mathcal N(0, 1).
$$
Training the last layer of a randomly initialized network via full batch gradient descent for an infinite amount of time corresponds to Gaussian process inference with kernel $K$ [[Lee et al. 2018](http://arxiv.org/abs/1711.00165), [2019](http://arxiv.org/abs/1902.06720)].
Thus, the more the GP prior (governed by the CK) is consistent with the ground truth function $f^*$, the more we expect the Gaussian process inference and GD training to generalize well.
We can measure this consistency in the "alignment" between the eigenvalues $\lambda_i$ and the squared coefficients $a_i^2$ of $f^*$'s expansion in the $\{u_i\}_i$ basis.
The former can be interpreted as the expected magnitude (squared) of the $u_i$-component of a sample $f \sim \mathcal N(0, K)$, and the latter can be interpreted as the actual magnitude squared of such component of $f^*$.
Here and in this paper, we will investigate an even cleaner setting where $f^* = u_i$ is an eigenfunction.
Thus we would hope to use a kernel whose $i$th eigenvalue $\lambda_i$ is as large as possible.

A similar intuition holds for NTK, because training all parameters of the network for an infinite amount of time yields the mean prediction of the GP $\mathcal N(0, \text{NTK})$ in expectation [[Lee et al. 2019](http://arxiv.org/abs/1902.06720)].

## A brief summary of the spectral theory of CK and NTK

Now, if the CK and the NTK have spectra difficult to compute, then this perspective is not so useful.
But in idealized settings, where the data distribution is uniform over the boolean cube, the sphere, or from the standard Gaussian, a complete (or almost complete in the Gaussian case) eigendecomposition of the kernel can be obtained, thanks to the symmetry of the domain.
Here and in the paper, we focus on the boolean cube, since in high dimensions, all three distributions are very similar, and the boolean cube eigenvalues are much easier to compute (see paper for more details).

We briefly summarize the spectral theory of CK and NTK (of multilayer perceptrons, or MLPs) on the boolean cube.
First, these kernels are always diagonalized by the *boolean Fourier basis*, which are just monomial functions like $x_1 x_3 x_{10}$.
These Fourier basis functions are naturally graded by their *degree*, ranging from 0 to the dimension $d$ of the cube.
Then the kernel has $d+1$ unique eigenvalues,
$$\mu_0, \ldots, \mu_d$$
corresponding to each of the degrees, so that the eigenspace associated to $\mu_k$ is a $\binom d k$ dimensional space of monomials with degree $k$.
These eigenvalues are simple linear functions of a small number of the kernel values, and can be easily computed.

Let's try computing them ourselves!

# Computing Eigenvalues over a Grid of Hyperparameters

```python
import numpy as np
import scipy as sp
from scipy.special import erf as erf
import matplotlib.pyplot as plt
from itertools import product
import seaborn as sns
sns.set()
from mpl_toolkits.axes_grid1 import ImageGrid
def tight_layout(plt):
    plt.tight_layout(rect=[0, 0.03, 1, 0.95])
```

Our methods for doing the theoretical computations lie in the `theory` module.

```python
from theory import *
```

First, let's compute the eigenvalues of erf CK and NTK over a large range of hyperparameters:

- $\sigma_w^2 \in [1, 5]$
- $\sigma_b^2 \in [0, 4]$
- dimension 128 boolean cube
- depth up to 128
- degree $k \le 8$.

Unless stated otherwise, all plots below use these hyperparameters.

We will do the same for relu kernels later.

```python
# range of $\sigma_b^2$
erfvbrange = np.linspace(0, 4, num=41)
# range of $\sigma_w^2$
erfvwrange = np.linspace(1, 5, num=41)
erfvws, erfvbs = np.meshgrid(erfvwrange, erfvbrange, indexing='ij')
# `dim` = $d$
dim = 128
depth = 128
# we will compute the eigenvalues $\mu_k$ for $k = 0, 1, ..., maxdeg$.
maxdeg = 8
```

As mentioned in the paper, any CK or NTK $K$ of multilayer perceptrons (MLPs) takes the form
$$K(x, y) = \Phi\left(\frac{\langle x, y \rangle}{\|x\|\|y\|}, \frac{\|x\|^2}d, \frac{\|y\|^2}d\right)$$
for some function $\Phi: \mathbb R^3 \to \mathbb R$.
On the boolean cube $\{1, -1\}^d$, $\|x\|^2 = d$ for all $x$, and $\langle x, y \rangle / d$ takes value in a discrete set $\{-1, -1+2/d, \ldots, 1-2/d, 1\}$.
Thus $K(x, y)$ only takes a finite number of different values as well.
We first compute these values (see paper for the precise formulas).

```python
# `erfkervals` has two entries, with keys `cks` and `ntks`.
# Each entry is an array with shape (`depth`, len(erfvwrange), len(erfvbrange), `dim`+1)
# The last dimension carries the entries $\Phi(-1), \Phi(-1 + 2/d), ..., \Phi(1)$
erfkervals = boolcubeFgrid(dim, depth, erfvws, erfvbs, VErf, VDerErf)
```

```python
erfkervals['cks'].shape
```

    (129, 41, 41, 129)

The eigenvalues $\mu_k, k = 0, 1, \ldots, d$, can be expressed as a simple linear function of $\Phi$'s values, as hinted before.
However, a naive evaluation would lose too much numerical precision because the number of alternating terms.
Instead, we do something more clever, resulting in the following algorithm:

- For $\Delta = 2/d$, we first evaluate $\Phi^{(a)}(x) = \frac 1 2 \left(\Phi^{(a-1)}(x) - \Phi^{(a-1)}(x - \Delta)\right)$ with base case $\Phi^{(0)} = \Phi$, for $a = 0, 1, \ldots$, and for various values of $x$.
- Then we just sum a bunch of nonnegative terms to get the eigenvalue $\mu_k$ associated to degree $k$ monomials
$$\mu_k = \frac 1{2^{d-k}} \sum_{r=0}^{d-k}\binom{d-k}r \Phi^{(k)}(1 - r \Delta).$$

We will actually use an even more clever algorithm here, but with the same line of the reasoning; see the paper and the `twostep` option in the source code for details.

Note that, here we will compute *normalized eigenvalues*, normalized by their trace.
So these normalized eigenvalues, with multiplicity, should sum up to 1.

```python
erfeigs = {}
# `erfeigs['ck']` is an array with shape (`maxdeg`, `depth`+1, len(erfvwrange), len(erfvbrange))
# `erfeigs['ck'][k, L] is the matrix of eigenvalue $\mu_k$ for a depth $L$ erf network,
# as a function of the values of $\sigma_w^2, \sigma_b^2$ in `erfvwrange` and `erfvbrange`
# Note that these eigenvalues are normalized by the trace
# (so that all normalized eigenvalues sum up to 1)
erfeigs['ck'] = relu(boolCubeMuAll(dim, maxdeg, erfkervals['cks']))
# similarly for `erfeigs['ntk']`
erfeigs['ntk'] = relu(boolCubeMuAll(dim, maxdeg, erfkervals['ntks']))
```

To perform fine-grained analysis of how hyperparameters affects the performance of the kernel and thus the network itself, we use a heuristic, the *fractional variance*, defined as
$$
\text{degree $k$ fractional variance} = \frac{\binom d k \mu_k}{\sum_{i=0}^d \binom d i \mu_i}.
$$
This terminology comes from the fact that, if we were to sample a function $f$ from a Gaussian process with kernel $K$, then we expect that $r\%$ of the total variance of $f$ comes from degree $k$ components of $f$, where $r\%$ is the degree $k$ fractional variance.
If we were to try to learn a homogeneous degree-$k$ polynomial using a kernel $K$, intuitively we should try to choose $K$ such that its $\mu_k$ is maximized.
In the paper, we present empirical evidence that fractional variance is indeed inversely correlated with test loss.

So let's compute them.

```python
# `erfeigs['ckfracvar']` is an array with shape (`maxdeg`, `depth`+1, len(erfvwrange), len(erfvbrange))
# just like `erfeigs['ck']`
erfeigs['ckfracvar'] = (
    sp.special.binom(dim, np.arange(0, maxdeg+1))[:, None, None, None]
    * erfeigs['ck']
)
# Same thing here
erfeigs['ntkfracvar'] = (
    sp.special.binom(dim, np.arange(0, maxdeg+1))[:, None, None, None]
    * erfeigs['ntk']
)
```

```python
erfeigs['ckfracvar'].shape
```

    (9, 129, 41, 41)

Similarly, let's compute the eigenvalues of ReLU CK and NTK over a large range of hyperparameters:

- $\sigma_w^2 = 2$
- $\sigma_b^2 \in [0, 4]$
- dimension 128 boolean cube
- depth up to 128
- degree $k \le 8$.

Unless stated otherwise, all plots below use these hyperparameters.

```python
reluvws, reluvbs = np.meshgrid([2], np.linspace(0, 4, num=401), indexing='ij')
dim = 128
depth = 128
maxdeg = 8

relukervals = boolcubeFgrid(dim, depth, reluvws, reluvbs, VReLU, VStep)

relueigs = {}
relueigs['ck'] = relu(boolCubeMuAll(dim, maxdeg, relukervals['cks']))
relueigs['ntk'] = relu(boolCubeMuAll(dim, maxdeg, relukervals['ntks']))
relueigs['ckfracvar'] = (
    sp.special.binom(dim, np.arange(0, maxdeg+1))[:, None, None, None]
    * relueigs['ck']
)
relueigs['ntkfracvar'] = (
    sp.special.binom(dim, np.arange(0, maxdeg+1))[:, None, None, None]
    * relueigs['ntk']
)
```

Now we have computed all the eigenvalues, let's take a look at them!

# Deeper Networks Learn More Complex Features --- But Not Too Deep

If $K$ were to be the CK or NTK of a relu or erf MLP, then we find that for higher $k$, depth of the network helps increase $\mu_k$.

```python
maxdeg = 8
plt.figure(figsize=(12, 4))

relueigs['ntkbestdepth'] = np.argmax(np.max(relueigs['ntk'][1:, :, ...], axis=(2, 3)), axis=1).squeeze()
relueigs['ckbestdepth'] = np.argmax(np.max(relueigs['ck'][1:, :, ...], axis=(2, 3)), axis=1).squeeze()
fig = plt.subplot(141)
plt.text(-.2, -.15, '(a)', fontsize=24, transform=fig.axes.transAxes)
plt.plot(np.arange(1, maxdeg+1), relueigs['ntkbestdepth'], label='ntk', markersize=4, marker='o')
plt.plot(np.arange(1, maxdeg+1), relueigs['ckbestdepth'], label='ck', markersize=4, marker='o')
plt.legend()
plt.xlabel('degree')
plt.ylabel('optimal depth')
plt.title('relu kernel optimal depths')

erfeigs['ntkbestdepth'] = np.argmax(np.max(erfeigs['ntk'][1:, :, ...], axis=(2, 3)), axis=1).squeeze()
erfeigs['ckbestdepth'] = np.argmax(np.max(erfeigs['ck'][1:, :, ...], axis=(2, 3)), axis=1).squeeze()
fig = plt.subplot(142)
plt.text(-.2, -.15, '(b)', fontsize=24, transform=fig.axes.transAxes)
plt.plot(np.arange(1, maxdeg+1), erfeigs['ntkbestdepth'], label='ntk', markersize=4, marker='o')
plt.plot(np.arange(1, maxdeg+1), erfeigs['ckbestdepth'], label='ck', markersize=4, marker='o')
plt.legend()
plt.xlabel('degree')
plt.title('erf kernel optimal depths')

fig = plt.subplot(143)
plt.text(-.5, -.15, '(c)', fontsize=24, transform=fig.axes.transAxes)
plt.imshow(relueigs['ntkfracvar'][3:, :20, 0, 0].T, aspect=12/20, origin='lower', extent=[2.5, 8.5, -.5, 20.5])
cb = plt.colorbar()
plt.xticks(range(3, 9, 2))
plt.xlabel('degree')
plt.ylabel('depth')
plt.grid()
plt.title(u'relu ntk, $\sigma_b^2=0$')

fig = plt.subplot(144)
plt.text(-.5, -.15, '(d)', fontsize=24, transform=fig.axes.transAxes)
plt.imshow(erfeigs['ntkfracvar'][3:, :, 0, 0].T, aspect=12/129, origin='lower', extent=[2.5, 8.5, -.5, 128.5], vmin=0, vmax=0.21)
cb = plt.colorbar()
cb.set_label('fractional variance')
plt.xticks(range(3, 9, 2))
plt.xlabel('degree')
plt.grid()
plt.title(u'erf ntk, $\sigma_w^2=1$, $\sigma_b^2=0$')

tight_layout(plt)
```

In **(a)** and **(b)** above, we plot, for each degree $k$, the depth that (with some combination of other hyperparameters like $\sigma_b^2$) maximizes degree $k$ fractional variance, for respectively relu and erf kernels.
Clearly, the maximizing depths are increasing with $k$ for relu, and also for erf when considering either odd $k$ or even $k$ only.
The slightly differing behavior between even and odd $k$ is expected, as seen in the form of Theorem 4.1 in the paper.
Note the different scales of y-axes for relu and erf --- the depth effect is much stronger for erf than relu.

For relu NTK and CK, $\sigma_b^2=0$ maximizes fractional variance in general, and the same holds for erf NTK and CK in the odd degrees (see our other notebook, [TheCompleteHyperparameterPicture]()).
In **(c)** and **(d)**, we give a more fine-grained look at the $\sigma_b^2=0$ slice, via heatmaps of fractional variance against degree and depth.
Brighter color indicates higher variance, and we see the optimal depth for each degree $k$ clearly increases with $k$ for relu NTK, and likewise for odd degrees of erf NTK.
However, note that as $k$ increases, the difference between the maximal fractional variance and those slightly suboptimal becomes smaller and smaller, reflected by suppressed range of color moving to the right.
The heatmaps for relu and erf CKs look similar (compute them yourself, as an exercise!).

In the paper, this trend of increasing optimal depth is backed up empirical data from training neural networks to learn polynomials of various degrees.

Note that implicit in our results here is a highly nontrivial observation:
Past some point (the *optimal depth*), high depth can be detrimental to the performance of the network, beyond just the difficulty to train, and this detriment can already be seen in the corresponding NTK or CK.
In particular, it's *not* true that the optimal depth is infinite.
This adds significant nuance to the folk wisdom that "depth increases expressivity and allows neural networks to learn more complex features."

# NTK Favors More Complex Features Than CK

We generally find the degree $k$ fractional variance of NTK to be higher than that of CK when $k$ is large, and vice versa when $k$ is small.

```python
plt.figure(figsize=(14, 4))
def convert2vb(i):
    return i/100

fig = plt.subplot(131)
plt.text(-.15, -.15, '(a)', fontsize=24, transform=fig.axes.transAxes)
cpal = sns.color_palette()
for i, (depth, vbid) in enumerate([(1, 10), (128, 300), (3, 0)]):
    color = cpal[i]
    plt.plot(relueigs['ntkfracvar'][:, depth, 0, vbid], c=color, label='{} | {}'.format(depth, convert2vb(vbid)), marker='o', markersize=4)
    plt.plot(relueigs['ckfracvar'][:, depth, 0, vbid], '--', c=color, marker='o', markersize=4)
plt.plot([], c='black', label='ntk')
plt.plot([], '--', c='black', label='ck')
plt.legend(title=u'depth | $\sigma_b^2$')
plt.xlabel('degree')
plt.ylabel('fractional variance')
plt.title('relu examples')
plt.semilogy()

def convert2vb(i):
    return i/10
def convert2vw(i):
    return i/10 + 1
cpal = sns.color_palette()
fig = plt.subplot(132)
plt.text(-.15, -.15, '(b)', fontsize=24, transform=fig.axes.transAxes)
for i, (depth, vwid, vbid) in enumerate([(1, 10, 1),(24, 0, 1), (1, 40, 40)]):
    color = cpal[i]
    plt.plot(erfeigs['ntkfracvar'][:, depth, vwid, vbid], c=color,
             label='{} | {} | {}'.format(depth, int(convert2vw(vwid)), convert2vb(vbid)), marker='o', markersize=4)
    plt.plot(erfeigs['ckfracvar'][:, depth, vwid, vbid], '--', c=color, marker='o', markersize=4)
plt.plot([], c='black', label='ntk')
plt.plot([], '--', c='black', label='ck')
plt.legend(title=u'depth | $\sigma_w^2$ | $\sigma_b^2$')
plt.xlabel('degree')
plt.title('erf examples')
plt.semilogy()

fig = plt.subplot(133)
plt.text(-.15, -.15, '(c)', fontsize=24, transform=fig.axes.transAxes)
# relu
balance = np.mean((relueigs['ntk'] > relueigs['ck']) & (relueigs['ntk'] > 1e-15), axis=(1, 2, 3))
# needed to filter out all situations where eigenval is so small that it's likely just 0
balance /= np.mean((relueigs['ntk'] > 1e-15), axis=(1, 2, 3))
plt.plot(np.arange(0, maxdeg+1), balance, marker='o', label='relu')
# erf
balance = np.mean((erfeigs['ntk'] > erfeigs['ck']) & (erfeigs['ntk'] > 1e-15), axis=(1, 2, 3))
# needed to filter out all situations where eigenval is so small that it's likely just 0
balance /= np.mean((erfeigs['ntk'] > 1e-15), axis=(1, 2, 3))
plt.plot(np.arange(0, maxdeg+1), balance, marker='o', label='erf')

plt.xlabel('degree')
plt.ylabel('fraction')
plt.legend()
plt.title('fraction of hyperparams where ntk > ck')

plt.suptitle('ntk favors higher degrees compared to ck')
tight_layout(plt)
```

In **(a)**, we give several examples of the fractional variance curves for relu CK and NTK across several representative hyperparameters.
In **(b)**, we do the same for erf CK and NTK.
In both cases, we clearly see that, while for degree 0 or 1, the fractional variance is typically higher for CK, the reverse is true for larger degrees.
In **(c)**, for each degree $k$, we plot the *fraction of hyperparameters* where the degree $k$ fractional variance of NTK is greater than that of CK.
Consistent with previous observations, this fraction increases with the degree.

This means that, if we train only the last layer of a neural network (i.e. CK dynamics), we intuitively should expect to learn *simpler* features faster and generalize better, while, if we train all parameters of the network (i.e. NTK dynamics), we should expect to learn more *complex* features faster and generalize better.
Similarly, if we were to sample a function from a Gaussian process with the CK as kernel (recall this is just the distribution of randomly initialized infinite width MLPs), this function is more likely to be accurately approximated by low degree polynomials than the same with the NTK.

Again, in the paper, we present empirical evidence that, training the last layer is better than training all layers only for degree 0 polynomials, i.e. constant functions, but is worse for all higher degree (homogenous) polynomials.
This corroborates the observations made from the spectra here.

# Conclusion

We have replicated the spectral plots of section 5 and 6 in the paper, concerning the generalization properties of neural networks.
As one can see, the spectral perspective is quite useful for understanding the effects of hyperparameters.
A complete picture of how the combination of $\sigma_w^2, \sigma_b^2$, depth, degree, and nonlinearity affects fractional variance is presented in the notebook *[The Complete Hyperparameter Picture](TheCompleteHyperparameterPicture.ipynb)*.
The fractional variance is, however, by no means a perfect indicator of generalization, and there's plenty of room for improvement, as mentioned in the main text.
We hope a better predictor of test loss can be obtained in future works.

Another interesting topic that spectral analysis can shed light on is the so-called "simplicity bias" of neural networks.
We discuss this in the notebook *[Clarifying Simplicity Bias](ClarifyingSimplicityBias.ipynb)*.


# Introduction to Time Series Models

# **Autoregressive (AR) Models**

$$ y_t = \beta + \epsilon_t + \sum_{i=1}^{p} \phi_i y_{t-i} $$
where $\beta$ is a constant, $\epsilon_t$ is the white noise at time t, and $p$ is the number of time lags to regress on. We refer to this as an AR($p$) model, an autoregressive model of order $p$.

* Forecast the variable of interest using a linear combination of past values of the variable.
* The term "Autoregression" indicates that it is a regression of the variable against itself.
* We normally restrict autoregressive models to **stationary data**, in which case some constraints on the values of the parameters are required.
  * AR($1$) : $-1< \phi_1 < 1$.
  * AR($2$) : $-1< \phi_2 < 1$, $\phi_1+\phi_2 < 1$, $\phi_2-\phi_1 < 1$.

Reference : [Autoregressive models](https://otexts.com/fpp2/AR.html#AR)

# **Moving average (MA) Models**

$$ y_t = \beta + \epsilon_t + \sum_{i=1}^{q} \theta_i \epsilon_{t-i} $$
where $\beta$ is a constant, $\epsilon_t$ is the white noise at time t, and $q$ is the number of time lags of the error term to regress on. We refer to this as an MA($q$) model, a moving average model of order $q$.
* Each value of $y_t$ can be thought of as a weighted moving average of the past few forecast errors.
* It is possible to write any stationary AR($p$) model as an MA($\infty$) model. For example, AR($1$) model
\begin{equation}
\begin{split}
  y_t &= \epsilon_t + \phi_1 y_{t-1} \\
      &= \epsilon_t + \phi_1 (\epsilon_{t-1} + \phi_1 y_{t-2}) \\
      &= \epsilon_t + \phi_1^2 y_{t-2} + \phi_1 \epsilon_{t-1} \\
      &= \epsilon_t + \phi_1^3 y_{t-3} + \phi_1^2 y_{t-2} + \phi_1 \epsilon_{t-1} \\
      &= \text{...... ,} \\   \\
\end{split}
\end{equation}
if $|\phi_1|<1$, then we obtain MA($\infty$)
$$y_t=\epsilon_t+\phi_1 \epsilon_{t-1}+\phi_1^2 \epsilon_{t-2} + \phi_1^3 \epsilon_{t_3} + \dots \text{ .}$$
* On the other hand, if we impose some constraints on the MA parameters (MA model is called **invertible**), we can write any invertible MA($q$) process as an AR($\infty$) process.

Reference : [Moving average models](https://otexts.com/fpp2/MA.html#MA)

# **Non-seasonal ARIMA models**

ARIMA is an acronym for AutoRegressive Integrated Moving Average.
$$y_{t}^{'} = \beta+\phi_1 y_{t-1}^{'}+\dots+\phi_p y_{t-p}^{'}+\theta_1 \epsilon_{t-1}+\dots+\theta_p \epsilon_{t-q}+\epsilon_{t} $$
where $y_t^{'}$ is the differenced series (it may have been differenced more than once). We call this an ARIMA($p$, $d$, $q$) model, where $p$ is the order of the autoregressive part, $d$ is the degree of first differencing involved, and $q$ is the order of the moving average part.

* The same stationarity and invertibility conditions that are used for autoregressive and moving average models also apply to an ARIMA model.
* The constant $\beta$ has an important effect on the long-term forecasts obtained from these models.
  * if $\beta=0$ and $d=0$, the long-term forecasts will go to zero.
  * if $\beta=0$ and $d=1$, the long-term forecasts will go to a non-zero constant.
  * if $\beta=0$ and $d=2$, the long-term forecasts will follow a straight line.
  * if $\beta\neq0$ and $d=0$, the long-term forecasts will go to the mean of the data.
  * if $\beta\neq0$ and $d=1$, the long-term forecasts will follow a straight line.
  * if $\beta\neq0$ and $d=2$, the long-term forecasts will follow a quadratic trend.
* The value of $d$ also has an effect on the prediction intervals — the higher the value of $d$, the more rapidly the prediction intervals increase in size.
* The value of $p$ is important if the data show cycles. To obtain cyclic forecasts, it is necessary to have $p\geq2$, along with some additional conditions on the parameters.
* It is sometimes possible to use the **ACF** plot, and the closely related **PACF** plot, to determine appropriate values for $p$ and $q$.

Reference : [Non-seasonal ARIMA models](https://otexts.com/fpp2/non-seasonal-arima.html#non-seasonal-arima)

# **SARIMAX Models**

SARIMAX (Seasonal Auto-Regressive Integrated Moving Average with eXogenous factors) is an updated version of the ARIMA model. SARIMAX includes **seasonal effects** (季節效應) and **eXogenous factors** (外部因素) with the autoregressive and moving average component in the model. SARIMAX model takes seasonality into account by essentially applying an ARIMA model to lags that are integer multiples of seasonality. Once the seasonality is modeled, an ARIMA model is applied to the leftover to capture non-seasonal structure. One shorthand notation for SARIMAX models is:
$$ \text{SARIMAX( }p, d, q\text{ )}\times\text{( }P, D, Q, S\text{ )}$$
where $p$ is the order of the non-seasonal AR part,
    $d$ is the non-seasonal differencing,
    $q$ is the order of the non-seasonal MA part,
    $P$ is the order of the seasonal AR part,
    $D$ is the seasonal differencing,
    $Q$ is the order of the seasonal MA part, and
    $S$ is the length of repeating seasonal pattern.

* The $S$ parameter influences the $P$, $D$, and $Q$ parameters. For example, an $S$ of 12 for monthly data suggests a yearly seasonal cycle.
* The SARIMAX model can subsume (包含) the AR, MA, and ARIMA models via model configuration parameters.
* The trend and seasonal hyperparameters of the model can be configured by analyzing [autocorrelation](https://colab.research.google.com/github/YenLinWu/Time_Series_Model/blob/main/Materials/Stationarity_Differencing_and_Autocorrelation.ipynb) and [partial autocorrelation plots](https://colab.research.google.com/github/YenLinWu/Time_Series_Model/blob/main/Materials/Stationarity_Differencing_and_Autocorrelation.ipynb), and this can take some expertise.
An alternative approach is to **grid search** a suite of model configurations and discover which configurations work best for a specific univariate time series. This approach can be faster on modern computers than an analysis process and can reveal surprising findings that might not be obvious and result in lower forecast error.
> *Seasonal ARIMA models can potentially have a large number of parameters and combinations of terms. Therefore, it is appropriate to try out a wide range of models when fitting to data and choose a best fitting model using an appropriate criterion …*
Quote : Pages 143-144, [Introductory Time Series with R](https://link.springer.com/book/10.1007/978-0-387-88698-5), Paul S.P. Cowpertwait and Andrew V. Metcalfe, 2009.


## Objective
Remove the signal that explains $Y$ in $X$

## Setting

Let $Y = [y_1, y_2, \ldots, y_n]' \in R^{n\times d}$, where $y_i = (y_{i1}, y_{i2}, \ldots, y_{id})\in R^d$. Let $X = [1_n, x_1, x_2, \ldots, x_n]' \in R^{n \times (p+1)}$ where $1_n$ is a ones vector and $x_i = (x_{i1}, \ldots, x_{ip})$. Assume that $X$ and $Y$ are continuous random variables, that there is a linear (or a monotonic relationship) between $Y$ and $X$ and that $Y$ is dependent on $X$. We can fit a multivariate linear model:
$$ Y = XB + \epsilon $$

$B\in R^{(p+1)\times d}$ represents the coefficient matrix, and $\epsilon \in R^{n\times d}$ is the residuals. Similar to OLS, we can solve the normal equations, and arrive at the usual form
\begin{align}
\hat{Y} &= X\hat{\beta}\\
&= X(X^TX)^{-1}X^TY
\end{align}
Thus,
\begin{align}
\epsilon &= Y - X\hat{\beta}\\
&= Y - X(X^TX)^{-1}X^T\hat{\beta}
\end{align}

This assumes that $X^TX$ is full rank and the covariance matrix of $\epsilon$ is a diagonal

```python
def compute_mvlr(X, Y):
    X_ = np.hstack([np.ones((X.shape[0], 1)), X])
    return Y - X_ @ np.linalg.inv(X_.T @ X_) @ X_.T @ Y
```

```python
def plot_pairwise(X, Y):
    p = X.shape[1]
    d = Y.shape[1]
    fig, ax = plt.subplots(nrows = d, ncols=p, figsize=(10, 10), sharex=True, sharey=True)

    for ydx in range(d):
        if ydx == d-1:
            for i in range(p):
                ax[ydx, i].set_xlabel('X dimension {}'.format(i + 1))
        for xdx in range(p):
            if xdx == 0:
                ax[ydx, xdx].set_ylabel('Y dimension {}'.format(ydx + 1))
            ax[ydx, xdx].scatter(X[:, xdx], Y[:, ydx])
```

### Sample data

$$
X \sim Unif(-1, 1) \in R^{n\times p} \\
Y \sim Xw + c \in R^{n\times d}
$$
where
$$
w \sim Unif(-1, 1) \in R^{p \times d}\\
c \sim Unif(1, 2) \in R^{1 \times d}
$$

We set the following:

$$
n = 100\\
p = 3\\
d = 5
$$

```python
n = 100
p = 2
d = 3

X = np.random.uniform(-1, 1, size=(n, p))
w = np.random.uniform(-1, 1, size=(p, d))
c = np.random.uniform(1, 2, size=(1, d))
Y = X @ w + c
```

```python
plot_pairwise(X, Y)
```

```python
E = compute_mvlr(X, Y)
```

```python
plot_pairwise(X, E)
```

```python

```