```
#hide
!pip install -Uqq fastbook
import fastbook
fastbook.setup_book()
```

```
#hide
from fastai.gen_doc.nbdoc import *
```

# A Neural Net from the Foundations

## Building a Neural Net Layer from Scratch

### Modeling a Neuron

### Matrix Multiplication from Scratch

```
import torch
from torch import tensor
```

```
def matmul(a,b):
    ar,ac = a.shape # n_rows * n_cols
    br,bc = b.shape
    assert ac==br
    c = torch.zeros(ar, bc)
    for i in range(ar):
        for j in range(bc):
            for k in range(ac): c[i,j] += a[i,k] * b[k,j]
    return c
```

```
m1 = torch.randn(5,28*28)
m2 = torch.randn(784,10)
```

```
%time t1=matmul(m1, m2)
```

```
%timeit -n 20 t2=m1@m2
```

### Elementwise Arithmetic

```
a = tensor([10., 6, -4])
b = tensor([2., 8, 7])
a + b
```

```
a < b
```

```
(a < b).all(), (a==b).all()
```

```
(a + b).mean().item()
```

```
m = tensor([[1., 2, 3], [4,5,6], [7,8,9]])
m*m
```

```
n = tensor([[1., 2, 3], [4,5,6]])
m*n
```

```
def matmul(a,b):
    ar,ac = a.shape
    br,bc = b.shape
    assert ac==br
    c = torch.zeros(ar, bc)
    for i in range(ar):
        for j in range(bc): c[i,j] = (a[i] * b[:,j]).sum()
    return c
```

```
%timeit -n 20 t3 = matmul(m1,m2)
```

### Broadcasting

#### Broadcasting with a scalar

```
a = tensor([10., 6, -4])
a > 0
```

```
m = tensor([[1., 2, 3], [4,5,6], [7,8,9]])
(m - 5) / 2.73
```

#### Broadcasting a vector to a matrix

```
c = tensor([10.,20,30])
m = tensor([[1., 2, 3], [4,5,6], [7,8,9]])
m.shape,c.shape
```

```
m + c
```

```
c.expand_as(m)
```

```
t = c.expand_as(m)
t.storage()
```

```
t.stride(), t.shape
```

```
c + m
```

```
c = tensor([10.,20,30])
m = tensor([[1., 2, 3], [4,5,6]])
c+m
```

```
c = tensor([10.,20])
m = tensor([[1., 2, 3], [4,5,6]])
c+m
```

```
c = tensor([10.,20,30])
m = tensor([[1., 2, 3], [4,5,6], [7,8,9]])
c = c.unsqueeze(1)
m.shape,c.shape
```

```
c+m
```

```
t = c.expand_as(m)
t.storage()
```

```
t.stride(), t.shape
```

```
c = tensor([10.,20,30])
c.shape, c.unsqueeze(0).shape,c.unsqueeze(1).shape
```

```
c.shape, c[None,:].shape,c[:,None].shape
```

```
c[None].shape,c[...,None].shape
```

```
def matmul(a,b):
    ar,ac = a.shape
    br,bc = b.shape
    assert ac==br
    c = torch.zeros(ar, bc)
    for i in range(ar):
#       c[i,j] = (a[i,:]          * b[:,j]).sum() # previous
        c[i]   = (a[i  ].unsqueeze(-1) * b).sum(dim=0)
    return c
```

```
%timeit -n 20 t4 = matmul(m1,m2)
```

#### Broadcasting rules

### Einstein Summation

```
def matmul(a,b): return torch.einsum('ik,kj->ij', a, b)
```

```
%timeit -n 20 t5 = matmul(m1,m2)
```

## The Forward and Backward Passes

### Defining and Initializing a Layer

```
def lin(x, w, b): return x @ w + b
```

```
x = torch.randn(200, 100)
y = torch.randn(200)
```

```
w1 = torch.randn(100,50)
b1 = torch.zeros(50)
w2 = torch.randn(50,1)
b2 = torch.zeros(1)
```

```
l1 = lin(x, w1, b1)
l1.shape
```

```
l1.mean(), l1.std()
```

```
x = torch.randn(200, 100)
for i in range(50): x = x @ torch.randn(100,100)
x[0:5,0:5]
```

```
x = torch.randn(200, 100)
for i in range(50): x = x @ (torch.randn(100,100) * 0.01)
x[0:5,0:5]
```

```
x = torch.randn(200, 100)
for i in range(50): x = x @ (torch.randn(100,100) * 0.1)
x[0:5,0:5]
```

```
x.std()
```

```
x = torch.randn(200, 100)
y = torch.randn(200)
```

```
from math import sqrt
w1 = torch.randn(100,50) / sqrt(100)
b1 = torch.zeros(50)
w2 = torch.randn(50,1) / sqrt(50)
b2 = torch.zeros(1)
```

```
l1 = lin(x, w1, b1)
l1.mean(),l1.std()
```

```
def relu(x): return x.clamp_min(0.)
```

```
l2 = relu(l1)
l2.mean(),l2.std()
```

```
x = torch.randn(200, 100)
for i in range(50): x = relu(x @ (torch.randn(100,100) * 0.1))
x[0:5,0:5]
```

```
x = torch.randn(200, 100)
for i in range(50): x = relu(x @ (torch.randn(100,100) * sqrt(2/100)))
x[0:5,0:5]
```

```
x = torch.randn(200, 100)
y = torch.randn(200)
```

```
w1 = torch.randn(100,50) * sqrt(2 / 100)
b1 = torch.zeros(50)
w2 = torch.randn(50,1) * sqrt(2 / 50)
b2 = torch.zeros(1)
```

```
l1 = lin(x, w1, b1)
l2 = relu(l1)
l2.mean(), l2.std()
```

```
def model(x):
    l1 = lin(x, w1, b1)
    l2 = relu(l1)
    l3 = lin(l2, w2, b2)
    return l3
```

```
out = model(x)
out.shape
```

```
def mse(output, targ): return (output.squeeze(-1) - targ).pow(2).mean()
```

```
loss = mse(out, y)
```

### Gradients and the Backward Pass

```
def mse_grad(inp, targ):
    # grad of loss with respect to output of previous layer
    inp.g = 2. * (inp.squeeze() - targ).unsqueeze(-1) / inp.shape[0]
```

```
def relu_grad(inp, out):
    # grad of relu with respect to input activations
    inp.g = (inp>0).float() * out.g
```

```
def lin_grad(inp, out, w, b):
    # grad of matmul with respect to input
    inp.g = out.g @ w.t()
    w.g = inp.t() @ out.g
    b.g = out.g.sum(0)
```

### Sidebar: SymPy

```
from sympy import symbols,diff
sx,sy = symbols('sx sy')
diff(sx**2, sx)
```

### End sidebar

```
def forward_and_backward(inp, targ):
    # forward pass:
    l1 = inp @ w1 + b1
    l2 = relu(l1)
    out = l2 @ w2 + b2
    # we don't actually need the loss in backward!
    loss = mse(out, targ)

    # backward pass:
    mse_grad(out, targ)
    lin_grad(l2, out, w2, b2)
    relu_grad(l1, l2)
    lin_grad(inp, l1, w1, b1)
```

### Refactoring the Model

```
class Relu():
    def __call__(self, inp):
        self.inp = inp
        self.out = inp.clamp_min(0.)
        return self.out

    def backward(self): self.inp.g = (self.inp>0).float() * self.out.g
```

```
class Lin():
    def __init__(self, w, b): self.w,self.b = w,b

    def __call__(self, inp):
        self.inp = inp
        self.out = inp@self.w + self.b
        return self.out

    def backward(self):
        self.inp.g = self.out.g @ self.w.t()
        self.w.g = self.inp.t() @ self.out.g
        self.b.g = self.out.g.sum(0)
```

```
class Mse():
    def __call__(self, inp, targ):
        self.inp = inp
        self.targ = targ
        self.out = (inp.squeeze() - targ).pow(2).mean()
        return self.out

    def backward(self):
        x = (self.inp.squeeze()-self.targ).unsqueeze(-1)
        self.inp.g = 2.*x/self.targ.shape[0]
```

```
class Model():
    def __init__(self, w1, b1, w2, b2):
        self.layers = [Lin(w1,b1), Relu(), Lin(w2,b2)]
        self.loss = Mse()

    def __call__(self, x, targ):
        for l in self.layers: x = l(x)
        return self.loss(x, targ)

    def backward(self):
        self.loss.backward()
        for l in reversed(self.layers): l.backward()
```

```
model = Model(w1, b1, w2, b2)
```

```
loss = model(x, y)
```

```
model.backward()
```

### Going to PyTorch

```
class LayerFunction():
    def __call__(self, *args):
        self.args = args
        self.out = self.forward(*args)
        return self.out

    def forward(self):  raise Exception('not implemented')
    def bwd(self):      raise Exception('not implemented')
    def backward(self): self.bwd(self.out, *self.args)
```

```
class Relu(LayerFunction):
    def forward(self, inp): return inp.clamp_min(0.)
    def bwd(self, out, inp): inp.g = (inp>0).float() * out.g
```

```
class Lin(LayerFunction):
    def __init__(self, w, b): self.w,self.b = w,b

    def forward(self, inp): return inp@self.w + self.b

    def bwd(self, out, inp):
        inp.g = out.g @ self.w.t()
        self.w.g = self.inp.t() @ self.out.g
        self.b.g = out.g.sum(0)
```

```
class Mse(LayerFunction):
    def forward (self, inp, targ): return (inp.squeeze() - targ).pow(2).mean()
    def bwd(self, out, inp, targ):
        inp.g = 2*(inp.squeeze()-targ).unsqueeze(-1) / targ.shape[0]
```

```
from torch.autograd import Function

class MyRelu(Function):
    @staticmethod
    def forward(ctx, i):
        result = i.clamp_min(0.)
        ctx.save_for_backward(i)
        return result

    @staticmethod
    def backward(ctx, grad_output):
        i, = ctx.saved_tensors
        return grad_output * (i>0).float()
```

```
import torch.nn as nn

class LinearLayer(nn.Module):
    def __init__(self, n_in, n_out):
        super().__init__()
        self.weight = nn.Parameter(torch.randn(n_out, n_in) * sqrt(2/n_in))
        self.bias = nn.Parameter(torch.zeros(n_out))

    def forward(self, x): return x @ self.weight.t() + self.bias
```

```
lin = LinearLayer(10,2)
p1,p2 = lin.parameters()
p1.shape,p2.shape
```

```
class Model(nn.Module):
    def __init__(self, n_in, nh, n_out):
        super().__init__()
        self.layers = nn.Sequential(
            nn.Linear(n_in,nh), nn.ReLU(), nn.Linear(nh,n_out))
        self.loss = mse

    def forward(self, x, targ): return self.loss(self.layers(x).squeeze(), targ)
```

```
class Model(Module):
    def __init__(self, n_in, nh, n_out):
        self.layers = nn.Sequential(
            nn.Linear(n_in,nh), nn.ReLU(), nn.Linear(nh,n_out))
        self.loss = mse

    def forward(self, x, targ): return self.loss(self.layers(x).squeeze(), targ)
```

## Conclusion

## Questionnaire

1. Write the Python code to implement a single neuron.
1. Write the Python code to implement ReLU.
1. Write the Python code for a dense layer in terms of matrix multiplication.
1. Write the Python code for a dense layer in plain Python (that is, with list comprehensions and functionality built into Python).
1. What is the "hidden size" of a layer?
1. What does the `t` method do in PyTorch?
1. Why is matrix multiplication written in plain Python very slow?
1. In `matmul`, why is `ac==br`?
1. In Jupyter Notebook, how do you measure the time taken for a single cell to execute?
1. What is "elementwise arithmetic"?
1. Write the PyTorch code to test whether every element of `a` is greater than the corresponding element of `b`.
1. What is a rank-0 tensor? How do you convert it to a plain Python data type?
1. What does this return, and why? `tensor([1,2]) + tensor([1])`
1. What does this return, and why? `tensor([1,2]) + tensor([1,2,3])`
1. How does elementwise arithmetic help us speed up `matmul`?
1. What are the broadcasting rules?
1. What is `expand_as`? Show an example of how it can be used to match the results of broadcasting.
1. How does `unsqueeze` help us to solve certain broadcasting problems?
1. How can we use indexing to do the same operation as `unsqueeze`?
1. How do we show the actual contents of the memory used for a tensor?
1. When adding a vector of size 3 to a matrix of size 3×3, are the elements of the vector added to each row or each column of the matrix? (Be sure to check your answer by running this code in a notebook.)
1. Do broadcasting and `expand_as` result in increased memory use? Why or why not?
1. Implement `matmul` using Einstein summation.
1. What does a repeated index letter represent on the left-hand side of einsum?
1. What are the three rules of Einstein summation notation? Why?
1. What are the forward pass and backward pass of a neural network?
1. Why do we need to store some of the activations calculated for intermediate layers in the forward pass?
1. What is the downside of having activations with a standard deviation too far away from 1?
1. How can weight initialization help avoid this problem?
1. What is the formula to initialize weights such that we get a standard deviation of 1 for a plain linear layer, and for a linear layer followed by ReLU?
1. Why do we sometimes have to use the `squeeze` method in loss functions?
1. What does the argument to the `squeeze` method do? Why might it be important to include this argument, even though PyTorch does not require it?
1. What is the "chain rule"? Show the equation in either of the two forms presented in this chapter.
1. Show how to calculate the gradients of `mse(lin(l2, w2, b2), y)` using the chain rule.
1. What is the gradient of ReLU? Show it in math or code. (You shouldn't need to commit this to memory—try to figure it using your knowledge of the shape of the function.)
1. In what order do we need to call the `*_grad` functions in the backward pass? Why?
1. What is `__call__`?
1. What methods must we implement when writing a `torch.autograd.Function`?
1. Write `nn.Linear` from scratch, and test it works.
1. What is the difference between `nn.Module` and fastai's `Module`?

### Further Research

1. Implement ReLU as a `torch.autograd.Function` and train a model with it.
1. If you are mathematically inclined, find out what the gradients of a linear layer are in mathematical notation. Map that to the implementation we saw in this chapter.
1. Learn about the `unfold` method in PyTorch, and use it along with matrix multiplication to implement your own 2D convolution function. Then train a CNN that uses it.
1. Implement everything in this chapter using NumPy instead of PyTorch.

```

```


```python
import numpy as np
import sympy as sy
import control.matlab as cm
import matplotlib.pyplot as plt
%matplotlib inline
```

```python
s,z,h = sy.symbols('s, z, h')
```

```python
Y = (s+1)/(s*s*(s+3))
sy.apart(Y)
```

    -2/(9*(s + 3)) + 2/(9*s) + 1/(3*s**2)

```python
H = sy.Integer(2)/sy.Integer(9) + h/(3*(z-1)) - 2*(z-1)/(9*(z-sy.exp(-3*h)))
print H
```

    h/(3*z - 3) - (2*z - 2)/(9*z - 9*exp(-3*h)) + 2/9

```python
H = sy.together(H)
print H
```

    (3*h*(z*exp(3*h) - 1) - 2*(z - 1)**2*exp(3*h) + 2*(z - 1)*(z*exp(3*h) - 1))/(9*(z - 1)*(z*exp(3*h) - 1))

```python
(B,A) = sy.fraction(H)
print B
```

    3*h*(z*exp(3*h) - 1) - 2*(z - 1)**2*exp(3*h) + 2*(z - 1)*(z*exp(3*h) - 1)

```python
sy.simplify(B - sy.exp(3*h)*(2*(z-1)*(z-sy.exp(-3*h)) + 3*h*(z-sy.exp(-3*h)) - 2*(z-1)**2))
```

    0

```python
np.log(2)/3
```

    0.23104906018664842

```python
Hc = sy.simplify(H/(H+1))
print Hc
```

    (3*h*(z*exp(3*h) - 1) - 2*(z - 1)**2*exp(3*h) + 2*(z - 1)*(z*exp(3*h) - 1))/(3*h*(z*exp(3*h) - 1) - 2*(z - 1)**2*exp(3*h) + 11*(z - 1)*(z*exp(3*h) - 1))

```python
Hc1 = Hc.subs(h, sy.log(2)/3)
(Bc,Ac) = sy.fraction(Hc1)
print sy.collect(sy.expand(Ac), z)
```

    18*z**2 + z*(-25 + 2*log(2)) - log(2) + 7

```python
sy.simplify(sy.expand( (11*z-11 +sy.log(2))*(z-0.5)-2*(z-1)**2) )
```

    9*z**2 - 12.5*z + z*log(2) - 0.5*log(2) + 3.5

```python
sy.simplify(sy.expand( A + B)).subs(h, sy.log(2)/3)
```

    18*z**2 - 25*z + 2*z*log(2) - log(2) + 7

```python
12.5-np.log(2)
```

    11.806852819440055

```python
3.5-np.log(2)/2
```

    3.1534264097200273

```python
Bcp = sy.Poly(Bc, z)
Acp = sy.Poly(Ac,z)
a = []
for el in Acp.coeffs():
    a.append(float(sy.N(el)))
print a
b = []
for el in Bcp.coeffs():
    b.append(float(sy.N(el)))
print b

sys = cm.tf(b,a,np.log(2)/3.0)
print sys
Td = np.log(2)/3*np.arange(80)
(yd,td) = cm.step(sys, Td)
plt.plot(td,yd[0], '*')
plt.xlim([0,20])
plt.savefig("stepresponse.pdf")
```

```python
Bp = sy.Poly(B.subs(h, sy.log(2)/3))
Ap = sy.Poly(A.subs(h, sy.log(2)/3))
a = []
for el in Ap.coeffs():
    a.append(float(sy.N(el)))
print a
b = []
for el in Bp.coeffs():
    b.append(float(sy.N(el)))
print b

sys_o = cm.tf(b,a,np.log(2)/3.0)
cm.nyquist(sys_o, np.linspace(0,2*np.pi, 800))
plt.ylim(-2,2)
plt.xlim(-1,1)
```

```python
H1 = cm.tf([1],[1, -2.2, 1.4, -0.2], 0.5)
H2 = cm.tf([1, 0],[1, -1.2, .2], 0.5)
H3 = cm.tf([4, 1],[1, 1, .16], 1)
H4 = cm.tf([0.25], [1, -1.5, 0.5])
```

```python
cm.nyquist(H4, np.linspace(0,2*np.pi, 800))
plt.ylim(-5,5)
plt.xlim(-2,2)
```

```python
import khcontrol as khc

z = np.exp(1j*np.linspace(0,np.pi, 800))
zz = H4(z)
(zr,zi) = khc.nyq(H4, 800)

#nq = []
#for w in np.linspace(0,2*np.pi, 200):
#    #nq.append(H4(np.exp(1j*w)))
#    nq.append(H4(1j*w))
plt.figure()
plt.plot(np.real(zz), np.imag(zz))
plt.ylim(-5,2)
plt.xlim(-2,1)
xy = np.column_stack((np.real(zz),np.imag(zz)))
np.savetxt('nyquist150907.out', xy)

```

```python
np.zeros(3)
```

    array([ 0.,  0.,  0.])

```python
1600.0/3
```

    533.3333333333334

```python

```


```
# default_exp oneDim
```

```
#hide
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib.cm as cm
plt.rcParams['figure.figsize'] = (10,6)
import sympy; sympy.init_printing()
# code for displaying matrices nicely
def display_matrix(m):
    display(sympy.Matrix(m))
```

# oneDim
> Code for a 1-D problem.

```
#hide
from nbdev.showdoc import *
```

# 1 dimensional case (ODE)

We consider the following 1-D problem:

$$-\frac{d}{dx}\left(p(x)\frac{du(x)}{dx}\right)=f(x) \hspace{0.5cm}\forall x\in[0,1]$$

$$u(0)=u(1)=0$$

where here $f$ is a random forcing term, assumed to be a GP in this work.

## Variational formulation

The variational formulation is given by:

$$a(u,v)=L(v)$$

where:

$$a(u,v)=\int_{0}^{1}pu^{\prime}v^{\prime}dx$$

and

$$L(v)=\int_{0}^{1}fvdx$$

We will make the following choices for $p,f$:

$$p(x)=1$$

$$f\sim\mathcal{G}\mathcal{P}(\bar{f},k_{f})$$

$$\bar{f}(x)=1$$

$$ k_{f}(x,y) = \sigma_f^{2}\exp\left(-\frac{|x-y|^2}{2l_f^2}\right)$$

$$ \sigma_{f} = 0.1$$

$$ l_f = 0.4 $$

## Difference between true prior mean and statFEM prior mean

Since the mean of $f$ is $\bar{f}(x)=1$ we have that the true mean of the solution $u$ is the solution of the ODE with forcing term set to the constant function 1. This has the exact analytic solution:

$$u(x)=\frac{1}{2}x(1-x)$$

as can be directly verified.

The FEM approximation to the solution distribution has mean $\boldsymbol{\Phi}(x)^{*}A^{-1}\bar{F}$ which is the solution to the approximate variational problem obtained by replacing $f$ with $\bar{f}$ in the linear form $L$.

We will utilise FEniCS to compute the error between these two as a function of $h$ the mesh size. To do this we first create a function `mean_assembler` which will assemble the mean for the statFEM prior

```
#export
from dolfin import *
import numpy as np
from scipy import integrate
from scipy.spatial.distance import cdist
from scipy.linalg import sqrtm
from scipy.sparse import csr_matrix
from scipy.sparse.linalg import spsolve
from scipy.interpolate import interp1d
from joblib import Parallel, delayed
import multiprocessing

# code to assemble the mean for a given mesh size
def mean_assembler(h,f_bar):
    "This function assembles the mean for the statFEM prior for our 1-D problem."
    # get size of the grid
    J = int(np.round(1/h))

    # set up the mesh and function space for FEM
    mesh = UnitIntervalMesh(J)
    V = FunctionSpace(mesh,'Lagrange',1)

    # set up boundary condition
    def boundary(x, on_boundary):
        return on_boundary

    bc = DirichletBC(V, 0.0, boundary)
    # set up the functions p and f
    p = Constant(1.0)
    f = f_bar

    # set up the bilinear form for the variational problem
    u = TrialFunction(V)
    v = TestFunction(V)
    a = inner(p*grad(u),grad(v))*dx

    # set up the linear form
    L = f*v*dx

    # solve the variational problem
    μ = Function(V)
    solve(a == L, μ, bc)

    return μ
```

`mean_assembler` takes in the mesh size `h` and the mean function `f_bar` for the forcing and computes the mean of the approximate statFEM prior, returning this as a FEniCS function.

> Important: `mean_assembler` requires `f_bar` to be represented as a FEniCS function/expression/constant.

Let's check that this is working:

```
h = 0.15
f_bar = Constant(1.0)
μ = mean_assembler(h,f_bar)
μ
```

```
# check the type of μ
assert type(μ) == function.function.Function
```

As explained above the true mean is the function $u(x)=\frac{1}{2}x(1-x)$. Let's check that the approximate mean resembles this by plotting both:

```
#hide_input
# use FEniCS to plot μ
x = np.linspace(0,1,100)
μ_true = 0.5*x*(1-x)
plt.plot(x,μ_true,label='true mean',color='red')
plot(μ,label='FEM approximation',color='blue')
plt.legend()
plt.xlabel(r'$x$')
plt.grid()
plt.show()
```

We can see that the FEM approximation does indeed resemble the true mean!

## Difference between true prior covariance and statFEM prior covariance

The solution $u$ has covariance function $c_u(x,y)$ given by the following expression:

$$c_u(x,y)=\int_{0}^{1}\int_{0}^{1}G(x,w)k_f(w,t)G(t,y)dtdw$$

Where $G(x,y)$ is the Green's function for our problem:

$$G(x,y) = x(1-y)\Theta(y-x) + (1-x)y\Theta(x-y) \quad \forall x,y\in[0,1]$$

(note: $\Theta(x)$ is the Heaviside Step function)

The statFEM covariance can be approximated as follows:

$$c_u^{\text{FEM}}(x,y)\approx\sum_{i,j=1}^{J}\varphi_{i}(x)Q_{ij}\varphi_{j}(y)$$

where $Q=A^{-1}MC_{f}M^{T}A^{-T}$ and where the $\{\varphi_{i}\}_{i=1}^{J}$ are the FE basis functions corresponding to the interior nodes of our domain. $C_f$ is the kernel matrix of $f$ (evaluated on the FEM grid).

The difference between the covariance operators we are interested in computing is the following contribution to the 2-Wasserstein distance between the true solution GP and the approximate FEM GP:

$$d_W(C_1,C_2) = \operatorname{tr} C_1 +\operatorname{tr} C_2-2\operatorname{tr}\sqrt{C_{1}^{1/2}C_{2}C_{1}^{1/2}}$$

where $C_1, C_2$ are the covariance operators corresponding to $c_u$ and $c_u^{\text{FEM}}$ respectively.

The above quantity will be approximated by fixing a fine grid and computing the cov matrices $\Sigma_1, \Sigma_2$ for the cov operators $C_1, C_2$, respectively, on this grid. We will then utilise the approximation:

$d_W(C_1,C_2)\approx \operatorname{tr} \Sigma_1 +\operatorname{tr} \Sigma_2-2\operatorname{tr}\sqrt{\Sigma_{1}^{1/2}\Sigma_{2}\Sigma_{1}^{1/2}}$

Thus, it will be necessary to write code to form the matrices $\Sigma_1,\Sigma_2$ above. The structure of the approximate $c_u^{\text{FEM}}$ will allow us to compute $\Sigma_2$ in a very efficient manner using FEniCS. This is achieved by noting that we can write:

$$c_u^{\text{FEM}}(x,y)\approx\boldsymbol{\phi}(x)^{T}Q\boldsymbol{\phi}(y)$$

where $\boldsymbol{\phi}(x):=\left(\varphi_1(x),\cdots,\varphi_J(x)\right)^{T}$

Written in this form, it is now easy to see that $\Sigma_2$, whose $ij$-th entry is given by $(\Sigma_2)_{ij}=\boldsymbol{\phi}(x_i)^{T}Q\boldsymbol{\phi}(x_j)$, can be expressed as follows:

$$\Sigma_2=\boldsymbol{\Phi}^{T}Q\boldsymbol{\Phi}$$

where $\boldsymbol{\Phi}$ is a $J\times N$ matrix whose $i$th column is given by $\boldsymbol{\phi}(x_i)$ where $\{x_i\}_{i=1}^{N}$ are the grid points.

Thus, provided we can efficiently compute the matrices $\boldsymbol{\Phi}$ and $Q$ with FEniCS we can efficiently compute the differene between the covariances required.

In order to compute $\Sigma_1$ and the matrix $C_f$ needed for $Q$ we will need to be able to construct a covariance matrix on a grid for a given cov function. We thus will first create a function `kernMat` which assembles the covariance matrix corresponding to the covariance function `k` on a grid `grid`.

```
#export
def kernMat(k,grid,parallel=True,translation_inv=False):
    "Function to compute the covariance matrix $K$ corresponding to the covariance kernel $k$ on a grid. This matrix has $ij-$th entry $K_{ij}=k(x_i,x_j)$ where $x_i$ is the $i$-th point of the grid."
    # get the length of the grid
    n = len(grid)
    # preallocate an n x n array of zeros to hold the cov matrix
    K = np.zeros((n,n))

    # check if the cov matrix should be computed in parallel
    if parallel:
        # compute the cov matrix in parallel by computing the upper triangular part column by column
        # set up function to compute the ith column of the upper triangular part:
        def processInput(i):
            return np.array([k(grid[i],grid[j]) for j in range(i,n)])

        # get the number of cpu cores present and compute the upper triangular columns in parallel
        num_cores = multiprocessing.cpu_count()
        results = Parallel(n_jobs=num_cores)(delayed(processInput)(i) for i in range(n))

        # store the results in the appropriate positions in K
        #for (i,v) in enumerate(results[0:n-1]):
        for (i,v) in enumerate(results):  # is this correct???
            K[i,i:] = v

        # only the upper triangular part has been formed, so use the symmetry of the cov mat to get full K:
        K = K + K.T - np.diag(K.diagonal())
        return K
    elif translation_inv:
        # reshape grid so that it has correct dimensions
        grid = grid.reshape(n,1)

        # compute the distance matrix D
        D = cdist(grid,grid)

        # evaluate the kernel function using D
        K = k(D)
        return K
    else:
        # compute the cov mat using a nested for loop
        for i in range(n):
            for j in range(i,n):
                K[i,j] = k(grid[i],grid[j])
        K = K + K.T - np.diag(K.diagonal())
        return K
```

> Note: This function takes in two optional boolean arguments `parallel` and `translation_inv`. The first of these specifies whether or not the cov matrix should be computed in parallel and the second specifies whether or not the cov kernel is translation invariant. If it is, the covariance matrix is computed more efficiently using the `cdist` function from scipy.

Let's quickly test if this function is working, by computing the cov matrix for white noise, which has kernel function $k(x,y)=\delta(x-y)$. For a grid of length $N$ this should be the $N\times N$ identity matrix.

```
# set up the kernel function
# set up tolerance for comparison
tol = 1e-16
def k(x,y):
    if np.abs(x-y) < tol:
        # x == y within the tolerance
        return 1.0
    else:
        # x != y within the tolerance
        return 0.0

# set up grid
N = 21
grid = np.linspace(0,1,N)
K = kernMat(k,grid,True,False) # parallel mode

# check that this is the N x N identity matrix
assert (K == np.eye(N)).all()
```

We now create a function `BigPhiMat` to utilise FEniCS to efficiently compute the matrix $\boldsymbol{\Phi}$ defined above.

```
#export
def BigPhiMat(J,grid):
    "Function to compute the $\Phi$ matrix."
    # create the FE mesh and function space
    mesh = UnitIntervalMesh(J)
    V = FunctionSpace(mesh,'Lagrange',1)
    # get the tree for the mesh
    tree = mesh.bounding_box_tree()
    # set up a function to compute the ith column of Phi corresponding to the ith grid point
    def Φ(i):
        x = grid[i]
        cell_index = tree.compute_first_entity_collision(Point(x))
        cell = Cell(mesh,cell_index)
        cell_global_dofs = V.dofmap().cell_dofs(cell_index)
        vertex_coordinates = cell.get_vertex_coordinates()
        cell_orientation = cell.orientation()
        data = V.element().evaluate_basis_all(x,vertex_coordinates,cell_orientation)
        return (data,cell_global_dofs,i*np.ones_like(cell_global_dofs))
    # compute all the columns of Phi using the function above
    res = [Φ(i) for i in range(len(grid))]
    # assemble the sparse matrix Phi using the results
    data = np.hstack([res[i][0] for i in range(len(grid))])
    row = np.hstack([res[i][1] for i in range(len(grid))])
    col = np.hstack([res[i][2] for i in range(len(grid))])
    return csr_matrix((data,(row,col)),shape=(V.dim(),len(grid)))
```

`BigPhiMat` takes in two arguments: `J`, which controls the FE mesh size ($h=1/J$), and `grid` which is the grid in the definition of $\boldsymbol{\Phi}$. `BigPhiMat` returns $\boldsymbol{\Phi}$ as a sparse `csr_matrix` for memory efficiency.

> Note: Note that since FEniCS works with the FE functions corresponding to all the FE dofs and our matrix $\Sigma_2$ only uses the FE functions corresponding to non-boundary dofs we need to account for this in the code. See the source code for `BigPhiMat` to see how this is done.

We now create a function `cov_asssembler` which assembles the approximate FEM covariance matrix on the grid.

```
#export
def cov_assembler(J,k_f,grid,parallel,translation_inv):
    "Function to assemble the approximate FEM covariance matrix on the reference grid."

    # set up mesh and function space
    mesh = UnitIntervalMesh(J)
    V = FunctionSpace(mesh,'Lagrange',1)

    # set up FE grid
    x_grid = V.tabulate_dof_coordinates()

    # set up boundary condition
    def boundary(x, on_boundary):
        return on_boundary

    bc = DirichletBC(V, 0.0, boundary)

    # get the boundary and interior dofs
    bc_dofs = bc.get_boundary_values().keys()
    first, last = V.dofmap().ownership_range()
    all_dofs = range(last - first)
    interior_dofs = list(set(all_dofs) - set(bc_dofs))
    bc_dofs = list(set(bc_dofs))

    # set up the function p
    p = Constant(1.0)

    # get the mass and stiffness matrices as sparse csr_matrices
    u = TrialFunction(V)
    v = TestFunction(V)

    mass_form = u*v*dx
    a = inner(p*grad(u),grad(v))*dx

    M = assemble(mass_form)
    A = assemble(a)
    M = as_backend_type(M).mat()
    A = as_backend_type(A).mat()
    M = csr_matrix(M.getValuesCSR()[::-1],shape=M.size)
    A = csr_matrix(A.getValuesCSR()[::-1],shape=A.size)

    # extract the submatrices corresponding to the interior dofs
    M = M[interior_dofs,:][:,interior_dofs]
    A = A[interior_dofs,:][:,interior_dofs]

    # get the forcing cov matrix on the interior nodes of the grid
    Σ_int = kernMat(k_f,x_grid[interior_dofs],parallel,translation_inv)

    # form the matrix Q in the defintion of the approximate FEM cov mat
    # Note: overwrite Σ_int for memory efficiency.
#     Σ_int = M @ Σ_int @ M.T
    Σ_int = Σ_int @ M.T
    Σ_int = M @ Σ_int

    # form Q (storing this in Σ_int directly for memory efficiency)
    Σ_int = spsolve(A,Σ_int)
    Σ_int = spsolve(A,Σ_int.T).T

    # ensure Σ_int is symmetric
    Σ_int = 0.5*(Σ_int + Σ_int.T)

    # get big phi matrix on the grid (extracting only the rows corresponding to the
    # interior dofs)
    Phi = BigPhiMat(J,grid)[interior_dofs,:]

    # assemble cov mat on grid using Phi and Σ_int
    Σ = Phi.T @ Σ_int @ Phi

    # ensure Σ is symmetric and return
    Σ = 0.5*(Σ + Σ.T)
    return Σ
```

`cov_assembler` takes in several arguments which are explained below:

- `J`: controls the FE mesh size ($h=1/J)$
- `k_f`: the covariance function for the forcing $f$
- `grid`: the reference grid where the FEM cov matrix should be computed on
- `parallel`: boolean argument indicating whether the intermediate computation of $C_f$ should be done in parallel
- `translation_inv`: boolean argument indicating whether the intermediate computation of $C_f$ should be computed assuming `k_f` is translation invariant or not

As a quick demonstration that the code is working, we will compute the true and approximate covariance matrices for a relatively coarse grid. We first set up functions to compute the true covariance matrix $\Sigma_1$:

```
# set up kernel functions for f
l_f = 0.4
σ_f = 0.1

def c_f(x,y):
    return (σ_f**2)*np.exp(-(x-y)**2/(2*(l_f**2)))

# translation invariant form of c_f
def k_f(x):
    return (σ_f**2)*np.exp(-(x**2)/(2*(l_f**2)))

# use quadrature for the true cov function
from scipy import integrate
# compute inner integral over t
def η(w,y):
    I_1 = integrate.quad(lambda t: t*c_f(w,t),0.0,y)[0]
    I_2 = integrate.quad(lambda t: (1-t)*c_f(w,t),y,1.0)[0]
    return (1-y)*I_1 + y*I_2

# use this function η and compute the outer integral over w
def c_u(x,y):
    I_1 = integrate.quad(lambda w: (1-w)*η(w,y),x,1.0)[0]
    I_2 = integrate.quad(lambda w: w*η(w,y),0.0,x)[0]
    return x*I_1 + (1-x)*I_2
```

With these functions we can now compute $\Sigma_1$ as follows:

```
# set up a reference grid
N = 21
grid = np.linspace(0,1,N)

# compute Σ_1 using c_u
Σ_1 = kernMat(c_u,grid,True,False)
```

We now use our function `cov_assembler` to compute $\Sigma_2$:

```
J = 20 # choose a FE mesh size
Σ_2 = cov_assembler(J,k_f,grid,False,True)
```

Let's plot heatmaps of both $\Sigma_1, \Sigma_2$ to compare:

```
#hide_input
vmin = min(Σ_1.min(), Σ_2.min())
vmax = max(Σ_1.max(), Σ_2.max())
plt.rcParams['figure.figsize'] = (12,6)
fig, axs = plt.subplots(ncols=3, gridspec_kw=dict(width_ratios=[4,4,0.2]))
sns.heatmap(Σ_1,cbar=False,
                annot=False,
                xticklabels=False,
                yticklabels=False,
                cmap=cm.viridis,
                ax=axs[0])
axs[0].title.set_text(r'$\Sigma_1$')
sns.heatmap(Σ_2,cbar=False,
                annot=False,
                xticklabels=False,
                yticklabels=False,
                cmap=cm.viridis,
                ax=axs[1])
axs[1].title.set_text(r'$\Sigma_2$')
fig.colorbar(axs[np.argmax([Σ_1.max(), Σ_2.max()])].collections[0], cax=axs[2])
plt.tight_layout()
plt.show()
```

Even with a relatively coarse reference grid and a relatively coarse FE space it looks as if the approximate FEM covariance is quite similar to the true covariance matrix as can be seen from the heatmaps above. Let's also check how similar they are by utilising `np.linalg.norm` to compute the relative percentage difference:

```
#hide_input
print("Relative percentage difference is: %.2f" %(100*np.linalg.norm(Σ_1-Σ_2)/np.linalg.norm(Σ_1)) + "%")
```

    Relative percentage difference is: 0.68%

## Posterior from incorporating sensor readings

Denote by $\nu_{i}=\mathcal{N}(m_{i},\Sigma_{i})$, where $i$ is either the symbol $\star$ or $h$, the true and statFEM prior respectively. When we take $u\sim\nu_{i}$ as our prior, the resulting posterior after incorporating the noisy sensor readings $\mathbf{v}$ at the locations $\{y_{j}\}_{j=1}^{s}$ is given by:

$$u|\mathbf{v}\sim\mathcal{N}\left(m_{u|\mathbf{v}}^{(i)},\Sigma_{u|\mathbf{v}}^{(i)}\right)$$

where we have:

$$m_{u|\mathbf{v}}^{(i)}=m_{i} + \Sigma_{i}S^{\dagger}(\epsilon^{2}I+S\Sigma_{i}S^{\dagger})^{-1}(\mathbf{v}-Sm_{i})$$

$$\Sigma_{u|\mathbf{v}}^{(i)}=\Sigma_{i} - \Sigma_{i}S^{\dagger}(\epsilon^{2}I+S\Sigma_{i}S^{\dagger})^{-1}S\Sigma_{i}$$

where $S$ is the operator which maps a function $g$ to the vector $\left(g(y_1),\cdots,g(y_s)\right)^{T}$.

For brevity we will denote the $s\times s$ matrix which appears above as $B_{\epsilon,i}:=\epsilon^{2}I+S\Sigma_{i}S^{\dagger}=\epsilon^{2}I+C_{Y,i}$ where we have also defined $C_{Y,i}:=S\Sigma_{i}S^{\dagger}$. This matrix has $pq$*-th* entry $c^{(i)}(y_{p},y_{q})$ where $c^{(i)}$ is the covariance function associated with the covariance operator $\Sigma_{i}$.

### Posterior mean

Thus, our posterior mean in both cases has the form:

$$m^{(i)}_{u|\mathbf{v}}(x)=m_{i}(x)+\sum_{p,q=1}^{s}c^{(i)}(x,y_{p})\left(B_{\epsilon,i}^{-1}\right)_{pq}(v_{q}-m_{i}(y_{q}))$$

Note that this can be expressed as:

$$m^{(i)}_{u|\mathbf{v}}(x)=m_{i}(x) - \mathbf{c}^{(i)}(x)^{T}B_{\epsilon,i}^{-1}(\mathbf{m}^{(i)}-\mathbf{v})$$

where $\mathbf{m}^{(i)}:=Sm_{i}=(m_{i}(y_1),\cdots,m_{i}(y_s))^{T}$ and $\mathbf{c}^{(i)}(x):=(c^{(i)}(x,y_1),\cdots,c^{(i)}(x,y_s))^{T}$.

Thus, we require a function to evaluate the posterior means. We will thus create a function `m_post` which evaluates the posterior means.

```
# export
def m_post(x,m,c,v,Y,B):
    "This function evalutes the posterior mean at the point $x$."
    m_vect = np.array([m(y_i) for y_i in Y]).flatten()
    c_vect = c(x).flatten()

    # compute the update term
    update = c_vect @ np.linalg.solve(B,m_vect-v)

    # return m_post
    return (m(x) - update)
```

`m_post` takes in several arguments which are explained below:

- `x`: point where the posterior mean will be evaluated
- `m`: function which computes the prior mean at a given point y
- `c`: function which returns the vector (c(x,y)) for y in Y (note: c is the prior covariance function)
- `v`: vector of noisy sensor readings
- `Y`: vector of sensor locations
- `B`: the matrix $\epsilon^{2}I+C_Y$ to be inverted in order to obtain the posterior

As a quick test to see if the code is working, note that if we choose $\mathbf{c}$ above to be constant at the $j$*-th* standard basis vector and if we take $B$ to be the identity matrix then we should obtain the function $m(x)-m(y_j)+v_j$. This will give us the $v_j$ when evaluated at $y_j$. We will test that this is indeed what we get:

```
# choose several prior mean functions to try
m_list = [lambda x: 1.0, lambda x: 0.5*x*(1.0-x), lambda x: np.sin(2*np.pi*x)]

# set up Y and B and v:
Y = np.linspace(0.01,0.99,11)
s = len(Y)
B = np.eye(s)
np.random.seed(42)
v = np.random.randn(s)

# test that we get v_j when evaluated at y_j (and when c is set to be j-th basis vector)
for j in range(s):
    # define c to be j-th basis vector for all x
    def c(x):
        c_vect = np.zeros(s)
        c_vect[j] = 1.0
        return c_vect

    # evaluate the posterior mean at jth sensor location and check that this is v_j
    # (this check is done up to a tolerance tol)
    tol = 1e-15
    for m in m_list:
        assert np.abs(m_post(Y[j],m,c,v,Y,B) - v[j]) < tol
```

### Difference between posterior means

In order to compute the difference between the posterior means we require some more code.

Firstly, we will need code to generate samples from a GP with mean $m$ and cov function $k$ on a grid. We write the function `sample_gp` for this purpose.

```
#export
def sample_gp(n_sim,m,k,grid,par=False,trans=True, tol=1e-9):
    "Function to sample a GP with mean $m$ and cov $k$ on a grid."
    # get length of grid
    d = len(grid)

    # construct mean vector
    μ = np.array([m(x) for x in grid]).reshape(d,1)

    # construct covariance matrix
    Σ = kernMat(k,grid,parallel = par, translation_inv = trans)

    # construct the cholesky decomposition Σ = GG^T
    # we add a small diagonal perturbation to Σ to ensure it
    # strictly positive definite
    G = np.linalg.cholesky(Σ + tol * np.eye(d))

    # draw iid standard normal random vectors
    Z = np.random.normal(size=(d,n_sim))

    # construct samples from GP(m,k)
    Y = G@Z + np.tile(μ,n_sim)

    # return the sampled trajectories
    return Y
```

`sample_gp` takes in several arguments which are explained below:

- `n_sim`: number of trajectories to be sampled
- `m`: mean function for the GP
- `k`: cov function for the GP
- `grid`: grid of points on which to sample the GP
- `par`: boolean argument indicating whether the computation of the cov matrix should be done in parallel
- `trans`: boolean argument indicating whether the computation of the cov matrix should be computed assuming `k` is translation invariant or not
- `tol`: controls the size of the tiny diagonal perturbation added to cov matrix to ensure it is strictly positive definite (defaults to `1e-9`)

As a quick demonstration that the code is working lets generate 10 trajectories of white noise, using the kernel `k` from one of the previous tests:

```
#hide_input
# set up grid to sample on
N = 51
grid = np.linspace(0,1,N)

# set up mean
def m(x):
    return 0.0

# sample the GP
n_sim = 10
np.random.seed(235)
samples = sample_gp(n_sim,m,k,grid,True,False)
plt.plot(grid,samples)
plt.grid()
plt.xlabel(r'$x$')
plt.title('White noise trajectories')
plt.show()
```

The next bit of code we require is code to generate noisy sensor readings from our system. We write the function `gen_sensor` for this purpose.

```
#export
def gen_sensor(ϵ,m,k,Y,u_quad,grid,par=False,trans=True,tol=1e-9,maxiter=50,require_f=False):
    "Function to generate noisy sensor observations of the solution u on a sensor grid Y."

    # get number of sensors from the sensor grid Y
    s = len(Y)

    # sample a single f on the grid
    f_sim = sample_gp(1,m,k,grid,par=par,trans=trans,tol=tol)

    # create solution function
    # interpolate f to get a function
    f = interp1d(grid,f_sim.flatten(),kind='cubic')
    # use u_quad together with f to compute solution
    def u(x):
        return u_quad(x,f,maxiter=maxiter)

    # get solution on grid Y:
    u_Y = np.array([u(y_i) for y_i in Y])

    # add N(0,ϵ^2) to each evaluation point
    u_S = u_Y + ϵ*np.random.normal(size=s)

    # if require the simulated trajectory of f return this as well, if not just return u_S
    if require_f:
        return u_S, f_sim
    else:
        return u_S
```

`gen_sensor` takes in several arguments which are explained below:

- `ϵ`: controls the amount of sensor noise
- `m`: mean function for the forcing f
- `k`: cov function for the forcing f
- `Y`: vector of sensor locations
- `u_quad`: function to accurately compute the solution u given a realisation of the forcing f
- `grid`: grid where forcing f is sampled on
- `par`: boolean argument indicating whether the computation of the forcing cov matrix should be done in parallel
- `trans`: boolean argument indicating whether the computation of the forcing cov matrix should be computed assuming `k` is translation invariant or not
- `tol`: controls the size of the tiny diagonal perturbation added to forcing cov matrix to ensure it is strictly positive definite (defaults to `1e-9`)
- `maxiter`: parameter which controls the accuracy of the quadrature used in u_quad (defaults to `50`)
- `require_f` : boolean argument indicating whether or not to also return the realisation of the forcing f (defaults to `False`)

> Important: The function `u_quad` which is passed to `gen_sensor` is assumed to compute the solution using quadrature. This must be done in a particular way and will be demonstrated below. It is also important to choose a fine enough grid for the argument `grid` passed to `gen_sensor` as this affects the solution accuracy.

Let's demonstrate that this code is working. To start we note that due to the form of the Green's function for our problem, we can express the solution $u$ in terms of the forcing $f$ as follows:

$$u(x)=\int_{0}^{1}G(x,y)f(y)\mathrm{d}y=(1-x)\int_{0}^{x}yf(y)\mathrm{d}y+x\int_{x}^{1}(1-y)f(y)\mathrm{d}y$$

We will use this observation when setting up `u_quad` below. We now generate $s=20$ sensor observations with the sensors equally spaced in the interval $(0.01,0.99)$.

```
# set up mean and kernel functions for the forcing f
l_f = 0.4
σ_f = 0.1

def m_f(x):
    return 1.0

def k_f(x):
    return (σ_f**2)*np.exp(-(x**2)/(2*(l_f**2)))

# set up sensor grid and sensor noise level
s = 20
Y = np.linspace(0.01,0.99,s)
ϵ = 0.1

# set up grid to simulate f on
N = 40
grid = np.linspace(0,1,N+1)

# set up u_quad
def u_quad(x,f,maxiter=50):
    I_1 = integrate.quadrature(lambda w: w*f(w), 0.0, x,maxiter=maxiter)[0]
    I_2 = integrate.quadrature(lambda w: (1-w)*f(w),x, 1.0,maxiter=maxiter)[0]
    return (1-x)*I_1 + x*I_2

# generate the sensor observations
np.random.seed(534)
v_dat,f_sim = gen_sensor(ϵ,m_f,k_f,Y,u_quad,grid,maxiter=200,require_f=True)
```

Plotting these sensor observations with the solution for this particular realisation of the forcing gives:

```
#hide_input
# plot the sensor readings as well as the true mean
x_range = np.linspace(0,1,100)
f = interp1d(grid,f_sim.flatten(),kind='cubic')
def u(x):
    return u_quad(x,f,maxiter=200)
res = np.array([u(x) for x in x_range])
plt.scatter(Y,v_dat,c='r',label='noisy sensor observations')
plt.plot(x_range,res,c='b',label='solution')
plt.xlabel(r'$x$')
plt.ylim(-0.2,0.3)
plt.title('Noisy sensor observations')
plt.legend()
plt.grid()
plt.show()
```

The next bit of code needed in order to compute the difference between the posterior means is a way of comparing the two different mean functions. One possible solution is to overload the `UserExpression` class in FEniCS to create custom FEniCS expressions from user defined functions. This will allow us to use our function `m_post` together with `errornorm` from FEniCS to compute the L2 norm of the difference. We thus, create a class called `MyExpression`.

```
#export
class MyExpression(UserExpression):
    "Class to allow users to user their own functions to create a FEniCS UserExpression."
    def eval(self, value, x):
        value[0] = self.f(x)
    def value_shape(self):
        return ()
```

```
show_doc(MyExpression,title_level=4)
```

<h4 id="MyExpression" class="doc_header"><code>class</code> <code>MyExpression</code><a href="" class="source_link" style="float:right">[source]</a></h4>

> <code>MyExpression</code>(**\*`args`**, **\*\*`kwargs`**) :: `UserExpression`

Class to allow users to user their own functions to create a FEniCS UserExpression.

We will now demonstrate how this works, building on the sensor observation example above.

```
# set up the true prior mean and the true prior cov needed for the true posterior
μ_true = Expression('0.5*x[0]*(1-x[0])',degree=2)
C_true_s = kernMat(c_u,Y.flatten())
def c_u_vect(x):
    return np.array([c_u(x,y_i) for y_i in Y])
# set up matrix B for posterior
B_true = (ϵ**2)*np.eye(s) + C_true_s

# compute the true posterior mean
def true_post_mean(x):
        return m_post(x,μ_true,c_u_vect,v_dat,Y,B_true)

# set up MyExpression object
μ_true_post = MyExpression()
μ_true_post.f = true_post_mean
μ_true_post
```

`μ_true_post` now works like a usual FEniCS expression/function. We can evaluate it at a point:

```
μ_true_post(0.3)
```

Or even evaluate it on the nodes of a FEniCS mesh:

```
μ_true_post.compute_vertex_values(mesh=UnitIntervalMesh(5))
```

    array([0.        , 0.08179325, 0.12278251, 0.12279555, 0.08181789,
           0.        ])

> Warning: A mesh needs to be passed when using `MyExpression` objects with certain FEniCS methods

We now require code which will create the matrix $C_Y,h$ and the function $\mathbf{c}^{(h)}$ required for the statFEM posterior mean. We will create the function `fem_cov_assembler_post` for this purpose.

```
#export
def fem_cov_assembler_post(J,k_f,Y,parallel,translation_inv):
    "Function to create the matrix $C_{Y,h}$ and the vector function $c^{(h)}$ required for the statFEM posterior mean."

    # set up mesh and function space
    mesh = UnitIntervalMesh(J)
    V = FunctionSpace(mesh,'Lagrange',1)
    tree = mesh.bounding_box_tree()

    # set up grid
    x_grid = V.tabulate_dof_coordinates()

    # set up boundary condition
    def boundary(x, on_boundary):
        return on_boundary

    bc = DirichletBC(V, 0.0, boundary)

    # get the boundary and interior dofs
    bc_dofs = bc.get_boundary_values().keys()
    first, last = V.dofmap().ownership_range()
    all_dofs = range(last - first)
    interior_dofs = list(set(all_dofs) - set(bc_dofs))
    bc_dofs = list(set(bc_dofs))

    # set up the function p
    p = Constant(1.0)

    # get the mass and stiffness matrices
    u = TrialFunction(V)
    v = TestFunction(V)

    mass_form = u*v*dx
    a = inner(p*grad(u),grad(v))*dx

    M = assemble(mass_form)
    A = assemble(a)
    M = as_backend_type(M).mat()
    A = as_backend_type(A).mat()
    M = csr_matrix(M.getValuesCSR()[::-1],shape=M.size)
    A = csr_matrix(A.getValuesCSR()[::-1],shape=A.size)

    # extract the submatrices corresponding to the interior dofs
    M = M[interior_dofs,:][:,interior_dofs]
    A = A[interior_dofs,:][:,interior_dofs]

    # get the forcing cov matrix on the interior nodes of the grid
    Σ_int = kernMat(k_f,x_grid[interior_dofs],parallel,translation_inv)

    # form the matrix Q in the defintion of the approximate FEM cov mat
    # Note: overwrite Σ_int for memory efficiency
    Σ_int = M @ Σ_int @ M.T

    Σ_int = spsolve(A,Σ_int)
    Σ_int = spsolve(A,Σ_int.T).T

    # ensure Σ_int is symmetric
    Σ_int = 0.5*(Σ_int + Σ_int.T)

    # get big phi matrix on the sensor grid (only need the interior dofs)
    Phi = BigPhiMat(J,Y)[interior_dofs,:]

    # assemble the FEM cov mat on the sensor grid and ensure it is symmetric
    Σ_s = Phi.T @ Σ_int @ Phi
    Σ_s = 0.5*(Σ_s + Σ_s.T)

    # set up function to yield the vector (c(x,y)) for y in Y
    def Φ(x):
        cell_index = tree.compute_first_entity_collision(Point(x))
        cell_global_dofs = V.dofmap().cell_dofs(cell_index)
        cell = Cell(mesh, cell_index)
        vertex_coordinates = cell.get_vertex_coordinates()
        cell_orientation = cell.orientation()
        data = V.element().evaluate_basis_all(x,vertex_coordinates,cell_orientation)
        col = np.zeros_like(cell_global_dofs)
        res = csr_matrix((data,(cell_global_dofs,col)),shape=(V.dim(),1))[interior_dofs,:]
        return res

    def c_fem(x):
        return Φ(x).T @ Σ_int @ Phi

    #return Σ and c_fem
    return Σ_s, c_fem
```

`fem_cov_assembler_post` takes in several arguments which are explained below:

- `J`: controls the FE mesh size ($h=1/J$)
- `k_f`: the covariance function for the forcing $f$
- `Y`: vector of sensor locations
- `parallel`: boolean argument indicating whether the computation of the forcing cov mat should be done in parallel
- `translation_inv`: boolean argument indicating whether the computation of the forcing cov mat should be computed assuming `k_f` is translation invariant or not

With all of this code in place we can now finally write the function `m_post_fem_assmebler` which will assemble the statFEM posterior mean function.

```
#export
def m_post_fem_assembler(J,f_bar,k_f,ϵ,Y,v_dat,par=False,trans=True):
    "Function to assemble the statFEM posterior mean function."

    # get number of sensors
    s = len(Y)

    # set up mesh and function space
    mesh = UnitIntervalMesh(J)
    V = FunctionSpace(mesh,'Lagrange',1)

    # set up boundary condition
    def boundary(x, on_boundary):
        return on_boundary

    bc = DirichletBC(V, 0.0, boundary)

    # set up the functions p and f
    p = Constant(1.0)
    f = f_bar

    # set up the bilinear form for the variational problem
    u = TrialFunction(V)
    v = TestFunction(V)
    a = inner(p*grad(u),grad(v))*dx

    # set up linear form
    L = f*v*dx

    # solve the variational problem
    μ_fem = Function(V)
    solve(a == L, μ_fem, bc)

    # use fem_cov_assembler_post to obtain cov mat on sensor grid and function to compute vector
    # (c(x,y)) for y in Y
    B_fem_s, c_fem = fem_cov_assembler_post(J,k_f,Y.flatten(),parallel=par,translation_inv=trans)

    # form B_fem_s by adding noise contribution
    B_fem_s += (ϵ**2)*np.eye(s)

    # assemble function to compute posterior mean and return
    def m_post_fem(x):
        return m_post(x,μ_fem,c_fem,v_dat,Y,B_fem_s)

    return m_post_fem
```

`m_post_fem_assembler` takes in several arguments which are explained below:

- `J`: controls the FE mesh size ($h=1/J$)
- `f_bar`: the mean function for the forcing $f$
- `k_f`: the covariance function for the forcing $f$
- `ϵ`: controls the amount of sensor noise
- `Y`: vector of sensor locations
- `v_dat`: vector of noisy sensor observations
- `par`: boolean argument passed to `fem_cov_assembler_post`'s argument `parallel` (defaults to `False`)
- `trans`: boolean argument passed to `fem_cov_assembler_post`'s argument `translation_inv` (defaults to `True`)

> Important: `m_post_fem_assembler` requires `f_bar` to be represented as a FEniCS function/expression/constant.

Let's quickly check that this function is working.

```
J = 20
f_bar = Constant(1.0)
m_post_fem = m_post_fem_assembler(J,f_bar,k_f,ϵ,Y,v_dat)
# compute posterior mean at a location x in [0,1]
x = 0.3
m_post_fem(x)
```

Let's also plot the statFEM posterior mean together with the corresponding statFEM prior mean:

```
#hide_input
h = 1/J
m_prior = mean_assembler(h,f_bar)
x_range = np.linspace(0,1,100)
y_range = np.array([m_post_fem(x) for x in x_range])
plot(m_prior,label='prior')
plt.plot(x_range,y_range,label='posterior',c='r')
plt.grid()
plt.xlabel(r'$x$')
plt.title('statFEM prior and posterior means')
plt.legend()
plt.show()
```

### Posterior covariance

From the form of the posterior covariance operators $\Sigma_{u|\mathbf{v}}^{(i)}$ given in the section **"Posterior from incorporating sensor readings"** we can see that the posterior covariance functions both have the form:

$$c_{u|\mathbf{v}}^{(i)}(x,y) = c^{(i)}(x,y) - \sum_{p,q=1}^{s}c^{(i)}(x,y_p)(B_{\epsilon,i}^{-1})_{pq}c^{(i)}(y_q,y)$$

Note that this can be expressed as:

$$c_{u|\mathbf{v}}^{(i)}(x,y) = c^{(i)}(x,y) - \mathbf{c}^{(i)}(x)^{T}B_{\epsilon,i}^{-1}\mathbf{c}^{(i)}(y)$$

where we have utilised the fact that $c^{(i)}$ are covariance functions and are hence symmetric which allows us to put $\mathbf{c}^{(i)}(y)=(c^{(i)}(y,y_1),\cdots,c^{(i)}(y,y_s))^{T}=(c^{(i)}(y_1,y),\cdots,c^{(i)}(y_s,y))^{T}$.

Thus, we require a function to evalute the posterior covarainces. We will thus create a function `c_post` which evalutes the posterior covariances.

```
#export
def c_post(x,y,c,Y,B):
    "This function evaluates the posterior covariance at $(x,y)$"
    # compute vectors c_x and c_y:
    c_x = np.array([c(x,y_i) for y_i in Y])
    c_y = np.array([c(y_i,y) for y_i in Y])

    # compute update term
    update = c_x @ np.linalg.solve(B,c_y)

    # return c_post
    return (c(x,y) - update)
```

`c_post` takes in several arguments which are explained below:

- `x`,`y`: points to evaluate the covariance at
- `c`: function which returns the prior covariance at any given pair $(x,y)$
- `Y`: vector of sensor locations
- `B`: the matrix $\epsilon^{2}I+C_{Y}$ to be inverted in order to obtain the posterior

> Note: The function `c_post` will only be used for the true posterior covariances.

### Difference between posterior covariances

In order to compute the difference between the posterior covariances we require some more code. Since we will be comparing the posterior covariances on a fixed reference grid $\{x_{i}\}_{i=1}^{N}$ we will need to assemble the cov matrices on this grid. I.e. we will require the matrices $\tilde{C}_{X,i}$ with $pq$*-th* entry $c_{u|\mathbf{v}}^{(i)}(x_{p},x_{q})$ for $p,q=1,\cdots N$. For statFEM this matrix can be efficiently assembled by exploiting the form of the statFEM prior and posterior covariance functions, i.e. by noting that we have:

$$\tilde{C}_{X,h} = \Sigma_{X} - \Sigma_{XY}B_{\epsilon,h}^{-1}\Sigma_{XY}^{T}$$

where $\Sigma_{X}:=\Phi_{X}^{T}Q\Phi_{X}$, $\Sigma_{XY}=\Phi_{X}^{T}Q\Phi_{Y}$ and where $\Phi_{X}$ is a $J\times N$ matrix whose $i$*-th* column is given by $\phi(x_{i})$ and similarly $\Phi_{Y}$ is a $J\times s$ matrix whose $i$*-th* column is given by $\phi(y_{i})$ and $Q$ is the matrix defined in the section **"Difference between the true prior covariance and the statFEM prior covariance"**.

Thus, we can use our function `BigPhiMat` to compute $\tilde{C}_{X,h}$ efficiently. We start by creating the function `post_fem_cov_assembler` which assembles the matrices $\Sigma_{X}, \Sigma_{XY}$, and $\Sigma_{Y}:=\Phi_{Y}^{T}Q\Phi_{Y}$ required for the statFEM posterior covariance.

```
#export
def post_fem_cov_assembler(J,k_f,grid,Y,parallel,translation_inv):
    "Function which assembles the matrices $Σ_X,Σ_{XY}$, and $Σ_Y$ required for the statFEM posterior covariance."

    # set up mesh and function space
    mesh = UnitIntervalMesh(J)
    V = FunctionSpace(mesh,'Lagrange',1)

    # set up grid
    x_grid = V.tabulate_dof_coordinates()

    # set up boundary condition
    def boundary(x, on_boundary):
        return on_boundary

    bc = DirichletBC(V, 0.0, boundary)

    # get the boundary and interior dofs
    bc_dofs = bc.get_boundary_values().keys()
    first, last = V.dofmap().ownership_range()
    all_dofs = range(last - first)
    interior_dofs = list(set(all_dofs) - set(bc_dofs))
    bc_dofs = list(set(bc_dofs))

    # set up the function p
    p = Constant(1.0)

    # get the mass and stiffness matrices
    u = TrialFunction(V)
    v = TestFunction(V)

    mass_form = u*v*dx
    a = inner(p*grad(u),grad(v))*dx

    M = assemble(mass_form)
    A = assemble(a)
    M = as_backend_type(M).mat()
    A = as_backend_type(A).mat()
    M = csr_matrix(M.getValuesCSR()[::-1],shape=M.size)
    A = csr_matrix(A.getValuesCSR()[::-1],shape=A.size)

    # extract the submatrices corresponding to the interior dofs
    M = M[interior_dofs,:][:,interior_dofs]
    A = A[interior_dofs,:][:,interior_dofs]

    # get the forcing cov matrix on the interior nodes of the grid
    Σ_int = kernMat(k_f,x_grid[interior_dofs],parallel,translation_inv)

    # form the matrix Q in the defintion of the approximate FEM cov mat
    # Note: overwrite Σ_int for memory efficiency
    Σ_int = M @ Σ_int @ M.T

    Σ_int = spsolve(A,Σ_int)
    Σ_int = spsolve(A,Σ_int.T).T

    # ensure Σ_int is symmetric
    Σ_int = 0.5*(Σ_int + Σ_int.T)

    # get big phi matrix on the grid (on;y need the interior nodes)
    Phi_grid = BigPhiMat(J,grid)[interior_dofs,:]

    # get big phi matrix on the sensor grid (only need the interior nodes)
    Phi_Y = BigPhiMat(J,Y)[interior_dofs,:]

    # assemble the FEM cov mat on the sensor grid using Σ_int and Phi_Y
    Σ_Y = Phi_Y.T @ Σ_int @ Phi_Y

    # assemble the FEM cov mat on the grid using Σ_int and Phi_grid
    Σ_X = Phi_grid.T @ Σ_int @ Phi_grid

    # assemble cross term matrix (with ijth entry c(x_i,y_j))
    Σ_XY = Phi_grid.T @ Σ_int @ Phi_Y

    # return these sigma matrices
    return Σ_Y, Σ_X, Σ_XY
```

`post_fem_cov_assembler` takes in several arguments which are explained below:

- `J`: controls the FE mesh size ($h=1/J$)
- `k_f`: the covariance function for the forcing $f$
- `grid`: the fixed reference grid $\{x_{i}\}_{i=1}^{N}$ on which to assemble the posterior cov mat
- `Y`: vector of sensor locations.
- `parallel`: boolean argument indicating whether the computation of the forcing cov mat should be done in parallel
- `translation_inv`: boolean argument indicating whether the computation of the forcing cov mat should be computed assuming `k_f` is translation invariant or not

Finally, we create the function `c_post_fem_assembler` which assembles the statFEM posterior cov mat on the reference grid using the matrices `post_fem_cov_assembler` returns.

```
#export
def c_post_fem_assembler(J,k_f,grid,Y,ϵ,par,trans):
    "Function to assemble the statFEM posterior cov mat on a reference grid specified by grid."
    # use post_fem_cov_assembler to get the sigma matrices needed for posterior cov mat
    Σ_Y, Σ_X, Σ_XY = post_fem_cov_assembler(J,k_f,grid,Y,parallel=par,translation_inv=trans)

    # create the matrix B
    s = len(Y) # number of sensor points
    B = (ϵ**2)*np.eye(s) + Σ_Y

    #form the posterior cov matrix
    update = Σ_XY @ np.linalg.solve(B,Σ_XY.T)

    return Σ_X - update
```

Let's quickly demonstrate that this code is working by computing the statFEM posterior covariance matrix on a reference grid and comparing this to the corresponding statFEM prior.

```
# set up reference grid and J
N = 21
grid = np.linspace(0,1,N)
J = 20

# get statFEM prior cov mat on this grid
Σ_prior = cov_assembler(J,k_f,grid,False,True)

# get statFEM posterior cov mat on this grid
Σ_posterior = c_post_fem_assembler(J,k_f,grid,Y,ϵ,False,True)
```

```
#hide_input
vmin = min(Σ_prior.min(), Σ_posterior.min())
vmax = max(Σ_prior.max(), Σ_posterior.max())
plt.rcParams['figure.figsize'] = (12,6)
fig, axs = plt.subplots(ncols=3, gridspec_kw=dict(width_ratios=[4,4,0.2]))
sns.heatmap(Σ_prior,cbar=False,
                annot=False,
                xticklabels=False,
                yticklabels=False,
                cmap=cm.viridis,
                ax=axs[0])
axs[0].title.set_text('statFEM prior covariance')
sns.heatmap(Σ_posterior,cbar=False,
                annot=False,
                xticklabels=False,
                yticklabels=False,
                cmap=cm.viridis,
                ax=axs[1])
axs[1].title.set_text('statFEM posterior covariance')
fig.colorbar(axs[np.argmax([Σ_prior.max(), Σ_posterior.max()])].collections[0], cax=axs[2])
plt.tight_layout()
plt.show()
```

```
#hide
from nbdev.export import notebook2script; notebook2script()
```

    Converted 00_oneDim.ipynb.
    Converted index.ipynb.


# Probabilidad I:
Valor esperado e indicadores. Teorema de Bayes. Estimación Bayesiana.

# 0. PMFs y PDFs conjuntas.

- Una PMF conjunta $p_{X,Y}$ de las variables $X$ y $Y$ está definida como

\begin{equation}
p_{X,Y}(x,y)=P(X=x,Y=y)
\end{equation}

- La PMF marginal de X y Y puede ser obtenida a partir de la PMF conjunta, utilizando

\begin{equation}
p_{X}(x)=\sum_{y}{p_{X,Y}(x,y)}
\end{equation}
y
\begin{equation}
p_{Y}(y)=\sum_{x}{p_{X,Y}(x,y)}
\end{equation}

De manera análoga, para $X$ y $Y$ que sean variables aleatorias conjuntas con una PDF conjunta $f_{X,Y}$ se tiene que

\begin{equation}
f_{X,Y}(x,y)=f_{Y}(y)f_{X|Y}(x|y)
\end{equation}

\begin{equation}
f_{X}(x)=\int_{-\infty}^{\infty}f_{Y}(y)f_{X|Y}(x|y)dy
\end{equation}

#### Ejercicio 1: Realice dos gráficas de funciones de distribución conjuntas para variables aleatorias X,Y distribuidas normales para medias y varianzas iguales y medias y varianzas diferentes

# 1. Valor Esperado.

Para variables discretas se define el valor esperado de una variable discreta $X$ como

\begin{equation}
E[X] =\sum_{x}{x p_{X} (x)}
\end{equation}

y para una variable continua:

\begin{equation}
E[X] =\int_{x}{x p_{X} (x)}dx
\end{equation}

y la varianza se define como

\begin{equation}
var[X] = E[(X-E[X])^2]
\end{equation}

#### Ejercicio 2: Obtenga numéricamente el valor esperado y la varianza de una función de distribución Gamma generada aleatoriamente con parámetros $1/\lambda=0.5$ y $k=9$. Realice la gráfica de la distribución.

```python

```


# Vertical Line Test

## 1.1 Create two graphs, one that passes the vertical line test and one that does not.

```python
import matplotlib.pyplot as plt

plt.plot([0,1,2], [1,3,9])
plt.xlim(0, 9)
plt.ylim(0, 9)
plt.show();
```

```python
plt.plot([1,1,2], [1,3,9])
plt.xlim(0, 9)
plt.ylim(0, 9)
plt.show();
```

## 1.2 Why are graphs that don't pass the vertical line test not considered "functions?"

Functions have one and only one output for a given input.

# Functions as Relations

## 2.1 Which of the following relations are functions? Why?

\begin{align}
\text{Relation 1: } \{(1, 2), (3, 2), (1, 3)\}
\\
\text{Relation 2: } \{(1, 3), (2, 3), (6, 7)\}
\\
\text{Relation 3: } \{(9, 4), (2, 1), (9, 6)\}
\\
\text{Relation 4: } \{(6, 2), (8, 3), (6, 4)\}
\\
\text{Relation 5: } \{(2, 6), (2, 7), (2, 4)\}
\end{align}

Only Relation 2. The other functions have multiple outputs for the same input.

# Functions as a mapping between dimensions

## 3.1 for the following functions what is the dimensionality of the domain (input) and codomain (range/output)?

\begin{align}
m(𝑥_1,𝑥_2,𝑥_3)=(x_1+x_2, x_1+x_3, x_2+x_3)
\\
n(𝑥_1,𝑥_2,𝑥_3,𝑥_4)=(x_2^2 + x_3, x_2x_4)
\end{align}

**m**

input: R^3

output: R^3

**n**

input: R^3

output: R^2

## 3.2 Do you think it's possible to create a function that maps from a lower dimensional space to a higher dimensional space? If so, provide an example.

Sure
\begin{align}
m(𝑥_1,𝑥_2)=(x_1, x_2, x_1+x_2)
\end{align}

# Vector Transformations

## 4.1 Plug the corresponding unit vectors into each function. Use the output vectors to create a transformation matrix.

\begin{align}
p(\begin{bmatrix}x_1 \\ x_2 \end{bmatrix}) = \begin{bmatrix} x_1 + 3x_2 \\2 x_2 - x_1 \\  \end{bmatrix}
\\
\\
p(\begin{bmatrix}x_1 \\ x_2 \end{bmatrix}) = x_1\begin{bmatrix} 1 \\-1 \\  \end{bmatrix} + x_2\begin{bmatrix} 3 \\2 \\  \end{bmatrix}
\\
\\
p(\begin{bmatrix}x_1 \\ x_2 \end{bmatrix}) = \begin{bmatrix} x_1 \\x_2 \\  \end{bmatrix} \begin{bmatrix} 1 &3 \\-1 &2 \\  \end{bmatrix}
\\
\\
q(\begin{bmatrix}x_1 \\ x_2 \\ x_3\end{bmatrix}) = \begin{bmatrix} 4x_1 + x_2 + 2x_3 \\2 x_2 - x_1 + 3x_3 \\ 5x_1 - 2x_3 + x_2  \end{bmatrix}
\\
\\
q(\begin{bmatrix}x_1 \\ x_2 \\ x_3\end{bmatrix}) = \begin{bmatrix} x_1 \\ x_2 \\ x_3 \end{bmatrix}\begin{bmatrix} 4 & 1 & 2 \\ -1 & 2 & 3 \\ 5 & 1 & -2 \end{bmatrix}
\\
\end{align}

## 4.2 Verify that your transformation matrices are correct by choosing an input matrix and calculating the result both via the traditional functions above and also via vector-matrix multiplication.

```
p(4, 7) =
    x_1 = 4 + 3*7 = 25
    x_2 = 2*7 - 4 = 10
```

```python
import numpy as np

p_trans = np.array([[ 1, 3],
                    [-1, 2]])
p_test = np.array([[4],
                   [7]])
np.matmul(p_trans, p_test)
```

    array([[25],
           [10]])

```
q(1, 2, 3) =
    x_1 = 4*1 + 2 + 2*3 = 12
    x_2 = 2*2 - 1 + 3*3 = 12
    x_3 = 5*1 - 2*3 + 2 = 1
```

```python
q_trans = np.array([[ 4, 1,  2],
                    [-1, 2,  3],
                    [ 5, 1, -2]])
q_test = np.array([[1],
                   [2],
                   [3]])
np.matmul(q_trans, q_test)
```

    array([[12],
           [12],
           [ 1]])

# Eigenvalues and Eigenvectors

## 5.1 In your own words, give an explanation for the intuition behind eigenvalues and eigenvectors.

**Eigenvectors** are the unique vectors of a space-transformation pair that are not re-oriented by the transformation of that space. However, the eigenvecotr may be scaled by this transformation, in which case this scalar is known as the **eigenvalue**.

# The Curse of Dimensionality

## 6.1 What are some of the challenges of working with high dimensional spaces?

With more dimensions, the more "distant" vectors are from each other, the less meaningful that distance is (they become equidistant), and the greater the threat of over-fitting.

## 6.2 What is the rule of thumb for how many observations you should have compared to parameters in your model?

5 * number of dimensions

# Principal Component Analysis

## 7.1 Load the UCI Machine Learning Repository's [Iris Dataset](https://gist.githubusercontent.com/curran/a08a1080b88344b0c8a7/raw/d546eaee765268bf2f487608c537c05e22e4b221/iris.csv) and use PCA to isolate the dataset's first and second principal components and plot them on a graph.

```python
import pandas as pd

df = pd.read_csv('https://gist.githubusercontent.com/curran/a08a1080b88344b0c8a7/'
                 'raw/d546eaee765268bf2f487608c537c05e22e4b221/iris.csv')
target = 'species'
features = [col for col in df if col != target]
X = df[features].copy()

for feature in X:
    X[feature] = (X[feature] - X[feature].mean()) / X[feature].std(ddof=1)

covariance_matrix = np.cov(X.T)

eig_values, eig_vectors = np.linalg.eig(covariance_matrix)

P = eig_vectors.T.dot(X.T)

P
```

    array([[-2.25698063e+00, -2.07945912e+00, -2.36004408e+00,
            -2.29650366e+00, -2.38080159e+00, -2.06362348e+00,
            -2.43754534e+00, -2.22638327e+00, -2.33413810e+00,
            -2.18136797e+00, -2.15626287e+00, -2.31960686e+00,
            -2.21665672e+00, -2.63090249e+00, -2.18497165e+00,
            -2.24394778e+00, -2.19539570e+00, -2.18286636e+00,
            -1.88775015e+00, -2.33213620e+00, -1.90816387e+00,
            -2.19728429e+00, -2.76490710e+00, -1.81433338e+00,
            -2.22077769e+00, -1.95048969e+00, -2.04521166e+00,
            -2.16095426e+00, -2.13315968e+00, -2.26121491e+00,
            -2.13739396e+00, -1.82582143e+00, -2.59949432e+00,
            -2.42981077e+00, -2.18136797e+00, -2.20373717e+00,
            -2.03759040e+00, -2.18136797e+00, -2.42781878e+00,
            -2.16329995e+00, -2.27889274e+00, -1.86545777e+00,
            -2.54929405e+00, -1.95772074e+00, -2.12624970e+00,
            -2.06842817e+00, -2.37330742e+00, -2.39018435e+00,
            -2.21934620e+00, -2.19858869e+00,  1.10030752e+00,
             7.30035752e-01,  1.23796222e+00,  3.95980711e-01,
             1.06901266e+00,  3.83174476e-01,  7.46215186e-01,
            -4.96201068e-01,  9.23129797e-01,  4.95143781e-03,
            -1.24281108e-01,  4.37265239e-01,  5.49792127e-01,
             7.14770518e-01, -3.71339807e-02,  8.72966018e-01,
             3.46844441e-01,  1.52880381e-01,  1.21124542e+00,
             1.56417164e-01,  7.35791136e-01,  4.70792484e-01,
             1.22388808e+00,  6.27279600e-01,  6.98133985e-01,
             8.70620328e-01,  1.25003446e+00,  1.35370481e+00,
             6.59915360e-01, -4.71236447e-02,  1.21128417e-01,
             1.40710866e-02,  2.35222819e-01,  1.05316323e+00,
             2.20677797e-01,  4.30341477e-01,  1.04590946e+00,
             1.03241951e+00,  6.68436674e-02,  2.74505447e-01,
             2.71425765e-01,  6.21089831e-01,  3.28903506e-01,
            -3.72380115e-01,  2.81999618e-01,  8.87557702e-02,
             2.23607677e-01,  5.71967342e-01, -4.55486949e-01,
             2.51402252e-01,  1.84150339e+00,  1.14933941e+00,
             2.19898270e+00,  1.43388176e+00,  1.86165399e+00,
             2.74500070e+00,  3.57177896e-01,  2.29531637e+00,
             1.99505169e+00,  2.25998344e+00,  1.36134878e+00,
             1.59372546e+00,  1.87796051e+00,  1.24890257e+00,
             1.45917316e+00,  1.58649440e+00,  1.46636772e+00,
             2.42924030e+00,  3.29809227e+00,  1.24979406e+00,
             2.03368323e+00,  9.70663302e-01,  2.88838807e+00,
             1.32475564e+00,  1.69855041e+00,  1.95119099e+00,
             1.16799163e+00,  1.01637610e+00,  1.78004554e+00,
             1.85855159e+00,  2.42736549e+00,  2.30834923e+00,
             1.85415982e+00,  1.10756129e+00,  1.19347092e+00,
             2.79159729e+00,  1.57487926e+00,  1.34254677e+00,
             9.20349720e-01,  1.84736315e+00,  2.00942544e+00,
             1.89676253e+00,  1.14933941e+00,  2.03648602e+00,
             1.99500751e+00,  1.86427657e+00,  1.55328823e+00,
             1.51576710e+00,  1.37179555e+00,  9.56095566e-01],
           [-5.04015404e-01,  6.53216394e-01,  3.17413945e-01,
             5.73446613e-01, -6.72514411e-01, -1.51347827e+00,
            -7.43137171e-02, -2.46787172e-01,  1.09148977e+00,
             4.47131117e-01, -1.06702096e+00, -1.58057946e-01,
             7.06750478e-01,  9.35149145e-01, -1.88366805e+00,
            -2.71328133e+00, -1.50869601e+00, -5.12587094e-01,
            -1.42633236e+00, -1.15416686e+00, -4.29027880e-01,
            -9.49277150e-01, -4.87882574e-01, -1.06394362e-01,
            -1.61644638e-01,  6.05862870e-01, -2.65126115e-01,
            -5.50173363e-01, -3.35516398e-01,  3.13827252e-01,
             4.82326259e-01, -4.43780131e-01, -1.82237008e+00,
            -2.17809480e+00,  4.47131117e-01,  1.83722324e-01,
            -6.82669420e-01,  4.47131117e-01,  8.79223933e-01,
            -2.91749567e-01, -4.66429135e-01,  2.31991966e+00,
             4.52301130e-01, -4.95730895e-01, -1.16752081e+00,
             6.89607099e-01, -1.14679074e+00,  3.61180775e-01,
            -1.02205856e+00, -3.21302061e-02, -8.60230593e-01,
            -5.96636785e-01, -6.12769614e-01,  1.75229858e+00,
             2.11050863e-01,  5.89088966e-01, -7.76098609e-01,
             1.84269557e+00, -3.02295550e-02,  1.02596404e+00,
             2.64918765e+00,  5.86846859e-02,  1.76666308e+00,
             1.84815166e-01,  4.31350036e-01, -5.08295314e-01,
             1.89985179e-01,  7.88085297e-01,  1.62790202e+00,
             1.29875233e+00, -4.01126570e-01,  4.15217206e-01,
             9.37773165e-01,  4.15419947e-01,  6.32819273e-02,
            -2.49871518e-01,  8.23442389e-02, -3.27722366e-01,
             2.23597000e-01,  1.05368248e+00,  1.55837169e+00,
             1.56813894e+00,  7.73333046e-01,  6.34774729e-01,
             2.79909969e-01, -8.52281697e-01, -5.20453696e-01,
             1.38781717e+00,  2.11910814e-01,  1.32537578e+00,
             1.11570381e+00, -2.74506710e-02,  9.85598884e-01,
             2.01119458e+00,  8.51099455e-01,  1.74324544e-01,
             3.79214256e-01,  1.53206717e-01,  1.53432438e+00,
             5.93871222e-01, -8.68786147e-01,  6.98984451e-01,
            -5.52618781e-01,  4.98435418e-02, -2.90220536e-01,
            -7.85799704e-01,  1.55488557e+00, -4.08149357e-01,
             7.21448440e-01, -1.91502747e+00, -6.91631011e-01,
             4.26818953e-01, -4.12949339e-01,  1.16349352e+00,
             4.42664602e-01, -6.74774813e-01, -2.52347086e-01,
            -2.54822057e+00,  2.35343587e-03,  1.71184899e+00,
            -9.04369044e-01,  5.69267278e-01, -3.96463171e-01,
             4.85135293e-01, -1.01076228e+00, -9.99984474e-01,
             3.17831851e-01, -6.53241212e-02,  1.92627480e-01,
            -5.53527164e-01, -2.45830912e-01, -2.61741528e+00,
             1.84055790e-01,  2.94997832e-01,  8.14439294e-01,
            -8.41927658e-01, -1.06889360e+00, -4.20846092e-01,
            -1.91661621e-02, -6.70177572e-01, -6.08358978e-01,
            -6.83734258e-01,  6.98984451e-01, -8.61797778e-01,
            -1.04504904e+00, -3.81543631e-01,  9.02290843e-01,
            -2.65903772e-01, -1.01296839e+00,  2.22095406e-02],
           [-1.21536190e-01, -2.26492063e-01,  5.13077379e-02,
             9.85303586e-02,  2.13563034e-02,  3.05818064e-02,
             3.41055070e-01, -8.22987560e-02,  1.53049668e-01,
            -2.45736283e-01, -2.63127870e-01,  9.98311719e-02,
            -2.22469221e-01,  1.88937073e-01, -4.67912544e-01,
             3.24949356e-02, -1.35893901e-03, -3.84866610e-02,
            -3.69504958e-01,  1.36958894e-01, -4.14617866e-01,
             1.64188300e-01,  3.37705041e-01,  3.99277109e-02,
             1.23786731e-01, -2.97594061e-01,  9.17854889e-02,
            -2.00623375e-01, -2.64428684e-01,  7.52632970e-02,
            -6.76291966e-02, -2.64489180e-01,  5.12478331e-02,
            -7.90848134e-02, -2.45736283e-01, -2.17894561e-01,
            -4.77810860e-01, -2.45736283e-01,  2.00884605e-01,
            -1.69371127e-01,  4.06005234e-02, -1.93879097e-01,
             3.12524850e-01,  3.13704670e-01,  2.51949169e-01,
            -5.63701629e-02,  6.18945510e-02,  1.46365295e-01,
            -1.76055500e-01, -1.46104065e-01, -6.83266103e-01,
            -9.37527216e-02, -5.52993953e-01, -1.85074806e-02,
            -3.96120397e-01,  1.26374324e-01,  1.48159674e-01,
             2.54701975e-01, -5.93471704e-01,  5.41054466e-01,
            -3.96805148e-02,  2.06013327e-01, -7.58838045e-01,
            -6.70751344e-02,  1.97400140e-01, -5.01824672e-01,
             4.91185999e-01, -2.97607503e-01, -4.77809208e-01,
            -1.68023850e-01,  6.14713279e-01, -2.61841091e-01,
            -3.65480465e-01, -2.88994316e-01, -4.43282521e-01,
            -4.70572424e-01, -7.24416666e-01, -2.60585089e-01,
             8.70763929e-02, -3.14266373e-01, -1.44756789e-01,
            -2.35791505e-01, -1.47478817e-01,  1.06396795e-01,
             6.65330740e-01,  4.49226536e-01, -3.94819584e-01,
            -6.83145701e-01,  2.93146195e-01,  9.31327651e-02,
             9.78441041e-02, -1.92401979e-02, -1.95313754e-01,
             1.11809481e-01,  1.33671013e-01,  1.31009481e-01,
             1.58238888e-01, -2.69137780e-01,  1.95471326e-01,
             9.44335784e-02,  9.99412860e-01,  5.29690124e-01,
            -2.04809778e-01,  1.62844918e-01,  3.92688789e-01,
            -5.84275328e-01,  9.87691038e-01, -6.51982485e-01,
            -3.92754684e-01,  3.91207078e-01,  2.82333672e-01,
             2.32262719e-02,  2.44665886e-02,  5.80186592e-01,
             1.00075789e+00,  6.34525003e-01,  3.65351132e-02,
            -1.33752000e-01, -7.04573572e-01, -2.63738535e-01,
             2.31103895e-01,  8.26734145e-01, -8.58052287e-01,
            -4.69163193e-03,  2.94969701e-01, -4.21405305e-01,
             1.30215675e-01,  3.36913478e-01,  2.69101012e-01,
            -7.15114982e-01, -7.27795822e-01, -4.97951359e-01,
             3.52150542e-01, -1.82049724e-01, -1.62668826e-01,
            -5.45174572e-01,  9.40242708e-01,  1.79427607e-01,
             4.16000663e-01, -1.47708457e-02,  4.24492856e-01,
             1.27372654e-01,  5.29690124e-01,  3.34146638e-01,
             6.27167818e-01,  2.53682459e-01, -2.52971620e-02,
             1.78678613e-01,  9.28295177e-01,  5.27028592e-01],
           [ 2.29962838e-02,  1.02863642e-01,  2.77323192e-02,
            -6.60900544e-02, -3.72724209e-02,  4.38408784e-03,
            -3.79693659e-02, -2.54200029e-02, -2.67043670e-02,
            -3.97740567e-02,  1.52500486e-02, -1.34104994e-01,
             2.62183044e-03, -1.93571700e-02,  1.92138361e-01,
             4.69683560e-02,  1.86008435e-01,  9.15986427e-02,
             5.93293380e-02, -3.96952015e-02,  1.03256322e-02,
             5.75364099e-02,  1.77941178e-02,  1.49843605e-01,
            -2.70323255e-01,  4.36909201e-02,  6.63786282e-02,
             9.22964909e-03,  8.32649884e-02, -1.08485942e-01,
            -4.82172368e-02,  2.38342524e-01, -2.31148225e-01,
            -5.08506746e-02, -3.97740567e-02,  1.68056763e-01,
             1.94960180e-01, -3.97740567e-02, -9.92753256e-03,
             6.21944933e-03,  1.05365277e-01,  2.90719046e-01,
            -6.71860375e-02,  1.74954094e-01, -1.52717190e-01,
             1.39826548e-01, -1.53703647e-01, -4.93132199e-02,
            -1.63894036e-02,  4.86153365e-02,  3.48610735e-02,
             4.43889296e-03,  9.64105894e-03,  6.71728117e-02,
             1.05189268e-01, -2.39724981e-01, -7.80396265e-02,
            -3.92576228e-02, -2.90052501e-02, -2.82541090e-02,
             1.60866655e-02,  3.97183975e-02,  4.81922483e-02,
            -1.64006239e-01,  1.08661097e-01,  1.04790230e-01,
            -1.91418220e-01, -2.03639005e-01,  2.27452513e-01,
            -5.02448721e-02, -8.41695519e-02,  1.13863263e-01,
            -8.42013960e-03, -2.72581705e-01,  4.39341062e-02,
             1.01780030e-01,  4.06930920e-02,  6.67900380e-02,
            -3.62311585e-02,  6.57873163e-02, -7.84898495e-03,
            -3.10452570e-02,  2.43778864e-02, -1.82806816e-01,
            -2.54697124e-01, -1.10775062e-01,  3.71743283e-02,
             1.38664082e-01, -1.46998590e-01,  9.91430675e-03,
            -2.68941652e-01, -1.47229405e-01,  7.60105196e-03,
             2.10110819e-02, -1.06516920e-01, -2.29367584e-01,
            -1.32135972e-01, -1.93447983e-02,  2.00212649e-01,
            -5.81006331e-02, -5.08975254e-02, -4.02786435e-02,
             5.91025008e-02, -2.34972681e-01, -1.67257667e-02,
            -1.00542846e-01, -1.32543405e-01, -2.36420767e-01,
            -8.47100358e-02,  1.02563700e-01,  1.06653619e-01,
             5.87458960e-02,  1.45808492e-01,  9.93488551e-02,
             2.74103899e-01,  1.90009070e-01, -1.54916942e-01,
            -2.74741142e-01,  4.66000729e-02, -6.28568258e-02,
             1.66581983e-01,  2.72277324e-02, -1.25653335e-01,
             1.40128432e-01, -6.25308917e-02, -2.17729716e-01,
             1.25265815e-01, -9.03822942e-03,  3.11031008e-02,
            -2.06863755e-01, -1.67375294e-02, -2.12448695e-01,
             9.97054598e-02, -1.85120071e-01, -4.86773264e-01,
             2.95331758e-01,  3.34952108e-02, -2.15185647e-01,
             4.72840526e-03,  1.94224778e-01,  2.45940777e-01,
             4.67647757e-01, -4.02786435e-02,  4.41303571e-02,
             2.11878544e-01,  3.87592018e-01,  2.20583209e-01,
             1.18506037e-01,  2.40655734e-02, -1.63129308e-01]])

```python
# Using sklearn
from sklearn.decomposition import PCA

target = 'species'
features = [col for col in df if col != target]
X = df[features].copy()

pca = PCA(2)
pca.fit(X)
B = pca.transform(X)
```

```python
import seaborn as sns

sns.scatterplot(x=0, y=1, data=df[[target]].join(pd.DataFrame(B)), hue='species');
```

# Stretch Goal

1) Do NOT work on the stretch goal until you feel like you have a firm grasp of eigenvectors, eigenvalues, and PCA. Prioritize self-study over the stretch goal if you are not comfortable with those topics yet.

2) Explore further the intuition behind eigenvalues and eigenvectors by creating your very own eigenfaces:

You don't necessarily have to use this resource, but this will get you started:
[Eigenface Tutorial](https://sandipanweb.wordpress.com/2018/01/06/eigenfaces-and-a-simple-face-detector-with-pca-svd-in-python/)