```
# Alexander Hebert
# ECE 6390
# Computer Project #2
# Method 2/3
```

```
# Tested using Python v3.4 and IPython v2
```

```
# Import libraries and functions
```

```
import numpy as np
```

```
from PlaneRotationFn import planeRotation1
from PlaneRotationFn import planeRotation2
```

```
import scipy
```

```
import sympy
```

```
from IPython.display import display
```

```
from sympy.interactive import printing
```

```
np.set_printoptions(precision=6)
```

```
#np.set_printoptions(suppress=True)
```

```
# Original system:
```

```
A = np.loadtxt('A_ex1.txt')
```

```
A
```

    array([[ 1.38  , -0.2077,  6.715 , -5.676 ],
           [-0.5814, -4.29  ,  0.    ,  0.675 ],
           [ 1.067 ,  4.273 , -6.654 ,  5.893 ],
           [ 0.048 ,  4.273 ,  1.343 , -2.104 ]])

```
n,nc = A.shape
```

```
B = np.loadtxt('B_ex1.txt')
```

```
B
```

    array([[ 0.   ,  0.   ],
           [ 5.679,  0.   ],
           [ 1.136, -3.146],
           [ 1.136,  0.   ]])

```
nr,m = B.shape
```

```
# Compute eigenvalues/poles of A to determine system stability:
```

```
A_eigvals, M = np.linalg.eig(A)
```

```
A_eigvals
```

    array([ 1.99096 ,  0.063508, -5.056574, -8.665894])

```
# Two poles lie in the RHP and are unstable.
```

```
A_eigvals_desired = np.array([-0.2,-0.5,A_eigvals[2],A_eigvals[3]])
```

```
A_eigvals_desired
```

    array([-0.2     , -0.5     , -5.056574, -8.665894])

```
Lambda = np.diag(A_eigvals_desired)
```

```
Lambda
```

    array([[-0.2     ,  0.      ,  0.      ,  0.      ],
           [ 0.      , -0.5     ,  0.      ,  0.      ],
           [ 0.      ,  0.      , -5.056574,  0.      ],
           [ 0.      ,  0.      ,  0.      , -8.665894]])

```
# Pole Assignment Algorithm from journal paper
```

```
# Step A: Decomposition of B using SVD
# B = U*S*V.H
```

```
U, s, VH = np.linalg.svd(B)
```

```
U
```

    array([[  6.016779e-18,  -8.304906e-17,  -9.868038e-01,  -1.619203e-01],
           [ -9.460842e-01,  -2.577794e-01,   3.176055e-02,  -1.935609e-01],
           [ -2.628862e-01,   9.648268e-01,  -2.220446e-16,   0.000000e+00],
           [ -1.892501e-01,  -5.156495e-02,  -1.587748e-01,   9.676342e-01]])

```
s
```

    array([ 5.944254,  3.065158])

```
S = np.zeros((4, 2))
S[:2, :2] = np.diag(s)
```

```
S
```

    array([[ 5.944254,  0.      ],
           [ 0.      ,  3.065158],
           [ 0.      ,  0.      ],
           [ 0.      ,  0.      ]])

```
VH
```

    array([[-0.990274,  0.139133],
           [-0.139133, -0.990274]])

```
# Extract U_0 and U_1 from matrix U = [U_0,U_1]
```

```
U_0 = U[:n,:m]
```

```
U_0
```

    array([[  6.016779e-18,  -8.304906e-17],
           [ -9.460842e-01,  -2.577794e-01],
           [ -2.628862e-01,   9.648268e-01],
           [ -1.892501e-01,  -5.156495e-02]])

```
U_1 = U[:n,m:]
```

```
U_1
```

    array([[ -9.868038e-01,  -1.619203e-01],
           [  3.176055e-02,  -1.935609e-01],
           [ -2.220446e-16,   0.000000e+00],
           [ -1.587748e-01,   9.676342e-01]])

```
# B = [U_0,U_1][Z,0].T
# Compute Z from SVD of B
```

```
Z = np.diag(s).dot(VH)
```

```
Z
```

    array([[-5.886439,  0.82704 ],
           [-0.426464, -3.035345]])

```
# U_1.T *(A - lambda_j*I)
# Compute S_hat_j and S_j

for j in range(len(A_eigvals_desired)):

    lambda_j = A_eigvals_desired[j]

    # M_j is a temp matrix
    exec("M_%d = np.dot(U_1.T,(A - lambda_j*np.identity(n)))" %(j+1))

    # U_1.T *(A - lambda_j*I) = T_j *[Gamma_j,0]*[S_j_hat,S_j].T
    exec("T_%d, gamma_%d, SH_%d = np.linalg.svd(M_%d)" %(j+1,j+1,j+1,j+1))

    exec("S_hat_%d = SH_%d[:m,:].T" %(j+1,j+1))
    exec("S_%d = SH_%d[m:,:].T" %(j+1,j+1))

```

```
# Initial eigenvectors in X_tilde
X_tilde = np.eye(n)
X_tilde
```

    array([[ 1.,  0.,  0.,  0.],
           [ 0.,  1.,  0.,  0.],
           [ 0.,  0.,  1.,  0.],
           [ 0.,  0.,  0.,  1.]])

```
# Initial eigenvectors in X
X = np.zeros((n,n))
X
```

    array([[ 0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.]])

```
# Step X with Method 2/3

maxiter = 1
v4prev = 0
v4current = 0

for r in range(maxiter):

    for j in range(n):

        xt_j = X_tilde[:,j].reshape((n,1))

        for k in range(j+1,n,1):

            xt_k = X_tilde[:,k].reshape((n,1))

            exec("phi_j = np.linalg.norm(np.dot(S_hat_%d.T,xt_j))" %(j+1))
            exec("phi_k = np.linalg.norm(np.dot(S_hat_%d.T,xt_k))" %(k+1))

            if (phi_j < phi_k):

                sin_theta = phi_j
                cos_theta = np.sqrt(1 - phi_j**2)
                protation = planeRotation2(n,cos_theta,sin_theta,j,k)
                v4current = v4current + phi_j**2

            else:

                sin_theta = phi_k
                cos_theta = np.sqrt(1 - phi_k**2)
                protation = planeRotation2(n,cos_theta,sin_theta,j,k)
                v4current = v4current + phi_k**2

            X_tilde = np.dot(protation,X_tilde)

    v4current = np.sqrt(v4current)
    print(v4current - v4prev)
    v4prev = v4current

```

    1.18443024921

```
# Compute eigenvectors x_j in X from X_tilde

for j in range(n):

    xt_j = X_tilde[:,j].reshape((n,1))
    exec("x_j = np.dot(np.dot(S_%d,S_%d.T),xt_j) / np.linalg.norm(np.dot(S_%d.T,xt_j))" %(j+1,j+1,j+1))
    X[:,j] = x_j[:,0]

```

```
X
```

    array([[ 0.948809, -0.093524, -0.671977,  0.142779],
           [ 0.081417,  0.084568, -0.246883, -0.879926],
           [ 0.035477,  0.657038,  0.694792,  0.126892],
           [ 0.303108,  0.743238,  0.068987,  0.435021]])

```
np.linalg.matrix_rank(X)
```

    4

```
X_inv = np.linalg.inv(X)
```

```
X_inv
```

    array([[ 1.338859, -0.09351 ,  1.363573, -1.026314],
           [-0.554042,  0.641841, -0.468308,  1.61671 ],
           [ 0.46658 , -0.426158,  1.894783, -1.567827],
           [-0.060276, -0.963857, -0.450465,  0.500306]])

```
# M defined as A + BF
M = X.dot(Lambda).dot(X_inv)
```

```
M
```

    array([[ 1.38    , -0.2077  ,  6.715   , -5.676   ],
           [ 0.12447 , -7.907368, -1.071934,  1.806108],
           [-1.400423,  2.346898, -6.017361,  4.434204],
           [ 0.189199,  3.549399,  1.128575, -1.877739]])

```
M_eigvals, H = np.linalg.eig(M)
M_eigvals
```

    array([-8.665894, -5.056574, -0.5     , -0.2     ])

```
# Compute feedback matrix F
F = np.dot(np.linalg.inv(Z),np.dot(U_0.T,(M - A)))
```

```
F
```

    array([[ 0.124295, -0.636973, -0.188754,  0.199174],
           [ 0.829187,  0.382232, -0.270522,  0.535619]])

```
np.linalg.norm(F)
```

    1.300078330255144

```
np.linalg.cond(X)
```

    4.9355547197097671


```python
from IPython.core.display import HTML
css_file = './custom.css'
HTML(open(css_file, "r").read())
```

###### Content provided under a Creative Commons Attribution license, CC-BY 4.0; code under MIT License. (c)2015 [David I. Ketcheson](http://davidketcheson.info)

##### Version 0.2 - May 2021

```python
import numpy as np
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.animation
from IPython.display import HTML
font = {'size'   : 15}
matplotlib.rc('font', **font)
```

# The FFT, aliasing, and filtering

Welcome to lesson 3.  Here we'll learn about the Fast Fourier Transform, which we've been using all along.  We'll also learn about a numerical pathology of pseudospectral methods (known as *aliasing*) and one way to avoid it (known as *filtering* or *dealiasing*).

## The fast Fourier transform

We won't go into great detail regarding the FFT algorithm, since there is already an excellent explanation of the Fast Fourier Transform in Jupyter Notebook form available on the web:

- [Understanding the FFT Algorithm](https://jakevdp.github.io/blog/2013/08/28/understanding-the-fft/) by Jake Vanderplas

Suffice it to say that the FFT is a fast algorithm for computing the discrete Fourier transform (DFT):

$$
\hat{u}_\xi = \sum_{j=0}^{m-1} u_j e^{-2\pi i \xi j/m}
$$

or its inverse.  The DFT, as we know, is linear and can be computed by multiplying $u$ by a certain $m\times m$  dense matrix $F$.  Multiplication by a dense matrix requeires ${\mathcal O}(m^2)$ operations.
The FFT is a shortcut to compute that matrix-vector product in just ${\mathcal O}(m \log m)$ operations by taking advantage of the special structure of $F$.

This is very important for pseudospectral methods, since most of the computational work occurs in computing the Fourier transform and its inverse.  It's also important that we make use of a compiled version of the FFT, since native Python code is relatively slow.  The `np.fft` function is an interface to a compiled FFT library.

### Ordering of wavenumbers in FFT output
The vector returned by `np.fft` contains the Fourier coefficients of $u$ corresponding to the wavenumbers

$$
\frac{2\pi}{L} \{-m/2, -m/2 + 1, \dots, m/2 - 1\}.
$$

However, for computational efficiency the output vector does not use the natural ordering above.  The ordering it uses can be obtained with the following command.

```python
m = 16
L = 2*np.pi

xi=np.fft.fftfreq(m)*m/(L/(2*np.pi))
print(xi)
```

As you can see, the return vector starts with the nonnegative wavenumbers, followed by the negative wavenumbers.  It may seem strange to you that the range of wavenumbers returned is not symmetric; in the case above, it includes $-8$ but not $+8$.  This apparent asymmetry can be explained once one understands the phenomenon known as *aliasing*.

## Aliasing

A numerical grid has a limited resolution.  If you try to represent a rapidly-oscillating function with relatively few grid points, you will observe an effect known as **aliasing**.  This naturally limits the range of frequencies that can be modelled on a given grid.  It can also lead to instabilities in pseudospectral simulations, when generation of high frequencies leads to buildup of lower-frequency energy due to aliasing.

The code below plots a sine wave of a given frequency, along with its representation on a grid with $m$ points.  Try changing $p$ and notice how for $m<2p$ the function looks like a lower-frequency mode.

```python
from ipywidgets import widgets
from ipywidgets import interact, interactive

def plot_sine(wavenumber=4,grid_points=12,plot_sine=True):
    "Plot sin(2*pi*p), sampled at m equispaced points."
    x  = np.linspace(0,1,grid_points+1); # grid
    xf = np.linspace(0,1,1000) # fine grid
    y  = np.sin(wavenumber*np.pi*x)
    yf = np.sin(wavenumber*np.pi*xf)
    fig = plt.figure(figsize = (8, 6));
    ax = fig.add_subplot(1,1,1);
    if plot_sine:
        ax.plot(xf, yf, 'r-', linewidth=2);
    ax.plot(x,  y,  'o-', lw=2)

interact(plot_sine, wavenumber=(-30,30,1),
         grid_points=(10, 16, 1));
```

### Exercise

Try to answer the questions below with pencil and paper; then check them by modifying the code above.

1. For a given number of grid points $m$, which wavenumbers $p$ will be aliased to the $p=0$ mode?  Which will be aliased to $p=1$?  Can you explain why?
2. What is the highest frequency mode that can be represented on a given grid?

After completing the exercise, explain why the sequence of wavenumbers given by `np.fft.fftfreq` is not, in fact, asymmetric after all.

## Aliasing as a source of numerical instability

As we have seen, aliasing means that wavenumbers of magnitude greater than $\pi m/L$ are incorrectly represented as lower wavenumbers on a grid with $m$ points.  This suggests that we shouldn't allow larger wavenumbers in our numerical solution.  For linear problems, this simply means that we should represent the initial condition by a truncated Fourier series containing modes with wavenumbers less than $\pi m/L$.  This happens naturally when we sample the function at the grid points.  As we evolve in time, higher frequencies are not generated due to the linearity of the problem.

Nonlinear problems are a different story.  Let's consider what happens when we have a quadratic term like $u^2$, as in Burgers' equation.  In general, if the grid function $u$ contains wavenumbers up to $\pi m/L$, then $u^2$ contains frequencies up to $2 \pi m/L$.  So each time we compute this term, we generate high frequencies that get aliased back to lower frequencies on our grid.  Clearly this has nothing to do with the correct mathematical solution and will lead to errors.  Even worse, this aliasing effect can, as it is repeated at every step, lead to an instability that causes the numerical solution to blow up.

$$
\newcommand{\F}{\mathcal F}
\newcommand{\Finv}{{\mathcal F}^{-1}}
$$
## An illustration of aliasing instability: the Korteweg-de Vries equation

To see aliasing in practice, we'll consider the KdV equation, which describes certain kinds of water waves:

$$
u_t = -u u_x - u_{xxx}
$$

A natural pseudospectral discretization is obtained if we compute the spatial derivatives via

\begin{align}
u_x & = \Finv(i\xi \F(u)) \\
u_{xxx} & = \Finv(-i\xi^3 \F(u)).
\end{align}
This gives
$$
U'(t) = -D[U] \Finv(i\xi \F(U)) - \Finv(-i\xi^3 \F(U)).
$$

This is identical to our discretization of Burgers' equation, except that now we have a third-derivative term.  In Fourier space, the third derivative gives a purely imaginary factor, which -- like the first derivative -- causes the solution to travel over time.  Unlike the first derivative, the third derivative term causes different wavenumber modes to travel at different speeds; this is referred to as *dispersion*.

The largest-magnitude eigenvalues of the Jacobian for this semi-discretization are related to the 3rd-derivative term.  If we consider only that term, the eigenvalues are

$$-i \xi^3$$

where $\xi$ lies in the range $(-m/2, m/2)$.  So we need the time step to satisfy $\Delta t (m/2)^3 \in S$, where $S$ is the region of absolute stability of a given time integrator.

For this example we'll use a 3rd-order Runge-Kutta method:

```python
def rk3(u,xi,rhs):
    y2 = u + dt*rhs(u,xi)
    y3 = 0.75*u + 0.25*(y2 + dt*rhs(y2,xi))
    u_new = 1./3 * u + 2./3 * (y3 + dt*rhs(y3,xi))
    return u_new
```

Let's check the size of the imaginary axis interval contained in this method's absolute stability region:

```python
from nodepy import rk
ssp33 = rk.loadRKM('SSP33')
print(ssp33.imaginary_stability_interval())
```

Now we'll go ahead and implement our solution, making sure to set the time step according to the condition above.

```python
def rhs(u, xi, equation='KdV'):
    uhat = np.fft.fft(u)
    if equation == 'Burgers':
        return -u*np.real(np.fft.ifft(1j*xi*uhat)) + np.real(np.fft.ifft(-xi**2*uhat))
    elif equation == 'KdV':
        return -u*np.real(np.fft.ifft(1j*xi*uhat)) - np.real(np.fft.ifft(-1j*xi**3*uhat))

# Grid
m = 256
L = 2*np.pi
x = np.arange(-m/2,m/2)*(L/m)
xi = np.fft.fftfreq(m)*m*2*np.pi/L

dt = 1.73/((m/2)**3)

A = 25; B = 16;

#u = 3*A**2/np.cosh(0.5*(A*(x+2.)))**2 + 3*B**2/np.cosh(0.5*(B*(x+1)))**2
#tmax = 0.006

# Try this one first:
u = 1500*np.exp(-10*(x+2)**2)
tmax = 0.005

uhat2 = np.abs(np.fft.fft(u))

num_plots = 50
nplt = np.floor((tmax/num_plots)/dt)
nmax = int(round(tmax/dt))

fig = plt.figure(figsize=(12,8))
axes = fig.add_subplot(211)
axes2 = fig.add_subplot(212)
line, = axes.plot(x,u,lw=3)
line2, = axes2.semilogy(xi,uhat2)
xi_max = np.max(np.abs(xi))
axes2.semilogy([xi_max/2.,xi_max/2.],[1.e-3,25000],'--r')
axes2.semilogy([-xi_max/2.,-xi_max/2.],[1.e-3,25000],'--r')

frames = [u.copy()]
tt = [0]
uuhat = [uhat2]

for n in range(1,nmax+1):
    u_new = rk3(u,xi,rhs)

    u = u_new.copy()
    t = n*dt
    # Plotting
    if np.mod(n,nplt) == 0:
        frames.append(u.copy())
        tt.append(t)
        uhat2 = np.abs(np.fft.fft(u))
        uuhat.append(uhat2)

def plot_frame(i):
    line.set_data(x,frames[i])
    line2.set_data(np.sort(xi),uuhat[i][np.argsort(xi)])
    axes.set_title('t= %.2e' % tt[i])
    axes.set_xlim((-np.pi,np.pi))
    axes.set_ylim((-100,3000))

anim = matplotlib.animation.FuncAnimation(fig, plot_frame,
                                   frames=len(frames), interval=100)

HTML(anim.to_jshtml())
```

In the output, we're plotting the solution (top plot) and its Fourier transform (bottom plot).  There are a lot of interesting things to say about the solution, but for now let's focus on the Fourier transform.  Notice how the wavenumbers present in the solution remain in the lower half of those representable on the grid (this region is delimited by the dashed red lines).  Because of this, no aliasing occurs.

Now change the code above to use only $m=128$ grid points.  What happens?

## Explanation

Here we will give a somewhat simplified explanation of the blow-up just observed.  First, this blowup has nothing to do with the absoute stability condition -- when we change $m$, the time step is automatically changed in a way that will ensure absolute stability.  If you're not convinced, try taking the time step even smaller; you will still observe the blowup.

By taking $m=128$, we cut by half the wavenumbers that can be represented on the grid.  As you can see from the plots, this means that some of the wavenumbers present in the initial data are in the upper half of the representable range (i.e., outside the dashed red lines).  That means that the highest wavenumbers generated by the quadratic term will be aliased -- and they will be aliased back into that upper-half range.  This leads to a gradual accumulation of high-wavenumber modes, easily visible in both plots.  Eventually the high-wavenumber modes dominate the numerical solution and lead to blowup.

For a detailed discussion of aliasing instabilities, see Chapter 11 of John Boyd's "Chebyshev and Fourier Spectral Methods".

## Filtering

How can we avoid aliasing instability?  The proper approach is to ensure that the solution is well resolved, so that the instability never appears.  However, this may entail a very substantial computational cost.  One way to ensure stability even if the solution is underresolved is by *filtering*, which is also known as *dealiasing*.  In general it is unwise to rely on filtering, since it can mask the fact that the solution is not resolved (and hence not accurate).  But understanding filtering can give a bit more insight into aliasing instability itself.

At the most basic level, filtering means removing the modes that lead to aliasing.  This can be done by damping the high wavenumbers or simply zeroing them when computing the $(u^2)_x$ term.  The obvious approach would be to filter the upper half of all wavenumbers, but this is overkill.  In fact, it is sufficient to filter only the uppermost third.  To see why, notice that the aliased modes resulting from the lower two-thirds will appear in the uppermost third of the range of modes, and so will be filtered at the next step.

A simple 2/3 filter is implemented in the code below.

```python
def rhs(u, xi, filtr, equation='KdV'):
    uhat = np.fft.fft(u)
    if equation == 'Burgers':
        return -u*np.real(np.fft.ifft(1j*xi*uhat)) \
                + np.real(np.fft.ifft(-xi**2*uhat))
    elif equation == 'KdV':
        return -u*np.real(np.fft.ifft(1j*xi*uhat*filtr)) \
                - np.real(np.fft.ifft(-1j*xi**3*uhat))

def rk3(u,xi,rhs,filtr):
    y2 = u + dt*rhs(u,xi,filtr)
    y3 = 0.75*u + 0.25*(y2 + dt*rhs(y2,xi,filtr))
    u_new = 1./3 * u + 2./3 * (y3 + dt*rhs(y3,xi,filtr))
    return u_new

# Grid
m = 128
L = 2*np.pi
x = np.arange(-m/2,m/2)*(L/m)
xi = np.fft.fftfreq(m)*m*2*np.pi/L
filtr = np.ones_like(xi)
xi_max = np.max(np.abs(xi))
filtr[np.where(np.abs(xi)>xi_max*1./2)] = 0.

dt = 1.73/((m/2)**3)

A = 25; B = 16;
u = 3*A**2/np.cosh(0.5*(A*(x+2.)))**2 + 3*B**2/np.cosh(0.5*(B*(x+1)))**2
tmax = 0.006

uhat2 = np.abs(np.fft.fft(u))

num_plots = 50
nplt = np.floor((tmax/num_plots)/dt)
nmax = int(round(tmax/dt))

fig = plt.figure(figsize=(12,8))
axes = fig.add_subplot(211)
axes2 = fig.add_subplot(212)
line, = axes.plot(x,u,lw=3)
line2, = axes2.semilogy(xi,uhat2)
axes2.semilogy([xi_max/2.,xi_max/2.],[1.e-3,25000],'--r')
axes2.semilogy([-xi_max/2.,-xi_max/2.],[1.e-3,25000],'--r')

frames = [u.copy()]
tt = [0]
uuhat = [uhat2]

for n in range(1,nmax+1):
    u_new = rk3(u,xi,rhs,filtr)

    u = u_new.copy()
    t = n*dt
    # Plotting
    if np.mod(n,nplt) == 0:
        frames.append(u.copy())
        tt.append(t)
        uhat2 = np.abs(np.fft.fft(u))
        uuhat.append(uhat2)

def plot_frame(i):
    line.set_data(x,frames[i])
    line2.set_data(np.sort(xi),uuhat[i][np.argsort(xi)])
    axes.set_title('t= %.2e' % tt[i])
    axes.set_xlim((-np.pi,np.pi))
    axes.set_ylim((-100,3000))

anim = matplotlib.animation.FuncAnimation(fig, plot_frame,
                                   frames=len(frames), interval=20)

HTML(anim.to_jshtml())
```

Notice how the solution remains stable, but small wiggles appear throughout the domain.  These are a hint that something is not sufficiently resolved.


## Tutorial on how to combine different Fields into a `NestedField` object

In some applications, you may have access to different fields that each cover only part of the region of interest. Then, you would like to combine them all together. You may also have a field covering the entire region and another one only covering part of it, but with a higher resolution. The set of those fields form what we call nested fields.

It is possible to combine all those fields with kernels, either with different if/else statements depending on particle position, or using recovery kernels (if only two levels of nested fields).

However, an easier way to work with nested fields in Parcels is to combine all those fields into one `NestedField` object. The Parcels code will then try to successively interpolate the different fields.

For each Particle, the algorithm is the following:

1. Interpolate the particle onto the first `Field` in the `NestedFields` list.

2. If the interpolation succeeds or if an error other than `ErrorOutOfBounds` is thrown, the function is stopped.

3. If an `ErrorOutOfBounds` is thrown, try step 1) again with the next `Field` in the `NestedFields` list

4. If interpolation on the last `Field` in the `NestedFields` list also returns an `ErrorOutOfBounds`, then the Particle is flagged as OutOfBounds.

This algorithm means that **the order of the fields in the `NestedField` matters**. In particular, the smallest/finest resolution fields have to be listed _before_ the larger/coarser resolution fields.

This tutorial shows how to use these `NestedField` with a very idealised example.

```python
%matplotlib inline
from parcels import Field, NestedField, FieldSet, ParticleSet, JITParticle, plotTrajectoriesFile, AdvectionRK4
import numpy as np
```

First define a zonal and meridional velocity field defined on a high resolution (dx = 100m) 2kmx2km grid with a flat mesh. The zonal velocity is uniform and 1 m/s, and the meridional velocity is equal to 0.5 * cos(lon / 200 * pi / 2) m/s.

```python
dim = 21
lon = np.linspace(0., 2e3, dim, dtype=np.float32)
lat = np.linspace(0., 2e3, dim, dtype=np.float32)
lon_g, lat_g = np.meshgrid(lon, lat)
V1_data = np.cos(lon_g / 200 * np.pi/2)
U1 = Field('U1', np.ones((dim, dim), dtype=np.float32), lon=lon, lat=lat)
V1 = Field('V1', V1_data, grid=U1.grid)
```

Now define the same velocity field on a low resolution (dx = 2km) 20kmx4km grid.

```python
xdim = 11
ydim = 3
lon = np.linspace(-2e3, 18e3, xdim, dtype=np.float32)
lat = np.linspace(-1e3, 3e3, ydim, dtype=np.float32)
lon_g, lat_g = np.meshgrid(lon, lat)
V2_data = np.cos(lon_g / 200 * np.pi/2)
U2 = Field('U2', np.ones((ydim, xdim), dtype=np.float32), lon=lon, lat=lat)
V2 = Field('V2', V2_data, grid=U2.grid)
```

We now combine those fields into a `NestedField` and create the fieldset

```python
U = NestedField('U', [U1, U2])
V = NestedField('V', [V1, V2])
fieldset = FieldSet(U, V)
```

```python
pset = ParticleSet(fieldset, pclass=JITParticle, lon=[0], lat=[1000])
output_file = pset.ParticleFile(name='NestedFieldParticle.nc', outputdt=50)
pset.execute(AdvectionRK4, runtime=14000, dt=10, output_file=output_file)
output_file.export()  # export the trajectory data to a netcdf file

plt = plotTrajectoriesFile('NestedFieldParticle.nc', show_plt=False)
plt.plot([0,2e3,2e3,0,0],[0,0,2e3,2e3,0], c='orange')
plt.plot([-2e3,18e3,18e3,-2e3,-2e3],[-1e3,-1e3,3e3,3e3,-1e3], c='green');
```

As we observe, there is a change of dynamic at lon=2000, which corresponds to the change of grid.

The analytical solution to the problem:
\begin{align}
dx/dt &= 1;\\
dy/dt &= \cos(x \pi/400);\\
\text{with  } x(0) &= 0, y(0) = 1000
\end{align}

is
\begin{align}
x(t) &= t;\\
y(t) &= 1000 + 400/\pi \sin(t \pi / 400)
\end{align}
which is captured by the High Resolution field (orange area) but not the Low Resolution one (green area).

### Keep track of the field interpolated

For different reasons, you may want to keep track of the field you have interpolated. You can do that easily by creating another field that share the grid with original fields.
Watch out that this operation has a cost of a full interpolation operation.

```python
F1 = Field('F1', np.ones((U1.grid.ydim, U1.grid.xdim), dtype=np.float32), grid=U1.grid)
F2 = Field('F2', 2*np.ones((U2.grid.ydim, U2.grid.xdim), dtype=np.float32), grid=U2.grid)
F = NestedField('F', [F1, F2])
fieldset.add_field(F)
```

```python
from parcels import Variable

def SampleNestedFieldIndex(particle, fieldset, time):
    particle.f = fieldset.F[time, particle.depth, particle.lat, particle.lon]

class SampleParticle(JITParticle):
    f = Variable('f', dtype=np.int32)

pset = ParticleSet(fieldset, pclass= SampleParticle, lon=[1000], lat=[500])
pset.execute(SampleNestedFieldIndex, runtime=0, dt=0)
print('Particle (%g, %g) interpolates Field #%d' % (pset[0].lon, pset[0].lat, pset[0].f))

pset[0].lon = 10000
pset.execute(SampleNestedFieldIndex, runtime=0, dt=0)
print('Particle (%g, %g) interpolates Field #%d' % (pset[0].lon, pset[0].lat, pset[0].f))

```

    INFO: Compiled SampleParticleSampleNestedFieldIndex ==> /var/folders/h0/01fvrmn11qb62yjw7v1kn62r0000gq/T/parcels-503/57e7f8f8b9c55074a13384b46fceb0eb.so
    WARNING: dt or runtime are zero, or endtime is equal to Particle.time. The kernels will be executed once, without incrementing time

    Particle (1000, 500) interpolates Field #1
    Particle (10000, 500) interpolates Field #2

```python

```

```python

```


# Programming Exercise 5:
# Regularized Linear Regression and Bias vs Variance

## Introduction

In this exercise, you will implement regularized linear regression and use it to study models with different bias-variance properties. Before starting on the programming exercise, we strongly recommend watching the video lectures and completing the review questions for the associated topics.

All the information you need for solving this assignment is in this notebook, and all the code you will be implementing will take place within this notebook. The assignment can be promptly submitted to the coursera grader directly from this notebook (code and instructions are included below).

Before we begin with the exercises, we need to import all libraries required for this programming exercise. Throughout the course, we will be using [`numpy`](http://www.numpy.org/) for all arrays and matrix operations, [`matplotlib`](https://matplotlib.org/) for plotting, and [`scipy`](https://docs.scipy.org/doc/scipy/reference/) for scientific and numerical computation functions and tools. You can find instructions on how to install required libraries in the README file in the [github repository](https://github.com/dibgerge/ml-coursera-python-assignments).

```python
# used for manipulating directory paths
import os

# Scientific and vector computation for python
import numpy as np

# Plotting library
from matplotlib import pyplot

# Optimization module in scipy
from scipy import optimize

# will be used to load MATLAB mat datafile format
from scipy.io import loadmat

# library written for this exercise providing additional functions for assignment submission, and others
import utils

# define the submission/grader object for this exercise
grader = utils.Grader()

# tells matplotlib to embed plots within the notebook
%matplotlib inline
```

## Submission and Grading

After completing each part of the assignment, be sure to submit your solutions to the grader. The following is a breakdown of how each part of this exercise is scored.

| Section | Part                                             | Submitted Function                | Points |
| :-      |:-                                                |:-                                 | :-:    |
| 1       | [Regularized Linear Regression Cost Function](#section1)      | [`linearRegCostFunction`](#linearRegCostFunction) |  25    |
| 2       | [Regularized Linear Regression Gradient](#section2)           | [`linearRegCostFunction`](#linearRegCostFunction) |25      |
| 3       | [Learning Curve](#section3)                                   | [`learningCurve`](#func2)         | 20     |
| 4       | [Polynomial Feature Mapping](#section4)                       | [`polyFeatures`](#polyFeatures)          | 10     |
| 5       | [Cross Validation Curve](#section5)                           | [`validationCurve`](#validationCurve)       | 20     |
|         | Total Points                                     |                                   |100     |

You are allowed to submit your solutions multiple times, and we will take only the highest score into consideration.

At the end of each section in this notebook, we have a cell which contains code for submitting the solutions thus far to the grader. Execute the cell to see your score up to the current section. For all your work to be submitted properly, you must execute those cells at least once.

## 1 Regularized Linear Regression

In the first half of the exercise, you will implement regularized linear regression to predict the amount of water flowing out of a dam using the change of water level in a reservoir. In the next half, you will go through some diagnostics of debugging learning algorithms and examine the effects of bias v.s.
variance.

### 1.1 Visualizing the dataset

We will begin by visualizing the dataset containing historical records on the change in the water level, $x$, and the amount of water flowing out of the dam, $y$. This dataset is divided into three parts:

- A **training** set that your model will learn on: `X`, `y`
- A **cross validation** set for determining the regularization parameter: `Xval`, `yval`
- A **test** set for evaluating performance. These are “unseen” examples which your model did not see during training: `Xtest`, `ytest`

Run the next cell to plot the training data. In the following parts, you will implement linear regression and use that to fit a straight line to the data and plot learning curves. Following that, you will implement polynomial regression to find a better fit to the data.

```python
# Load from ex5data1.mat, where all variables will be store in a dictionary
data = loadmat(os.path.join('Data', 'ex5data1.mat'))

# Extract train, test, validation data from dictionary
# and also convert y's form 2-D matrix (MATLAB format) to a numpy vector
X, y = data['X'], data['y'][:, 0]
Xtest, ytest = data['Xtest'], data['ytest'][:, 0]
Xval, yval = data['Xval'], data['yval'][:, 0]

# m = Number of examples
m = y.size

# Plot training data
pyplot.plot(X, y, 'ro', ms=10, mec='k', mew=1)
pyplot.xlabel('Change in water level (x)')
pyplot.ylabel('Water flowing out of the dam (y)');
```

### 1.2 Regularized linear regression cost function

Recall that regularized linear regression has the following cost function:

$$ J(\theta) = \frac{1}{2m} \left( \sum_{i=1}^m \left( h_\theta\left( x^{(i)} \right) - y^{(i)} \right)^2 \right) + \frac{\lambda}{2m} \left( \sum_{j=1}^n \theta_j^2 \right)$$

where $\lambda$ is a regularization parameter which controls the degree of regularization (thus, help preventing overfitting). The regularization term puts a penalty on the overall cost J. As the magnitudes of the model parameters $\theta_j$ increase, the penalty increases as well. Note that you should not regularize
the $\theta_0$ term.

You should now complete the code in the function `linearRegCostFunction` in the next cell. Your task is to calculate the regularized linear regression cost function. If possible, try to vectorize your code and avoid writing loops.

```python
def linearRegCostFunction(X, y, theta, lambda_=0.0):
    """
    Compute cost and gradient for regularized linear regression
    with multiple variables. Computes the cost of using theta as
    the parameter for linear regression to fit the data points in X and y.

    Parameters
    ----------
    X : array_like
        The dataset. Matrix with shape (m x n + 1) where m is the
        total number of examples, and n is the number of features
        before adding the bias term.

    y : array_like
        The functions values at each datapoint. A vector of
        shape (m, ).

    theta : array_like
        The parameters for linear regression. A vector of shape (n+1,).

    lambda_ : float, optional
        The regularization parameter.

    Returns
    -------
    J : float
        The computed cost function.

    grad : array_like
        The value of the cost function gradient w.r.t theta.
        A vector of shape (n+1, ).

    Instructions
    ------------
    Compute the cost and gradient of regularized linear regression for
    a particular choice of theta.
    You should set J to the cost and grad to the gradient.
    """
    # Initialize some useful values
    m = y.size # number of training examples

    # You need to return the following variables correctly
    J = 0
    grad = np.zeros(theta.shape)

    # ====================== YOUR CODE HERE ======================
    J = 1/(2*m) * np.sum(np.power((np.dot(X, theta) - y),2)) + lambda_/(2*m)*np.sum(theta[1:]**2)
    grad = 1/m * np.dot(X.T, (np.dot(X, theta) - y))
    grad[1:] = grad[1:] + lambda_/m*theta[1:]
    # ============================================================
    return J, grad
```

When you are finished, the next cell will run your cost function using `theta` initialized at `[1, 1]`. You should expect to see an output of 303.993.

```python
theta = np.array([1, 1])
J, _ = linearRegCostFunction(np.concatenate([np.ones((m, 1)), X], axis=1), y, theta, 1)

print('Cost at theta = [1, 1]:\t   %f ' % J)
print('This value should be about 303.993192)\n' % J)
```

    Cost at theta = [1, 1]:	   303.993192
    This value should be about 303.993192)

After completing a part of the exercise, you can submit your solutions for grading by first adding the function you modified to the submission object, and then sending your function to Coursera for grading.

The submission script will prompt you for your login e-mail and submission token. You can obtain a submission token from the web page for the assignment. You are allowed to submit your solutions multiple times, and we will take only the highest score into consideration.

*Execute the following cell to grade your solution to the first part of this exercise.*

```python
grader[1] = linearRegCostFunction
grader.grade()
```

    Submitting Solutions | Programming Exercise regularized-linear-regression-and-bias-variance

                                      Part Name |     Score | Feedback
                                      --------- |     ----- | --------
    Regularized Linear Regression Cost Function |  25 /  25 | Nice work!
         Regularized Linear Regression Gradient |   0 /  25 |
                                 Learning Curve |   0 /  20 |
                     Polynomial Feature Mapping |   0 /  10 |
                               Validation Curve |   0 /  20 |
                                      --------------------------------
                                                |  25 / 100 |

### 1.3 Regularized linear regression gradient

Correspondingly, the partial derivative of the cost function for regularized linear regression is defined as:

$$
\begin{align}
& \frac{\partial J(\theta)}{\partial \theta_0} = \frac{1}{m} \sum_{i=1}^m \left( h_\theta \left(x^{(i)} \right) - y^{(i)} \right) x_j^{(i)} & \qquad \text{for } j = 0 \\
& \frac{\partial J(\theta)}{\partial \theta_j} = \left( \frac{1}{m} \sum_{i=1}^m \left( h_\theta \left( x^{(i)} \right) - y^{(i)} \right) x_j^{(i)} \right) + \frac{\lambda}{m} \theta_j & \qquad \text{for } j \ge 1
\end{align}
$$

In the function [`linearRegCostFunction`](#linearRegCostFunction) above, add code to calculate the gradient, returning it in the variable `grad`. Do not forget to re-execute the cell containing this function to update the function's definition.

When you are finished, use the next cell to  run your gradient function using theta initialized at `[1, 1]`. You should expect to see a gradient of `[-15.30, 598.250]`.

```python
theta = np.array([1, 1])
J, grad = linearRegCostFunction(np.concatenate([np.ones((m, 1)), X], axis=1), y, theta, 1)

print('Gradient at theta = [1, 1]:  [{:.6f}, {:.6f}] '.format(*grad))
print(' (this value should be about [-15.303016, 598.250744])\n')
```

    Gradient at theta = [1, 1]:  [-15.303016, 598.250744]
     (this value should be about [-15.303016, 598.250744])

*You should now submit your solutions.*

```python
grader[2] = linearRegCostFunction
grader.grade()
```

    Submitting Solutions | Programming Exercise regularized-linear-regression-and-bias-variance

                                      Part Name |     Score | Feedback
                                      --------- |     ----- | --------
    Regularized Linear Regression Cost Function |  25 /  25 | Nice work!
         Regularized Linear Regression Gradient |  25 /  25 | Nice work!
                                 Learning Curve |   0 /  20 |
                     Polynomial Feature Mapping |   0 /  10 |
                               Validation Curve |   0 /  20 |
                                      --------------------------------
                                                |  50 / 100 |

### Fitting linear regression

Once your cost function and gradient are working correctly, the next cell will run the code in `trainLinearReg` (found in the module `utils.py`) to compute the optimal values of $\theta$. This training function uses `scipy`'s optimization module to minimize the cost function.

In this part, we set regularization parameter $\lambda$ to zero. Because our current implementation of linear regression is trying to fit a 2-dimensional $\theta$, regularization will not be incredibly helpful for a $\theta$ of such low dimension. In the later parts of the exercise, you will be using polynomial regression with regularization.

Finally, the code in the next cell should also plot the best fit line, which should look like the figure below.

The best fit line tells us that the model is not a good fit to the data because the data has a non-linear pattern. While visualizing the best fit as shown is one possible way to debug your learning algorithm, it is not always easy to visualize the data and model. In the next section, you will implement a function to generate learning curves that can help you debug your learning algorithm even if it is not easy to visualize the
data.

```python
# add a columns of ones for the y-intercept
X_aug = np.concatenate([np.ones((m, 1)), X], axis=1)
theta = utils.trainLinearReg(linearRegCostFunction, X_aug, y, lambda_=0)

#  Plot fit over the data
pyplot.plot(X, y, 'ro', ms=10, mec='k', mew=1.5)
pyplot.xlabel('Change in water level (x)')
pyplot.ylabel('Water flowing out of the dam (y)')
pyplot.plot(X, np.dot(X_aug, theta), '--', lw=2)
```

## 2 Bias-variance

An important concept in machine learning is the bias-variance tradeoff. Models with high bias are not complex enough for the data and tend to underfit, while models with high variance overfit to the training data.

In this part of the exercise, you will plot training and test errors on a learning curve to diagnose bias-variance problems.

### 2.1 Learning Curves

You will now implement code to generate the learning curves that will be useful in debugging learning algorithms. Recall that a learning curve plots training and cross validation error as a function of training set size. Your job is to fill in the function `learningCurve` in the next cell, so that it returns a vector of errors for the training set and cross validation set.

To plot the learning curve, we need a training and cross validation set error for different training set sizes. To obtain different training set sizes, you should use different subsets of the original training set `X`. Specifically, for a training set size of $i$, you should use the first $i$ examples (i.e., `X[:i, :]`
and `y[:i]`).

You can use the `trainLinearReg` function (by calling `utils.trainLinearReg(...)`) to find the $\theta$ parameters. Note that the `lambda_` is passed as a parameter to the `learningCurve` function.
After learning the $\theta$ parameters, you should compute the error on the training and cross validation sets. Recall that the training error for a dataset is defined as

$$ J_{\text{train}} = \frac{1}{2m} \left[ \sum_{i=1}^m \left(h_\theta \left( x^{(i)} \right) - y^{(i)} \right)^2 \right] $$

In particular, note that the training error does not include the regularization term. One way to compute the training error is to use your existing cost function and set $\lambda$ to 0 only when using it to compute the training error and cross validation error. When you are computing the training set error, make sure you compute it on the training subset (i.e., `X[:n,:]` and `y[:n]`) instead of the entire training set. However, for the cross validation error, you should compute it over the entire cross validation set. You should store
the computed errors in the vectors error train and error val.

```python
def learningCurve(X, y, Xval, yval, lambda_=0):
    """
    Generates the train and cross validation set errors needed to plot a learning curve
    returns the train and cross validation set errors for a learning curve.

    In this function, you will compute the train and test errors for
    dataset sizes from 1 up to m. In practice, when working with larger
    datasets, you might want to do this in larger intervals.

    Parameters
    ----------
    X : array_like
        The training dataset. Matrix with shape (m x n + 1) where m is the
        total number of examples, and n is the number of features
        before adding the bias term.

    y : array_like
        The functions values at each training datapoint. A vector of
        shape (m, ).

    Xval : array_like
        The validation dataset. Matrix with shape (m_val x n + 1) where m is the
        total number of examples, and n is the number of features
        before adding the bias term.

    yval : array_like
        The functions values at each validation datapoint. A vector of
        shape (m_val, ).

    lambda_ : float, optional
        The regularization parameter.

    Returns
    -------
    error_train : array_like
        A vector of shape m. error_train[i] contains the training error for
        i examples.
    error_val : array_like
        A vecotr of shape m. error_val[i] contains the validation error for
        i training examples.

    Instructions
    ------------
    Fill in this function to return training errors in error_train and the
    cross validation errors in error_val. i.e., error_train[i] and
    error_val[i] should give you the errors obtained after training on i examples.

    Notes
    -----
    - You should evaluate the training error on the first i training
      examples (i.e., X[:i, :] and y[:i]).

      For the cross-validation error, you should instead evaluate on
      the _entire_ cross validation set (Xval and yval).

    - If you are using your cost function (linearRegCostFunction) to compute
      the training and cross validation error, you should call the function with
      the lambda argument set to 0. Do note that you will still need to use
      lambda when running the training to obtain the theta parameters.

    Hint
    ----
    You can loop over the examples with the following:

           for i in range(1, m+1):
               # Compute train/cross validation errors using training examples
               # X[:i, :] and y[:i], storing the result in
               # error_train[i-1] and error_val[i-1]
               ....
    """
    # Number of training examples
    m = y.size

    # You need to return these values correctly
    error_train = np.zeros(m)
    error_val   = np.zeros(m)

    # ====================== YOUR CODE HERE ======================

    for i in range(1, m+1):
        theta = utils.trainLinearReg(linearRegCostFunction, X[:i], y[:i], lambda_)
        error_train[i-1], _ = linearRegCostFunction(X[:i], y[:i], theta)
        error_val[i-1], _ = linearRegCostFunction(Xval, yval, theta)

    # =============================================================
    return error_train, error_val
```

When you are finished implementing the function `learningCurve`, executing the next cell prints the learning curves and produce a plot similar to the figure below.

In the learning curve figure, you can observe that both the train error and cross validation error are high when the number of training examples is increased. This reflects a high bias problem in the model - the linear regression model is too simple and is unable to fit our dataset well. In the next section, you will implement polynomial regression to fit a better model for this dataset.

```python
X_aug = np.concatenate([np.ones((m, 1)), X], axis=1)
Xval_aug = np.concatenate([np.ones((yval.size, 1)), Xval], axis=1)
error_train, error_val = learningCurve(X_aug, y, Xval_aug, yval, lambda_=0)

pyplot.plot(np.arange(1, m+1), error_train, np.arange(1, m+1), error_val, lw=2)
pyplot.title('Learning curve for linear regression')
pyplot.legend(['Train', 'Cross Validation'])
pyplot.xlabel('Number of training examples')
pyplot.ylabel('Error')
pyplot.axis([0, 13, 0, 150])

print('# Training Examples\tTrain Error\tCross Validation Error')
for i in range(m):
    print('  \t%d\t\t%f\t%f' % (i+1, error_train[i], error_val[i]))
```

*You should now submit your solutions.*

```python
grader[3] = learningCurve
grader.grade()
```

    Submitting Solutions | Programming Exercise regularized-linear-regression-and-bias-variance

                                      Part Name |     Score | Feedback
                                      --------- |     ----- | --------
    Regularized Linear Regression Cost Function |  25 /  25 | Nice work!
         Regularized Linear Regression Gradient |  25 /  25 | Nice work!
                                 Learning Curve |  20 /  20 | Nice work!
                     Polynomial Feature Mapping |   0 /  10 |
                               Validation Curve |   0 /  20 |
                                      --------------------------------
                                                |  70 / 100 |

## 3 Polynomial regression

The problem with our linear model was that it was too simple for the data
and resulted in underfitting (high bias). In this part of the exercise, you will address this problem by adding more features. For polynomial regression, our hypothesis has the form:

$$
\begin{align}
h_\theta(x)  &= \theta_0 + \theta_1 \times (\text{waterLevel}) + \theta_2 \times (\text{waterLevel})^2 + \cdots + \theta_p \times (\text{waterLevel})^p \\
& = \theta_0 + \theta_1 x_1 + \theta_2 x_2 + \cdots + \theta_p x_p
\end{align}
$$

Notice that by defining $x_1 = (\text{waterLevel})$, $x_2 = (\text{waterLevel})^2$ , $\cdots$, $x_p =
(\text{waterLevel})^p$, we obtain a linear regression model where the features are the various powers of the original value (waterLevel).

Now, you will add more features using the higher powers of the existing feature $x$ in the dataset. Your task in this part is to complete the code in the function `polyFeatures` in the next cell. The function should map the original training set $X$ of size $m \times 1$ into its higher powers. Specifically, when a training set $X$ of size $m \times 1$ is passed into the function, the function should return a $m \times p$ matrix `X_poly`, where column 1 holds the original values of X, column 2 holds the values of $X^2$, column 3 holds the values of $X^3$, and so on. Note that you don’t have to account for the zero-eth power in this function.

```python
def polyFeatures(X, p):
    """
    Maps X (1D vector) into the p-th power.

    Parameters
    ----------
    X : array_like
        A data vector of size m, where m is the number of examples.

    p : int
        The polynomial power to map the features.

    Returns
    -------
    X_poly : array_like
        A matrix of shape (m x p) where p is the polynomial
        power and m is the number of examples. That is:

        X_poly[i, :] = [X[i], X[i]**2, X[i]**3 ...  X[i]**p]

    Instructions
    ------------
    Given a vector X, return a matrix X_poly where the p-th column of
    X contains the values of X to the p-th power.
    """
    # You need to return the following variables correctly.
    X_poly = np.zeros((X.shape[0], p))

    # ====================== YOUR CODE HERE ======================

    for i in range(1,p+1):
        X_poly[:,i-1] = (X**i).flatten()

    # ============================================================
    return X_poly
```

Now you have a function that will map features to a higher dimension. The next cell will apply it to the training set, the test set, and the cross validation set.

```python
p = 8

# Map X onto Polynomial Features and Normalize
X_poly = polyFeatures(X, p)
X_poly, mu, sigma = utils.featureNormalize(X_poly)
X_poly = np.concatenate([np.ones((m, 1)), X_poly], axis=1)

# Map X_poly_test and normalize (using mu and sigma)
X_poly_test = polyFeatures(Xtest, p)
X_poly_test -= mu
X_poly_test /= sigma
X_poly_test = np.concatenate([np.ones((ytest.size, 1)), X_poly_test], axis=1)

# Map X_poly_val and normalize (using mu and sigma)
X_poly_val = polyFeatures(Xval, p)
X_poly_val -= mu
X_poly_val /= sigma
X_poly_val = np.concatenate([np.ones((yval.size, 1)), X_poly_val], axis=1)

print('Normalized Training Example 1:')
X_poly[0, :]
```

    Normalized Training Example 1:

    array([ 1.        , -0.36214078, -0.75508669,  0.18222588, -0.70618991,
            0.30661792, -0.59087767,  0.3445158 , -0.50848117])

*You should now submit your solutions.*

```python
grader[4] = polyFeatures
grader.grade()
```

    Submitting Solutions | Programming Exercise regularized-linear-regression-and-bias-variance

                                      Part Name |     Score | Feedback
                                      --------- |     ----- | --------
    Regularized Linear Regression Cost Function |  25 /  25 | Nice work!
         Regularized Linear Regression Gradient |  25 /  25 | Nice work!
                                 Learning Curve |  20 /  20 | Nice work!
                     Polynomial Feature Mapping |  10 /  10 | Nice work!
                               Validation Curve |   0 /  20 |
                                      --------------------------------
                                                |  80 / 100 |

## 3.1 Learning Polynomial Regression

After you have completed the function `polyFeatures`, we will proceed to train polynomial regression using your linear regression cost function.

Keep in mind that even though we have polynomial terms in our feature vector, we are still solving a linear regression optimization problem. The polynomial terms have simply turned into features that we can use for linear regression. We are using the same cost function and gradient that you wrote for the earlier part of this exercise.

For this part of the exercise, you will be using a polynomial of degree 8. It turns out that if we run the training directly on the projected data, will not work well as the features would be badly scaled (e.g., an example with $x = 40$ will now have a feature $x_8 = 40^8 = 6.5 \times 10^{12}$). Therefore, you will
need to use feature normalization.

Before learning the parameters $\theta$ for the polynomial regression, we first call `featureNormalize` and normalize the features of the training set, storing the mu, sigma parameters separately. We have already implemented this function for you (in `utils.py` module) and it is the same function from the first exercise.

After learning the parameters $\theta$, you should see two plots generated for polynomial regression with $\lambda = 0$, which should be similar to the ones here:

You should see that the polynomial fit is able to follow the datapoints very well, thus, obtaining a low training error. The figure on the right shows that the training error essentially stays zero for all numbers of training samples. However, the polynomial fit is very complex and even drops off at the extremes. This is an indicator that the polynomial regression model is overfitting the training data and will not generalize well.

To better understand the problems with the unregularized ($\lambda = 0$) model, you can see that the learning curve  shows the same effect where the training error is low, but the cross validation error is high. There is a gap between the training and cross validation errors, indicating a high variance problem.

```python
lambda_ = 0
theta = utils.trainLinearReg(linearRegCostFunction, X_poly, y,
                             lambda_=lambda_, maxiter=55)

# Plot training data and fit
pyplot.plot(X, y, 'ro', ms=10, mew=1.5, mec='k')

utils.plotFit(polyFeatures, np.min(X), np.max(X), mu, sigma, theta, p)

pyplot.xlabel('Change in water level (x)')
pyplot.ylabel('Water flowing out of the dam (y)')
pyplot.title('Polynomial Regression Fit (lambda = %f)' % lambda_)
pyplot.ylim([-20, 50])

pyplot.figure()
error_train, error_val = learningCurve(X_poly, y, X_poly_val, yval, lambda_)
pyplot.plot(np.arange(1, 1+m), error_train, np.arange(1, 1+m), error_val)

pyplot.title('Polynomial Regression Learning Curve (lambda = %f)' % lambda_)
pyplot.xlabel('Number of training examples')
pyplot.ylabel('Error')
pyplot.axis([0, 13, 0, 100])
pyplot.legend(['Train', 'Cross Validation'])

print('Polynomial Regression (lambda = %f)\n' % lambda_)
print('# Training Examples\tTrain Error\tCross Validation Error')
for i in range(m):
    print('  \t%d\t\t%f\t%f' % (i+1, error_train[i], error_val[i]))
```

One way to combat the overfitting (high-variance) problem is to add regularization to the model. In the next section, you will get to  try different $\lambda$ parameters to see how regularization can lead to a better model.

### 3.2 Optional (ungraded) exercise: Adjusting the regularization parameter

In this section, you will get to observe how the regularization parameter affects the bias-variance of regularized polynomial regression. You should now modify the the lambda parameter and try $\lambda = 1, 100$. For each of these values, the script should generate a polynomial fit to the data and also a learning curve.

For $\lambda = 1$, the generated plots should look like the the figure below. You should see a polynomial fit that follows the data trend well (left) and a learning curve (right) showing that both the cross validation and training error converge to a relatively low value. This shows the $\lambda = 1$ regularized polynomial regression model does not have the high-bias or high-variance problems. In effect, it achieves a good trade-off between bias and variance.

For $\lambda = 100$, you should see a polynomial fit (figure below) that does not follow the data well. In this case, there is too much regularization and the model is unable to fit the training data.

*You do not need to submit any solutions for this optional (ungraded) exercise.*

### 3.3 Selecting $\lambda$ using a cross validation set

From the previous parts of the exercise, you observed that the value of $\lambda$ can significantly affect the results of regularized polynomial regression on the training and cross validation set. In particular, a model without regularization ($\lambda = 0$) fits the training set well, but does not generalize. Conversely, a model with too much regularization ($\lambda = 100$) does not fit the training set and testing set well. A good choice of $\lambda$ (e.g., $\lambda = 1$) can provide a good fit to the data.

In this section, you will implement an automated method to select the $\lambda$ parameter. Concretely, you will use a cross validation set to evaluate how good each $\lambda$ value is. After selecting the best $\lambda$ value using the cross validation set, we can then evaluate the model on the test set to estimate
how well the model will perform on actual unseen data.

Your task is to complete the code in the function `validationCurve`. Specifically, you should should use the `utils.trainLinearReg` function to train the model using different values of $\lambda$ and compute the training error and cross validation error. You should try $\lambda$ in the following range: {0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10}.

```python
def validationCurve(X, y, Xval, yval):
    """
    Generate the train and validation errors needed to plot a validation
    curve that we can use to select lambda_.

    Parameters
    ----------
    X : array_like
        The training dataset. Matrix with shape (m x n) where m is the
        total number of training examples, and n is the number of features
        including any polynomial features.

    y : array_like
        The functions values at each training datapoint. A vector of
        shape (m, ).

    Xval : array_like
        The validation dataset. Matrix with shape (m_val x n) where m is the
        total number of validation examples, and n is the number of features
        including any polynomial features.

    yval : array_like
        The functions values at each validation datapoint. A vector of
        shape (m_val, ).

    Returns
    -------
    lambda_vec : list
        The values of the regularization parameters which were used in
        cross validation.

    error_train : list
        The training error computed at each value for the regularization
        parameter.

    error_val : list
        The validation error computed at each value for the regularization
        parameter.

    Instructions
    ------------
    Fill in this function to return training errors in `error_train` and
    the validation errors in `error_val`. The vector `lambda_vec` contains
    the different lambda parameters to use for each calculation of the
    errors, i.e, `error_train[i]`, and `error_val[i]` should give you the
    errors obtained after training with `lambda_ = lambda_vec[i]`.

    Note
    ----
    You can loop over lambda_vec with the following:

          for i in range(len(lambda_vec))
              lambda = lambda_vec[i]
              # Compute train / val errors when training linear
              # regression with regularization parameter lambda_
              # You should store the result in error_train[i]
              # and error_val[i]
              ....
    """
    # Selected values of lambda (you should not change this)
    lambda_vec = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]

    # You need to return these variables correctly.
    error_train = np.zeros(len(lambda_vec))
    error_val = np.zeros(len(lambda_vec))

    # ====================== YOUR CODE HERE ======================
    for i, lambda_ in enumerate(lambda_vec):
        theta = utils.trainLinearReg(linearRegCostFunction, X, y, lambda_)
        error_train[i], _ = linearRegCostFunction(X, y, theta)
        error_val[i], _ = linearRegCostFunction(Xval, yval, theta)

    # ============================================================
    return lambda_vec, error_train, error_val
```

After you have completed the code, the next cell will run your function and plot a cross validation curve of error v.s. $\lambda$ that allows you select which $\lambda$ parameter to use. You should see a plot similar to the figure below.

In this figure, we can see that the best value of $\lambda$ is around 3. Due to randomness
in the training and validation splits of the dataset, the cross validation error can sometimes be lower than the training error.

```python
lambda_vec, error_train, error_val = validationCurve(X_poly, y, X_poly_val, yval)

pyplot.plot(lambda_vec, error_train, '-o', lambda_vec, error_val, '-o', lw=2)
pyplot.legend(['Train', 'Cross Validation'])
pyplot.xlabel('lambda')
pyplot.ylabel('Error')

print('lambda\t\tTrain Error\tValidation Error')
for i in range(len(lambda_vec)):
    print(' %f\t%f\t%f' % (lambda_vec[i], error_train[i], error_val[i]))
```

*You should now submit your solutions.*

```python
grader[5] = validationCurve
grader.grade()
```

    Submitting Solutions | Programming Exercise regularized-linear-regression-and-bias-variance

                                      Part Name |     Score | Feedback
                                      --------- |     ----- | --------
    Regularized Linear Regression Cost Function |  25 /  25 | Nice work!
         Regularized Linear Regression Gradient |  25 /  25 | Nice work!
                                 Learning Curve |  20 /  20 | Nice work!
                     Polynomial Feature Mapping |  10 /  10 | Nice work!
                               Validation Curve |  20 /  20 | Nice work!
                                      --------------------------------
                                                | 100 / 100 |

### 3.4  Optional (ungraded) exercise: Computing test set error

In the previous part of the exercise, you implemented code to compute the cross validation error for various values of the regularization parameter $\lambda$. However, to get a better indication of the model’s performance in the real world, it is important to evaluate the “final” model on a test set that was not used in any part of training (that is, it was neither used to select the $\lambda$ parameters, nor to learn the model parameters $\theta$). For this optional (ungraded) exercise, you should compute the test error using the best value of $\lambda$ you found. In our cross validation, we obtained a test error of 3.8599 for $\lambda = 3$.

*You do not need to submit any solutions for this optional (ungraded) exercise.*

```python

```

### 3.5 Optional (ungraded) exercise: Plotting learning curves with randomly selected examples

In practice, especially for small training sets, when you plot learning curves to debug your algorithms, it is often helpful to average across multiple sets of randomly selected examples to determine the training error and cross validation error.

Concretely, to determine the training error and cross validation error for $i$ examples, you should first randomly select $i$ examples from the training set and $i$ examples from the cross validation set. You will then learn the parameters $\theta$ using the randomly chosen training set and evaluate the parameters $\theta$ on the randomly chosen training set and cross validation set. The above steps should then be repeated multiple times (say 50) and the averaged error should be used to determine the training error and cross validation error for $i$ examples.

For this optional (ungraded) exercise, you should implement the above strategy for computing the learning curves. For reference, the figure below  shows the learning curve we obtained for polynomial regression with $\lambda = 0.01$. Your figure may differ slightly due to the random selection of examples.

*You do not need to submit any solutions for this optional (ungraded) exercise.*

```python

```

Coursera Machine Learning Python Assignments/Exercise5/exercise5.ipynb


# Financial Network

**Author**: [Erika Fille Legara](http://www.erikalegara.net/)

You are free to use (or change) this notebook for any purpose you'd like. However, please respect the MIT License that governs its use, and for copying permission.

Copyright © 2016 Erika Fille Legara

---
## Description
I have been receiving requests to release the Python code I wrote to produce the financial network discussed in my blog post at [erikafille.ph](http://erikafille.ph) titled [PSE Correlation-Based Network](https://erikafille.ph/2016/08/16/pse-correlation-based-network/). Here it is.

In this notebook, we build a correlation-based network (minimum spanning tree (MST)) of companies listed in the [Philippine Stock Exchange](https://www.pse.com.ph/stockMarket/home.html). The entire process can be summarized into three steps:
1. Set up the **correlation matrix** for the stock prices in the Philippine Stock Exchange.
2. Convert the resulting correlation matrix into a **distance matrix**.
3. Build a **minimum spanning tree** from the distance matrix.

### Data
I got the link to the data from this [stock forum post](http://www.stockmarketpilipinas.com/thread-500.html). In this post it says [Drop Box: 2006-present worth of CSV files uploaded by Mr. Coelacanth](https://www.dropbox.com/sh/1dluf0lawy9a7rm/fHREemAjWS). The last I checked, the 2016 historical dataset is only until the 15th of July.

---
## Let's Dig In
Import the necessary packages.

```python
from __future__ import division

try:
    import networkx as nx
    import pandas as pd
    import os
    import matplotlib.pyplot as plt
    import seaborn as sns
    import math
    import numpy as np
    from datetime import datetime
    %matplotlib inline

except:
    import traceback
    traceback.print_exc()
    raise ImportError('Something failed, see above.')
```

### Load Data
We first load the list of companies in the PSE and store it in a Pandas data frame we call `pse_companies`.

```python
pse_companies = pd.read_csv("PSE-listed-companies.csv")
pse_companies = pse_companies[["Company Name", "Stock Symbol", "Sector", "Subsector", "Listing Date"]]
pse_companies.head()
```

      Company Name
      Stock Symbol
      Sector
      Subsector
      Listing Date

      0
      Unioil Resources & Holdings Company, Inc.
      UNI
      Holding Firms
      Holding Firms
      27 July 1987

      1
      Union Bank of the Philippines, Inc.
      UBP
      Financials
      Banks
      29 June 1992

      2
      United Paragon Mining Corporation
      UPM
      Mining and Oil
      Mining
      2 April 1973

      3
      Universal Robina Corporation
      URC
      Industrial
      Food, Beverage & Tobacco
      25 March 1994

      4
      Uniwide Holdings, Inc.
      UW
      Property
      Property
      19 August 1996

Then, load all files under the folder "2016" to load all 2016 data. Each file inside a folder contains the prices of all stock quotes in the PSE for the day (as indicated in the filename). As mentioned, the dataset I have for the year 2016 only runs until the 15th of July.

The cell block below loads all files in the directory `"./2016/"`.

```python
files2016 = os.listdir("./2016/")
```

Let's now explore the content of a file (stock prices for 1 day) inside the directory (folder).

*I am not exactly sure what the last two columns are, so I'm assigning them the variables X1 and X2, respectively.*

```python
print "Day: ", files2016[0]
df0 = pd.read_csv("./2016/" + files2016[0], header=None)
df0.head()
```

    Day:  stockQuotes_01042016.csv

      0
      1
      2
      3
      4
      5
      6
      7

      0
      ^FINANCIAL
      01/04/2016
      1554.07
      1555.61
      1531.13
      1531.13
      298140
      59289

      1
      AUB
      01/04/2016
      46.30
      46.30
      45.00
      46.15
      12900
      411515

      2
      BDO
      01/04/2016
      106.00
      106.00
      101.90
      101.90
      551000
      10878135

      3
      BPI
      01/04/2016
      83.70
      83.75
      83.00
      83.20
      384540
      9318360

      4
      CHIB
      01/04/2016
      37.50
      37.50
      37.05
      37.05
      33200
      193819

```python
df0.columns = ["Company_Index", "Date", "Open", "High", "Low", "Close", "X1", "X2"]
df0.head()
```

      Company_Index
      Date
      Open
      High
      Low
      Close
      X1
      X2

      0
      ^FINANCIAL
      01/04/2016
      1554.07
      1555.61
      1531.13
      1531.13
      298140
      59289

      1
      AUB
      01/04/2016
      46.30
      46.30
      45.00
      46.15
      12900
      411515

      2
      BDO
      01/04/2016
      106.00
      106.00
      101.90
      101.90
      551000
      10878135

      3
      BPI
      01/04/2016
      83.70
      83.75
      83.00
      83.20
      384540
      9318360

      4
      CHIB
      01/04/2016
      37.50
      37.50
      37.05
      37.05
      33200
      193819

After my initial data exploration, I made a list of companies I am discarding in this analysis. I am also excluding the indices. In any case, you may add or delete stocks to your liking.

```python
pse_comp = list(pse_companies["Stock Symbol"])
discard = ['UW', 'VMC', 'VVT', 'PRIM', 'MJIC', 'MACAY', 'PMT', 'REG', 'ROX', \
           'RCI', 'SPC', 'SPM', 'STR', 'STN', 'SRDC', 'SGP', 'MAH', 'MGH', \
           'NXGEN', 'PCP', 'PMPC', 'PAX', 'PHC', 'H2O', 'PNC', 'PRC', 'PTT', \
           'PTC', 'PORT', 'GPH', 'GREEN', 'KPH', 'LMG', 'LSC', 'CHP', 'CAT', \
           'CIP', 'CSB', 'DWC', 'ECP', 'EVER', 'EIBA', 'FEU', 'FFI', 'FYN', \
           'FAF', 'ABC', 'AAA', 'ATI', 'AB', 'BH', 'CHI', 'CPV', "BCOR"]
companies = [company for company in pse_comp if company not in discard ]
print companies
```

    ['UNI', 'UBP', 'UPM', 'URC', 'V', 'VLL', 'VITA', 'VUL', 'WPI', 'WIN', 'X', 'YEHEY', 'ZHI', 'IPO', 'PHN', 'PNX', 'PSPC', 'PHA', 'PLC', 'POPI', 'PRMX', 'PPC', 'PGOLD', 'RFM', 'RCB', 'RLC', 'RRHI', 'ROCK', 'SBS', 'SM', 'SMPH', 'SOC', 'SSI', 'STI', 'SMC', 'PF', 'SECB', 'SCC', 'SHNG', 'SGI', 'SPH', 'SLI', 'SLF', 'SUN', 'SFI', 'T', 'PSE', 'OV', 'TFHI', 'TA', 'TAPET', 'TBGI', 'RWM', 'MJC', 'MA', 'MWC', 'MFC', 'MARC', 'MAXS', 'MWIDE', 'MEG', 'MCP', 'MPI', 'MRSGI', 'MBT', 'MG', 'NRCP', 'NI', 'NIKL', 'NOW', 'OM', 'ORE', 'OPM', 'PAL', 'TEL', 'TFC', 'LOTO', 'PA', 'PIP', 'PERC', 'PCOR', 'WEB', 'PX', 'PXP', 'PBC', 'PBB', 'PHES', 'PNB', 'RLT', 'PSB', 'SEVN', 'GTCAP', 'GSMI', 'FNI', 'GERI', 'GLO', 'HVN', 'TUGS', 'HLCM', 'HI', 'I', 'EG', 'IPM', 'IRC', 'ISM', 'IMP', 'IMI', 'ICT', 'ION', 'IS', 'IDC', 'JGS', 'JAS', 'JFC', 'JOH', 'KEP', 'LBC', 'LTG', 'LR', 'LC', 'LFM', 'LIB', 'LIHC', 'LPZ', 'MED', 'MRC', 'MHC', 'MVC', 'MAC', 'MFIN', 'MBC', 'MB', 'MER', 'CEU', 'CNPF', 'CPM', 'CPG', 'CHIB', 'TECH', 'LAND', 'CDC', 'COAL', 'CIC', 'CA', 'COSCO', 'CROWN', 'CEI', 'CYBR', 'DNL', 'DFNN', 'DMC', 'DAVIN', 'DMPL', 'DIZ', 'DD', 'EEI', 'EW', 'EMP', 'ELI', 'EDC', 'EURO', 'FJP', 'FDC', 'FLI', 'FGEN', 'FMETF', 'FPH', 'FPI', 'GEO', 'GMAP', 'GMA7', '2GO', 'HOUSE', 'BRN', 'ANS', 'ABS', 'ABSP', 'AGF', 'APC', 'ATN', 'ABA', 'AEV', 'AP', 'AR', 'ACE', 'ANI', 'AGI', 'FOOD', 'ACR', 'ALT', 'ALHI', 'APO', 'APX', 'ARA', 'ALCO', 'AUB', 'ABG', 'AT', 'AC', 'ALI', 'BLFI', 'BDO', 'BPI', 'BSC', 'BEL', 'BC', 'BLOOM', 'BMM', 'BHI', 'BKR', 'COL', 'CAL', 'CEB']

We then filter the dataframe to companies listed in `companies`. We also drop the other columns and only retain the following  `["Company_Index", "Date", "Close"]` columns

```python
allprices = pd.DataFrame()
for f in files2016:
    df = pd.read_csv("./2016/" + f, header=None)
    df.columns = ["Company_Index", "Date", "Open", "High", "Low", "Close", "Volume1", "Volume2"]
    df = df[df.Company_Index.isin(companies)]
    df.Date = pd.to_datetime(df.Date)
    df = df[["Company_Index", "Date", "Close"]]
    allprices = pd.concat([allprices,df], ignore_index = True)
```

```python
allprices.head()
```

      Company_Index
      Date
      Close

      0
      AUB
      2016-01-04
      46.15

      1
      BDO
      2016-01-04
      101.90

      2
      BPI
      2016-01-04
      83.20

      3
      CHIB
      2016-01-04
      37.05

      4
      EW
      2016-01-04
      18.22

```python
## List of all comapnies we are considering in this notebook

print set(list(allprices.Company_Index))
```

    set(['PXP', 'CPM', 'AGI', 'BC', 'JFC', 'AGF', 'POPI', 'FMETF', 'CPG', 'RWM', 'IDC', 'SCC', 'BMM', 'PAL', 'FNI', 'MED', 'PERC', 'RRHI', 'LIB', 'GMA7', 'GERI', 'CROWN', 'SPH', 'ION', 'LBC', 'MFIN', 'YEHEY', 'PIP', 'LR', 'WEB', 'LPZ', 'URC', 'VITA', 'OM', 'GMAP', 'GSMI', '2GO', 'T', 'PGOLD', 'X', 'MRC', 'APX', 'UBP', 'PHN', 'SMPH', 'HOUSE', 'PHA', 'CDC', 'IPO', 'IPM', 'MAC', 'APO', 'SFI', 'APC', 'TBGI', 'FDC', 'DMC', 'BEL', 'AEV', 'MWC', 'HI', 'EEI', 'EMP', 'GTCAP', 'MFC', 'VLL', 'PX', 'CEI', 'PRMX', 'CEU', 'PA', 'PF', 'GEO', 'EG', 'UPM', 'BLFI', 'FOOD', 'BDO', 'CNPF', 'STI', 'PHES', 'TUGS', 'NRCP', 'PBC', 'PBB', 'ABSP', 'EW', 'JGS', 'RFM', 'MG', 'MA', 'OV', 'PSB', 'MWIDE', 'MPI', 'AR', 'HLCM', 'ANS', 'OPM', 'NOW', 'AT', 'TAPET', 'ACE', 'GLO', 'ATN', 'SUN', 'VUL', 'ACR', 'PCOR', 'ISM', 'CHIB', 'FJP', 'SMC', 'COL', 'LAND', 'NI', 'ALHI', 'SECB', 'TEL', 'DMPL', 'LFM', 'TECH', 'WPI', 'BKR', 'ICT', 'BSC', 'AUB', 'IMI', 'DNL', 'COSCO', 'JOH', 'MVC', 'LTG', 'SBS', 'V', 'UNI', 'FLI', 'EURO', 'EDC', 'MEG', 'FGEN', 'ALI', 'WIN', 'MB', 'PSPC', 'RLC', 'BRN', 'PLC', 'SM', 'MER', 'RLT', 'ALT', 'ORE', 'TFHI', 'CIC', 'ALCO', 'KEP', 'DAVIN', 'BPI', 'DD', 'ANI', 'ELI', 'SHNG', 'TFC', 'DIZ', 'CAL', 'ZHI', 'PSE', 'RCB', 'PPC', 'SOC', 'LC', 'LIHC', 'MJC', 'SEVN', 'MBC', 'SGI', 'COAL', 'MHC', 'HVN', 'LOTO', 'ROCK', 'MBT', 'SSI', 'DFNN', 'BLOOM', 'CA', 'TA', 'CYBR', 'ABA', 'AC', 'ABG', 'I', 'IS', 'FPI', 'FPH', 'JAS', 'AP', 'ABS', 'SLI', 'SLF', 'IRC', 'CEB', 'BHI', 'PNX', 'MARC', 'MRSGI', 'IMP', 'MAXS', 'NIKL', 'ARA', 'PNB', 'MCP'])

In the variable `subset` below, we further filter the dataset by date range. Here, I only want to look at the prices from January 01, 2016 to the end of July. I then reshape the data frame `subset` and store it in the variable `final_df` where the columns are the assets and the rows are the prices of the assets on a particular day.

```python
subset = allprices[(allprices.Date > datetime(2016,1,1,0,0,0)) & (allprices.Date < datetime(2016,7,31,23,59,59))]
final_df = subset.pivot(index="Date", columns="Company_Index", values="Close")
```

```python
final_df.head()
```

      Company_Index
      2GO
      ABA
      ABG
      ABS
      ABSP
      AC
      ACE
      ACR
      AEV
      AGF
      ...
      V
      VITA
      VLL
      VUL
      WEB
      WIN
      WPI
      X
      YEHEY
      ZHI

      Date

      2016-01-04
      7.00
      0.370
      10.30
      62.40
      64.0
      756
      NaN
      1.38
      57.20
      2.80
      ...
      3.20
      0.59
      5.00
      1.05
      23.0
      NaN
      NaN
      15.62
      4.10
      NaN

      2016-01-05
      6.91
      0.370
      10.28
      62.30
      63.0
      742
      NaN
      1.37
      57.75
      2.71
      ...
      1.83
      0.59
      5.00
      1.08
      22.7
      0.21
      0.33
      15.52
      4.12
      0.250

      2016-01-06
      6.80
      0.365
      10.20
      62.00
      62.6
      732
      1.14
      1.39
      57.90
      2.58
      ...
      1.80
      0.62
      4.97
      1.06
      21.5
      NaN
      0.34
      15.20
      3.88
      NaN

      2016-01-07
      6.63
      0.370
      10.22
      61.75
      62.1
      709
      1.15
      1.36
      56.00
      2.65
      ...
      1.76
      0.58
      4.82
      1.03
      21.0
      0.20
      0.33
      14.48
      3.62
      0.231

      2016-01-08
      6.55
      0.360
      10.18
      61.10
      62.0
      700
      1.12
      1.35
      56.30
      2.50
      ...
      1.70
      0.56
      4.93
      1.06
      21.0
      0.19
      0.32
      14.64
      3.38
      0.231

5 rows × 213 columns

---
### Step 1: Build Correlation Matrix
From the data frame above, we now build the **correlation matrix**.

In finance/stock trading, **correlation** is just a measure of the extent at which two equities behave with respect to each other.

Below, we build the correlation matrix from `final_df` and store it in the variable `price_corr`. The matrix provides us with the corresponding **correlation coefficients (-1.0 to +1.0)** for all stock pairs in the list of companies. [Straightforward](http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.corr.html), isn't it?

```python
price_corr = final_df.corr()
```

When two assets are *positively correlated*, it means that the general trends of the two stocks are similar; when one goes up, the other one goes up as well. When, on the other hand, two assets are *negatively correlated*, their trends go in opposite directions.

I made a quick sketch below to illustrate these relationships.

For a more concrete example, let us have a look at the relationships between (`$ABS` and `$ACR`), (`$FMETF` and `$ALI`) and (`$BLOOM` and `$BDO`); the `$` sign is the hashtag used for stock quotes. I chose these securities based on the correlation values of the pairs in `price_corr`. Can you tell which of the pairs are positively correlated? Negatively correlation?

```python
## ABS and ACR
plt.figure(figsize=(8,5))
plt.plot(final_df['ABS'], final_df["ACR"], '.', markersize=10)
plt.xlabel("Price of $ABS (PhP)")
plt.ylabel("Price of $ACR (PhP)")
_ = plt.show()
```

```python
## FMETF and ALI

plt.figure(figsize=(8,5))
plt.plot(final_df['FMETF'], final_df["ALI"], '.', markersize=10)
plt.xlabel("Price of $FMETF (PhP)")
plt.ylabel("Price of $ALI (PhP)")
_ = plt.show()
```

```python
## BLOOM and BDO

plt.figure(figsize=(8,5))
plt.plot(final_df['BLOOM'], final_df["BDO"], '.', markersize=10)
plt.xlabel("Price of $BLOOM (PhP)")
plt.ylabel("Price of $BDO (PhP)")
_ = plt.show()
```

Below, we draw the heatmap of the resulting correlation matrix.

```python
## Source: https://stanford.edu/~mwaskom/software/seaborn/examples/many_pairwise_correlations.html

## Generate a mask for the upper triangle
mask = np.zeros_like(price_corr, dtype=np.bool)
mask[np.triu_indices_from(mask)] = True

## Set up the matplotlib figure
f, ax = plt.subplots(figsize=(15, 15))

## Generate a custom diverging colormap
cmap = sns.diverging_palette(220, 10, as_cmap=True)

## Draw the heatmap with the mask and correct aspect ratio
sns.heatmap(price_corr, mask=mask, cmap=cmap, vmax=.3,
            square=True, xticklabels=2, yticklabels=2,
            linewidths=.5, cbar_kws={"shrink": .5}, ax=ax)

_ = plt.show()
```

---
### Step 2: Build Distance Matrix
As mentioned in the blog post, we will use two distance metrics in building the distance matrices. The first metric is from Bonanno et al.

\begin{equation}d_{ij} = \sqrt{2 \times (1 - c_{ij})}\end{equation}

where $c_{ij}$ is the correlation cofficient of stocks $i$ and $j$. In the equation, when $c_{ij}=1$, $d_{ij}=0$; and, when $c_{ij}=-1$, $d_{ij}=2$. That is, when there is a perfectly positive correlation (+1), the distance is 0; and, when the correlation is perfectly negative, the distance is the farthest at 2. The next distance measure is from [SliceMatrix](http://SliceMatrix.com) (mktstk) and it is given by

\begin{equation}d_{ij} = 1 - |c_{ij}|.\end{equation}

This equation does not distinguish between a positively or a negatively correlated pair of stocks; as long as two stocks are highly correlated, the distance is minimized.

Here, we define the distance matrices as `dist_bonanno` and `dist_mktstk`.

```python
dist_bonanno = np.sqrt(2*(1-(price_corr)))
dist_mktstk = 1-abs(price_corr)

## I am just defining the labels
labs_bonanno = list(dist_bonanno.index)
labs_mktstk = list(dist_mktstk.index)
```

---
### Step 3: Build Minimum Spanning Tree (MST)

Now, we are ready to build the minimum spanning tree. The idea is to connect the ones that have the closest distance to each other, i.e. connect those that are highly correlated. Let's first build the "weighted" networks `G_bonanno` and `G_mktstk` from the distance matrices `dist_bonanno` and `dist_mktstk`, respectively. Using the Python package NetworkX, that's pretty straightforward to do.

```python
G_bonanno = nx.from_numpy_matrix(dist_bonanno.as_matrix())
G_mktstk = nx.from_numpy_matrix(dist_mktstk.as_matrix())
```

Once we have the distance networks, we can already build minimum spanning trees (MST). Here, we use Kruskal's algorithm. Below is the pseudo-code copied from the Wikipedia entry on the algorithm.

KRUSKAL(G):
1 A = ∅
2 foreach v ∈ G.V:
3    MAKE-SET(v)
4 foreach (u, v) in G.E ordered by weight(u, v), increasing:
5    if FIND-SET(u) ≠ FIND-SET(v):
6       A = A ∪ {(u, v)}
7       UNION(u, v)
8 return A

Again, we can use NetworkX to build the MST with the graphs as inputs.

```python
MST_b = nx.minimum_spanning_tree(G_bonanno)
MST_m = nx.minimum_spanning_tree(G_mktstk)
```

Finally, let's add more attributes to the "nodes" or the stocks. The attributes that I want to include here are:

- `label` (the stock symbol)
- `sector` (which sector the stock belongs)
- `change` (the $\%$ change of the stock for the period under study)

This way, when we draw the MSTs, we can choose to color the nodes by either `sector` or `change`.

```python
change = (final_df.iloc[-1] - final_df.iloc[0]) * 100 / final_df.iloc[0]
```

```python
for node in MST_b.nodes():
    sector = pse_companies[pse_companies["Stock Symbol"] == labs_bonanno[node]].Sector.iloc[0]
    MST_b.node[node]["sector"] = sector
    MST_b.node[node]["label"] = labs_bonanno[node]
    if math.isnan(change[labs_bonanno[node]]):
        MST_b.node[node]["color"] = "black"
    elif change[labs_bonanno[node]] < -10:
        MST_b.node[node]["color"] = "red"
    elif change[labs_bonanno[node]] > 10:
        MST_b.node[node]["color"] = "green"
    else:
        MST_b.node[node]["color"] = "blue"
```

```python
for node in MST_m.nodes():
    sector = pse_companies[pse_companies["Stock Symbol"] == labs_mktstk[node]].Sector.iloc[0]
    MST_m.node[node]["sector"] = sector
    MST_m.node[node]["label"] = labs_mktstk[node]
    if math.isnan(change[labs_mktstk[node]]):
        #print change[labs_mktstk[node]], labs_mktstk[node]
        #Gm.node[node]["change"] = 101
        MST_m.node[node]["color"] = "black"
    elif change[labs_mktstk[node]] < -10:
        MST_m.node[node]["color"] = "red"
    elif change[labs_mktstk[node]] > 10:
        MST_m.node[node]["color"] = "green"
    else:
        MST_m.node[node]["color"] = "blue"
```

### Drawing the MSTs

```python
plt.figure(figsize=(10,10))
nx.draw_networkx(MST_b)
```

```python
plt.figure(figsize=(10,10))
nx.draw_networkx(MST_m)
```

### Write out MSTs
Below, we write the MSTs as `gexf` files so we can use them in [Gephi](https://gephi.org/) (open-source and free) to generate much prettier networks/trees.

Gephi is the leading visualization and exploration software for all kinds of graphs and networks. Gephi is open-source and free.

```python
nx.write_gexf(MST_b, "corrmat_bonanno.gexf")
nx.write_gexf(MST_m, "corrmat_mktstk.gexf")
```

Below is the resulting network (`MST_b`) drawn using Gephi.

```python

```


```
import bluefog.torch as bf
from bluefog.common import topology_util

import matplotlib.pyplot as plt
import numpy as np
import networkx as nx
import torch

%matplotlib inline
```

# Dynamic Topology over Exponential Two Graph

When running average consensus based decentralized algorithm, the communication cost is not neglectable usually. In one round, the vector length of communication is proportional to the number of edges -- $O(E)$. Even for exponential graph each node will contain only $O(log(|V|))$ edge, the total communication cost will be $O(|V|log(|V|))$, i.e. $O(Nlog(N))$. This is actually worse than the [ring-allreduce](https://arxiv.org/pdf/1802.05799.pdf) based decentralized algorithm, which only requires $O(2N)$ communication cost. Although ring-structed neighbor allreduce will have similar communication cost, the convergence speed of ring-structure is so slow that it can hinder the gradient descent. Actually, it is shown under some assumptions that ring-allreduce algorithm actually achieve the optimal bandwidth usage for static topology.

Fortunately, AWC algorithms or other variantions also work under dynamic topology. It means in each round, the underlying topology is dynamic, i.e., the communication graph $\mathcal{G}^k$ changes with time/iteration $k$. There are many choices for generating dynamic topologies. Among them, this notebook focuses on the *dynamic one peer strategy*, which means all agents only selection one neighbor agents (peer) at each communication around. You can immediate find that the communication cost decreased to $O(N)$ now.
When it combined with Exponential graph, you will see that it not only reduces the communications cost but surprisingly accelerates the average consensus speed under mild conditions.

```
# Consider a static exponential-2 graph with size 16
nx.draw_circular(topology_util.ExponentialTwoGraph(16))
```

Under exponential-2 graph, we know the agent $i$ will send the informaiton to $[i+1, i+2, i+4, \ldots, i+2^n]$ module $N$. Then, the dynamic one peer strategy can be easily generated through at iteration $k$, agent $i$ selects the neighbro $i+2^{k\%n}$. It is easy to see that dynamic topology is periodical and the period is $n$. Let's plot the case for exponential-2 graph with size 16, which should have 4 distinct one peer graphs.

```
n = 4
size = 2 ** n
G_full = topology_util.ExponentialTwoGraph(size=size)
for k in range(4):
    plt.figure(figsize=(8, 6))
    x = np.array([1.0 if i == 2 ** (k % n) else 0 for i in range(size)])
    x /= x.sum()
    topo = np.empty((size, size))
    for i in range(size):
        topo[i] = np.roll(x, i)
    G = nx.from_numpy_array(topo, create_using=nx.DiGraph)

    nx.draw_circular(G_full, node_size=800, width=2, edge_color="#C2C2C2")
    nx.draw_circular(G, node_size=800, width=2)
    plt.text(
        1,
        -1.2,
        "Iteration K" + ("+" + str(k) if k != 0 else "  "),
        horizontalalignment="right",
        verticalalignment="center",
        fontsize=25,
    )
    plt.show()
```

It is not surprising that the communication cost is reduced. But you may wonder when we decrease the communication, will it hinder our convergence speed and eventually undermine the speedup brought by the communication? There is one simple lemma:

**Lemma** Suppose $W^{(k)}$ is the combination matrix at iteration $k$ over the dynamic one-peer exponential-2 graph with $N=2^n$ nodes. It holds that each $W^{(k)}$ is doubly-stochastic, i.e., $W^{(k)} \mathbb{1} = \mathbb{1}$ and $\mathbb{1}^T W^{(k)} = \mathbb{1}^T$. Furthermore, it holds that
\begin{align}\label{finite-time-average}
W^{(k+\ell)} W^{(k+\ell-1)} \cdots W^{(k+1)} W^{(k)}= \frac{1}{n}\mathbb{1}\mathbb{1}^T
\end{align}
for any integer $k \ge0$ and $\ell \ge n$.

Let's verify this in code.

```
all_combined_matrix = np.eye(size)
for k in range(4):
    x = np.array([1.0 if i == 2 ** (k % n) else 0 for i in range(size)])
    x /= x.sum()
    topo = np.empty((size, size))
    for i in range(size):
        topo[i] = np.roll(x, i)  # topo does not include self
    W = (np.eye(size) + topo) / 2
    all_combined_matrix = all_combined_matrix @ W
```

```
uniform_matrix = np.ones([size, size]) / size
np.all(np.isclose(all_combined_matrix - uniform_matrix, 0))
```

    True

# Dynamic One Peer Strategy over General Topology

Similarly, we can design one peer strategy over general topology, i.e. every node pick one peer to send at each iterations. BlueFog provided several topology utility functions for several common dynamic topology cases. This dynamic one peer strategy is definitely covered -- `topology_util.GetDynamicOnePeerSendRecvRanks`.

It is not a function but [generator](https://docs.python.org/3/tutorial/classes.html#generators) actually. If you are not familar with the concept of generator, don't be afraid. It is easy. Check the following example

```
def simple_generator():
    yield 1
    yield 2

g = simple_generator()
print(next(g))
print(next(g))
```

    1
    2

You should use the `topology_util.GetDynamicOnePeerSendRecvRanks` in a similar style. It takes the `topology` and `self_rank` as input and at each iteration, it returns two lists -- one for the ranks of sending to and another for the ranks of receiving from with respect to that `self_rank`.

```
generator = topology_util.GetDynamicOnePeerSendRecvRanks(
    topology_util.ExponentialTwoGraph(size=16), self_rank=0
)
print(next(generator))
print(next(generator))
print(next(generator))
print(next(generator))
print(next(generator))
```

    ([1], [15])
    ([2], [14])
    ([4], [12])
    ([8], [8])
    ([1], [15])

Another example of using it over mesh grid topology.

```
generator = topology_util.GetDynamicOnePeerSendRecvRanks(
    topology_util.MeshGrid2DGraph(size=16), self_rank=0
)
print(next(generator))
print(next(generator))
print(next(generator))
print(next(generator))
print(next(generator))
```

    ([1], [])
    ([4], [])
    ([1], [1, 4])
    ([4], [])
    ([1], [])


**Brief Honor Code**. Do the homework on your own. You may discuss ideas with your classmates, but DO NOT copy the solutions from someone else or the Internet. If stuck, discuss with TA.

**1**. (20 points)

Consider the linear transformation $f(x)$ on $\mathbb{R}^3$ that takes the standard basis $\left\{e_1,e_2,e_3\right\}$ to $\left\{v_1,v_2,v_3\right\}$ where

$$v_1=\left(\begin{matrix}10\\-10\\16\end{matrix}\right), v_2=\left(\begin{matrix}2\\-5\\20\end{matrix}\right) \textrm {and } v_3=\left(\begin{matrix}1\\-4\\13\end{matrix}\right)$$

1. Write a matrix $A$ that represents the same linear transformation. (4 points)

2. Compute the rank of $A$ using two different methods (do not use `matrix_rank`!). (4 points)

3. Find the eigenvalues and eigenvectors of $A$. (4 points)

4. What is the matrix representation of $f$ with respect to the eigenbasis? (8 points)

```python
import numpy as np
import scipy.linalg as la
```

```python
A = np.array([[10,2,1],[-10,-5,-4], [16,20,13]])
A
```

    array([[ 10,   2,   1],
           [-10,  -5,  -4],
           [ 16,  20,  13]])

```python
#np.linalg.matrix_rank(A)
U, s, Vt = la.svd(A)
tol = s.max() * max(A.shape) * np.finfo(float).eps
rank1 = np.sum(s > tol)
(q, r) = la.qr(A)
#to search how many colums have pivots
rank2 = np.sum((r != 0).any(axis=0))
rank1,rank2
```

    (3, 3)

```python
eg_value, eg_vector = np.linalg.eig(A)
```

```python
eg_value
```

    array([ 9.,  3.,  6.])

```python
eg_vector
```

    array([[  5.77350269e-01,   6.31950024e-16,  -1.20385853e-01],
           [ -5.77350269e-01,  -4.47213595e-01,  -2.40771706e-01],
           [  5.77350269e-01,   8.94427191e-01,   9.63086825e-01]])

```python
np.linalg.inv(eg_vector) @ A @ eg_vector
```

    array([[  9.00000000e+00,   7.75444829e-15,   6.25430592e-15],
           [  1.64008779e-14,   3.00000000e+00,  -5.98520757e-15],
           [ -5.19475027e-15,   3.56964333e-15,   6.00000000e+00]])

```python
np.diag(eg_value)
```

    array([[ 9.,  0.,  0.],
           [ 0.,  3.,  0.],
           [ 0.,  0.,  6.]])

**2**. (20 points)

You are given the following x-y coordinates (first column is x, second is y)

```
array([[  0.        ,   4.12306991],
       [  3.        , -15.47355729],
       [  4.        , -11.68725507],
       [  3.        , -20.33756693],
       [  5.        ,  -6.06401989],
       [  6.        ,  32.79353057],
       [  8.        ,  82.48658405],
       [  9.        ,  84.02971858],
       [  4.        ,  -1.30587276],
       [  8.        ,  68.59409878]])
```

- Find the coefficients $(a, b, c)$ of the least-squares fit of a quadratic function $y = a + bx + cx^2$ to the data.
- Plot the data and fitted curve using `matplotlib`.

```python
xs = np.array([
    [  0.        ,   4.12306991],
    [  3.        , -15.47355729],
    [  4.        , -11.68725507],
    [  3.        , -20.33756693],
    [  5.        ,  -6.06401989],
    [  6.        ,  32.79353057],
    [  8.        ,  82.48658405],
    [  9.        ,  84.02971858],
    [  4.        ,  -1.30587276],
    [  8.        ,  68.59409878]])
```

```python
length = len(xs)
x1 = [xs[i][0] for i in range(length)]
X = np.vstack([np.power(x1, 0),np.power(x1, 1),np.power(x1, 2)]).T
y = [xs[i][1] for i in range(length)]
```

```python
beta = np.linalg.inv(X.T @ X) @X.T @y
beta = beta.reshape(-1,1)
beta
```

    array([[ -0.35762896],
           [-11.78531232],
           [  2.53125199]])

```python
yhat = X @ beta
xcon = np.linspace(0, 9 ,10)
Xfull = np.vstack([np.power(xcon, 0),np.power(xcon, 1),np.power(xcon, 2)]).T
ycon1 = Xfull@ beta
```

```python
import matplotlib.pyplot as plt
plt.plot(x1,y, "ro")
plt.plot(ycon1, label="Prediction")
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plt.show()
```

    <matplotlib.figure.Figure at 0x7ff89984b8d0>

**3**. (20 points)

Use the `svd` function to solve the least squares problem above, and repeat the same plot. Calculate the residual error $\lvert y - X\beta \rvert$.

```python
U, S, V, = np.linalg.svd(X, y)
ps_inv_sigma = np.r_[np.linalg.inv(np.diag(S.tolist())),np.zeros((7,3))].T
```

```python
beta1 = V.T@(ps_inv_sigma) @U.T @ y
yhat1 = X @ beta1
```

```python
abserr = np.abs(y - yhat1)
abserr
```

    array([  4.48069887,   2.54125925,   4.68840861,   7.40526889,
            10.06112898,  12.73796195,  15.12658447,  14.57625244,
             5.6929737 ,   1.2340992 ])

```python
#sum abs error
np.sum(abserr)
```

    78.544636354965434

```python
#sum l_2 norm error
np.sqrt(np.sum((y - yhat1)**2))
```

    29.018088805093509

```python
ycon2 = Xfull@ beta1
```

```python
plt.plot(x1,y, "ro")
plt.plot(ycon2, label="Prediction")
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plt.show()
```

**4**. (20 points)

Avoiding catastrophic cancellation.

Read the Wikipedia entry on [loss of significance](https://en.wikipedia.org/wiki/Loss_of_significance). Then answer the following problem:

The tail of the standard logistic distribution is given by $1 - F(t) = 1 - (1+e^{-t})^{-1}$.

- Define a function `f1` to calculate the tail probability of the logistic distribution using the formula given above
- Use [`sympy`](http://docs.sympy.org/latest/index.html) to find the exact value of the tail distribution (using the same symbolic formula) to 20 decimal digits
- Calculate the *relative error* of `f1` when $t = 25$ (The relative error is given by `abs(exact - approximate)/exact`)
- Rewrite the expression for the tail of the logistic distribution using simple algebra so that there is no risk of cancellation, and write a function `f2` using this formula. Calculate the *relative error* of `f2` when $t = 25$.
- How much more accurate is `f2` compared with `f1` in terms of the relative error?

```python
def f1(t):
    """
    return right tail of logistic distribution
    """
    return 1-(1+np.exp(-t))**(-1)
```

```python
from sympy import exp
def fexact(t, n=20):
    """
    return right tail of logistic distribution
    """
    return (1 - (1 + exp(-t))**(-1)).evalf(n=n)
```

```python
e1= abs(fexact(25) - f1(25))/fexact(25)
e1
```

    4.1759147665982646285e-6

```python
def f2(t):
    """
    return right tail of logistic distribution
    """
    return 1/(1 + np.exp(t))
```

```python
e2 =abs(fexact(25) - f2(25))/fexact(25)
e2
```

    2.3034925262183876741e-16

```python
(e1-e2)/e2
```

    18128623031.494952608

**5**. (20 points)

Read in `figs/elephant.jpg` as a gray-scale image. The image has $1066 \times 1600$ values. Using SVD, recreate the image with a relative error of less than 0.5%. What is the relative size of the compressed image as a percentage?

```python
from skimage import io
img = io.imread('figs/elephant.jpg', as_grey=True)
plt.imshow(img)
plt.show()
```

```python
U.shape, V.shape
```

    ((1066, 1066), (1600, 1600))

```python
U, S, V = np.linalg.svd(img)
i = 1
s = np.sum(S[:i]**2)
while s/np.sum(S**2) < 1 - 0.005:
        i +=1
        s = np.sum(S[:i]**2)
```

```python
i
```

    21

```python
(i*1066 + i + i*1600)/(1066*1600)
```

    0.032837124765478426

In this case, it takes about 3.3% spaces of the orginal plot.

```python
#np.where(np.cumsum(S**2)/np.sum(S**2) > 1 - 0.005**2) start from 0? index is 562
```