# INF-510, v0.31, Claudio Torres, ctorres@inf.utfsm.cl. DI-UTFSM
## Textbook: Lloyd N. Trefethen, Spectral Methods in MATLAB, SIAM, Philadelphia, 2000

# More on Spectral Matrices

```python
import matplotlib.pyplot as plt
%matplotlib inline
import numpy as np
import scipy.sparse.linalg as sp
from scipy import interpolate
import scipy as spf
from sympy import *
import sympy as sym
from scipy.linalg import toeplitz
from ipywidgets import interact
from ipywidgets import IntSlider
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
from matplotlib.ticker import LinearLocator, FormatStrFormatter
# The variable M is used for changing the default size of the figures
M=5
import ipywidgets as widgets
import matplotlib as mpl
mpl.rcParams['font.size'] = 14
mpl.rcParams['axes.labelsize'] = 20
mpl.rcParams['xtick.labelsize'] = 14
mpl.rcParams['ytick.labelsize'] = 14
sym.init_printing()
```

### Chebyshev differentiation matrix

```python
def cheb(N):
    if N==0:
        D=0
        x=1
        return D,x
    x = np.cos(np.pi*np.arange(N+1)/N)
    c=np.hstack((2,np.ones(N-1),2))*((-1.)**np.arange(N+1))
    X=np.tile(x,(N+1,1)).T
    dX=X-X.T
    D = np.outer(c,1./c)/(dX+np.eye(N+1))
    D = D - np.diag(np.sum(D.T,axis=0))
    return D,x
```

## Understanding how the np.FFT does the FFT

```python
def show_spectral_derivative_example(N):
    x=np.linspace(2*np.pi/N,2*np.pi,N)
    u = lambda x: np.sin(x)
    up = lambda x: np.cos(x)

    #u = lambda x: np.sin(x)*np.cos(x)
    #up = lambda x: np.cos(x)*np.cos(x)-np.sin(x)*np.sin(x)

    v=u(x)
    K=np.fft.fftfreq(N)*N
    iK=1j*K
    vhat=np.fft.fft(v)

    W=iK*vhat
    W[int(N/2)]=0

    vp=np.real(np.fft.ifft(W))

    plt.figure(figsize=(10,10))
    plt.plot(x,v,'ks-',markersize=12,markeredgewidth=3,label='$\sin(x)$',linewidth=3)
    plt.plot(x,up(x),'b.-',markersize=24,markeredgewidth=3,label='Exact derivative: $\cos(x)$',linewidth=3)
    plt.plot(x,np.real(vp),'rx-',markersize=10,markeredgewidth=3,label='spectral derivative',linewidth=3)
    plt.grid(True)
    plt.legend(loc='best')
    plt.xlabel('$x$')
    plt.show()

    print('v    :',v)
    print('vhat :',vhat)
    print('K    :',K)
    print('W    :',W)
    print('vprime: ',vp)
widgets.interact(show_spectral_derivative_example,N=(2,40,2))
```

```python
def spectralDerivativeByFFT(v,nu=1):
    if not np.all(np.isreal(v)):
        raise ValueError('The input vector must be real')
    N=v.shape[0]
    K=np.fft.fftfreq(N)*N
    iK=(1j*K)**nu
    v_hat=np.fft.fft(v)
    w_hat=iK*v_hat
    if np.mod(nu,2)!=0:
        w_hat[int(N/2)]=0
    return np.real(np.fft.ifft(w_hat))

def my_D2_spec_2pi(N):
    h=(2*np.pi/N)
    c=np.zeros(N)
    j=np.arange(1,N)
    c[0]=-np.pi**2/(3.*h**2)-1./6.
    c[1:]=-0.5*((-1)**j)/(np.sin(j*h/2.)**2)
    D2=toeplitz(c)
    return D2
```

# Fractional derivative application

```python
def fractional_derivative(N=10,nu=1):
    x=np.linspace(2*np.pi/N,2*np.pi,N)
    u = lambda x: np.sin(x)
    up = lambda x: np.cos(x)
    v = u(x)
    vp=spectralDerivativeByFFT(v,nu)
    plt.figure(figsize=(10,10))
    plt.plot(x,v,'ks-',markersize=12,markeredgewidth=3,label='$\sin(x)$',linewidth=3)
    plt.plot(x,up(x),'b.-',markersize=24,markeredgewidth=3,label='Exact derivative: $\cos(x)$',linewidth=3)
    plt.plot(x,np.real(vp),'rx-',markersize=10,markeredgewidth=3,label=r'$\frac{d^{\nu}u}{dx^{\nu}}$',linewidth=3)
    plt.grid(True)
    plt.legend(loc='best')
    plt.xlabel('$x$')
    plt.show()
d_nu=0.1
widgets.interact(fractional_derivative,N=(4,100),nu=(d_nu,1,d_nu))
```

# Example 1: Computing Eigenvalues

We are solving: $-u''(x)+x^2\,u(x)=\lambda\, u(x)$ on $\mathbb{R}$

```python
L=8.0
def show_example_1(N=6):
    h=2*np.pi/N
    x=np.linspace(h,2*np.pi,N)
    x=L*(x-np.pi)/np.pi
    D2=(np.pi/L)**2*my_D2_spec_2pi(N)
    w, v = np.linalg.eig(-D2+np.diag(x**2))
    # eigenvalues = np.sort(np.linalg.eigvals(-D2+np.diag(x**2)))
    ii = np.argsort(w)
    w=w[ii]
    v=v[:,ii]

    plt.figure(figsize=(2*M,2*M))

    for i in np.arange(1,5):
        plt.subplot(2,2,i)
        plt.title(r'$u_{:d}(x),\, \lambda_{:d}={:f}$'.format(i,i,w[i-1]))
        plt.plot(x,v[:,i],'kx',markersize=16,markeredgewidth=3)
        plt.grid(True)
    plt.show()
widgets.interact(show_example_1,N=(6,100,1))
```

# Example 2:  Solving ODE

Solving the following BVP $u_{xx}=\exp(4\,x)$ with $u(-1)=u(1)=0$

```python
def example_2(N=16):
    D,x = cheb(N)
    D2  = np.dot(D,D)
    D2  = D2[1:-1,1:-1]
    f   = np.exp(4*x[1:-1])
    u   = np.linalg.solve(D2,f)
    u   = np.concatenate(([0],u,[0]),axis=0)

    plt.figure(figsize=(M,M))
    plt.plot(x,u,'k.')
    xx  = np.linspace(-1,1,1000)
    P   = np.polyfit(x, u, N)
    uu  = np.polyval(P, xx)
    plt.plot(xx,uu,'b-')
    plt.grid(True)
    exact = (np.exp(4*xx)-np.sinh(4.)*xx-np.cosh(4.))/16.
    plt.title('max error= '+str(np.linalg.norm(exact-uu,np.inf)))
    plt.ylim([-2.5,0.5])
    plt.show()
interact(example_2,N=(2,35))
```

# Example 3: Solving ODE

Solving the following BVP $u_{xx}=\exp(u)$ with $u(-1)=u(1)=0$

```python
def example_3(N=16,IT=20):
    D,x = cheb(N)
    D2  = np.dot(D,D)
    D2  = D2[1:-1,1:-1]

    u   = np.zeros(N-1)
    for i in np.arange(IT):
        u_new = np.linalg.solve(D2,np.exp(u))
        change = np.linalg.norm(u_new-u,np.inf)
        u = u_new

    u = np.concatenate(([0],u,[0]),axis=0)

    plt.figure(figsize=(M,M))
    plt.plot(x,u,'k.')
    xx  = np.linspace(-1,1,1000)
    P   = np.polyfit(x, u, N)
    uu  = np.polyval(P, xx)
    plt.plot(xx,uu,'b-')
    plt.grid(True)
    plt.title('IT= '+str(IT)+' u(0)= '+str(u[int(N/2)]))
    plt.ylim([-0.5,0.])
    plt.show()

interact(example_3,N=(2,30),IT=(0,100))
```

# Example 4: Eigenvalue BVP

Solve $u_{xx}=\lambda\,u$ with $u(-1)=u(1)=0$

```python
N_widget = IntSlider(min=2, max=50, step=1, value=10)
j_widget = IntSlider(min=1, max=49, step=1, value=5)

def update_j_range(*args):
    j_widget.max = N_widget.value-1
j_widget.observe(update_j_range, 'value')

def example_4(N=36,j=5):
    D,x = cheb(N)
    D2  = np.dot(D,D)
    D2  = D2[1:-1,1:-1]

    lam, V = np.linalg.eig(D2)

    ii=np.argsort(-np.real(lam))

    lam=lam[ii]
    V=V[:,ii]

    u = np.concatenate(([0],V[:,j-1],[0]),axis=0)

    plt.figure(figsize=(2*M,M))
    plt.plot(x,u,'k.')
    xx  = np.linspace(-1,1,1000)
    P   = np.polyfit(x, u, N)
    uu  = np.polyval(P, xx)
    plt.plot(xx,uu,'b-')
    plt.grid(True)
    plt.title('eig '+str(j)+' = '+str(lam[j-1]*4./(np.pi**2))+' pi**2/4'+'     ppw '+str(4*N/(np.pi*j)))
    plt.show()
interact(example_4,N=N_widget,j=j_widget)
```

# Example 5:  (2D) Poisson equation $u_{xx}+u_{yy}=f$ with u=0 on $\partial\Gamma$

```python
elev_widget = IntSlider(min=0, max=180, step=10, value=40)
azim_widget = IntSlider(min=0, max=360, step=10, value=230)

def example_5(N=10,elev=40,azim=230):
    D,x = cheb(N)
    y=x
    D2  = np.dot(D,D)
    D2  = D2[1:-1,1:-1]

    xx,yy=np.meshgrid(x[1:-1],y[1:-1])
    xx = xx.flatten()
    yy = yy.flatten()

    f = 10*np.sin(8*xx*(yy-1))

    I = np.eye(N-1)
    # The Laplacian
    L = np.kron(I,D2)+np.kron(D2,I)

    u = np.linalg.solve(L,f)

    fig = plt.figure(figsize=(2*M,2*M))

    # The spy of the Laplacian
    plt.subplot(221)
    plt.spy(L)

    # Plotting the approximation and its interpolation

    # The numerical approximation
    uu = np.zeros((N+1,N+1))
    uu[1:-1,1:-1]=np.reshape(u,(N-1,N-1))
    xx,yy=np.meshgrid(x,y)
    value = uu[int(N/4),int(N/4)]

    plt.subplot(222,projection='3d')
    ax = fig.gca()
    #surf = ax.plot_surface(xxx, yyy, uuu_n, rstride=1, cstride=1, cmap=cm.coolwarm,
    #                       linewidth=0, antialiased=False)
    ax.plot_wireframe(xx, yy, uu)
    ax.view_init(elev,azim)

    # The INTERPOLATED approximation

    N_fine=4*N
    finer_mesh=np.linspace(-1,1,N_fine)
    xxx,yyy=np.meshgrid(finer_mesh,finer_mesh)
    uuu = spf.interpolate.interp2d(xx, yy, uu, kind='linear')
    uuu_n=np.reshape(uuu(finer_mesh,finer_mesh),(N_fine,N_fine))

    plt.subplot(224,projection='3d')
    ax = fig.gca()
    surf = ax.plot_surface(xxx, yyy, uuu_n, rstride=1, cstride=1, cmap=cm.coolwarm,
                           linewidth=0, antialiased=False)
    #ax.plot_wireframe(xxx, yyy, uuu_n)
    fig.colorbar(surf)
    ax.view_init(elev,azim)

    plt.subplot(223)
    ax = fig.gca()
    #surf = ax.plot_surface(xxx, yyy, uuu_n, rstride=1, cstride=1, cmap=cm.coolwarm,
    #                       linewidth=0, antialiased=False)
    extent = [x[0], x[-1], y[0], y[-1]]
    plt.imshow(uu, extent=extent)
    plt.ylabel('$y$')
    plt.xlabel('$x$')
    plt.colorbar()

    plt.show()
interact(example_5,N=(3,20),elev=elev_widget,azim=azim_widget)
```

# Example 6:  (2D) Helmholtz equation $u_{xx}+u_{yy}+k^2\,u=f$ with u=0 on $\partial\Gamma$

```python
elev_widget = IntSlider(min=0, max=180, step=10, value=40)
azim_widget = IntSlider(min=0, max=360, step=10, value=230)

def example_6(N=10,elev=40,azim=230,k=9,n_contours=8):
    D,x = cheb(N)
    y=x
    D2  = np.dot(D,D)
    D2  = D2[1:-1,1:-1]

    xx,yy=np.meshgrid(x[1:-1],y[1:-1])
    xx = xx.flatten()
    yy = yy.flatten()

    f = np.exp(-10.*((yy-1.)**2+(xx-.5)**2))

    I = np.eye(N-1)
    # The Laplacian
    L = np.kron(I,D2)+np.kron(D2,I)+k**2*np.eye((N-1)**2)

    u = np.linalg.solve(L,f)

    fig = plt.figure(figsize=(2*M,2*M))

    # Plotting the approximation and its interpolation

    # The numerical approximation
    uu = np.zeros((N+1,N+1))
    uu[1:-1,1:-1]=np.reshape(u,(N-1,N-1))
    xx,yy=np.meshgrid(x,y)
    value = uu[int(N/4),int(N/4)]

    plt.subplot(221,projection='3d')
    ax = fig.gca()
    #surf = ax.plot_surface(xxx, yyy, uuu_n, rstride=1, cstride=1, cmap=cm.coolwarm,
    #                       linewidth=0, antialiased=False)
    ax.plot_wireframe(xx, yy, uu)
    ax.view_init(elev,azim)

    plt.subplot(222)
    plt.contour(xx, yy, uu, n_contours,
                 colors='k',  # negative contours will be dashed by default
                 )

    # The INTERPOLATED approximation
    N_fine=4*N
    finer_mesh=np.linspace(-1,1,N_fine)
    xxx,yyy=np.meshgrid(finer_mesh,finer_mesh)
    uuu = spf.interpolate.interp2d(xx, yy, uu, kind='linear')
    uuu_n=np.reshape(uuu(finer_mesh,finer_mesh),(N_fine,N_fine))

    plt.subplot(223,projection='3d')
    ax = fig.gca()
    #surf = ax.plot_surface(xxx, yyy, uuu_n, rstride=1, cstride=1, cmap=cm.coolwarm,
    #                       linewidth=0, antialiased=False)
    ax.plot_wireframe(xxx, yyy, uuu_n)
    ax.view_init(elev,azim)

    plt.subplot(224)
    plt.contour(xxx, yyy, uuu_n, n_contours,
                 colors='k',  # negative contours will be dashed by default
                 )

    plt.show()
interact(example_6,N=(3,30),elev=elev_widget,azim=azim_widget,k=(1,20),n_contours=(5,12))
```

# Example 7:  (2D)  $-(u_{xx}+u_{yy})=\lambda\,u$ with u=0 on $\partial\Gamma$

```python
elev_widget = IntSlider(min=0, max=180, step=10, value=40)
azim_widget = IntSlider(min=0, max=360, step=10, value=230)
N_widget = IntSlider(min=2, max=30, step=1, value=10)
j_widget = IntSlider(min=1, max=20, step=1, value=1)

def update_j_range(*args):
    j_widget.max = (N_widget.value-1)**2
j_widget.observe(update_j_range, 'value')

def example_7(N=10,elev=40,azim=230,n_contours=8,j=1):
    D,x = cheb(N)
    y=x
    D2  = np.dot(D,D)
    D2  = D2[1:-1,1:-1]

    xx,yy=np.meshgrid(x[1:-1],y[1:-1])
    xx = xx.flatten()
    yy = yy.flatten()

    I = np.eye(N-1)
    # The Laplacian
    L = (np.kron(I,-D2)+np.kron(-D2,I))

    lam, V = np.linalg.eig(L)

    ii=np.argsort(np.real(lam))
    lam=lam[ii]
    V=V[:,ii]

    fig = plt.figure(figsize=(2*M,M))

    # Plotting the approximation and its interpolation

    # The numerical approximation
    vv = np.zeros((N+1,N+1))
    vv[1:-1,1:-1]=np.reshape(np.real(V[:,j-1]),(N-1,N-1))
    xx,yy=np.meshgrid(x,y)

    plt.subplot(221,projection='3d')
    ax = fig.gca()
    #surf = ax.plot_surface(xxx, yyy, uuu_n, rstride=1, cstride=1, cmap=cm.coolwarm,
    #                       linewidth=0, antialiased=False)
    ax.plot_wireframe(xx, yy, vv)
    plt.title('eig '+str(j)+'/ (pi/2)**2= '+str(lam[j-1]/((np.pi/2)**2)))
    ax.view_init(elev,azim)

    plt.subplot(222)
    plt.contour(xx, yy, vv, n_contours,
                 colors='k',  # negative contours will be dashed by default
                 )

    # The INTERPOLATED approximation
    N_fine=4*N
    finer_mesh=np.linspace(-1,1,N_fine)
    xxx,yyy=np.meshgrid(finer_mesh,finer_mesh)
    vvv = spf.interpolate.interp2d(xx, yy, vv, kind='linear')
    vvv_n=np.reshape(vvv(finer_mesh,finer_mesh),(N_fine,N_fine))

    plt.subplot(223,projection='3d')
    ax = fig.gca()
    #surf = ax.plot_surface(xxx, yyy, uuu_n, rstride=1, cstride=1, cmap=cm.coolwarm,
    #                       linewidth=0, antialiased=False)
    ax.plot_wireframe(xxx, yyy, vvv_n)
    ax.view_init(elev,azim)

    plt.subplot(224)
    plt.contour(xxx, yyy, vvv_n, n_contours,
                 colors='k',  # negative contours will be dashed by default
                 )

    plt.show()
interact(example_7,N=N_widget,elev=elev_widget,azim=azim_widget,n_contours=(5,12),j=j_widget)
```

# In-class work

## [Flash back] Implement Program 6, 7 and 12.

## [Today] Implement Program 19, 20, 21, 22 and 23.

```python

```

SC5/04 Numerical Example of Spectral Differentiation.ipynb


```python
from logicqubit.logic import *
from cmath import *
import numpy as np
import sympy as sp
import scipy
from scipy.optimize import *
import matplotlib.pyplot as plt
```

    Cuda is not available!
    logicqubit version 1.5.8

https://arxiv.org/abs/1304.3061

https://cpb-us-w2.wpmucdn.com/voices.uchicago.edu/dist/0/2327/files/2020/10/SimultaneousMeasurementVQE.pdf1

```python
gates = Gates(1)

ID = gates.ID()
X = gates.X()
Y = gates.Y()
Z = gates.Z()
```

```python
II = ID.kron(ID)

XX = X.kron(X)
XI = X.kron(ID)
IX = ID.kron(X)
XY = X.kron(Y)
XZ = X.kron(Z)

YY = Y.kron(Y)
YI = Y.kron(ID)
IY = ID.kron(Y)
YX = Y.kron(X)
YZ = Y.kron(Z)

ZZ = Z.kron(Z)
ZI = Z.kron(ID)
IZ = ID.kron(Z)
ZX = Z.kron(X)
ZY = Z.kron(Y)

sig_is = np.kron([1, 1], [1, -1])
sig_si = np.kron([1, -1], [1, 1])
ZZ.get()
```

    array([[ 1,  0,  0,  0],
           [ 0, -1,  0,  0],
           [ 0,  0, -1,  0],
           [ 0,  0,  0,  1]])

```python
#H = XZ + ZI*2 # -ZX + 2*ZI

H = XX*3 + ZZ*7

min(scipy.linalg.eig(H.get())[0])
```

    (-10+0j)

```python
def ansatz(reg, params):
    n_qubits = len(reg)
    depth = n_qubits
    for i in range(depth):
        for j in range(n_qubits):
            if(j < n_qubits-1):
                reg[j+1].CNOT(reg[j])
            reg[i].RY(params[j])

def ansatz_2q(q1, q2, params):
    q2.CNOT(q1)
    q1.RY(params[0])
    q2.RY(params[1])
    q1.CNOT(q2)
    q1.RY(params[2])
    q2.RY(params[3])
    q2.CNOT(q1)
    q1.RY(params[4])
    q2.RY(params[5])
    q1.CNOT(q2)
```

```python
def expectation_2q(params):
    logicQuBit  = LogicQuBit(2)
    q1 = Qubit()
    q2 = Qubit()

    ansatz_2q(q1,q2,params)
    psi = logicQuBit.getPsi()

    return (psi.adjoint()*H*psi).get()[0][0]

minimum = minimize(expectation_2q, [0,0,0,0,0,0], method='Nelder-Mead', options={'xtol': 1e-10, 'ftol': 1e-10})
print(minimum)
```

    /home/cleoner/anaconda3/lib/python3.8/site-packages/scipy/optimize/optimize.py:586: ComplexWarning: Casting complex values to real discards the imaginary part
      fsim[k] = func(sim[k])
    /home/cleoner/anaconda3/lib/python3.8/site-packages/scipy/optimize/optimize.py:611: ComplexWarning: Casting complex values to real discards the imaginary part
      fsim[-1] = fxe
    /home/cleoner/anaconda3/lib/python3.8/site-packages/scipy/optimize/optimize.py:618: ComplexWarning: Casting complex values to real discards the imaginary part
      fsim[-1] = fxr
    /home/cleoner/anaconda3/lib/python3.8/site-packages/scipy/optimize/optimize.py:614: ComplexWarning: Casting complex values to real discards the imaginary part
      fsim[-1] = fxr
    /home/cleoner/anaconda3/lib/python3.8/site-packages/scipy/optimize/optimize.py:637: ComplexWarning: Casting complex values to real discards the imaginary part
      fsim[-1] = fxcc
    /home/cleoner/anaconda3/lib/python3.8/site-packages/scipy/optimize/optimize.py:627: ComplexWarning: Casting complex values to real discards the imaginary part
      fsim[-1] = fxc

     final_simplex: (array([[ 5.31847022, -0.42327127, -3.1307354 ,  1.06588762,  1.01423211,
            -3.42077752],
           [ 5.31847022, -0.42327127, -3.1307354 ,  1.06588762,  1.01423211,
            -3.42077752],
           [ 5.31847022, -0.42327127, -3.1307354 ,  1.06588762,  1.01423211,
            -3.42077752],
           [ 5.31847022, -0.42327127, -3.1307354 ,  1.06588762,  1.01423211,
            -3.42077752],
           [ 5.31847022, -0.42327127, -3.1307354 ,  1.06588762,  1.01423211,
            -3.42077752],
           [ 5.31847022, -0.42327127, -3.1307354 ,  1.06588762,  1.01423211,
            -3.42077752],
           [ 5.31847022, -0.42327127, -3.1307354 ,  1.06588762,  1.01423211,
            -3.42077752]]), array([-10., -10., -10., -10., -10., -10., -10.]))
               fun: -10.000000000000004
           message: 'Optimization terminated successfully.'
              nfev: 1065
               nit: 614
            status: 0
           success: True
                 x: array([ 5.31847022, -0.42327127, -3.1307354 ,  1.06588762,  1.01423211,
           -3.42077752])

    /home/cleoner/anaconda3/lib/python3.8/site-packages/scipy/optimize/optimize.py:644: ComplexWarning: Casting complex values to real discards the imaginary part
      fsim[j] = func(sim[j])

```python
def expectation_value(measurements, base = np.array([1,-1,-1,1])):
    probabilities = np.array(measurements)
    expectation = np.sum(base * probabilities)
    return expectation

def sigma_xx(params):
    logicQuBit  = LogicQuBit(2, first_left = False)
    q1 = Qubit()
    q2 = Qubit()

    ansatz_2q(q1,q2,params)

    # medidas em XX
    q1.RY(-pi/2)
    q2.RY(-pi/2)

    result = logicQuBit.Measure([q1,q2])
    result = expectation_value(result)
    return result

def sigma_yy(params):
    logicQuBit  = LogicQuBit(2, first_left = False)
    q1 = Qubit()
    q2 = Qubit()

    ansatz_2q(q1,q2,params)

    # medidas em YY
    q1.RX(pi/2)
    q2.RX(pi/2)

    result = logicQuBit.Measure([q1,q2])
    yy = expectation_value(result)
    iy = expectation_value(result, sig_is)
    yi = expectation_value(result, sig_si)
    return yy, iy, yi

def sigma_zz(params):
    logicQuBit  = LogicQuBit(2, first_left = False)
    q1 = Qubit()
    q2 = Qubit()

    ansatz_2q(q1,q2,params)

    result = logicQuBit.Measure([q1,q2])
    zz = expectation_value(result)
    iz = expectation_value(result, sig_is) # [zz, iz] = 0
    zi = expectation_value(result, sig_si) # [zz, zi] = 0
    return zz, iz, zi

def sigma_zx(params):
    logicQuBit  = LogicQuBit(2, first_left = False)
    q1 = Qubit()
    q2 = Qubit()

    ansatz_2q(q1,q2,params)

    # medidas em X
    #q1.RY(-pi/2)
    q2.RY(-pi/2)

    result = logicQuBit.Measure([q1,q2])
    zx = expectation_value(result)
    return zx

def sigma_xx_zz(params):
    logicQuBit  = LogicQuBit(2)
    q1 = Qubit()
    q2 = Qubit()

    ansatz_2q(q1,q2,params)

    # medida na base de bell
    q2.CX(q1)
    q1.H()    # Hx1

    result = logicQuBit.Measure([q1,q2])
    zi = expectation_value(result, sig_si)
    iz = expectation_value(result, sig_is)
    xx = zi
    zz = iz
    return xx, zz

def expectation_energy(params):
    #xx =  sigma_xx(params)
    #yy =  sigma_yy(params)

    #zz, iz, zi = sigma_zz(params)
    #zx = sigma_zx(params)
    #result = -zx + zi*2

    xx, zz = sigma_xx_zz(params)
    result = xx*3 + zz*7
    return result
```

```python
minimum = minimize(expectation_energy, [0,0,0,0,0,0], method='Nelder-Mead', options={'xtol': 1e-10, 'ftol': 1e-10})
print(minimum)
```

     final_simplex: (array([[ 2.19396352, -0.53207923,  0.91245827, -3.03402174,  0.31749551,
            -1.89342844],
           [ 2.19396352, -0.53207923,  0.91245827, -3.03402174,  0.31749551,
            -1.89342844],
           [ 2.19396352, -0.53207923,  0.91245827, -3.03402174,  0.31749551,
            -1.89342844],
           [ 2.19396352, -0.53207923,  0.91245827, -3.03402174,  0.31749551,
            -1.89342844],
           [ 2.19396352, -0.53207923,  0.91245828, -3.03402174,  0.31749551,
            -1.89342844],
           [ 2.19396352, -0.53207923,  0.91245827, -3.03402174,  0.31749551,
            -1.89342844],
           [ 2.19396352, -0.53207923,  0.91245827, -3.03402174,  0.31749551,
            -1.89342844]]), array([-10., -10., -10., -10., -10., -10., -10.]))
               fun: -10.000000000000005
           message: 'Optimization terminated successfully.'
              nfev: 932
               nit: 543
            status: 0
           success: True
                 x: array([ 2.19396352, -0.53207923,  0.91245827, -3.03402174,  0.31749551,
           -1.89342844])

```python
def gradient(params, evaluate, shift = pi/2):
    n_params = params.shape[0]
    gradients = np.zeros(n_params)

    for i in range(n_params):
        #parameter shift rule
        shift_vect = np.array([shift if j==i else 0 for j in range(n_params)])
        shift_right = params + shift_vect
        shift_left = params - shift_vect

        expectation_right = evaluate(shift_right)
        expectation_left = evaluate(shift_left)

        gradients[i] = expectation_right - expectation_left

    return gradients
```

```python
params = np.random.uniform(-np.pi, np.pi, 6)
last_params = np.zeros(6)
```

```python
lr = 0.1
err = 1
while err > 1e-3:
    grad = gradient(params, expectation_energy, pi/1.3)
    params = params - lr*grad
    err = abs(sum(params - last_params))
    last_params = np.array(params)
    print(err)
```

    2.740998722971736
    0.7664059946733903
    2.6551149987384
    1.2608214802431843
    1.6463140203823585
    0.9882522116289967
    0.7597045148619697
    0.7288509454263662
    0.48616133345178747
    0.5674605061569935
    0.38406516697773163
    0.43028564541804637
    0.29331113347599436
    0.3180409806639399
    0.22291660083314208
    0.23564204761260749

    /tmp/ipykernel_13066/2399136664.py:14: ComplexWarning: Casting complex values to real discards the imaginary part
      gradients[i] = expectation_right - expectation_left

    0.17091061131625096
    0.17690361978602842
    0.13286436551320124
    0.1350978736282606
    0.1047919107121954
    0.10491825192171336
    0.0837058159106186
    0.08266834303990159
    0.06753875093640191
    0.06589648464516769
    0.05490283477970613
    0.0529959455511186
    0.04486783316222952
    0.04290418378493158
    0.03679987111384837
    0.03490346410358293
    0.030255093195600533
    0.02849544969493395
    0.024912658187603715
    0.023323817622205006
    0.0205332435402516
    0.019126519558116256
    0.016933316866161174
    0.015705975012091533
    0.013968936519843905
    0.012910073278434617
    0.011525275889824105
    0.010619738007284787
    0.009509607723144992
    0.008740537754570302
    0.007846417010252585
    0.0071968454167906926
    0.006473860085199415
    0.005927646503809669
    0.005341106487352892
    0.004883446257502394
    0.004406284195538374
    0.004023932295785418
    0.0036348551808148
    0.0033161729228793763
    0.002998310149989969
    0.002733206203465899
    0.0024731080036946296
    0.0022529213447672614
    0.002039807784688666
    0.0018571634170972545
    0.00168235488315098
    0.0015310116825909825
    0.0013874924506602632
    0.001262194729184235
    0.0011442752984405363
    0.0010406145568409575
    0.0009436681184822188

```python
expectation_energy(params)
```

    (-9.999986367379375+0j)

```python

```


# Лаба 3

# Метод сеток решения одномерного нестационарного уравнения теплопроводности

## Необходимые импорты

```python
import numpy as np
import sympy as sp
import matplotlib.pyplot as plot
import matplotlib.patches as mpatches
import scipy.integrate as intg
from collections import namedtuple

HeatSource = namedtuple("HeatSource", "power location")
```

## Задание 1

Промоделировать стационарные процессы теплопроводности стержня в зависимости от входных данных задачи.

Стационарное уравнение теплопроводности:

$$
\begin{cases}
    -\dfrac {d}{dx}(k(x) \cdot \dfrac {du}{dx}) = f(x), \\
    u(a)=U_a, u(b)=U_b
\end{cases}
$$

Задать точечный источник тепла можно следующим образом:
$$ f(x) = c \cdot \delta(x-x_0),$$
где $c$ - мощность источника,
$\delta(x)$ - дельта  функция,
$x_0$ - точка где располагается источник.

Решить эту задачу можно использовав интегральное уравнения баланса (при $q(x)=0$ по условию) в частном виде на отрезке $[a, b]$:

$$ W(a) - W(b) + \int_{a}^{b} f(x)dx = 0, $$ где $W(x) = -k(x) \dfrac{du}{dx}$ - плотность потока тепла.

Далее посторим разностную схему:

$$ x_n = n \cdot h, n = 0, 1, ..., N $$

с промежуточными, так называемыми, потоковыми узлами:

$$ x_{n \pm 0.5} = x_n \pm 0.5h $$

Запишем уравнение баланса на промежутке $[x_{n-0.5}, x_{n+0.5}]$:

$$ W(x_{n-0.5}) - W(x_{n+0.5}) + \int_{x_{n-0.5}}^{x_{n+0.5}} f(x)dx = 0 $$

Далее нехитрыми преобразованиями получаем:

$$ W(x_{n-0.5}) = -a_n \cdot \dfrac {u_n - u_{n-1}}{h} $$

$$ a_n=h \cdot ({\int_{x_{n-1}}^{x_n} \dfrac {dx}{k(x)}})^{-1} $$

$$ W(x_{n+0.5}) = -b_n \cdot \dfrac {u_{n+1} - u_{n}}{h} $$

$$ b_n = h \cdot (\int_{x_n}^{x_{n+1}} \dfrac {dx}{k(x)})^{-1} $$

$$ \phi_n = \dfrac {1}{h} \cdot \int_{x_{n-0.5}}^{x_{n+0.5}} f(x)dx = \dfrac {1}{h} \cdot \int_{x_{n-0.5}}^{x_{n+0.5}} c \cdot \delta(x-x_0) dx $$

Подставляем в исходное уравнение и получаем:

$$
\begin{cases}
    \dfrac {1}{h} \cdot (b_n \dfrac {u_{n+1} - u_{n}}{h} - a_n \dfrac {u_n - u_{n-1}}{h}) = -\phi_n, n = 1, 2, ..., N-1, \\
    u_0=U_0, u_N=U_N
\end{cases}
$$

Преобразуем и получим:

$$
\begin{cases}
   \dfrac {1}{h^2} \cdot (-a_n u_{n-1} + (a_n + b_n)u_n - b_n u_{n+1}) = \phi_n, n = 1, 2, ..., N-1, \\
    u_0=U_0, u_N=U_N
\end{cases}
$$

Данная разностная схема аппроксимирует исходное уравнение с порядком $O(h^2)$.

```python
x_0 = 0
x_n = 1

u_0 = 0
u_n = 0

N = 151
h = (x_n - x_0) / (N - 1)
print(f"Шаг h = {h:.3}")
```

    Шаг h = 0.00667

```python
def accuracy_for(element):
    return round(element, 5)

def delta(x_a, x_b, x0):
    return 1 if abs(x_a - x0) <= h/2 else 0

def phi(c, x_a, x_b, x0):
    return c/h * delta(x_a, x_b, x0)

def k(x):
    if len(k_list) == 1:
        return k_list[0]

    if len(k_list) == 2:
        def k2(x):
            if x_0 <= x <= 1/2 * (x_n + x_0):
                return k_list[0]
            else:
                return k_list[1]
        return k2(x)

    elif len(k_list) == 3:
        def k3(x):
            if x_0 <= x <= x_0 + 1/3 * (x_n + x_0):
                return k_list[0]
            elif x_0 + 1/3 * (x_n + x_0) <= x <= x_0 + 2/3 * (x_n - x_0):
                return k_list[1]
            else:
                return k_list[2]
        return k3(x)

def kk(x):
    return 1/k(x)

def a(i, x_list):
     return h / intg.quad(kk, x_list[i - 1], x_list[i])[0]

def b(i, x_list):
    return h / intg.quad(kk, x_list[i], x_list[i + 1])[0]

def solution(x: list, heat_sources):
    matrix = np.zeros((N, N))
    vector = np.zeros(N)
    matrix[0, 0] = matrix[-1, -1] = 1
    vector[0], vector[-1] = u_0, u_n
    for i in range(1, N - 1):
        matrix[i, i - 1] = -1/h**2 * a(i, x)
        matrix[i, i] = 1/h**2 * (a(i, x) + b(i, x))
        matrix[i, i + 1] = -1/h**2 * b(i, x)
        vector[i] = sum(phi(c, x[i], x[i + 1], x0) for c, x0 in heat_sources)
    return np.linalg.solve(matrix, vector)
```

```python
k_super_list = [
    [1],
    [10, 20],
    [20, 10],
    [25, 50, 75],
    [75, 50, 25],
    [10, 20, 10],
    [20, 1, 20]
]

heat_sources = [
    [
        HeatSource(power=1, location=x_0 + (x_n - x_0) / 2)
    ],
    [
        HeatSource(power=1, location=x_0 + (x_n - x_0) / 4),
        HeatSource(power=1, location=x_n - (x_n - x_0) / 4)
    ],
    [
        HeatSource(power=2, location=x_0 + (x_n - x_0) / 4),
        HeatSource(power=5, location=x_n - (x_n - x_0) / 4)
    ],
    [
        HeatSource(power=2, location=x_0 + 0.1),
        HeatSource(power=2, location=x_n - 0.1)
    ]
]

x = [accuracy_for(x_0 + h * i) for i in range(N)]

for k_i in k_super_list:
    k_list = k_i
    for heat_source in heat_sources:
        y = solution(x, heat_source)
        patches = []
        for point in heat_source:
            plot.axvline(x=point.location, color='orange', linestyle='--')
            patches.append(mpatches.Patch(color='orange', label=fr'$x_0$={point.location:.2f}'))
            patches.append(mpatches.Patch(color='green', label=fr'$c_0$={point.power}'))
        plot.plot(x, y)
        patches.append(mpatches.Patch(label=f"k={', '.join(map(str, k_list))}"))
        plot.xlabel("x")
        plot.ylabel("U(x)")
        plot.legend(handles=patches)
        plot.show()
```

## Задание 2

Промоделировать нестационарные процессы теплопроводности для уравнения:

$$
\begin{cases}
    \dfrac {\partial u}{\partial t} = \dfrac {\partial}{\partial x} (k(x) \dfrac {\partial u}{\partial x}) + f(x) \cdot (1-e^{-t}), a < x < b, 0 < t < T, \\
    u(a, t)=U_a, u(b, t)=U_b, 0 \le t \le T, \\
    u(x, 0)=\phi(x), a \le x \le b.
\end{cases}
$$

Для решения уравнения используем явную разностную схему.

Распишем производную произведения в конечных разностях:

$$
    \dfrac {\partial}{\partial x} (k(x) \dfrac {\partial u}{\partial x}) = \dfrac {k(x+0.5h) \cdot u'(x+0.5h, t)-k(x-0.5h) \cdot u'(x-0.5h, t)} {h}
$$

Здесь:

$$ u'(x+0.5h, t) = \dfrac {u(x+h, t) - u(x, t)} {h}$$

$$u'(x-0.5h, t) = \dfrac {u(x, t) - u(x-h, t)}{h} $$

Подставим и получим:

$$\dfrac {\partial}{\partial x} (k(x) \dfrac {\partial u}{\partial x}) = \dfrac {k(x+0.5h) \cdot u(x+h, t) + k(x-0.5h) \cdot u(x-h, t) - u(x, t) \cdot (k(x+0.5h) + k(x-0.5h))} {h^2}$$

Распишем производную по времени:

$$ \dfrac {\partial u}{\partial t} = \dfrac {u(x, t+\tau) - u(x, t)} {\tau}  $$

Подставляем в исходное уравнение и получаем:

$$
    u(x, t + \tau) = \dfrac {1} {h^2} \cdot (C_1 \cdot u(x+h, t) + C_2 \cdot u(x-h, t) + C_3 \cdot u(x, t) + \tau \cdot f(x) \cdot (1 - e^{-t})
$$

Где:
$$  C_1 = \tau \cdot k(x+0.5h)$$
$$C_2 = \tau \cdot k(x-0.5h) $$
$$C_3 = (h^2 - \tau \cdot (k(x+0.5h) + k(x-0.5h)) $$

```python
def k(x):
    return 1/x**2

def f(x):
    return -2 * x**2 - 2 * x

def phi(x):
    return -200 * x**2 + 404 * x - 150
```

```python
h = 0.01
a = 0.5
b = 1.5
U_a = 2
U_b = 6
```

```python
N = int((b - a) / h) + 1
x = np.linspace(a, b, N)
```

Из условия устойчивости $ \tau < \dfrac {h^2}{2 \cdot k} $

```python
tau = h**2 / (2 * max(k(x))) / 2
T = int(0.05 / tau) + 1

print(f"\u03C4 = {tau}")
```

    τ = 6.25e-06

```python
matrix = np.zeros((T, N))
matrix[0] = phi(x)
matrix[..., 0] = U_a
matrix[..., -1] = U_b
```

```python
for j in range(T - 1):
    for i in range(1, N - 1):
        ui_plus = 1/h**2 * tau * k(x[i] + 0.5*h) * matrix[j, i + 1]
        ui_minus = 1/h**2 * tau * k(x[i] - 0.5*h) * matrix[j, i - 1]
        ui = (1 - tau/h**2 * (k(x[i] + 0.5*h) + k(x[i] - 0.5*h))) * matrix[j, i]
        matrix[j + 1, i] = ui_plus + ui_minus + ui + tau * f(x[i]) * (1 - np.exp(-(tau * j)))
```

```python
for i in range(0, T, 400):
    plot.plot(x, matrix[i])
```

## Задание 3

Решение аналлогинчо заданию 2, только $ k = const $

```python
a = -1
b = 1
k = 2
T = 0.1

def g1(t):
    return 1

def g2(t):
    return 1

def f(x, t):
    return 0

def phi(x):
    return x**2
```

```python
h = (b - a) / 10
print(f"h = {h}")
```

    h = 0.2

```python
tau = h**2 / (2 * k) / 2
print(f"\u03C4 = {tau}")
```

    τ = 0.005000000000000001

```python
Nt = int(T / tau) + 1
N = int((b - a) / h) + 1
x = np.linspace(a, b, N)
```

```python
matrix = np.zeros((Nt, N))
matrix[0] = phi(x)
matrix[..., 0] = g1(np.linspace(0, T, Nt))
matrix[..., -1] = g2(np.linspace(0, T, Nt))
```

```python
for j in range(Nt - 1):
    for i in range(1, N - 1):
        ui_plus = 1/h**2 * tau * k * matrix[j, i + 1]
        ui_minus = 1/h**2 * tau * k * matrix[j, i - 1]
        ui = (1 - tau/h**2 * 2*k) * matrix[j, i]
        matrix[j + 1, i] = ui_plus + ui_minus + ui + tau * f(x[i], tau * j)
```

```python
for i in range(0, Nt, 5):
    plot.plot(x, matrix[i])
```


Lambda School Data Science

*Unit 4, Sprint 2, Module 2*

---

# Backpropagation & Gradient Descent (Prepare)

## Learning Objectives
* Part 1: Explain the intutition behind backproprogation
* Part 2: Implement gradient descent + backpropagation on a feedforward neural network.

## Summary of Yesterday

Yesterday, we learned about some of the principal components of Neural Networks: Neurons, Weights, Activation Functions, and layers (input, output, & hidden). Today, we will reinfornce our understanding of those components and introduce the mechanics of training a neural network. Feedforward neural networks, such as multi-layer perceptrons (MLPs), are almost always trained using some variation of gradient descent where the gradient has been calculated by backpropagation.

- There are three kinds of layers: input, hidden, and output layers.
- Each layer is made up of **n** individual neurons (aka activation units) which have a corresponding weight and bias.
- Signal is passed from layer to layer through a network by:
 - Taking in inputs from the training data (or previous layer)
 - Multiplying each input by its corresponding weight (think arrow/connecting line)
 - Adding a bias to this weighted some of inputs and weights
 - Activating this weighted sum + bias by squishifying it with sigmoid or some other activation function. With a single perceptron with three inputs, calculating the output from the node is done like so:
\begin{align}
 y = sigmoid(\sum(weight_{1}input_{1} + weight_{2}input_{2} + weight_{3}input_{3}) + bias)
\end{align}
 - this final activated value is the signal that gets passed onto the next layer of the network.

## Training a Neural Network: *Formal Summary*

0. Pick a network architecture
   - No. of input units = No. of features
   - No. of output units = Number of Classes (or expected targets)
   - Select the number of hidden layers and number of neurons within each hidden layer
1. Randomly initialize weights
2. Implement forward propagation to get $h_{\theta}(x^{(i)})$ for any $x^{(i)}$
3. Implement code to compute a cost function $J(\theta)$
4. Implement backpropagation to compute partial derivatives $\frac{\delta}{\delta\theta_{jk}^{l}}{J(\theta)}$
5. Use gradient descent (or other advanced optimizer) with backpropagation to minimize $J(\theta)$ as a function of parameters $\theta\$
6. Repeat steps 2 - 5 until cost function is 'minimized' or some other stopping criteria is met. One pass over steps 2 - 5 is called an iteration or epoch.

## Calculating *"cost"*, *"loss"* or *"error"*

We've talked about how in order to evaluate a network's performance, the data is "fed forward" until predictions are obtained and then the "loss" or "error" for a given observation is ascertained by looking at what the network predicted for that observation and comparing it to what it *should* have predicted.

The error for a given observation is calculated by taking the square of the difference between the predicted value and the actual value.

We can summarize the overal quality of a network's predictions by finding the average error across all observations. This gives us the "Mean Squared Error." which hopefully is a fairly familiar model evaluation metric by now. Graphing the MSE over each epoch (training cycle) is a common practice with Neural Networks. This is what you're seeing in the top right corner of the Tensorflow Playground website as the number of "epochs" climbs higher and higher.

## What is an "Epoch"?

An "Epoch" is one cycle of passing our data forward through the network, measuring error given our specified cost function, and then -via gradient descent- updating weights within our network to hopefully improve the quality of our predictions on the next iteration.

### Batch vs Minibatch vs Stochastic Gradient Descent Epochs

You may have heard these variations on the training process referenced in the 3Blue1Brown videos about backpropagation. "Minibatch" Gradient Descent means that instead of passing all of our data through the network for a given epoch (Batch GD), we just pass a randomized portion of our data through the network for each epoch.

Stochastic Gradient Descent is when we make updates to our weights after forward propagating each individual training observation.

### A note about Hyperparameters

Neural Networks have many more hyperparameters than other machine learning algorithms which is part of what makes them a beast to train.

1. You need more data to train them on.
2. They're complex so they take longer to train.
3. They have lots and lots of hyperparameters which we need to find the most optimal combination of, so we might end up training our model dozens or hundreds of times with different combinations of hyperparameters in order to try and squeeze out a few more tenths of a percent of accuracy.

# Backpropagation (Learn)

## Overview

Backpropagation is short for ["Backwards Propagation of errors"](https://en.wikipedia.org/wiki/Backpropagation) and refers to a specific (rather calculus intensive) algorithm for how weights in a neural network are updated in reverse order at the end of each training epoch. Our purpose today is to demonstrate the backpropagation algorithm on a simple Feedforward Neural Network and in so doing help you get a grasp on the main process. If you want to understand all of the underlying calculus of how the gradients are calculated then you'll need to dive into it yourself, [3Blue1Brown's video is a great starting place](https://www.youtube.com/watch?v=tIeHLnjs5U8). I also highly recommend this Welch Labs series [Neural Networks Demystified](https://www.youtube.com/watch?v=bxe2T-V8XRs) if you want a rapid yet orderly walkthrough of the main intuitions and math behind the backpropagation algorithm.

### What is a Gradient?

> In vector calculus, the gradient is a multi-variable generalization of the derivative.

The gradients that we will deal with today will be vector representations of the derivative of the activation function.

## Follow Along

In this section, we will again implement a multi-layer perceptron using numpy. We'll focus on using a __Feed Forward Neural Network__ to predict test scores.

### Generate some Fake Data

```python
import numpy as np

np.random.seed(812)

# hours studying, hours sleep
X = np.array(([2,9],
              [1,5],
              [3,6]), dtype=float)

# Exam Scores
y = np.array(([92],
              [86],
              [89]), dtype=float)
```

### Feature Normalization

```python
# Normalizing Data on feature
# Neural Network would probably do this on its own, but it will help us converge on a solution faster
X = X / np.amax(X, axis=0)
y = y / 100

print("Studying, Sleeping \n", X)
print("Test Score \n", y)
```

    Studying, Sleeping
     [[0.66666667 1.        ]
     [0.33333333 0.55555556]
     [1.         0.66666667]]
    Test Score
     [[0.92]
     [0.86]
     [0.89]]

### Neural Network Architecture
Lets create a Neural_Network class to contain this functionality

```python
class NeuralNetwork:
    def __init__(self):
        # Set up Architecture of Neural Network
        self.inputs = 2
        self.hiddenNodes = 3
        self.outputNodes = 1

        # Initial Weights
        # 2x3 Matrix Array for the First Layer
        self.weights1 = np.random.rand(self.inputs, self.hiddenNodes)

        # 3x1 Matrix Array for Hidden to Output
        self.weights2 = np.random.rand(self.hiddenNodes, self.outputNodes)
```

### Randomly Initialize Weights
How many random weights do we need to initialize? "Fully-connected Layers"

```python
nn = NeuralNetwork()
```

```python
print("Layer 1 wieghts: \n", nn.weights1)
print("Layer 2 wieghts: \n", nn.weights2)
```

    Layer 1 wieghts:
     [[0.5049808  0.60592761 0.45748719]
     [0.32659171 0.59345002 0.25569456]]
    Layer 2 wieghts:
     [[0.23870931]
     [0.95553049]
     [0.95889787]]

### Implement Feedforward Functionality

After this step our neural network should be able to generate an output even though it has not been trained.

```python
class NeuralNetwork:
    def __init__(self):
        # Set up Architecture of Neural Network
        self.inputs = 2
        self.hiddenNodes = 3
        self.outputNodes = 1

        # Initial Weights
        # 2x3 Matrix Array for the First Layer
        self.weights1 = np.random.rand(self.inputs, self.hiddenNodes)

        # 3x1 Matrix Array for Hidden to Output
        self.weights2 = np.random.rand(self.hiddenNodes, self.outputNodes)

    def sigmoid(self, s):
        return 1 / (1+np.exp(-s))

    def feed_forward(self, X):
        """
        Calculate the NN inference using feed forward.
        aka "predict"
        """

        # Weighted sum of inputs => hidden layer
        self.hidden_sum = np.dot(X, self.weights1)

        # Activations of weighted sum
        self.activated_hidden = self.sigmoid(self.hidden_sum)

        # Weight sum between hidden and output
        self.output_sum = np.dot(self.activated_hidden, self.weights2)

        # Final activation of output
        self.activated_output = self.sigmoid(self.output_sum)

        return self.activated_output

```

### Make a Prediction

```python
# Try to make a prediction with our updated 'net

nn = NeuralNetwork()
output = nn.feed_forward(X[0])
print("intput", X[0])
print("output", output)
```

    intput [0.66666667 1.        ]
    output [0.81245642]

### Calculate Error

```python
error = y[0] - output
error
```

    array([0.10754358])

```python
output_all = nn.feed_forward(X)
error_all = y - output_all
print(output_all)
print(error_all)
```

    [[0.81245642]
     [0.78438969]
     [0.81529149]]
    [[0.10754358]
     [0.07561031]
     [0.07470851]]

Why is my error so big?

My error is so big because my prediction is low.

Why are my prediction low?

Because either:

  1) Second layer **weights** are low

  (or)

  2) Activations coming from the first layer are low

How are activations from the first layer determined?

  1) By inputs - fixed

  2) by **weights** - variable

The only thing that I have control over throughout this process in order to increase the value of my final predictions is to either increase weights in layer 2 or increase weights in layer 1.

Imagine that you could only change your weights by a fixed amount. Say you have .3 and you have to split that up and disperse it over your weights so as to increase your predictions as much as possible. (This isn't actually what happens, but it will help us identify which weights we would benefit the most from moving.)

I need to increase weights of my model somewhere, I'll get the biggest bang for my buck if I increase weights in places where I'm already seeing high activation values -because they end up getting multiplied together before being passed to the sigmoid function.

> "Neurons that fire together, wire together"

### Implement Backpropagation

> *Assigning blame for bad predictions and delivering justice - repeatedly and a little bit at a time*

What in our model could be causing our predictions to suck so bad?

Well, we know that our inputs (X) and outputs (y) are correct, if they weren't then we would have bigger problems than understanding backpropagation.

We also know that our activation function (sigmoid) is working correctly. It can't be blamed because it just does whatever we tell it to and transforms the data in a known way.

So what are the potential culprits for these terrible predictions? The **weights** of our model. Here's the problem though. I have weights that exist in both layers of my model. How do I know if the weights in the first layer are to blame, or the second layer, or both?

Lets investigate. And see if we can just eyeball what should be updated.

```python
attributes = ['weights1', 'hidden_sum', 'activated_hidden', 'weights2', 'activated_output']
[print(i+'\n', getattr(nn,i), '\n'+'---'*3) for i in attributes if i[:2]!= '__']
```

    weights1
     [[0.5049808  0.60592761 0.45748719]
     [0.32659171 0.59345002 0.25569456]]
    ---------
    hidden_sum
     [[0.66324558 0.99740176 0.56068602]
     [0.34976677 0.53167033 0.29454826]
     [0.72270861 1.00156095 0.62795023]]
    ---------
    activated_hidden
     [[0.65998908 0.73054743 0.63661126]
     [0.58656102 0.6298726  0.57310926]
     [0.67320319 0.73136537 0.65202454]]
    ---------
    weights2
     [[0.23870931]
     [0.95553049]
     [0.95889787]]
    ---------
    activated_output
     [[0.81245642]
     [0.78438969]
     [0.81529149]]
    ---------

    [None, None, None, None, None]

### Backpropagation by Hand (Not Recommended)

Our model has 9 total weights (6 in the first layer, 3 in the last layer) that could be off.

1) Calculate Error for a given each observation

2) Does the error indicate that I'm overestimating or underestimating in my prediction?

3) Look at final layer weights to get an idea for which weights are helping pass desireable signals and which are stifling desireable signals

4) Also go to the previous layer and see what can be done to boost activations that are associated with helpful weights, and limit activations that are associated with unhelpful weights.

```python
# We want activated that correspond to negative weights to be lower
# And we want more higher activation for positivie weights
```

### Update Weights Based on Gradient

Repeat steps 1-4 for every observation in a given batch, and then given the network's cost function, calculate its gradient using calculus and update weights associated with the (negative) gradient of the cost function.

Remember that we have 9 weights in our network therefore the gradient that comes from our gradient descent calculation will be the vector that takes us in the most downward direction along some function in 9-dimensional hyperspace.

\begin{align}
C(w1, w2, w3, w4, w5, w6, w7, w8, w9)
\end{align}

You should also know that with neural networks it is common to have gradients that are not convex (like what we saw when we applied gradient descent to linear regression). Due to the high complexity of these models and their nonlinearity, it is common for gradient descent to get stuck in a local minimum, but there are ways to combat this:

1) Stochastic Gradient Descent

2) More advanced Gradient-Descent-based "Optimizers" - See Stretch Goals on assignment.

```python
# I want activations that correspond to negative weights to be lower
# and activations that correspond to positive weights to be higher

class NeuralNetwork:
    def __init__(self):
        # Set up Architecture of Neural Network
        self.inputs = 2
        self.hiddenNodes = 3
        self.outputNodes = 1

        # Initial Weights
        # 2x3 Matrix Array for the First Layer
        self.weights1 = np.random.rand(self.inputs, self.hiddenNodes)

        # 3x1 Matrix Array for Hidden to Output
        self.weights2 = np.random.rand(self.hiddenNodes, self.outputNodes)

    def sigmoid(self, s):
        return 1 / (1+np.exp(-s))

    def sigmoidPrime(self, s):
        return s * (1 - s)

    def feed_forward(self, X):
        """
        Calculate the NN inference using feed forward.
        aka "predict"
        """

        # Weighted sum of inputs => hidden layer
        self.hidden_sum = np.dot(X, self.weights1)

        # Activations of weighted sum
        self.activated_hidden = self.sigmoid(self.hidden_sum)

        # Weight sum between hidden and output
        self.output_sum = np.dot(self.activated_hidden, self.weights2)

        # Final activation of output
        self.activated_output = self.sigmoid(self.output_sum)

        return self.activated_output

    def backward(self, X,y,o):
        """
        Backward propagate through the network
        """

        # Error in Output
        self.o_error = y - o

        # Apply Derivative of Sigmoid to error
        # How far off are we in relation to the Sigmoid f(x) of the output
        # ^- aka hidden => output
        self.o_delta = self.o_error * self.sigmoidPrime(o)

        # z2 error
        self.z2_error = self.o_delta.dot(self.weights2.T)

        # How much of that "far off" can explained by the input => hidden
        self.z2_delta = self.z2_error * self.sigmoidPrime(self.activated_hidden)

        # Adjustment to first set of weights (input => hidden)
        self.weights1 += X.T.dot(self.z2_delta)
        # Adjustment to second set of weights (hidden => output)
        self.weights2 += self.activated_hidden.T.dot(self.o_delta)

    def train(self, X, y):
        o = self.feed_forward(X)
        self.backward(X,y,o)
```

#### Let's look at the shape of the Gradient Componets

```python
nn = NeuralNetwork()

nn.train(X,y)
```

##### Our Error Associated with Each Observation
aka how wrong were we?

```python
nn.o_error
```

    array([[0.12894158],
           [0.1009682 ],
           [0.09106923]])

##### 1st Gradient
Simple interpretation - how much more sigmoid activation would have pushed us towards the right answer?

```python
nn.o_delta
```

    array([[0.02131211],
           [0.01846734],
           [0.0146294 ]])

##### 2nd Error
Justice hasn't been served yet - tho. We still have neurons to blame. Let's go back another layer.

__Discussion:__ Why is this shape different?

```python
nn.z2_error
```

    array([[0.00923415, 0.01205355, 0.01769813],
           [0.00800156, 0.01044462, 0.01533576],
           [0.00633865, 0.00827399, 0.01214864]])

##### 2nd Gradient
For each observation, how much more sigmoid activation from this layer would have pushed us towards the right answer?

```python
nn.z2_delta
```

    array([[0.00145207, 0.00254479, 0.0036546 ],
           [0.00173798, 0.00249254, 0.00363744],
           [0.00103473, 0.00161844, 0.00229352]])

##### Descent

*Discussion:* Input to Hidden Weight Update
- We multiply the gradient by the inputs. Why?
- Why do we need to transpose the inputs?

```python
X.T.dot(nn.z2_delta)
```

    array([[0.0025821 , 0.00414581, 0.0059424 ],
           [0.00310744, 0.00500849, 0.00720441]])

*Discussion:* Hidden to Output Weight Update
- Why is output the shape 3x1?
- We multiply the gradient by the inputs. Why?
- Why do we need to transpose the inputs?

```python
nn.activated_hidden.T.dot(nn.o_delta)
```

    array([[0.04134839],
           [0.03678658],
           [0.03735949]])

### Train the Network (fo real this time)

```python
# Train my 'net
nn = NeuralNetwork()

# Number of Epochs / Iterations
for i in range(10000):
    if (i+1 in [1,2,3,4,5]) or ((i+1) % 1000 ==0):
        print('+' + '---' * 3 + f'EPOCH {i+1}' + '---'*3 + '+')
        print('Input: \n', X)
        print('Actual Output: \n', y)
        print('Predicted Output: \n', str(nn.feed_forward(X)))
        print("Loss: \n", str(np.mean(np.square(y - nn.feed_forward(X)))))
    nn.train(X,y)
```

    +---------EPOCH 1---------+
    Input:
     [[0.66666667 1.        ]
     [0.33333333 0.55555556]
     [1.         0.66666667]]
    Actual Output:
     [[0.92]
     [0.86]
     [0.89]]
    Predicted Output:
     [[0.71543321]
     [0.6919486 ]
     [0.71585904]]
    Loss:
     0.033471307436290655
    +---------EPOCH 2---------+
    Input:
     [[0.66666667 1.        ]
     [0.33333333 0.55555556]
     [1.         0.66666667]]
    Actual Output:
     [[0.92]
     [0.86]
     [0.89]]
    Predicted Output:
     [[0.74804112]
     [0.72168355]
     [0.74881477]]
    Loss:
     0.02287818856456615
    +---------EPOCH 3---------+
    Input:
     [[0.66666667 1.        ]
     [0.33333333 0.55555556]
     [1.         0.66666667]]
    Actual Output:
     [[0.92]
     [0.86]
     [0.89]]
    Predicted Output:
     [[0.77166203]
     [0.74351875]
     [0.77264889]]
    Loss:
     0.01644777248478282
    +---------EPOCH 4---------+
    Input:
     [[0.66666667 1.        ]
     [0.33333333 0.55555556]
     [1.         0.66666667]]
    Actual Output:
     [[0.92]
     [0.86]
     [0.89]]
    Predicted Output:
     [[0.78950105]
     [0.76020747]
     [0.79062316]]
    Loss:
     0.012288094320433563
    +---------EPOCH 5---------+
    Input:
     [[0.66666667 1.        ]
     [0.33333333 0.55555556]
     [1.         0.66666667]]
    Actual Output:
     [[0.92]
     [0.86]
     [0.89]]
    Predicted Output:
     [[0.80343568]
     [0.77338065]
     [0.80464591]]
    Loss:
     0.00945849116382972
    +---------EPOCH 1000---------+
    Input:
     [[0.66666667 1.        ]
     [0.33333333 0.55555556]
     [1.         0.66666667]]
    Actual Output:
     [[0.92]
     [0.86]
     [0.89]]
    Predicted Output:
     [[0.90056238]
     [0.86920438]
     [0.89950773]]
    Loss:
     0.00018431280905304145
    +---------EPOCH 2000---------+
    Input:
     [[0.66666667 1.        ]
     [0.33333333 0.55555556]
     [1.         0.66666667]]
    Actual Output:
     [[0.92]
     [0.86]
     [0.89]]
    Predicted Output:
     [[0.90135083]
     [0.86936811]
     [0.89839871]]
    Loss:
     0.0001686970889116981
    +---------EPOCH 3000---------+
    Input:
     [[0.66666667 1.        ]
     [0.33333333 0.55555556]
     [1.         0.66666667]]
    Actual Output:
     [[0.92]
     [0.86]
     [0.89]]
    Predicted Output:
     [[0.90197396]
     [0.869568  ]
     [0.8974644 ]]
    Loss:
     0.0001574007581152533
    +---------EPOCH 4000---------+
    Input:
     [[0.66666667 1.        ]
     [0.33333333 0.55555556]
     [1.         0.66666667]]
    Actual Output:
     [[0.92]
     [0.86]
     [0.89]]
    Predicted Output:
     [[0.90248053]
     [0.8697795 ]
     [0.89666494]]
    Loss:
     0.0001489972452723368
    +---------EPOCH 5000---------+
    Input:
     [[0.66666667 1.        ]
     [0.33333333 0.55555556]
     [1.         0.66666667]]
    Actual Output:
     [[0.92]
     [0.86]
     [0.89]]
    Predicted Output:
     [[0.90290085]
     [0.869989  ]
     [0.89597438]]
    Loss:
     0.0001426180578336025
    +---------EPOCH 6000---------+
    Input:
     [[0.66666667 1.        ]
     [0.33333333 0.55555556]
     [1.         0.66666667]]
    Actual Output:
     [[0.92]
     [0.86]
     [0.89]]
    Predicted Output:
     [[0.90325469]
     [0.87018892]
     [0.8953747 ]]
    Loss:
     0.00013770225181712454
    +---------EPOCH 7000---------+
    Input:
     [[0.66666667 1.        ]
     [0.33333333 0.55555556]
     [1.         0.66666667]]
    Actual Output:
     [[0.92]
     [0.86]
     [0.89]]
    Predicted Output:
     [[0.90355559]
     [0.8703752 ]
     [0.8948525 ]]
    Loss:
     0.00013387002357282302
    +---------EPOCH 8000---------+
    Input:
     [[0.66666667 1.        ]
     [0.33333333 0.55555556]
     [1.         0.66666667]]
    Actual Output:
     [[0.92]
     [0.86]
     [0.89]]
    Predicted Output:
     [[0.90381323]
     [0.87054593]
     [0.89439724]]
    Loss:
     0.00013085453274904526
    +---------EPOCH 9000---------+
    Input:
     [[0.66666667 1.        ]
     [0.33333333 0.55555556]
     [1.         0.66666667]]
    Actual Output:
     [[0.92]
     [0.86]
     [0.89]]
    Predicted Output:
     [[0.90403488]
     [0.87070057]
     [0.89400019]]
    Loss:
     0.00012846293590844602
    +---------EPOCH 10000---------+
    Input:
     [[0.66666667 1.        ]
     [0.33333333 0.55555556]
     [1.         0.66666667]]
    Actual Output:
     [[0.92]
     [0.86]
     [0.89]]
    Predicted Output:
     [[0.90422616]
     [0.87083944]
     [0.89365391]]
    Loss:
     0.0001265528979812011

## Challenge

In the module project, you will implement backpropagation inside a multi-layer perceptron (aka a feedforward neural network).

# Stochastic Gradient Descent (Learn)

## Overview

The What - Stochastic Gradient Descent calculates an approximation of the gradient over the entire dataset by reviewing the predictions of a random sample.

The Why - *Speed*. Calculating the gradient over the entire dataset is extremely expensive computationally.

## Follow Along

A true Stochastic GD-based implementation from [Welch Labs](https://www.youtube.com/watch?v=bxe2T-V8XRs)

```python
class Neural_Network(object):
    def __init__(self):
        #Define Hyperparameters
        self.inputLayerSize = 2
        self.outputLayerSize = 1
        self.hiddenLayerSize = 3

        #Weights (parameters)
        self.W1 = np.random.randn(self.inputLayerSize,self.hiddenLayerSize)
        self.W2 = np.random.randn(self.hiddenLayerSize,self.outputLayerSize)

    def forward(self, X):
        #Propogate inputs though network
        self.z2 = np.dot(X, self.W1)
        self.a2 = self.sigmoid(self.z2)
        self.z3 = np.dot(self.a2, self.W2)
        yHat = self.sigmoid(self.z3)
        return yHat

    def sigmoid(self, z):
        #Apply sigmoid activation function to scalar, vector, or matrix
        return 1/(1+np.exp(-z))

    def sigmoidPrime(self,z):
        #Gradient of sigmoid
        return np.exp(-z)/((1+np.exp(-z))**2)

    def costFunction(self, X, y):
        #Compute cost for given X,y, use weights already stored in class.
        self.yHat = self.forward(X)
        J = 0.5*sum((y-self.yHat)**2)
        return J

    def costFunctionPrime(self, X, y):
        #Compute derivative with respect to W and W2 for a given X and y:
        self.yHat = self.forward(X)

        delta3 = np.multiply(-(y-self.yHat), self.sigmoidPrime(self.z3))
        dJdW2 = np.dot(self.a2.T, delta3)

        delta2 = np.dot(delta3, self.W2.T)*self.sigmoidPrime(self.z2)
        dJdW1 = np.dot(X.T, delta2)

        return dJdW1, dJdW2

    #Helper Functions for interacting with other classes:
    def getParams(self):
        #Get W1 and W2 unrolled into vector:
        params = np.concatenate((self.W1.ravel(), self.W2.ravel()))
        return params

    def setParams(self, params):
        #Set W1 and W2 using single paramater vector.
        W1_start = 0
        W1_end = self.hiddenLayerSize * self.inputLayerSize
        self.W1 = np.reshape(params[W1_start:W1_end], (self.inputLayerSize , self.hiddenLayerSize))
        W2_end = W1_end + self.hiddenLayerSize*self.outputLayerSize
        self.W2 = np.reshape(params[W1_end:W2_end], (self.hiddenLayerSize, self.outputLayerSize))

    def computeGradients(self, X, y):
        dJdW1, dJdW2 = self.costFunctionPrime(X, y)
        return np.concatenate((dJdW1.ravel(), dJdW2.ravel()))
```

```python
from scipy import optimize
class trainer(object):
    def __init__(self, N):
        #Make Local reference to network:
        self.N = N

    def callbackF(self, params):
        self.N.setParams(params)
        self.J.append(self.N.costFunction(self.X, self.y))

    def costFunctionWrapper(self, params, X, y):
        self.N.setParams(params)
        cost = self.N.costFunction(X, y)
        grad = self.N.computeGradients(X,y)

        return cost, grad

    def train(self, X, y):
        #Make an internal variable for the callback function:
        self.X = X
        self.y = y

        #Make empty list to store costs:
        self.J = []

        params0 = self.N.getParams()

        options = {'maxiter': 200, 'disp' : True}
        _res = optimize.minimize(self.costFunctionWrapper, params0, jac=True, method='BFGS', \
                                 args=(X, y), options=options, callback=self.callbackF)

        self.N.setParams(_res.x)
        self.optimizationResults = _res
```

```python
NN = Neural_Network()
```

```python
T = trainer(NN)
```

```python
T.train(X,y)
```

    Optimization terminated successfully.
             Current function value: 0.000177
             Iterations: 50
             Function evaluations: 54
             Gradient evaluations: 54

```python
print("Predicted Output: \n" + str(NN.forward(X)))
print("Loss: \n" + str(np.mean(np.square(y - NN.forward(X))))) # mean sum squared loss
```

    Predicted Output:
    [[0.9057561 ]
     [0.87225846]
     [0.89058644]]
    Loss:
    0.00011783410464925031

```python
import matplotlib.pyplot as plt
plt.plot(T.J)
plt.xlabel('Iterations')
plt.ylabel('Cost')
plt.show()
```

## Challenge

This is a reference implementation for you to explore. You will not be expected to apply it to today's module project.