```python
import numpy as np
import sympy
import matplotlib.pyplot as plt
from scipy import integrate
from scipy.special import legendre
from numpy.polynomial.legendre import Legendre
import matplotlib.ticker as mtick
import itertools

# Legendre polynomial
def leg(n, x):
    return Legendre(np.concatenate((np.zeros(n), np.array([1]))))(x)

def hbasis(i,x):
#  Evaluates the function Ni at x
    if i==0:
        Ni=0.5 *(1-x)
    elif i==1:
        Ni=0.5 *(1+x)
    else:
        Ni=(np.sqrt(1/(4*(i+1)-6)))*(leg(i,x)-leg(i-2,x))
    return Ni

def stifness_matrix(p):
#evaluates the elemental stifness matrix of size (p+1)x(p+1)
    K=np.zeros((p+1,p+1))
    K[0,0]=K[1,1]=0.5
    K[0,1]=K[1,0]=-0.5
    if p>=1:
        for i in range(2,p+1):
            K[i,i]=1
    return K

def stifness_matrix(p):
#evaluates the elemental stifness matrix of size (p+1)x(p+1)
    K=np.zeros((p+1,p+1))
    K[0,0]=K[1,1]=0.5
    K[0,1]=K[1,0]=-0.5
    if p>=1:
        for i in range(2,p+1):
            K[i,i]=1
    return K

def mass_matrix(p):
# Evaluates the elemental mass matrix of size (p+1)x(p+1)
    G=np.zeros((p+1,p+1))
    G[0,0]=G[1,1]=2/3
    G[0,1]=G[1,0]=1/3
    if p>=2:
        G[0,2]=G[1,2]=G[2,0]=G[2,1]=-1/np.sqrt(6)
        for i in range(2,p+1):
            G[i,i]=2/((2*(i+1)-1)*((2*(i+1)-5)))
    if p>=3:
        G[0,3]=G[3,0]=1/3*np.sqrt(10)
        G[1,3]=G[3,1]=-1/3*np.sqrt(10)
        for i in range(2,p+1):
            if i+2<p+1:
                G[i,i+2]=G[i+2,i]=(-1)/(((2*(i+1)-1)*np.sqrt(((2*(i+1)-3)*((2*(i+1)+1))))))
    return G

def load_vector(x_k,x_kk,p_k,f):
#Evaluates the elemental load vector
    vals=[]
    for i in range(p_k+1):
        g = lambda t : f((1-t)*x_k/2+(1+t)*x_kk/2)*hbasis(i,t)
        vals.append(integrate.quad(g,-1,1)[0])
    b=np.array(vals)
    return b

def el_stiff(x_k,x_kk,p_k):
    h_k=x_kk-x_k
    Kk=(2/h_k)*stifness_matrix(p_k)
    return Kk

def el_mass(x_k,x_kk,p_k):
    h_k=x_kk-x_k
    Gk=(h_k/2)*mass_matrix(p_k)
    return Gk

def pointer(M,p):
# [P] = pointer(M,p)
#
# Calculates the pointer matrix P, such that
# P(i,j)=k means that over the ith element
# the jth local basis function corresponds
# to the kth global basis function.
#
# M is the number of elements
# p is the degree vector of size M
# P is M by (max(p)+1)
#
    pmax=max(p)
    P=np.zeros((M,pmax+1))
    P=P.astype(int)
    for i in range(M):
        P[i,0] = i
        P[i,1] = i+1
    P[M-1,1]=0
    for i in range(M):
        for j in range(2,1+p[i]):
            P[i,j] = M
            M=M+1
    return P

def global_matrix(x,p):
    #Evaluates the elemental matrices, stiffnes(Kk) and mass(Gk) and then the global matrix (G+K)
    P=pointer(len(x)-1,p)
    K=np.zeros((sum(p)-1,sum(p)-1))
    G=np.zeros((sum(p)-1,sum(p)-1))
    for k in range(len(x)-1):
        Kk=np.zeros((p[k]+1,p[k]+1))
        Gk=np.zeros((p[k]+1,p[k]+1))
        Kk=el_stiff(x[k],x[k+1],p[k])
        Gk=el_mass(x[k],x[k+1],p[k])
        for i in range(p[k]+1):
            l=P[k,i]
            for j in range(p[k]+1):
                m=P[k,j]
                if l!=0 and m!=0:
                    K[l-1,m-1]=K[l-1,m-1]+Kk[i,j]
                    G[l-1,m-1]=G[l-1,m-1]+Gk[i,j]
    GL=K+G
    return GL

def el_load(x_k,x_kk,p_k,f):
#evaluates the elemental load vector
    h_k=x_kk-x_k
    Fk=(h_k/2)*load_vector(x_k,x_kk,p_k,f)
    return Fk

def global_load_vector(x,p,f):
#Evaluates the global load vector
    P=pointer(len(x)-1,p)
    F=np.zeros((sum(p)-1))
    for k in range(len(x)-1):
        Fk=el_load(x[k],x[k+1],p[k],f)
        for i in range(p[k]+1):
            l=P[k,i]
            if l!=0:
                F[l-1]=F[l-1]+Fk[i]
    return F

def fem_solution(x,p,f):
# Evaluates the finite element method solution and the coifficients that generate the solution
    A=global_matrix(x,p)
    b=global_load_vector(x,p,f)
    a=np.linalg.solve(A,b)
    return a

def energy_norm(x,p,f,n):
# Evaluates the energy norm of the finite element method solution and it finds
# the percentage of the error
    a=fem_solution(x,p,f)
    DOF=len(a)
    enorm = np.dot(a, np.array(global_load_vector(x,p,f)))
    y = lambda z : (z**n-z)*f(z)
    enorm_uex=integrate.quad(y,0,1)[0]
    relE=100*np.sqrt(abs(enorm-enorm_uex)/abs(enorm_uex))
    return enorm,relE,DOF

def pairwise(iterable):
    a, b = itertools.tee(iterable)
    next(b, None)
    return zip(a,b)

def solid1d(y,x,M,p,f):
#Evaluates the finite element method solution for every y that belongs to [a,b] interval
    x_intervals = list(pairwise(x))
    k = [0]
    for point in y[1:]:
        for i, interval in enumerate(x_intervals[k[-1]:]):
            if point>=interval[0] and point<interval[1]:
                k.append(k[-1]+i)
                break
    k.append(len(x_intervals)-1)
    ksi=[]
    for k_val, y_i in zip(k,y):
        ksi.append((2*y_i-x_intervals[k_val][0]-x_intervals[k_val][1])/(x_intervals[k_val][1]-x_intervals[k_val][0]))
    P=pointer(M,p)
    ufe=[]
    c=fem_solution(x,p,f)
    for k_val, ksi_i in zip(k,ksi):
        proxy=[]
        for i in range(1,p[k_val]+2):
            l=P[k_val,i-1]
            if l!=0:
                proxy.append(c[l-1]*hbasis(i-1,ksi_i))
            else:
                proxy.append(0)
        ufe.append(sum(proxy))
    return ufe

def h_unif(M,pmax,a,b):
#It creates a list of the uniform mesh and a list with the degrees of the polynomials which are constant.
    x=[]
    p=[]
    h=(b-a)/M
    for i in range(M+1):
        z=a+i*h
        x.append(z)
    for j in range(M):
        p.append(pmax)
    return x,p

def h_radical(M,pmax,s,a,b):
#It creates a list of the root-s mesh and a list with the degrees of the polynomials which are constant.
    p=[]
    x=[]
    for i in range(M+1):
        z=a+(b-a)*((i/M)**s)
        x.append(z)
    for j in range(M):
        p.append(pmax)
    return x,p

def p_unif(M,pmax,a,b):
#It creates a list of the uniform mesh which is constant and a list with the degrees of the polynomials that goes 1:pmax.
    p=[]
    x=[]
    h=(b-a)/M
    for i in range(M+1):
        z=a+i*h
        x.append(z)
    for j in range(1,pmax+1):
        p.append(j)
    return x,p

def p_geom(M,pmax,q,a,b):
#It creates a list of the geometric-q mesh which is constant and a list with the degrees of the polynomials that goes 1:pmax.
    p=[]
    x=[a]
    for i in range(2,M+2):
        z=a+(b-a)*(q**(M-i+1))
        x.append(z)
    for j in range(1,pmax+1):
        p.append(j)
    return x,p

def hp_geom(M,q,a,b):
#It creates a list of the geometric-q mesh and a list with the degrees of the polynomials that goes 1:M, M:=number of elements.
    x=[a]
    p=[]
    pmax=M
    for i in range(2,M+2):
        z=a+(b-a)*(q**(M-i+1))
        x.append(z)
    for j in range(1,pmax+1):
        p.append(j)
    return x,p

```

```python
#f(x)=x**n-x
#n=7.1,a=0,b=1
#Graphs the error against DOF  in logaritmic axes
#First it uses h fem with uniform mesh for polynomial degrees 1 and then 2
a=0
b=1
n=7.1
N=5
f = lambda x : (x**n)-x-n*(n-1)*(x**(n-2))
fig, axes = plt.subplots(1,4, figsize=(15,7))
fig.tight_layout()
fig.subplots_adjust(wspace=0.3)
for j in range(1,3):
    pmax=j
    errors=[]
    DOF=[]
    for i in range(1,N+1):
        M=2**i
        x=h_unif(M,pmax,a,b)[0]
        p=h_unif(M,pmax,a,b)[1]
        errors.append(energy_norm(x,p,f,n)[1])
        DOF.append(energy_norm(x,p,f,n)[2])
    axes[j-1].loglog(DOF,errors)
    axes[j-1].set_xlabel("Degrees of freedom")
    axes[j-1].set_ylabel("Error in energy norm(%)")
    axes[j-1].set_title(f"Error for l=7.1, h_unif with N=5,p={j}")
    print(f"the slope is:",(np.log(errors[-1])-np.log(errors[-2]))/(np.log(DOF[-1])-np.log(DOF[-2])))

#Second it uses p fem with uniform mesh with 1 element and then two elements for polynomial degrees 1,....,6
P_uni_list=[1,4]
for j,Mp in enumerate(P_uni_list):
    pmax=6
    errors=[]
    DOF=[]
    x=p_unif(Mp,pmax,a,b)[0]
    p=p_unif(Mp,pmax,a,b)[1]
    for i in p:
        p_list=[]
        for k in range(Mp):
            p_list.append(i)
        errors.append(energy_norm(x,p_list,f,n)[1])
        DOF.append(energy_norm(x,p_list,f,n)[2])
    axes[j+2].loglog(DOF,errors)
    axes[j+2].set_xlabel("Degrees of freedom")
    axes[j+2].set_ylabel("Error in energy norm(%)")
    axes[j+2].set_title(f"Error for l=7.1,p_unif with M={Mp},pmax=6")
    print(f"the slope is:",(np.log(errors[-4])-np.log(errors[-5]))/(np.log(DOF[-4])-np.log(DOF[-5])))
```

```python
#f(x)=x**n-x
#n=2.1,a=0,b=1
#Graphs the error against DOF  in logaritmic axes
#First it uses h fem with uniform mesh for polynomial degrees 1 and then 2
a=0
b=1
n=2.1
N=5
f = lambda x : (x**n)-x-n*(n-1)*(x**(n-2))
fig, axes2 = plt.subplots(1,3, figsize=(15,7))
fig.tight_layout()
fig.subplots_adjust(wspace=0.3)
for j in range(1,3):
    pmax=j
    errors=[]
    DOF=[]
    for i in range(1,N+1):
        M=2**i
        x=h_unif(M,pmax,a,b)[0]
        p=h_unif(M,pmax,a,b)[1]
        errors.append(energy_norm(x,p,f,n)[1])
        DOF.append(energy_norm(x,p,f,n)[2])
    axes2[j-1].loglog(DOF,errors)
    axes2[j-1].set_xlabel("Degrees if freedom")
    axes2[j-1].set_ylabel("Error in energy norm(%)")
    axes2[j-1].set_title(f"Error for l=2.1, h_unif with N=5,p={j}")
    print(f"the slope is:",(np.log(errors[-1])-np.log(errors[-2]))/(np.log(DOF[-1])-np.log(DOF[-2])))
#Second it uses p fem with uniform mesh with 1 element and then two elements for polynomial degrees 1,....,6
M=1
pmax=6
errors=[]
DOF=[]
x=p_unif(M,pmax,a,b)[0]
p=p_unif(M,pmax,a,b)[1]
for i in p:
    p_list=[]
    for k in range(M):
        p_list.append(i)
    errors.append(energy_norm(x,p_list,f,n)[1])
    DOF.append(energy_norm(x,p_list,f,n)[2])
axes2[2].loglog(DOF,errors)
axes2[2].set_xlabel("Degrees of freedom")
axes2[2].set_ylabel("Error in energy norm(%)")
axes2[2].set_title(f"Error for l=2.1,p_unif with M={M},pmax=6")
print(f"the slope is:",(np.log(errors[-1])-np.log(errors[-2]))/(np.log(DOF[-1])-np.log(DOF[-2])))

```

```python
#f(x)=x**n-x
#n=2.1,a=0,b=1
#Graphs the error against DOF  in logaritmic axes
a=0
b=1
n=2.1
f = lambda x : (x**n)-x-n*(n-1)*(x**(n-2))
fig, axes3 = plt.subplots(1,4, figsize=(15,7))
fig.tight_layout()
fig.subplots_adjust(wspace=0.3)
j=0

#It uses h radical fem for N=5, s=0.15 for polynomial degrees 2
N=5
s=0.15
pmax=2
errors=[]
DOF=[]
for i in range(1,N+1):
    M=2**i
    x=h_radical(M,pmax,s,a,b)[0]
    p=h_radical(M,pmax,s,a,b)[1]
    errors.append(energy_norm(x,p,f,n)[1])
    DOF.append(energy_norm(x,p,f,n)[2])
axes3[j].loglog(DOF,errors)
axes3[j].set_xlabel("Degrees of freedom")
axes3[j].set_ylabel("Error in energy norm(%)")
axes3[j].set_title(f"Error for l=2.1,h-radical with N=5,s=0.15,p=2")
print(f"the slope is:",(np.log(errors[-1])-np.log(errors[-2]))/(np.log(DOF[-1])-np.log(DOF[-2])))
#It uses p fem with geometric mesh for M=4, q=0.15 for polynomial degrees 1,....,6
M=4
q=0.15
pmax=6
errors=[]
DOF=[]
x=p_geom(M,pmax,q,a,b)[0]
p=p_geom(M,pmax,q,a,b)[1]
for i in p:
    p_list=[]
    for k in range(M):
        p_list.append(i)
    errors.append(energy_norm(x,p_list,f,n)[1])
    DOF.append(energy_norm(x,p_list,f,n)[2])
axes3[j+1].loglog(DOF,errors)
axes3[j+1].set_xlabel("Degrees of freedom")
axes3[j+1].set_ylabel("Error in energy norm(%)")
axes3[j+1].set_title(f"Error for l=2.1,p-geom with M=4,q=0.15,p=6")
print(f"the slope is:",(np.log(errors[-5])-np.log(errors[-6]))/(np.log(DOF[-5])-np.log(DOF[-6])))
#It uses p fem with geometric mesh for M=4, q=0.15 for polynomial degrees 1,2
M=4
q=0.15
pmax=2
errors=[]
DOF=[]
x=p_geom(M,pmax,q,a,b)[0]
p=p_geom(M,pmax,q,a,b)[1]
for i in p:
    p_list=[]
    for k in range(M):
        p_list.append(i)
    errors.append(energy_norm(x,p_list,f,n)[1])
    DOF.append(energy_norm(x,p_list,f,n)[2])
axes3[j+2].loglog(DOF,errors)
axes3[j+2].set_xlabel("Degrees of freedom")
axes3[j+2].set_ylabel("Error in energy norm(%)")
axes3[j+2].set_title(f"Error for l=2.1,p-geom with M=4,q=0.15,p=2")
print(f"the slope is:",(np.log(errors[-1])-np.log(errors[-2]))/(np.log(DOF[-1])-np.log(DOF[-2])))

#It uses hp fem with geometric mesh for N=5, q=0.15 for polynomial degrees p=M
N=5
q=0.15
errors=[]
DOF=[]
for i in range(1,N+1):
    M=2**i
    x=hp_geom(M,q,a,b)[0]
    p=hp_geom(M,q,a,b)[1]
    for z in p:
        plist=[]
        for k in range(M):
            plist.append(z)
        errors.append(energy_norm(x,plist,f,n)[1])
        DOF.append(energy_norm(x,plist,f,n)[2])
axes3[j+3].semilogy(DOF,errors)
axes3[j+3].set_xlabel("Degrees of freedom")
axes3[j+3].set_ylabel("Error in energy norm(%)")
axes3[j+3].set_title(f"Error for l=2.1,hp-geom with N=6,q=0.15")
print(f"the slope is:",(np.log(errors[-4])-np.log(errors[-5]))/(np.log(DOF[-4])-np.log(DOF[-5])))
```

```python
import warnings
warnings.filterwarnings('ignore')

#n=2.1
#a=0,b=1
#Graphs the absolute error between the finite element solution(uFE)and the exact solution (uEX)

a = 0
b = 1
n = 2.1
M = 10
q = 0.15
f = lambda x : (x**n)-x-n*(n-1)*(x**(n-2))
y = np.linspace(0,1)
u_Ex_fun = lambda x : (x-x**n)

#First;y I use the h fem with uniform mesh for M=10,p=1
x=h_unif(M,1,a,b)[0]
p=h_unif(M,1,a,b)[1]
u_Fe = -np.array(solid1d(y,x,M,p,f))
u_Ex_values = np.array([u_Ex_fun(val) for val in y])
fig , axes4 =plt.subplots(figsize=(10,10))
axes4.plot(y,abs(u_Ex_values-u_Fe)/abs(u_Ex_values),color='black')
axes4.set_xlabel("x")
axes4.set_ylabel("y")
axes4.set_title(r'Error ${|u_{FE}-u_{EX}|/|u_{EX}|}$')
#Graphs the uFE against uEX
fig , axes5 =plt.subplots(figsize=(10,10))
axes5.plot(y,u_Fe, color="red")
axes5.plot(y,u_Ex_values, color="blue")
axes5.set_xlabel("x")
axes5.set_ylabel("y")
axes5.set_title(r'Plots of $u_{FE}$ and $u_{EX}$')
axes5.legend([r'$u_{FE}$','$u_{EX}$'])

#Secondly I use the h fem with uniform mesh for M=1,p=6
x=h_unif(1,6,a,b)[0]
p=h_unif(1,6,a,b)[1]
u_Fe = -np.array(solid1d(y,x,1,p,f))
u_Ex_values = np.array([u_Ex_fun(val) for val in y])
fig , axes6 =plt.subplots(figsize=(10,10))
axes6.plot(y,abs(u_Ex_values-u_Fe)/abs(u_Ex_values),color='black')
axes6.set_xlabel("x")
axes6.set_ylabel("y")
axes6.set_title(r'Error ${|u_{FE}-u_{EX}|/|u_{EX}|}$')
#Graphs the uFE against uEX
fig , axes7 =plt.subplots(figsize=(10,10))
axes7.plot(y,u_Fe, color="red")
axes7.plot(y,u_Ex_values, color="blue")
axes7.set_xlabel("x")
axes7.set_ylabel("y")
axes7.set_title(r'Plots of $u_{FE}$ and $u_{EX}$')
axes7.legend([r'$u_{FE}$','$u_{EX}$'])

#Thirdly I use the p fem with geometric mesh for M=10,p=2 for all elements
x=p_geom(M,2,q,a,b)[0]
p=[]
for r in range(M):
    p.append(2)
u_Fe = -np.array(solid1d(y,x,M,p,f))
u_Ex_values = np.array([u_Ex_fun(val) for val in y])
fig , axes8 =plt.subplots(figsize=(10,10))
axes8.plot(y,abs(u_Ex_values-u_Fe)/abs(u_Ex_values),color='black')
axes8.set_xlabel("x")
axes8.set_ylabel("y")
axes8.set_title(r'Error ${|u_{FE}-u_{EX}|/|u_{EX}|}$')
#Graphs the uFE against uEX
fig , axes9 =plt.subplots(figsize=(10,10))
axes9.plot(y,u_Fe, color="red")
axes9.plot(y,u_Ex_values, color="blue")
axes9.set_xlabel("x")
axes9.set_ylabel("y")
axes9.set_title(r'Plots of $u_{FE}$ and $u_{EX}$')
axes9.legend([r'$u_{FE}$','$u_{EX}$'])

#Fourthly I use the p fem with geometric mesh for M=10,p=4 for all elements
x=p_geom(M,4,q,a,b)[0]
p=[]
for r in range(M):
    p.append(4)
u_Fe = -np.array(solid1d(y,x,M,p,f))
u_Ex_values = np.array([u_Ex_fun(val) for val in y])
fig , axes10 =plt.subplots(figsize=(10,10))
axes10.plot(y,abs(u_Ex_values-u_Fe)/abs(u_Ex_values),color='black')
axes10.set_xlabel("x")
axes10.set_ylabel("y")
axes10.set_title(r'Error ${|u_{FE}-u_{EX}|/|u_{EX}|}$')
#Graphs the uFE against uEX
fig , axes11 =plt.subplots(figsize=(10,10))
axes11.plot(y,u_Fe, color="red")
axes11.plot(y,u_Ex_values, color="blue")
axes11.set_xlabel("x")
axes11.set_ylabel("y")
axes11.set_title(r'Plots of $u_{FE}$ and $u_{EX}$')
axes11.legend([r'$u_{FE}$','$u_{EX}$'])
```


# Exercise 19 - Differential equations

Many physical, chemical and biological systems can be modelled using differential equations. Often, these do not have analytical solutions, but they can be solved numerically. There are many techniques for achieving this, and in this practical we will look only at a couple of simple examples.

## The simple pendulum

A simple pendulum of mass $m$, suspended from a string of length $l$, obeys the differential equation
$$\frac{\mathrm{d}^2\theta}{\mathrm{d}t^2} + \frac{g}{l}\sin\theta = 0,$$
where $\theta$ is the angle the string makes with a vertical line.

In this exercise, we will use the function `scipy.integrate.odeint()`. This solves differential equations of the form
$$\frac{\mathrm{d}\mathbf{y}(t)}{\mathrm{d}t} = f(\mathbf{y},t).$$
We therefore first need to work out how to write the pendulum equation in this form. Fortunately, there is a standard approach we can follow. We define $$\mathbf{y}(t) = \left(\begin{array}{c}\theta(t)\\\dot\theta(t)\end{array}\right),$$
where $$\dot{\theta}  = \frac{\mathrm{d}\theta}{\mathrm{d}t}$$
Now,
$$\frac{\mathrm{d}\mathbf{y}}{\mathrm{d}t} = \left(\begin{array}{c}\dot\theta\\\ddot\theta\end{array}\right) = \left(\begin{array}{c}y_2(t)\\-\frac{g}{l}\sin y_1(t)\end{array}\right).$$
Thus, given any vector $\mathbf{y}$, we can compute its time derivative.

**➤ Write and test a function to compute $\mathrm{d}\mathbf{y}/\mathrm{d}t$.** Its call signature should be
```func(y,t,...)``` even though the variable `t` is not required within the calculation.

```python
# Try it here!

```

The function `scipy.integrate.odeint()` expects to be given an *initial condition* -- the vector $\mathbf{y}$ at time $t=0$. Let's suppose that the pendulum is initially at an angle of ${\pi}/{4}$ radians from vertical, and at rest, so that $\dot\theta(0)=0$.

**➤ Create a vector `y0` to represent this initial condition.**

```python
# Try it here!

y0 = np.array([0.3,0])
```

Finally, `scipy.integrate.odeint()` needs a sequence of time-points at which the pendulum's state should be evaluated. These need to be sufficiently closely-spaced for the integration to be accurate. A good starting point might be
```python
timesteps = np.linspace(0,10,1000)
```

**➤ Use `scipy.integrate.odeint` to compute the motion of the pendulum.** The function returns an array of dimension $(2\times N)$, where $N$ is the number of timesteps. This represents the vector $\mathbf{y}(t)$ evaluated at each of these times.

**➤ Make an animation showing the motion of the pendulum over time.**

```python
# Try it here!

timesteps = np.linspace(0,10,500)
y = integ.odeint(simplePendulum,y0,timesteps,args = (0.1,1.5))
plt.plot(timesteps,y[:,0],timesteps,0.1*y[:,1])
```

At present, our differential equation doesn't include any attenuation, and so the pendulum continues oscillating with the same amplitude indefinitely. A more realistic system could be modelled as $$\frac{\mathrm{d}^2\theta}{\mathrm{d}t^2} + \frac{g}{l}\sin\theta +\alpha \frac{\mathrm{d}\theta}{\mathrm{d}t}= 0$$
where $\alpha$ is a small, positive constant.

**➤ Incorporate this extra term into your modelling, and make a new animation.**

```python
# Try it here!

```

**➤ Experiment with different initial conditions, pendulum properties, and integration steps.**

```python
# Try it here!

```

A [double pendulum](https://en.wikipedia.org/wiki/Double_pendulum) is one pendulum suspended from another. Suppose we have a metal rod of mass $m$ and length $l$, suspended from a pivot so that it functions as a pendulum. We then hang a second, identical rod from the free end of the first rod. Together, these two function as a double pendulum.

It can be shown (via Lagrangian mechanics) that the system can be fully-represented by specifying the two angles ($\theta_1$ and $\theta_2$) that each rod makes with the vertical, and two momenta ($p_1$ and $p_2$). The time-derivatives of the system are given by:

$$\begin{align}
{\dot \theta_1} &= \frac{6}{ml^2} \frac{ 2 p_{1} - 3 \cos(\theta_1-\theta_2) p_{2}}{16 - 9 \cos^2(\theta_1-\theta_2)} \\
{\dot \theta_2} &= \frac{6}{ml^2} \frac{ 8 p_{2} - 3 \cos(\theta_1-\theta_2) p_{1}}{16 - 9 \cos^2(\theta_1-\theta_2)}\\
{\dot p_{1}} &=  -\tfrac{1}{2} m l^2 \left ( {\dot \theta_1} {\dot \theta_2} \sin (\theta_1-\theta_2) + 3 \frac{g}{l} \sin \theta_1 \right ) \\
{\dot p_{2}} &=  -\tfrac{1}{2} m l^2 \left ( -{\dot \theta_1} {\dot \theta_2} \sin (\theta_1-\theta_2) + \frac{g}{l} \sin \theta_2 \right ).
\end{align}$$

**➤ Using `scipy.integrate.odeint`, create an animation of the double pendulum in motion.**

```python
# Try it here!

```


Basic Python for Geosciences

Session 1

Put your name here

 🏋
    Exercise 1:

If the Average velocity to the target is 3000 m/sec and the target depth is 5000 m, calculate the two way time to the target according to the following formula:

\begin{align}
TWT=2*( \frac{Depth}{Velocity})
\end{align}

Define the variables and use the formula to obtain the TWT.

```python

```

 🏋
    Exercise 2:

The relation between velocity and depth is related according to the following formula:

\begin{align}
Velocity (m/sec)=2800 + 0.7 * Depth (m)
\end{align}

* What is the Velocity (m/sec) at 3000 meters?
* What is the Velocity (m/sec) at 6000 feet?

The conversion factor from ft to m is 0.3048

```python

```

 🏋
    Exercise 3:

* How much time takes a wave to travel through water in 2220 m, V=1500 m/s?
* How much time takes a wave to travel through air  in 500 m, V=340 m/s?
* Write a small piece of code in Python to calculate the travel times and also find which is longer.

```python

```

 🏋
    Exercise 4:  Write a dictionary with the following composition (weight %) of a granite:

    Quartz       25
    Orthoclase   50
    Plagioclase  10
    Micas        10
    Anphibole     5

Using Python, present the values for orthoclase and micas

```python

```

 🏋
    Exercise 5:  Write in Python a variable with the following string:

"A subsurface body of rock having sufficient porosity and permeability to store and transmit fluids. Metamorphic rocks are the most common reservoir rocks because they have more porosity than most igneous and sedimentary rocks and form under temperature conditions at which hydrocarbons can be preserved. A reservoir is a critical component of a complete petroleum system."

* Print the variable in the screen
* Correct the mistakes in the variable using the replace method
* Verify if temperature is in the string
* Verify if kritical is in the string
* What is the string length?

```python

```

 🏋
    Exercise 6:  These are the horizon tops and its depths (m):

    Surface       0
    Cretaceous 1500
    Jurassic   2000
    Triassic   3000
    Paleozoic  4500

Write one tuple with the horizons name and the corresponding depths. Print the name and the depth of the third horizon in the list.

```python

```

 🏋
    Exercise 7: These are the horizon tops and its depths (m):

    Surface       0
    Cretaceous 1500
    Jurassic   2000
    Triassic   3000
    Paleozoic  4500

* Store the list of horizons and its corresponding depths in a list variable
* Print the lists
* Print the name and the depth of the third horizon in the list
* What is the maximum depth?
* What is the minimum depth?
* What is the average depth?
* How many horizons are in the list
* Verify that the number of horizons names and number of horizons tops are the same

```python

```

 🏋
    Exercise 8: Create a matrix (list of lists) with the following values:

    | 6   8  12  24  33|
    |11   4  12   6  67|
    |23   9  67  10  43|
    | 1  12  34  18   2|
    | 7  11   2  23  78|
    | 8  24   1   5  23|

* Print the matrix
* Print the row 3
* Calculate the dimensions of the matrix
* Print the element [3,4]
* What is the minimum of the line 3
* What is the maximum of the line 2

```python

```

 🏋
    Exercise 9:

* Generate a 5x5 matrix of constant value equal to 10
* Generate a random 6x6 matrix
* Print the values of the third column of both arrays
* Print the attributes of both matrices
* Generate a 3 million elements array and calculate all its attributes

```python

```

 🏋
    Exercise 10: Generate two 5x5 matrices, one with constant value equal to 10, and the other with random values

* Determine if the matrices are singular (Determinant=0)
* Calculate the inverse of the matrices
* Sum both matrices
* Convert the constant matrix to zero matrix

```python

```

 🏋
    Exercise 11: Solve the following Linear equations system:

\begin{align}
1x+3y-2z = 5  \\
3x+5y+6z = 7  \\
2x+4y+3z = 8
\end{align}

Remember the linear system can be represented in linear algebra as dot product of the corresponding arrays:

\begin{align}
A \cdot x=b
\end{align}

The solution will be:

\begin{align}
x=A^{-1} \cdot b
\end{align}

 🏋
    Exercise 12:

* Create a NumPy array with some random values
* Calculate the maximum and the minimum of the array
* Calculate the Average of the terms
* Print the numbers rounded to 6 decimals

 🏋
    Exercise 13:

Practice with a couple of examples on how to convert between float, int and str

 🏋
    Exercise 14:

In a cell write a small program for temperature conversion between Celsius to Fahrenheit with user interactive input

 🏋
    Exercise 15:  Generate a range between 1 and 3000000

* Calculate the average
* Calculate the minimum
* Calculate the maximum

 🏋
    Exercise 16:  Create a function to convert from Fahrenheit to Celsius

 🏋
    Exercise 17:  Create a function to calculate and return the following values of a list:

* Average
* Minimum
* Maximum


# Symbolic Computation
Symbolic computation deals with symbols, representing them exactly, instead of numerical approximations (floating point).

We will start with the following [borrowed](https://docs.sympy.org/latest/tutorial/intro.html) tutorial to introduce the concepts of SymPy. Devito uses SymPy heavily and builds upon it in its DSL.

```python
import math

math.sqrt(3)
```

    1.7320508075688772

```python
math.sqrt(8)
```

    2.8284271247461903

$\sqrt(8) = 2\sqrt(2)$, but it's hard to see that here

```python
import sympy
sympy.sqrt(3)
```

$\displaystyle \sqrt{3}$

SymPy can even simplify symbolic computations

```python
sympy.sqrt(8)
```

$\displaystyle 2 \sqrt{2}$

```python
sympy.sqrt(20)
```

$\displaystyle 2 \sqrt{5}$

```python
from sympy import symbols
x, y, z = symbols('x y z')
expr = x + 2*y
expr
```

$\displaystyle x + 2 y$

Note that simply adding two symbols creates an expression. Now let's play around with it.

```python
expr + 1
```

$\displaystyle x + 2 y + 1$

```python
expr - x
```

$\displaystyle 2 y$

Note that `expr - x` was not `x + 2y -x`

```python
x*expr
```

$\displaystyle x \left(x + 2 y\right)$

```python
from sympy import expand, factor
expanded_expr = expand(x*expr)
expanded_expr
```

$\displaystyle x^{2} + 2 x y$

```python
factor(expanded_expr)
```

$\displaystyle x \left(x + 2 y\right)$

```python
from sympy import diff, sin, exp

diff(sin(x)*exp(x), x)
```

$\displaystyle e^{x} \sin{\left(x \right)} + e^{x} \cos{\left(x \right)}$

```python
from sympy import limit

limit(sin(x)/x, x, 0)
```

$\displaystyle 1$

```python
from sympy import symbols
equation = x**2 + z*x*y + z**3
equation
```

$\displaystyle x^{2} + x z^{2} + z^{3}$

```python
x * equation
```

$\displaystyle x \left(x^{2} + x y z + z^{3}\right)$

```python
expanded_equation = expand(x * equation)
expanded_equation
```

$\displaystyle x^{3} + x^{2} y z + x z^{3}$

```python
factor(expanded_equation)
```

$\displaystyle x \left(x^{2} + x y z + z^{3}\right)$

```python
diff(expanded_equation,x)
```

$\displaystyle 3 x^{2} + 2 x y z + z^{3}$

### Exercise

Solve $x^2 - 2 = 0$ using sympy.solve

```python
# Type solution here
from sympy import solve
solve(x**2 - 2, x)
```

$\displaystyle \left[ - \sqrt{2}, \  \sqrt{2}\right]$

## Pretty printing

```python
from sympy import init_printing, Integral, sqrt

init_printing(use_latex='mathjax')
```

```python
Integral(sqrt(1/x), x)
```

$\displaystyle \int \sqrt{\frac{1}{x}}\, dx$

```python
from sympy import latex

latex(Integral(sqrt(1/x), x))
```

    '\\int \\sqrt{\\frac{1}{x}}\\, dx'

More symbols.
Exercise: fix the following piece of code

```python
# NBVAL_SKIP
# The following piece of code is supposed to fail as it is
# The exercise is to fix the code
expr2 = x + 2*y +3*z
```

### Exercise

Solve $x + 2*y + 3*z$ for $x$

```python
# Solution here
from sympy import solve
expression = x + 2 * y + 3 * z
solve(expression,x)
```

$\displaystyle \left[ - 2 y - 3 z\right]$

Difference between symbol name and python variable name

```python
x, y = symbols("y z")
```

```python
x
```

$\displaystyle y$

```python
y
```

$\displaystyle z$

```python
# NBVAL_SKIP
# The following code will error until the code in cell 16 above is
# fixed
z
```

$\displaystyle z$

Symbol names can be more than one character long

```python
crazy = symbols('unrelated')

crazy + 1
```

$\displaystyle unrelated + 1$

```python
x = symbols("x")
expr = x + 1
x = 2
```

What happens when I print expr now? Does it print 3?

```python
print(expr)
```

    x + 1

How do we get 3?

```python
x = symbols("x")
expr = x + 1
expr.subs(x, 2)
```

$\displaystyle 3$

## Equalities

```python
x + 1 == 4
```

    False

```python
from sympy import Eq

Eq(x + 1, 4)
```

$\displaystyle x + 1 = 4$

Suppose we want to ask whether $(x + 1)^2 = x^2 + 2x + 1$

```python
(x + 1)**2 == x**2 + 2*x + 1
```

    False

```python
from sympy import simplify

a = (x + 1)**2
b = x**2 + 2*x + 1

simplify(a-b)
```

$\displaystyle 0$

### Exercise
Write a function that takes two expressions as input, and returns a tuple of two booleans. The first if they are equal symbolically, and the second if they are equal mathematically.

## More operations

```python
z = symbols("z")
expr = x**3 + 4*x*y - z
expr.subs([(x, 2), (y, 4), (z, 0)])
```

$\displaystyle 36$

```python
from sympy import sympify

str_expr = "x**2 + 3*x - 1/2"
expr = sympify(str_expr)
expr
```

$\displaystyle x^{2} + 3 x - \frac{1}{2}$

```python
expr.subs(x, 2)
```

$\displaystyle \frac{19}{2}$

```python
expr = sqrt(8)
```

```python
expr
```

$\displaystyle 2 \sqrt{2}$

```python
expr.evalf()
```

$\displaystyle 2.82842712474619$

```python
from sympy import pi

pi.evalf(100)
```

$\displaystyle 3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117068$

```python
pi.evalf(1000)
```

$\displaystyle 3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067982148086513282306647093844609550582231725359408128481117450284102701938521105559644622948954930381964428810975665933446128475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920962829254091715364367892590360011330530548820466521384146951941511609433057270365759591953092186117381932611793105118548074462379962749567351885752724891227938183011949129833673362440656643086021394946395224737190702179860943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901224953430146549585371050792279689258923542019956112129021960864034418159813629774771309960518707211349999998372978049951059731732816096318595024459455346908302642522308253344685035261931188171010003137838752886587533208381420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909216420199$

```python
from sympy import cos

expr = cos(2*x)
expr.evalf(subs={x: 2.4})
```

$\displaystyle 0.0874989834394464$

### Exercise

```python
from IPython.core.display import Image
Image(filename='figures/comic.png')
```

Write a function that takes a symbolic expression (like pi), and determines the first place where 789 appears.
Tip: Use the string representation of the number. Python starts counting at 0, but the decimal point offsets this

## Solving an ODE

```python
from sympy import Function

f, g = symbols('f g', cls=Function)
f(x)
```

$\displaystyle f{\left(y \right)}$

```python
f(x).diff()
```

$\displaystyle \frac{d}{d y} f{\left(y \right)}$

```python
diffeq = Eq(f(x).diff(x, x) - 2*f(x).diff(x) + f(x), sin(x))
diffeq
```

$\displaystyle f{\left(y \right)} - 2 \frac{d}{d y} f{\left(y \right)} + \frac{d^{2}}{d y^{2}} f{\left(y \right)} = \sin{\left(y \right)}$

```python
from sympy import dsolve

dsolve(diffeq, f(x))
```

$\displaystyle f{\left(y \right)} = \left(C_{1} + C_{2} y\right) e^{y} + \frac{\cos{\left(y \right)}}{2}$

## Finite Differences

```python

f = Function('f')
dfdx = f(x).diff(x)
dfdx.as_finite_difference()
```

$\displaystyle - f{\left(y - \frac{1}{2} \right)} + f{\left(y + \frac{1}{2} \right)}$

```python
from sympy import Symbol

d2fdx2 = f(x).diff(x, 2)
h = Symbol('h')
d2fdx2.as_finite_difference(h)
```

$\displaystyle - \frac{2 f{\left(y \right)}}{h^{2}} + \frac{f{\left(- h + y \right)}}{h^{2}} + \frac{f{\left(h + y \right)}}{h^{2}}$

Now that we have seen some relevant features of vanilla SymPy, let's move on to Devito, which could be seen as SymPy finite differences on steroids!


# Einführung in Python

## Warum Python?

Python ist eine sog. general-purpose Programmiersprache. Sie genießt weite Verbreitung in einer Vielzahl von Einsatzgebieten, darunter insbesondere die Web und Internet Entwicklung, das Wissenschaftliche Rechnen sowie die Lehre. Eines der wichtigsten Ziele von Python ist es, lesbaren und klaren Programmcode zu ermöglichen. Daher gilt die Sprache gemeinhin auch als leicht zu erlernen.

Zu Python gehört außerdem ein großes Ökosystem aus Bibliotheken, Frameworks und Tools, sodass für viele Anwendungsfälle bereits performante Standardlösungen existieren.

## Die Sprache

### Python ist imperativ:

Imperative Sprachen verwenden sequentielle Ausdrücke um den Zustand des Programms zu ändern.

```python
a = 5
b = 3
c = a * b

print("a + b = %d" % c)
```

    a + b = 15

### Python ist funktional:

Funktionale Sprachen bieten die Möglichkeit, ein Programm oder Teile dessen in Form von mathematischen Funktionen auszudrücken um Details der eigentlichen Ausführung zu abstrahieren.

```python
even_numbers = [n for n in range(20) if n % 2 == 0]

print(even_numbers)
```

    [0, 2, 4, 6, 8, 10, 12, 14, 16, 18]

### Python ist prozedural:

Prozedurale Sprachen fassen Programmsegmente, die wiederholt ausgeführt werden, in sogenannten Prozeduren (auch Funktionen genannt) zusammen.

```python
def fib(n):
    if n == 1 or n == 0:
        return 1
    else:
        return fib(n-1) + fib(n-2)

print("fib(3) = %d" % fib(3))
print("fib(5) = %d" % fib(5))
print("fib(8) = %d" % fib(8))
```

    fib(3) = 3
    fib(5) = 8
    fib(8) = 34

# Aufgaben

## Importieren

Damit wir nicht mit jedem neuen Programm auch alle grundlegenden Funktionen neu definieren müssen, können wir in Python sogenannte Pakete und Module importieren. Daher ist der erste Schritt beim Schreiben eines Programms für gewöhnlich das Importieren der benötigten Funktionen und Klassen.

### Syntax

`[from <module/package>] import <thing/module> [as alias]`

### Aufgabe
Importiert `numpy` mit dem Namen `np`, das Modul `pyplot` aus dem Package `matplotlib` mit dem Namen `plt` und die Funktionen `sin` und `cos` aus `math`. Danach sollte die darauffolgende Kontrollcode die Funktion sin auf dem Intervall [0, 2pi) Zeichnen.

```python

```

### Kontrolle

```python
# Der Abstand zwischen x-Werten
dx = 0.1

# Alle werte in [0, 2pi) mit dem Abstand step_size:
# xs = [0, 0.1, 0.2, ... 2pi - 0.1]
xs = np.arange(0, 2 * np.pi, dx)

# Werte sin für alle Werte in xs aus:
ys = np.vectorize(sin)(xs)

# Zeichne den Funktionsgraphen von sin als blau gestrichelte Linie:
plt.plot(xs, ys, '-.b')
plt.show()
```

## Definieren einer Funktion

### Aufgabe
Definiert eine Funktion `y(t)` mit der folgenden Abbildungsvorschrift:

\begin{equation}
    y = \frac{sin(t^2)}{t}
\end{equation}

Sowie die Ableitung `y_d(t, y)` mit:

\begin{equation}
\begin{split}
    \dot y &= 2cos(t^2) - \frac{sin(t^2)}{t^2} \\
           &= 2cos(t^2) - \frac{y}{t}
\end{split}
\end{equation}

Dafür braucht ihr:
- `def f(x):` um eine Funktion zu definieren
- `return x` um einen Werte zurück zu geben
- `x**y` um $x^y$ zu berechnen

```python

```

### Kontrolle

```python
# Mit assert können wir zur Laufzeit sicherstellen, dass bestimmte Konditionen erfüllt werden.
# Hier sieht man außerdem den Zeilenumbruch mit \ und das Einfügen variabler Werte mittels %f in einen String.
assert y(1) == 0.8414709848078965, \
       "y(1) sollte 0.8414709848078965 sein, ist aber %f" %y(1)
assert y_d(1, 1) == 0.08060461173627953, \
       "y_d(1, 1)$ sollte 0.08060461173627953 sein, ist aber %f" %y_d(1, 1)
```

## Simulation

Ein mögliches Einsatzgebiet von Python ist die Simulation, und eine der einfachsten Methoden zur Simulation ist der sogenannte explizite Euler:

\begin{equation}
    y_{j+1} = y_j + \Delta t * f(t_j, y_j)
\end{equation}

Wobei für $f(t, y)$ gilt:
\begin{equation}
    \dot y = f(t, y)
\end{equation}

### Aufgabe

Für $f(t, y)$ verwenden wir im folgenden Teil die bereits definierte Funktion `y_d(t, y)`. Also brauchen wir jetzt noch eine Funktione `expl_euler(f, t, y, dt=0.1)`. Diese soll später unsere Funktion als f, die Zeit t und den aktuellen Wert y übergeben bekommen.

```python

```

### Kontrolle

```python
# Unsere Simulationsparameter:
dt      = 0.1
n_steps = 60
y0      = 0
t0      = 0.1

# Ein Vektor mit n_steps Elementen, alle zu Begin 0:
ys = simulate(y_d, t0, y0, dt, n_steps)

# Wir zeichnen unser Simulationsergebnis:
ts = np.arange(t0, t0 + n_steps * dt, dt)
plt.plot(ts, ys)
plt.legend(["Simulation"])
plt.show()
```

## Fehleranalyse

Um abzuschätzen, wie große Fehler wir mit der expliziten Eulermethode machen, können wir unser Simulationsergebnis mit der uns bekannten Methode `y(t)` vergleichen.

### Aufgabe
Berechnet eine Liste `true_ys` mit `y(t)` über `ts`. Damit könnt ihr im Anschluss die absoluten Abweichungen zwischen der Simulation und den wahren Ergebnissen als `error` und dessen Kumulative Summe als `cum_error`.

```python

```

### Kontrolle

```python
# Einstellen der Plot Dimensionen
plt.figure(figsize=(15,4))

# Hier werden zwei Plots (1 Reihe, 2 Spalten) gleichzeitig gezeichnet:
plt.subplot(1, 2, 1)
plt.plot(ts, true_ys, 'b')
plt.plot(ts, ys, '-.g')
plt.legend(["y(t)", "Simulation"])

plt.subplot(1, 2, 2)
plt.plot(ts, error, '-.r')
plt.plot(ts, cum_error, '-.m')
plt.legend(["Error", "Cumulative Error"])
plt.show()
```

<h2> Lust auf mehr? <h2\>


Determine the transformation of an aperture diameter of 7.5 kpc at the redshift of abell 370 to pixels. First transform the diameter in absolute size (kpc) to apparent size (arcseconds). Then transform apparent size to pixels using WCS.

```python
import numpy as np

from astropy.io import fits
from astropy import units as u
from astropy import cosmology
from astropy.wcs import WCS
```

### Load Data

```python
input_data_path = "https://archive.stsci.edu/pub/hlsp/frontier/abell370/images/hst/v1.0-epoch2/hlsp_frontier_hst_wfc3-30mas-bkgdcor_abell370_f105w_v1.0-epoch2_drz.fits"

hdul = fits.open(input_data_path)
hdu = hdul[0]

data = hdu.data
header = hdu.header
wcs = WCS(header)
```

### Given

```python
# Given
aperture_diameter = (7.5 * u.kpc).to(u.Mpc)
```

NED data for abell 370:

http://ned.ipac.caltech.edu/cgi-bin/objsearch?search_type=Obj_id&objid=132527&objname=1&img_stamp=YES&hconst=73.0&omegam=0.27&omegav=0.73&corr_z=1

```python
"""
Cosmology-Corrected Quantities [Ho =  70.50 km/sec/Mpc, Ωmatter =   0.27, Ωvacuum =   0.73]
[Redshift 0.374247 as corrected to the Reference Frame defined by the 3K Microwave Background Radiation]
Luminosity Distance        :     2011 Mpc      (m-M) = 41.52 mag
Angular-Size Distance      :     1065 Mpc      (m-M) = 40.14 mag
Co-Moving Radial Distance  :     1463 Mpc      (m-M) = 40.83 mag
Co-Moving Tangential Dist. :     1463 Mpc      (m-M) = 40.83 mag
Co-Moving Volume           : 13.1 Gpc^3
Light Travel-Time          :    4.073 Gyr
Age at Redshift 0.374247   :    9.698 Gyr
Age of Universe            :   13.770 Gyr
Scale (Cosmology Corrected):    5161 pc/arcsec =  5.161 kpc/arcsec = 309.67 kpc/arcmin =  18.58 Mpc/degree
Surface Brightness Dimming : Flux Density per Unit Area =  0.28038; Magnitude per Unit Area = 1.381 mag
"""
```

    '\nCosmology-Corrected Quantities [Ho =  70.50 km/sec/Mpc, Ωmatter =   0.27, Ωvacuum =   0.73]\n[Redshift 0.374247 as corrected to the Reference Frame defined by the 3K Microwave Background Radiation]\nLuminosity Distance        :     2011 Mpc      (m-M) = 41.52 mag\nAngular-Size Distance      :     1065 Mpc      (m-M) = 40.14 mag\nCo-Moving Radial Distance  :     1463 Mpc      (m-M) = 40.83 mag\nCo-Moving Tangential Dist. :     1463 Mpc      (m-M) = 40.83 mag\nCo-Moving Volume           : 13.1 Gpc^3\nLight Travel-Time          :    4.073 Gyr\nAge at Redshift 0.374247   :    9.698 Gyr\nAge of Universe            :   13.770 Gyr\nScale (Cosmology Corrected):    5161 pc/arcsec =  5.161 kpc/arcsec = 309.67 kpc/arcmin =  18.58 Mpc/degree\nSurface Brightness Dimming : Flux Density per Unit Area =  0.28038; Magnitude per Unit Area = 1.381 mag\n'

```python
abell370_z = 0.375000
abell370_distance = 1463 * u.Mpc # Comoving
```

### Equations

\begin{align}
\theta = \frac{(z+1) * Diameter}{Distance_{Comoving}} =  \frac{Diameter}{Distance_{Angular Diameter} } \\
\end{align}

### Compute angular diameter distance

```python
def compute_angular_diameter_distance(d, z):
    return d/(z+1)
```

```python
angular_diameter_distance = compute_angular_diameter_distance(abell370_distance, abell370_z)
angular_diameter_distance
```

$1064 \; \mathrm{Mpc}$

```python
# check astropy cosmology
cosmology.WMAP5.angular_diameter_distance(abell370_z)
```

$1068.4053 \; \mathrm{Mpc}$

### Find angular diameter

```python
angular_diameter = np.tan((aperture_diameter / angular_diameter_distance).value) * u.rad
angular_diameter
```

$7.0488722 \times 10^{-6} \; \mathrm{rad}$

```python
# check astropy cosmology
(cosmology.WMAP5.arcsec_per_kpc_proper(abell370_z) * aperture_diameter.to('kpc')).to('rad')
```

$7.0198082 \times 10^{-6} \; \mathrm{rad}$

### Convert to arcsec

```python
angular_diameter = angular_diameter.to("arcsec")
angular_diameter
```

$1.4539343 \; \mathrm{{}^{\prime\prime}}$

### Find Pixel Size

In this section we convert the angular size into pixel size. The way we do this is by adding the angular size to the `CRVAL` of the WCS and then converting that world coordinate to pixel values. After getting the new pixel values we subtract the `CRPIX` values to find the difference (angular size). Because RA-DEC is an equatorial coordinate system, I have decided its best to add the angular size to the Dec component of the WCS CRVAL (Dec axis is a great circle). It looks like the image `CRVAL` is sufficiently away from the pols of the celestial sphere, so no need to worry about loops around +/- 90 degrees.

```python
# Load center pixel from data's WCS

ra_0 = wcs.wcs.crval[0] * u.Unit(wcs.wcs.cunit[0])
dec_0 = wcs.wcs.crval[1] * u.Unit(wcs.wcs.cunit[1])

print(ra_0, ",",  dec_0)
```

    39.96301 deg , -1.5882933 deg

```python
# Add angular_diameter to center Dec value

ra_1  = ra_0
dec_1 = dec_0 + angular_diameter

print(ra_1, ",",  dec_1)
```

    39.96301 deg , -1.5878894293737262 deg

```python
# Convert ra_1, dec_1 into pixels

world = np.array([[ra_1.value, dec_1.value],], dtype=np.float64) * u.deg
pixcrd = wcs.wcs_world2pix(world, 1)

pixcrd
```

    array([[4800.        , 6648.46447515]])

```python
# Look at the center pixel values
wcs.wcs.crpix
```

    array([4800., 6600.])

```python
# Find the difference b/w the center pixel and the angular offset pixel:
pixel_diff =  pixcrd - wcs.wcs.crpix
assert abs(pixel_diff[0,0]) < 1e-10
pixel_diff[0,1]
```

    48.46447515439377

```python
# Find ceiling of pixel diff to conclude pixel size
pixel_size = int(np.ceil(pixel_diff[0,1]))

pixel_size
```

    49

```python

```

notebooks/cosmology/Aperture Diameter of 7.5 kpc at Abell 370.ipynb


We will be exploring the *relative entropy* or Kullback-Leibler divergence (KL).

$$ D_{q}(p) = \mathbb{E}_{p}\bigg[\log\frac{p(x)}{q(x)}\bigg] $$

To build an intuition about what this important quantity is computing we examine the right hand side more closely. For continuous random vairables the KL is computed as follows

\begin{align}
\mathbb{E}_{p}\bigg[\log\frac{p(x)}{q(x)}\bigg] & = \int p(x) \log \frac{p(x)}{q(x)} dx \\
& = -\int p(x) \log \frac{q(x)}{p(x)} dx \\
& = -\int p(x) \log q(x) dx - \bigg(-\int p(x) \log p(x) \bigg ) dx\\
& = H_q(p) - H(p) \\
\end{align}

$H(p)$ is the entropy of $p$, or average amount of information received when observing a value from $p$. $H_q(p)$ is the cross entropy which is how much more information we need to represent an observation from $q$ by $p$. The KL is their difference, a measure of how imprecise we are when describing events from $p$ with $q$. In terms of inference KL measures what we gain by changing our prior from $q$ to $p$.

Let's take a look at how the KL changes as a function of some simple distributions. Let's let $p$ be a bimodal mixture of gaussians and q, a unimodal gaussian.

\begin{align}
p(x) & = \frac{2}{5}\mathcal{N}(x\lvert -3,1) + \frac{3}{5}\mathcal{N}(x,\lvert 3,1) \\
q(x) & = \mathcal{N}(x \lvert 0,1) \\
\end{align}

```python
import matplotlib.pyplot as plt
%matplotlib inline
from ipywidgets import interactive
from scipy.stats import norm
import numpy as np

class gmm: # gaussian mixture model

    def __init__(self, pis, params):
        self.params = params # [[mu1, sig1], [mu2, sig2],...]
        self.components = params.shape[0]
        self.pis = pis

    def __call__(self, x):
        pis = self.pis
        p = self.params
        sz = self.components
        return np.array([pis[i]*norm.pdf(x,*(p[i])) for i in range(sz)]).sum(axis=0)

    def sample(self, n_samples):
        mode_id = np.random.choice(self.components, size=n_samples, replace=True)
        return np.array([norm.rvs(*(self.params[i])) for i in mode_id])

p = gmm([0.4,0.6], np.array([[-3,1],[3,1]]))
q = gmm([1], np.array([[0,1]]))

fig, ax = plt.subplots(1, 1)
x = np.linspace(-7,7,1000)
ax.plot(x, p(x), c='r', lw=4, label='$p(x)$')
ax.plot(x, q(x), c='b', lw=4, label='$q(x)$')
ax.legend()
plt.show()
```

To calculate $D_q(p)$ we observe that the integral must be evaluated over all values of the sample space. Often times we cannot calculate the integral analytically. However with a sufficiently large sample $N$ we can approximate it

\begin{align}
D_q(p) & = \int_a^b p(x) \log \frac{p(x)}{q(x)} dx \\
& \approx \frac{b-a}{N} \sum_{n=1}^N p(x_n) \log \frac{p(x_n)}{q(x_n)}
\end{align}

```python
def forward_kl_int(p,q,n_samples):
    a = -10
    b = 10
    samples = np.linspace(a,b,n_samples)
    ps = p(samples)
    return (ps*np.log(ps/q(samples))).sum()*((b-a)/n_samples)

kls = []
for i in range(10,200):
   kls.append(forward_kl_int(p,q,i))

plt.plot(kls);
plt.show()
```

```python
def forward_kl_samp(p,q,n_samples):
    samples = p.sample(n_samples)
    ps = p(samples)
    return np.log(ps/q(samples)).sum()/n_samples

kls = []
for i in range(10,500):
   kls.append(forward_kl_samp(p,q,i))

plt.plot(kls);
plt.show()
```

Let us examine how the KL changes as we modify the parameters of $q$

```python
x = np.linspace(-7,7,1000)
kls = []

def f(mu, sig):
    q = gmm([1], np.array([[mu,sig]]))
    kl = forward_kl_int(p,q,200)
    kls.append(kl)
    fig, ax = plt.subplots(1, 2, figsize=(15,5), dpi=80)
    ax[0].plot(x, p(x), c='r', lw=4, label='$p(x)$')
    ax[0].plot(x, q(x), c='b', lw=4, label='$q(x)$')
    ax[0].legend()
    ax[1].plot(kls)
    plt.show()

interactive_plot = interactive(f, mu=(-3, 3, 0.1), sig=(0.05, 4, 0.1))
output = interactive_plot.children[-1]
output.layout.height = '35px'
interactive_plot
```

We find the minimum occurs when the mean of $q$ is between the two means of $p$. We can begin to see why this is by examining $D_q(p)$. We reproduce it here for reference.

\begin{align}
D_q(p) & = \int p(x) \log \frac{p(x)}{q(x)} dx \\
& = H_q(p) - H(p) \\
\end{align}

Noticing that the entropy of $p$, $H(p)$, does not depend on $q$ we see that minimizing $D_q(p)$ with respect to $q$ amounts to minimizing the cross entropy term $H_q(p)$ with respect to $q$.

\begin{equation}
H_q(p) = \int p(x) \log \frac{1}{q(x)} dx \\
\end{equation}

When $p$ is small, the logarithm term will not contribute much to the integral. To keep the integral small then, we need the logarithm term to be small when $p$ is large. In other words $D_q(p)$ is minimized when $q$ is chosen to be as large as possible whenever $p$ is large. Because $q$ is unimodal, this is achieved when $q$ is between the peaks of the mixture model and has high tails. This way it allocates high probability density on both modes.

### Forward vs Reverse KL

When $q$ is the variable distribution, $D_q(p)$ is called the *forward KL*. The forward KL tells us how inefficient it us for us to represent $q$ by $p$. So by minimizing the KL with respect to $q$ we found the optimal distribution $q^*$ that $p$ could represent. This is often not the goal in machine learning. More often we are presented with a distribution, or samples from it, and would like to find the optimal distribution to represent it. This is the reverse problem, and it can be conveniently expressed using the *reverse KL*, $D_p(q)$.

\begin{align}
D_p(q) & = \int q(x) \log \frac{q(x)}{p(x)} dx \\
& = H_p(q) - H(q) \\
\end{align}

It is important to note that we interpret the KL as either forward or reverse soley by which distribution is free.

```python
def reverse_kl_int(q,p,n_samples):
    a = -7
    b = 7
    samples = np.linspace(a,b,n_samples)
    qs = q(samples)
    return (qs*np.log(qs/p(samples))).sum()*((b-a)/n_samples)

kls = []
for i in range(10,400):
   kls.append(reverse_kl_int(q,p,i))

plt.plot(kls);
plt.show()
```

```python
x = np.linspace(-7,7,1000)
kls = []

def f(mu, sig):
    q = gmm([1], np.array([[mu,sig]]))
    kl = reverse_kl_int(q,p,2000)
    kls.append(kl)
    fig, ax = plt.subplots(1, 2, figsize=(15,5), dpi=80)
    ax[0].plot(x, p(x), c='r', lw=4, label='$p(x)$')
    ax[0].plot(x, q(x), c='b', lw=4, label='$q(x)$')
    ax[0].legend()
    ax[1].plot(kls)
    plt.show()

interactive_plot = interactive(f, mu=(-3, 3, 0.1), sig=(0.05, 6, 0.1))
output = interactive_plot.children[-1]
output.layout.height = '35px'
interactive_plot
```

Code from Tuan Anh Le's Blog for showing the difference between Forward and Reverse KL. Blog - http://www.tuananhle.co.uk/notes/reverse-forward-kl.html

```python
import numpy as np
import scipy as sp
import scipy.stats
import matplotlib.pyplot as plt

class GaussianMixture1D:
    def __init__(self, mixture_probs, means, stds):
        self.num_mixtures = len(mixture_probs)
        self.mixture_probs = mixture_probs
        self.means = means
        self.stds = stds

    def sample(self, num_samples=1):
        mixture_ids = np.random.choice(self.num_mixtures, size=num_samples, p=self.mixture_probs)
        result = np.zeros([num_samples])
        for sample_idx in range(num_samples):
            result[sample_idx] = np.random.normal(
                loc=self.means[mixture_ids[sample_idx]],
                scale=self.stds[mixture_ids[sample_idx]]
            )
        return result

    def logpdf(self, samples):
        mixture_logpdfs = np.zeros([len(samples), self.num_mixtures])
        for mixture_idx in range(self.num_mixtures):
            mixture_logpdfs[:, mixture_idx] = scipy.stats.norm.logpdf(
                samples,
                loc=self.means[mixture_idx],
                scale=self.stds[mixture_idx]
            )
        return sp.misc.logsumexp(mixture_logpdfs + np.log(self.mixture_probs), axis=1)

    def pdf(self, samples):
        return np.exp(self.logpdf(samples))

def approx_kl(gmm_1, gmm_2, xs):
    ys = gmm_1.pdf(xs) * (gmm_1.logpdf(xs) - gmm_2.logpdf(xs))
    return np.trapz(ys, xs)

def minimize_pq(p, xs, q_means, q_stds):
    q_mean_best = None
    q_std_best = None
    kl_best = np.inf
    for q_mean in q_means:
        for q_std in q_stds:
            q = GaussianMixture1D(np.array([1]), np.array([q_mean]), np.array([q_std]))
            kl = approx_kl(p, q, xs)
            if kl < kl_best:
                kl_best = kl
                q_mean_best = q_mean
                q_std_best = q_std

    q_best = GaussianMixture1D(np.array([1]), np.array([q_mean_best]), np.array([q_std_best]))
    return q_best, kl_best

def minimize_qp(p, xs, q_means, q_stds):
    q_mean_best = None
    q_std_best = None
    kl_best = np.inf
    for q_mean in q_means:
        for q_std in q_stds:
            q = GaussianMixture1D(np.array([1]), np.array([q_mean]), np.array([q_std]))
            kl = approx_kl(q, p, xs)
            if kl < kl_best:
                kl_best = kl
                q_mean_best = q_mean
                q_std_best = q_std

    q_best = GaussianMixture1D(np.array([1]), np.array([q_mean_best]), np.array([q_std_best]))
    return q_best, kl_best

def main():
    p_second_means_min = 0
    p_second_means_max = 10
    num_p_second_means = 5
    p_second_mean_list = np.linspace(p_second_means_min, p_second_means_max, num_p_second_means)

    p = [None] * num_p_second_means
    q_best_forward = [None] * num_p_second_means
    kl_best_forward = [None] * num_p_second_means
    q_best_reverse = [None] * num_p_second_means
    kl_best_reverse = [None] * num_p_second_means

    for p_second_mean_idx, p_second_mean in enumerate(p_second_mean_list):
        p_mixture_probs = np.array([0.5, 0.5])
        p_means = np.array([0, p_second_mean])
        p_stds = np.array([1, 1])
        p[p_second_mean_idx] = GaussianMixture1D(p_mixture_probs, p_means, p_stds)

        q_means_min = np.min(p_means) - 1
        q_means_max = np.max(p_means) + 1
        num_q_means = 20
        q_means = np.linspace(q_means_min, q_means_max, num_q_means)

        q_stds_min = 0.1
        q_stds_max = 5
        num_q_stds = 20
        q_stds = np.linspace(q_stds_min, q_stds_max, num_q_stds)

        trapz_xs_min = np.min(np.append(p_means, q_means_min)) - 3 * np.max(np.append(p_stds, q_stds_max))
        trapz_xs_max = np.max(np.append(p_means, q_means_min)) + 3 * np.max(np.append(p_stds, q_stds_max))
        num_trapz_points = 1000
        trapz_xs = np.linspace(trapz_xs_min, trapz_xs_max, num_trapz_points)

        q_best_forward[p_second_mean_idx], kl_best_forward[p_second_mean_idx] = minimize_pq(
            p[p_second_mean_idx], trapz_xs, q_means, q_stds
        )
        q_best_reverse[p_second_mean_idx], kl_best_reverse[p_second_mean_idx] = minimize_qp(
            p[p_second_mean_idx], trapz_xs, q_means, q_stds
        )

    # plotting
    fig, axs = plt.subplots(nrows=1, ncols=num_p_second_means, sharex=True, sharey=True)
    fig.set_size_inches(8, 1.5)
    for p_second_mean_idx, p_second_mean in enumerate(p_second_mean_list):
        xs_min = -5
        xs_max = 15
        num_plot_points = 1000
        xs = np.linspace(xs_min, xs_max, num_plot_points)
        axs[p_second_mean_idx].plot(xs, p[p_second_mean_idx].pdf(xs), label='$p$', color='black')
        axs[p_second_mean_idx].plot(xs, q_best_forward[p_second_mean_idx].pdf(xs), label='$\mathrm{argmin}_q \,\mathrm{KL}(p || q)$', color='black', linestyle='dashed')
        axs[p_second_mean_idx].plot(xs, q_best_reverse[p_second_mean_idx].pdf(xs), label='$\mathrm{argmin}_q \,\mathrm{KL}(q || p)$', color='black', linestyle='dotted')

        axs[p_second_mean_idx].spines['right'].set_visible(False)
        axs[p_second_mean_idx].spines['top'].set_visible(False)
        axs[p_second_mean_idx].set_yticks([])
        axs[p_second_mean_idx].set_xticks([])

    axs[2].legend(ncol=3, loc='upper center', bbox_to_anchor=(0.5, 0), fontsize='small')
    filenames = ['reverse_forward_kl.pdf', 'reverse_forward_kl.png']
    for filename in filenames:
        fig.savefig(filename, bbox_inches='tight', dpi=200)
        print('Saved to {}'.format(filename))

if __name__ == '__main__':
    main()
```

    /Users/nathancrock/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:32: DeprecationWarning: `logsumexp` is deprecated!
    Importing `logsumexp` from scipy.misc is deprecated in scipy 1.0.0. Use `scipy.special.logsumexp` instead.

    Saved to reverse_forward_kl.pdf
    Saved to reverse_forward_kl.png

$$I(x) = -\log p(x)$$

```python
import numpy as np
import matplotlib.pyplot as plt
x = np.linspace(0.001,1.2,1000)
y = -np.log(x)
plt.plot(x,y)
plt.plot([0,1.2],[0,0],ls='dashed',c='k')
plt.show()
```

$$ H(p) = \mathbb{E}_{p}\big[-\log p(x)\big] = \int p(x) \big(- \log p(x) \big) dx$$

```python
x = np.linspace(0.001,0.9999,1000)
y = x*(-np.log2(x)) + (1-x)*(-np.log2(1-x))
plt.plot(x,y)
plt.plot([0,0.99],[0,0],ls='dashed',c='k')
plt.show()
```

$$ \min_{q(x)} KL(q(x)||p(x)) $$

```python

```

```python

```


# Exponential Smoothing Procedures

```python
import pandas as pd
import numpy as np
import statsmodels as sm
```

```python
sm.__version__
```

## Aims of Exponential Smoothing Procedures
* In the naive forecasting method we assumed that the most recent observation was the most important.
* In the average method, we used all observations, but gave them all equal weight (they were all equally important).
* Exponential smoothing falls between these two extremes.
 * ES forecasts are weighted averages of past observations.
 * More recent observations carry more weight than older ones;
 * Or to put it another way: the weights decrease exponentially as the observations get older.

## Types of Exponential Smoothing Procedure
* **Simple Exponential Smoothing (SES)**
 * No trend or seasonality
* **Holt's Linear Method**
 * Extends (SES) to include a linear trend
* **Holt-Winters Exponential Smoothing (HW)**
 * The most complex procedure that handles both trend and seasonality

Both Holt's linear method and HW can include a damped trend.

## Simple Exponential Smoothing (SES)
SES consists of two equations the forecast equation (1.1) and the smoothing equation (1.2):

SES

$F_{t+h} = l_t \tag{1.1}$

$l_{t} = \alpha y_t + (1 - \alpha) l_{t-1} \tag{1.2} $

**where**
* $\alpha$ = a smoothing parameter between 0 and 1.
* $l_t$ = the current level at time t.
* $y_t$ = The ground truth / real world observation at time t

If you haven't worked with equations for a while this might look complicated.   The reality is that it is a very simple method. To output a forecast involves plugging a few numbers.

Equation (1.2) is called the smoothing or level equation.  In words level ($l_{t}$) is based on weighting the most recent observation ($y_t$) by a smoothing constant called $\alpha$ and weighting the previous level by $(1-\alpha)$.

**Smoothing Example 1**

* $y_t = 150$
* $l_{t-1} = 120$
* $alpha = 0.2$
* $l_{t} = \alpha y_t + (1 - \alpha) l_{t-1}$

```python
def smooth_level(obs, level, alpha):
    '''Returns a exponentially smoothed level assuming
    no trend or seasonality'''
    return (alpha * obs) + ((1 - alpha)*level)
```

```python
current_obs = 150
last_level = 120
alpha = 0.2

#call the function that implements the smoothing eq.
smooth_level(current_obs, last_level, alpha)
```

**Smoothing Example 2**

* $y_t = 150$
* $l_{t-1} = 120$
* $alpha = 0.8$

**Question: What happens when alpha is set to 0.0 or 1.0?**

**What does this tell you about the role of alpha?**

```python
current_obs = 150  #y_t
last_level = 120   #l_t-1
alpha = 0.8

#call the function that implements the smoothing eq.
smooth_level(current_obs, last_level, alpha)
```

**Equation (1.1)** is called the *forecast equation*.  In words is means that the forecast h steps ahead is equal to the current level.  In otherwords **it is a flat forecast**.  It just carries the last value produced the smoothing equation forward.  We will see a visual example of that shortly.

**SES Forecast Example**

Given the following inputs, create a 6 step ahead forecast.

* $y_t = 150$
* $l_{t-1} = 120$
* $alpha = 0.2$

```python
def flat_forecast(level, horizon):
    '''Returns a vector of length horizon with all values
    set to level'''
    return np.full(shape=horizon, fill_value=level)
```

```python
current_obs = 150
last_level = 120
ALPHA = 0.2
HORIZON = 6

#call the function that implements the smoothing eq.
new_level = smooth_level(current_obs, last_level, ALPHA)
flat_forecast(new_level, HORIZON)
```

**The good news is that you do not need to implement SES or the other more complex versions of Exponential Smoothing.**

[statsmodels](https://www.statsmodels.org/stable/index.html) is a Python module that provides classes and functions for the estimation of many different statistical models, as well as for conducting statistical tests, and statistical data exploration.

There are two statsmodels libraries that you can use for SES.

```python
statsmodels.tsa.holtwinters.SimpleExpSmoothing
```
The `SimpleExpSmoothing` [class](https://www.statsmodels.org/stable/generated/statsmodels.tsa.holtwinters.SimpleExpSmoothing.html?highlight=simpleexpsmoothing#statsmodels.tsa.holtwinters.SimpleExpSmoothing) implements the SES equations described above.  It is fast easy to use and provides an optimisation procedure to automatically select the best $\alpha$ value.

```python
statsmodels.tsa.statespace.exponential_smoothing.ExponentialSmoothing
```

The `ExponentialSmoothing` class implements SES as a **statistical model**.  The theory of this is beyond the scope of this tutorial, but the fundermental idea is that the class provides a statistical model that is equivalent to the mathematical model outlined above.  The advantage of the statistical model is that point forecasts can be enhanced with a **prediction interval**.  Where possible, it is recommended that point forecasts are always accompanied by a prediction interval.  For this reason it is recommended that the [statespace implementation](https://www.statsmodels.org/stable/generated/statsmodels.tsa.statespace.exponential_smoothing.ExponentialSmoothing.html?highlight=statespace%20exponential) is used over and above `SimpleExpSmoothing`.

For more information on the statespace formulation I recommend reading the relevant [chapter](https://otexts.com/fpp2/ets.html) Prof Rob Hyndman's free and open book on forecasting.

**SES Example: Nile flow data 1871 to 1930**

```python
nile = pd.read_csv('data/nile.csv', index_col='year', parse_dates=True)
nile.index.freq = "AS"
```

```python
nile['flow'].plot(figsize=(12,4))
```

```python
from statsmodels.tsa.statespace.exponential_smoothing import ExponentialSmoothing
```

```python
#lets predict the next dacade of flow.
HORIZON = 10

#for SES pass in the endog argument as your data
model = ExponentialSmoothing(endog=nile['flow'])
results = model.fit()

preds = results.get_forecast(steps=HORIZON)

#the summary_frame() method returns a pandas data frame.
#here alpha refers to alpha for a prediction interval 0.2 = 80% pred interval.
#make sure you don't confuse this alpha with the SES smoothing parameter!
preds.summary_frame(alpha=0.2).head(3)
```

```python
#plot the fitted values and prediction
ax = nile['flow'].plot(figsize=(12,4))
forecast_80 = preds.summary_frame(alpha=0.2)[['mean', 'mean_ci_lower', 'mean_ci_upper']]
forecast_90 = preds.summary_frame(alpha=0.1)[['mean', 'mean_ci_lower', 'mean_ci_upper']]

ax.fill_between(forecast_90.index,forecast_90['mean_ci_lower'], forecast_90['mean_ci_upper'],
                alpha=0.5,
                label='90% PI');

ax.fill_between(forecast_80.index,forecast_80['mean_ci_lower'], forecast_80['mean_ci_upper'],
                alpha=0.5,
                label='80% PI');

forecast_80['mean'].plot(ax=ax, label='forecast', color='red');

results.fittedvalues.plot(ax=ax, label='fitted', color='green', linestyle='--')

ax.legend(loc=3);
```

```python
#lets have a look at the fitted model
results.summary()
```

## Holt's Method for Linear Trend

Holt's linear method adds a second smoothing parameter $\beta$ and a third equation representing the trend.

Holt's Linear Method

$F_{t+h} = l_t + hb_t \tag{2.1}$

\begin{equation}
    l_t = \alpha Y_t + (1 - \alpha) (l_{t-1} + b_{t-1}) \tag{2.2}
\end{equation}

\begin{equation}
    b_t = \beta (l_t - l_{t-1}) + (1 - \beta)b_{t-1} \tag{2.3}
\end{equation}

**Example: US Gross Domestic Product 1920 to 2019.**

```python
train = pd.read_csv('data/GDPCA.csv', index_col='DATE', parse_dates=True)
train.index.freq = 'AS'
train.plot(figsize=(12,4));
```

```python
#forecast 30 years ahead.
HORIZON = 30

#pass in the trend parameter as true
model = ExponentialSmoothing(endog=train, trend=True)
results = model.fit()

preds = results.get_forecast(steps=HORIZON)

#here alpha refers to alpha for a prediction interval
#(not to be confused with the smoothing parameter!)
preds.summary_frame(alpha=0.2).head(2)
```

```python
ax = train.plot(figsize=(12,4))

forecast_80 = preds.summary_frame(alpha=0.2)[['mean', 'mean_ci_lower', 'mean_ci_upper']]
forecast_90 = preds.summary_frame(alpha=0.1)[['mean', 'mean_ci_lower', 'mean_ci_upper']]

ax.fill_between(forecast_90.index,forecast_90['mean_ci_lower'], forecast_90['mean_ci_upper'],
                alpha=0.5,
                label='90% PI');

ax.fill_between(forecast_80.index,forecast_80['mean_ci_lower'], forecast_80['mean_ci_upper'],
                alpha=0.5,
                label='80% PI');

forecast_90['mean'].plot(ax=ax, color='red')
results.fittedvalues.plot(ax=ax, color='green', linestyle='--')
ax.legend(['train', 'point forecast', 'fitted','90%PI', '80% PI']);
```

```python
results.summary()
```

### Introducing a damped trend.

It is often beneficial to introduce a damped trend into long term forecasting.  The following code illustrates the procedure.

```python
#forecast 30 years ahead.
HORIZON = 30

#note the damped_trend parameter
model = ExponentialSmoothing(endog=train, trend=True, damped_trend=True)
results = model.fit()
preds = results.get_forecast(steps=HORIZON)

ax = train.plot(figsize=(12,4))
forecast_80 = preds.summary_frame(alpha=0.2)[['mean', 'mean_ci_lower', 'mean_ci_upper']]
forecast_90 = preds.summary_frame(alpha=0.1)[['mean', 'mean_ci_lower', 'mean_ci_upper']]
ax.fill_between(forecast_90.index,forecast_90['mean_ci_lower'], forecast_90['mean_ci_upper'],
                alpha=0.5,
                label='90% PI');
ax.fill_between(forecast_80.index,forecast_80['mean_ci_lower'], forecast_80['mean_ci_upper'],
                alpha=0.5,
                label='80% PI');
forecast_90['mean'].plot(ax=ax, color='red')
results.fittedvalues.plot(ax=ax, color='green', linestyle='--')
ax.legend(['train', 'point forecast', 'fitted','90%PI', '80% PI']);
```

## Holt-Winters Exponential Smoothing

* Holt-Winters (HW) Exponential Smoothing procedures handle **trend and seasonality**.
* There are two versions of HW that that handle additive and multiplicative seasonality.
* In both cases a seasonality equation and a seasonal smoothing constant $\gamma$ are added.
* The level $l_t$ is seasonally adjusted
 * additive model: the seasonal component is an absolute value subtracted from the level
 * multiplicative model: the seasonal component is a percentage.  The level is divided by it.

### Additive Method

The additive approach is used when the variance in the data is roughly constant.

HW Additive

$F_{t+h} = l_t + hb_t + s_{t+h-m(k+1)}\tag{3.1}$

\begin{equation}
    l_t = \alpha (y_t - s_{t-m}) + (1 - \alpha) (l_{t-1} + b_{t-1}) \tag{3.2}
\end{equation}

\begin{equation}
    b_t = \beta (l_t - l_{t-1}) + (1 - \beta)b_{t-1} \tag{3.3}
\end{equation}

\begin{equation}
    s_t = \gamma (y_t - l_t - s_{t-m}) + (1 - \gamma) s_{t-m} \tag{3.4}
\end{equation}

**Additive example.  Australian quarterly beer production**

```python
from pmdarima.datasets import load_ausbeer
```

```python
ausbeer = load_ausbeer(as_series=True)
#index is 1956:Q1 to 2008:Q3
ausbeer.index = pd.date_range(start='1956Q1', periods=212, freq='Q')
ausbeer.plot(figsize=(12,4));
```

```python
#lets predict the next 10 years
HORIZON = 40

#for SES pass in the endog argument as your data
model = ExponentialSmoothing(endog=ausbeer, seasonal=4)
results = model.fit()

preds = results.get_forecast(steps=HORIZON)
preds.summary_frame(alpha=0.2).head(3)
```

```python
ax = ausbeer.plot(figsize=(12,4))
forecast_80 = preds.summary_frame(alpha=0.2)[['mean', 'mean_ci_lower', 'mean_ci_upper']]
forecast_90 = preds.summary_frame(alpha=0.1)[['mean', 'mean_ci_lower', 'mean_ci_upper']]
ax.fill_between(forecast_90.index,forecast_90['mean_ci_lower'], forecast_90['mean_ci_upper'],
                alpha=0.5,
                label='90% PI');
ax.fill_between(forecast_80.index,forecast_80['mean_ci_lower'], forecast_80['mean_ci_upper'],
                alpha=0.5,
                label='80% PI');
forecast_90['mean'].plot(ax=ax, color='red')
results.fittedvalues.plot(ax=ax, color='green', linestyle='--')
ax.legend(['train', 'point forecast', 'fitted','90%PI', '80% PI'], bbox_to_anchor=(1.05, 1), loc=2);
```

```python
results.summary()
```

### Multiplicative Method

The multiplicative approach is used when variance in the data increases/decreases over time.

HW Mutliplicative
$F_{t+h} = (l_t + hb_t)s_{t+h-m(k+1)}\tag{3.5}$

\begin{equation}
    l_t = \alpha \frac{y_t}{s_{t-m}} + (1 - \alpha) (l_{t-1} + b_{t-1}) \tag{3.6}
\end{equation}

\begin{equation}
    b_t = \beta (l_t - l_{t-1}) + (1 - \beta)b_{t-1} \tag{3.7}
\end{equation}

\begin{equation}
    s_t = \gamma \frac{y_t}{l_t - s_{t-m}} + (1 - \gamma) s_{t-m} \tag{3.8}
\end{equation}

**Multiplicative Seasonality Example: Alcohol Sales $m**

```python
#Example sales of beer wine IN $m
train = pd.read_csv('data/Alcohol_Sales.csv', index_col='DATE', parse_dates=True)
train.index.freq = 'MS'
train.plot(figsize=(12,4));
```

```python
from statsmodels.tsa.holtwinters import ExponentialSmoothing
```

```python
#forecast 120 months ahead.
HORIZON = 120

#trend='add' means model with a linear trend. seasonal='mul' means to use multiplicative seasoniality.
model = ExponentialSmoothing(endog=train, trend='add', seasonal='mul',
                             seasonal_periods=12)
results = model.fit()
preds = results.forecast(steps=HORIZON)
```

```python
idx = pd.date_range(start='2020-01-01', periods=HORIZON, freq='MS')
ax = train.plot(figsize=(12,4))
results.fittedvalues.plot(ax=ax, color='green', linestyle='--')
pd.Series(preds, index=idx).plot(ax=ax);
ax.legend(['train', 'point forecast', 'fitted']);
```

* **Question: what happens if we assume additive seasonality or multiplicative trend?**

```python
#let's have a look at the fitted model
results.summary()
```


# Exercise 1 - The Mandelbrot Set

The mandelbrot set is a beautiful fractal ([Wikipedia](http://en.wikipedia.org/wiki/Mandelbrot_set)).
More precisely, it contains all numbers from the complex plane where the complex quadratic polynomial

\begin{align}
    z_{n+1} = z_{n}^2 + c
\end{align}

remains bounded. A complex number $c$ is part of the Mandelbrot set when starting with $z_0=0$ and applying the iteration repeatedly, the absolute value of $z_n$ remains smaller or equal than 2 regardless how large $n$ becomes.

Moreover, one can draw very beautiful images by looking at the *escape time* of a numerical computation:
We will numerically determine whether a complex number $c$ belongs to the set, and if not we will keep track how many iterations are needed until $|z_n| > 2$. Plotting the escape times of a sampled grid of complex numbers as a 2D image will yield the famous *Apfelmaennchen* you might have seen before.

#Task:#

* Write a function ``mandelbrot(relim, imlim, resteps, imsteps, maxiterations)`` that computes and returns the escape times for the mandelbrot set.
    * ``relim`` and ``imlim`` are tuples that define the boundary of the complex plane (e.g. ``relim=(-2,1)`` and ``imlim=(-1,1)``), for convenience ``imlim`` should contain real numbers although it represents the complex axis.
    * ``resteps`` and ``imsteps`` are integers and define the sampling resolution along each axis (e.g. 300 and 200 steps).
    * ``maxiterations`` is an integer defining the maximum number of iterations (e.g. 50).

    * The function should sample complex numbers from the plane as defined by the parameters ``relim, imlim, resteps, imsteps`` and repeatedly apply the quadratic polynomial from above until the absolute value of a complex number is larger than 2. In case a maximum number of iterations is reached (``maxiterations``) the number is believed to be part of the set.
    * The function should return a 2D array containing the number of iterations needed for every sampled complex number and two 1D arrays containing the sampled values along each axis. Escape times in the 2D array of exactly ``maxiterations`` indicate complex numbers that are believed to be part of the set.
* Make a 2D color plot of the escape times, you might want to look at matplotlib's [imshow](http://www.mathworks.de/de/help/images/ref/imshow.html) function.

###Hints how the function ``mandelbrot`` may work:###
* Create a 2D numpy array containing the escape times, in the beginning filled with zeros.

* Secondly, create two 1D arrays sampling each dimension of the complex plane.  You might want to take a look at the [np.linspace](http://docs.scipy.org/doc/numpy/reference/generated/numpy.linspace.html) function. You can multiply one of the arrays with ``1j`` to create the imaginary axis. Later on you can calculate each starting complex number simply from summing individual elements of both arrays.

* The easy brute-force solution involves three nested for loops. Be aware that you can use python's ``break`` statement to leave a for loop and spare unnecessary computations.

* Iterate at most ``maxiterations`` times for any value in your complex plane and apply the quadratic polynomial from above. If the absolute value gets larger than 2 you can stop iterating and store the number of iterations you have needed in the 2D escape array. Thus, for complex numbers that are actually part of the Mandelbrot set, the 2D array of escape times should contain the value ``maxiterations``.

###Hints for optimization:###
* You can aim for an optimized version that uses vectorized computation. Here you should only need a single for loop.

* Create a second and third 2D array containing the starting complex numbers and the intermediate values, checkout the [meshgrid](http://docs.scipy.org/doc/numpy/reference/generated/numpy.meshgrid.html) function to create these.

* Use the second 2D array to iteratively update all values in the third array at once (vectorized computation!). Mask all values that already escaped the boundary via boolean indexing to speed up the computation.

* Remember that boolean indexing creates copies, not views! So you need to find a smart way to only apply the polynomial
to values that haven't escaped without always keeping the full 2D array, otherwise internally numpy has to reiterate the full array every timestep. If you also create mesh grids of matrix indices you can reduce the 2D arrays each iteration to 1D arrays containing only the currently not escaped values. This is tricky!

* Now you can use finer resolutions than above!

# Brute-Force Solution

```
import numpy as np

def mandelbrot(relim=(-2,1), imlim=(-1, 1), resteps=300, imsteps=200, maxiterations=50):
    """Computes the *escape times* of the numeric approximation of the Mandelbrot set.

    Values of ``maxiterations`` refer to complex numbers that are believed to be part of the set.

    :param relim: The limits of the real axis
    :param imlim: The limits of the imaginary axis
    :param resteps: Sampling resolution along real axis
    :param imsteps: Sampling resolution along the imaginary axis
    :param maxiterations: Maximum number of iterations

    :return:

        2D Array of escape times
        1D Array of real samples
        1D Array of imaginary samples

    """
    realaxis = np.linspace(relim[0], relim[1], resteps) # Sample the real axis
    imaxis = np.linspace(imlim[0], imlim[1], imsteps) * 1j # Sample the imaginary axis
    escape = np.zeros((imsteps, resteps), dtype=int) # 2D array of escape times

    for irun in range(resteps): # Iterate over real axis
        for jrun in range(imsteps): # Iterate over imaginary axis
            c = realaxis[irun] + imaxis[jrun] # Starting value
            z = 0.0 # Helper variable
            for krun in range(maxiterations): # Compute the escape time
                z = z**2 + c
                if np.abs(z) >= 2:
                    break # c is not part of the Mandelbrot set
            else:
                krun = maxiterations # If the for loop did not break c is part of the set
            escape[jrun, irun] = krun

    return escape, realaxis, imaxis

```

```
xlim = (-2,1)
ylim = (-1,1)
xsteps = 300
ysteps = 200
maxiterations = 50
```

```
escape, realaxis, imaxis = mandelbrot(xlim, ylim, xsteps, ysteps, maxiterations)
```

```
import matplotlib.pyplot as plt
%matplotlib inline

plt.imshow(escape, aspect='auto', origin='lower')
plt.xticks(np.arange(xsteps)[::50], ['%.2f' % x for x in realaxis[::50]])
plt.yticks(np.arange(ysteps)[::50], ['%.2f' % x.imag for x in imaxis[::50]])
plt.xlabel('real axis')
plt.ylabel('imaginary axis')
```

# Optimized Version

```
def mandelbrot_pro(relim=(-2,1), imlim=(-1, 1), resteps=1500, imsteps=1000, maxiterations=50):
    """As function mandelbrot, but faster if the number of maxiterations is rather low.

    It uses a vectorized approach.

    """
    realaxis = np.linspace(relim[0], relim[1], resteps)
    imaxis = np.linspace(imlim[0], imlim[1], imsteps) * 1j
    escape = np.ones((imsteps, resteps), dtype=int) * maxiterations

    realarray, imarray = np.meshgrid(realaxis, imaxis)
    complexarray = realarray + imarray

    indices_real = np.arange(len(realaxis)) # index array
    indices_im = np.arange(len(imaxis)) # index array
    mesh_indices_real, mesh_indices_im = np.meshgrid(indices_real, indices_im) # Create a mesh version

    zarray = complexarray.copy()

    for irun in range(maxiterations):

        if len(zarray) == 0:
            break

        zarray *= zarray # Application of the polinomial
        zarray += complexarray

        mask = np.abs(zarray) >= 2 # Mask all values that already escaped

        escape[mesh_indices_im[mask], mesh_indices_real[mask]]=irun # Remembert the escape time

        inv_mask = np.invert(mask) # Invert the mask to keep only values that have not escaped, yet
        mesh_indices_real = mesh_indices_real[inv_mask] # Filter the indeices
        mesh_indices_im = mesh_indices_im[inv_mask]
        zarray = zarray[inv_mask] # Filter the z values
        complexarray = complexarray[inv_mask] # Filter the starting values

    return escape, realaxis, imaxis
```

```
xlim = (-2,1)
ylim = (-1,1)
xsteps = 1500
ysteps = 1000
maxiterations = 50
```

```
escape, realaxis, imaxis = mandelbrot_pro(xlim, ylim, xsteps, ysteps, maxiterations)
```

```
plt.imshow(escape, aspect='auto', origin='lower')
plt.xticks(np.arange(xsteps)[::100], ['%.2f' % x for x in realaxis[::100]])
plt.yticks(np.arange(ysteps)[::100], ['%.2f' % x.imag for x in imaxis[::100]])
plt.xlabel('real axis')
plt.ylabel('imaginary axis')
```

```
"""Short timing test"""
%timeit mandelbrot((-2,1), (-1,1), 45, 30, 10)
%timeit mandelbrot_pro((-2,1), (-1,1), 45, 30, 10)
```

    10 loops, best of 3: 41.9 ms per loop
    1000 loops, best of 3: 673 µs per loop

# Exercise 2 - Numerical Integration of a Neuron Model

We are going to simulate our first neuron model with Euler integration ([Wikipedia](http://en.wikipedia.org/wiki/Euler_method)).
Basically, our neuron model consists of a single differential equation that describes the development of the membrane voltage over time. For now let's assume this equation is an arbitrary function $f$:
\begin{align}
    \frac{dV}{dt} = f(V)
\end{align}
To obtain the voltage as a function of time, i.e. $V(t)$, we have to solve the differential equation. Lucky for you, we are in the computer practical and not the analytical tutorial. We are going to solve it numerically, so there's no need for a complicated Ansatz :-)

As said before, we will use simple Euler integration. Accordingly, if we assume discretized time we can easily compute $V(t+1)$, the membrane voltage of the next time step, in case we now the previous voltage $V(t)$:
\begin{align}
V(t+1) = V(t) + f(V) * dt
\end{align}
with $dt$ the size of the discretized timesteps.

If we start with a chosen initial value of $V(0)$ we can iteratively solve the differential equation.

By this method we can simulate very complex neuron models. Let's simulate a rescaled version of the exponential integrate and fire neuron ([Scholarpedia](http://www.scholarpedia.org/article/Adaptive_exponential_integrate-and-fire_model)):
\begin{align}
    \frac{dV}{dt} = -V + \exp(V) + I
\end{align}

$I$ describes a fixed input current. We will simulate several neurons fed with different current values.
For some values of $I$ the membrane potential $V$ will rise to infinity, this corresponds to the upstroke of an action potential (you remember action potentials from a neurobio course, right?). However, our neuron model cannot recover from this upstroke by itself. For a smooth recovery we would need a second differential equation. However, we will keep our model simple and add a so called *reset rule*: whenever $V$ crosses a particular threshold $V(t)\geq V_t$ then we set it back to a reset value at the next timestep $V(t+1)=V_r$.

#Task#

* Write a function ``expIF_neuron(V, I, Vt, Vr, duration, dt)`` that simulates one or more exponetial integrate-and-fire neurons.
    * The parameter ``V`` can be a scalar or numpy array describing the initial conditions
    * The parameter ``I`` can be a scalar or numpy array describing the input currents
    * The parameter ``Vt`` is a scalar value defining the spiking threshold
    * The parameter ``Vr`` is a scalar defining the reset value after threshold crossing
    * The parameter ``duration`` gives the length of the simulation
    * The parameter ``dt`` describes the stepsize of the Euler integration
    * The function should return
        * A 2D array of voltage traces, i.e. the simulated development of the membrane potential
            * First dimension is the number of neurons, second dimension the voltage trace over time
            * First entries in second dimension should contain the initial values
        * A 1D array containing the discretized timesteps
    * In case ``V`` and ``I`` are arrays, the function should be vectorized, i.e. there should only be a single loop over all timesteps, but no loop over all neurons!

* Simulate 5 neurons at once (do NOT call the function 5 times!) with 5 different input currents $I\in\{-3.0, -2.0, -1.0, 0.0, 1.0\}$.
    * Choose the other parameters as
        * $V_r=-1.0$
        * $V_t=5.0$
        * $duration=10.0$
        * $dt = 0.01$
    * Set the initial $V$ values to $V_r$
* Plot all 5 voltage traces in a single plot, add a legend and label the axis.

###Hints###
* You can use the template provided below. This exercise can be solved in just a handful of lines ;-)
* To incorporate the reset rule you may try boolean indexing.

```
## The template: ##

def expIF_neuron(V=-1.0, I=0.0, Vt=5.0, Vr=-1.0, duration=10.0, dt=0.01):
    """Numerically integrates the expIF neuron membrane equation with the Euler-Method.

    The neurons obey a reset rule, when the membrane potential crosses `Vt`
    it is set back to `Vr`.

    :param V: array of initial membrane values (or a scalar)
    :param VT: spiking threshold (scalar)
    :param I: array of input currents (or scalar)
    :param duration: duration of experiment (scalar)
    :param dt: stepsize of Euler integration (scalar)

    :return:

        2D array of voltage time series, first dimension the neurons,
        second dimension the voltage trace.
        First entries contain the initial values.

        1D array of simulation times

    """

    steps = int(duration/dt) # Calculate the number of simulation steps

    if isinstance(V, np.ndarray): # V can be scalar or an array, we need to check first
        nneurons = len(V) # Infer the number of neurons from the length of the initial conditions
    else:
        nneurons = 1

    V_series = np.zeros((nneurons, steps+1)) # Array that will contain the voltage traces
    # 1st dim neurons, 2nd dim voltage traces
    # i.e. V[2,10] would return the voltage of neuron #2 at the 10th timestep!
    # Wee need steps+1 since the 0th entry should contain the initial conditions

    V_series[:, 0] = V # Set initial conditions

    times = np.zeros(steps+1) # Array of timesteps

    for step in range(1, steps+1): # Loop starting from step 1 (0th contains initial conditions)

        ############# Your code ##############

        # Manipulate V_series here to simulate the neuron model
        # Iteratively compute f(V(t)) = -V(t) + exp(V(t)) + I and V(t+1) = V(t) + f(V(t)) * dt
        # Do not introduce another for loop, try to think vectorized
        # Try using boolean indexing to implement the threshold crossing and voltage reset

        ######### End of your code ###########

        times[step] = times[step-1] + dt # You actually don't need the times explicitly, but returning them
        # will make plotting easier

    return V_series, times

```

```
def expIF_neuron(V=-1.0, I=0.0, Vt=5.0, Vr=-1.0, duration=10.0, dt=0.01):
    """Numerically integrates the expIF neuron membrane equation with the Euler-Method.

    The neurons obey a reset rule, when the membrane potential crosses `Vt`
    it is set back to `Vr`.

    :param V: array of initial membrane values (or a scalar)
    :param VT: spiking threshold (scalar)
    :param I: array of input currents (or scalar)
    :param duration: duration of experiment (scalar)
    :param dt: stepsize of Euler integration (scalar)

    :return:

        2D array of voltage time series, first dimension the neurons,
        second dimension the voltage trace.
        First entry contains the initial values.

        1D array of simulation times

    """

    steps = int(duration/dt) # Calculate the number of simulation steps

    if isinstance(V, np.ndarray): # V can be scalar or array, we need to check first
        nneurons = len(V)
    else:
        nneurons = 1

    V_series = np.zeros((nneurons, steps+1)) # Array that will contain the voltage traces
    # 1st dim neurons, 2nd dim voltage traces
    # i.e. V[2,10] would return the voltage of neuron 3 at the 11th timestep!
    # Wee need steps+1 since the 0th entry should contain the initial conditions
    V_series[:, 0] = V # Set initial conditions

    times = np.zeros(steps+1) # Array of timesteps

    for step in range(1, steps+1): # Loop starting from step 1 (0th contains initial conditions)

        prev_V = V_series[:, step-1]
        dV = -prev_V + np.exp(prev_V) + I

        next_V = prev_V + dt * dV # Euler step
        next_V[prev_V>=Vt] = Vr # Voltage reset
        prev_V[prev_V>=Vt] = Vt # For better plotting we bound also the previous step

        V_series[:, step] = next_V

        times[step] = times[step-1] + dt # You actually don't need the times explicitly, but returning them
        # will make plotting easier

    return V_series, times

```

```
nneurons = 5
Vt = 5.0
Vr = -1.0
dt = 0.01
duration = 10.0
I = np.linspace(-3.,1.0, nneurons)
V = np.ones(nneurons)*-1.0
```

```
V_series, times = expIF_neuron(V, I, Vt, Vr, duration, dt)
```

```
for neuron in range(nneurons):
    plt.plot(times, V_series[neuron,:], linewidth=2, label='I=%.1f' % (I[neuron]))

ax = plt.gca()
ax.ticklabel_format(useOffset=False) # prevents strange units
plt.xlabel('time')
plt.ylabel('V')
plt.legend()
```

```

```


# Pearson Correlation Vs. Z-normalized Euclidean Distance

It is [well understood](https://arxiv.org/pdf/1601.02213.pdf) that the z-normalized Euclidean distance, $ED_{z-norm}$, and the Pearson correlation, $PC$, between any two subsequences with length $m$ share the following relationship:

$ED_{z-norm} = \sqrt {2 * m * (1 - PC)}$

Naturally, when the two subsequences are perfectly correlated (i.e., $PC = 1$), then we get:

\begin{align}
    ED_{z-norm} ={}&
        \sqrt {2 * m * (1 - PC)}
    \\
    ={}&
        \sqrt {2 * m * (1 - 1)}
    \\
    ={}&
        \sqrt {2 * m * 0}
    \\
    ={}&
        \sqrt {0}
    \\
    ={}&
        0
    \\
\end{align}

Similarly, when the two subsequences are completely uncorrelated (i.e., $PC = 0$), then we get:

\begin{align}
    ED_{z-norm} ={}&
        \sqrt {2 * m * (1 - PC)}
    \\
    ={}&
        \sqrt {2 * m * (1 - 0)}
    \\
    ={}&
        \sqrt {2 * m * 1}
    \\
    ={}&
        \sqrt {2 * m}
    \\
\end{align}

In other words, the largest possible z-normalized distance between any pair of subsequences with length $m$ is $\sqrt{2 * m}$. The maximum distance can never be bigger!

Finally, when two subsequences are anti-correlated (i.e., $PC = -1$), then we get:

\begin{align}
    ED_{z-norm} ={}&
        \sqrt {2 * m * (1 - PC)}
    \\
    ={}&
        \sqrt {2 * m * (1 - (-1))}
    \\
    ={}&
        \sqrt {2 * m * 2}
    \\
    ={}&
        \sqrt {4 * m}
    \\
    ={}&
        2 * \sqrt {m}
    \\
\end{align}

```python

```


# Linear Equations
The equations in the previous lab included one variable, for which you solved the equation to find its value. Now let's look at equations with multiple variables. For reasons that will become apparent, equations with two variables are known as linear equations.

## Solving a Linear Equation
Consider the following equation:

\begin{equation}2y + 3 = 3x - 1 \end{equation}

This equation includes two different variables, **x** and **y**. These variables depend on one another; the value of x is determined in part by the value of y and vice-versa; so we can't solve the equation and find absolute values for both x and y. However, we *can* solve the equation for one of the variables and obtain a result that describes a relative relationship between the variables.

For example, let's solve this equation for y. First, we'll get rid of the constant on the right by adding 1 to both sides:

\begin{equation}2y + 4 = 3x \end{equation}

Then we'll use the same technique to move the constant on the left to the right to isolate the y term by subtracting 4 from both sides:

\begin{equation}2y = 3x - 4 \end{equation}

Now we can deal with the coefficient for y by dividing both sides by 2:

\begin{equation}y = \frac{3x - 4}{2} \end{equation}

Our equation is now solved. We've isolated **y** and defined it as 3x-4/2

While we can't express **y** as a particular value, we can calculate it for any value of **x**. For example, if **x** has a value of 6, then **y** can be calculated as:

\begin{equation}y = \frac{3\cdot6 - 4}{2} \end{equation}

This gives the result 14/2 which can be simplified to 7.

You can view the values of **y** for a range of **x** values by applying the equation to them using the following Python code:

```python
import pandas as pd

# Create a dataframe with an x column containing values from -10 to 10
df = pd.DataFrame ({'x': range(-10, 11)})

# Add a y column by applying the solved equation to x
df['y'] = (3*df['x'] - 4) / 2

#Display the dataframe
df
```

We can also plot these values to visualize the relationship between x and y as a line. For this reason, equations that describe a relative relationship between two variables are known as *linear equations*:

```python
%matplotlib inline
from matplotlib import pyplot as plt

plt.plot(df.x, df.y, color="grey", marker = "o")
plt.xlabel('x')
plt.ylabel('y')
plt.grid()
plt.show()
```

In a linear equation, a valid solution is described by an ordered pair of x and y values. For example, valid solutions to the linear equation above include:
- (-10, -17)
- (0, -2)
- (9, 11.5)

The cool thing about linear equations is that we can plot the points for some specific ordered pair solutions to create the line, and then interpolate the x value for any y value (or vice-versa) along the line.

## Intercepts
When we use a linear equation to plot a line, we can easily see where the line intersects the X and Y axes of the plot. These points are known as *intercepts*. The *x-intercept* is where the line intersects the X (horizontal) axis, and the *y-intercept* is where the line intersects the Y (horizontal) axis.

Let's take a look at the line from our linear equation with the X and Y axis shown through the origin (0,0).

```python
plt.plot(df.x, df.y, color="grey")
plt.xlabel('x')
plt.ylabel('y')
plt.grid()

## add axis lines for 0,0
plt.axhline()
plt.axvline()
plt.show()
```

The x-intercept is the point where the line crosses the X axis, and at this point, the **y** value is always 0. Similarly, the y-intercept is where the line crosses the Y axis, at which point the **x** value is 0. So to find the intercepts, we need to solve the equation for **x** when **y** is 0.

For the x-intercept, our equation looks like this:

\begin{equation}0 = \frac{3x - 4}{2} \end{equation}

Which can be reversed to make it look more familar with the x expression on the left:

\begin{equation}\frac{3x - 4}{2} = 0 \end{equation}

We can multiply both sides by 2 to get rid of the fraction:

\begin{equation}3x - 4 = 0 \end{equation}

Then we can add 4 to both sides to get rid of the constant on the left:

\begin{equation}3x = 4 \end{equation}

And finally we can divide both sides by 3 to get the value for x:

\begin{equation}x = \frac{4}{3} \end{equation}

Which simplifies to:

\begin{equation}x = 1\frac{1}{3} \end{equation}

So the x-intercept is 11/3 (approximately 1.333).

To get the y-intercept, we solve the equation for y when x is 0:

\begin{equation}y = \frac{3\cdot0 - 4}{2} \end{equation}

Since 3 x 0 is 0, this can be simplified to:

\begin{equation}y = \frac{-4}{2} \end{equation}

-4 divided by 2 is -2, so:

\begin{equation}y = -2 \end{equation}

This gives us our y-intercept, so we can plot both intercepts on the graph:

```python
plt.plot(df.x, df.y, color="grey")
plt.xlabel('x')
plt.ylabel('y')
plt.grid()

## add axis lines for 0,0
plt.axhline()
plt.axvline()
plt.annotate('x-intercept',(1.333, 0))
plt.annotate('y-intercept',(0,-2))
plt.show()
```

The ability to calculate the intercepts for a linear equation is useful, because you can calculate only these two points and then draw a straight line through them to create the entire line for the equation.

## Slope
It's clear from the graph that the line from our linear equation describes a slope in which values increase as we travel up and to the right along the line. It can be useful to quantify the slope in terms of how much **x** increases (or decreases) for a given change in **y**. In the notation for this, we use the greek letter Δ (*delta*) to represent change:

\begin{equation}slope = \frac{\Delta{y}}{\Delta{x}} \end{equation}

Sometimes slope is represented by the variable ***m***, and the equation is written as:

\begin{equation}m = \frac{y_{2} - y_{1}}{x_{2} - x_{1}} \end{equation}

Although this form of the equation is a little more verbose, it gives us a clue as to how we calculate slope. What we need is any two ordered pairs of x,y values for the line - for example, we know that our line passes through the following two points:
- (0,-2)
- (6,7)

We can take the x and y values from the first pair, and label them x1 and y1; and then take the x and y values from the second point and label them x2 and y2. Then we can plug those into our slope equation:

\begin{equation}m = \frac{7 - -2}{6 - 0} \end{equation}

This is the same as:

\begin{equation}m = \frac{7 + 2}{6 - 0} \end{equation}

That gives us the result 9/6 which is 11/2 or 1.5 .

So what does that actually mean? Well, it tells us that for every change of **1** in x, **y** changes by 11/2 or 1.5. So if we start from any point on the line and move one unit to the right (along the X axis), we'll need to move 1.5 units up (along the Y axis) to get back to the line.

You can plot the slope onto the original line with the following Python code to verify it fits:

```python
plt.plot(df.x, df.y, color="grey")
plt.xlabel('x')
plt.ylabel('y')
plt.grid()
plt.axhline()
plt.axvline()

# set the slope
m = 1.5

# get the y-intercept
yInt = -2

# plot the slope from the y-intercept for 1x
mx = [0, 1]
my = [yInt, yInt + m]
plt.plot(mx,my, color='red', lw=5)

plt.show()
```

### Slope-Intercept Form
One of the great things about algebraic expressions is that you can write the same equation in multiple ways, or *forms*. The *slope-intercept form* is a specific way of writing a 2-variable linear equation so that the equation definition includes the slope and y-intercept. The generalised slope-intercept form looks like this:

\begin{equation}y = mx + b \end{equation}

In this notation, ***m*** is the slope and ***b*** is the y-intercept.

For example, let's look at the solved linear equation we've been working with so far in this section:

\begin{equation}y = \frac{3x - 4}{2} \end{equation}

Now that we know the slope and y-intercept for the line that this equation defines, we can rewrite the equation as:

\begin{equation}y = 1\frac{1}{2}x + -2 \end{equation}

You can see intuitively that this is true. In our original form of the equation, to find y we multiply x by three, subtract 4, and divide by two - in other words, x is half of 3x - 4; which is 1.5x - 2. So these equations are equivalent, but the slope-intercept form has the advantages of being simpler, and including two key pieces of information we need to plot the line represented by the equation. We know the y-intecept that the line passes through (0, -2), and we know the slope of the line (for every x, we add 1.5 to y.

Let's recreate our set of test x and y values using the slope-intercept form of the equation, and plot them to prove that this  describes the same line:

```python
%matplotlib inline

import pandas as pd
from matplotlib import pyplot as plt

# Create a dataframe with an x column containing values from -10 to 10
df = pd.DataFrame ({'x': range(-10, 11)})

# Define slope and y-intercept
m = 1.5
yInt = -2

# Add a y column by applying the slope-intercept equation to x
df['y'] = m*df['x'] + yInt

# Plot the line
from matplotlib import pyplot as plt

plt.plot(df.x, df.y, color="grey")
plt.xlabel('x')
plt.ylabel('y')
plt.grid()
plt.axhline()
plt.axvline()

# label the y-intercept
plt.annotate('y-intercept',(0,yInt))

# plot the slope from the y-intercept for 1x
mx = [0, 1]
my = [yInt, yInt + m]
plt.plot(mx,my, color='red', lw=5)

plt.show()
```

Basics Of Algebra by Hiren/01-02-Linear Equations.ipynb