# Esempi

## prime due es

```python
#Scrivere un programma Python per calcolare la distanza tra due punti

import math
p1=[4,0]
p2=[6,6]

distanza= math.sqrt((p1[0]-p2[0])**2+(p1[1]-p2[1])**2)

Creare una lista con 100 elementi uguali a 0.

lista=[0]*100
print(lista)

'''
Esercizio 20
Data una lista annidata a=[[2,1,3],[4,5,6],[7,8,9]], scrivere un programma Python che la trasformi
nella lista semplice ottenuta con i suoi valori, cioè a1=[2,1,3,4,5,6,7,8,9]
'''

a=[[2,1,3],[4,5,6],[7,8,9]]

a1=a[0]+a[1]+a[2]
print(a1)

z = np.arange(0,30.5,0.5)
print("Numpy Array z ",z)
t = np.linspace(1,2,100)

ps= np.dot(a,b)
print("Prodotto scalare tra a e b ",ps)

D=np.diag(a)

a=1
c=1

k=np.arange(1,9)
b=10.0**k;
Delta= b**2-4*a*c
'''
La formula del calcolo della soluzione x1 può dare problemi numerici in base ai valori di b:
quando b assume un valore molto elevato b**2-4ac è approssimabile a b**2, quindi nel calcolo
di x1 si può veriificare il fenomento di cancellazione dovuto al calcolo -b+ sqrt(Delta)
in quanto sqrt(Delta) in questo caso è approssimabile a b.
Per la formula del calcolo della soluzione x2 questo problema non si verifica.
'''
x1=(-b+np.sqrt(Delta))/(2*a)
x2=(-b-np.sqrt(Delta))/(2*a)

'''
Usiamo una formula alternativa, nota in algebra,
per il calcolo della soluzione x1 a partire dalla soluzione x2
'''

x1new=c/(a*x2)

------

```

# Malcondizionamento sistema al variare di x

```python
import numpy as np
import numpy.linalg as npl
import scipy.linalg  as spl
import matplotlib.pyplot as plt

A=np.array([[3,5],[3.01,5.01]])
b=np.array([10, 1])

x= spl.solve(A,b)  #soluzione del sistema: x=[-2255; 1355]

# perturbo il coeff. di x della seconda equazione:
dA=np.array([[0, 0], [0.01, 0]])

err_dati= npl.norm(dA,np.inf)/npl.norm(A,np.inf)  #errore percentuale=0.12%
print("Errore relativo sui dati Sistema 1 pert. in percentuale", err_dati*100,"%")
# soluzione del sistema perturbato:
x1=spl.solve(A+dA,b)

err_rel_sol=npl.norm(x-x1,np.inf)/npl.norm(x,np.inf) #%errore percentuale=71.43%
print("Errore relativo sulla soluzione Sistema 1 pert. in percentuale", err_rel_sol*100,"%")
# quindi ad una piccola perturbazione sui dati corrisponde una grossa
# perturbazione sui risultati --> problema mal condizionato

"""
Esercizio 4

"""
import numpy as np
import numpy.linalg as npl
import scipy.linalg as spl

#Esempi di sistemi malcondizionati

A=np.array([[6, 63, 662.2],[63, 662.2, 6967.8],[662.2, 6967.8, 73393.5664]])
b=np.array([1.1, 2.33, 1.7])

KA= npl.cond(A,np.inf)

x=spl.solve(A,b)

#perturbare la matrice

A1=A.copy()
A1[0,0]=A[0,0]+0.01
x_per=spl.solve(A1,b)

#Errore relativo sui dati
err_dati=npl.norm(A-A1,np.inf)/npl.norm(A,np.inf)
print("Errore relativo sui dati  in percentuale ", err_dati*100,"%")

err_rel_sol=npl.norm(x_per-x,np.inf)/npl.norm(x,np.inf)
print("Errore relativo sulla soluzione  in percentuale ", err_rel_sol*100,"%")

```

    Errore relativo sui dati Sistema 1 pert. in percentuale 0.12468827930174566 %
    Errore relativo sulla soluzione Sistema 1 pert. in percentuale 71.42857142857439 %

# Zeri

## esami

```python
"""
Ese1
"""

import numpy as np
import matplotlib.pyplot as plt
from funzioni_zeri import newton_m, stima_ordine
import sympy as sym
from sympy.utilities.lambdify import lambdify
from scipy.optimize import fsolve

x=sym.symbols('x')
fx= x-1/3*sym.sqrt(30*x-25)
dfx=sym.diff(fx,x,1)
print(dfx)

x0=4
f=lambdify(x,fx,np)
alfa=fsolve(f,x0)
print("La funzione ha uno zero in ", alfa)
fp=lambdify(x,dfx,np)
fp_alfa=fp(alfa)
print("La derivata prima  in ", alfa, 'vale ',fp_alfa)
x=np.linspace(5/6,25/6,100)
plt.plot(x,f(x))
plt.plot(x,[0]*100)
plt.plot(alfa,0,'ro')
plt.show()

#In alfa=1.66666667 si annula sia la funzione che la sua derivata prima,
#la funzione ha in x=1.66666667 uno xero con molteplicita=2
m=2
tolx=1e-12
tolf=1e-12
nmax=100

#Metodo iterativo che converge quadraticamente al alfa: metodo di Newton Modificato con m=2
x1,it,xk= newton_m(f,fp,x0,m,tolx,tolf,nmax)
#Verifico l'ordine di convergenza
ordine=stima_ordine(xk,it)

plt.plot(range(it),np.abs(xk))
plt.show()

#Il metodo non converge se scelgo come iterato iniziale x0=5/6,
#perchè la derivata prima in  5/6 diverge  va a -infinito
```

```python
# -*- coding: utf-8 -*-
"""
Created on Mon May 24 19:09:59 2021

@author: damia
"""

import numpy as np
import matplotlib.pyplot as plt
from funzioni_zeri import iterazione
import sympy as sym
from sympy.utilities.lambdify import lambdify
n=35
u1=np.zeros((n,),dtype=float)
u2=np.zeros((n,),dtype=float)
u3=np.zeros((n,),dtype=float)

for i in range(1,n+1):
    u1[i-1]=15*((3/5)**(i)+1)/(5*(3/5)**(i)+3)

u2[0]=4
for i in range(1,n):
    u2[i]=8-15/u2[i-1]

u3[0]=4
u3[1]=17/4
for i in range(3,n+1):
    u3[i-1]=108-815/u3[i-2]+1500/(u3[i-2]*u3[i-3])

plt.plot(range(n),u2)
plt.title('Formula 2')
plt.show()

plt.plot(range(n),u3)
plt.title('Formula 3')
plt.show()

err_rel2=np.abs(u2-u1)/np.abs(u1)

err_rel3=np.abs(u3-u1)/np.abs(u1)

plt.semilogy(range(n),err_rel2,range(n),err_rel3)
plt.legend(['Errore relativo formula 2', 'Errore relativo formula 3'])
plt.show()

g1=lambda x:  8-15/x

g2=lambda x:  108-815/x+ 1500/(x**2);

x=sym.symbols('x')

#Definisco funzione g1
g1x= 8-15/x
dg1x=sym.diff(g1x,x,1)
dg1=lambdify(x,dg1x,np)
g1=lambdify(x,g1x,np)

#Definisco funzione g2
g2x=108-815/x+1500/(x**2)
dg2x=sym.diff(g2x,x,1)
dg2=lambdify(x,dg2x,np)
g2=lambdify(x,g2x,np)

x=np.linspace(4,100 ,100)
plt.plot(x,g1(x))
plt.plot(x,x)
plt.legend(['g1(x)','y=x'])
plt.show()

#La g2 interseca la bisettrice in 2 punti, ha due punti fissi (x=5 ed 100) , ma la derivata prima della g2
# non soddisfa le ipotesi del teorema di convergenza locale in un intorno del primo punto fisso,5
plt.plot(x,g2(x))
plt.plot(x,x)
plt.legend(['g2(x)','y=x'])
plt.show()

tolx=1e-5
nmax=100
x0=4
x1,xk,it=iterazione(g1,x0,tolx,nmax)
x2,xk2,it2=iterazione(g2,x0,tolx,nmax)
print("Punto fisso della funzione g1 -->  ",x1)
print("Punto fisso della funzione g1 -->  ",x2)

#Visualizziamo la derivata prima di g1 in un intorno di 5, sono soddisfatte le iptesi del teorema di convergenza
#locale
xx=np.linspace(2,6,100)
plt.semilogy(xx,dg1(xx))
plt.plot([2,6],[1,1])
plt.plot([2,6],[-1,-1])
plt.legend(['derivata prima di g1 in un intorno di 5 ', 'y=1','y=-1'])
plt.show()
xx=np.linspace(2,6,100)
plt.semilogy(xx,dg2(xx))
plt.plot([2,6],[1,1])
plt.plot([2,6],[-1,-1])
plt.legend(['derivata prima di g2 in un intorno di 5 ', 'y=1','y=-1'])

plt.show()

xx=np.linspace(95,105,100)
plt.plot(xx,dg2(xx))
plt.plot([95,105],[1,1])
plt.plot([95,105],[-1,-1])

plt.legend(['Derivata prima  di g2 in un intorno di 100','y=1','y=-1'])

```

```python
import numpy as np
import sympy as sym
import funzioni_zeri
import matplotlib.pyplot as plt
from sympy.utilities.lambdify import lambdify
from scipy.optimize import fsolve
tolx=1e-7
nmax=1000

f= lambda x: np.tan(3/2*x)-2*np.cos(x)-x*(7-x)

#Utilizzo il metodo fsolve di scipy.optimize per calcolare lo zero alfa della funzione f,
#prende in input l'iterato iniziale x0

x0=0.0
alfa=fsolve(f, x0)
print("Lo zero della funzione e' ",alfa)
#Disegno: l'asse x e la funzione f valutata in un intervallo opportuno
xx=np.linspace(-1.0,1.0,100)
plt.plot(xx,0*xx,xx,f(xx),alfa,0,'ro')
plt.legend(['Funzione f', 'zero'])
plt.show()

'''
Definisco la funzione g in formato simbolico perchè poi utilizzo la funzione diff
di sympy per calcolare l'espressione analitica della derivata prima'
'''
x=sym.symbols('x')

#Considero la funzione g indicata dalla traccia del compito
gx=sym.tan(3/2*x)-2*sym.cos(x)-x*(6-x)

#Disegno la funzione g(x) e la bisettrice y=x
g=lambdify(x,gx,np)
plt.plot(xx,xx,xx,g(xx))
plt.title('funzione g(x) e y=x')
plt.show()

#Calcolo la derivata prima di gx espressione simbolica tramite la funzione diff del modulo sym
dgx=sym.diff(gx,x,1)

dg=lambdify(x,dgx,np)

#Disegno la funzione dg(x)

#Posso giustifcare la convergenza del procedimento iterativo guardando la derivata prima di g(x)
#in un intorno della soluzione: il metodo genera una successione di iterati convergenti alla radice alfa
# ed appartenenti a questo intorno se |g'(x)|< 1 in un intorno della soluzione

plt.plot(xx ,dg(xx ))
plt.plot(alfa,0,'ro')
#Disegno la retta y=1
plt.plot([-1,1],[1,1],'--')
#Disegno la retta y=-1
plt.plot([-1,1],[-1,-1],'--')
plt.title('funzione dg(x) proposta dalla traccia - Ipotesi per la convergenza non soddisfatte')

plt.legend(['Grafico derivata prima di g1 (x)', 'Zero', 'Retta y=1', 'Retta y=-1'])
plt.show()

#Dal grafico vedo che per la funzione g proposta dalla traccia
#non sono soddisfatte le ipotesi del teorema di convergenza locale

#Ricavo la funzione gx per la quale ci sia convergenza
gx1= (sym.tan(3.0/2.0*x)-2*sym.cos(x)+x**2)/7
#Disegno la funzione g(x) e la bisettrice y=x
g1=lambdify(x,gx1,np)
plt.plot(xx,xx,xx,g1(xx))
plt.title('funzione g1(x) ricavata e y=x')
plt.show()

#Calcolo la derivata prima di gx espressione simbolica tramite la funzione diff del modulo sym
dgx1=sym.diff(gx1,x,1)

dg1=lambdify(x,dgx1,np)

#Disegno la funzione dg1(x)

#Posso giustifcare la convergenza del procedimento iterativo guardando la derivata prima di g(x)
#in un intorno della soluzione: il metodo genera una successione di iterati convergenti alla radice alfa
# ed appartenenti a questo intorno se |g'(x)|< 1 in un intorno della soluzione

plt.plot(xx ,dg1(xx ))
plt.plot(alfa,0,'ro')
#Disegno la retta y=1
plt.plot([-1,1],[1,1],'--')
#Disegno la retta y=-1
plt.plot([-1,1],[-1,-1],'--')
plt.title('funzione dg1(x) Ricavata - Ipotesi di convergenza soddisfatte')
plt.legend(['Grafico derivata prima di g1 (x)', 'Zero', 'Retta y=1', 'Retta y=-1'])
plt.show()

'''
Dal grafico vedo che per la funzione g  ceh ho ricavato
soddisfatte le ipotesi del teorema di convergenza locale: esiste un intorno della soluzione per cui
|g'(x)|<1

'''

#Utilizzo il metodo di iterazione funzionale per calcolare il punto fisso di g1
x1,it,xk=funzioni_zeri.iterazione(g1,x0,tolx,nmax)
print('iterazioni= {:d}, soluzione={:e} \n\n'.format(it,x1))

#Calcolo l'ordine del metodo
ordine_iter= funzioni_zeri.stima_ordine(xk,it)
#Essendo il metodo con ordine di convergenza lineare, la costante asintotica di convergenza è data
#da |g'(alfa)| dove alfa è la radice.

print("Iterazione it={:d}, ordine di convergenza {:e}".format(it,ordine_iter ))

plt.plot(range(it+1),xk)

```

## Newton, Newton modificato(molteplicità) e stima ordine

```python
import math
import numpy as np
import sympy as sym
import funzioni_zeri
import matplotlib.pyplot as plt
from sympy.utilities.lambdify import lambdify

tolx=1e-12;
tolf=1e-12;

x=sym.symbols('x')

fx=fx=x**3+x**2-33*x+63
dfx=sym.diff(fx,x,1)

#Trasformo in numeriche la funzione e la sua derivata
f=lambdify(x,fx,np)
df=lambdify(x,dfx,np)

#Disegno nello stesso grafico l'asse x e la funzione f valutata in un intervallo opportuno, [-10,10]
z=np.linspace(-10,10,100)
plt.plot(z,0*z,z,f(z),'r-')

nmax=500
x0=1
xNew,itNew,xkNew=funzioni_zeri.newton(f,df,x0,tolx,tolf,nmax)
print('X0= {:e} , zero Newton= {:e} con {:d} iterazioni \n'.format(x0,xNew,itNew))
ordine_New =funzioni_zeri.stima_ordine(xkNew,itNew)
print("Newton it={:d}, ordine di convergenza {:e}".format(itNew,ordine_New))

#Utilizzando il metodo di Newton modifica e ponendo m uguale alla molteplicità della radice
# si ottiene un metodo con ordine di convergenza 2

m=2
xNew_m,itNew_m,xkNew_m=funzioni_zeri.newton_m(f,df,x0,m,tolx,tolf,nmax)
print('X0= {:e} , zero Newton Mod= {:e} con {:d} iterazioni \n'.format(x0,xNew_m,itNew_m))
ordine_New_m =funzioni_zeri.stima_ordine(xkNew_m,itNew_m)
print("Newton Mod it={:d}, ordine di convergenza {:e}".format(itNew_m,ordine_New_m))
```

## Costate asintotica

```python
"""
esercizio 6
"""

import numpy as np
import sympy as sym
import funzioni_zeri
import matplotlib.pyplot as plt
from sympy.utilities.lambdify import lambdify

tolx=1.e-7
nmax=1000

f= lambda x: x**3+4*x**2-10

#Disegno: l'asse x e la funzione f valutata in un intervallo opportuno
xx=np.linspace(0.0,1.6,100)
plt.plot(xx,0*xx,xx,f(xx))
plt.show()

x0=1.5;

x=sym.symbols('x')
#Definisco le possibili espressioni della funzione f(x)=0 nella forma x=g(x)
#Per come ricavare le diverse g(X) , analizzare file pdf su virtuale con spiegazione teorica dell'esercizio 6

#................................
#gx=sym.sqrt(10/(x+4));       #p=1, 0.127229401770925
gx=1/2*sym.sqrt(10-x**3)      #p=1, C=0.511961226874885
#gx=(10+x)/(x**2+4*x+1);         #p=1, C=0.983645643784931
#gx=sym.sqrt(10/x-4*x)         # non converge

#Disegno la funzione g(x) e la bisettrice y=x
g=lambdify(x,gx,np)
plt.plot(xx,xx,xx,g(xx))
plt.title('funzione g(x) e y=x')
plt.show()

#Calcolo la derivata prima di gx espressione simbolica tramite la funzione diff del modulo sym
dgx=sym.diff(gx,x,1)

dg=lambdify(x,dgx,np)

#Disegno la funzione dg(x)

#Posso giustifcare la convergenza del procedimento iterativo guardando la derivata prima di g(x)
#in un intorno della soluzione: il metodo genera una successione di iterati convergenti alla radice alfa
# ed appartenenti a questo intorno se |g'(x)|< 1 in un intorno della soluzione
plt.plot(xx,dg(xx))
plt.title('funzione dg(x)')
plt.show()

x1,it,xk=funzioni_zeri.iterazione(g,x0,tolx,nmax)
print('iterazioni= {:d}, soluzione={:e} \n\n'.format(it,x1))

#Calcolo l'ordine del metodo
ordine_iter= funzioni_zeri.stima_ordine(xk,it)
#Essendo il metodo con ordine di convergenza lineare, la costante asintotica di convergenza è data
#da |g'(alfa)| dove alfa è la radice.

C=abs(dg(x1))
print("Iterazione it={:d}, ordine di convergenza {:e}, Costante asintotica di convergenza {:e}".format(it,ordine_iter,C))

plt.plot(xx,xx,'k-',xx,g(xx))
plt.title("abs(g'(alfa))="+str(C))
Vx=[]
Vy=[]
for k in range(it):
    Vx.append(xk[k])
    Vy.append(xk[k])
    Vx.append(xk[k])
    Vy.append(xk[k+1])

Vy[0]=0
plt.plot(Vx,Vy,'r',xk,[0]*(it+1),'or-')
plt.show()

# Si osserva che a parità di ordine di convergenza, più piccola è la costante asintotica di convergenza,
#maggiore è la velocità del metodo.

```

## Funzioni

```python
import numpy as np
import math

'''
Il core Python non possiede la funzione sign.
La funzione copysign(a,b)  del modulo math restituisce un valore numerico che ha il valore assoluto di
a e segno di b.
Per avere il segno di un valore numerico b si può usare math.copysign(1,b)
che resistuisce 1 se b>0, -1 se b<0, 0 se b è zero
'''

def sign(x): return math.copysign(1, x)

#Bisezione
def bisez(fname,a,b,tol):
    eps=np.spacing(1)      # np.spacing(x) Restituisce la distanza tra x e il numero adiacente più vicino.
                           # np.spacing(1)  restituisce quindi l' eps di macchina.
    fa=fname(a)
    fb=fname(b)
    if sign(fa)==sign(fb):
       print('intervallo non corretto --Metodo non applicabile')
       return [],0,[]
    else:
        maxit=int(math.ceil(math.log((b-a)/tol)/math.log(2)))
        print('n. di passi necessari=',maxit,'\n');
        xk=[]
        it=0
        #while it<maxit and  abs(b-a)>=tol+eps*max(abs(a),abs(b)):
        while it<maxit and  abs(b-a)>=tol:
            c=a+(b-a)*0.5   #formula stabile per il calcolo del punto medio dell'intervallo
            xk.append(c)
            it+=1
            if c==a or c==b:
                break
            fxk=fname(c)
            if fxk==0:
                break
            elif sign(fxk)==sign(fa):
                a=c
                fa=fxk
            elif sign(fxk)==sign(fb):
                b=c
                fb=fxk

        x=c

    return x,it,xk

def regula_falsi(fname,a,b,tol,nmax):
#Regula Falsi
        eps=np.spacing(1)
        xk=[]
        fa=fname(a)
        fb=fname(b)
        if sign(fa)==sign(fb):
          print('intervallo non corretto --Metodo non applicabile')
          return [],0,[]
        else:
            it=0
            fxk=fname(a)
            while it<nmax and  abs(b-a)>=tol+eps*max(abs(a),abs(b)) and abs(fxk)>=tol :
                x1=a-fa*(b-a)/(fb-fa);
                xk.append(x1)
                it+=1
                fxk=fname(x1);
                if fxk==0:
                    break
                elif sign(fxk)==sign(fa):
                    a=x1;
                    fa=fxk;
                elif sign(fxk)==sign(fb):
                    b=x1;
                    fb=fxk;

            if it==nmax :
                print('Regula Falsi: Raggiunto numero max di iterazioni')

        return x1,it,xk

def corde(fname,fpname,x0,tolx,tolf,nmax):
 #Corde
        xk=[]
        m=fpname(x0)               #m= Coefficiente angolare della tangente in x0
        fx0=fname(x0)
        d=fx0/m
        x1=x0-d
        fx1=fname(x1)
        xk.append(x1)
        it=1
        while it<nmax and  abs(fx1)>=tolf and abs(d)>=tolx*abs(x1) :
           x0=x1
           fx0=fname(x0)
           d=fx0/m
           '''
           #x1= ascissa del punto di intersezione tra  la retta che passa per il punto
           (xi,f(xi)) e ha pendenza uguale a m  e l'asse x
           '''
           x1=x0-d
           fx1=fname(x1)
           it=it+1
           xk.append(x1)

        if it==nmax:
            print('raggiunto massimo numero di iterazioni \n')

        return x1,it,xk

#Secanti
def secanti(fname,xm1,x0,tolx,tolf,nmax):
        xk=[]
        fxm1=fname(xm1);
        fx0=fname(x0);
        d=fx0*(x0-xm1)/(fx0-fxm1)
        x1=x0-d;
        xk.append(x1)
        fx1=fname(x1);
        it=1

        while it<nmax and abs(fx1)>=tolf and abs(d)>=tolx*abs(x1):
            xm1=x0
            x0=x1
            fxm1=fname(xm1)
            fx0=fname(x0)
            d=fx0*(x0-xm1)/(fx0-fxm1)
            x1=x0-d
            fx1=fname(x1)
            xk.append(x1);
            it=it+1;

        if it==nmax:
           print('Secanti: raggiunto massimo numero di iterazioni \n')

        return x1,it,xk

def newton(fname,fpname,x0,tolx,tolf,nmax):
#Newton
        xk=[]
        fx0=fname(x0)
        dfx0=fpname(x0)
        if abs(dfx0)>np.spacing(1):
            d=fx0/dfx0
            x1=x0-d
            fx1=fname(x1)
            xk.append(x1)
            it=0

        else:
            print('Newton:  Derivata nulla in x0 - EXIT \n')
            return [],0,[]

        it=1
        while it<nmax and abs(fx1)>=tolf and  abs(d)>=tolx*abs(x1):
            x0=x1
            fx0=fname(x0)
            dfx0=fpname(x0)
            if abs(dfx0)>np.spacing(1):
                d=fx0/dfx0
                x1=x0-d
                fx1=fname(x1)
                xk.append(x1)
                it=it+1
            else:
                 print('Newton: Derivata nulla in x0 - EXIT \n')
                 return x1,it,xk

        if it==nmax:
            print('Newton: raggiunto massimo numero di iterazioni \n');

        return x1,it,xk

def stima_ordine(xk,iterazioni):
      p=[]

      for k in range(iterazioni-3):
         p.append(np.log(abs(xk[k+2]-xk[k+3])/abs(xk[k+1]-xk[k+2]))/np.log(abs(xk[k+1]-xk[k+2])/abs(xk[k]-xk[k+1])));

      ordine=p[-1]
      return ordine

#Newton Modificato
def newton_m(fname,fpname,x0,m,tolx,tolf,nmax):
        eps=np.spacing(1)
        xk=[]
        #xk.append(x0)
        fx0=fname(x0)
        dfx0=fpname(x0)
        if abs(dfx0)>eps:
            d=fx0/dfx0
            x1=x0-m*d
            fx1=fname(x1)
            xk.append(x1)
            it=0

        else:
            print('Newton:  Derivata nulla in x0  \n')
            return [],0,[]

        it=1
        while it<nmax and abs(fx1)>=tolf and  abs(d)>=tolx*abs(x1):
            x0=x1
            fx0=fname(x0)
            dfx0=fpname(x0)
            if abs(dfx0)>eps:
                d=fx0/dfx0
                x1=x0-m*d
                fx1=fname(x1)
                xk.append(x1)
                it=it+1
            else:
                 print('Newton Mod: Derivata nulla   \n')
                 return x1,it,xk

        if it==nmax:
            print('Newton Mod: raggiunto massimo numero di iterazioni \n');

        return x1,it,xk

def iterazione(gname,x0,tolx,nmax):

        xk=[]
        xk.append(x0)
        x1=gname(x0)
        d=x1-x0
        xk.append(x1)
        it=1
        while it<nmax and  abs(d)>=tolx*abs(x1) :
            x0=x1
            x1=gname(x0)
            d=x1-x0
            it=it+1
            xk.append(x1)

        if it==nmax:
            print('Raggiunto numero max di iterazioni \n')

        return x1, it,xk
```

# Sistemi Lineari e QR

## esame

```python
# -*- coding: utf-8 -*-
"""
Created on Tue Dec  7 12:09:22 2021

@author: HWRUser
"""

import numpy as np
import Sistemi_lineari as sl

# CHOLESKY --> autovalori > 0 e simmetrica
# LU --> determinante delle sottomatrici != 0

A = np.matrix('10 -4 4 0; -4 10 0 2; 4 0 10 2; 0 2 2 0')
print(np.linalg.eigvals(A))
#A ha autovalore negativo, quindi non ammette cholesky

B = np.matrix('5 -2 2 0; -2 5 0 1; 2 0 5 1; 0 1 1 5')
print(np.linalg.eigvals(B))
#B ha autovalori positivi e simmetrica, ammette cholesky

for i in range(1, 5):
    if np.linalg.det(A[0:i, 0:i]) == 0:
        print('Errore: determinante di A = 0')

#A ha tutte le sottomatrici con determinante diverso da 0, ammette LU

for i in range(1, 5):
    if np.linalg.det(B[0:i, 0:i]) == 0:
        print('Errore: determinante di B = 0')

#B ha tutte le sottomatrici con determinante diverso da 0, ammette LU

Ainv = np.linalg.inv(A)
Binv = np.linalg.inv(B)

P, L, U, flag = sl.LU_nopivot(A)
print(P)
print(L)
print(U)

detA = np.prod(np.diag(U))
detInvA = 1/detA

P, L, U, flag = sl.LU_nopivot(B)
print(P)
print(L)
print(U)

detB = np.prod(np.diag(U))
detInvB = 1/detB

# PA = LU --> det(P) * det(A) = det(L) * det(U)
```

## Funzioni

```python
import numpy as np

def Lsolve(L,b):
    """
    Risoluzione con procedura forward di Lx=b con L triangolare inferiore
     Input: L matrice triangolare inferiore
            b termine noto
    Output: x: soluzione del sistema lineare
            flag=  0, se sono soddisfatti i test di applicabilità
                   1, se non sono soddisfatti
    """
#test dimensione
    m,n=L.shape
    flag=0;
    if n != m:
        print('errore: matrice non quadrata')
        flag=1
        x=[]
        return x, flag

     # Test singolarita'
    if np.all(np.diag(L)) != True:
         print('el. diag. nullo - matrice triangolare inferiore')
         x=[]
         flag=1
         return x, flag
    # Preallocazione vettore soluzione
    x=np.zeros((n,1))

    for i in range(n):
         s=np.dot(L[i,:i],x[:i]) #scalare=vettore riga * vettore colonna
         x[i]=(b[i]-s)/L[i,i]

    return x,flag

def Usolve(U,b):

    """
    Risoluzione con procedura backward di Rx=b con R triangolare superiore
     Input: U matrice triangolare superiore
            b termine noto
    Output: x: soluzione del sistema lineare
            flag=  0, se sono soddisfatti i test di applicabilità
                   1, se non sono soddisfatti

    """
#test dimensione
    m,n=U.shape
    flag=0;
    if n != m:
        print('errore: matrice non quadrata')
        flag=1
        x=[]
        return x, flag

     # Test singolarita'
    if np.all(np.diag(U)) != True:
         print('el. diag. nullo - matrice triangolare superiore')
         x=[]
         flag=1
         return x, flag
    # Preallocazione vettore soluzione
    x=np.zeros((n,1))

    for i in range(n-1,-1,-1):
         s=np.dot(U[i,i+1:n],x[i+1:n]) #scalare=vettore riga * vettore colonna
         x[i]=(b[i]-s)/U[i,i]

    return x,flag

def LUsolve(L,U,P,b):
     """
     Risoluzione a partire da PA =LU assegnata
     """
     Pb=np.dot(P,b)
     y,flag=Lsolve(L,Pb)
     if flag == 0:
         x,flag=Usolve(U,y)
     else:
        return [],flag

     return x,flag

def LU_nopivot(A):
    """
    % Fattorizzazione PA=LU senza pivot   versione vettorizzata
    In output:
    L matrice triangolare inferiore
    U matrice triangolare superiore
    P matrice identità
    tali che  LU=PA=A
    """
    # Test dimensione
    m,n=A.shape

    flag=0;
    if n!=m:
      print("Matrice non quadrata")
      L,U,P,flag=[],[],[],1
      return P,L,U,flag

    P=np.eye(n);
    U=A.copy();
 # Fattorizzazione
    for k in range(n-1):
       #Test pivot
          if U[k,k]==0:
            print('elemento diagonale nullo')
            L,U,P,flag=[],[],[],1
            return P,L,U,flag

  #     Eliminazione gaussiana
          U[k+1:n,k]=U[k+1:n,k]/U[k,k]                                   # Memorizza i moltiplicatori
          U[k+1:n,k+1:n]=U[k+1:n,k+1:n]-np.outer(U[k+1:n,k],U[k,k+1:n])  # Eliminazione gaussiana sulla matrice

    L=np.tril(U,-1)+np.eye(n)  # Estrae i moltiplicatori
    U=np.triu(U)           # Estrae la parte triangolare superiore + diagonale
    return P,L,U,flag

def LU_nopivotv(A):
    """
    % Fattorizzazione PA=LU senza pivot   versione vettorizzata intermedia
    In output:
    L matrice triangolare inferiore
    U matrice triangolare superiore
    P matrice identità
    tali che  LU=PA=A
    """
    # Test dimensione
    m,n=A.shape

    flag=0;
    if n!=m:
      print("Matrice non quadrata")
      L,U,P,flag=[],[],[],1
      return P,L,U,flag

    P=np.eye(n);
    U=A.copy();
 # Fattorizzazione
    for k in range(n-1):
       #Test pivot
          if U[k,k]==0:
            print('elemento diagonale nullo')
            L,U,P,flag=[],[],[],1
            return P,L,U,flag

  #     Eliminazione gaussiana
          for i in range(k+1,n):
             U[i,k]=U[i,k]/U[k,k]                                   # Memorizza i moltiplicatori
             U[i,k+1:n]=U[i,k+1:n]-U[i,k]*U[k,k+1:n]  # Eliminazione gaussiana sulla matrice

    L=np.tril(U,-1)+np.eye(n)  # Estrae i moltiplicatori
    U=np.triu(U)           # Estrae la parte triangolare superiore + diagonale
    return P,L,U,flag

def LU_nopivotb(A):
    """
    % Fattorizzazione PA=LU senza pivot  versione base
    In output:
    L matrice triangolare inferiore
    U matrice triangolare superiore
    P matrice identità
    tali che  LU=PA=A
    """
    # Test dimensione
    m,n=A.shape
    flag=0;
    if n!=m:
      print("Matrice non quadrata")
      L,U,P,flag=[],[],[],1
      return P,L,U,flag

    P=np.eye(n);
    U=A.copy();
 # Fattorizzazione
    for k in range(n-1):
         #Test pivot

          if U[k,k]==0:
            print('elemento diagonale nullo')
            L,U,P,flag=[],[],[],1
            return P,L,U,flag

  #     Eliminazione gaussiana
          for i in range(k+1,n):
                U[i,k]=U[i,k]/U[k,k]
                for j in range(k+1,n):                                 # Memorizza i moltiplicatori
                  U[i,j]=U[i,j]-U[i,k]*U[k,j]  # Eliminazione gaussiana sulla matrice

    L=np.tril(U,-1)+np.eye(n)  # Estrae i moltiplicatori
    U=np.triu(U)           # Estrae la parte triangolare superiore + diagonale
    return P,L,U,flag

def swapRows(A,k,p):
    A[[k,p],:] = A[[p,k],:]

def LU_pivot(A):
    """
    % Fattorizzazione PA=LU con pivot
    In output:
    L matrice triangolare inferiore
    U matrice triangolare superiore
    P matrice di permutazione
    tali che  PA=LU
    """
    # Test dimensione
    m,n=A.shape
    flag=0;
    if n!=m:
      print("Matrice non quadrata")
      L,U,P,flag=[],[],[],1
      return P,L,U,flag

    P=np.eye(n);
    U=A.copy();
 # Fattorizzazione
    for k in range(n-1):
       #Scambio di righe nella matrice U e corrispondente scambio nella matrice di permutazione per
       # tenere traccia degli scambi avvenuti

       #Fissata la colonna k-esima calcolo l'indice di riga p a cui appartiene l'elemento di modulo massimo a partire dalla riga k-esima
          p = np.argmax(abs(U[k:n,k])) + k
          if p != k:
              swapRows(P,k,p)
              swapRows(U,k,p)

  #     Eliminazione gaussiana
          U[k+1:n,k]=U[k+1:n,k]/U[k,k]                                   # Memorizza i moltiplicatori
          U[k+1:n,k+1:n]=U[k+1:n,k+1:n]-np.outer(U[k+1:n,k],U[k,k+1:n])  # Eliminazione gaussiana sulla matrice

    L=np.tril(U,-1)+np.eye(n)  # Estrae i moltiplicatori
    U=np.triu(U)           # Estrae la parte triangolare superiore + diagonale
    return P,L,U,flag

def solve_nsis(A,B):
  # Test dimensione
    m,n=A.shape
    flag=0;
    if n!=m:
      print("Matrice non quadrata")

      return

    Y= np.zeros((n,n))
    X= np.zeros((n,n))
    P,L,U,flag= LU_nopivot(A)

    if flag==0:
        for i in range(n):
            y,flag=Lsolve(L,np.dot(P,B[:,i]))
            Y[:,i]=y.squeeze(1)
            x,flag= Usolve(U,Y[:,i])
            X[:,i]=x.squeeze(1)
    else:
        print("Elemento diagonale nullo")
        X=[]
    return X

```

## Es4

```python
import numpy as np
import funzioni_Sistemi_lineari as fSl
import scipy.linalg as spl
import matplotlib.pyplot as plt

xesatta=np.array([[2],[2]])

err_rel_nopivot=[]
err_rel_pivot=[]
for k in range(2,19,2):
    A= np.array([[10.0**(-k),1],[1,1]])
    b=np.array([[2+10.0**(-k)],[4]])

    P,L,U,flag, = fSl.LU_nopivot(A)

    if flag==0:
        x_nopivot,flag=fSl.LUsolve(L,U,P,b)
    else:
        print("Sistema non risolubile senza strategia pivotale")

    err_rel_nopivot.append(np.linalg.norm(x_nopivot-xesatta,1)/np.linalg.norm(xesatta,1))

    P_pivot,Lpivot,Upivot,flagpivot, = fSl.LU_pivot(A)

    if flagpivot==0:
        x_pivot,flag=fSl.LUsolve(Lpivot,Upivot,P_pivot,b)
    else:
        print("Sistema non risolubile con strategia pivotale")

    err_rel_pivot.append(np.linalg.norm(x_pivot-xesatta,1)/np.linalg.norm(xesatta,1))

plt.semilogy(range(2,19,2),err_rel_nopivot,range(2,19,2),err_rel_pivot)
plt.legend(['No pivot','Pivot'])
plt.show()
```

## esame qr

```python
"""
Created on Sat May  1 11:23:27 2021

@author: damia
"""
import numpy as np
import matplotlib.pyplot as plt
import scipy.linalg as spl

#Funzioni necessarie
def Usolve(U,b):

    """
    Risoluzione con procedura backward di Rx=b con R triangolare superiore
     Input: U matrice triangolare superiore
            b termine noto
    Output: x: soluzione del sistema lineare
            flag=  0, se sono soddisfatti i test di applicabilità
                   1, se non sono soddisfatti

    """
#test dimensione
    m,n=U.shape
    flag=0;
    if n != m:
        print('errore: matrice non quadrata')
        flag=1
        x=[]
        return x, flag

     # Test singolarita'
    if np.all(np.diag(U)) != True:
         print('el. diag. nullo - matrice triangolare superiore')
         x=[]
         flag=1
         return x, flag
    # Preallocazione vettore soluzione
    x=np.zeros((n,1))

    for i in range(n-1,-1,-1):
         s=np.dot(U[i,i+1:n],x[i+1:n]) #scalare=vettore riga * vettore colonna
         x[i]=(b[i]-s)/U[i,i]

    return x,flag

def metodoQR(x,y,n):
    """
    INPUT
    x vettore colonna con le ascisse dei punti
    y vettore colonna con le ordinate dei punti
    n grado del polinomio approssimante
    OUTPUT
     a vettore colonna contenente i coefficienti incogniti
 """

    H=np.vander(x,n+1)
    Q,R=spl.qr(H)
    y1=np.dot(Q.T,y)
    a,flag=Usolve(R[0:n+1,:],y1[0:n+1])
    return  a

#------------------------------------------------------
#Script
#----------------------------------------------------
m=12

x=np.linspace(1900,2010,12)
y=np.array([76.0,92.0,106.0,123.0,132.0,151.0,179.0,203.0,226.0,249.0,281.0,305.0])

xmin=np.min(x)
xmax=np.max(x)
xval=np.linspace(xmin,xmax,100)

for n in range(1,4):
    a=metodoQR(x,y,n)
    residuo=np.linalg.norm(y-np.polyval(a,x))**2
    print("Norma del residuo al quadrato",residuo)
    p=np.polyval(a,xval)
    plt.plot(xval,p)

plt.legend(['n=1','n=2','n=3'])

plt.plot(x,y,'o')
```

## Metodo QR

```python
import numpy as np
import scipy.linalg as spl
from funzioni_Sistemi_lineari import Usolve

def metodoQR(x,y,n):
    """
    INPUT
    x vettore colonna con le ascisse dei punti
    y vettore colonna con le ordinate dei punti
    n grado del polinomio approssimante
    OUTPUT
     a vettore colonna contenente i coefficienti incogniti
 """

    H=np.vander(x,n+1)
    Q,R=spl.qr(H)
    y1=np.dot(Q.T,y)
    a,flag=Usolve(R[0:n+1,:],y1[0:n+1])
    return  a

```

```python
import numpy as np
import matplotlib.pyplot as plt
from funzioni_Approssimazione_MQ import metodoQR

x = np.array([0.0004, 0.2507, 0.5008, 2.0007, 8.0013])
y = np.array([0.0007, 0.0162,0.0288, 0.0309, 0.0310]);

#Calcolo della retta di regessione
a=metodoQR(x,y,1)
residuo=np.linalg.norm(y-np.polyval(a,x))**2
print("Norma al quadrato del residuo Retta di regressione",residuo)
xmin=np.min(x)
xmax=np.max(x)
xval=np.linspace(xmin,xmax,100)
p=np.polyval(a,xval)
plt.plot(xval,p,'r-',x,y,'o')
plt.legend(['Retta di regressione', 'Dati'])
plt.show()

#Calcolo della parabola di approssimazione nel senso dei minimi quuadrati
a=metodoQR(x,y,2)
residuo=np.linalg.norm(y-np.polyval(a,x))**2
print("Norma al quadrato del residuo Polinomio di approssimazione di grado 2",residuo)
xmin=np.min(x)
xmax=np.max(x)
xval=np.linspace(xmin,xmax,100)
p=np.polyval(a,xval)
plt.plot(xval,p,'r-',x,y,'o')
plt.legend(['Polinomio di approssimazione di grado 2', 'Dati'])
plt.show()

```

# Interpolazione

## Funzione

```python
import numpy as np
def plagr(xnodi,k):
    """
    Restituisce i coefficienti del k-esimo pol di
    Lagrange associato ai punti del vettore xnodi
    """
    xzeri=np.zeros_like(xnodi)
    n=xnodi.size
    if k==0:
       xzeri=xnodi[1:n]
    else:
       xzeri=np.append(xnodi[0:k],xnodi[k+1:n])

    num=np.poly(xzeri)
    den=np.polyval(num,xnodi[k])

    p=num/den

    return p

def InterpL(x, f, xx):
     """"
        %funzione che determina in un insieme di punti il valore del polinomio
        %interpolante ottenuto dalla formula di Lagrange.
        % DATI INPUT
        %  x  vettore con i nodi dell'interpolazione
        %  f  vettore con i valori dei nodi
        %  xx vettore con i punti in cui si vuole calcolare il polinomio
        % DATI OUTPUT
        %  y vettore contenente i valori assunti dal polinomio interpolante
        %
     """
     n=x.size
     m=xx.size
     L=np.zeros((n,m))
     for k in range(n):
        p=plagr(x,k)
        L[k,:]=np.polyval(p,xx)

     return np.dot(f,L)
```

## Es

```python

import numpy as np
from funzioni_Interpolazione_Polinomiale import InterpL
import matplotlib.pyplot as plt
#nodi del problema di interpolazione

T=np.array([-55, -45, -35, -25, -15, -5, 5, 15, 25, 35, 45, 55, 65])
L=np.array([3.7, 3.7,3.52,3.27, 3.2, 3.15, 3.15, 3.25, 3.47, 3.52, 3.65, 3.67, 3.52])

# punti di valutazione per l'interpolante
xx=np.linspace(np.min(T),np.max(T),200);
pol=InterpL(T,L,xx);

pol42=InterpL(T,L,np.array([42]))
pol_42=InterpL(T,L,np.array([-42]))

plt.plot(xx,pol,'b--',T,L,'r*',42,pol42,'og',-42,pol_42,'og');
plt.legend(['interpolante di Lagrange','punti di interpolazione','stima 1', 'stima2']);
plt.show()

```

# Integrazione

## esame

```python
"""
Created on Sat Jun  5 08:29:40 2021

@author: damia
"""
import numpy as np
import matplotlib.pyplot as plt
import math

#funzioni per l'integrazione Simposon Composita e con ricerca automatica del numero di N
#di sottointervalli

def SimpComp(fname,a,b,n):
    h=(b-a)/(2*n)
    nodi=np.arange(a,b+h,h)
    f=fname(nodi)
    I=(f[0]+2*np.sum(f[2:2*n:2])+4*np.sum(f[1:2*n:2])+f[2*n])*h/3
    return I

def simptoll(fun,a,b,tol):

    Nmax=4096
    err=1

    N=1;
    IN=SimpComp(fun,a,b,N);

    while N<=Nmax and err>tol :
        N=2*N
        I2N=SimpComp(fun,a,b,N)
        err=abs(IN-I2N)/15
        IN=I2N

    if N>Nmax:
        print('Raggiunto nmax di intervalli con simptoll')
        N=0
        IN=[]

    return IN,N

#Funzioni per l'interpolazione di Lagrange
def plagr(xnodi,k):
    """
    Restituisce i coefficienti del k-esimo pol di
    Lagrange associato ai punti del vettore xnodi
    """
    xzeri=np.zeros_like(xnodi)
    n=xnodi.size
    if k==0:
       xzeri=xnodi[1:n]
    else:
       xzeri=np.append(xnodi[0:k],xnodi[k+1:n])

    num=np.poly(xzeri)
    den=np.polyval(num,xnodi[k])

    p=num/den

    return p

def InterpL(x, f, xx):
     """"
        %funzione che determina in un insieme di punti il valore del polinomio
        %interpolante ottenuto dalla formula di Lagrange.
        % DATI INPUT
        %  x  vettore con i nodi dell'interpolazione
        %  f  vettore con i valori dei nodi
        %  xx vettore con i punti in cui si vuole calcolare il polinomio
        % DATI OUTPUT
        %  y vettore contenente i valori assunti dal polinomio interpolante
        %
     """
     n=x.size
     m=xx.size
     L=np.zeros((n,m))
     for k in range(n):
        p=plagr(x,k)
        L[k,:]=np.polyval(p,xx)

     return np.dot(f,L)

# Script principale
tol=1e-08
x=np.zeros((6,))
y=np.zeros((6,))
N=np.zeros((6,))
fig=1

#Funzione integranda
f= lambda x: 2/math.pi*(5.5*(1-np.exp(-0.05*x))*np.sin(x**2))

for i in range(0,6):
    #Estremo destro dell'intervallo di integrazione
    x[i]=0.5+2*i

    #Costruisco punti nell'intervallo [0,x[i]] in cui valutare e poi disegnare la funzione integranda
    xi=np.linspace(0,x[i],100)
    plt.subplot(2,3,fig)
    plt.plot(xi,f(xi))
    plt.legend([ 'x['+str(i)+']='+str(x[i])])
    fig+=1
    #Calcolo il valore dell'integrale i-esimo  con a=0 e b= x[i]con la precisione richiesta
    y[i],N[i]=simptoll(f,0,x[i],tol)

plt.show()
xx=np.linspace(min(x),max(x),100)

#Calcolo il polinomio che interpola le coppie (x,y)
pol=InterpL(x, y, xx)
plt.plot(xx,pol,x,y,'ro')
plt.legend(['Polinomio interpolante','Nodi di interpolazione'])
plt.show()
print("Numero di sottointervalli per ciascuni il calcolo di ciascun integrale \n",N)

```

```python

import numpy as np
import f_Interpol_Polinomial as fip
import f_Integrazi_numerica as fin
import matplotlib.pyplot as plt

A = 1
B = 3
n = 3

f = lambda x: x-np.sqrt(x-1)

punti = np.linspace(A, B, 50)
nodi = np.linspace(A, B, n+1)
pol = fip.InterpL(nodi, f(nodi), punti)
print(pol)

def fp(val):
    return fip.InterpL(nodi, f(nodi), val)

plt.plot(punti, f(punti), 'r', punti, pol, 'b', nodi, f(nodi), 'go')
plt.show()

tol = 1e-5
I1 = 2.114381916835873
I2 = 2.168048769926493

I1a, N1 = fin.simptoll(f, A, B, tol)
I2a, N2 = fin.simptoll(fp, A, B, tol)

print(N1)
print(N2)
print(np.abs(I1a - I1))
print(np.abs(I2a - I2))
```

```python

import numpy as np
import f_Interpol_Polinomial as ip
import matplotlib.pyplot as plt

f = lambda x: 1/(1+900*x**2)

a = -1
b = 1

punti = np.linspace(a, b, 50)

for n in range(5, 35, 5):
    x = np.array(list())
    for i in range(1, n+2):
        x = np.append(x, -1+(2*(i-1)/n))

    pe = ip.InterpL(x, f(x), punti)
    re = np.abs(f(punti) - pe)
    plt.plot(punti, re)
    plt.show()

for n in range(5, 35, 5):
    x = np.array(list())
    for i in range(n+1, 0, -1):
        x = np.append(x, np.cos(((2*i-1)*np.pi)/(2*(n+1))))

    pe = ip.InterpL(x, f(x), punti)
    re = np.abs(f(punti) - pe)
    plt.plot(punti, re)
    plt.show()
```

```python

import numpy as np
import f_Interpol_Polinomial as fip
import matplotlib.pyplot as plt

f = lambda x: np.cos(np.pi * x) + np.sin(np.pi * x)
x = np.array([1.0, 1.5, 1.75])

a = 0
b = 2
xx = np.linspace(a, b, 50)

p = fip.InterpL(x, f(x), xx)

plt.plot(xx, p, 'r', xx, f(xx), 'g', x, f(x), 'o')
plt.legend(['Polinomio', 'Funzione', 'Nodi'])
plt.show()

xo = np.array([0.75])
r = np.abs(f(0.75) - fip.InterpL(x, f(x), xo))
print('Resto: ', r)

x = np.array([0.75, 1.0, 1.5, 1.75])
p2 = fip.InterpL(x, f(x), xx)

plt.plot(xx, p2, 'r', xx, f(xx), 'g', x, f(x), 'o')
plt.legend(['Polinomio 2', 'Funzione', 'Nodi'])
plt.show()
```

## Funzioni

```python
import numpy as np

def TrapComp(fname,a,b,n):
    h=(b-a)/n
    nodi=np.arange(a,b+h,h)
    f=fname(nodi)
    I=(f[0]+2*np.sum(f[1:n])+f[n])*h/2
    return I

def SimpComp(fname,a,b,n):
    h=(b-a)/(2*n)
    nodi=np.arange(a,b+h,h)
    f=fname(nodi)
    I=(f[0]+2*np.sum(f[2:2*n:2])+4*np.sum(f[1:2*n:2])+f[2*n])*h/3
    return I

def traptoll(fun,a,b,tol):

    Nmax=2048
    err=1

    N=1;
    IN=TrapComp(fun,a,b,N);

    while N<=Nmax and err>tol :
        N=2*N
        I2N=TrapComp(fun,a,b,N)
        err=abs(IN-I2N)/3
        IN=I2N

    if N>Nmax:
        print('Raggiunto nmax di intervalli con traptoll')
        N=0
        IN=[]

    return IN,N

def simptoll(fun,a,b,tol):

    Nmax=2048
    err=1

    N=1;
    IN=SimpComp(fun,a,b,N);

    while N<=Nmax and err>tol :
        N=2*N
        I2N=SimpComp(fun,a,b,N)
        err=abs(IN-I2N)/15
        IN=I2N

    if N>Nmax:
        print('Raggiunto nmax di intervalli con traptoll')
        N=0
        IN=[]

    return IN,N
```

## Es1

```python
import sympy as sym
import Funzioni_Integrazione as FI
from sympy.utilities.lambdify import lambdify
import numpy as np
import matplotlib.pyplot as plt
scelta=input("Scegli funzione ")

x=sym.symbols('x')
scelta_funzione = {
        '1': [x**10,0.0,1.0],
        '2': [sym.asin(x),0.0,1.0],
        '3': [sym.log(1+x), 0.0,1.0]
}
fx,a,b=scelta_funzione.get(scelta)

Iesatto=float(sym.integrate(fx,(x,a,b)))

f= lambdify(x,fx,np)
N=[1, 2, 4, 8, 16, 32 ,64 ,128, 256]
i=0
InT=[]
InS=[]
for n in N:
    InT.append(FI.TrapComp(f,a,b,n))
    InS.append(FI.SimpComp(f,a,b,n))

ET=np.zeros((9,))

ES=np.zeros((9,))

ET=np.abs(np.array(InT)-Iesatto)/abs(Iesatto)

ES=np.abs(np.array(InS)-Iesatto)/abs(Iesatto)

plt.semilogy(N,ET,'ro-',N,ES,'b*-')
plt.legend(['Errore Trapezi Composita', 'Errore Simpson Composita'])
plt.show()
```

## Es2 toll

```python
import sympy as sym
import Funzioni_Integrazione as FI
from sympy.utilities.lambdify import lambdify
import numpy as np
import matplotlib.pyplot as plt
scelta=input("Scegli funzione ")

x=sym.symbols('x')
scelta_funzione = {
        '1': [sym.log(x),1.0,2.0],
        '2': [sym.sqrt(x),0.0,1.0],
        '3': [sym.Abs(x), -1.0,1.0]
}
fx,a,b=scelta_funzione.get(scelta)

Iesatto=float(sym.integrate(fx,(x,a,b)))

f= lambdify(x,fx,np)

tol=1e-6

IT,NT=FI.traptoll(f,a,b,tol)

print("Il valore dell'integrale esatto e' ", Iesatto)
if NT>0:
    print("Valore con Trapezi Composito Automatica ",IT," numero di suddivisoini ",NT)

IS,NS=FI.simptoll(f,a,b,tol)

if NS>0:
    print("Valore con Simpson Composito Automatica ",IS," numero di suddivisoini ",NS)
```


# Markov chain Monte Carlo

A major takeaway from the previous section is the inherent difficulty in calculating or estimating posterior distributions for use in Bayesian inference. The two alternative strategies to obtaining posteriors for moderate to large models involve either analytic approximations or stochastic sampling. Approximations are usually valid conditional on assumptions regarding the true posterior distribution, which are typically impossible to validate. Direct sampling strategies rely on our ability to sample from the posterior distribution, and this is frequently not possible. Indirect sampling methods, such as rejection sampling, can be plagued with sampling efficiency issues.

The sampling approaches we have introduced so far have each attempted to obtain *independent* samples from the posterior distribution. It turns out, however, that it is possible to generate samples from the posterior distribution using a *dependent* sampling algorithm, and despite the dependence of the samples, one may extract valid inference from them. A class of algorithms called *Markov chain Monte Carlo* yields a Markovian sample (explained below) which, provided that certain conditions are satisfied, is guaranteed to be indistinguishable from a sample drawn from the true posterior itself.

## Markov Chains

A Markov chain is a special type of *stochastic process*. The standard definition of a stochastic process is an ordered collection of random variables:

$$\begin{gathered}
\begin{split}\{X_t:t \in T\}\end{split}\notag\\\begin{split}\end{split}\notag
\end{gathered}$$

where $t$ is frequently (but not necessarily) a time index. If we think of $X_t$ as a state $X$ at time $t$, and invoke the following dependence condition on each state:

$$\begin{gathered}
\begin{split}Pr(X_{t+1}=x_{t+1} | X_t=x_t, X_{t-1}=x_{t-1},\ldots,X_0=x_0) = Pr(X_{t+1}=x_{t+1} | X_t=x_t)\end{split}\notag\\\begin{split}\end{split}\notag\end{gathered}$$

then the stochastic process is known as a Markov chain. This conditioning specifies that the future depends on the current state, but not past states. Thus, the Markov chain wanders about the state space,
remembering only where it has just been in the last time step. The collection of transition probabilities is sometimes called a *transition matrix* when dealing with discrete states, or more generally, a
*transition kernel*.

In the context of Markov chain Monte Carlo, it is useful to think of the Markovian property as “mild non-independence”. MCMC allows us to indirectly generate independent samples from a particular posterior distribution.

### Markovian Jargon

Before we move on, it is important to define some general properties of Markov chains. They are frequently encountered in the MCMC literature, and some will help us decide whether MCMC is producing a useful sample from the posterior.

---

**Homogeneity**: A Markov chain is homogeneous at step $t$ if the transition probabilities are independent of time $t$.

**Irreducibility** A Markov chain is irreducible if every state is accessible in one or more steps from any other state. That is, the chain contains no absorbing states. This implies that there is a non-zero probability of eventually reaching state $k$ from any other state in the chain.

**Recurrence** States which are visited repeatedly are *recurrent*. If the expected time to return to a particular state is bounded, this is known as *positive recurrence*, otherwise the recurrent state is *null recurrent*. Further, a chain is *Harris recurrent* when it visits all states $X \in S$ infinitely often in the limit as $t \to \infty$; this is an important characteristic when dealing with unbounded, continuous state spaces. Whenever a chain ends up in a closed, irreducible set of Harris recurrent states, it stays there forever and visits every state with probability one.

**Stationarity** A stationary Markov chain produces the same marginal distribution when multiplied by the transition kernel. Thus, if $P$ is some $n \times n$ transition matrix:

$${\bf \pi P} = {\bf \pi}$$

for Markov chain $\pi$. Thus, $\pi$ is no longer subscripted, and is referred to as the *limiting distribution* of the chain. In MCMC, the chain explores the state space according to its limiting marginal distribution.

**Ergodicity**: Ergodicity is an emergent property of Markov chains which are irreducible, positive Harris recurrent and aperiodic. Ergodicity is defined as:

$$\lim_{n \to \infty} Pr^{(n)}(\theta_i \rightarrow \theta_j) = \pi(\theta) \quad \forall \theta_i, \theta_j \in \Theta$$

or in words, after many steps the marginal distribution of the chain is the
same at one step as at all other steps. This implies that our Markov chain,
which we recall is dependent, can generate samples that are independent if
we wait long enough between samples. If it means anything to you,
ergodicity is the analogue of the strong law of large numbers for Markov
chains. For example, take values $\theta_{i+1},\ldots,\theta_{i+n}$
from a chain that has reached an ergodic state. A statistic of interest can
then be estimated by:

$$\frac{1}{n}\sum_{j=i+1}^{i+n} h(\theta_j) \approx \int f(\theta) h(\theta) d\theta$$

---

## Why MCMC Works: Reversible Markov Chains

Markov chain Monte Carlo simulates a Markov chain for which some function of interest
(*e.g.* the joint distribution of the parameters of some model) is the unique, invariant limiting distribution. An invariant distribution with respect to some Markov chain with transition kernel $Pr(y \mid x)$ implies that:

$$\begin{gathered}
\begin{split}\int_x Pr(y \mid x) \pi(x) dx = \pi(y).\end{split}\notag\\\begin{split}\end{split}\notag
\end{gathered}$$

Invariance is guaranteed for any *reversible* Markov chain. Consider a Markov chain in reverse sequence:
$\{\theta^{(n)},\theta^{(n-1)},...,\theta^{(0)}\}$. This sequence is still Markovian, because:

$$\begin{gathered}
\begin{split}Pr(\theta^{(k)}=y \mid \theta^{(k+1)}=x,\theta^{(k+2)}=x_1,\ldots ) = Pr(\theta^{(k)}=y \mid \theta^{(k+1)}=x)\end{split}\notag\\\begin{split}\end{split}\notag\end{gathered}$$

Forward and reverse transition probabilities may be related through Bayes theorem:

$$\begin{gathered}
\begin{split}\end{split}\notag\end{gathered}$$

$$\begin{gathered}
\begin{split}\frac{Pr(\theta^{(k+1)}=x \mid \theta^{(k)}=y) \pi^{(k)}(y)}{\pi^{(k+1)}(x)}\end{split}\notag\\\begin{split}\end{split}\notag\end{gathered}$$

Though not homogeneous in general, $\pi$ becomes homogeneous if:

-   $n \rightarrow \infty$

-   $\pi^{(i)}=\pi$ for some $i < k$

If this chain is homogeneous it is called reversible, because it satisfies the ***detailed balance equation***:

$$\begin{gathered}
\begin{split}\pi(x)Pr(y \mid x) = \pi(y) Pr(x \mid y)\end{split}\notag\\\begin{split}\end{split}\notag\end{gathered}$$

Reversibility is important because it has the effect of balancing movement through the entire state space. When a Markov chain is reversible, $\pi$ is the unique, invariant, stationary distribution of that chain. Hence, if $\pi$ is of interest, we need only find the reversible Markov chain for which $\pi$ is the limiting distribution.
This is what MCMC does!

## Gibbs Sampling

The Gibbs sampler is the simplest and most prevalent MCMC algorithm. If a posterior has $k$ parameters to be estimated, we may condition each parameter on current values of the other $k-1$ parameters, and sample from the resultant distributional form (usually easier), and repeat this operation on the other parameters in turn. This procedure generates samples from the posterior distribution. Note that we have now combined Markov chains (conditional independence) and Monte Carlo techniques (estimation by simulation) to yield Markov chain Monte Carlo.

Here is a stereotypical Gibbs sampling algorithm:

1.  Choose starting values for states (parameters):
    ${\bf \theta} = [\theta_1^{(0)},\theta_2^{(0)},\ldots,\theta_k^{(0)}]$

2.  Initialize counter $j=1$

3.  Draw the following values from each of the $k$ conditional
    distributions:

    $$\begin{aligned}
    \theta_1^{(j)} &\sim& \pi(\theta_1 | \theta_2^{(j-1)},\theta_3^{(j-1)},\ldots,\theta_{k-1}^{(j-1)},\theta_k^{(j-1)}) \\
    \theta_2^{(j)} &\sim& \pi(\theta_2 | \theta_1^{(j)},\theta_3^{(j-1)},\ldots,\theta_{k-1}^{(j-1)},\theta_k^{(j-1)}) \\
    \theta_3^{(j)} &\sim& \pi(\theta_3 | \theta_1^{(j)},\theta_2^{(j)},\ldots,\theta_{k-1}^{(j-1)},\theta_k^{(j-1)}) \\
    \vdots \\
    \theta_{k-1}^{(j)} &\sim& \pi(\theta_{k-1} | \theta_1^{(j)},\theta_2^{(j)},\ldots,\theta_{k-2}^{(j)},\theta_k^{(j-1)}) \\
    \theta_k^{(j)} &\sim& \pi(\theta_k | \theta_1^{(j)},\theta_2^{(j)},\theta_4^{(j)},\ldots,\theta_{k-2}^{(j)},\theta_{k-1}^{(j)})\end{aligned}$$

4.  Increment $j$ and repeat until convergence occurs.

As we can see from the algorithm, each distribution is conditioned on the last iteration of its chain values, constituting a Markov chain as advertised. The Gibbs sampler has all of the important properties outlined in the previous section: it is aperiodic, homogeneous and ergodic. Once the sampler converges, all subsequent samples are from the target distribution. This convergence occurs at a geometric rate.

## Example: Inferring patterns in UK coal mining disasters

Let's try to model a more interesting example, a time series of recorded coal mining
disasters in the UK from 1851 to 1962.

Occurrences of disasters in the time series is thought to be derived from a
Poisson process with a large rate parameter in the early part of the time
series, and from one with a smaller rate in the later part. We are interested
in locating the change point in the series, which perhaps is related to changes
in mining safety regulations.

```python
import numpy as np

disasters_array = np.array([4, 5, 4, 0, 1, 4, 3, 4, 0, 6, 3, 3, 4, 0, 2, 6,
                         3, 3, 5, 4, 5, 3, 1, 4, 4, 1, 5, 5, 3, 4, 2, 5,
                         2, 2, 3, 4, 2, 1, 3, 2, 2, 1, 1, 1, 1, 3, 0, 0,
                         1, 0, 1, 1, 0, 0, 3, 1, 0, 3, 2, 2, 0, 1, 1, 1,
                         0, 1, 0, 1, 0, 0, 0, 2, 1, 0, 0, 0, 1, 1, 0, 2,
                         3, 3, 1, 1, 2, 1, 1, 1, 1, 2, 4, 2, 0, 0, 1, 4,
                         0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1])
```

```python
%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_context('notebook')

plt.figure(figsize=(12.5, 3.5))
n_count_data = len(disasters_array)
plt.bar(np.arange(1851, 1962), disasters_array, color="#348ABD")
plt.xlabel("Year")
plt.ylabel("Disasters")
plt.title("UK coal mining disasters, 1851-1962")
plt.xlim(1851, 1962);
```

We are going to use Poisson random variables for this type of count data. Denoting year $i$'s accident count by $y_i$,

$$ y_i \sim \text{Poisson}(\lambda)  $$

The modeling problem revolves around estimating the values of the $\lambda$ parameters. Looking at the time series above, it appears that the rate declines later in the time series.

A ***changepoint model*** identifies a point (year) during the observation period (call it $\tau$) after which the parameter $\lambda$ drops to a lower value. So we are estimating two $\lambda$ parameters: one for the early period and another for the late period.

$$
\lambda =
\begin{cases}
\lambda_1  & \text{if } t \lt \tau \cr
\lambda_2 & \text{if } t \ge \tau
\end{cases}
$$

We need to assign prior probabilities to both $\lambda$ parameters. The gamma distribution not only provides a continuous density function for positive numbers, but it is also **conjugate** with the Poisson sampling distribution. We will specify suitably vague hyperparameters $\alpha$ and $\beta$ for both priors.

\begin{align}
&\lambda_1 \sim \text{Gamma}( \alpha, \beta ) \\\
&\lambda_2 \sim \text{Gamma}( \alpha, \beta )
\end{align}

Since we do not have any intuition about the location of the changepoint (prior to viewing the data), we will assign a discrete uniform prior over all years 1851-1962.

\begin{align}
& \tau \sim \text{DiscreteUniform(1851,1962) }\\\\
& \Rightarrow P( \tau = k ) = \frac{1}{111}
\end{align}

### Implementing Gibbs sampling

We are interested in estimating the joint posterior of $\lambda_1$, $\lambda_2$ and $\tau$ given the array of annnual disaster counts $\mathbf{y}$. This gives:

$$
 P( \lambda_1, \lambda_2, \tau | \mathbf{y} ) \propto P(\mathbf{y} | \lambda_1, \lambda_2, \tau ) P(\lambda_1, \lambda_2, \tau)
$$

To employ Gibbs sampling, we need to factor the joint posterior into the product of conditional expressions:

$$
 P( \lambda_1, \lambda_2, \tau | \mathbf{y} ) \propto P(y_{t<\tau} | \lambda_1, \tau) P(y_{t\ge \tau} | \lambda_2, \tau) P(\lambda_1) P(\lambda_2) P(\tau)
$$

which we have specified as:

$$\begin{aligned}
P( \lambda_1, \lambda_2, \tau | \mathbf{y} ) &\propto \left[\prod_{t=1851}^{\tau} \text{Poi}(y_t|\lambda_1) \prod_{t=\tau+1}^{1962} \text{Poi}(y_t|\lambda_2) \right] \text{Gamma}(\lambda_1|\alpha,\beta) \text{Gamma}(\lambda_2|\alpha, \beta) \frac{1}{111} \\
&\propto \left[\prod_{t=1851}^{\tau} e^{-\lambda_1}\lambda_1^{y_t} \prod_{t=\tau+1}^{1962} e^{-\lambda_2} \lambda_2^{y_t} \right] \lambda_1^{\alpha-1} e^{-\beta\lambda_1} \lambda_2^{\alpha-1} e^{-\beta\lambda_2} \\
&\propto \lambda_1^{\sum_{t=1851}^{\tau} y_t +\alpha-1} e^{-(\beta+\tau)\lambda_1} \lambda_2^{\sum_{t=\tau+1}^{1962} y_i + \alpha-1} e^{-\beta\lambda_2}
\end{aligned}$$

So, the full conditionals are known, and critically for Gibbs, can easily be sampled from.

$$\lambda_1 \sim \text{Gamma}(\sum_{t=1851}^{\tau} y_t +\alpha, \tau+\beta)$$
$$\lambda_2 \sim \text{Gamma}(\sum_{t=\tau+1}^{1962} y_i + \alpha, 1962-\tau+\beta)$$
$$\tau \sim \text{Categorical}\left( \frac{\lambda_1^{\sum_{t=1851}^{\tau} y_t +\alpha-1} e^{-(\beta+\tau)\lambda_1} \lambda_2^{\sum_{t=\tau+1}^{1962} y_i + \alpha-1} e^{-\beta\lambda_2}}{\sum_{k=1851}^{1962} \lambda_1^{\sum_{t=1851}^{\tau} y_t +\alpha-1} e^{-(\beta+\tau)\lambda_1} \lambda_2^{\sum_{t=\tau+1}^{1962} y_i + \alpha-1} e^{-\beta\lambda_2}} \right)$$

Implementing this in Python requires random number generators for both the gamma and discrete uniform distributions. We can leverage NumPy for this:

```python
# Function to draw random gamma variate
rgamma = np.random.gamma

# Function to draw random categorical variate
rcategorical = lambda probs, n=None: np.array(probs).cumsum().searchsorted(
                                                        np.random.sample(n))
```

Next, in order to generate probabilities for the conditional posterior of $\tau$, we need the kernel of the gamma density:

```python
dgamma = lambda lam, a, b: lam**(a-1) * np.exp(-b*lam)
```

Diffuse hyperpriors for the gamma priors on $\lambda_1, \lambda_2$:

```python
alpha, beta = 1., 10
```

For computational efficiency, it is best to pre-allocate memory to store the sampled values. We need 3 arrays, each with length equal to the number of iterations we plan to run:

```python
# Specify number of iterations
n_iterations = 1000

# Initialize trace of samples
lambda1, lambda2, tau = np.empty((3, n_iterations+1))
```

The penultimate step initializes the model paramters to arbitrary values:

```python
lambda1[0] = 6
lambda2[0] = 2
tau[0] = 50
```

Now we can run the Gibbs sampler.

```python
# Sample from conditionals
for i in range(n_iterations):

    # Sample early mean
    lambda1[i+1] = rgamma(disasters_array[:int(tau[i])].sum() + alpha, 1./(tau[i] + beta))

    # Sample late mean
    lambda2[i+1] = rgamma(disasters_array[int(tau[i]):].sum() + alpha,
                          1./(n_count_data - tau[i] + beta))

    # Sample changepoint
    p = np.array([dgamma(lambda1[i+1], disasters_array[:t].sum() + alpha, t + beta)*
             dgamma(lambda2[i+1], disasters_array[t:].sum() + alpha, n_count_data - t + beta)
             for t in range(n_count_data)])
    tau[i+1] = rcategorical(p/p.sum())

```

Plotting the trace and histogram of the samples reveals the marginal posteriors of each parameter in the model.

```python
for samples in lambda1, lambda2, tau:
    fig, axes = plt.subplots(1, 2, figsize=(10, 3))
    axes[0].plot(samples[100:])
    axes[1].hist(samples[int(n_iterations/2):])
```

## The Metropolis-Hastings Algorithm

The key to success in applying the Gibbs sampler to the estimation of Bayesian posteriors is being able to specify the form of the complete conditionals of
${\bf \theta}$. In fact, the algorithm cannot be implemented without them. Of course, the posterior conditionals cannot always be neatly specified. In contrast to the Gibbs algorithm, the Metropolis-Hastings algorithm generates candidate state transitions from an alternate distribution, and accepts or rejects each candidate probabilistically.

Let us first consider a simple Metropolis-Hastings algorithm for a single parameter, $\theta$. We will use a standard sampling distribution, referred to as the *proposal distribution*, to produce candidate variables $q_t(\theta^{\prime} | \theta)$. That is, the generated value, $\theta^{\prime}$, is a *possible* next value for
$\theta$ at step $t+1$. We also need to be able to calculate the probability of moving back to the original value from the candidate, or
$q_t(\theta | \theta^{\prime})$. These probabilistic ingredients are used to define an *acceptance ratio*:

$$\begin{gathered}
\begin{split}a(\theta^{\prime},\theta) = \frac{q_t(\theta^{\prime} | \theta) \pi(\theta^{\prime})}{q_t(\theta | \theta^{\prime}) \pi(\theta)}\end{split}\notag\\\begin{split}\end{split}\notag\end{gathered}$$

The value of $\theta^{(t+1)}$ is then determined by:

$$\begin{gathered}
\begin{split}\theta^{(t+1)} = \left\{\begin{array}{l@{\quad \mbox{with prob.} \quad}l}\theta^{\prime} & \min(a(\theta^{\prime},\theta^{(t)}),1) \\ \theta^{(t)} & 1 - \min(a(\theta^{\prime},\theta^{(t)}),1) \end{array}\right.\end{split}\notag\\\begin{split}\end{split}\notag\end{gathered}$$

This transition kernel implies that movement is not guaranteed at every step. It only occurs if the suggested transition is likely based on the acceptance ratio.

A single iteration of the Metropolis-Hastings algorithm proceeds as follows:

The original form of the algorithm specified by Metropolis required that
$q_t(\theta^{\prime} | \theta) = q_t(\theta | \theta^{\prime})$, which reduces $a(\theta^{\prime},\theta)$ to
$\pi(\theta^{\prime})/\pi(\theta)$, but this is not necessary. In either case, the state moves to high-density points in the distribution with high probability, and to low-density points with low probability. After convergence, the Metropolis-Hastings algorithm describes the full target posterior density, so all points are recurrent.

1.  Sample $\theta^{\prime}$ from $q(\theta^{\prime} | \theta^{(t)})$.

2.  Generate a Uniform[0,1] random variate $u$.

3.  If $a(\theta^{\prime},\theta) > u$ then
    $\theta^{(t+1)} = \theta^{\prime}$, otherwise
    $\theta^{(t+1)} = \theta^{(t)}$.

### Random-walk Metropolis-Hastings

A practical implementation of the Metropolis-Hastings algorithm makes use of a random-walk proposal.
Recall that a random walk is a Markov chain that evolves according to:

$$
\theta^{(t+1)} = \theta^{(t)} + \epsilon_t \\
\epsilon_t \sim f(\phi)
$$

As applied to the MCMC sampling, the random walk is used as a proposal distribution, whereby dependent proposals are generated according to:

$$\begin{gathered}
\begin{split}q(\theta^{\prime} | \theta^{(t)}) = f(\theta^{\prime} - \theta^{(t)}) = \theta^{(t)} + \epsilon_t\end{split}\notag\\\begin{split}\end{split}\notag\end{gathered}$$

Generally, the density generating $\epsilon_t$ is symmetric about zero,
resulting in a symmetric chain. Chain symmetry implies that
$q(\theta^{\prime} | \theta^{(t)}) = q(\theta^{(t)} | \theta^{\prime})$,
which reduces the Metropolis-Hastings acceptance ratio to:

$$\begin{gathered}
\begin{split}a(\theta^{\prime},\theta) = \frac{\pi(\theta^{\prime})}{\pi(\theta)}\end{split}\notag\\\begin{split}\end{split}\notag\end{gathered}$$

The choice of the random walk distribution for $\epsilon_t$ is frequently a normal or Student’s $t$ density, but it may be any distribution that generates an irreducible proposal chain.

An important consideration is the specification of the scale parameter for the random walk error distribution. Large values produce random walk steps that are highly exploratory, but tend to produce proposal values in the tails of the target distribution, potentially resulting in very small acceptance rates. Conversely, small values tend to be accepted more frequently, since they tend to produce proposals close to the current parameter value, but may result in chains that mix very slowly.
Some simulation studies suggest optimal acceptance rates in the range of 20-50%. It is often worthwhile to optimize the proposal variance by iteratively adjusting its value, according to observed acceptance rates early in the MCMC simulation .

## Example: Linear model estimation

This very simple dataset is a selection of real estate prices, with the associated age of each house. We wish to estimate a simple linear relationship between the two variables, using the Metropolis-Hastings algorithm.

$$\mu_i = \beta_0 + \beta_1 a_i$$

$$p_i \sim N(\mu_i, \tau)$$

```python
age = np.array([13, 14, 14,12, 9, 15, 10, 14, 9, 14, 13, 12, 9, 10, 15, 11,
                15, 11, 7, 13, 13, 10, 9, 6, 11, 15, 13, 10, 9, 9, 15, 14,
                14, 10, 14, 11, 13, 14, 10])
price = np.array([2950, 2300, 3900, 2800, 5000, 2999, 3950, 2995, 4500, 2800,
                  1990, 3500, 5100, 3900, 2900, 4950, 2000, 3400, 8999, 4000,
                  2950, 3250, 3950, 4600, 4500, 1600, 3900, 4200, 6500, 3500,
                  2999, 2600, 3250, 2500, 2400, 3990, 4600, 450,4700])/1000.
```

This function calculates the joint log-posterior, conditional on values for each paramter:

```python
from scipy.stats import distributions
dgamma = distributions.gamma.logpdf
dnorm = distributions.norm.logpdf

def calc_posterior(a, b, t, y=price, x=age):
    # Calculate joint posterior, given values for a, b and t

    # Priors on a,b
    logp = dnorm(a, 0, 10000) + dnorm(b, 0, 10000)
    # Prior on t
    logp += dgamma(t, 0.001, 0.001)
    # Calculate mu
    mu = a + b*x
    # Data likelihood
    logp += sum(dnorm(y, mu, t**-0.5))

    return logp
```

```python
rnorm = np.random.normal
runif = np.random.rand

def metropolis(n_iterations, initial_values, prop_var=1):

    n_params = len(initial_values)

    # Initial proposal standard deviations
    prop_sd = [prop_var]*n_params

    # Initialize trace for parameters
    trace = np.empty((n_iterations+1, n_params))

    # Set initial values
    trace[0] = initial_values

    # Calculate joint posterior for initial values
    current_log_prob = calc_posterior(*trace[0])

    # Initialize acceptance counts
    accepted = [0]*n_params

    for i in range(n_iterations):

        if not i%1000: print('Iteration %i' % i)

        # Grab current parameter values
        current_params = trace[i]

        for j in range(n_params):

            # Get current value for parameter j
            p = trace[i].copy()

            # Propose new value
            if j==2:
                # Ensure tau is positive
                theta = np.exp(rnorm(np.log(current_params[j]), prop_sd[j]))
            else:
                theta = rnorm(current_params[j], prop_sd[j])

            # Insert new value
            p[j] = theta

            # Calculate log posterior with proposed value
            proposed_log_prob = calc_posterior(*p)

            # Log-acceptance rate
            alpha = proposed_log_prob - current_log_prob

            # Sample a uniform random variate
            u = runif()

            # Test proposed value
            if np.log(u) < alpha:
                # Accept
                trace[i+1,j] = theta
                current_log_prob = proposed_log_prob
                accepted[j] += 1
            else:
                # Reject
                trace[i+1,j] = trace[i,j]

    return trace, accepted
```

Let's run the MH algorithm with a very small proposal variance:

```python
n_iter = 10000
trace, acc = metropolis(n_iter, (1,0,1), 0.001)
```

We can see that the acceptance rate is way too high:

```python
np.array(acc, float)/n_iter
```

```python
for param, samples in zip(['intercept', 'slope', 'precision'], trace.T):
    fig, axes = plt.subplots(1, 2, figsize=(8, 2))
    axes[0].plot(samples)
    axes[0].set_ylabel(param)
    axes[1].hist(samples[int(n_iter/2):])
```

Now, with a very large proposal variance:

```python
trace_hivar, acc = metropolis(n_iter, (1,0,1), 10)
```

```python
np.array(acc, float)/n_iter
```

```python
for param, samples in zip(['intercept', 'slope', 'precision'], trace_hivar.T):
    fig, axes = plt.subplots(1, 2, figsize=(8, 2))
    axes[0].plot(samples)
    axes[0].set_ylabel(param)
    axes[1].hist(samples[int(n_iter/2):])
```

In order to avoid having to set the proposal variance by trial-and-error, we can add some tuning logic to the algorithm. The following implementation of Metropolis-Hastings reduces proposal variances  by 10% when the acceptance rate is low, and increases it by 10% when the acceptance rate is high.

```python
def metropolis_tuned(n_iterations, initial_values, prop_var=1,
                     tune_for=None, tune_interval=100):

    n_params = len(initial_values)

    # Initial proposal standard deviations
    prop_sd = [prop_var] * n_params

    # Initialize trace for parameters
    trace = np.empty((n_iterations+1, n_params))

    # Set initial values
    trace[0] = initial_values
    # Initialize acceptance counts
    accepted = [0]*n_params

    # Calculate joint posterior for initial values
    current_log_prob = calc_posterior(*trace[0])

    if tune_for is None:
        tune_for = n_iterations/2

    for i in range(n_iterations):

        if not i%1000: print('Iteration %i' % i)

        # Grab current parameter values
        current_params = trace[i]

        for j in range(n_params):

            # Get current value for parameter j
            p = trace[i].copy()

            # Propose new value
            if j==2:
                # Ensure tau is positive
                theta = np.exp(rnorm(np.log(current_params[j]), prop_sd[j]))
            else:
                theta = rnorm(current_params[j], prop_sd[j])

            # Insert new value
            p[j] = theta

            # Calculate log posterior with proposed value
            proposed_log_prob = calc_posterior(*p)

            # Log-acceptance rate
            alpha = proposed_log_prob - current_log_prob

            # Sample a uniform random variate
            u = runif()

            # Test proposed value
            if np.log(u) < alpha:
                # Accept
                trace[i+1,j] = theta
                current_log_prob = proposed_log_prob
                accepted[j] += 1
            else:
                # Reject
                trace[i+1,j] = trace[i,j]

            # Tune every 100 iterations
            if (not (i+1) % tune_interval) and (i < tune_for):

                # Calculate aceptance rate
                acceptance_rate = (1.*accepted[j])/tune_interval
                if acceptance_rate<0.2:
                    prop_sd[j] *= 0.9
                elif acceptance_rate>0.5:
                    prop_sd[j] *= 1.1
                accepted[j] = 0

    return trace[tune_for:], accepted
```

```python
trace_tuned, acc = metropolis_tuned(n_iter, (1,0,1), tune_for=9000)
```

```python
np.array(acc, float)/1000
```

```python
for param, samples in zip(['intercept', 'slope', 'precision'], trace_tuned.T):
    fig, axes = plt.subplots(1, 2, figsize=(8, 2))
    axes[0].plot(samples)
    axes[0].set_ylabel(param)
    axes[1].hist(samples[int(len(samples)/2):])
```

50 random regression lines drawn from the posterior:

```python
plt.plot(age, price, 'bo')
plt.xlabel('age (years)'); plt.ylabel('price ($1000\'s)')
xvals = np.linspace(age.min(), age.max())
for i in range(50):
    b0,b1,tau = trace_tuned[np.random.randint(0, 1000)]
    plt.plot(xvals, b0 + b1*xvals, 'r-', alpha=0.2)
```

## Exercise: Bioassay analysis

Gelman et al. (2003) present an example of an acute toxicity test, commonly performed on animals to estimate the toxicity of various compounds.

In this dataset `log_dose` includes 4 levels of dosage, on the log scale, each administered to 5 rats during the experiment. The response variable is `death`, the number of positive responses to the dosage.

The number of deaths can be modeled as a binomial response, with the probability of death being a linear function of dose:

$$\begin{aligned}
y_i &\sim \text{Bin}(n_i, p_i) \\
\text{logit}(p_i) &= a + b x_i
\end{aligned}$$

The common statistic of interest in such experiments is the **LD50**, the dosage at which the probability of death is 50%.

Use Metropolis-Hastings sampling to fit a Bayesian model to analyze this bioassay data, and to estimate LD50.

```python
# Log dose in each group
log_dose = [-.86, -.3, -.05, .73]

# Sample size in each group
n = 5

# Outcomes
deaths = [0, 1, 3, 5]
```

```python
# Write your answer here
```

## Slice Sampling

Though Gibbs sampling is very computationally efficient, it can be difficult to implement in a general way, whereas the Metropolis-Hastings algorithm is relatively inefficient, while being easy to implement for a variety of models. We have seen that it is possible to tune Metropolis samplers, but it would be nice to have a "black-box" method that works for arbitrary continuous distributions, which we may know little about *a priori*.

The ***slice sampler*** bridges this gap by being both efficient and easy to program generally.
The idea is to first sample from the conditional distribution for $y$ given some current value of $x$, which is uniform over the $(0, f (x))$, and conditional on this value for $y$, then sample $x$, which is uniform on $S = {x : y < f (x)},$, that is the “slice” defined by the $y$ value. Hence, this algorithm automatically adapts to the local characteristics of the posterior.

The steps required to perform a single iteration of the slice sampler to update the current value of $x_i$ is as follows:

1. Sample $y$ uniformly on $(0,f(x_i))$.
2. Use this value $y$ to define a horizontal *slice* $S = \{x : y < f (x)\}$.
3. Establish an interval, $I = (x_{a}, x_{b})$, around $x_i$ that contains most of the slice.
4. Sample $x_{i+1}$ from the region of the slice overlaping $I$.

Hence, slice sampling employs an *auxilliary variable* ($y$) that is not retained at the end of the iteration. Note that in practice one may operate on the log scale such that $g(x) = \log(f (x))$ to avoid floating-point underflow. In this case, the auxiliary variable becomes $z = log(y) = g(x_i) − e$, where $e \sim \text{Exp}(1)$, resulting in the slice $S = \{x : z < g(x)\}$.

There are many ways of establishing and sampling from the interval $I$, with the only restriction being that the resulting Markov chain leaves $f(x)$ *invariant*. The objective is to include as much of the slice as possible, so that the potential step size can be large, but not (much) larger than the slice, so that the sampling of invalid points is minimized. Ideally, we would like it to be the slice itself, but it may not always be feasible to determine (and certainly not automatically).

### Stepping out

One method for determining a sampling interval for $x_{i+1}$  involves specifying an initial "guess" at the slice width $w$, and iteratively moving the endpoints out (growing the interval) until either (1) the interval reaches a maximum pre-speficied width or (2) $y$ is less than the $f(x)$ evaluated both at the left and the right interval endpoints.

```python
def step_out(func, x0, y, w, m=np.inf):
    """
    func: target function (evaluates at x)
    x0: current value of x
    y: y-value that defines slice
    w: estimate of slice width
    m: factor limiting interval to size m*w (defaults to infinity)

    Returns end points of a sampling interval for slice at y
    """

    left = x0 - w*np.random.random()
    right = left + w

    i = np.floor(m*np.random.random())
    j = (m-1) - i

    while (i > 0) and (y < func(left)):
        left -= w
        i -= 1

    while (j > 0) and (y < func(right)):
        right += w
        j -= 1

    return left, right
```

```python
from scipy.stats import gamma
gam = gamma(2, scale=1).pdf

xvals = np.linspace(0, 10)
plt.plot(xvals, gam(xvals))

x, y = 1, 0.03
l,r = step_out(gam, x, y, 1, 5)
plt.plot((l,r), (y,y), 'r-')

x, y = 1, 0.3
l,r = step_out(gam, x, y, 1, 5)
plt.plot((l,r), (y,y), 'm-')
```

### Doubling

The efficiency of stepping out depends largely on the ability to pick a reasonable interval $w$ from which to sample. Otherwise, the doubling procedure may be preferable, as it can be expanded faster. It simply doubles the size of the interval until both endpoints are outside the slice.

```python
def doubling(func, x0, y, w, p=10):
    """
    func: target function (evaluates at x)
    x0: current value of x
    y: y-value that defines slice
    w: estimate of slice width
    p: integer limiting interval to size m*w (defaults to infinity)

    Returns end points of a sampling interval for slice at y
    """

    left = x0 - w*np.random.random()
    right = left + w

    while (p > 0) and ((y < func(left)) or (y < func(right))):

        if np.random.random() < 0.5:

            left -= right - left

        else:

            right += right - left

        p -= 1

    return left, right
```

```python
gam = gamma(2, scale=1).pdf

xvals = np.linspace(0, 10)
plt.plot(xvals, gam(xvals))

x, y = 1, 0.03
l,r = doubling(gam, x, y, 1)
plt.plot((l,r), (y,y), 'r-')

x, y = 1, 0.3
l,r = doubling(gam, x, y, 1)
plt.plot((l,r), (y,y), 'm-')
```

Irrespective of which method for interval determination is used, the next step is to draw a value from this interval. One condition that applies to the new point is that it should be as likely to draw the interval $I$ from the new value as it was from the current value.

There are two approaches that could be taken:

1. Draw from $I$ until a suitable point is obtained
2. Draw from $I$, which shrinks by some factor each time an unsuitable point is drawn, until a suitable point is obtained

A shrinkage function might proceed as follows:

```python
def shrink(func, x0, y, left, right, w):

    l, r = left, right

    while True:

        x1 = l + np.random.random()*(r - l)

        if (y < func(x1)) and check_value(x0, x1, l, r, y, func, w):
            return x1

        if x1 < x0:
            l = x1
        else:
            r = x1
```

If a doubling procedure was used to expand the sampling interval, we require an acceptance function to ascertain whether a given point satisfies the criterion above.

```python
def check_value(x0, x1, left, right, y, func, w):

    l, r = left, right

    m = (l + r)/2.

    differ = False

    while (r - l) > (1.1*w):

        if ((x0 < m) and (x1 >= m)) or ((x0 >= m) and (x1 < m)):
            # Intervals generated from new point likely different
            differ = True

        # Reduce interval
        if x1 < m:
            r = m
        else:
            l = m

        if differ and (y >= func(l)) and (y >= func(r)):
            # Point is not acceptable
            return False

    # If not rejected above, accept
    return True
```

Otherwise we can just substitute a trivial function for `check_value`:

    check_value = lambda *args, **kwargs: return True

```python
uniform = np.random.uniform

def slice(n_iterations, logp, initial_values, w=1, tune=True):

    n_params = len(initial_values)

    # Initialize trace for parameters
    trace = np.empty((n_iterations+1, n_params))

    # Set initial values
    trace[0] = initial_values

    w_tune = []

    for i in range(n_iterations):

        if not i%1000: print('Iteration %i' % i)

        q = trace[i]
        q0 = q.copy()

        w = np.resize(w, len(q0))

        y = logp(*q0) - np.random.exponential()

        # Stepping out procedure
        ql = q0.copy()
        ql -= uniform(0, w)
        qr = q0.copy()
        qr = ql + w

        yl = logp(*ql)
        yr = logp(*qr)

        while((y < yl).all()):
            ql -= w
            yl = logp(*ql)

        while((y < yr).all()):
            qr += w
            yr = logp(*qr)

        q_next = q0.copy()

        while True:

            # Sample uniformly from slice
            qi = uniform(ql, qr)

            yi = logp(*qi)

            if yi > y:
                q = qi
                break
            elif (qi > q).all():
                qr = qi
            elif (qi < q).all():
                ql = qi

        if tune:
            # Tune sampler parameters
            w_tune.append(abs(q0 - q))
            w = 2 * sum(w_tune, 0) / len(w_tune)

        trace[i+1] = q

    return trace
```

```python
n_iterations = 5000
trace = slice(n_iterations, calc_posterior, (1,0,1))
```

```python
for param, samples in zip(['intercept', 'slope', 'precision'], trace.T):
    fig, axes = plt.subplots(1, 2, figsize=(8, 2))
    axes[0].plot(samples )
    axes[0].set_ylabel(param)
    axes[1].hist(samples[int(n_iterations/2):])
```

## References

Gelman, A., Carlin, J. B., Stern, H. S., & Rubin, D. B. (2003). Bayesian Data Analysis, Second Edition (Chapman & Hall/CRC Texts in Statistical Science) (2nd ed.). Chapman and Hall/CRC.

Neal, R. M. (2003). Slice sampling. The Annals of Statistics, 31(3), 705–767. doi:10.1111/1467-9868.00198


# Tagging

## Tagging cells

### SIR model without vital dynamics

The [Susceptible-Infectious-Recovered (SIR) model](https://en.wikipedia.org/wiki/Compartmental_models_in_epidemiology) is an example of a compartmental model that is widely used to forecast the progress of disease epidemics.

\begin{align}
\frac{dS}{dt} & = -\frac{\beta I S}{N} \\
\frac{dI}{dt} & = \frac{\beta I S}{N} - \gamma I \\
\frac{dR}{dt} & = \gamma I
\end{align}

Where the **basic reproduction number** $R_0 = \frac{\beta}{\gamma}$

```python
%matplotlib inline

import numpy as np
import matplotlib.pyplot as plt
from scipy.integrate import solve_ivp

N = 10000.0
R0 = 2.4
gamma = 1/8.0
beta = gamma*R0

def deriv(t, y):
    S, I, R = y
    dSdt = -beta*I*S/N
    dIdt = beta*I*S/N - gamma*I
    dRdt = gamma*I
    return [dSdt, dIdt, dRdt]

t = np.linspace(0, 180, 181)
y0 = [N - 1.0, 1.0, 0.0]
soln = solve_ivp(deriv, [t[0], t[-1]], y0, t_eval=t)

plt.plot(soln.t, soln.y[0], label='Susceptible')
plt.plot(soln.t, soln.y[1], label='Infectious')
plt.plot(soln.t, soln.y[2], label='Recoverd')
plt.legend()
plt.xlabel('days')
plt.grid(True)
```

# Special formatting for special tags

### SIR model without vital dynamics

The [Susceptible-Infectious-Recovered (SIR) model](https://en.wikipedia.org/wiki/Compartmental_models_in_epidemiology) is an example of a compartmental model that is widely used to forecast the progress of disease epidemics.

\begin{align}
\frac{dS}{dt} & = -\frac{\beta I S}{N} \\
\frac{dI}{dt} & = \frac{\beta I S}{N} - \gamma I \\
\frac{dR}{dt} & = \gamma I
\end{align}

Where the **basic reproduction number** $R_0 = \frac{\beta}{\gamma}$

This cell has been tagged with "home-activity".

This cell has been tagged with "class-activity".

This cell has been tagged with "important-note".

## Tagging solutions in code cells

For teaching purposes, it is often useful to selecting remove code from cells a

```python
# write a function that returns the square of a number

assert sqr(2) == 4
assert sqr(-1) == 1

```

```python
assert sqr('string') == 12
```


# Symbolic Mathematics in Python

There are times when you need to solve a difficult problem symbollically or analytically. If you have ever used Wolfram Alpha, then you have already done this. Sympy is a python library that allows you to do symbolic mathematics in python.

```python
import sympy as sym
```

## 1. Introduction

### Example 1.1

If you try to write the follwing in python by itself, you will get an error telling you x is undefined:
$$x-x$$

```python
x-x
```

(The error above is on purpose). Variables in python need to be defined before you can say something specific about them

```python
x=102
x-x
```

If you are trying to show that $x-x=0$ is true for any $x$, the above answer would not be valid. Instead you can use a symbolic expression to show that it is true

**First we define the variable as a symmbolic expression**

```python
x = sym.symbols('x')
```

**Now we can use the variable in a symbolic expression**

```python
x-x
```

### Example 1.2

Sympy can be used to perform algebreic operations (among other things). Consider the following expression: $$(3a-4b)^3$$

We can use symppy to expand the expression algebraically.

**First we need to define the variables as symbolic expressions**

```python
a,b = sym.symbols('a,b')
```

**Side note** Notice that the left hand side of the epression has two variables being defined. Python can define more than one variable at a time:

```python
x1,y1 =10,20
print(x1)
print(y1)
```

    10
    20

**Back to the expression** We can define an expression using the variables $a$ and $b$.

```python
expr = (3*a-4*b)**3
print(expr)
```

    (3*a - 4*b)**3

We can also make it look nicer in our notebook. This doesn't affect the math, but it makes our notebook more readable.

```python
sym.init_printing()
```

```python
expr
```

**Now we expand the function algebreically**

```python
expr.expand()
```

Sympy can also factor the equation

```python
sym.factor(26*a**3-108*a**2*b+144*a*b**2-64*b**3)
```

If you want to copy and paste a result, you print the result.

```python
print(sym.factor(26*a**3-108*a**2*b+144*a*b**2-64*b**3))
```

    2*(a - 2*b)*(13*a**2 - 28*a*b + 16*b**2)

You can also chain together functions

```python
expr.expand().factor()
```

### Exercise 1.1

Show that the following two expressions are true.
$$(2w-3z)(2w+3z)=4w^2-9z^2$$
$$(2w-3z)^2\ne4w^2-9z^2$$

```python
w,z = sym.symbols('w,z')
expr = (2*w-3*z)*(2*w+3*z)
expr.expand()
expr2= (2*w-3*z)**2
expr2.expand()
print(expr.expand())
print(expr2.expand())
print("This proves the second equation is not true")

```

    4*w**2 - 9*z**2
    4*w**2 - 12*w*z + 9*z**2
    This proves the second equation is not true

```python
## 2. Solving Equations
#Sympy can be used to symbolilically solve equations. As before, you need to define which variables are symbols
```

### Example 2.1

Use sympy to solve the following equation
$$ax^3+bx^2+cx+d=0$$

```python
# Define the variables
a,b,c,d,x = sym.symbols('a,b,c,d,x')
```

```python
# Define the expression
expr=a*x**3+b*x**2+c*x+d
expr
```

We can use the `solvset` function to solve this equation

```python
solutions=sym.solveset(expr,x)
```

```python
print(solutions)
```

    {-(-3*c/a + b**2/a**2)/(3*(sqrt(-4*(-3*c/a + b**2/a**2)**3 + (27*d/a - 9*b*c/a**2 + 2*b**3/a**3)**2)/2 + 27*d/(2*a) - 9*b*c/(2*a**2) + b**3/a**3)**(1/3)) - (sqrt(-4*(-3*c/a + b**2/a**2)**3 + (27*d/a - 9*b*c/a**2 + 2*b**3/a**3)**2)/2 + 27*d/(2*a) - 9*b*c/(2*a**2) + b**3/a**3)**(1/3)/3 - b/(3*a), -(-3*c/a + b**2/a**2)/(3*(-1/2 - sqrt(3)*I/2)*(sqrt(-4*(-3*c/a + b**2/a**2)**3 + (27*d/a - 9*b*c/a**2 + 2*b**3/a**3)**2)/2 + 27*d/(2*a) - 9*b*c/(2*a**2) + b**3/a**3)**(1/3)) - (-1/2 - sqrt(3)*I/2)*(sqrt(-4*(-3*c/a + b**2/a**2)**3 + (27*d/a - 9*b*c/a**2 + 2*b**3/a**3)**2)/2 + 27*d/(2*a) - 9*b*c/(2*a**2) + b**3/a**3)**(1/3)/3 - b/(3*a), -(-3*c/a + b**2/a**2)/(3*(-1/2 + sqrt(3)*I/2)*(sqrt(-4*(-3*c/a + b**2/a**2)**3 + (27*d/a - 9*b*c/a**2 + 2*b**3/a**3)**2)/2 + 27*d/(2*a) - 9*b*c/(2*a**2) + b**3/a**3)**(1/3)) - (-1/2 + sqrt(3)*I/2)*(sqrt(-4*(-3*c/a + b**2/a**2)**3 + (27*d/a - 9*b*c/a**2 + 2*b**3/a**3)**2)/2 + 27*d/(2*a) - 9*b*c/(2*a**2) + b**3/a**3)**(1/3)/3 - b/(3*a)}

```python
solutions
```

What if I need help. You can do this with any python function. `function?`

```python
# Run this command to see a help box
sym.solveset?
```

### Exercise 2.1

Use the `solveset` function to solve the following chemical problem.

Phosgene gas, $\text{COCl}_2$, dissociates at high temperatures according to the following equilibrium:

$$ \text{COCl}_2 \rightleftharpoons  \text{CO} + \text{Cl}_2 $$

At $\text{400 C}$, the equilibrium constant $K_c=8.05$.

If you start with a  $\text{0.250 M}$ phosgene sample at $\text{400 C}$, determine the concentrations of all species at equilibrium.

```python
# Your code here
#x^2/(.250-x) - 8.05
#sym.solveset(f, symbol=x)
x = sym.symbols('x')
sym.solveset(x**2/(.250-x)-8.05,x)

```

```python
#Why did you pick your answer?
print("The answer should be 0.242683798033777 because the concentration should be positive")
```

    The answer should be 0.242683798033777 because the concentration should be positive

## 3. Calculus

We can use also Sympy to differentiate and integrate. Let us experiment with differentiating the following expression:

$$x ^ 2 - \cos(x)$$

```python
sym.diff(x ** 2 - sym.cos(x), x)
```

Similarly we can integrate:

```python
sym.integrate(x ** 2 - sym.cos(x), x)
```

We can also carry out definite integrals:

```python
sym.integrate(x ** 2 - sym.cos(x), (x, 0, 5))
```

### Exercise 3.1

Use Sympy to calculate the following:

1. $\frac{d(x ^2 + xy - \ln(y))}{dy}$
1. $\int_0^5 e^{2x}\;dx$

```python
x,y = sym.symbols('x,y')
sym.diff(x ** 2 + x*y-sym.ln(y), x)
```

```python
#This took me forevverrrr
import sympy as sym
import numpy as num
import math
from sympy import exp
x = sym.symbols('x')
integ = exp(2*x)
sym.integrate(integ,(x,0,5))

```

    -1/2 + exp(10)/2

### Exercise 3.2
Solve the following definate integral
$$\int\limits_{ - \infty }^\infty  {\frac{1}{{\sigma \sqrt {2\pi } }}{e^{ - \frac{1}{2}{{\left( {\frac{{x - \mu }}{\sigma }} \right)}^2}}}}$$

Hint, the sympy symbol for infinity is `oo`

```python
# Your code here
import math
import sympy as sym
import numpy as num
from sympy import exp
from sympy import sqrt

sym.init_printing()

z,b,c = sym.symbols('z,b,c')
g1= exp((-1/2)*((z - b)/c)**2)
g2=(1/c*sqrt(2*num.pi))
gaussian= g1*g2
sym.integrate(gaussian,(x,'-oo','oo'))

```

Lookup Gaussian functions: https://en.wikipedia.org/wiki/Gaussian_function
Does your answer maake sense?

## 4. Plotting with Sympy

Finally Sympy can be used to plot functions. Note that this makes use of [matplotlib](http://matplotlib.org/).

Let us plot $x^2$:

```python
expr = x **2
p=sym.plot(expr)
```

### Exercise 4.1 Plot the following function:

1. $y=x + cos(x)$
1. ${\frac{1}{{ \sqrt {2\pi } }}{e^{ - \frac{x^2}{2}}}}$

```python
x = sym.symbols('x')
e1 = x+sym.cos(x)
p1 = sym.plot(e1)
```

```python
x = sym.symbols('x')
e1 = (1/sqrt(2*num.pi))
e2 = exp((-x**2)/2)
gaussian= e1*e2
p1 = sym.plot(gaussian)
```

# Lecture

## L1. Hydrogen Atom

Sympy has built in modules for the eigenfunctions of the hydrogen atom.

```python

```

```python
import sympy.physics.hydrogen
import numpy as np
```

You can caluclate the eigenvalues ($E$) in Hartrees

`sym.physics.hydrogen.E_nl(n,Z)`

```python
sym.physics.hydrogen.E_nl(1,1)
```

We can use a loop to print out many energies

```python
for n in range(1,5):
    print(sym.physics.hydrogen.E_nl(n,1))
```

    -1/2
    -1/8
    -1/18
    -1/32

We can plot the hydrogen radial wavefunction (1s orbital)

```python
r = sympy.symbols('r')
sympy.physics.hydrogen.R_nl(1, 0, r, 1)
#Principle quantum, angular momentum, variable, and Z
```

```python
sym.plot(sympy.physics.hydrogen.R_nl(1, 0, r, 1),(r,0,10.50))
```

And the probablity distribution function

```python
sympy.symbols('r')
prob_1s=sympy.physics.hydrogen.R_nl(1, 0, r, 1)*sympy.physics.hydrogen.R_nl(1, 0, r, 1)
prob_1s
#Principle quantum, angular momentum, variable, and Z
```

```python
sym.plot(prob_1s,(r,0,10))
```

Plot a 2s orbital

```python
sympy.symbols('r')
prob_2s=sympy.physics.hydrogen.R_nl(2, 0, r, 1)*sympy.physics.hydrogen.R_nl(2, 0, r, 1)
prob_2s
#Principle quantum, angular momentum, variable, and Z
```

```python
sym.plot(prob_2s,(r,0,10))
```

We can change the range to see the node better.

```python
sym.plot(prob_2s,(r,1,8))
```

Notice the node!

### Exercise L1.1

Plot the radial distriubution function for a 2p, 3s, 4s, and 3d orbital.

```python
#I STILL HAVE TO DO 4S, AND 3D
print("2p")
sympy.symbols('r')
prob_2p=sympy.physics.hydrogen.R_nl(2, 1, r, 1)*sympy.physics.hydrogen.R_nl(2, 1, r, 1)
prob_2p
#Principle quantum, angular momentum, variable, and Z
```

```python
print("2p")
sym.plot(prob_2p,(r,-1.5,12))
```

```python
print("3s")
sympy.symbols('r')
prob_3s=sympy.physics.hydrogen.R_nl(3, 0, r, 1)*sympy.physics.hydrogen.R_nl(3, 0, r, 1)
prob_3s
```

```python
print("3s")
sym.plot(prob_3s,(r,-1.5,12))
```

```python
print("4s")
sympy.symbols('r')
prob_4s=sympy.physics.hydrogen.R_nl(4, 0, r, 1)*sympy.physics.hydrogen.R_nl(4, 0, r, 1)
prob_4s
```

```python
print("4s")
sym.plot(prob_4s,(r,-1.5,12))
```

```python
print("3d")
sympy.symbols('r')
prob_3d=sympy.physics.hydrogen.R_nl(3, 2, r, 1)*sympy.physics.hydrogen.R_nl(3, 2, r, 1)
prob_3d
```

```python
print("3d")
sym.plot(prob_3d,(r,-1.5,12))
```

```python

```


# Spectral Expansions for Asian Options

## Distributional properties

We assume that  under the risk-neutral probability measure, the underlying asset price follows a geometric Brownian motion process:
$$S_t = S_0 e^{\sigma B_t + (r-q-\sigma^2 /2)t}$$

Let $\mathscr{A}_t$ be the continuously sampled arithmetic average price:
$$\mathscr{A}_t=\frac{1}{t}\int_0^t S_u du$$

From the general form of option pricing formula $e^{-rT}\mathbb{E}[(K-\mathscr{A}_T)^+]$, make the following derivation:
$$\left(K-\frac{1}{T}\int^T_0S_udu\right)^+=\frac{4S_0}{\sigma^2T}\left(k-\frac{\sigma^2}{4S_0}\int^{\tau}_0S_0e^{\sigma B_u+(r-q-\sigma^2/2)u}d\left(\frac{\sigma^2 u}{4}\right)\right)^+\\
=\frac{4S_0}{\sigma^2T}\left(k-\int^{\tau}_0e^{2(B_t+\nu t)}dt\right)$$

Define *Brownian exponential functional* $A_{\tau}^{(\nu)}$:
$$A_{\tau}^{(\nu)}:=\int^{\tau}_0e^{2(B_t+\nu t)}dt$$

From the invariance by time reversal of the distribution of a Levy process, the following process $X_t$ has an identity in law, $X_t \overset{(law)}{=}A_{t}^{(\nu)} $:
$$X_t = \int_0^T \exp{\left(2(B_t-B_u)+2\nu (t-u)\right)}du, X_0=0$$

From Ito's fomula, we express the SDE of $X_t$ as:
$$dX_t = \left[2(\nu+1)X_t+1\right]+2X_tdB_t$$
This $X_t$ is a one-dimensional diffusion process on $[0,\infty)$ started at $X_0 = 0$ and with the infinitesimal diffusion $a(x)=2x$, infinitesimal drift $b(x)=2(\nu+1)x+1$, and infinitesimal generator
$$
\begin{align}
(\mathscr{G}f)(x)&=\frac{1}{2}a^2(x)f''(x)+b(x)f'(x)\\&=2x^2f''(x)+[2(\nu+1)+1]f'(x)
\end{align}
$$
This diffusion has scale and speed densities:
$$\mathscr{s}(x)=\exp{\left(-\int\frac{2b(x)}{a^2(x)}dx\right)}=x^{-\nu-1}e^{1/2x}$$
$$\mathscr{m}(x)=\frac{2}{a^2(x)\mathscr{s}(x)}=\frac{1}{2}x^{\nu-1}e^{-1/2x}$$

In addition, this paper also consider *up-and-out puts* on the diffusion X. For $b>k$, consider functions
$$P^{(\nu)}(k,\tau):=\mathbb{E}\left[(k-X_t)^+\right],P_b^{(\nu)}(k,\tau):=\mathbb{E}\left[1_{\{\mathscr{T}_b>\tau\}}(k-X_\tau)^+\right]$$
where $\mathscr{T}_b:=inf{t\ge0:X_t=b}$ is the first hitting time of $b$ and $1_{\{\mathscr{T}_b>\tau\}}=1$ if $\mathscr{T}_b>\tau$. Their strategy is to first calculate the function $P_b^{(\nu)}(k,\tau)$ and then take the limit $b\uparrow\infty$ to recover the function $P^{(\nu)}(k,\tau)$

## Whittaker functions

The Whittaker functions are solutions to the ODE
$$\frac{d^2w}{dz^2}+\left(-\frac{1}{4}+\frac{\kappa}{z}+\frac{1/4-\mu^2}{z^2}\right)w=0$$

There are two linearly independent solutions to the ODE:
$$M_{\kappa,\mu}=\exp{(-z/2)}z^{\mu+1/2}M\left(\mu-\kappa+\frac{1}{2},1+2\mu;z\right),$$
$$W_{\kappa,\mu}=\exp{(-z/2)}z^{\mu+1/2}U\left(\mu-\kappa+\frac{1}{2},1+2\mu;z\right)$$
where
$$M(a,b,z)=\sum_{k=0}^\infty\frac{(a)_k}{(b)_k}\frac{z^k}{k!}$$
$$U(a,b,z)=\frac{\pi}{\sin{b\pi}}\left(\frac{M(a,b,z)}{\Gamma(1+a-b)\Gamma(b)}-z^{1-b}\frac{M(1+a-b,2-b,z)}{\Gamma(a)\Gamma(2-b)}\right)$$

## Up-and-out puts price derivation

From the last section, $P_b^{(\nu)}(k,\tau)$ is what we care the most, we need the following integral to calculate:
$$P_b^{(\nu)}(k,\tau)=\int_0^k (k-y)p_b(\tau;0,y)\mathscr{m}(y)dy$$
where $p_b(t;x,y)$ is the transition probability density w.r.t. the speed measure $\mathscr{m}(y)dy$ of the infinitesimal generator $(\mathscr{G}f)(x)$ started at $x\in[0,b)$ and killed at $b$.

This prob density admit a spectral representation
$$p_b(t;x,y)=\sum_{n=1}^\infty e^{-\lambda_n \tau}\varphi_n(x)\varphi_n(y)$$
where $\{\lambda_n,\varphi_n\}_{n=1}^\infty$ are eigenvalues and (normalized) eigenfunctions of the Strum-Liouville problem:
$$-(\mathscr{G}u)(x)\equiv \lambda u(x)$$
with the boundary conditions at 0 and $b$:
$$\underset{x\downarrow0}{lim}\frac{u'(x)}{\mathscr{s}(x)}=0, u(b)=0$$

This linear ODE turns out to be the solution of *Whittaker's form of the confluent hypergeometric equation*, with the solution:
$$\psi(x,\lambda)=(2x)^{(1-\nu)/2}e^{1/4x}W_{(1-\nu)/2,(1/2)\sqrt{\nu^2-2\lambda}}\left(\frac{1}{2x}\right)$$

From Theorem 4.1 in Kent (1980), for fixed $x > 0$ the zeros of $\psi(x,\lambda)$ are simple and positive, it can be written as a canonical product
$$\psi(x,\lambda)=\prod_{n=1}^\infty\left(1-\frac{\lambda}{\lambda_{n,x}}\right)$$
by finding root $\psi(x,\lambda)=0$, we solve the eigenvalues $\lambda_n=\lambda_{n,b}$ as the zeros of $\psi(b,\lambda)$. Consider the two cases: $0<\lambda\le\nu^2/2$ and $\lambda>\nu^2/2$

### Approximation for $0<\lambda\le\nu^2/2$

By setting $\lambda = (\nu^2-q^2)/2$, we are equivalently solving $q\in[0,|\nu|)$. To get precise numerical values of $q_{n,b}$, we need to find the roots of $W_{(1-\nu)/2,q/2}\left(\frac{1}{2x}\right)$ numerically. However, for large $b$ we can get estimates by using the Whittaker function
asymptotics for $\mu>0$ and $z > 0$:
$$W_{\kappa,\mu}(z)\sim\frac{\Gamma(2\mu)}{\Gamma(1/2+\mu-\kappa)}z^{1/2-\mu}e^{-z/2}$$

This gives
$$\psi(b,(\nu^2-q^2)/2)\sim\frac{\Gamma(q)}{\Gamma((\nu+q)/2)}\left(\frac{1}{2b}\right)^{(\nu-q)/2}$$

For $\nu<0$,  the reciprocal of the gamma function $1/\Gamma((\nu+q)/2)$ has zeros $(\nu+q)/2=-n+1, n = 1,...,[|\nu|/2+1]$. Thus, for $\nu < 0$ and *large enough* b, the total number of zeros in $[0,|\nu|)$ is equal to $N_\nu(b)=[|\nu|/2+1]$ and
$$q_{n,b}=(|\nu|-2n+2), n=1,...,[|\nu|/2+1]$$

For $\nu>0$, $N_\nu(b)\equiv0, \forall b>0$

Then the non-normalized eigenfunctions are:
$$\psi(x,\lambda_{n,b})=(2x)^{(1-\nu)/2}e^{1/4x}W_{(1-\nu)/2,q_{n,b}/2}\left(\frac{1}{2x}\right)$$

### Approximation for $\lambda>\nu^2/2$

Setting $\lambda = (\nu^2+p^2)/2$. We can obtain estimates that improve with increasing n by using the following estimate of the Whittaker function with purely imaginary second index $\mu=i\rho, \rho >0, \kappa\in\mathbb{R}$ and fixed $z>0$:
$$W_{\kappa,i\rho}=\sqrt{2z}e^{\pi\rho/2}\rho^{\kappa-1/2}\cos{\left(\rho\ln{\left(\frac{z}{4\rho}\right)}+\rho-\left(\kappa-\frac{1}{2}\right)\frac{\pi}{2}\right)}\times[1+O(\rho^{-1})]$$

In this case, the root is related to the cosine function, let
$$\frac{p}{2}\left(-\ln{(4bp)}+1\right)+\frac{1}{4}\pi\nu=\frac{\pi}{2}+n\pi $$
then the solution of following equation gives the estimate of $p_{n,b}$
$$p_{n,b}[\ln{(4bp_{n,b})}-1]=2\pi\left(n+\frac{\nu}{4}-\frac{1}{2}\right)$$

Then the non-normalized eigenfunctions are:
$$\psi(x,\lambda_{N_\nu(b)+n,b})=(2x)^{(1-\nu)/2}e^{1/4x}W_{(1-\nu)/2,ip_{n,b}/2}\left(\frac{1}{2x}\right)$$

### Normalization of the eigenfunctions

The normalizing factor is related to the other linearly independent solution to the ODE. By calculating the Wronskian, define
$$\eta_{n,b}^{(\nu)}:=\frac{\partial}{\partial p}\left.W_{(1-\nu)/2,ip_{n,b}/2}\left(\frac{1}{2b}\right)\right|_{p=p_{n,b}}$$
$$\xi_{n,b}^{(\nu)}:=-\frac{\partial}{\partial q}\left.W_{(1-\nu)/2,q_{n,b}/2}\left(\frac{1}{2b}\right)\right|_{q=q_{n,b}}$$

The norms of the non-normalized eigenfunctions are:

For $\lambda\le\nu^2/2,\nu<0$ and $N_\nu>0$, the norms are given by:
$$\frac{1}{||\psi(\bullet,\lambda_{n,b})||^2}=2^\nu q_{n,b}\frac{\Gamma(\tfrac{1}{2}(\nu+q_{n,b}))}{\eta_{n,b}^{(\nu)}\Gamma(1+q_{n,b})}M_{(1-\nu)/2,q_{n,b}/2}\left(\frac{1}{2b}\right), n=1,...,N_\nu(b)$$

For $\lambda\in(\nu^2/2,\infty)$, the norms are given by:
$$\frac{1}{||\psi(\bullet,\lambda_{N_\nu(b)+n,b})||^2}=2^\nu p_{n,b}\frac{\Gamma(\tfrac{1}{2}(\nu+ip_{n,b}))}{\xi_{n,b}^{(\nu)}\Gamma(1+ip_{n,b})}M_{(1-\nu)/2,ip_{n,b}/2}\left(\frac{1}{2b}\right),n=1,2,...$$

### Spectral representation for $P_b^{(\nu)}(k,\tau)$

The transition probability density takes the form
$$p_b(t;x,y)=\sum_{n=1}^\infty e^{-t\lambda_{n,b}}\frac{\psi(x,\lambda_{n,b})\psi(y,\lambda_{n,b})}{||\psi(\bullet,\lambda_{n,b})||^2}$$

Define $c(\lambda)$ as
$$
\begin{align}
c(\lambda)&=\int_0^k(k-y)\psi(y,\lambda)\mathscr{m}(y)dy\\
&=2^{-(1+\nu)/2}k^{(\nu+3)/2}e^{-1/4k}W_{-(\nu+3)/2,\sqrt(\nu^2-2\lambda)/2}\left(\frac{1}{2k}\right)
\end{align}$$

Then the transformed payoff $P_b^{(\nu)}(k,\tau)$ have the spectral representation:
$$P_b^{(\nu)}(k,\tau)=\sum_{n=1}^\infty e^{-\tau\lambda_{n,b}}\frac{c(\lambda_{n,b})}{||\psi(\bullet,\lambda_{n,b})||^2}$$

## Proposition 1

The function $P_b^{(\nu)}(k,\tau)$ is given by the following series:
$$\begin{align}
P_b^{(\nu)}(k,\tau)\\
=&\sum_{n=1}^\infty e^{-\tau(\nu^2+p^2_{n,b})/2}p_{n,b}\frac{\Gamma(\tfrac{1}{2}(\nu+ip_{n,b}))}{4\xi_{n,b}^{(\nu)}\Gamma(1+ip_{n,b})}\\
&\times(2k)^{(\nu+3)/2}e^{-1/4k}W_{-(\nu+3)/2,(ip_{n,b})/2}\left(\frac{1}{2k}\right)M_{(1-\nu)/2,ip_{n,b}/2}\left(\frac{1}{2b}\right)\\
&+\sum_{n=1}^{N_\nu(b)} e^{-\tau(\nu^2-q^2_{n,b})/2}q_{n,b}\frac{\Gamma(\tfrac{1}{2}(\nu+q_{n,b}))}{4\eta_{n,b}^{(\nu)}\Gamma(1+q_{n,b})}\\
&\times(2k)^{(\nu+3)/2}e^{-1/4k}W_{-(\nu+3)/2,q_{n,b}/2}\left(\frac{1}{2k}\right)M_{(1-\nu)/2,q_{n,b}/2}\left(\frac{1}{2b}\right)
\end{align}$$

```python

```