```python
import math
import numpy as np;
import matplotlib.pyplot as plt
```

```python
def inf_n(z, a):
    return 1-(9*a)/(8*z)+math.pow(a,3)/(2*math.pow(z,3))-math.pow(a,5)/(8*math.pow(z,5))

def inf_t(z, a):
    return 1-(9*a)/(16*z)+2*math.pow(a,3)/(16*math.pow(z,3))-math.pow(a,5)/(16*math.pow(z,5))

def channel_n(z,a,L):
    dom = 12
    term1 = 0
    term2 = 0

    for n in range(0,dom):
        term1 = term1 + math.pow(-1,n)*(1/inf_n(n*L+z,a) -1)

    for n in range(0,dom):
        term2 = term2 + math.pow(-1,n)*(1/inf_n((n+1)*L-z,a) -1)

    return 1/(term1+term2+1)

channel_n_vec = np.vectorize(channel_n)

def channel_t(z,a,L):
    dom = 12
    term1 = 0
    term2 = 0

    for n in range(0,dom):
        term1 = term1 + math.pow(-1,n)*(1/(inf_t(z,a)*(n*L+a)) -1)

    for n in range(0,dom):
        term2 = term2 + math.pow(-1,n)*(1/(inf_t(z,a)*((n+1)*L-a)) -1)

    return 1/(term1+term2+1)

channel_t_vec = np.vectorize(channel_t)

def fit_wall_mob(h, p1, p2, p3, p4, p5, p6, p7):
    return p1 + p2/pow(p3  + h,5) - p4/pow(p5   + h,3) - p6/(p7  + h)

def inf_n(z, a):
    return 1-(9*a)/(8*z)+math.pow(a,3)/(2*math.pow(z,3))-math.pow(a,5)/(8*math.pow(z,5))

def inf_t(z, a):
    return 1-(9*a)/(16*z)+math.pow(a,3)/(8*math.pow(z,3))-math.pow(a,5)/(16*math.pow(z,5))

def channel_n(z,a,L):
    dom = 8
    term1 = 0
    term2 = 0
    for n in range(0,dom):
        term1 = term1 + math.pow(-1,n)*(1/inf_n(n*L+z,a) -1)
    for n in range(0,dom):
        term2 = term2 + math.pow(-1,n)*(1/inf_n((n+1)*L-z,a) -1)
    return 1/(term1+term2+1)

def channel_t(z,a,L):
    dom = 8
    term1 = 0
    term2 = 0
    for n in range(0,dom):
        term1 = term1 + math.pow(-1,n)*(1/inf_t(n*L+z,a) -1)
    for n in range(0,dom):
        term2 = term2 + math.pow(-1,n)*(1/inf_t((n+1)*L-z,a) -1)
    return 1/(term1+term2+1)

channel_n_vec = np.vectorize(channel_n)
channel_t_vec = np.vectorize(channel_t)
```

```python
L=19.125
a=1
x=np.linspace(1,10,50)
y=channel_n_vec(x,a,L)

plt.plot(x,y)
```

```python
L=9.56
x=np.linspace(1,4,50)
y=channel_t_vec(x,1,L)

plt.plot(x,y)
```

```python
L=3e-7
totalRad=1.56875e-08
wetRad=3.1375e-08
dryRad=3.1375e-08
x=0.5e-7
totalm=channel_n_vec(x,totalRad,L)
wetm=channel_n_vec(x,wetRad,L)
drym = (totalm/totalRad - wetm/wetRad)*dryRad

print(totalm)
print(wetm)
print(drym)

```

    0.6570385461720548
    0.40081817918271434
    0.9132589131613952

```python

```

```python
from scipy.optimize import curve_fit

def fit_wall_mob(h,p1, p2, p3, p4, p5, p6, p7):
    return p1 + p2/pow(p3  + h,5) - p4/pow(p5   + h,3) - p6/(p7  + h)

L=3e-7
totalRad=1.56875e-08
xpos=np.array([0.5*L/totalRad,0.25*L/totalRad,0.125*L/totalRad,0.0625*L/totalRad,0.03125*L/totalRad,0.015625*L/totalRad,0.0078125*L/totalRad,0])

wetRad=3.1375e-08
xpos50=np.array([0.5*L/wetRad,0.25*L/wetRad,0.125*L/wetRad,0.0625*L/wetRad,0.03125*L/wetRad,0.015625*L/wetRad,0.0078125*L/wetRad,0])

tmob=np.array([0.8955,0.8642,0.7653,0.5755,0.2764,0.07962,0.01990,0])
nmob=np.array([0.8479,0.7659,0.5708,0.3141,0.1232,0.03737,0.01060,0])

#tmob50=np.array([0.8981,0.8605,0.7711,0.5709,0.2795,0.07817,0.02066,0])
#nmob50=np.array([0.8480,0.7658,0.5710,0.3140,0.1232,0.03737,0.01071,0])

tmob50=np.array([0.8041,0.7418,0.5674,0.2741,0.07903,0.01976,0.004939,0])
nmob50=np.array([0.7040,0.5645,0.3134,0.1231,0.03734,0.01059,0.002872,0])

poptt, pcovt = curve_fit(fit_wall_mob, xpos, tmob, p0=[1,1,1,16,4,1,1],maxfev=10000000)
poptn, pcovn = curve_fit(fit_wall_mob, xpos, nmob, p0=[1,1,1,16,4,1,1],maxfev=10000000)

poptt50, pcovt50 = curve_fit(fit_wall_mob, xpos50, tmob50, p0=[1,1,1,16,4,1,1],maxfev=10000000)
poptn50, pcovn50 = curve_fit(fit_wall_mob, xpos50, nmob50, p0=[1,1,1,16,4,1,1],maxfev=10000000)
```

```python
poptn50
```

    array([-3.11038612e+01,  7.24698118e+02,  2.59720646e+00,  4.13848704e+01,
            1.79959078e+00, -3.20502020e+04,  9.99299583e+02])

```python
plt.rcParams['mathtext.fontset'] = 'cm'
#plt.rc('text', usetex=True)
plt.rc('xtick', labelsize=13)
plt.rc('ytick', labelsize=13)
plt.rc('axes', linewidth=1.5)
fig = plt.figure(figsize=[21,10])
plt.subplots_adjust(wspace=0.20)

#xf1=np.linspace(0,2,50)
#f0=fit_wall_mob(xf1,*poptt15r)
#f1=fit_wall_mob(xf1,*poptt3)
#f2=fit_wall_mob(xf1,*poptt6)

#f3=fit_wall_mob(xf1,*poptn15)
#f4=fit_wall_mob(xf1,*poptn3)
#f5=fit_wall_mob(xf1,*poptn6)

ax1 = fig.add_subplot(1,2,1)
ax1.set_yscale('log')
#ax1.plot(xf1,f0,color='black', linewidth=2, label='1.5nm fit')
#ax1.plot(xf1,f2,color='red', linestyle='dashed',linewidth=3.5, label='6nm fit')
ax1.scatter(xpos,tmob,s=150,color='black',facecolors='none',marker='o', linewidths=2, label='100\% wet')
ax1.scatter(xpos50,tmob50,s=150,color='red',marker='x', linewidths=3, label='50\% wet')
ax1.set_xlabel(r'$\tilde{y}/a_t$', fontsize=30)
ax1.set_ylabel(r'$\gamma_{\parallel}$', fontsize=30,rotation=0, labelpad=15)
ax1.set_ylim([0.009,1.05])
ax1.tick_params(labelsize=20)
ax1.legend(loc=(0.65,0.04),fontsize=20)
ax1.xaxis.set_tick_params(width=1.5)
ax1.yaxis.set_tick_params(width=1.5)

ax2 = fig.add_subplot(1,2,2)
ax2.set_yscale('log')
#ax2.plot(xf1,f3,color='black', linewidth=2, label='1.5nm fit')
#ax2.plot(xf1,f5,color='red', linestyle='dashed',linewidth=3.5, label='6nm fit')
ax2.scatter(xpos,nmob,s=150,color='black',facecolors='none',marker='o', linewidths=2, label='100\% wet')
ax2.scatter(xpos50,nmob50,s=150,color='red',marker='x', linewidths=3, label='50\% wet')
ax2.set_xlabel(r'$\tilde{y}/a_t$', fontsize=30)
ax2.set_ylabel(r'$\gamma_{\bot}$', fontsize=30,rotation=0, labelpad=15)
ax2.set_ylim([0.005,1.05])
ax2.tick_params(labelsize=20)
ax2.legend(loc=(0.65,0.04),fontsize=20)
ax2.xaxis.set_tick_params(width=1.5)
ax2.yaxis.set_tick_params(width=1.5)
fig.savefig("mob3.pdf", bbox_inches = 'tight',
    pad_inches = 0.05)
```

```python
#1.5nm
L=1.5e-7
wetRad=1.56875e-08
xpos15r=np.array([0.5*L/wetRad,0.25*L/wetRad,0.125*L/wetRad,0.0625*L/wetRad,0.03125*L/wetRad,0.015625*L/wetRad,0.0078125*L/wetRad,0])
tmob15r=np.array([0.7935,0.7358,0.5653,0.2734,0.07888,0.01972,0.004930,0])
nmob15r=np.array([0.7040,0.5645,0.3134,0.1231,0.03734,0.01059,0.002872,0])

poptt15r, pcovt15r = curve_fit(fit_wall_mob, xpos15r, tmob15r, p0=[1,1,1,16,4,1,1],maxfev=10000000)
poptn15r, pcovn15r = curve_fit(fit_wall_mob, xpos15r, nmob15r, p0=[1,1,1,16,4,1,1],maxfev=10000000)

xpos15=np.array([2,1,0.5,0.25,0.125,0.0635,0])
tmob15=np.array([0.7081,0.4923,0.2100 ,0.05522,0.01381 ,0.003563,0])
nmob15=np.array([0.4986,0.2580,0.09141,0.02714,0.007602,0.002104,0])

poptt15, pcovt15 = curve_fit(fit_wall_mob, xpos15, tmob15, p0=[1,1,1,16,4,1,1],maxfev=10000000)
poptn15, pcovn15 = curve_fit(fit_wall_mob, xpos15, nmob15, p0=[1,1,1,16,4,1,1],maxfev=10000000)

#3nm
xpos=np.array([2,1,0.5,0.25,0.125,0.0635,0])
tmob3=np.array([0.7310,0.4997,0.2120 ,0.05574,0.01393 ,0.003596,0])
nmob3=np.array([0.5022,0.2584,0.09148,0.02716,0.007605,0.002105,0])

poptt3, pcovt3 = curve_fit(fit_wall_mob, xpos, tmob3, p0=[1,1,1,16,4,1,1],maxfev=10000000)
poptn3, pcovn3 = curve_fit(fit_wall_mob, xpos, nmob3, p0=[1,1,1,16,4,1,1],maxfev=10000000)

#6nm
xpos=np.array([2,1,0.5,0.25,0.125,0.0635,0])
tmob6=np.array([0.7349,0.5008,0.2123 ,0.05581,0.01395 ,0.003601,0])
nmob6=np.array([0.5023,0.2584,0.09149,0.02716,0.007605,0.002105,0])

poptt6, pcovt6 = curve_fit(fit_wall_mob, xpos, tmob6, p0=[1,1,1,16,4,1,1],maxfev=10000000)
poptn6, pcovn6 = curve_fit(fit_wall_mob, xpos, nmob6, p0=[1,1,1,16,4,1,1],maxfev=10000000)
```

    /usr/local/lib/python3.8/dist-packages/scipy/optimize/minpack.py:833: OptimizeWarning: Covariance of the parameters could not be estimated
      warnings.warn('Covariance of the parameters could not be estimated',

```python
print(poptn15r)
```

    [-3.11038612e+01  7.24698118e+02  2.59720646e+00  4.13848704e+01
      1.79959078e+00 -3.20502020e+04  9.99299583e+02]

```python
plt.rcParams['mathtext.fontset'] = 'cm'
#plt.rc('text', usetex=True)
plt.rc('xtick', labelsize=13)
plt.rc('ytick', labelsize=13)
plt.rc('axes', linewidth=1.5)
fig = plt.figure(figsize=[21,10])
plt.subplots_adjust(wspace=0.20)

xf1=np.linspace(0,2,50)
f0=fit_wall_mob(xf1,*poptt15r)
f1=fit_wall_mob(xf1,*poptt3)
f2=fit_wall_mob(xf1,*poptt6)

f3=fit_wall_mob(xf1,*poptn15)
f4=fit_wall_mob(xf1,*poptn3)
f5=fit_wall_mob(xf1,*poptn6)

ax1 = fig.add_subplot(1,2,1)
ax1.set_yscale('log')
ax1.plot(xf1,f0,color='black', linewidth=2, label='1.5nm fit')
ax1.plot(xf1,f2,color='red', linestyle='dashed',linewidth=3.5, label='6nm fit')
ax1.scatter(xpos15,tmob15,s=150,color='black',facecolors='none',marker='o', linewidths=2, label='1.5nm data')
ax1.scatter(xpos,tmob6,s=150,color='red',marker='x', linewidths=3, label='6nm data')
ax1.set_xlabel(r'$\tilde{y}/a_t = \tilde{y}/a_w$', fontsize=30)
ax1.set_ylabel(r'$\gamma_{\parallel}\quad$', fontsize=30,rotation=0, labelpad=15)
ax1.set_ylim([0.009,1.05])
ax1.tick_params(labelsize=20)
ax1.legend(loc=(0.65,0.04),fontsize=20)
ax1.xaxis.set_tick_params(width=1.5)
ax1.yaxis.set_tick_params(width=1.5)

ax2 = fig.add_subplot(1,2,2)
ax2.set_yscale('log')
ax2.plot(xf1,f3,color='black', linewidth=2, label='1.5nm fit')
ax2.plot(xf1,f5,color='red', linestyle='dashed',linewidth=3.5, label='6nm fit')
ax2.scatter(xpos15,nmob15,s=150,color='black',facecolors='none',marker='o', linewidths=2, label='1.5nm data')
ax2.scatter(xpos,nmob6,s=150,color='red',marker='x', linewidths=3, label='6nm data')
ax2.set_xlabel(r'$\tilde{y}/a_t = \tilde{y}/a_w$', fontsize=30)
ax2.set_ylabel(r'$\gamma_{\bot}\quad$', fontsize=30,rotation=0, labelpad=15)
ax2.set_ylim([0.005,1.05])
ax2.tick_params(labelsize=20)
ax2.legend(loc=(0.65,0.04),fontsize=20)
ax2.xaxis.set_tick_params(width=1.5)
ax2.yaxis.set_tick_params(width=1.5)
fig.savefig("mob2.pdf", bbox_inches = 'tight',
    pad_inches = 0.05)
```

```python
fit_wall_mob(0,*poptt)
```

    -0.0007420656601707654

```python

plt.rcParams['mathtext.fontset'] = 'cm'
#plt.rc('text', usetex=True)
plt.rc('xtick', labelsize=13)
plt.rc('ytick', labelsize=13)
plt.rc('axes', linewidth=1.5)
fig = plt.figure(figsize=[21,10])
plt.subplots_adjust(wspace=0.20)

L=19.125
a=1

x1=np.linspace(1,9.56,50)
y1=channel_t_vec(x1,a,L)

xf1=np.linspace(0,9.56,1000000)
f1=fit_wall_mob(xf1,*poptt)

x2=np.linspace(1,9.56,50)
y2=channel_n_vec(x2,a,L)

xf2=np.linspace(0,9.56,100000)
f2=fit_wall_mob(xf2,*poptn)

ax1 = fig.add_subplot(1,2,1)
ax1.set_yscale('log')
ax1.plot(x1,y1,color='red', linestyle='dashed',linewidth=3.5, label='Theory')
ax1.plot(xf1,f1,color='black', linewidth=2, label='Fit')
ax1.scatter(xpos,tmob,s=150,color='black',facecolors='none',marker='o', linewidths=2, label='Measured')
ax1.set_xlabel(r'$\tilde{y}/a_t = \tilde{y}/a_w$', fontsize=30)
ax1.set_ylabel(r'$\gamma_{\parallel}$', fontsize=30,rotation=0, labelpad=15)
ax1.set_ylim([0.018,1.05])
ax1.tick_params(labelsize=20)
ax1.legend(loc=(0.65,0.04),fontsize=20)
ax1.xaxis.set_tick_params(width=1.5)
ax1.yaxis.set_tick_params(width=1.5)

ax2 = fig.add_subplot(1,2,2)
ax2.set_yscale('log')
ax2.plot(x2,y2,color='red', linestyle='dashed',linewidth=3.5, label='Theory')
ax2.plot(xf2,f2,color='black', linewidth=2, label='Fit')
ax2.scatter(xpos,nmob,s=150,color='black',facecolors='none',marker='o', linewidths=2, label='Measured')
ax2.set_xlabel(r'$\tilde{y}/a_t = \tilde{y}/a_w$', fontsize=30)
ax2.set_ylabel(r'$\gamma_{\bot}$', fontsize=30,rotation=0, labelpad=20)
ax2.set_ylim([0.009,1.05])
ax2.tick_params(labelsize=20)
ax2.legend(loc=(0.65,0.04),fontsize=20)
ax2.xaxis.set_tick_params(width=1.5)
ax2.yaxis.set_tick_params(width=1.5)
fig.savefig("mob1.pdf", bbox_inches = 'tight',
    pad_inches = 0.05)
```

```python
kB=1.38064852e-16
eta=1e-2
T=300
totalDiff_small=1.331e-05
dx=3e-07/24
wetRad=dx*1.255 # pkernel=4
a=kB*T/(totalDiff_small*eta*math.pi*6.0)
print(3e-07/a)
```

    18.171720440165384

```python
fig.savefig("test.pdf", bbox_inches = 'tight',
    pad_inches = 0.05)
```

```python
from sympy import *
z = Symbol('z')
p1 = Symbol('p1')
p2 = Symbol('p2')
p3 = Symbol('p3')
p4 = Symbol('p4')
p5 = Symbol('p5')
p6 = Symbol('p6')
p7 = Symbol('p7')
aa = Symbol('aa')

diff(p1 + p2/pow(p3  + z/aa,5) - p4/pow(p5   + z/aa,3) - p6/(p7  + z/aa), z)
```

$\displaystyle - \frac{5 p_{2}}{aa \left(p_{3} + \frac{z}{aa}\right)^{6}} + \frac{3 p_{4}}{aa \left(p_{5} + \frac{z}{aa}\right)^{4}} + \frac{p_{6}}{aa \left(p_{7} + \frac{z}{aa}\right)^{2}}$

```python

```


# Motivación: Redes Neuronales Convolucionales

La información que extraemos de las entradas sensoriales a menudo está determinada por su contexto. Con las imágenes, podemos suponer que los píxeles cercanos están estrechamente relacionados y su información colectiva es más relevante cuando se toma como una unidad. Por el contrario, podemos suponer que los píxeles individuales no transmiten información relacionada entre sí. Por ejemplo, para reconocer letras o dígitos, necesitamos analizar la dependencia de píxeles cercanos, porque determinan la forma del elemento. De esta manera, podríamos calcular la diferencia entre, por ejemplo, un 0 o un 1. Los píxeles de una imagen están organizados en una cuadrícula bidimensional, y si la imagen no es en escala de grises, tendremos una tercera dimensión para Los mapas de colores. Alternativamente, una imagen de resonancia magnética (MRI) también usa espacio tridimensional. Puede recordar que, hasta ahora, si queríamos alimentar una imagen a una red neuronal, teníamos que cambiarla de una matriz bidimensional a una matriz unidimensional. Las CNN están diseñadas para abordar este problema: cómo hacer que la información perteneciente a las neuronas que están más cerca sea más relevante que la información proveniente de las neuronas que están más separadas. En problemas visuales, esto se traduce en hacer que las neuronas procesen información proveniente de píxeles que están cerca uno del otro. Con CNNs, podremos alimentar entradas de una, dos o tres dimensiones y la red producirá una salida de la misma dimensionalidad. Como veremos más adelante, esto nos dará varias ventajas

Cuando tratamos de clasificar las imágenes CIFAR-10 usando una red de capas completamente conectadas con poco éxito. Una de las razones es que se sobreajustan. Si miramos la primera capa oculta de esa red, que tiene 1.024 neuronas. El tamaño de entrada de la imagen es 32x32x3 = 3,072. Por lo tanto, la primera capa oculta tenía un total de 2072 * 1024 = 314, 5728 pesos. ¡Ese no es un número pequeño! No solo es fácil sobreajustar una red tan grande, sino que también es ineficiente en la memoria. Además, cada neurona de entrada (o píxel) está conectada a cada neurona en la capa oculta. Debido a esto, la red no puede aprovechar la proximidad espacial de los píxeles, ya que no tiene una manera de saber qué píxeles están cerca uno del otro. Por el contrario, las CNN tienen propiedades que proporcionan una solución efectiva a estos problemas:

- Conectan neuronas, que solo corresponden a píxeles vecinos de la imagen. De esta manera, las neuronas están "forzadas" a recibir información de otras neuronas que están espacialmente cercanas. Esto también reduce el número de pesos, ya que no todas las neuronas están interconectadas.

- Una CNN utiliza el uso compartido de parámetros. En otras palabras, se comparte un número limitado de pesos entre todas las neuronas de una capa. Esto reduce aún más la cantidad de pesas y ayuda a combatir el sobreajuste. Puede sonar confuso, pero quedará claro en la siguiente sección.

La capa convolucional es el bloque de construcción más importante de una CNN. Consiste en un conjunto de filtros (también conocidos como núcleos o detectores de características), donde cada filtro se aplica en todas las áreas de los datos de entrada. Un filtro se define por un conjunto de pesos aprendibles. Como un guiño al tema en cuestión, la siguiente imagen ilustra esto muy bien:

Se muestra una capa de entrada bidimensional de una red neuronal. Por el bien de la simplicidad, asumiremos que esta es la capa de entrada, pero puede ser cualquier capa de la red. Como hemos visto en los capítulos anteriores, cada neurona de entrada representa la intensidad de color de un píxel (asumiremos que es una imagen en escala de grises por simplicidad). Primero, aplicaremos un filtro 3x3 en la parte superior derecha esquina de la imagen. Cada neurona de entrada está asociada con un solo peso del filtro. Tiene nueve pesos, debido a las nueve neuronas de entrada, pero, en general, el tamaño es arbitrario (2x2, 4x4, 5x5, etc.). La salida del filtro es una suma ponderada de sus entradas (el activaciones de las neuronas de entrada). Su propósito es resaltar una característica específica en la entrada, por ejemplo, una arista o una línea. El grupo de neuronas cercanas, que participan en la entrada. se llaman el campo receptivo. En el contexto de la red, la salida del filtro representa el valor de activación de una neurona en la siguiente capa. La neurona estará activa, si la función es presente en esta ubicación espacial.

Para cada nueva neurona, deslizaremos el filtro por la imagen de entrada y calcularemos su salida (la suma ponderada) con cada nuevo conjunto de neuronas de entrada. En el siguiente diagrama, puede ver cómo calcular las activaciones de las siguientes dos posiciones (un píxel para derecho):

- Al decir "arrastrar", queremos decir que los pesos del filtro no cambian en la imagen. En efecto, utilizaremos los mismos nueve pesos de filtro para calcular las activaciones de todas las neuronas de salida, cada vez con un conjunto diferente de neuronas de entrada. Llamamos a este parámetro compartir, y lo hacemos por dos razones:

- Al reducir el número de pesos, reducimos la huella de la memoria y evitamos el sobreajuste. El filtro resalta características específicas. Podemos suponer que esta característica es útil, independientemente de su posición en la imagen. Al compartir pesos, garantizamos que el filtro podrá ubicar la función en toda la imagen.

Hasta ahora, hemos descrito la relación de corte uno a uno, donde la salida es un solo corte, que toma la entrada de otro segmento (o una imagen). Esto funciona bien en escala de grises, pero cómo ¿Lo adaptamos para imágenes en color (relación n a 1)? ¡Una vez más, es simple! Primero, dividiremos el imagen en canales de color. En el caso de RGB, serían tres. Podemos pensar en cada color canal como un segmento de profundidad, donde los valores son las intensidades de píxeles para el color dado (R, G, o B), como se muestra en el siguiente ejemplo:

La combinación de sectores se denomina volumen de entrada con una profundidad de 3. Un filtro único de 3x3 es
aplicado a cada rebanada. La activación de una neurona de salida es solo la suma ponderada de filtros aplicados en todos los sectores. En otras palabras, combinaremos los tres filtros en un gran 3 x 3 x 3 + 1 filtro con 28 pesos (agregamos profundidad y un solo sesgo). Entonces, calcularemos el suma ponderada aplicando los pesos relevantes a cada segmento.

    - Los mapas de características de entrada y salida tienen diferentes dimensiones. Digamos que tenemos una capa de entrada con tamaño (ancho, alto) y un filtro con dimensiones (filter_w, filter_h). Después de aplicar la convolución, las dimensiones de la capa de salida son (ancho - filtro_w + 1, altura - filtro_h + 1).

Como mencionamos, un filtro resalta una característica específica, como bordes o líneas. Pero, en general, muchas características son importantes y nos interesarán todas. ¿Cómo los destacamos a todos? Como de costumbre, es simple. Aplicaremos varios filtros en el conjunto de sectores de entrada. Cada filtro generará un segmento de salida único, que resalta la característica, detectada por el filtro (relación de n a m). Un sector de salida puede recibir información de:

- Todos los sectores de entrada, que es el estándar para capas convolucionales. En este escenario, un segmento de salida único es un caso de la relación n-a-1, que describimos anteriormente. Con múltiples segmentos de salida, la relación se convierte en n-m. En otras palabras, cada segmento de entrada contribuye a la salida de cada segmento de salida.

- Una sola porción de entrada. Esta operación se conoce como convolución profunda. Es un
tipo de reversión del caso anterior. En su forma más simple, aplicamos un filtro sobre un único segmento de entrada para producir un único segmento de salida. Este es un caso de la relación uno a uno, que describimos en la sección anterior. Pero también podemos especificar un multiplicador de canal (un entero m), donde aplicamos filtros m sobre un solo sector de salida para producir m sectores de salida. Este es un caso de relación de 1 a m. El número total de segmentos de salida es n * m.

Denotemos el ancho y la altura del filtro con Fw y Fh, la profundidad del volumen de entrada con D y la profundidad del volumen de salida con M. Luego, podemos calcular el número total de pesos W en una capa convolucional con el siguiente ecuación:
\begin{equation}
W=(D*F_w *F_h+1)*M
\end{equation}

Digamos que tenemos tres sectores y queremos aplicarles cuatro filtros de 5x5. Entonces la la capa convolucional tendrá un total de (3x5x5 + 1) * 4 = 304 pesos, y cuatro cortes de salida (volumen de salida con una profundidad de 4), un sesgo por corte. El filtro para cada segmento de salida tendrá tres parches de filtro de 5x5 para cada uno de los tres segmentos de entrada y un sesgo para un total de 3x5x5 + 1 = 76 pesos. La combinación de los mapas de salida se denomina volumen de salida con una profundidad de cuatro.

```python
import numpy as np
def conv(image, im_filter):
    """
    :param image: grayscale image as a 2-dimensional numpy array
    :param im_filter: 2-dimensional numpy array
    """
    # input dimensions
    height = image.shape[0]
    width = image.shape[1]
    # output image with reduced dimensions
    im_c = np.zeros((height - len(im_filter) + 1,width - len(im_filter) + 1))
    # iterate over all rows and columns
    for row in range(len(im_c)):
        for col in range(len(im_c[0])):
            # apply the filter
            for i in range(len(im_filter)):
                for j in range(len(im_filter[0])):
                    im_c[row, col] += image[row + i, col + j] *im_filter[i][j]
    # fix out-of-bounds values
    im_c[im_c > 255] = 255
    im_c[im_c < 0] = 0
    # plot images for comparison
    import matplotlib.pyplot as plt
    import matplotlib.cm as cm
    plt.figure()
    plt.imshow(image, cmap=cm.Greys_r)
    plt.show()
    plt.imshow(im_c, cmap=cm.Greys_r)
    plt.show()

import requests
from PIL import Image
from io import BytesIO
# Cargar la imagen
url ="https://upload.wikimedia.org/wikipedia/commons/thumb/8/88/Commander_Eileen_Collins_-_GPN-2000-001177.jpg/382px-Commander_Eileen_Collins_-_GPN-2000-001177.jpg?download"
resp = requests.get(url)
image_rgb =np.asarray(Image.open(BytesIO(resp.content)).convert("RGB"))
# Convertirla a escala de grises
image_grayscale = np.mean(image_rgb, axis=2, dtype=np.uint)
# Aplicar filtro de blur
blur = np.full([10, 10], 1. / 100)
conv(image_grayscale, blur)
sobel_x = [[-1, -2, -1],[0, 0, 0],
[1, 2, 1]]
conv(image_grayscale, sobel_x)
sobel_y = [[-1, 0, 1],
[-2, 0, 2],
[-1, 0, 1]]
conv(image_grayscale, sobel_y)
```

# Stride y relleno en capas convolucionales
Hasta ahora, asumimos que el deslizamiento del filtro ocurre un píxel a la vez, pero ese no es siempre el caso. Podemos deslizar el filtro en múltiples posiciones. Este parámetro de las capas convolucionales se llama zancada. Por lo general, el paso es el mismo en todas las dimensiones de la entrada. En el siguiente diagrama, podemos ver una capa convolucional con un paso de 2:

Al usar una zancada mayor que 1, reducimos el tamaño del segmento de salida. En la sección anterior, presentamos una fórmula simple para el tamaño de salida, que incluía los tamaños de la entrada y el núcleo. Ahora, lo ampliaremos para incluir también el paso: ((ancho - filtro_w) / stride_w + 1, ((altura - filtro_h) / stride_h + 1).
Por ejemplo, el tamaño de salida de un corte cuadrado generado por una imagen de entrada de 28x28, convolucionado con un filtro 3x3 con zancada 1, sería 28-3 + 1 = 26. Pero con zancada 2, obtenemos (28-3) / 2 + 1 = 13. El efecto principal del paso más grande es un aumento en el campo receptivo de las neuronas de salida. Vamos a explicar esto con un ejemplo. Si usamos Stride 2, el tamaño del segmento de salida será aproximadamente cuatro veces menor que el de entrada. En otras palabras, una neurona de salida "cubrirá" el área, que es cuatro veces más grande, en comparación con las neuronas de entrada. Las neuronas en el
Las siguientes capas capturarán gradualmente la entrada de regiones más grandes de la imagen de entrada. Esto es importante, porque les permitiría detectar características más grandes y más complejas de la entrada.

Las operaciones de convolución que hemos discutido hasta ahora han producido una salida menor que la entrada. Pero, en la práctica, a menudo es deseable controlar el tamaño de la salida. Podemos resolver esto rellenando los bordes del segmento de entrada con filas y columnas de ceros antes de la operación de convolución. La forma más común de usar relleno es producir resultados con las mismas dimensiones que la entrada. En el siguiente diagrama, podemos ver una capa convolucional con relleno de 1:

Las neuronas blancas representan el relleno. Los segmentos de entrada y salida tienen las mismas dimensiones (neuronas oscuras). Esta es la forma más común de usar relleno. Los ceros recién rellenados participarán en la operación de convolución con el corte, pero no afectarán el resultado. La razón es que, aunque las áreas rellenadas estén conectadas con pesos a la siguiente capa, siempre multiplicaremos esos pesos por el valor rellenado, que es 0. Ahora agregaremos relleno a la fórmula del tamaño de salida. Deje que el tamaño del segmento de entrada sea I = (Iw, Ih), el tamaño del filtro F = (Fw, Fh), la zancada S = (Sw, Sh) y el relleno P = (Pw, Ph). Entonces el tamaño O = (Ow, Oh) del segmento de salida viene dado por las siguientes ecuaciones:

\begin{equation}
O_w=\frac{I_w+2P_w-F_w}{S_w}+1
\end{equation}

\begin{equation}
O_h=\frac{I_h+2P_h-F_h}{S_h}+1
\end{equation}

# Capas de pooling
En la sección anterior, explicamos cómo aumentar el campo receptivo de las neuronas usando un paso más grande que 1. Pero también podemos hacer esto con la ayuda de la agrupación de capas. Una capa de agrupación divide la porción de entrada en una cuadrícula, donde cada celda de la cuadrícula representa un campo receptivo de varias neuronas (tal como lo hace una capa convolucional). Luego, se aplica una operación de agrupación sobre cada celda de la cuadrícula. Existen diferentes tipos de capas de agrupación. Las capas de agrupación no cambian la profundidad del volumen, porque la operación de agrupación se realiza de forma independiente en cada segmento.

- Max pooling: es la forma más popular de pooling. La operación de agrupación máxima lleva a la neurona con el valor de activación más alto en cada campo receptivo local (celda de cuadrícula) y propaga solo ese valor hacia adelante. En la siguiente figura, podemos ver un ejemplo de agrupación máxima con un campo receptivo de 2x2:

- Average Pooling: es otro tipo de agrupación, donde la salida de cada campo receptivo es el valor medio de todas las activaciones dentro del campo. El siguiente es un ejemplo de agrupación promedio

Las capas de agrupación se definen por dos parámetros:
- Stride, que es lo mismo que con las capas convolucionales
- Tamaño del campo receptivo, que es el equivalente del tamaño del filtro en capas convolucionales.

# Estructura de una red neuronal convolucional

Normalmente, alternaríamos una o más capas convolucionales con una capa de agrupación. De esta forma, las capas convolucionales pueden detectar características en cada nivel del tamaño del campo receptivo. El tamaño del campo receptivo agregado de las capas más profundas es mayor que las del comienzo de la red. Esto les permite capturar características más complejas de regiones de entrada más grandes. Vamos a ilustrar esto con un ejemplo. Imagine que la red utiliza convoluciones de 3x3 con zancada 1 y agrupación de 2x2 con zancada 2:

- las neuronas de la primera capa convolucional recibirán información de 3x3 píxeles de la imagen.
- Un grupo de neuronas de salida 2x2 de la primera capa tendrá un tamaño de campo receptivo combinado de 4x4 (debido a la zancada).
- Después de la primera operación de agrupación, este grupo se combinará en una sola neurona de la capa de agrupación.
- La segunda operación de convolución toma información de las neuronas de agrupación 3x3. Por lo tanto, recibirá la entrada de un cuadrado con lados 3x4 = 12 (o un total de 12x12 = 144) píxeles de la imagen de entrada.

Utilizamos las capas convolucionales para extraer características de la entrada. Las características detectadas por las capas más profundas son muy abstractas, pero tampoco son legibles por los humanos. Para resolver este problema, generalmente agregamos una o más capas completamente conectadas después de la última capa convolucional / agrupación. En este ejemplo, la última capa (salida) completamente conectada utilizará softmax para estimar las probabilidades de clase de la entrada. Puede pensar en las capas totalmente conectadas como traductores entre el idioma de la red (que no entendemos) y el nuestro. Las capas convolucionales más profundas generalmente tienen más filtros (por lo tanto, mayor profundidad de volumen), en comparación con los iniciales. Un detector de características al comienzo de la red funciona en un pequeño campo receptivo. Solo puede detectar un número limitado de características, como bordes o líneas, compartidas entre todas las clases. Por otro lado, una capa más profunda detectaría características más complejas y numerosas. Por ejemplo, si tenemos varias clases, como automóviles, árboles o personas, cada una tendrá su propio conjunto de características, como neumáticos, puertas, hojas y caras, etc. Esto requeriría más detectores de características.

```python

#Importen tensorflow, keras, de keras, importen Sequential, Dense, Activation,
#Convolution2D, MaxPooling, Flatten y np_utils

#Primero: Carguen datos de mnist
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.layers import Convolution2D, MaxPooling2D
from keras.layers import Flatten
from keras.utils import np_utils
(X_train, Y_train), (X_test, Y_test) = mnist.load_data()
X_train = X_train.reshape(60000, 28, 28, 1)
X_test = X_test.reshape(10000, 28, 28, 1)
Y_train = np_utils.to_categorical(Y_train, 10)
Y_test = np_utils.to_categorical(Y_test, 10)

#Segundo: creen una red neuronal convolucional

model = Sequential([Convolution2D(filters=64,kernel_size=(3, 3),input_shape=(28, 28, 1)),Activation('sigmoid'),Convolution2D(filters=32,kernel_size=(3, 3)), Activation('sigmoid'),MaxPooling2D(pool_size=(4, 4)),  Flatten(),  Dense(64),  Activation('relu'), Dense(10),  Activation('softmax')])
model.compile(loss='categorical_crossentropy',metrics=['accuracy'], optimizer='adadelta')

model.fit(X_train, Y_train, batch_size=100, epochs=5,validation_split=0.1, verbose=1)

score = model.evaluate(X_test, Y_test, verbose=1)
print('Test accuracy:', score[1])

```

```python

import warnings
warnings.filterwarnings("ignore")

```

# Preprocesamiento de datos
Hasta ahora, hemos alimentado la red con entradas no modificadas. En el caso de las imágenes, estas son intensidades de píxeles en el rango [0: 255]. Pero eso no es óptimo. Imagine que tenemos una imagen RGB, donde las intensidades en uno de los canales de color son muy altas en comparación con los otros dos. Cuando alimentamos la imagen a la red, los valores de este canal serán dominantes, disminuyendo los demás. Esto podría sesgar los resultados, porque en realidad cada canal tiene la misma importancia. Para resolver esto, necesitamos preparar (o normalizar) los datos, antes de alimentarlos a la red. En la práctica, usaremos dos tipos de normalización:

- Feature Scaling: Esta operación escala todas las entradas en el rango [0,1]. Por ejemplo, un píxel con intensidad 125 tendría un valor escalado de. El escalado de características es rápido y fácil de implementar.

- Standard Score: Aquí μ y σ son la media y la desviación estándar de todos los datos de entrenamiento. Por lo general, se calculan por separado para cada dimensión de entrada. Por ejemplo, en una imagen RGB, calcularíamos la media μ y σ para cada canal. Debemos tener en cuenta que μ y σ deben calcularse solo en los datos de entrenamiento y luego aplicarse a los datos de la prueba.

# Dropout

Dropout es una técnica de regularización, que se puede aplicar a la salida de algunas de las capas de red. El dropout aleatorio y periódico elimina algunas de las neuronas (junto con sus conexiones de entrada y salida) de la red. Durante un mini lote de entrenamiento, cada neurona tiene una probabilidad p de ser descartada estocásticamente. Esto es para asegurar que ninguna neurona termine confiando demasiado en otras neuronas y "aprenda" algo útil para la red. El abandono se puede aplicar después de capas convolucionales, de agrupación o completamente conectadas. En la siguiente ilustración, podemos ver un abandono de capas completamente conectadas:

# Aumento de datos

Una de las técnicas de regularización más eficientes es el aumento de datos. Si los datos de entrenamiento son demasiado pequeños, la red podría comenzar a sobreajustarse. El aumento de datos ayuda a contrarrestar esto al aumentar artificialmente el tamaño del conjunto de entrenamiento. Usemos un ejemplo. En los ejemplos de MNIST y CIFAR-10, hemos entrenado la red en varias épocas. La red "verá" cada muestra del conjunto de datos una vez por época. Para evitar esto, podemos aplicar aumentos aleatorios a las imágenes, antes de usarlas para el entrenamiento. Las etiquetas permanecerán igual. Algunos de los aumentos de imagen más populares son:

```python
import keras
from keras.datasets import cifar10
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Dense, Dropout, Activation, Flatten,BatchNormalization
from keras.models import Sequential
from keras.preprocessing.image import ImageDataGenerator

batch_size = 50
(X_train, Y_train), (X_test, Y_test) = cifar10.load_data()
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
Y_train = keras.utils.to_categorical(Y_train, 10)
Y_test = keras.utils.to_categorical(Y_test, 10)

data_generator = ImageDataGenerator(rotation_range=90, width_shift_range=0.1,height_shift_range=0.1, featurewise_center=True, featurewise_std_normalization=True, horizontal_flip=True)
data_generator.fit(X_train)
# standardize the test set
for i in range(len(X_test)):
    X_test[i] = data_generator.standardize(X_test[i])

model = Sequential()
model.add(Conv2D(32, (3, 3), padding='same',input_shape=X_train.shape[1:]))
model.add(BatchNormalization())
model.add(Activation('elu'))
model.add(Conv2D(32, (3, 3), padding='same'))
model.add(BatchNormalization())
model.add(Activation('elu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Dropout(0.2))
model.add(Conv2D(64, (3, 3), padding='same'))
model.add(BatchNormalization())
model.add(Activation('elu'))
model.add(Conv2D(64, (3, 3), padding='same'))
model.add(BatchNormalization())
model.add(Activation('elu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(Conv2D(128, (3, 3)))
model.add(BatchNormalization())
model.add(Activation('elu'))
model.add(Conv2D(128, (3, 3)))
model.add(BatchNormalization())
model.add(Activation('elu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(10, activation='softmax'))

model.compile(loss='categorical_crossentropy', optimizer='adam',
metrics=['accuracy'])

model.fit_generator( generator=data_generator.flow(x=X_train, y=Y_train, batch_size=batch_size),steps_per_epoch=len(X_train),epochs=100, validation_data=(X_test, Y_test),workers=4)
```

```python

```


## Developing A Theoretical Understanding on the Limitations of ML interpretation methods

### Linear Regression

Let's start with a simple 10-variable regression problem.

\begin{equation}
y = 2.0x_1 + 1.5x_2 + 1.2x_3 + 0.5x_4 + 0.2x_5 + x_6 + x_7 + x_8 + x_9 + x_{10} + \epsilon
\end{equation}

where $x_i = \mathcal{U}(-1,1)$ and $\epsilon=\mathcal{N}(\mu=0, \sigma=0.1)$. In the special case where $\epsilon = 0$ (or at least sufficiently small), the predictor ranking is based on the magnitude of the coefficients.

```python
import sys, os
current_dir = os.getcwd()
path = os.path.dirname(current_dir)
sys.path.append(path)
import mintpy
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
from sklearn.model_selection import train_test_split
```

```python
random_state=np.random.RandomState(42)
n_examples = 1000
n_vars = 5
weights = [2.0, 1.5, 1.2, 0.5, 0.2]
X = np.stack([random_state.uniform(0,1, size=n_examples) for _ in range(n_vars)], axis=-1)
feature_names = [f'X_{i+1}' for i in range(n_vars)]
X = pd.DataFrame(X, columns=feature_names)

error = np.random.normal(loc=0, scale=0.1, size=n_examples)
y = X.dot(weights)#+error
```

```python
X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=0, test_size=0.25)

lr = LinearRegression()
lr.fit(X_train, y_train)
print('Learned Coefficients: ', lr.coef_)
```

    Learned Coefficients:  [2.  1.5 1.2 0.5 0.2]

The coefficients approximate the true coefficients quite well given the error.

```python
myInterpreter = mintpy.InterpretToolkit(models=lr, model_names='Linear Regression', examples=X,targets=y,)
```

```python
results = myInterpreter.calc_ale(features=feature_names, n_bins=30)
myInterpreter.plot_ale(features='X_1')
```

```python
results['X_1__bin_values'].values
#[0.09082125, 0.27714732, 0.48378955, 0.6950751 , 0.89978646]
```

    array([0.04810705, 0.13429628, 0.2289725 , 0.32910934, 0.43682521,
           0.54333059, 0.63934867, 0.74412883, 0.84971491, 0.94964612])

```python
lr = LinearRegression()

print(np.shape(results['X_1__bin_values'].values), np.shape(results['X_1__Linear Regression__ale'][0]))

lr.fit(results['X_1__bin_values'].values.reshape(-1, 1), results['X_1__Linear Regression__ale'].values[0,:])
print(lr.coef_[0])
```

    (10,) (10,)
    1.999999999999998

```python
results = myInterpreter.calc_permutation_importance(
                                               n_vars=len(feature_names),
                                               evaluation_fn='mse',
                                               n_bootstrap=1000,
                                               subsample=1.0,
                                               n_jobs=10,
                                              )
```

```python
results['multipass_rankings__Linear Regression']
```

<svg style="position: absolute; width: 0; height: 0; overflow: hidden">
<defs>
<symbol id="icon-database" viewBox="0 0 32 32">
<path d="M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z"></path>
<path d="M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z"></path>
<path d="M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z"></path>
</symbol>
<symbol id="icon-file-text2" viewBox="0 0 32 32">
<path d="M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z"></path>
<path d="M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z"></path>
<path d="M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z"></path>
<path d="M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z"></path>
</symbol>
</defs>
</svg>

<xarray.DataArray 'multipass_rankings__Linear Regression' (n_vars_multipass: 5)>
array(['X_1', 'X_2', 'X_3', 'X_4', 'X_5'], dtype='<U3')
Dimensions without coordinates: n_vars_multipass

xarray.DataArray

'multipass_rankings__Linear Regression'

n_vars_multipass: 5

<input id='section-5eb583bc-0821-4724-84ce-764d9d3f737d' class='xr-array-in' type='checkbox' checked><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg>
'X_1' 'X_2' 'X_3' 'X_4' 'X_5'

array(['X_1', 'X_2', 'X_3', 'X_4', 'X_5'], dtype='<U3')

<input id='section-7ead4048-8a21-49cc-b304-5d519ab4e004' class='xr-section-summary-in' type='checkbox' disabled >Coordinates: (0)

<input id='section-670117a9-1245-4da7-9bd7-459456817369' class='xr-section-summary-in' type='checkbox' disabled >Attributes: (0)

```python
ale_var_results = myInterpreter.calc_ale_variance(model_name='Linear Regression')
```

```python
ale_var_results
```

<svg style="position: absolute; width: 0; height: 0; overflow: hidden">
<defs>
<symbol id="icon-database" viewBox="0 0 32 32">
<path d="M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z"></path>
<path d="M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z"></path>
<path d="M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z"></path>
</symbol>
<symbol id="icon-file-text2" viewBox="0 0 32 32">
<path d="M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z"></path>
<path d="M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z"></path>
<path d="M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z"></path>
<path d="M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z"></path>
</symbol>
</defs>
</svg>

<xarray.Dataset>
Dimensions:                                   (n_bootstrap: 1, n_vars_ale_variance: 5)
Dimensions without coordinates: n_bootstrap, n_vars_ale_variance
Data variables:
    ale_variance_rankings__Linear Regression  (n_vars_ale_variance) <U3 'X_1'...
    ale_variance_scores__Linear Regression    (n_vars_ale_variance, n_bootstrap) float64 ...
Attributes:
    model_output:      raw
    models used:       Linear Regression
    method:            ale_variance
    dimension:         1D
    n_multipass_vars:  5
    perm_method:       marginal
    evaluation_fn:     mse
    model output:      probability

xarray.Dataset

<input id='section-38a0190e-be5f-4874-b8e0-6dce5365bdfb' class='xr-section-summary-in' type='checkbox' disabled >Dimensions:

n_bootstrap: 1

n_vars_ale_variance: 5

<input id='section-323347d1-2de9-4b54-bfff-da9127425803' class='xr-section-summary-in' type='checkbox' disabled >Coordinates: (0)

<input id='section-8c712a2f-fe5e-49a3-8ce9-1bb55d4255fa' class='xr-section-summary-in' type='checkbox'  checked>Data variables: (2)

ale_variance_rankings__Linear Regression

(n_vars_ale_variance)

<U3

'X_1' 'X_2' 'X_3' 'X_4' 'X_5'
<input id='attrs-27fed55b-fbb4-45fd-9425-eff4aed5f89e' class='xr-var-attrs-in' type='checkbox' disabled><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg><input id='data-d89b02e2-1b75-4d3d-bf88-4cc058d2ee7b' class='xr-var-data-in' type='checkbox'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg>

array(['X_1', 'X_2', 'X_3', 'X_4', 'X_5'], dtype='<U3')

ale_variance_scores__Linear Regression

(n_vars_ale_variance, n_bootstrap)

float64

0.6143 0.4562 0.3655 0.1509 0.06001
<input id='attrs-1528a7d1-2a7f-44fe-825a-f683a3f171cd' class='xr-var-attrs-in' type='checkbox' disabled><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg><input id='data-ccca786f-e5b5-4ea0-aa88-3e90d7343fd3' class='xr-var-data-in' type='checkbox'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg>

array([[0.61428985],
       [0.45621367],
       [0.36549296],
       [0.15087281],
       [0.0600097 ]])

<input id='section-734e0c5c-4089-4538-8288-cdd988e03048' class='xr-section-summary-in' type='checkbox'  checked>Attributes: (8)

model_output :

raw

models used :

Linear Regression

method :

ale_variance

dimension :

1D

n_multipass_vars :

5

perm_method :

marginal

evaluation_fn :

mse

model output :

probability

```python
myInterpreter.plot_importance(method='singlepass')
```

Since the features are completely independent, the single-pass permutation method is good option for ranking the features. The results show the correct rankings:
* $X_1$, $X_2$, and $X_3$ are the most important
* $X_6-X_{10}$ have equivalent ranking (discerned by the CIs)
* $X_4-X_5$ are the least important

```python
myInterpreter.plot_importance(method='multipass')
```

# Increasing error

The following test includes increasing the error and seeing if the rankings are altered.

```python
scales = np.arange(0.1, 1.7, 0.2)

ranked_features=[]
for s in scales:
    error = np.random.normal(loc=0, scale=s, size=n_examples)
    y = X.dot(weights)+error
    X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=0, test_size=0.25)

    lr = LinearRegression()
    lr.fit(X_train, y_train)
    results = myInterpreter.calc_permutation_importance(
                                               n_vars=len(feature_names),
                                               evaluation_fn='mse',
                                               n_bootstrap=100,
                                               subsample=1.0,
                                               n_jobs=10,
                                              )
    ranked_features.append(results['multipass_rankings__Linear Regression'].values)
```

```python
def get_avg_ranking(ranked_features, feature_names):
    """Compute the average rankings for a set of features"""

    rankings ={f:[] for f in feature_names}
    for rank in ranked_features:
        for f in feature_names:
            rankings[f].append(np.where(rank==f))

    for f in feature_names:
        rankings[f] = np.mean(rankings[f])

    return rankings

rankings = get_avg_ranking(ranked_features, feature_names)
```

```python
fig, ax = plt.subplots(dpi=150)
avg_rankings = [rankings[f] for f in feature_names]
ax.plot(np.arange(n_vars), avg_rankings, color='xkcd:medium blue', linewidth=2.5)
ax.set_xticks(np.arange(n_vars))
ax.set_xticklabels([f'${f}$' for f in feature_names])
ax.tick_params(labelsize=10)
ax.set_xlabel('Features')
ax.set_ylabel('Ranking (lower is better!)')
ax.grid(alpha=0.6)
```

## Introducing colinearity

Takeaways
*
*

```python
fig, ax = plt.subplots(dpi=150)

scales = np.arange(0.1, 1.0, 0.1)
for s in scales:
    data1 = [np.random.uniform(-1,1, size=n_examples) for _ in range(int(n_vars/2))]
    data2 = [d+np.random.normal(0,s, size=n_examples) for d in data1]

    X = np.stack(data1+data2, axis=-1)
    feature_names = [f'X_{i+1}' for i in range(n_vars)]
    X = pd.DataFrame(X, columns=feature_names)

    print(X.corr().abs()['X_1']['X_6'])

    error = np.random.normal(loc=0, scale=0.1, size=n_examples)
    y = X.dot(weights)+error
    X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=0, test_size=0.25)

    lr = LinearRegression()
    lr.fit(X_train, y_train)

    myInterpreter = mintpy.InterpretToolkit(models=lr,
                                            model_names='Linear Regression',
                                            examples=X_train,targets=y_train,
                                           )

    results = myInterpreter.calc_permutation_importance(
                                               n_vars=len(feature_names),
                                               evaluation_fn='mse',
                                               n_bootstrap=100,
                                               subsample=1.0,
                                               n_jobs=10,
                                              )

    rankings = results['multipass_rankings__Linear Regression'].values
    ranked_features.append(rankings)
    rankings = [np.where(rankings==f)[0] for f in feature_names]

    ax.plot(np.arange(n_vars), rankings, linewidth=2.5, label=f'$\sigma$={s:.1f}')

ax.set_xticks(np.arange(n_vars))
ax.set_xticklabels([f'${f}$' for f in feature_names])
ax.tick_params(labelsize=10)
ax.set_xlabel('Features')
ax.set_ylabel('Ranking (lower is better!)')
ax.grid(alpha=0.6)
ax.legend()
ax.set_title('10-var Linear Regression\nWith Varying levels of Correlations')

```

```python
fig, ax = plt.subplots(dpi=150)

scales = np.arange(0.1, 1.7, 0.2)
for s in scales:
    data1 = [np.random.uniform(-1,1, size=n_examples) for _ in range(int(n_vars/2))]
    data2 = [d+np.random.normal(0,s, size=n_examples) for d in data1]

    X = np.stack(data1+data2, axis=-1)
    feature_names = [f'X_{i+1}' for i in range(n_vars)]
    X = pd.DataFrame(X, columns=feature_names)

    error = np.random.normal(loc=0, scale=0.5, size=n_examples)
    y = X.dot(weights)+error
    X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=0, test_size=0.25)

    rf = RandomForestRegressor()
    rf.fit(X_train, y_train)

    myInterpreter = mintpy.InterpretToolkit(
                                            models=rf,
                                            model_names='Random Forest',
                                            examples=X_train,targets=y_train,
                                           )
    results = myInterpreter.calc_permutation_importance(
                                               n_vars=len(feature_names),
                                               evaluation_fn='mse',
                                               n_bootstrap=100,
                                               subsample=1.0,
                                               n_jobs=10,
                                              )

    rankings = results['multipass_rankings__Random Forest'].values
    ranked_features.append(rankings)
    rankings = [np.where(rankings==f)[0] for f in feature_names]

    ax.plot(np.arange(n_vars), rankings, linewidth=2.5, label=f'$\sigma$={s:.1f}')

ax.set_xticks(np.arange(n_vars))
ax.set_xticklabels([f'${f}$' for f in feature_names])
ax.tick_params(labelsize=10)
ax.set_xlabel('Features')
ax.set_ylabel('Ranking (lower is better!)')
ax.grid(alpha=0.6)
ax.legend()
ax.set_title('10-var Random Forest\nWith Varying levels of Correlations')

```

```python

```


 $\quad \quad \quad $  *Credit: [Japan Times](https://www.japantimes.co.jp/news/2017/11/10/national/japans-casinos-work-keep-yakuza-deal-problem-drinking-experts/#.W-RJPJMzaUk).*

# From Scratch: Bayesian Inference, Markov Chain Monte Carlo and Metropolis Hastings, in python

In one of the courses during my data science degree, I came across a technique called Markov Chain Monte Carlo, or as it's more commonly referred to, MCMC. The description for this method stated something along the lines of: MCMC is a class of techniques for sampling from a probability distribution and can be used to estimate the distribution of parameters given a set of observations.

Back then, I did not think much of it. I thought, "oh it's just another sampling technique", and I decided I'd read on it when I'd practically need it. This need never emerged, or perhaps it did and I wrongly used something else.

## So why the interest now?

Recently, I have seen a few discussions about MCMC and some of its implementations, specifically the Metropolis-Hastings algorithm and the PyMC3 library. $Markov \: Chain\: Monte\: Carlo\: in\: Python\: - \:A \:Complete\: Real$-$World \:Implementation$ was the article that caught my attention the most. In it, William Koehrsen explains how he was able to learn the approach by applying it to a real world problem: to estimate the parameters of a logistic function that represents his sleeping patterns.

\begin{equation} P(sleep/t,\alpha,\beta)=\dfrac{1}{1+e^{\beta t + \alpha}}  \end{equation}

Mr. Koehrsen uses the PyMC3 implementation of the Metropolis-Hastings algorithm to estimate $\beta$ and $\alpha$, thus computing the entire parameter space, and deriving the most likely logistic model.

## So why am I talking about all that?

I this article, I propose to implement from scratch, my own version of the Metropolis-Hastings algorithm to find parameter distributions for a dummy data example and then of a real world problem.

I figured that if I get my hands dirty, I might finally be able to understand it. I will only use numpy to implement the algorithm, and matplotlib to draw pretty things. Alternatively, scipy can be used to compute the density functions, but I will also show how to implement them using numpy.

## Flow of the article:

    * At first, I will introduce Bayesian inference, MCMC-MH and their mathematical components.
    * Second, I will explain the algorithm using dummy data.
    * Third, I will apply it to a real world problem.

# Part 1: Bayesian inference, Markov Chain Monte Carlo, and Metropolis-Hastings

## A bird's eye view on the philosophy of probabilities

In order to talk about Bayesian inference and what follows it, I shall first explain what the Bayesian view of probability is, and situate it within its historical context

### Frequentist vs Bayesian thinking

There are two major interpretations to probabilities: Bayesian and Frequentist.

From a **Frequentist's** perspective, probabilities represent long term frequencies with which events occur. A frequentist can say that the probability of having tails from a coin toss is equal to 0.5 *on the long run*. Each new experiment, can be considered as one of an infinite sequence of possible repetitions of the same experiment. The idea is that there is *no* belief in a frequentist's view of probability. The probability of event $x$ happening out of n trials is equal to the following frequency: $P(x)=\dfrac{n_x}{n}$, and the true probability is reached when $n->\infty$. Frequentists will never say "I am 45% (0.45) sure that there is lasagna for lunch today", since this does not happen on the long run. Commonly, a frequentist approach is referred to as the *objective* approach since there is no expression of belief and/or prior events in it.

On the other hand, in **Bayesian** thinking, probabilities are treated as an expression of belief. Therefore it is perfectly reasonable for a Bayesian to say "I am 50% (0.5) sure that there is lasagna for lunch today". By combining *prior* beliefs and current events (the *evidence*) one can compute the *posterior*, the belief that there is lasagna today. The idea behind Bayesian thinking is to keep updating the beliefs as more evidence is provided. Since this approach deals with belief, it is usually referred to as the *subjective* view on probability.

### Bayesian inference

 $\quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad $  *A supposed portrait of Thomas Bayes, an English statistician, philosopher, and theologian. Image Credit: [Farnam Street](https://fs.blog/2018/09/bayes-theorem/).*

In the philosophy of decision making, Bayesian inference is closely related to Bayesian probability, in the sense that it manipulates priors, evidence, and likelihood to compute the posterior. Given some event B, what is the probability that event A occurs?. This is answered by Bayes' famous formula: $P(A/B)=\dfrac{P(B/A)P(A)}{P(B)}$

* $P(A/B)$ is the **posterior**. What we wish to compute.

* $P(B/A)$ is the **likelihood**. Assuming A occured, how likely is B.

* $P(A)$ is the **prior**. How likely the event $A$ is regardless of evidence.

* $P(B)$ is the **evidence**. How likely the evidence $B$ is regardless of the event.

In our case, we are mostly interested in the specific formulation of Bayes' formula:

$P(\theta/D)=\dfrac{P(D/\theta)P(\theta)}{P(D)}$ where, $P(\theta/D)$ is the **posterior**, $P(D/\theta)$ is the **likelihood**, $P(\theta)$ is the **prior** and $P(D)$ is the **evidence**.

We would like to find the most likely distribution of $\theta$, the parameters of the model explaining the data, D.

Computing some of these probabilities can be tedious, especially the evidence $P(D)$. Also, other problems can arise such as those of ensuring conjugacy, which I will not dive into in this article. Luckily, some techniques, namely MCMC, allow us to sample from the posterior, and a draw distributions over our parameters without having to worry about computing the evidence, nor about conjugacy.

### Markov Chain Monte Carlo

MCMC allows us to draw samples from any distribution that we can't sample from directly. It can be used to sample from the posterior distribution over parameters.
It has seen much success in many applications, such as computing the distribution of parameters, given a set of observations and some prior belief, and also computing high dimensional integrals in physics and in digital communications.

Bottom line: **It can be used to compute the distribution over the parameters, given a set of observations and a prior belief.**

### Metropolis-Hastings

MCMC is a class of methods. Metropolis-Hastings is a specific implementation of MCMC. It works well in high dimensional spaces as opposed to Gibbs sampling and rejection sampling.

This technique requires a simple distribution called the **proposal distribution** (Which I like to call **transition model**) $Q(\theta^\prime/\theta)$ to help draw samples from an intractable posterior distribution $P(\Theta=\theta/D)$.

Metropolis-Hastings uses $Q$ to randomly walk in the distribution space, accepting or rejecting jumps to new positions based on how likely the sample is. This "memoriless" random walk is the "Markov Chain" part of MCMC.

The "likelihood" of each new sample is decided by a function $f$ . That's why $f$ must be proportional to the posterior we want to sample from. f is commonly chosen to be a probability density function that expresses this proportionality.

To get a new position of the parameter, just take our current one $\theta$, and propose a new one $\theta^\prime$, that is a random sample drawn from $Q(\theta^\prime/\theta)$. Often this is a symmetric distribution. For instance, a normal distribution with mean $\theta$ and some standard deviation $\sigma$: $Q(\theta^\prime/\theta) = \mathcal{N}(\theta, \sigma)$

To decide if $\theta^\prime$ is to be accepted or rejected, the following ratio must be computed for each new value of $\theta^\prime$: $\dfrac{P(\theta^\prime/D)}{P(\theta/D)}$. Using Bayes' formula this can be easily re-formulated as: $\dfrac{P(D/\theta^\prime)P(\theta^\prime)}{P(D/\theta)P(\theta)}$ (The evidence $P(D)$ is simply crossed out during the division).  $\dfrac{P(D/\theta^\prime)P(\theta^\prime)}{P(D/\theta)P(\theta)}$ is also equivalent to $\dfrac{\prod_i^nf(d_i/\Theta=\theta^\prime)P(\theta^\prime)}{\prod_i^nf(d_i/\Theta=\theta)P(\theta)}$

$$\begin{equation} P(\text{accept}) = \begin{cases}\dfrac{\prod_i^nf(d_i/\Theta=\theta^\prime)P(\theta^\prime)}{\prod_i^nf(d_i/\Theta=\theta)P(\theta)}, & \prod_i^nf(d_i/\Theta=\theta)P(\theta)>\prod_i^nf(d_i/\Theta=\theta^\prime)P(\theta^\prime) \\  1, & \prod_i^nf(d_i/\Theta=\theta)P(\theta)\leq \prod_i^nf(d_i/\Theta=\theta^\prime)P(\theta^\prime) \end{cases} \end{equation}$$

This means that if a θ' is more likely than the current θ, then we always accept θ'. If it is less likely than the current θ, then we might accept it or reject it randomly with decreasing probability, the less likely it is.

*Note: The prior components are often crossed if there is no preference or restrictions on the parameters.*

#### Metropolis-Hastings Algorithm:
* given:
    * $f$, the PDF of the distribution to sample from
    * $Q$, the transition model
    * $\theta_0$, a first guess for $\theta$
    * $\theta = \theta_0$

* for $n$ iterations
    * $p =  f(D/\Theta=\theta)P(\theta)$
    *  $\theta^\prime = Q(\theta_i)$
    *  $p^\prime = f(D/\Theta=\theta^\prime)P(\theta^\prime)$
    *  $ratio = \dfrac{p^\prime}{p}$
    * generate a uniform random number $r$ in [0,1]
    *  if $r<ratio$:
        * set $\theta_i = \theta^\prime$

```python
import numpy as np
import scipy
import scipy.stats
import matplotlib as mpl
import matplotlib.pyplot as plt
%matplotlib inline
```

# Part 2: Dummy data example
## Step 1: Data generation

We generate 30,000 samples from a normal distribution with $\mu$ = 10, and $\sigma$= 3, but we can only observe 1000 of them.

```python
mod1=lambda t:np.random.normal(10,3,t)

#Form a population of 30,000 individual, with average=10 and scale=3
population = mod1(30000)
#Assume we are only able to observe 1,000 of these individuals.
observation = population[np.random.randint(0, 30000, 1000)]

fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(1,1,1)
ax.hist( observation,bins=35 ,)
ax.set_xlabel("Value")
ax.set_ylabel("Frequency")
ax.set_title("Figure 1: Distribution of 1000 observations sampled from a population of 30,000 with $\mu$=10, $\sigma$=3")
mu_obs=observation.mean()
mu_obs
```

## Step 2: What do we want?

We would like to find a distribution for $\sigma_{obs}$ using the 1000 observed samples. Those of you who are adept at mathematics will say that there is a formula for computing $\sigma$ ( $\sigma=\sqrt{\dfrac{1}{n}\sum_i^n(d_i-\mu)^2}$)! Why do we want to sample and whatnot?? Well, this is just a dummy data example, the real problem is in part 3, where the parameters cannot be computed directly. Plus here, we are not trying to find *a* value for $\sigma$, but rather, we are trying to compute a distribution of the possible values of $\sigma$.

## Step 3: Define the PDF and the transition model.

From Figure 1, we can see that the data is normally distributed. The mean can be easily computed by taking the average of the values of the 1000 samples. By doing that, we get for example $\mu_{obs}=9.8$.

### For the transition model/ proposal distribution:
I have no specific distribution in mind, so I will choose a simple one: the Normal distribution!

\begin{equation} Q(\sigma_{new} / \sigma_{current}) = N(\mu=\sigma_{current},\sigma'=1) \end{equation}

Note that $\sigma'$ is unrelated to $\sigma_{new}$ and $\sigma_{current}$. It simply specifies the standard deviation of the parameter space. It can be any value desired. It only affects the convergence time of the algorithm.

### For the PDF:
Since f should be proportional to the posterior, we choose f to be the following Probability Density Function (PDF), for each data point $d_i$ in the data D:

\begin{equation} f(d_i/ \mu,\sigma^2) = \dfrac{1}{\sqrt{2\pi\sigma^2}}e^{-\dfrac{(d_i-\mu)^2}{2\sigma^2}} \end{equation}

In our case, $\theta$ is made up of two values: $[\mu,\sigma]$, and that $\mu$ is a constant, $\mu = \mu_{obs}$.

## Step 4: Define when we accept or reject $\sigma_{new}$:
We accept $\sigma_{new}$ if:

$\dfrac{Likelihood(D/\mu_{obs},\sigma_{new})*prior(\mu_{obs},\sigma_{new})}{Likelihood(D/\mu_{obs},\sigma_{current})*prior(\mu_{obs},\sigma_{current})}>1     \quad \quad \quad \quad \quad      (1)$

If this ratio is smaller or equal to 1, then we compare it to a uniformly generated random number in the closed set [0,1]. If the ratio is larger than the random number, we accept $\sigma_{new}$, otherwise we reject it.

*Note: Since we will be computing this ratio to decide which parameters should be accepted, it is imperative to make sure that the adopted function $f$  is proportional to the posterior itself, $P(\sigma/ D,\mu)$, which in that case is verified. ($f$ is the PDF of P)*

## Step 5: Define the prior and the likelihood:
### For the Prior $P(\theta)$ which we can alternatively note $P(\sigma)$ since $\mu$ is constant:
We don't have any preferences for the values that $\sigma_{new}$ and $\sigma_{current}$ can take. The only thing worth noting is that they should be positive. Why? Intuitively, the standard deviation measures dispersion. Dispersion is a distance, and distances cannot be negative. Mathematically, $\sigma=\sqrt{\dfrac{1}{n}\sum_i^n(d_i-\mu)^2}$, and the square root of a number cannot be negative. We strictly enforce this in the prior.

### For the likelihood :
The total likelihood for a set of observation $D$ is: $Likelihood(D/\mu_{obs},\sigma_{a}) = \prod_i^n f(d_i/\mu_{obs},\sigma_{a}) $, where $a=new \: or \: current$.

In our case, we will log both the prior and the likelihood function. Why log? Simply because it helps with numerical stability, i.e. multiplying thousands of small values (probabilities, likelihoods, etc..) can cause an underflow in the system's memory, and the log is a perfect solution because it transforms multiplications to additions and small positive numbers into non-small negative numbers.

Therefore our acceptance condition from equation $(1)$ becomes:

Accept $\sigma_{new}$ if:

$\quad \quad \quad \quad \quad Log(Likelihood(D/\mu_{obs},\sigma_{new})) + Log(prior(\mu_{obs},\sigma_{new})) - (Log(Likelihood(D/\mu_{obs},\sigma_{current}))+$

$Log(prior(\mu_{obs},\sigma_{current})))>0$

 $\quad$

 Equivalent to:

 $\sum_i^nLog(f(d_i/\mu_{obs},\sigma_{new})) + Log(prior(\mu_{obs},\sigma_{new})) - \sum_i^nLog(f(d_i/\mu_{obs},\sigma_{current}))-Log(prior(\mu_{obs},\sigma_{current}))>0$

 $\quad$

  Equivalent to:

  $\sum_i^nLog(f(d_i/\mu_{obs},\sigma_{new})) + Log(prior(\mu_{obs},\sigma_{new})) > \sum_i^nLog(f(d_i/\mu_{obs},\sigma_{current}))+Log(prior(\mu_{obs},\sigma_{current}))$

   $\quad$

  Equivalent to:

  $\sum_i^n -nLog(\sigma_{new}\sqrt{2\pi})-\dfrac{(d_i-\mu_{obs})^2}{2\sigma_{new}^2} + Log(prior(\mu_{obs},\sigma_{new})) \quad > $

 $ \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad \quad(2)$
 $ \quad \quad \quad \quad \quad \sum_i^n -nLog(\sigma_{current}\sqrt{2\pi})-\dfrac{(d_i-\mu_{obs})^2}{2\sigma_{current}^2}+Log(prior(\mu_{obs},\sigma_{current}))  $

  This form can be reduced even more by taking the square root and the multiplication out of the log.

```python
#The tranistion model defines how to move from sigma_current to sigma_new
transition_model = lambda x: [x[0],np.random.normal(x[1],0.5,(1,))[0]]

def prior(x):
    #x[0] = mu, x[1]=sigma (new or current)
    #returns 1 for all valid values of sigma. Log(1) =0, so it does not affect the summation.
    #returns 0 for all invalid values of sigma (<=0). Log(0)=-infinity, and Log(negative number) is undefined.
    #It makes the new sigma infinitely unlikely.
    if(x[1] <=0):
        return 0
    return 1

#Computes the likelihood of the data given a sigma (new or current) according to equation (2)
def manual_log_like_normal(x,data):
    #x[0]=mu, x[1]=sigma (new or current)
    #data = the observation
    return np.sum(-np.log(x[1] * np.sqrt(2* np.pi) )-((data-x[0])**2) / (2*x[1]**2))

#Same as manual_log_like_normal(x,data), but using scipy implementation. It's pretty slow.
def log_lik_normal(x,data):
    #x[0]=mu, x[1]=sigma (new or current)
    #data = the observation
    return np.sum(np.log(scipy.stats.norm(x[0],x[1]).pdf(data)))

#Defines whether to accept or reject the new sample
def acceptance(x, x_new):
    if x_new>x:
        return True
    else:
        accept=np.random.uniform(0,1)
        # Since we did a log likelihood, we need to exponentiate in order to compare to the random number
        # less likely x_new are less likely to be accepted
        return (accept < (np.exp(x_new-x)))

def metropolis_hastings(likelihood_computer,prior, transition_model, param_init,iterations,data,acceptance_rule):
    # likelihood_computer(x,data): returns the likelihood that these parameters generated the data
    # transition_model(x): a function that draws a sample from a symmetric distribution and returns it
    # param_init: a starting sample
    # iterations: number of accepted to generated
    # data: the data that we wish to model
    # acceptance_rule(x,x_new): decides whether to accept or reject the new sample
    x = param_init
    accepted = []
    rejected = []
    for i in range(iterations):
        x_new =  transition_model(x)
        x_lik = likelihood_computer(x,data)
        x_new_lik = likelihood_computer(x_new,data)
        if (acceptance_rule(x_lik + np.log(prior(x)),x_new_lik+np.log(prior(x_new)))):
            x = x_new
            accepted.append(x_new)
        else:
            rejected.append(x_new)

    return np.array(accepted), np.array(rejected)
```

## Step 6: Run the algorithm with initial parameters and collect accepted and rejected samples

```python
accepted, rejected = metropolis_hastings(manual_log_like_normal,prior,transition_model,[mu_obs,0.1], 50000,observation,acceptance)
```

    /usr/local/lib/python3.7/site-packages/ipykernel_launcher.py:17: RuntimeWarning: invalid value encountered in log
    /usr/local/lib/python3.7/site-packages/ipykernel_launcher.py:51: RuntimeWarning: divide by zero encountered in log

### The algorithm accepted 8803 samples (which might be different on each new run). The last 10 samples contain the following  values for $\sigma$:

```python
print(accepted.shape)
accepted[-10:,1]
```

    (8118, 2)

    array([3.06423543, 3.0703956 , 3.06228605, 2.91587806, 2.92883687,
           2.88441621, 2.93353383, 2.89699706, 2.88554012, 2.88552288])

```python
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(2,1,1)

ax.plot( rejected[0:50,1], 'rx', label='Rejected',alpha=0.5)
ax.plot( accepted[0:50,1], 'b.', label='Accepted',alpha=0.5)
ax.set_xlabel("Iteration")
ax.set_ylabel("$\sigma$")
ax.set_title("Figure 2: MCMC sampling for $\sigma$ with Metropolis-Hastings. First 50 samples are shown.")
ax.grid()
ax.legend()

ax2 = fig.add_subplot(2,1,2)
to_show=-accepted.shape[0]
ax2.plot( rejected[to_show:,1], 'rx', label='Rejected',alpha=0.5)
ax2.plot( accepted[to_show:,1], 'b.', label='Accepted',alpha=0.5)
ax2.set_xlabel("Iteration")
ax2.set_ylabel("$\sigma$")
ax2.set_title("Figure 3: MCMC sampling for $\sigma$ with Metropolis-Hastings. All samples are shown.")
ax2.grid()
ax2.legend()

fig.tight_layout()
accepted.shape
```

So, starting from an initial σ of 0.1, the algorithm converged pretty quickly to the expected value of 3. That said, it's only sampling in a 1D space…. so it's not very surprising.

### We consider the initial 25% of the values of $\sigma$ to be "burn-in", so we drop them.
### Let's visualize the trace of  $\sigma$ and the histogram of the trace.

```python

show=int(-0.75*accepted.shape[0])
hist_show=int(-0.75*accepted.shape[0])

fig = plt.figure(figsize=(20,10))
ax = fig.add_subplot(1,2,1)
ax.plot(accepted[show:,1])
ax.set_title("Figure 4: Trace for $\sigma$")
ax.set_ylabel("$\sigma$")
ax.set_xlabel("Iteration")
ax = fig.add_subplot(1,2,2)
ax.hist(accepted[hist_show:,1], bins=20,density=True)
ax.set_ylabel("Frequency (normed)")
ax.set_xlabel("$\sigma$")
ax.set_title("Figure 5: Histogram of $\sigma$")
fig.tight_layout()

ax.grid("off")

```

### The most likely value for $\sigma$ is around 3.1. This is a bit more than the original value of 3.0. The difference is due to us observing only 3.33% of the original population (1,000 out of 30,000)

## Predictions:
First, we average the last 75% of accepted samples of σ, and we generate 30,000 random individuals from a normal distribution with μ=9.8 and σ=3.05 (the average of the last 75% of accepted samples) which is actually better than the most likely value of 3.1.

```python
mu=accepted[show:,0].mean()
sigma=accepted[show:,1].mean()
print(mu, sigma)
model = lambda t,mu,sigma:np.random.normal(mu,sigma,t)
observation_gen=model(population.shape[0],mu,sigma)
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(1,1,1)
ax.hist( observation_gen,bins=70 ,label="Predicted distribution of 30,000 individuals")
ax.hist( population,bins=70 ,alpha=0.5, label="Original values of the 30,000 individuals")
ax.set_xlabel("Mean")
ax.set_ylabel("Frequency")
ax.set_title("Figure 6: Posterior distribution of predicitons")
ax.legend()
```

# Part 3: A real world example

*Credit: Amateur astronomer Alan Friedman on July 10, 2012.*

A sunspot is a region on the Sun's surface (photosphere) that is marked by a lower temperature than its environment. These reduced temperatures are caused by concentrations of magnetic field flux that inhibit convection by an effect similar to [eddy current brakes](https://en.wikipedia.org/wiki/Eddy_current). Sunspots usually appear in pairs of opposite magnetic polarity. Their number varies according to the approximately 11-year solar cycle.

The data we will be working on is the "Monthly mean total sunspot number", for each month from January 1749 to November 2018. This is data collected, curated and made publicly available by the [World Data Center for the production, preservation and dissemination of the international sunspot number](http://www.sidc.be/silso/home).

## Let's plot the data over the years to see what the distribution might be like.

```python
sunspots = np.loadtxt("data/SN_m_tot_V2.0.csv", delimiter=';')

#years is the third column
years = sunspots[:,2]
#activity is the 4th column
activity = sunspots[:,3] + 0.1
print(years.shape,activity.shape)
fig = plt.figure(figsize=(20,20))
ax = fig.add_subplot(2,1,1)

ax.plot(years, activity, label="Sunspot count")
ax.set_yticks(np.arange(0,500,100))
ax.set_xlabel("Years")
ax.set_ylabel("Monthly mean count of sunspots")
ax.set_title("Figure 7: Sunspots, 1749-2018")
ax.legend()

ax = fig.add_subplot(2,1,2)
print(years[432])
ax.plot(years[:432], activity[:432], label="Sunspot count")
ax.set_yticks(np.arange(0,500,100))
ax.set_xlabel("Years")
ax.set_ylabel("Monthly mean count of sunspots")
ax.set_title("Figure 8: Sunspots, 1749-1785")
ax.legend()
```

## It seems like we could model this phenomenon with a gamma distribution, with a new cycle resetting every 12 years.

A gamma distribution Γ is a two-parameter family of continuous probability distributions, the parameters are the shape a and the scale b. A random variable X that is gamma-distributed is noted X~Γ(a, b), and in our case X is the count of sunspots. The two parameters a and b are the unknowns that we would like to calculate distributions for.
*Credit: Wikipedia Commons.*

For example, in the first cycle, the sunspot counts start from their highest at about 300 at the end of 1749, and fall to their lowest 6 years after, during 1755. Then the number rises up again to it's maximum during 1761 and 1762 before falling again during 1766 and so on...

## Let's make sure by plotting a histogram of sunspot counts:

```python
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(1,1,1)
ax.hist(activity, bins=40, density=True)
ax.set_xlabel("Sunspot count")
ax.set_ylabel("Frequency")
ax.set_title("Figure 9: Histogram showing the frequency of sunspot counts over 270 years (1749-2018)")
```

## Indeed, it does seem like the frequency of counts follows a gamma distribution

The gamma distribution, has for PDF, $f/f(x;a,b) =\dfrac{b^a x^{a-1}e^{-b x}}{\Gamma{(a)}}$ where $\Gamma$ is the gamma function (not to be confused with the gamma distribution which requires 2 parameters): $\Gamma{(a)}=(a-1)!$

Following the same procedure as in the dummy data example, we can write down the log likelihood from this pdf (see code below). Alternatively, one could use the $scipy.stats.gamma(a,b).pdf(x)$ function to compute it.

Since a and b must be positive, we enforce this in the prior.

```python
transition_model = lambda x: np.random.normal(x,[0.05,5],(2,))
import math
def prior(w):
    if(w[0]<=0 or w[1] <=0):
        return 0
    else:
        return 1

def manual_log_lik_gamma(x,data):
    return np.sum((x[0]-1)*np.log(data) - (1/x[1])*data - x[0]*np.log(x[1]) - np.log(math.gamma(x[0])))

def log_lik_gamma(x,data):
    return np.sum(np.log(scipy.stats.gamma(a=x[0],scale=x[1],loc=0).pdf(data)))

```

```python
accepted, rejected = metropolis_hastings(manual_log_lik_gamma,prior,transition_model,[4, 10], 50000,activity,acceptance)
```

```python
print(accepted.shape)
accepted[-10:]
```

    (8537, 2)

    array([[ 0.97009904, 85.34205726],
           [ 0.97018033, 88.24737974],
           [ 0.97439353, 87.74024406],
           [ 0.97891065, 85.53414066],
           [ 0.96527862, 86.48928538],
           [ 0.98343021, 84.39404222],
           [ 1.00547929, 82.69265895],
           [ 0.95034478, 86.55690013],
           [ 0.98935646, 85.16989734],
           [ 1.00038475, 82.32425891]])

Starting from a=4, and b =10, the algorithm accepted 8561 pairs of samples, the last value for a is 1.01307402 and the last value for b is 83.40995308, which are pretty far off the initial values.

## As with the dummy data example, let's see how the algorithm worked its way to these values:

```python

fig = plt.figure(figsize=(10,20))
ax = fig.add_subplot(3,1,1)
ax.plot(accepted[:50,0], accepted[:50,1], label="Path")
ax.plot(accepted[:50,0], accepted[:50,1], 'b.', label='Accepted')
ax.plot(rejected[:50,0], rejected[:50,1], 'rx', label='Rejected')
ax.set_xlabel("a")
ax.set_ylabel("b")
ax.legend()
ax.set_title("Figure 10: MCMC sampling for $a$ and $b$ with Metropolis-Hastings. First 50 samples are shown.")

ax = fig.add_subplot(3,1,2)
ax.plot(accepted[:,0], accepted[:,1], label="Path")
ax.plot(accepted[:,0], accepted[:,1], 'b.', label='Accepted',alpha=0.3)
ax.plot(rejected[:,0], rejected[:,1], 'rx', label='Rejected',alpha=0.3)
ax.set_xlabel("a")
ax.set_ylabel("b")
ax.legend()
ax.set_title("Figure 11: MCMC sampling for $a$ and $b$ with Metropolis-Hastings. All samples are shown.")

to_show=50
ax = fig.add_subplot(3,1,3)
ax.plot(accepted[-to_show:,0], accepted[-to_show:,1], label="Path")
ax.plot(accepted[-to_show:,0], accepted[-to_show:,1], 'b.', label='Accepted',alpha=0.5)
ax.plot(rejected[-to_show:,0], rejected[-to_show:,1], 'rx', label='Rejected',alpha=0.5)
ax.set_xlabel("a")
ax.set_ylabel("b")
ax.legend()
ax.set_title("Figure 12: MCMC sampling for $a$ and $b$ with Metropolis-Hastings. Last 50 samples are shown.")
```

As we can see from figures 10, 11, and 12, the algorithm converges quickly to the [a=1,b=85] zone.

Tip: when the algorithm starts to heavily reject samples, that means that we have reached a zone of saturation of the likelihood. Commonly, this can be interpreted as having reached the optimal parameter space from which we can sample, i.e. there is very little reason for the algorithm to accept new values. This is marked in figures 11, and 12 where the algorithm no longer accepts any values outside of a small range.

### We consider the initial 50% of the values of a and b to be "burn-in", so we drop them.
### Let's visualize the traces of  and b and the histogram of the traces.

```python
show=int(-0.5*accepted.shape[0])
hist_show=int(-0.50*accepted.shape[0])

fig = plt.figure(figsize=(15,7))
ax = fig.add_subplot(1,2,1)
ax.plot(accepted[show:,0])
ax.set_title("Figure 13: Trace for $a$")
ax.set_xlabel("Iteration")
ax.set_ylabel("a")
ax = fig.add_subplot(1,2,2)
ax.hist(accepted[hist_show:,0], bins=20, density=True)
ax.set_ylabel("Frequency (normed)")
ax.set_xlabel("a")
ax.set_title("Figure 14: Histogram of $a$")
fig.tight_layout()

fig = plt.figure(figsize=(15,7))
ax = fig.add_subplot(1,2,1)
ax.plot(accepted[show:,1])
ax.set_title("Figure 15: Trace for $b$")
ax.set_xlabel("Iteration")
ax.set_ylabel("b")
ax = fig.add_subplot(1,2,2)
ax.hist(accepted[hist_show:,1], bins=20, density=True)
ax.set_ylabel("Frequency (normed)")
ax.set_xlabel("b")
ax.set_title("Figure 16: Histogram of $b$")
fig.tight_layout()

fig = plt.figure(figsize=(10,7))
ax = fig.add_subplot(1,1,1)
xbins, ybins = np.linspace(0.8,1.2,30), np.linspace(75,90,30)
counts, xedges, yedges, im = ax.hist2d(accepted[hist_show:,0], accepted[hist_show:,1], density=True, bins=[xbins, ybins])
ax.set_xlabel("a")
ax.set_ylabel("b")
fig.colorbar(im, ax=ax)
ax.set_title("2D histogram showing the joint distribution of $a$ and $b$")
```

## Prediction time

First, we average the last 50% of accepted samples of a and b, and we generate random individuals from a Γ distribution. $a_{average}$=0.9866200759935773 and $b_{average}$=83.70749712447888.

```python
show=-int(0.5*accepted.shape[0])

mu=accepted[show:,0].mean()
sigma=accepted[show:,1].mean()
print(mu, sigma)
model = lambda t,mu,sigma:np.random.gamma(mu,sigma,t)
t=np.arange(activity.shape[0])
observation_gen=model(t.shape[0],mu,sigma)
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(1,1,1)

ax.hist( observation_gen,bins=np.linspace(0,500,50) ,density=True,label="Predicted values")
ax.hist( activity,bins=np.linspace(0,500,50) ,alpha=0.5,density=True, label="Original values")
ax.set_xlabel("Count")
ax.set_ylabel("Frequency")
ax.set_title("Figure 17: Posterior distribution of predicitons")
ax.legend()
```

## Evaluation
### Evaluation of the proposal distribution
How do we specify the parameters for the distribution Q? Should we move far from the current sample θ, or stay relatively close? These questions can be answered by measuring the auto-correlation between accepted samples: we do not want the accepted samples to be too correlated between one another. We don't want distant samples to be too correlated as we are trying to implement a markov chain, i.e. a sample should only depend on its previous sample, and the auto-correlation plot should show a quick, exponential decrease between the correlation of sample i and i-1,i-2,...i-n

The auto-correlation is defined by computing the following function for each lag $k$:
$r_k=\dfrac{\sum_{i=1}^{N-k}(Y_i-Y_{avg})(Y_{i+k}-Y_{avg})}{\sum_{i=1}^{N}(Y_i-Y_{avg})^2}$

The lag $k$, is basically the *range* around a sample $Y_i$ in which we would like to measure the correlation.

The plots below show the auto-correlation for a, b for k going from 1 to 100. A lag of k=1 means that we are measuring the correlation of a sample with itself, so we expect it to be equal to 1. The higher k goes, the lower that correlation ought to be.

```python
mean_acc_0=accepted[show:,0].mean()
mean_acc_1=accepted[show:,1].mean()
print(mean_acc_0,mean_acc_1)

lag=np.arange(1,100)
def autocorr(accepted,lag):
    num_0=0
    denom_0=0
    num_1=0
    denom_1=0
    for i in range(accepted.shape[0]-lag):
        num_0+=(accepted[i,0]-mean_acc_0)*(accepted[i+lag,0]-mean_acc_0)
        num_1+=(accepted[i,1]-mean_acc_1)*(accepted[i+lag,1]-mean_acc_1)
        denom_0+=(mean_acc_0-accepted[i,0])**2
        denom_1+=(mean_acc_1-accepted[i,1])**2
    rk_0=num_0/denom_0
    rk_1=num_1/denom_1
    return rk_0, rk_1

accepted_reversed=accepted[show:,:]
result=np.zeros((2,lag.shape[0]))
#print(lag)
for l in lag:
    result[:,l-1]=autocorr(accepted_reversed,l)

###Instead of writing an autocorrelation function, one could simply use thee autocorr function provided in pymc3
#from pymc3.stats import autocorr

fig, ax = plt.subplots()
#ax.plot(lag, [autocorr(accepted[show:,1], l) for l in lags], label='auto b')
#ax.plot(lag, [autocorr(accepted[show:,0], l) for l in lags], label='auto a')
ax.plot(lag, result[1,:], label='Auto correlation for b')
ax.plot(lag, result[0,:], label='Auto correlation for a')
ax.legend(loc=0)
ax.set_title("Figure 18: Auto-correlation for a and b, for k=1..100")
ax.set(xlabel='lag', ylabel='autocorrelation', ylim=(-.1, 1))
```

In our case, we are lucky to have a low enough correlation. In general, we might want to setup the parameters of the proposal distribution, Q, automatically, one common method is to keep adjusting the proposal parameters so that more than 50% proposals are rejected. Alternatively, one could use an enhanced version of MCMC called Hamiltonian Monte Carlo, which reduces the correlation between successive sampled states and reaches the stationary distribution quicker.

## Conclusion
While the abstraction behind this algorithm may seem out of grasp at first, the implementation is actually pretty simple, and gives awesome results. In fact, the great thing about probabilistic programming, notably MCMC is that you only need to write down the model and then run it. There is no need to compute evidence, or ensure some constraining mathematical properties.

## Resources
Peter Driscoll, "A comparison of least-squares and Bayesian fitting techniques to radial velocity data sets"

Carson Chow, "MCMC and fitting models to data"

John H. Williamson, "Data Fundamentals - Probabilities"

Simon Rogers, "A first course in machine learning"


# Homework 6

### Submit the MP4 files you make along with a PDF of this notebook!

This week we will learn about how to turn the plots we have been making into animations! This will be really helpful for any groups that want to make simulations or other animations for their final project. This will build off of the differential equations we learned last week as well as the plotting skills we have learned so far.

If you are having trouble running the popup window with the %matplotlib commands, you can just comment the lines out. The animations will not show up in a popup window, but they will still save and you can open them from your current directory.

First let us import the neccessary libraries,

```python
import numpy as np
import matplotlib.pyplot as plt
import scipy
import scipy.integrate

from mpl_toolkits.mplot3d import Axes3D
from matplotlib.animation import FFMpegWriter

#%config InlineBackend.figure_format='retina' # UNCOMMENT IF USING MAC makes display better

%matplotlib osx
# ^ UNCOMMENT THIS LINE IF USING MAC

# %matplotlib qt
# ^ UNCOMMENT THIS LINE IF USING WINDOWS
```

# Problem 1 (10 points)

## Animating a Cool Function

In homework 5, we plotted the following function:

\begin{equation}
f = \frac{\sin(10(x^2 + y^2))}{10}
\end{equation}

```python
x, y = np.meshgrid(np.linspace(0, 2*np.pi, 100),
                   np.linspace(0, 2*np.pi, 100))

f = np.sin(10*(x**2 + y**2))/10

plt.figure(figsize=(12,12))
plt.imshow(f)          # Plot the 2D array
plt.set_cmap('hot')    # Set the color scheme ("jet" is matplotlib's default)
plt.colorbar()         # Display a legend for the color values
plt.show()
```

Now let's animate it!

For each iteration, add 1 more to x and y. For example, add 1 to x and y for the first iteration, and add 10 to both x and y for the tenth iteration.

Rember to use plt.imshow( ) so that your function is shown.

```python
## SAVE AS MP4 ##
fig, ax = plt.subplots(figsize=(5,5))

num_iterations = 20

with writer.saving(fig, "3Dfunction.mp4", dpi=200):
    for i in range(num_iterations):

        ax.clear() # first clear the figure

        ### YOUR CODE HERE ###
        f = np.sin(10*((x+i)**2 + (y+i)**2))/10
        plt.imshow(f)

        plt.draw()
        plt.pause(0.01)
        writer.grab_frame() # save the current frame to mp4
```

****

# Problem 2 (20 points)
## Binary Star System

In lecture we animated a single planet orbiting a star, but now let's animate a binary star system — two stars orbiting eachother. Because these two objects are of similar mass, we need to update the movements of both stars instead of just one.

First we'll define our initial conditions. For the sake of simplicity, don't worry about units for this problem.
- Star 1: mass $=3$, $x_i=-1$, $y_i=0$, $v_{xi}=0$, $v_{yx}=1$

- Star 2: mass $=3$, $x_i=1$, $y_i=0$, $v_{xi}=0$, $v_{yx}=-1$

```python
G = 1 # gravitational constant

# Define masses (in units of solar mass)
m1 = 3.0
m2 = 3.0

# Define initial position vectors
r1 = np.array([-1, 0])
r2 = np.array([1, 0])

# Define initial velocities
v1 = np.array([0, 1])
v2 = np.array([0, -1])
```

The stars motions are characterized by the Law of Universal Gravitation:

\begin{equation}
 F = G \frac{m_1 m_2}{r^2}
\end{equation}

Ultimately, all we need to plot the stars motion is the position coordinates of the two stars. But to correctly update the stars positions we need the stars velocity and the change in velocity (acceleration). Instead of working with forces, let us work with accelerations. We can then rewrite the equations for star 1 and star 2 like this:

- $ a_1 = \frac{m_2(r_2 - r_1)}{r_{12}^3} $

- $ a_2 = \frac{m_1(r_1 - r_2)}{r_{12}^3} $

Fill out the following cell to define a function that we will integrate over to obtain our star positions.

Refer to the lecture demo notebook for help! OrbitEquation( ) that was used in the demo is set up the same way, but now we want to change position and velocity for another object as well.

```python
def TwoBodyEquations(w, t, m1, m2): # w is an array containing positions and velocities
    r1 = w[:2]
    r2 = w[2:4]
    v1 = w[4:6]
    v2 = w[6:8]

    r12 = np.linalg.norm(r2-r1)

    dv1bydt = m2*(r2-r1)/r12**3  # derivative of velocity
    dv2bydt = m1*(r1-r2)/r12**3

    dr1bydt = v1 # derivative of position
    dr2bydt = v2

    r_derivs = np.concatenate((dr1bydt,dr2bydt)) # joining arrays together
    v_derivs = np.concatenate((dv1bydt,dv2bydt))
    derivs = np.concatenate((r_derivs,v_derivs))

    return derivs
```

Now we want to run the ordinary differential equation solver. Then set r1_sol equal to the parts of two_body_sol that correspond to the first star. Do the same for r2_sol but for the second star.

```python
# Package initial parameters into one array (just easier to work with this way)
init_params = np.array([r1,r2,v1,v2])
init_params = init_params.flatten()
time_span = np.linspace(0, 20, 5000)  # run for t=20 (5000 points)

# Run the ordinary differential equation solver
two_body_sol = scipy.integrate.odeint(TwoBodyEquations, init_params, time_span, args=(m1,m2)) # use scipy.integrate.odeint()

r1_sol = two_body_sol[:,:2]
r2_sol = two_body_sol[:,2:4]
```

Now we have all of the data we want to plot. Using FFMpegWriter, let us loop through all of our data and save each frame to our MP4 file. Set your x and y axes to range from -2 to 2.

Again, this code is very similar to what was shown in the lecture demo!

```python
# Initilize writer
metadata = dict(title='2D animation', artist='Matplotlib')
writer = FFMpegWriter(fps=50, metadata=metadata, bitrate=200000)
fig = plt.figure(dpi=200)

## SAVE AS MP4 ##
fig, ax = plt.subplots()

with writer.saving(fig, "binary_stars.mp4", dpi=200):
    for i in range(len(time_span)):

        ax.clear()

        ### YOUR CODE HERE ###
        ax.plot(r1_sol[:i,0],r1_sol[:i,1],color="red", alpha=0.5)
        ax.plot(r2_sol[:i,0],r2_sol[:i,1],color="orange", alpha=0.5)

        ax.scatter(r1_sol[i,0],r1_sol[i,1],color="red",marker="o",s= m1*30, zorder=5)
        ax.scatter(r2_sol[i,0],r2_sol[i,1],color="orange",marker="o",s=m2*30, zorder=5)

        ax.set_xlim(-3, 3)
        ax.set_ylim(-3, 3)
        ###

        plt.draw()
        plt.pause(0.01)
        writer.grab_frame()
```

**Optional**: try changing the star masses, initial postions, and/or initial velocities and show us an animation that you think looks cool!

```python
# can be whatever other initial conditions
```

****

# Problem 3 (20 points)
## Swinging Pendulum
Now we will animate a basic swinging pendulum that you have probably seen in your Physics classes! The steps for animating this are similar to how we animated the binary star system. The main difference is how we define our ode_func and the differential equations.
For our pendulum system, we will have a point mass $m$ connected to a string of length $l$ with slight damping:
* $m = 1 kg$ is the mass,
* $g = 9.81 m/s^2$ is gravity,
* $l = 1 m$ is the length of the massless string
* $b = 0.05$ is the damping coefficient, and
* $t = 0$ to $10 s$ is the time duration with 100 intervals.
\
\
We can derive the change in the pendulum system as:
\
\
$$\frac{d^2\theta}{dt} = - \frac{b}{m} \frac{d\theta}{dt} - \frac{g  *  sin(\theta)}{l}$$
\
To further simplify this second order differential equation, we can rewrite this as two linear equations by defining

$$\theta_{1} = \theta  ,  \theta_{2} = \frac{d\theta}{dt}$$

$$\frac{d\theta_{1}}{dt} = \theta_{2}$$

$$\frac{d\theta_{2}}{dt} = -\frac{b  *  \theta_{2}}{m} - \frac{g  *  sin(\theta_{1})}{l}$$

Don't worry if you do not understand the physics equations and math for this derivation! We will only be needing the last two equations for this problem. Now use ode_func as an argument to scipy.integrate.odeint to solve for the theta1 and theta2 values for all time steps. Use the initial conditions of
* $\theta_{1} = \frac{\pi}{10} $ as the initial angle and
* $\theta_{2} = 0$ as the initial angular velocity.

```python
def ode_func(theta, t, g, l, m, b):
    '''
    Given a list theta that is [theta1, theta2] and the conditions of the system
    Computes the differential change for theta1 and theta2 following the last two eqs.
    Returns dtheta1dt and dtheta2dt as a list in that order
    '''
    # YOUR CODE HERE
    theta1, theta2 = theta
    dtheta1_dt = theta2
    dtheta2_dt = (- b * theta2 / m - g * np.sin(theta1) / l)
    return [dtheta1_dt, dtheta2_dt]

# Define system variables l, m, g, b, t and initial conditions
# YOUR CODE HERE
l = 1
m = 1
g = 9.81
b = 0.05
t = np.linspace(0, 10, 100)
initial_conditions = [np.pi/10,1]

# Calculate theta1 and theta2 at all time steps
# HINT: scipy.integrate.odeint returns a 2D matrix where
# the first column is the theta1 values and the second column is the theta2 values
# YOUR CODE HERE
theta = scipy.integrate.odeint(ode_func, initial_conditions, t, args=(g, l, m, b))
```

Now that we have all of the $\theta_{1}$ and $\theta_{2}$ values from scipy.integrate.odeint, we can find the position of the point mass (x, y) values as
\
\
$$ x = l*sin(\theta_{1})$$
$$ y = - l*cos(\theta_{1})$$
\
For every time step, plot and save the (x, y) position using the $\theta$ values calculated above as an animation.
* Save the animation as "pendulum.mp4"
* Set y-axis limits as -1.1, 0
* Set x-axis limits as -0.5, 0.5
* If you would like to animate the string, include a line at each frame:
    "plt.plot( [ 0, x[i] ], [ 0, y[i] ] )" that will plot a line between the origin and the pendulum in the current frame i

```python
# Calculate x and y from theta1
# YOUR CODE HERE
theta1 = theta[:,0]
x = [l * np.sin(th) for th in theta1]
y = [-l * np.cos(th) for th in theta1]

# Initialize writer
# YOUR CODE HERE
metadata = dict(title='2D animation', artist='Matplotlib')
writer = FFMpegWriter(fps=100, metadata=metadata, bitrate=200000)
fig = plt.figure(dpi=200)

## SAVE AS MP4 ##
# YOUR CODE HERE
fig = plt.figure()

with writer.saving(fig, "pendulum.mp4", dpi=200):
    for i in range(len(t)):

        fig.clear()

        plt.plot([0, x[i]], [0, y[i]]) # plots the string
        plt.plot(x[i], y[i], marker="o", markersize=10) # plots the point mass
        plt.xlim([-0.5,0.5])
        plt.ylim([-1.1,0])

        plt.draw()
        plt.pause(0.01)
        writer.grab_frame()
```

****

# Problem 4 (Extra Credit 5 points)
## Random Walk

An optional problem in homework 4 was plotting 7 different random walk runs. We'll be using the same base code but instead of just plotting, let's animate the random walks!

Here's a function that defines the random walk (check out homework 4 for a more detailed explanation):

```python
N=int(1e3)  #step number
interval=0.5  #interval
n = 7 #number of particles

def random_walk(step):
    """
    A function that takes in the number of random
    steps you want to take and returns an x and y
    1D array corresponding to the path the particle
    took in the xy plane. It also returns roughly
    the limit of how far the particle can travel in
    N steps.
    """

    N1 = int(step)
    theta = np.random.uniform(0, 2*np.pi, size=N1)
    x = np.cos(theta).cumsum(0)
    y = np.sin(theta).cumsum(0)
    return x,y,np.sqrt(N1)
```

Now use whatever animation method you want (FFMpegWriter, FuncAnimation, etc.) to animate the random walk!

```python
# YOUR CODE HERE
xarr, yarr = np.zeros((n,N+1)), np.zeros((n,N+1))
for trail in range(n):
    xarr[trail][1:],yarr[trail][1:],lim = random_walk(N)

fig, ax = plt.subplots(figsize=(8,8))
ax.set_xlim(-lim*1.3,lim*1.3)
ax.set_ylim(-lim*1.3,lim*1.3)
circle = plt.Circle((0, 0), lim, color='k',ec='white')
ax.add_patch(circle)
ax.set_facecolor('k')
ax.grid(1, zorder=-10, alpha=0.3)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)

lines = [ax.plot([], [], lw=0.5)[0] for _ in range(n)]
texts = [ax.text(0, lim*1.15, '', fontsize=15, color='white', ha='center', va='center') for _ in range(n)]

def init():
    for line in lines:
        line.set_data([], [])

def animate(i):
    for j, line in enumerate(lines):
        line.set_data(xarr[j][:i], yarr[j][:i])
        texts[j].set_text('N={}'.format(i))
    return lines

ani = animation.FuncAnimation(fig, func=animate, frames=np.arange(0,len(xarr[0])), \
                    interval = interval, repeat=False)

ani.save('random_walkh.gif', writer=animation.PillowWriter(fps=120))

plt.show()
```