# Solution to the Initial Value Problem:

We solve
\begin{equation}
\epsilon\frac{\partial \theta}{\partial t} + \cos{(y)}\frac{\partial\theta}{\partial x} = \frac{1}{Pe}\frac{\partial^2\theta}{\partial x^2} + \epsilon\frac{\partial^2\theta}{\partial y^2} ,
\end{equation}

in the domain $0<y<2\pi$, $-2 \pi < x < 2 \pi$, $t>0$. Thus smallest the wavenumber (other than zero) is $k=0.5$.

The initial condition is the Gaussian

\begin{equation}
\theta(x, y, 0) = \theta_{0}\exp{(-x^2/\sigma^2)} .
\end{equation}

### We only calculate the average solution below

```python
import matplotlib.pyplot as plt
import numpy as np
from matplotlib import gridspec
import xarray as xr
```

```python
from mathieu_functions import mathieu_functions as mfs
from mathieu_functions import A_coefficients
```

```python
# =================================
# Important parameters to define
# =================================
L = np.pi # Half of channel width (y-direction)
N = 50  # length of k-array
alpha = 2  # length of channel periodic in x. I have used alpha=10 before, but for the gaussian initial condition a value of 2 is better.
Nx = 500  # length of x-array
sigma=0.5  # changes width of gaussian

eps = 0.05  # ta / td << 1 for weakly diffusive processes.
Pe = 1 / eps

x = np.linspace(-alpha * L, alpha * L, Nx)
y = np.linspace(0, L, Nx//5)
X, Y = np.meshgrid(x, y)

K = np.arange(0, N / alpha, 1 / alpha)  # wavenumber array.
K_test = np.linspace(0, N/alpha, 1000)
Q = (1j) * 2 * K / eps  # Canonical Mathieu parameter
qf = Q[-1].imag  # Largest value of Mathieu's parameter.

M = 75  # matrix size
t = np.linspace(0, .25, 100)
colors = ['#0000cc', '#990000', 'darkgreen',
          'indigo', '#009999', 'orange',
         'k']
```

```python
K
```

    array([ 0. ,  0.5,  1. ,  1.5,  2. ,  2.5,  3. ,  3.5,  4. ,  4.5,  5. ,
            5.5,  6. ,  6.5,  7. ,  7.5,  8. ,  8.5,  9. ,  9.5, 10. , 10.5,
           11. , 11.5, 12. , 12.5, 13. , 13.5, 14. , 14.5, 15. , 15.5, 16. ,
           16.5, 17. , 17.5, 18. , 18.5, 19. , 19.5, 20. , 20.5, 21. , 21.5,
           22. , 22.5, 23. , 23.5, 24. , 24.5])

```python
print('Last element of K, q, size(q)')
K[-1], Q[-1], len(Q)
```

## Calculate Mathieu Eigenfunctions and Eigenvalues

We don't need to calculate the Eigenfunctions, as we are only interested in calculated the averaged (exact) solution. We only
need the eigenvalues $a_{2n}$ and the Fourier Coefficients $A_{2r}^{(2n)}$.

```python
A_vals = A_coefficients(Q, M, 'even', 'one')  ## NOTE: slow since Q and M are large.
```

```python
fig, ax = plt.subplots(figsize=(20, 8))
gs = gridspec.GridSpec(3,2)
gs.update(hspace=0.05)
ax1 = plt.subplot(gs[4])
plt.plot(Q.imag, A_vals['a24'].real,'k', lw=4, label='n=12')
plt.plot(Q.imag, A_vals['a26'].real, '#808080', lw=4, label='n=13')
plt.xlim(Q[0].imag, Q[-1].imag)
# plt.ylim(0, 10)
plt.yticks(size=15)
plt.xlabel('q', fontsize=25)
plt.xticks(size=15)
plt.legend(fontsize=20, frameon=False)
ax2 = plt.subplot(gs[5])
plt.plot(Q.imag, A_vals['a24'].imag, 'k', lw=4, ls='--', label='n=12')
plt.plot(Q.imag, A_vals['a26'].imag, '#808080', lw=4, ls='--', label='n=13')
plt.xlim(Q[0].imag, Q[-1].imag)
plt.yticks(size=15)
plt.xlabel('q', fontsize=25)
plt.xticks(size=15)
ax3 = plt.subplot(gs[2])
plt.plot(Q.imag, A_vals['a28'].real, 'k', lw=4, label='n=14')
plt.plot(Q.imag, A_vals['a30'].real, '#808080', lw=4, label='n=15')
plt.xlim(Q[0].imag, Q[-1].imag)
plt.yticks(size=15)
plt.legend(fontsize=20, frameon=False)
plt.setp(ax3.get_xticklabels(), visible=False)
ax4 = plt.subplot(gs[3])
plt.plot(Q.imag, A_vals['a28'].imag, 'k', lw=4, ls='--', label='n=14')
plt.plot(Q.imag, A_vals['a30'].imag, '#808080',lw=4, ls='--', label='n=15')
plt.xlim(Q[0].imag, Q[-1].imag)
plt.yticks(size=15)
plt.setp(ax4.get_xticklabels(), visible=False)
# plt.legend(fontsize=20, frameon=False)
ax5 = plt.subplot(gs[0])
plt.plot(Q.imag, A_vals['a32'].real, 'k', lw=4, label='n=16')
plt.plot(Q.imag, A_vals['a34'].real, '#808080', lw=4, label='n=17')
plt.xlim(Q[0].imag, Q[-1].imag)
plt.yticks(size=15)
plt.legend(fontsize=20, frameon=False)
plt.setp(ax5.get_xticklabels(), visible=False)
plt.title(r'$Re(a_{2n})$', fontsize=25)
ax6 = plt.subplot(gs[1])
plt.plot(Q.imag, A_vals['a32'].imag, 'k', lw=4, ls='--', label='n=16')
plt.plot(Q.imag, A_vals['a34'].imag, '#808080', lw=4, ls='--', label='n=17')
plt.xlim(Q[0].imag, Q[-1].imag)
plt.yticks(size=15)
plt.setp(ax6.get_xticklabels(), visible=False)
plt.title(r'$Im(a_{2n})$', fontsize=25)
plt.show()
```

```python
fig, ax = plt.subplots(figsize=(20, 8))
gs = gridspec.GridSpec(3,2)
gs.update(hspace=0.05)
ax1 = plt.subplot(gs[4])
plt.plot(Q.imag, A_vals['a12'].real,'k', lw=4, label='n=6')
plt.plot(Q.imag, A_vals['a14'].real, '#808080', lw=4, label='n=7')
plt.xlim(Q[0].imag, Q[-1].imag)
# plt.ylim(0, 10)
plt.yticks(size=15)
plt.xlabel('q', fontsize=25)
plt.xticks(size=15)
plt.legend(fontsize=20, frameon=False)
ax2 = plt.subplot(gs[5])
plt.plot(Q.imag, A_vals['a12'].imag, 'k', lw=4, ls='--', label='n=6')
plt.plot(Q.imag, A_vals['a14'].imag, '#808080', lw=4, ls='--', label='n=7')
plt.xlim(Q[0].imag, Q[-1].imag)
plt.yticks(size=15)
plt.xlabel('q', fontsize=25)
plt.xticks(size=15)
ax3 = plt.subplot(gs[2])
plt.plot(Q.imag, A_vals['a16'].real, 'k', lw=4, label='n=8')
plt.plot(Q.imag, A_vals['a18'].real, '#808080', lw=4, label='n=9')
plt.xlim(Q[0].imag, Q[-1].imag)
plt.yticks(size=15)
plt.legend(fontsize=20, frameon=False)
plt.setp(ax3.get_xticklabels(), visible=False)
ax4 = plt.subplot(gs[3])
plt.plot(Q.imag, A_vals['a16'].imag, 'k', lw=4, ls='--', label='n=8')
plt.plot(Q.imag, A_vals['a18'].imag, '#808080',lw=4, ls='--', label='n=9')
plt.xlim(Q[0].imag, Q[-1].imag)
plt.yticks(size=15)
plt.setp(ax4.get_xticklabels(), visible=False)
# plt.legend(fontsize=20, frameon=False)
ax5 = plt.subplot(gs[0])
plt.plot(Q.imag, A_vals['a20'].real, 'k', lw=4, label='n=10')
plt.plot(Q.imag, A_vals['a22'].real, '#808080', lw=4, label='n=11')
plt.xlim(Q[0].imag, Q[-1].imag)
plt.yticks(size=15)
plt.legend(fontsize=20, frameon=False)
plt.setp(ax5.get_xticklabels(), visible=False)
plt.title(r'$Re(a_{2n})$', fontsize=25)
ax6 = plt.subplot(gs[1])
plt.plot(Q.imag, A_vals['a20'].imag, 'k', lw=4, ls='--', label='n=10')
plt.plot(Q.imag, A_vals['a22'].imag, '#808080', lw=4, ls='--', label='n=11')
plt.xlim(Q[0].imag, Q[-1].imag)
plt.yticks(size=15)
plt.setp(ax6.get_xticklabels(), visible=False)
plt.title(r'$Im(a_{2n})$', fontsize=25)
plt.show()
```

```python
fig, ax = plt.subplots(figsize=(20, 8))
gs = gridspec.GridSpec(3,2)
gs.update(hspace=0.05)
ax1 = plt.subplot(gs[4])
plt.plot(Q.imag, A_vals['a0'].real, color=colors[0], lw=4, label='n=0')
plt.plot(Q.imag, A_vals['a2'].real, color=colors[1], lw=4, label='n=1')
plt.xlim(0, qf)
plt.ylim(0, 10)
plt.yticks(size=15)
plt.xlabel('q', fontsize=25)
plt.xticks(size=15)
plt.legend(fontsize=20, frameon=False)
ax2 = plt.subplot(gs[5])
plt.plot(Q.imag, A_vals['a0'].imag, color=colors[0], lw=4, ls='--', label='n=0')
plt.plot(Q.imag, A_vals['a2'].imag, color=colors[1], lw=4, ls='--', label='n=1')
plt.xlim(0, qf)
plt.yticks(size=15)
plt.xlabel('q', fontsize=25)
plt.xticks(size=15)
ax3 = plt.subplot(gs[2])
plt.plot(Q.imag, A_vals['a4'].real, color=colors[2], lw=4, label='n=2')
plt.plot(Q.imag, A_vals['a6'].real, color=colors[3], lw=4, label='n=3')
plt.xlim(0, qf)
plt.yticks(size=15)
plt.legend(fontsize=20, frameon=False)
plt.setp(ax3.get_xticklabels(), visible=False)
ax4 = plt.subplot(gs[3])
plt.plot(Q.imag, A_vals['a4'].imag, color=colors[2], lw=4, ls='--', label='n=2')
plt.plot(Q.imag, A_vals['a6'].imag, color=colors[3], lw=4, ls='--', label='n=3')
plt.xlim(0, qf)
plt.yticks(size=15)
plt.setp(ax4.get_xticklabels(), visible=False)
# plt.legend(fontsize=20, frameon=False)
ax5 = plt.subplot(gs[0])
plt.plot(Q.imag, A_vals['a8'].real, color=colors[4], lw=4, label='n=4')
plt.plot(Q.imag, A_vals['a10'].real, color=colors[5], lw=4, label='n=5')
plt.xlim(0, qf)
plt.yticks(size=15)
plt.legend(fontsize=20, frameon=False)
plt.setp(ax5.get_xticklabels(), visible=False)
plt.title(r'$Re(a_{2n})$', fontsize=25)
ax6 = plt.subplot(gs[1])
plt.plot(Q.imag, A_vals['a8'].imag, color=colors[4], lw=4, ls='--', label='n=4')
plt.plot(Q.imag, A_vals['a10'].imag, color=colors[5], lw=4, ls='--', label='n=5')
plt.xlim(0, qf)
plt.yticks(size=15)
plt.setp(ax6.get_xticklabels(), visible=False)
plt.title(r'$Im(a_{2n})$', fontsize=25)
plt.show()
```

## Create Ordered Lists

```python
OM = []
AA = []
for k in range(M // 2):
    OM.append(0.25*np.copy(A_vals['a' + str(2 * k)]))
    AA.append(np.copy(A_vals['A' + str(2 * k)][:, 0]))

COS = [np.exp(K[i] * x*(1j)) for i in range(N)]
```

## Gaussian Initial condition

```python
G = np.exp(-(x/sigma)**2)
```

```python
fac = np.sqrt(np.pi)*sigma/(2*L)
arg = ((2 * L) / (np.pi*sigma))**2
cn = []
for n in range(N):
    cn.append(fac * np.exp(-K[n]**2/(arg**2)))
```

## Define a function to construct the solution

```python
def evolve_ds(AA, OM, K, cn, sigma, X, Y, t):
    """Constructs the solution to the IVP. So far, only works in case Phi=1"""
    ## Initialize the array
    coords = {"time": t, "x": X[0, :]}
    Temp = xr.DataArray(np.nan, coords=coords, dims=["time", 'x'])
    ds = xr.Dataset({'Theta': Temp})
    N = len(K)
    for i in range(len(t)):
#         print(i)
        coeff=[]
        for k in range(N):
            CE2n = [2 * (AA[r][k]**2) * np.exp(-(OM[r][k] + K[k]**2)*t[i]) for r in range(len(AA))]
            CE2n = sum(CE2n) # r-sum
            coeff.append(cn[k] * CE2n * COS[k])
        T0 = (sigma**2)*np.sum(coeff, axis=0).real # k-sum
        ds['Theta'].data[i, :] = T0
    return ds
```

## Construct solution

```python
ds = evolve_ds(AA, OM, K, cn, sigma, X, Y, t)
```

## Initial condition

```python
fig, ax = plt.subplots(figsize=(14, 8))
ds.Theta.isel(time=0).plot(lw=4)
plt.xticks(size=15)
plt.yticks(size=15)
plt.xlabel('x', fontsize=25)
plt.ylabel(r'$\theta(x, y, 0)$', fontsize=25, rotation=0, labelpad=45)
plt.show()
```

## Hovmoeller plot of (cross-channel) mean solution to the Advection Diffusion Equation

```python
T, Xt = np.meshgrid(x, t)
```

```python
cmap='nipy_spectral'
fig, ax = plt.subplots(figsize=(14, 8))
cf=plt.contourf(T, Xt, ds.Theta,  levels=np.linspace(0, 1, 1000), cmap=cmap)
plt.xticks(size=15)
plt.yticks([0, .1, .2], size=15)
plt.ylim(0, 0.2)
plt.xlabel('x', fontsize=25)
plt.xlim(-alpha * L, alpha * L)
plt.ylabel(r'$\frac{t}{t_{d}}$', fontsize=35, rotation=0, labelpad = 35)
cbaxes = fig.add_axes([0.675, 0.935, 0.225, 0.03])
clb1 = plt.colorbar(cf,cax=cbaxes,ticks=[0, 0.5, 1],orientation='horizontal')
clb1.ax.tick_params(labelsize=15)
plt.show()
```

```python

```


```python
%autosave 0
%matplotlib notebook
import numpy as np
import matplotlib.pyplot as plt
from IPython.display import display
import ipywidgets as widgets
from matplotlib import animation
from functools import partial
slider_layout = widgets.Layout(width='600px', height='20px')
slider_style = {'description_width': 'initial'}
IntSlider_nice = partial(widgets.IntSlider, style=slider_style, layout=slider_layout)
FloatSlider_nice = partial(widgets.FloatSlider, style=slider_style, layout=slider_layout)
SelSlider_nice = partial(widgets.SelectionSlider, style=slider_style, layout=slider_layout)
```

# Aprendizaje Supervisado

Esquema donde se busca aprender un mapeo o función

$$
f_{\theta}: \mathcal{X} \rightarrow \mathcal{Y},
$$

donde $\mathcal{X}$ es el dominio de nuestros datos (entrada) e $\mathcal{Y}$ es un objetivo (salida)

Entrenamos nuestro modelo a partir de un conjunto de $N$ ejemplos:

$$
\{(x_1, y_1), (x_2, y_2), \ldots, (x_N, y_N)\},
$$

donde cada ejemplo es una tupla formada de datos $x_i \in \mathcal{X}$ y objetivo $y_i \in \mathcal{Y}$

Si la variable objetivo es
- continua: hablamos de un problema de regresión o aproximación de funciones
- categórica: hablamos de un problema de clasificación

La naturaleza de los datos  depende del problema

Lo más común es que los datos $x_i$ se estructuren como arreglos de $M$ componentes

A los componentes los llamamos atributos o *features*

### Aprendizaje

El vector $\theta$ corresponde a los **parámetros** del modelo

> Aprender o ajustar el modelo corresponde a encontrar el valor "óptimo" de $\theta$

Usamos una **función de pérdida/costo** $L(\theta)$ para medir el error de nuestro modelo

> **Minimizamos** la función de costo para encontrar el mejor $\theta$

Cuando hablamos de $\theta$ "óptimo" lo decimos en el sentido de una función de costo particular

### Optimización

Queremos resolver el siguiente problema
$$
\min_\theta L(\theta)
$$

Una opción es evaluar $L()$ en todo el espacio de posibles $\theta$: Fuerza Bruta

Pero en general esto no es computacionalmente posible

Podemos usar **técnicas de optimización** para encontrar el mejor $\theta$

Si $L(\theta)$ es continua y derivable podemos escribir

$$
\nabla_\theta L(\theta) = \vec 0
$$

e intentar despejar $\theta~$

## Regresión lineal

Modelo para aprender una mapeo entre una o más variables continuas (atributos) hacia una variable continua (objetivo)

En un esquema con $M$ atributos y $N$ ejemplos tenemos

$$
y_i = f_\theta(\vec x_i) = \vec w^T \vec x_i + b = \sum_{j=1}^M w_j x_{ij} + b,
$$

donde $f_\theta$ es un modelo parámetrico (hiperplano) con $M+1$ parámetros

$$
\theta= \begin{pmatrix} b \\ w_1 \\ w_2 \\ \vdots \\ w_M \end{pmatrix}
$$

También podemos escribir el sistema matricialmente como

$$
Y = X \theta
$$

donde $Y= \begin{pmatrix} y_1 \\ y_2 \\ \vdots \\y_N\end{pmatrix} \in \mathbb{R}^N$, $X = \begin{pmatrix} 1 & x_{11} & x_{12}& \ldots& x_{1M} \\ \vdots & \vdots & \vdots& \ddots& \vdots \\ 1 & x_{N1} & x_{N2}& \ldots& x_{NM} \\ \end{pmatrix} \in \mathbb{R}^{N\times M}$ y $\theta \in \mathbb{R}^M$

Una función de costo razonable es

$$
L(\theta) = \frac{1}{2} (Y - X\theta)^T (Y - X\theta)
$$

que corresponde al cuadrado de los errores

> Esta función de costo se conoce como Error Cuadrático Medio

Luego si derivamos e igualamos a cero obtenemos

$$
\nabla_\theta L(\theta) = -X^T(Y-X\theta) = 0,
$$

y despejando

$$
\hat \theta = (X^T X)^{-1} X^T Y
$$

siempre y cuando podamos invertir $X^T X$

> Esto se conoce como solución de mínimos cuadrados

### Regresión lineal con funciones base

Podemos generalizar el regresor lineal aplicando transformaciones a $X$

Por ejemplo una regresión polinomial de grado $M$ sería

$$
y_i = f_\theta(x_i) = \sum_{j=1}^M w_j x_{i}^j + b,
$$

y su solución sería

$$
\hat \theta = (\Phi^T \Phi)^{-1} \Phi^T Y,
$$

donde $\Phi = \begin{pmatrix} 1 & x_1 & x_1^2& \ldots& x_1^M \\ \vdots & \vdots & \vdots& \ddots& \vdots \\ 1 & x_N & x_N^2& \ldots& x_N^M \\ \end{pmatrix}$

### Ejemplo

Considere el siguiente conjunto de datos $X,Y$ sintéticos con $Y$ continuo separados en conjunto de entrenamiento y validación

Usaremos $2/3$ de la data para entrenar y $1/3$ para validar

La función anónima `poly_basis` recibe $X$ y retorna $\Phi$

```python
poly_basis = lambda x, M : np.vstack([x**k for k in range(M)]).T
theta_real = np.array([10, -2, -0.3, 0.1])

x = np.linspace(-5, 6, num=21);
X = poly_basis(x, len(theta_real))
y = np.dot(X, theta_real)

rseed, sigma = 0, 1.
np.random.seed(rseed);
Y = y + sigma*np.random.randn(len(x))
P = np.random.permutation(len(x))
train_idx, valid_idx = P[:2*len(x)//3], P[2*len(x)//3:]
```

La función `np.linalg.lstsq` recibe $\Phi$ e $Y$ y retorna el resultado de mínimos cuadrados $\hat \theta$

El siguiente ejemplo estudia el resultado de la regresión usando distintos valores de $M$, es decir el grado del polinomio

- $M=0 \rightarrow$  constante
- $M=1 \rightarrow$  recta
- $M=2 \rightarrow$  parábola
- ...

En este ejemplo
- El "modelo real" es el que se usó para generar los datos. Normalmente no está disponible.
- El "modelo aprendido" es el que ajustamos con la regresión polinomial. Idealmente queremos que este modelo se parezca al "modelo real"

Modifica $M$ y analiza los resultados del ajuste

```python
plt.close('all'); fig, ax = plt.subplots(figsize=(6, 4), tight_layout=True)

def update_plot(ax, M):
    ax.cla();
    Phi = poly_basis(x, M)
    theta_hat = np.linalg.lstsq(Phi[train_idx, :], Y[train_idx], rcond=None)[0]
    ax.plot(x, y, 'g-', linewidth=2, label='Modelo real', alpha=0.6, zorder=-100)
    ax.scatter(x[train_idx], Y[train_idx], s=50, label='Entrenamiento')
    ax.scatter(x[valid_idx], Y[valid_idx], s=50, label='Validación')
    ax.vlines(x[train_idx], np.dot(Phi[train_idx, :], theta_hat), Y[train_idx])
    ax.vlines(x[valid_idx], np.dot(Phi[valid_idx, :], theta_hat), Y[valid_idx])
    x_plot = np.linspace(-5, 6, num=100);
    ax.plot(x_plot, np.dot(poly_basis(x_plot, M), theta_hat), 'k-', linewidth=2, label='Modelo aprendido')
    ax.set_ylim([-5, 15]); plt.legend()

widgets.interact(update_plot,
                 ax=widgets.fixed(ax),
                 M=IntSlider_nice(description='M: grado del polinomio', min=1, max=11));
```

# Complejidad y Sobreajuste

En el ejemplo anterior vimos que se puede obtener modelos más flexibles si aumentamos el grado del polinomio

> Aumentar la cantidad de parámetros (grados de libertad) hace al modelo más flexible y más complejo

Si la flexibilidad es excesiva aproximamos los datos con cero error

Esto no es bueno ya que estamos aprendiendo "de memoria" los datos y ajustandonos al ruido

> Sobreajuste: Aprender perfectamente los datos usados para entrenar

El modelo sobreajustado predice muy mal los datos "que no ha visto"

> El sobreajuste es inversamente proporcional a la capacidad de generalización

Por esta razón usamos conjuntos de validación

Figura: https://www.d2l.ai/chapter_multilayer-perceptrons/underfit-overfit.html

# Representatividad y Validación

En Machine Learning el primer paso para entrenar nuestro modelo es obtener datos

Es crítico que los datos que utilizemos **representen** adecuadamente el problema que queremos resolver

Sea un espacio de datos (circulo negro) y muestras (puntos azules), ¿que puede decir de los siguientes casos?

```python
from matplotlib.patches import Circle
fig, ax = plt.subplots(1, 2, figsize=(7, 3))
np.random.seed(19)
for ax_ in ax:
    ax_.axis('off')
    ax_.set_xlim([-3, 3])
    ax_.set_ylim([-3, 3])
r = 2.5*np.random.rand(100); t = np.random.rand(100);
ax[0].scatter(r*np.cos(2.0*np.pi*t), r*np.sin(2.0*np.pi*t))
p = Circle((0, 0), 3, fill=False, ec='k')
ax[0].add_artist(p)
r = 2.5*np.random.rand(100); t = np.random.rand(100);
ax[1].scatter(r*np.cos(t), r*np.sin(t))
p = Circle((0, 0), 3, fill=False, ec='k')
ax[1].add_artist(p);
```

Siempre que podamos controlar el proceso de muestreo debemos poner atención a evitar **sesgos**

Asumiendo que nuestro dataset es representativo el siguiente paso es **entrenar**

Para combatir el sobreajuste podemos usar **estrategias de validación**

Consisten en separar el conjunto en dos o más subconjuntos
- Holdout: Entrenamiento/Validación/Prueba
- K-fold cross-validation y Leave one-out (N-fold) cross-validation
- Versiones estratificadas/balanceadas de las anteriores

Para que nuestro conjuntos de entrenamiento y validación sigan siendo representativos del total los **seleccionamos aleatoriamente**

Medimos $L(\theta)$ en entrenamiento y validación

> Optimizamos nuestro modelo minimizando el error de entrenamiento

> Seleccionamos los parámetros e hiper-parámetros que dan mínimo error de validación

> Comparamos distintas familias de modelos  con el error de prueba

### Ejemplo

Apliquemos esto al ejemplo de regresión polinomial

¿Cómo varía el error de entrenamiento y validación con $M$?

```python
fig, ax = plt.subplots(figsize=(6, 4), tight_layout=True)
M_values = np.arange(1, 11)
mse = np.zeros(shape=(len(M_values), 2))
for i, M in enumerate(M_values):
    Phi = poly_basis(x, M)
    theta_hat = np.linalg.lstsq(Phi[train_idx, :], Y[train_idx], rcond=None)[0]
    mse[i, 0] = np.mean(np.power(Y[train_idx] - np.dot(Phi[train_idx, :], theta_hat), 2))
    mse[i, 1] = np.mean(np.power(Y[valid_idx] - np.dot(Phi[valid_idx, :], theta_hat), 2))
ax.plot(M_values, mse[:, 0], label='Entrenamiento')
ax.plot(M_values, mse[:, 1], label='Validación')
plt.legend()
ax.set_ylim([1e-4, 1e+4])
ax.set_yscale('log')
ax.set_xlabel('Grado del polinomio')
ax.set_ylabel('Loss');

# El punto representa el mejor M de validación
idx_best = np.argmin(mse[:, 1])
ax.scatter(M_values[idx_best], mse[idx_best, 1], c='k', s=100);
```

En resumen

- Bajo error de entrenamiento y de validación: **Ideal**

- Bajo error de entrenamiento y alto error de validación: **Modelo sobreajustado**

- Alto error de entrenamiento y de validación: Considera otro modelo y/o revisa tu código

## Neurona artificial o regresor logístico

Es un modelo para aprender un mapeo desde una o más variables continuas (atributos) hacia una variable binaria (objetivo)

Definamos primero la función sigmoide o logística

$$
\mathcal{S}(z) = \frac{1}{1+\exp(-z)}
$$

Esta función mapea un variable real al espacio $[0, 1]$

```python
fig, ax = plt.subplots(figsize=(5, 2.5), tight_layout=True)

sigmoid = lambda z : 1./(1+np.exp(-z))
z = np.linspace(-6, 6, num=1000)
ax.plot(z, sigmoid(z))
dot = ax.plot(0, sigmoid(0), 'ko')

def update_plot(z):
    dot[0].set_xdata(z)
    dot[0].set_ydata(sigmoid(z))

widgets.interact(update_plot, z=FloatSlider_nice(min=-5, max=5, value=0));
```

Luego definamos la siguiente transformación lineal de los datos de entrada

$$
z_i = \theta_0 + \sum_{j=1}^M \theta_j x_{ij}
$$

El modelo de regresión logística aplica la función sigmoide a la transformación lineal

$$
f_\theta(\vec x_i) = \mathcal{S} \left(\theta_0 + \sum_{j=1}^M \theta_j x_{ij} \right)
$$

La salida del regresor está en el intervalo $[0,1]$, podemos interpretarla como la probabilidad de "pertenecer" a la clase $1$ (o la probabilidad de "no pertenecer" a la clase 0)

> Modelo de clasificación binaria (dos clases)

### Ejemplo

Considera los siguientes datos sintéticos

```python
np.random.seed(1234)
data = np.concatenate((np.random.randn(50, 2),
                       2 + np.random.randn(50, 2)), axis=0)
#data = data - np.mean(data, axis=0, keepdims=True)
label = np.array([0]*50 + [1]*50)

fig, ax = plt.subplots(1, figsize=(6, 3), tight_layout=True)
ax.scatter(data[label==0, 0], data[label==0, 1], c='k', s=20, label='clase 0')
ax.scatter(data[label==1, 0], data[label==1, 1], c='k', s=20, marker='x', label='clase 1')
plt.legend();
```

El siguiente ejemplo permite modificar los parámetros $\theta_0$, $\theta_1$ y $\theta_2$ de un regresor logístico

> ¿Puedes obtener un plano que separé las clases lo mejor posible?

```python
fig, ax = plt.subplots(1, figsize=(7, 3), tight_layout=True)

from matplotlib import cm
fig.colorbar(cm.ScalarMappable(cmap=plt.cm.RdBu_r), ax=ax)
x_min, x_max = data[:, 0].min() - 0.5, data[:, 0].max() + 0.5
y_min, y_max = data[:, 1].min() - 0.5, data[:, 1].max() + 0.5
xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.05), np.arange(y_min, y_max, 0.05))

def sigmoid(X, w, b):
    Z = np.dot(X, w) + b # Transformación lineal
    return 1./(1 + np.exp(-Z)) # Función sigmoide

def update_plot(t1, t2, t0):
    ax.cla()
    ax.contourf(xx, yy, sigmoid(np.c_[xx.ravel(), yy.ravel()], np.array([t1, t2]), t0).reshape(xx.shape),
                cmap=plt.cm.RdBu_r, alpha=0.75)
    ax.scatter(data[label==0, 0], data[label==0, 1], c='k', s=20)
    ax.scatter(data[label==1, 0], data[label==1, 1], c='k', s=20, marker='x')

widgets.interact(update_plot,
                 t1=FloatSlider_nice(value=-1, min=-10, max=10),
                 t2=FloatSlider_nice(value=1, min=-10, max=10),
                 t0=FloatSlider_nice(value=0, min=-10, max=10));
```

## Función de costo para clasificación binaria

La función de costo del regresor lineal no es apropiada ya que queremos comparar categorías discretas (clases)

Para un problema de clasificación binario podemos usar la **Entropía Cruzada Binaria**

$$
L(\theta) = \sum_{i=1}^N  -y_i \log( f_\theta(\vec x_i) ) - (1-y_i) \log(1 - f_\theta(\vec x_i))
$$

Calculemos el gradiente de la entropía cruzada

$$
\begin{align}
\frac{d}{d \theta_j} L(\theta) &= \sum_{i=1}^N  \left(-\frac{y_i}{f_\theta(\vec x_i)} + \frac{1-y_i}{1 - f_\theta(\vec x_i)}\right) \frac{d f_\theta(\vec x_i)}{d\theta_j} \nonumber \\
&= -\sum_{i=1}^N (y_i - f_\theta(\vec x_i)) \frac{d z_i}{d\theta_j} \nonumber \\
&= -\sum_{i=1}^N \left(y_i -  \mathcal{S} \left(z_i\right)  \right) x_{ij} \nonumber
\end{align}
$$

donde se uso que

$$
\frac{d \mathcal{S}(z)}{dz} = \mathcal{S}(z) (1- \mathcal{S}(z))
$$

Si observamos

$$
\frac{d}{d \theta_j} L(\theta) = -\sum_{i=1}^N \left(y_i -  \mathcal{S} \left(z_i\right)  \right) x_{ij}
$$

podemos notar que no es posible despejar analiticamente $\theta_j$ como en el caso de la regresión lineal. Esto se debe a la no-linealdad que introduce la función sigmoide

> Un opción es proponer una $\theta$ inicial y mejorarlo interativamente: Métodos de optimización iterativos

## Optimización: Método de Newton

Sea el valor actual del vector de parámetros $\theta_t$

Queremos encontrar el mejor "próximo valor" según nuestra función objetivo
$$
\theta_{t+1} = \theta_t + \Delta \theta
$$
Consideremos la aproximación de Taylor de segundo orden de $f$
$$
f(\theta_{t} + \Delta \theta) \approx f(\theta_t) + \nabla f (\theta_t) \Delta \theta + \frac{1}{2} \Delta \theta^T H_f (\theta_t) \Delta \theta
$$
Derivando en función de $\Delta \theta$ e igualando a cero tenemos
$$
\begin{align}
\nabla f (\theta_t)  +  H_f (\theta_t) \Delta \theta &= 0 \nonumber \\
\Delta \theta &= - [H_f (\theta_t)]^{-1}\nabla f (\theta_t)  \nonumber \\
\theta_{t+1} &= \theta_{t} - [H_f (\theta_t)]^{-1}\nabla f (\theta_t)  \nonumber \\
\end{align}
$$

- Se obtiene una regla iterativa en función del **Gradiente** y del **Hessiano**
- La solución depende de $\theta_0$
- "Asumimos" que la aproximación de segundo orden es "buena"
- Si nuestro modelo tiene $M$ parámetros el Hessiano es de $M\times M$, ¿Qué pasa si $M$ es grande?

## Optimización: Gradiente descendente

Muchas veces el Hessiano es prohibitivo

Podemos sacrificar calidad y reducir cómputo usando una aproximación de primer orden

El método más clásico es el **gradiente descendente**

$$
\theta_{t+1} = \theta_{t} - \eta \nabla f (\theta_t)
$$

donde hemos reemplazado el Hessiano por una constante $\eta$ llamado "paso" o "tasa de aprendizaje"

- ¿Cómo cambia la optimización con distintos $\eta$?
- ¿Qué ocurre cuando la superficie de error tiene mínimos locales?

### Ejercicio

- Considere la función $5 + (\theta -1)^2$, calcule su gradiente con respecto a $\theta$, complete la función `df` con su solución
- Repita para la función $5 + (\theta -1)^2 + 10\sin(\theta)$

```python
f = lambda theta : 5+ (theta-1.)**2 #+ 10*np.sin(theta)
df = 0 # Completa aquí!
```

- Estudie como se optimiza un conjunto de soluciones iniciales usando distintos valores de la tasa de aprendizaje $\eta$
- Considere las funciones de costo anterior por separado

```python
plt.close('all'); fig, ax = plt.subplots(2, figsize=(7, 4), tight_layout=True, sharex=True)
x = np.linspace(-4, 6, num=100)

t = 10*np.random.rand(10) - 4. # valores iniciales de theta
ax[0].plot(x, f(x))
sc = ax[0].scatter(t, f(t), s=100)
ax[1].plot(x, -df(x))
ax[1].set_xlabel(r'$\theta$')
ax[0].set_ylabel(r'$f(\theta)$')
ax[1].set_ylabel(r'$-\nabla f(\theta)$')

eta = 0.01 # tasa de aprendizaje

def update(n):
    t = sc.get_offsets()[:, 0]
    t -= eta*df(t) # Gradiente descendente
    sc.set_offsets(np.c_[t, f(t)])

anim = animation.FuncAnimation(fig, update, frames=100, interval=200, repeat=False, blit=True)
```

## Gradiente descendente para el regresor logístico

Ajustemos el regresor del ejemplo anterior usando gradiente descedente

```python
fig, ax = plt.subplots(1, 2, figsize=(9, 3), tight_layout=True)
fig.colorbar(cm.ScalarMappable(cmap=plt.cm.RdBu_r), ax=ax[0])

def sigmoid(X, theta):
    Z = np.dot(X, theta[1:]) + theta[0] # Transformación lineal
    return 1./(1 + np.exp(-Z)) # Función sigmoide

def loss(theta, X, Y, tol=1e-8): # Función de costo
    f = sigmoid(X, theta)
    return -np.sum(Y*np.log(f + tol) + (1.-Y)*np.log(1.-f+tol))

def grad_loss(theta, X, Y): # Gradiente
    f = sigmoid(X, theta)
    return -np.dot(Y-f, np.hstack((np.ones(shape=(len(data), 1)), data)))

eta = 1e-2 # Tasa de aprendizaje
np.random.seed(1234)
t = 0.5*np.random.randn(3) # Solución inicial
L_history = []

def update_plot(k):
    global t, L_history
    L_history.append(loss(t, data, label)) # Valor histórico de la loss

    gradL = grad_loss(t, data, label)
    t -= eta*gradL # Gradiente descendente

    ax[0].cla()
    ax[0].contourf(xx, yy, sigmoid(np.c_[xx.ravel(), yy.ravel()], t).reshape(xx.shape),
                cmap=plt.cm.RdBu_r, alpha=0.75)
    ax[0].scatter(data[label==0, 0], data[label==0, 1], c='k', s=20)
    ax[0].scatter(data[label==1, 0], data[label==1, 1], c='k', s=20, marker='x')
    ax[1].cla()
    ax[1].plot(L_history)

anim = animation.FuncAnimation(fig, update_plot, frames=50, interval=250, repeat=False, blit=True)
```

# Métricas: Evaluando un clasificador binario

La salida de este clasificador es un valor en el rango $[0, 1]$

Para tomar un decisión binaria se debe seleccionar un umbral $\mathcal{T}$ tal que

$$
d_i =
\begin{cases}
0, & \text{si } f_\theta(\vec x_i)  < \mathcal{T} \\
1, & \text{si } f_\theta(\vec x_i) \geq \mathcal{T}
\end{cases}
$$

Una vez seleccionado el umbral se puede contar la cantidad de
- **True positives** (TP): Era clase (1) y lo clasifico como (1)
- **True negative** (TN): Era clase (0) y lo clasifico como (0)
- **False positives** (FP): Era clase (0) y lo clasifico como (1): Error tipo I
- **False negative** (FN): Era clase (1) y lo clasifico como (0): Error tipo II

A partir de estas métricas se construye la **matriz de confusión** del clasificador

|Clasificado como/En realidad era|Positivo|Negativo|
|---|---|---|
|Positivo:|TP | FP |
|Negativo:| FN | TN |

En base a estas métricas se construyen otras

$$
\text{Recall} = \frac{TP}{TP + FN}
$$

también conocida como la **Tasa de verdaderos positivos** (TPR) o sensitividad

> TPR: La proporción de positivos correctamente clasificados respecto al total de positivos

$$
\text{FPR} = \frac{FP}{TN + FP} = 1 - \frac{TN}{TN + FP}
$$

la **tasa de falsos positivos** (FPR) también representada como "1 - especificidad"

> FPR: La proporción de negativos incorrectamente clasificados respecto al total de negativos

$$
\text{Precision} = \frac{TP}{TP + FP}
$$

también conocido como pureza

> Precision: La proporción de positivos correctamente clasificados respecto a todos los ejemplos clasificados como positivo

$$
\text{Accuracy} = \frac{TP+TN}{TP + FP + FN+ TN}
$$

> Accuracy: La proporción de ejemplos correctamente clasificados

$$
\text{f1-score} = \frac{2\cdot\text{Recall}\cdot\text{Precision}}{\text{Recall} + \text{Precision}}
$$

> f1-score: Media armónica entre Recall y Precision asumiendo igual ponderación

Si las clases son desbalanceadas entonces f1-score es más aconsejable que accuracy

### Ejemplo: Evaluando el regresor logístico

Podemos usar las funciones de `sklearn.metrics` para obtener matrices de confusión, reportes y curvas de desempeño

```python
from sklearn.metrics import confusion_matrix, classification_report

probability = sigmoid(data, t)

display("Matriz de confusión:")
display(confusion_matrix(y_true=label, y_pred=probability>0.5))

display("Matriz de confusión:")
print(classification_report(y_true=label, y_pred=probability>0.5))
```

```python
from sklearn.metrics import roc_curve, precision_recall_curve, auc

fpr, tpr, th = roc_curve(y_true=label, y_score=probability)

fig, ax = plt.subplots(1, 2, figsize=(7, 3), tight_layout=True)
ax[0].plot(fpr, tpr);
ax[0].set_xlabel('Tasa de Falsos positivos')
ax[0].set_ylabel('Tasa de Verdaderos\nPositivos (Recall)')
ax[0].set_title(f'Curva ROC AUC: {auc(fpr, tpr)}')

prec, rec, th = precision_recall_curve(y_true=label, probas_pred=probability)

ax[1].plot(rec, prec, '-');
ax[1].set_xlabel('Recall')
ax[1].set_ylabel('Precision')
ax[1].set_title('Curva PR');
```

# Ojo con:

#### La distribución de los datos donde se aplicará el clasificador es distinta a la que usaste para entrenar/validar

Actualiza tus conjuntos de datos para que sean representativos!

#### Usa los subconjuntos adecuadamente

Ajusta los parámetros con el set de validación

Compara distintas familias de modelos con el set de prueba

#### La métrica que usas no es la adecuada para el problema

Si el problema tiene clases desbalanceadas el *accuracy* puede ser muy alto, contrasta usando métricas sencibles al desbalance (e.g. *f1-score*)

#### Mi modelo se sobreajusta de inmediato

Prueba disminuyendo la complejidad/arquitectura del modelo o añadiendo **regularización**

Esto también puede ser señal de que necesitas más ejemplos para entrenar

Se pueden usar **Técnicas de aumentación** de datos

#### Mi modelo no está aprendiendo

Si estas seguro que no hay bugs prueba aumentando la complejidad del modelo

#### Estudia los errores de tu modelo para mejorarlo

Analiza los datos mal clasificados y busca patrones

Revisa que las etiquetas estén correctas

Revisa que los atributos estén adecuadamente calculados

Propon nuevos atributos que ayuden a clasificador los ejemplos difíciles

```python

```


# Multi-output Gaussian processes in GPflow

This notebook shows how to construct a multi-output GP model using GPflow, together with different interdomain inducing variables which lead to different approximation properties. GPflow provides a framework for specifying multioutput GP priors, and interdomain approximations which is
- modular, by providing a consistent interface for the user of the resulting `SVGP` model,
- extensible, by allowing new interdomain variables and kernels to be specified while reusing exising code where possible,
- efficient, by allowing the most efficient custom code path to be specified where desired.

Getting to grips with the maths and code can be a bit daunting, so to accompany the documentation there is an [in-depth review on arXiv](https://arxiv.org/abs/2003.01115), which provides a unified mathematical framework, together with a high-level description of software design choices in GPflow.

This notebook shows the various design choices that can be made, to show the reader the flexibility of the framework. This is done in the hope that an example is provided that can be easily adapted to the special case that the reader wants to implement.

A reader who just wants to use a multioutput kernel should simply choose the most efficient set of inducing variables.

To cite this framework, please reference our [arXiv paper](https://arxiv.org/abs/2003.01115).
```
@article{GPflow2020multioutput,
  author = {{van der Wilk}, Mark and Dutordoir, Vincent and John, ST and
            Artemev, Artem and Adam, Vincent and Hensman, James},
  title = {A Framework for Interdomain and Multioutput {G}aussian Processes},
  year = {2020},
  journal = {arXiv:2003.01115},
  url = {https://arxiv.org/abs/2003.01115}
}
```

\begin{equation}
\newcommand{\GP}{\mathcal{GP}}
\newcommand{\NN}{\mathcal{N}}
\newcommand{\LL}{\mathcal{L}}
\newcommand{\RR}{\mathbb{R}}
\newcommand{\EE}{\mathbb{E}}
\newcommand{\valpha}{\boldsymbol\alpha}
\newcommand{\vf}{\mathbf{f}}
\newcommand{\vF}{\mathbf{F}}
\newcommand{\vg}{\mathbf{g}}
\newcommand{\vW}{\mathbf{W}}
\newcommand{\vI}{\mathbf{I}}
\newcommand{\vZ}{\mathbf{Z}}
\newcommand{\vu}{\mathbf{u}}
\newcommand{\vU}{\mathbf{U}}
\newcommand{\vX}{\mathbf{X}}
\newcommand{\vY}{\mathbf{Y}}
\newcommand{\identity}{\mathbb{I}}
\end{equation}

## Task
We will consider a regression problem for functions $f: \mathbb{R}^D \rightarrow \mathbb{R}^P$. We assume that the dataset is of the form $(X, f_1), \dots, (X, f_P)$, that is, we observe all the outputs for a particular input location (for cases where there are **not** fully observed outputs for each input, see [A simple demonstration of coregionalization](./coregionalisation.ipynb)).

Here we assume a model of the form:
\begin{equation}
f(x) = W g(x),
\end{equation}
where $g(x) \in \mathbb{R}^L$, $f(x) \in \mathbb{R}^P$ and $W \in \mathbb{R}^{P \times L}$. We assume that the outputs of $g$ are uncorrelated, and that by *mixing* them with $W$ they become correlated. In this notebook, we show how to build this model using Sparse Variational Gaussian Process (SVGP) for $g$, which scales well with the numbers of data points and outputs.

Here we have two options for $g$:
1. The output dimensions of $g$ share the same kernel.
1. Each output of $g$ has a separate kernel.

In addition, we have two further suboptions for the inducing inputs of $g$:
1. The instances of $g$ share the same inducing inputs.
1. Each output of $g$ has its own set of inducing inputs.

The notation is as follows:
- $X \in \mathbb{R}^{N \times D}$ denotes the input
- $Y \in \RR^{N \times P}$ denotes the output
- $k_{1..L}$, $L$ are kernels on $\RR^{N \times D}$
- $g_{1..L}$, $L$ are independent $\GP$s  with $g_l \sim \GP(0,k_l)$
- $f_{1..P}$, $P$ are correlated  $\GP$s  with $\vf = \vW \vg$

```python
import numpy as np
import matplotlib.pyplot as plt
import gpflow as gpf
import tensorflow as tf

from gpflow.utilities import print_summary
from gpflow.ci_utils import ci_niter

gpf.config.set_default_float(np.float64)
gpf.config.set_default_summary_fmt("notebook")
np.random.seed(0)
%matplotlib inline

MAXITER = ci_niter(2000)
```

## Generate synthetic data
We create a utility function to generate synthetic data. We assume that:

```python
N = 100  # number of points
D = 1  # number of input dimensions
M = 15  # number of inducing points
L = 2  # number of latent GPs
P = 3  # number of observations = output dimensions
```

```python
def generate_data(N=100):
    X = np.random.rand(N)[:, None] * 10 - 5  # Inputs = N x D
    G = np.hstack((0.5 * np.sin(3 * X) + X, 3.0 * np.cos(X) - X))  # G = N x L
    W = np.array([[0.5, -0.3, 1.5], [-0.4, 0.43, 0.0]])  # L x P
    F = np.matmul(G, W)  # N x P
    Y = F + np.random.randn(*F.shape) * [0.2, 0.2, 0.2]

    return X, Y
```

```python
X, Y = data = generate_data(N)
Zinit = np.linspace(-5, 5, M)[:, None]
```

We create a utility function for plotting:

```python
def plot_model(m, lower=-8.0, upper=8.0):
    pX = np.linspace(lower, upper, 100)[:, None]
    pY, pYv = m.predict_y(pX)
    if pY.ndim == 3:
        pY = pY[:, 0, :]
    plt.plot(X, Y, "x")
    plt.gca().set_prop_cycle(None)
    plt.plot(pX, pY)
    for i in range(pY.shape[1]):
        top = pY[:, i] + 2.0 * pYv[:, i] ** 0.5
        bot = pY[:, i] - 2.0 * pYv[:, i] ** 0.5
        plt.fill_between(pX[:, 0], top, bot, alpha=0.3)
    plt.xlabel("X")
    plt.ylabel("f")
    plt.title(f"ELBO: {m.elbo(data):.3}")
    plt.plot(Z, Z * 0.0, "o")
```

## Model the outputs of $f(x)$ directly
The three following examples show how to model the outputs of the model $f(x)$ directly. Mathematically, this case is equivalent to having:
\begin{equation}
f(x) = I g(x),
\end{equation}
i.e. $W = I$ and $P = L$.

### 1. Shared independent multi-output kernel (MOK) and shared independent inducing variables
Here the priors on all outputs are constrained to have the same kernel hyperparameters. We also share the inducing inputs between all outputs. The different GPs are independent both in the prior and the approximate posterior.

```python
# create multi-output kernel
kernel = gpf.kernels.SharedIndependent(
    gpf.kernels.SquaredExponential() + gpf.kernels.Linear(), output_dim=P
)
# initialization of inducing input locations (M random points from the training inputs)
Z = Zinit.copy()
# create multi-output inducing variables from Z
iv = gpf.inducing_variables.SharedIndependentInducingVariables(
    gpf.inducing_variables.InducingPoints(Z)
)
```

```python
# create SVGP model as usual and optimize
m = gpf.models.SVGP(kernel, gpf.likelihoods.Gaussian(), inducing_variable=iv, num_latent_gps=P)
print_summary(m)
```

name                                      	class    	transform       	prior  	trainable  	shape      	dtype  	value

SVGP.kernel.kernel.kernels[0].variance    	Parameter	Softplus        	       	True       	()         	float64	1.0

SVGP.kernel.kernel.kernels[0].lengthscales	Parameter	Softplus        	       	True       	()         	float64	1.0

SVGP.kernel.kernel.kernels[1].variance    	Parameter	Softplus        	       	True       	()         	float64	1.0

SVGP.likelihood.variance                  	Parameter	Softplus + Shift	       	True       	()         	float64	1.0

SVGP.inducing_variable.inducing_variable.Z	Parameter	                	       	True       	(15, 1)    	float64	[[-5....

SVGP.q_mu                                 	Parameter	                	       	True       	(15, 3)    	float64	[[0., 0., 0....

SVGP.q_sqrt                               	Parameter	FillTriangular  	       	True       	(3, 15, 15)	float64	[[[1., 0., 0....

```python
def optimize_model_with_scipy(model):
    optimizer = gpf.optimizers.Scipy()
    optimizer.minimize(
        model.training_loss_closure(data),
        variables=model.trainable_variables,
        method="l-bfgs-b",
        options={"disp": True, "maxiter": MAXITER},
    )

optimize_model_with_scipy(m)
```

```python
print_summary(m)
```

name                                      	class    	transform       	prior  	trainable  	shape      	dtype  	value

SVGP.kernel.kernel.kernels[0].variance    	Parameter	Softplus        	       	True       	()         	float64	0.9282983778062053

SVGP.kernel.kernel.kernels[0].lengthscales	Parameter	Softplus        	       	True       	()         	float64	0.7983434071745613

SVGP.kernel.kernel.kernels[1].variance    	Parameter	Softplus        	       	True       	()         	float64	1.2108813104578107

SVGP.likelihood.variance                  	Parameter	Softplus + Shift	       	True       	()         	float64	0.03966202904211286

SVGP.inducing_variable.inducing_variable.Z	Parameter	                	       	True       	(15, 1)    	float64	[[-4.83952592...

SVGP.q_mu                                 	Parameter	                	       	True       	(15, 3)    	float64	[[-0.90201946, 0.69934369, -1.47565105...

SVGP.q_sqrt                               	Parameter	FillTriangular  	       	True       	(3, 15, 15)	float64	[[[1.71316406e-02, 0.00000000e+00, 0.00000000e+00...

```python
# Plot predictions and observations
plot_model(m)
```

```python
print_summary(m.kernel)
m.kernel.kernel.kernels[0].lengthscales
```

name                                            	class    	transform  	prior  	trainable  	shape  	dtype  	   value

SharedIndependent.kernel.kernels[0].variance    	Parameter	Softplus   	       	True       	()     	float64	0.928298

SharedIndependent.kernel.kernels[0].lengthscales	Parameter	Softplus   	       	True       	()     	float64	0.798343

SharedIndependent.kernel.kernels[1].variance    	Parameter	Softplus   	       	True       	()     	float64	1.21088

    <gpflow.Parameter 'Variable:0' dtype=float64 unconstrained-shape=() unconstrained-value=0.20037196803209242 constrained-shape=() constrained-value=0.7983434071745613>

### 2. Separate independent MOK and shared independent inducing variables
Here we allow different hyperparameters for the priors of each output. We still share the inducing inputs between all outputs.

```python
# Create list of kernels for each output
kern_list = [gpf.kernels.SquaredExponential() + gpf.kernels.Linear() for _ in range(P)]
# Create multi-output kernel from kernel list
kernel = gpf.kernels.SeparateIndependent(kern_list)
# initialization of inducing input locations (M random points from the training inputs)
Z = Zinit.copy()
# create multi-output inducing variables from Z
iv = gpf.inducing_variables.SharedIndependentInducingVariables(
    gpf.inducing_variables.InducingPoints(Z)
)
```

```python
# create SVGP model as usual and optimize
m = gpf.models.SVGP(kernel, gpf.likelihoods.Gaussian(), inducing_variable=iv, num_latent_gps=P)
```

```python
optimize_model_with_scipy(m)
```

```python
print_summary(m.kernel)
```

name                                                  	class    	transform  	prior  	trainable  	shape  	dtype  	   value

SeparateIndependent.kernels[0].kernels[0].variance    	Parameter	Softplus   	       	True       	()     	float64	0.863243

SeparateIndependent.kernels[0].kernels[0].lengthscales	Parameter	Softplus   	       	True       	()     	float64	0.93434

SeparateIndependent.kernels[0].kernels[1].variance    	Parameter	Softplus   	       	True       	()     	float64	0.863065

SeparateIndependent.kernels[1].kernels[0].variance    	Parameter	Softplus   	       	True       	()     	float64	0.751096

SeparateIndependent.kernels[1].kernels[0].lengthscales	Parameter	Softplus   	       	True       	()     	float64	0.970686

SeparateIndependent.kernels[1].kernels[1].variance    	Parameter	Softplus   	       	True       	()     	float64	0.530254

SeparateIndependent.kernels[2].kernels[0].variance    	Parameter	Softplus   	       	True       	()     	float64	1.10592

SeparateIndependent.kernels[2].kernels[0].lengthscales	Parameter	Softplus   	       	True       	()     	float64	0.748322

SeparateIndependent.kernels[2].kernels[1].variance    	Parameter	Softplus   	       	True       	()     	float64	2.26849

```python
plot_model(m)
```

```python
[k.kernels[0].lengthscales for k in m.kernel.kernels]
```

    [<gpflow.Parameter 'Variable:0' dtype=float64 unconstrained-shape=() unconstrained-value=0.4353688299101709 constrained-shape=() constrained-value=0.9343400554372966>,
     <gpflow.Parameter 'Variable:0' dtype=float64 unconstrained-shape=() unconstrained-value=0.4945470958443274 constrained-shape=() constrained-value=0.970686268477486>,
     <gpflow.Parameter 'Variable:0' dtype=float64 unconstrained-shape=() unconstrained-value=0.1074634330801334 constrained-shape=() constrained-value=0.748321751704559>]

### 3. Separate independent kernel and separate independent inducing variables
Here we allow different hyperparameters for the priors of each output. We now allow different inducing inputs for each output.

```python
# Create list of kernels for each output
kern_list = [gpf.kernels.SquaredExponential() + gpf.kernels.Linear() for _ in range(P)]
# Create multi-output kernel from kernel list
kernel = gpf.kernels.SeparateIndependent(kern_list)
# initialization of inducing input locations, one set of locations per output
Zs = [Zinit.copy() for _ in range(P)]
# initialize as list inducing inducing variables
iv_list = [gpf.inducing_variables.InducingPoints(Z) for Z in Zs]
# create multi-output inducing variables from iv_list
iv = gpf.inducing_variables.SeparateIndependentInducingVariables(iv_list)
```

**NOTE:** While the inducing points are independent, there needs to be the same number of inducing points per dimension.

```python
# create SVGP model as usual and optimize
m = gpf.models.SVGP(kernel, gpf.likelihoods.Gaussian(), inducing_variable=iv, num_latent_gps=P)
```

```python
optimize_model_with_scipy(m)
```

```python
plot_model(m)
```

The following plot shows that we use different inducing *inputs* in each output dimension.

```python
for i in range(len(m.inducing_variable.inducing_variable_list)):
    q_mu_unwhitened, q_var_unwhitened = m.predict_f(m.inducing_variable.inducing_variable_list[i].Z)
    plt.plot(
        m.inducing_variable.inducing_variable_list[i].Z.numpy(),
        q_mu_unwhitened[:, i, None].numpy(),
        "o",
    )
plt.gca().set_xticks(np.linspace(-6, 6, 20), minor=True)
plt.gca().set_yticks(np.linspace(-9, 9, 20), minor=True)
plt.grid(which="minor")
```

```python
m.inducing_variable.inducing_variable_list
```

    ListWrapper([<gpflow.inducing_variables.inducing_variables.InducingPoints object at 0x7f64fc2ba9e8>, <gpflow.inducing_variables.inducing_variables.InducingPoints object at 0x7f64fc2baf60>, <gpflow.inducing_variables.inducing_variables.InducingPoints object at 0x7f64fc2ba588>])

## Model $f(x)$ by doing inference in the $g$ space
### Mixed kernel and uncorrelated inducing variables

Remember the general case: $f(x) = W g(x)$, where $g(x) \in \mathbb{R}^L$, $f(x) \in \mathbb{R}^P$ and $W \in \mathbb{R}^{P \times L}$, where $L \leq P$.
We assume that the outputs of $g$ are uncorrelated, and by *mixing* them with $W$ they become correlated.
With this setup we perform the optimal routine to calculate the conditional. Namely, calculate the conditional of the uncorrelated latent $g$ and afterwards project the mean and variance using the mixing matrix: $\mu_f = W \mu_g$ and $\Sigma_f = W\Sigma_g W^\top$

- $K_{uu} = L \times M \times M$
- $K_{uf} = L \times M \times N$

```python
# Create list of kernels for each output
kern_list = [gpf.kernels.SquaredExponential() + gpf.kernels.Linear() for _ in range(L)]
# Create multi-output kernel from kernel list
kernel = gpf.kernels.LinearCoregionalization(
    kern_list, W=np.random.randn(P, L)
)  # Notice that we initialise the mixing matrix W
# initialisation of inducing input locations (M random points from the training inputs)
Z = Zinit.copy()
# create multi-output inducing variables from Z
iv = gpf.inducing_variables.SharedIndependentInducingVariables(
    gpf.inducing_variables.InducingPoints(Z)
)
```

```python
# initialize mean of variational posterior to be of shape MxL
q_mu = np.zeros((M, L))
# initialize \sqrt(Σ) of variational posterior to be of shape LxMxM
q_sqrt = np.repeat(np.eye(M)[None, ...], L, axis=0) * 1.0

# create SVGP model as usual and optimize
m = gpf.models.SVGP(
    kernel, gpf.likelihoods.Gaussian(), inducing_variable=iv, q_mu=q_mu, q_sqrt=q_sqrt
)
```

```python
optimize_model_with_scipy(m)
```

```python
plot_model(m)
```

## Illustration of GPflow's multi-output capabilities
This section shows the inheritance structure in GPflow's multi-output framework.

### Multi-output kernels (MOK) class diagram
We include three multi-output kernels:
- `SharedIndependent`: This kernel is included mainly as an illustration of specifying a conditional using the multiple dispatch framework. The same functionality is provided by using a normal kernel and passing in multiple approximate posteriors by stacking `q_mu` and `q_sqrt`.
- `SeparateIndependent`: This kernel allows you to use different priors for each output GP.
- `LinearCoregionalization`: This kernel describes the prior of the linear model of coregionalization. As shown previously, this implementation supports various inducing point approximations.

We include several base classes. Two are noteworthy:
- `MultioutputKernel` is included to be the base class for all multi-output kernels.
- `IndepedentLatent` is the base class for all multi-output kernels which are constructed from independent latent processes. Including this kernel allows the specification of a default approximation method which, while not the most efficient, does take advantage of _some_ structure. It can be applied to _any_ kernel constructed from independent latent processes.

There is a similarity in the meaning of `SeparateIndependent` and `IndependentLatent`. Both kernels indicate that independent processes are used, and that $\mathbf{K}_{\bf uu}$ can therefore be represented as a `[L, M, M]` tensor. It could therefore be suggested that `SeparateIndependent` be the parent class of all "independent latent" kernels, instead of having a separate `IndependentLatent` class. We decided against this because:
- this would increase the complexity in specifying `conditionals()` for the otherwise simple multi-output kernels `SeparateIndependent` and `SharedIndependent`.
- we did not want to specify too much of an implementation in `IndependentLatent`, leaving implementation details to child classes. Using `SeparateIndependent` as the base class would force all child classes to be a `Combination` kernel.

### Multi-output inducing variables class diagram

#### Inducing points
The goal of this class is to provide inducing variables that can be used with _any_ kernel, even if the method ends up being slow.

The multiouput framework extends `InducingPoints` to work with multi-output kernels. Just like for single-output kernels, we want `InducingPoints` to work for all `MultioutputKernel`s. We do this by defining `InducingPoints` to take _all_ outputs for specific inducing inputs as inducing variables.

#### Fallback shared/separate independent inducing variables
The goal of these classes is to provide a reasonably efficient implementation for kernels that give exploitable independence structure in the prior of inducing variables (that is, subclasses of `IndependentLatent`), while only needing to implement `Kuu()` and `Kuf()` methods.

#### Shared/separate independent inducing variables
The goal of these classes is to provide the most efficient code path for kernels that allow exploiting independence structure in the prior of inducing variables.

For more specialized multi-output kernels (i.e. `{Shared|Separate}Independent` or `LinearCoregionalization`) we define `{Shared|Separate}IndependentInducingVariables`. These wrap (a list of) single-output inducing variables to define groups of a-priori independent inducing variables, which leads to a $\mathbf{K}_{\bf uu}$ that can be represented as a `[L, M, M]` tensor. We saw the use of these previously.

`{Shared|Separate}IndependentInducingVariables` inherit from `Fallback{Shared|Separate}IndependentInducingVariables`, so the multiple dispatch will fall back on the slower but general implementation.

### Implemented combinations
Multiple dispatch is applied to both `Kuu()`, `Kuf()`, and `conditional()`. The return values of the covariances can therefore be tailored to a specific implementation of `conditional()`. The following table lists combinations which are currently available in GPflow. Thanks to the multiple dispatch code, implementing your own outside of GPflow should require only a small amount of code!

| Inducing variable class                                      | Kernel                  | Kuu           | Kuf           | conditional                         | note                                                                                |
|----------------------------------------------|-------------------------|---------------|---------------|-------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `InducingPoints`                               | `MultioutputKernel`       | `[M, P, M, P]` | `[M, P, N, P]` | `inducing_point_conditional()`, which calls `fully_correlated_conditional()`       | Works for all kernels, but might be very inefficient. In this case `q_mu` and `q_sqrt` should have shapes of `[1, MP]` and `[1, MP, MP]`  |
| `SharedIndependentInducingVariables`           | `SharedIndependent`       | `[M, M]`         | `[M, N]`         | `shared_independent_conditional()`, which calls `base_conditional()`                    | The combination of these two classes is in a sense redundant, because we can achieve the same behavior by using the single output Kernel and InducingVariable classes. They are added for illustrative purposes. Thanks to the conditional dispatch, the most efficient code path is used. |
| `SeparateIndependentInducingVariables`         | `SharedIndependent`       | `[P, M, M]`     | `[P, M, N]`     | `separate_independent_conditional()`, which calls `base_conditional()` P times               | We loop P times over the `base_conditional()`         |
| `SeparateIndependentInducingVariable`         | `SeparateIndependent`     | `[P, M, M]`     | `[P, M, N]`     |  `separate_independent_conditional()`, which calls `base_conditional()` P times                | We loop P times over the `base_conditional()`           |
| `SharedIndependentInducingVariables`           | `SeparateIndependent`     | `[P, M, M]`     | `[P, M, N]`     | `separate_independent_conditional()`, which calls `base_conditional()` P times                | We loop P times over the `base_conditional()`        |
| `FallbackSharedIndependentInducingVariables`   | `IndependentLatent`       | `[L, M, M]`     | `[M, L, N, P]` | `fallback_independent_latent_conditional()`, which calls `independent_interdomain_conditional()` | Implementation which only requires custom `Kuu()` and `Kuf()` |
| `FallbackSeparateIndependentInducingVariable` | `IndependentLatent`       | `[L, M, M]`     | `[M, L, N, P]` | `fallback_independent_latent_conditional()`, which calls `independent_interdomain_conditional()` | Implementation which only requires custom `Kuu()` and `Kuf()`  |
| `SharedIndependentInducingVariables`           | `LinearCoregionalization` | `[L, M, M]`     | `[L, M, N]`     | `coregionalization_conditional()`, which calls `base_conditional()`                    | This is the most efficient implementation for linear coregionalization. The inducing outputs live in g-space. Here we use the output of the base conditional and project the mean and covariance with the mixing matrix W.                                                                    |
| `SeparateIndependentInducingVariables`         | `LinearCoregionalization` | `[L, M, M]`     | `[L, M, N]`     | base_conditional                    | This is the most efficient implementation for linear coregionalization. The inducing outputs live in g-space. Here we use the output of the base conditional and project the mean and covariance with the mixing matrix W.                                                                    |

## Debugging: introspect

Given all these possibilities it can be hard to determine which conditional will be called for which set of kernel and inducing variable. The following method lets you proactively introspect which implementation will be executed. This can be useful when debugging new code.

```python
def inspect_conditional(inducing_variable_type, kernel_type):
    """
    Helper function returning the exact implementation called
    by the multiple dispatch `conditional` given the type of
    kernel and inducing variable.

    :param inducing_variable_type:
        Type of the inducing variable
    :param kernel_type:
        Type of the kernel

    :return: String
        Contains the name, the file and the linenumber of the
        implementation.
    """
    import inspect
    from gpflow.conditionals import conditional

    implementation = conditional.dispatch(object, inducing_variable_type, kernel_type, object)
    info = dict(inspect.getmembers(implementation))
    return info["__code__"]

# Example:
inspect_conditional(
    gpf.inducing_variables.SharedIndependentInducingVariables, gpf.kernels.SharedIndependent
)
```

## Further Reading:
- [A simple demonstration of coregionalization](./coregionalisation.ipynb), which details other GPflow features for multi-output prediction without fully observed outputs.


```python
import numpy as np
from bokeh.plotting import figure, output_notebook, show

output_notebook()
np.random.seed(103472)
```

    Loading BokehJS ...

Let's simulate a process where an individual flips a coin, and moves forward by one step if the coin lands heads, otherwise makes a step backward. The individual will flip the coin 16 times. Now, let's repeat the experiment 1,000 times and record how many steps forward or backward each individual moved.

```python
results = list()
cumsums = list()
for i in range(1000):
    moves = np.where(np.round(np.random.uniform(low=0, high=1, size=16)) == 0, -1, 1)
    cumsums.append(np.cumsum(moves))
    final = np.sum(moves)
    results.append(final)

p = figure(title='Results of a 1,000 coin flip simulations')
p.title.align = 'center'
high, edges = np.histogram(results, bins=30)
p.quad(top=high, bottom=0, left=edges[:-1], right=edges[1:])
show(p)
```

```python
p = figure(x_axis_label='step number', y_axis_label='position')
p.multi_line(xs=[list(range(16)) for i in range(1000)], ys=cumsums, alpha=.10)
p.line(x=(3, 3), y=(-15, 15), line_dash='dotted', color='red')
p.line(x=(7, 7), y=(-15, 15), line_dash='dotted', color='red')
p.line(x=(15, 15), y=(-15, 15), line_dash='dotted', color='red')
show(p)
```

```python

```

```python

```

```python

```

```python

```

```python

```

```python

```

```python

```

```python
%matplotlib inline

import os

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
import scipy.stats as stats
from sklearn.preprocessing import scale

from rethinking.stan import StanCache
from rethinking.utils import hpdi

np.set_printoptions(precision=3)
```

```python

```

```python

```

```python
DATA_FOLDER = '../data'

HOWELL_PATH = os.path.join(DATA_FOLDER, 'howell1.csv')
```

```python
howell_full = pd.read_csv(HOWELL_PATH)
howell = howell_full[howell_full.age >= 18]
howell.head()
```

      height
      weight
      age
      male

      0
      151.765
      47.825606
      63.0
      1

      1
      139.700
      36.485807
      63.0
      0

      2
      136.525
      31.864838
      65.0
      0

      3
      156.845
      53.041915
      41.0
      1

      4
      145.415
      41.276872
      51.0
      0

```python
howell.describe()
```

      height
      weight
      age
      male

      count
      352.000000
      352.000000
      352.000000
      352.000000

      mean
      154.597093
      44.990486
      41.138494
      0.468750

      std
      7.742332
      6.456708
      15.967855
      0.499733

      min
      136.525000
      31.071052
      18.000000
      0.000000

      25%
      148.590000
      40.256290
      28.000000
      0.000000

      50%
      154.305000
      44.792210
      39.000000
      0.000000

      75%
      160.655000
      49.292693
      51.000000
      1.000000

      max
      179.070000
      62.992589
      88.000000
      1.000000

```python
height = howell.loc[howell.age >= 18, 'height']
```

```python
sns.kdeplot(height)
```

$$
\begin{align}
    h_i &\sim Normal(\mu, \sigma) \\
    \mu &\sim Normal(178, 20) \\
    \sigma &\sim Uniform(0, 50)
\end{align}
$$

```python
x = np.linspace(start=100, stop=250, num=50)
y = stats.norm(178, 20).pdf(x)
plt.plot(x, y)
plt.title('Prior for $\mu$')
plt.show()
```

```python
x = np.linspace(start=-10, stop=60, num=50)
y = stats.uniform(0, 50).pdf(x)
plt.plot(x, y)
plt.title('Prior for $\sigma$')
plt.show()
```

```python
x = np.linspace(start=100, stop=1000, num=100)
sample_mu = np.random.normal(loc=178, scale=20, size=1000)
sample_sigma = np.random.uniform(low=0, high=50, size=1000)
prior_h = np.random.normal(loc=sample_mu, scale=sample_sigma, size=1000)
sns.kdeplot(prior_h)
plt.title('Combined prior for height')
plt.show()
```

```python
m4_1 = StanCache(filename='../models/height.stan', cache_path='../cache').compile()
m4_1.model_code
```

    data {
        int<lower=0> N;                      // number of observations
        real<lower=0, upper=200> height[N];
    }
    parameters {
        real<lower=0, upper=200> mu;
        real<lower=0, upper=50> sigma;
    }
    model {
        sigma ~ uniform(0, 50);
        mu ~ normal(178, 20);
        height ~ normal(mu, sigma);
    }

```python
height_data = dict(
    N=len(height),
    height=height
)

fit4_1 = m4_1.sampling(data=height_data, iter=1000, chains=4)
fit4_1
```

    /home/gianluca/.virtualenvs/rethinking/lib/python3.6/site-packages/pystan/misc.py:399: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
      elif np.issubdtype(np.asarray(v).dtype, float):

    Inference for Stan model: anon_model_4a78e70403dbc23b3cffbc3b6451b058.
    4 chains, each with iter=1000; warmup=500; thin=1;
    post-warmup draws per chain=500, total post-warmup draws=2000.

            mean se_mean     sd   2.5%    25%    50%    75%  97.5%  n_eff   Rhat
    mu    154.61  9.4e-3   0.42 153.79 154.33  154.6 154.89 155.47   2000    1.0
    sigma   7.76  8.1e-3    0.3    7.2   7.56   7.75   7.94   8.38   1352    1.0
    lp__  -892.2    0.04   1.05 -895.0 -892.6 -891.8 -891.4 -891.2    707    1.0

    Samples were drawn using NUTS at Sun Mar 11 22:07:12 2018.
    For each parameter, n_eff is a crude measure of effective sample size,
    and Rhat is the potential scale reduction factor on split chains (at
    convergence, Rhat=1).

```python
fit4_1.plot()
plt.tight_layout()
plt.show()
```

```python
posterior = fit4_1.extract(permuted=True)
sns.jointplot(x='mu', y='sigma', data=posterior, kind='kde')
```

# Sampling from the posterior

```python
sample_rows = np.random.choice(a=range(len(posterior['lp__'])), replace=True, size=10000)
sample_mu = posterior['mu'][sample_rows]
sample_sigma = posterior['sigma'][sample_rows]
```

```python
sns.jointplot(x=sample_mu, y=sample_sigma, kind='kde')
```

```python
hpdi_mu = hpdi(samples=sample_mu, prob=.95)
hpdi_sigma = hpdi(samples=sample_sigma, prob=.95)
print(f'HPDI mu: {hpdi_mu}\nHPDI sigma: {hpdi_sigma}')
```

    HPDI mu: (153.77454003602867, 155.4335418396508)
    HPDI sigma: (7.179855609761088, 8.363976760260666)

# Adding a predictor

```python
plt.scatter(x='weight', y='height', data=howell)
plt.xlabel('Weight (in kg)')
plt.ylabel('Height (in cm)')
plt.show()
```

$$
\begin{align}
    h_i &\sim Normal(\mu_i, \sigma) \\
    \mu_i &= \alpha + \beta x_i \\
    \alpha &\sim Normal(178, 100) \\
    \beta &\sim Normal(0, 10) \\
    \sigma &\sim Uniform(0, 50)
\end{align}
$$

```python
sm2 = StanCache(filename='../models/predictor.stan', cache_path='../cache').compile()
sm2.model_code
```

    data {
        int<lower=0> N;
        vector[N] x;
        vector[N] h;
    }
    parameters {
        real alpha;
        real beta;
        real<lower=0> sigma;
    }
    model {
        //sigma ~ uniform(0, 50);
        alpha ~ normal(178, 100);
        beta ~ normal(0, 10);
        h ~ normal(alpha + beta * x, sigma);
    }

We found that the Uniform(0, 50) prior for $\sigma$ as suggest by the author (McElreath) causes an extremely low effective sample size (ESS). We commented out the prior to fix the issue. Further analyses should be conducted to understand why this is happening since:
* The posterior distribution of $\sigma$ is within the interval (0, 50)
* The default prior for sigma, when not manually specified by the user, is a uniform prior bounded between $0$ and $\inf$. This is much larger than the interval (0, 50)

```python
predictor_data = dict(
    N=len(howell.height),
    x=howell.weight,
    h=howell.height)
```

```python
fit2 = sm2.sampling(data=predictor_data, iter=4000)
fit2
```

    /home/gianluca/.virtualenvs/rethinking/lib/python3.6/site-packages/pystan/misc.py:399: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
      elif np.issubdtype(np.asarray(v).dtype, float):
    /home/gianluca/git/rethinking/src/rethinking/stan.py:95: UserWarning: Pickling fit objects is an experimental feature!
    The relevant StanModel instance must be pickled along with this fit object.
    When unpickling the StanModel must be unpickled first.
      pickle.dump(results, f)

    Inference for Stan model: anon_model_3af82d1625525dbd730bf0dcb2fe1b7f.
    4 chains, each with iter=4000; warmup=2000; thin=1;
    post-warmup draws per chain=2000, total post-warmup draws=8000.

            mean se_mean     sd   2.5%    25%    50%    75%  97.5%  n_eff   Rhat
    alpha 113.89    0.04   1.91 110.19 112.53  113.9 115.19 117.61   2591    1.0
    beta     0.9  8.3e-4   0.04   0.82   0.88    0.9   0.93   0.99   2566    1.0
    sigma    5.1  3.3e-3    0.2   4.74   4.96   5.09   5.23    5.5   3554    1.0
    lp__  -747.6    0.02    1.2 -750.7 -748.2 -747.3 -746.7 -746.2   2711    1.0

    Samples were drawn using NUTS at Mon Mar 12 19:21:05 2018.
    For each parameter, n_eff is a crude measure of effective sample size,
    and Rhat is the potential scale reduction factor on split chains (at
    convergence, Rhat=1).

```python
samples2 = fit2.extract(permuted=True)
s = pd.DataFrame(samples2)
s.describe()
```

      alpha
      beta
      sigma
      lp__

      count
      8000.000000
      8000.000000
      8000.000000
      8000.000000

      mean
      113.889942
      0.904791
      5.101670
      -747.644842

      std
      1.914472
      0.042099
      0.195439
      1.202966

      min
      106.120375
      0.760510
      4.492672
      -754.868032

      25%
      112.534350
      0.876111
      4.964124
      -748.214691

      50%
      113.901202
      0.904980
      5.094346
      -747.331305

      75%
      115.189788
      0.934110
      5.230897
      -746.760659

      max
      120.615644
      1.067379
      5.851855
      -746.133715

The program doesn't seem to contain bugs. Samples from the priors looks okay.

```python
np.corrcoef(s.drop('lp__', axis='columns').T
            .as_matrix())
```

    array([[ 1.   , -0.99 ,  0.009],
           [-0.99 ,  1.   , -0.011],
           [ 0.009, -0.011,  1.   ]])

The correlation matrix shows an extremely high negative correlation between $\alpha$ and $\beta$. This can be fixed centering the predictors.

# Centering

```python
howell['weight_c'] = scale(X=howell['weight'], with_mean=True, with_std=False)
```

    /home/gianluca/.virtualenvs/rethinking/lib/python3.6/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning:
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead

    See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
      """Entry point for launching an IPython kernel.

```python
plt.hist(howell.weight_c, bins=20)
plt.show()
```

```python
predictor_data = dict(
    N=len(howell.height),
    x=howell.weight_c,
    h=howell.height)

fit3 = sm2.sampling(data=predictor_data, iter=4000)
fit3
```

    /home/gianluca/.virtualenvs/rethinking/lib/python3.6/site-packages/pystan/misc.py:399: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
      elif np.issubdtype(np.asarray(v).dtype, float):
    /home/gianluca/git/rethinking/src/rethinking/stan.py:95: UserWarning: Pickling fit objects is an experimental feature!
    The relevant StanModel instance must be pickled along with this fit object.
    When unpickling the StanModel must be unpickled first.
      pickle.dump(results, f)

    Inference for Stan model: anon_model_3af82d1625525dbd730bf0dcb2fe1b7f.
    4 chains, each with iter=4000; warmup=2000; thin=1;
    post-warmup draws per chain=2000, total post-warmup draws=8000.

            mean se_mean     sd   2.5%    25%    50%    75%  97.5%  n_eff   Rhat
    alpha  154.6  3.0e-3   0.27 154.07 154.41  154.6 154.78 155.13   8000    1.0
    beta    0.91  4.7e-4   0.04   0.82   0.88   0.91   0.93   0.99   8000    1.0
    sigma    5.1  2.2e-3   0.19   4.74   4.97    5.1   5.23    5.5   8000    1.0
    lp__  -747.4    0.02    1.2 -750.5 -747.9 -747.1 -746.5 -746.0   4043    1.0

    Samples were drawn using NUTS at Mon Mar 12 19:21:06 2018.
    For each parameter, n_eff is a crude measure of effective sample size,
    and Rhat is the potential scale reduction factor on split chains (at
    convergence, Rhat=1).

```python
samples3 = fit3.extract(permuted=True)
s3 = pd.DataFrame(samples3)

np.corrcoef(s3.drop('lp__', axis='columns').T
            .as_matrix())
```

    array([[ 1.   , -0.004, -0.008],
           [-0.004,  1.   , -0.027],
           [-0.008, -0.027,  1.   ]])

Note that the posterior mean and standard deviation for $\beta$ and $\sigma$ is the same as the one in the model without centering the predictors. The effective sample size though is much larger when the model's predictors are centered. Of course, centering the predictors affects the intercept ($\alpha$).

# Plotting prosterior inference against the data

```python
plt.scatter(x='weight_c', y='height', data=howell, alpha=.35)
x = np.array([-20, 20])
for i in np.random.choice(range(8000), size=20, replace=False):
    f = lambda x: s3.loc[i, 'alpha'] + s3.loc[i, 'beta'] * x
    plt.plot(x, f(x), linewidth=0.25, c='grey')
plt.show()
```

# Polynomial regression

```python
plt.scatter(x='weight', y='height', data=howell_full, alpha=.35)
plt.xlabel('Weight (in kg)')
plt.ylabel('Height (in cm)')
plt.show()
```

```python
howell_full['weight_s'] = scale(
    X=howell_full.weight, with_mean=True, with_std=True)
```

```python
plt.subplot(121)
plt.hist(howell_full.weight)
plt.title('Original predictor')

plt.subplot(122)
plt.hist(howell_full.weight_s)
plt.title('Scaled predictor')
plt.show()
```

```python
sm3 = StanCache(filename='../models/poly.stan', cache_path='../cache').compile()
sm3.model_code
```

    data {
        int<lower=0> N;                  // n observations
        int<lower=0> M;                  // n features
        vector[N] weight;
        vector[N] weight_2;              // weight^2
        vector[N] height;
    }
    parameters {
        real alpha;
        real beta[M];
        real<lower=0> sigma;
    }
    model {
        alpha ~ normal(178, 100);
        beta ~ normal(0, 10);
        height ~ normal(alpha + beta[1] * weight + beta[2] * weight_2, sigma);
    }

```python
poly_data = dict(
    N=len(howell_full.height),
    M=2,
    weight=howell_full.weight_s,
    weight_2=np.power(howell_full.weight_s, 2),
    height=howell_full.height)

fit4 = sm3.sampling(data=poly_data, iter=4000)
fit4
```

    /home/gianluca/.virtualenvs/rethinking/lib/python3.6/site-packages/pystan/misc.py:399: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
      elif np.issubdtype(np.asarray(v).dtype, float):

    Inference for Stan model: anon_model_8a020c4da28c7aa4a258598b82a44c16.
    4 chains, each with iter=4000; warmup=2000; thin=1;
    post-warmup draws per chain=2000, total post-warmup draws=8000.

              mean se_mean     sd   2.5%    25%    50%    75%  97.5%  n_eff   Rhat
    alpha   146.67  6.0e-3   0.38 145.93 146.42 146.67 146.92 147.41   3930    1.0
    beta[0]  21.38  4.2e-3   0.29   20.8  21.18  21.37  21.57  21.95   4845    1.0
    beta[1]   -8.4  4.9e-3   0.29  -8.97  -8.59   -8.4  -8.21  -7.84   3458    1.0
    sigma     5.78  2.1e-3   0.18   5.45   5.66   5.78   5.89   6.13   6983    1.0
    lp__     -1226    0.02   1.45  -1230  -1227  -1226  -1225  -1224   3430    1.0

    Samples were drawn using NUTS at Mon Mar 12 19:20:09 2018.
    For each parameter, n_eff is a crude measure of effective sample size,
    and Rhat is the potential scale reduction factor on split chains (at
    convergence, Rhat=1).

```python
x = np.arange(start=-2.5, stop=2.5, step=0.1)
f = np.vectorize(lambda x: 146.67 + 21.38 * x -8.41 * np.power(x, 2))
y = f(x)
plt.scatter(x='weight_s', y='height', data=howell_full, alpha=.35)
plt.plot(x, y)
plt.xlabel('Scaled weight')
plt.ylabel('Height')
plt.show()
```