```
import math
import matplotlib.pyplot as plt
import numpy as np
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
from sympy import Symbol
from scipy import integrate
from sympy import Integral, integrate
from scipy.integrate import quad
from sympy import simplify
from sympy import integrate, init_printing
from sympy.abc import T,S,U
u=np.linspace(-1.5,1.5,10)
v=np.linspace(-1.5,1.5,10)
w=np.arange(0,1,0.8)
t=np.linspace(0,10)
s=np.linspace(0,2*np.pi)
## parte uno Graficar Fx,Fy,Fz
umesh,vmesh,wmesh=np.meshgrid(u,v,w,indexing='ij')
def F(u,v,w):
    l=w*(math.e**u)
    m=(math.e**v)-u
    o=w
    return np.array([l,m,o])
lmesh,mmesh,omesh=F(umesh,vmesh,wmesh)
fig1=plt.figure('Campo vectorial F')
ax=fig1.gca(projection='3d')
ax.set_xlabel('$Fx $')
ax.set_ylabel('$Fy $')
ax.set_zlabel('$Fz $')
ax.quiver(umesh,vmesh,wmesh,lmesh,mmesh,omesh,length=0.3)
### grafica de campo en una Esfera
fig2 = plt.figure('Representacion de F en una Superficie')
am = fig2.add_subplot(111, projection='3d')

def sphere(r):
    p = np.linspace(0, 2 * np.pi, 20)
    q = np.linspace(0, np.pi, 20)
    x = r * np.outer(np.cos(p), np.sin(q))
    y = r * np.outer(np.sin(p), np.sin(q))
    z = r * np.outer(np.ones(np.size(p)), np.cos(q))
    return x,y,z

x,y,z = sphere(3)
xmesh,ymesh,zmesh=F(x,y,z)
am.set_xlabel('$x $')
am.set_ylabel('$y $')
am.set_zlabel('$z $')
am.quiver(x,y,z,xmesh,ymesh,zmesh,length=0.15)
am.plot_surface(x, y, z, rstride=4, cstride=4, color='darkviolet')

##graficas de campo en curvas
print('se tomara la curva recta ')
X=t
Y=2*t-5
Z= 64-14*t
a=Z*(math.e**X)
b=(math.e**Y)-X
c=Z
fig=plt.figure('F representado en una traectoria recta')
ay=fig.gca(projection='3d')
ax.set_xlabel('$x $')
ax.set_ylabel('$y $')
ax.set_zlabel('$z $')
ay.plot(X,Y,Z,'g')
ay.plot(a,b,c,'y')
##calculo de integrales de linea recta
init_printing(use_latex="mathjax")
fT= 2*math.e**(2*T-5)+(64-14*T)*math.e**T+194*T-896
f=lambda  T:2*math.e**(2*T-5)+(64-14*T)*math.e**T+194*T-896
I1=quad(f, 0, 10)
print(integrate(fT,T))
print('el trabajo ejercido sobre la trayectoria recta definida es: ',I1)
### Grafica de campo en una curva cerrada en este caso seria un circulo
L=3*np.cos(s)
M=3*np.sin(s)
N=5
A=N*(math.e**L)
B=(math.e**M)-L
C=N
fig3=plt.figure('F representado en una traectoria curva cerrada')
az=fig3.gca(projection='3d')
az.plot(L,M,N,'b')
az.plot(A,B,C,'g')
U=Symbol('U')
### calcu de integrales de de linea curva cerrada
f1=lambda  U:3*np.cos(U)*(math.e**(3*np.sin(U))-3*np.sin(U))-15*np.sin(U)*math.e**(3*np.cos(U))
print('np.exp(3*np.sin(t))+5*np.exp(3*np.cos(t))+3*np.cos(t)')
I2=quad(f1, 0, 2*np.pi)
#print(integrate(3*np.cos(U)*(np.e(3*sen(U))-3*np.sin(U))-15*np.sin(U)*np.e(3*np.cos(U)),U).doit())
print('el trabajo ejercido sobre la trayectoria curva cerrada definida es: ',I2)

plt.show()
```

```
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

n = 100

theta = np.linspace(0, 2.*np.pi, n)
phi = np.linspace(0, 2.*np.pi, n)
theta, phi = np.meshgrid(theta, phi)
c, a = 4, 1
x = (c + a*np.cos(theta)) * np.cos(phi)
y = (c + a*np.cos(theta)) * np.sin(phi)
z = a * np.sin(theta)

fig = plt.figure()
ax1 = fig.add_subplot(121, projection='3d')
ax1.set_zlim(-3,3)
ax1.plot_surface(x, y, z, rstride=5, cstride=5, color='r', edgecolors='k')
ax1.view_init(36, 26)
ax2 = fig.add_subplot(122, projection='3d')
ax2.set_zlim(-3,3)
ax2.plot_surface(x, y, z, rstride=5, cstride=5, color='b', edgecolors='w')
ax2.view_init(0, 0)
ax2.set_xticks([])
plt.show()
```

```
import numpy as np
from scipy.integrate import tplquad

R = 4
r = 1

# Volume integral
# Order of integration is z, rho, theta
f1 = lambda z, rho, theta: 2 * rho
# set theta limits
a, b = 0, 2 * np.pi
# set rho limits
rho_low = lambda rho: R - r
rho_high = lambda rho: R + r
# set z limits
z_low = lambda theta, rho: 0
z_high = lambda theta, rho: np.sqrt(r**2 - (rho - R)**2)
# calculate integral
V, _ = tplquad(f1, a, b, rho_low, rho_high, z_low, z_high)
print(V)
```

    78.95683520871499


# Lecture 24 - Sequential Monte Carlo in `PyMC3`

```python
import numpy as np
import pymc3 as pm
import theano as T
from theano import shared, function, tensor as tt
from sample_smc import sample_smc

try:
    import sympy
except:
    _=!pip install sympy
    import sympy

import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

import logging

%matplotlib inline
```

## Objectives

+ Compute the model evidence using `PyMC3`.
+ Do model selection with `PyMC3`.

## Sanity check - Does the calculation of the evidence with PySMC work?

Let
$$
p(\theta) = \mathcal{N}(\theta|0, 1),
$$
and
$$
p(y|\theta) = \mathcal{N}(y|\theta,0).
$$
The posterior of $\theta$ given $y$ is:
$$
p(\theta|y) = \frac{p(y|\theta)p(\theta)}{Z},
$$
where
$$
Z = \int_{-\infty}^{\infty} p(y|\theta)p(\theta)d\theta.
$$
Let's first calculate $Z$ analytically.

```python
import sympy.stats
sympy.init_printing()
y, t = sympy.symbols('y \\theta')
q = 1. / sympy.sqrt(2. * sympy.pi) * sympy.exp(-0.5 * (y - t) ** 2) * \
    1. / sympy.sqrt(2. * sympy.pi) * sympy.exp(-0.5 * t ** 2)
sympy.simplify(sympy.integrate(q, (t, -sympy.oo, sympy.oo)))
```

So, if the observed $y$ was zero, then the Z should be:
$$
Z = \frac{1}{2\sqrt{\pi}}.
$$

```python
Z = 1 /  2. / np.sqrt(np.pi)
print('log Z = {0:.3f}'.format(np.log(Z)))
```

    log Z = -1.266

All, right. Now, let's program this thing in pysmc and compare the results.

We start with the model:

```python
model = pm.Model()
yobs = 0.
with model:
    # prior over theta
    theta = pm.Normal('theta', mu=0., sigma=1.,testval=0.)

    # log likelihood
    llk = pm.Potential('llk', pm.Normal.dist(theta, 1.).logp(yobs))

    trace, smcres = sample_smc(1000)
```

    Sample initial stage: ...
    Stage:   0 Beta: 1.000 Steps:  25 Acce: 1.000

```python
# get the model evidence
log_evidence_smc = np.log(smcres.model.marginal_likelihood)
print('True log evidence: %.4f \nSMC log evidence: %.4f'%(np.log(Z), log_evidence_smc))
```

    True log evidence: -1.2655
    SMC log evidence: -1.2737

Which is close to the truth.

### Questions

+ Repeat the calculations above for a varying number of SMC particles. Start from 10 and go up to 10,000.
    - It get closer to the true value as SMC particles increase

## Polynomial Regression

```python
def compute_design_matrix(X, phi):
    """
    Arguments:

    X   -  The observed inputs (1D array)
    phi -  The basis functions.
    """
    num_observations = X.shape[0]
    num_basis = phi.num_basis
    Phi = np.ndarray((num_observations, num_basis))
    for i in range(num_observations):
        Phi[i, :] = phi(X[i, :])
    return Phi

class PolynomialBasis(object):
    """
    A set of linear basis functions.

    Arguments:
    degree  -  The degree of the polynomial.
    """
    def __init__(self, degree):
        self.degree = degree
        self.num_basis = degree + 1
    def __call__(self, x):
        return np.array([x[0] ** i for i in range(self.degree + 1)])

class FourierBasis(object):
    """
    A set of linear basis functions.

    Arguments:
    num_terms  -  The number of Fourier terms.
    L          -  The period of the function.
    """
    def __init__(self, num_terms, L):
        self.num_terms = num_terms
        self.L = L
        self.num_basis = 2 * num_terms
    def __call__(self, x):
        res = np.ndarray((self.num_basis,))
        for i in range(num_terms):
            res[2 * i] = np.cos(2 * i * np.pi / self.L * x[0])
            res[2 * i + 1] = np.sin(2 * (i+1) * np.pi / self.L * x[0])
        return res

class RadialBasisFunctions(object):
    """
    A set of linear basis functions.

    Arguments:
    X   -  The centers of the radial basis functions.
    ell -  The assumed lengthscale.
    """
    def __init__(self, X, ell):
        self.X = X
        self.ell = ell
        self.num_basis = X.shape[0]
    def __call__(self, x):
        return np.exp(-.5 * (x - self.X) ** 2 / self.ell ** 2).flatten()
```

Let's generate some fake data.

```python
np.random.seed(12345)
def getdata(N, sigma2):
    X = 2 * np.random.rand(N) - 1.
    y = 0.5 * X  ** 3 - 0.3 * X ** 2 + np.sqrt(sigma2) * np.random.rand(N)
    return X, y

num_samples = 50
sigma2 = 1e-3
X, y = getdata(num_samples, sigma2)
plt.figure(figsize=(8, 6))
plt.plot(X, y, 'o', markeredgewidth=2, label='Data')
plt.xlabel('$x$', fontsize=20)
plt.ylabel('$y$', fontsize=20)
plt.legend(loc='best', fontsize=20)
plt.tight_layout()
```

We are going to implement a standard Bayesian linear regression and train it with `PyMC3`.
We will compute the evidence in order to select the best class of basis functions.
The model is as follows:

The output $y$ conditioned on the input $x$, the weights of the basis functions $w$ and
the noise variance $\sigma^2$ has likelihood:
$$
p(y|x,w,\sigma, \mathcal{M}) = \mathcal{N}(y|w^T\phi_{\mathcal{M}}(x), \sigma^2),
$$
where $\phi_{\mathcal{M},1}(\cdot), \dots, \phi_{\mathcal{M},m_{\mathcal{M}}}(\cdot)$ are the
$m_{\mathcal{M}}$ basis functions of the model $\mathcal{M}$.
We put a normal prior on the weights:
$$
p(w|\alpha) = \mathcal{N}(w|0, \alpha I_{m_{\mathcal{M}}}),
$$
and an inverse Gamma prior for $\sigma$ and $\alpha$:
$$
p(\sigma^2) = \mathrm{IG}(\sigma^2|1, 1),
$$
and
$$
p(\alpha) = \mathrm{IG}(\alpha|1,1).
$$

Assume that the data we have observed are:
$$
x_{1:n} = \{x_1,\dots,x_n\},\;\mathrm{and}\;y_{1:n} = \{y_1,\dots,y_n\}.
$$
Consider the design matrix $\Phi_{\mathcal{M}}\in\mathbb{R}^{n\times m}$:
$$
\Phi_{\mathcal{M},ij} = \phi_{\mathcal{M},j}(x_i).
$$
The likelihood of the data is:
$$
p(y_{1:n} | x_{1:n}, w, \sigma, \mathcal{M}) = \mathcal{N}(y_{1:n}|\Phi_{\mathcal{M}}w, \sigma^2I_n).
$$
Let's turn this into `PyMC3` code.

```python
def make_model(Phi, y):
    """
    INPUTS:
        Phi -> Design matrix.
        y   -> Target vector.

    RETURNS:
        model -> `pymc3.model` context.
    """
    num_data, num_features = Phi.shape

    # define the model
    with pm.Model() as model:
        # prior on the weights
        alpha = pm.InverseGamma('alpha', alpha=1., beta=1.)
        w = pm.Normal('w', mu=0., tau=alpha, shape=num_features)

        # prior on the likelihood noise variance
        sigma2 = pm.InverseGamma('sigma2', alpha=5., beta=0.1)

        # the data likelihood mean
        ymean = pm.Deterministic('ymean', tt.dot(Phi, w))

        # likelihood
        y = pm.Normal('y', ymean, sigma2, shape=num_data, observed=y)
        #llk = pm.Potential('llk', pm.Normal.dist(ymean, tt.sqrt(sigma2)).logp_sum(y))
    return model
```

Now, let's create a function that trains the model using pysmc for a polynomial basis with a given order.

```python
def fit_poly(phi, X, y, num_particles=100):
    """

    RETURNS:
        1. An instance of pymc3.Model for the SMC model.
        2. The SMC trace.
        3. An instance of pymc3.smc.SMC containing sampling information.
    """
    Phi = compute_design_matrix(X[:, None], phi)
    smcmodel = make_model(Phi, y)
    trace, res = sample_smc(draws=num_particles,
                         model=smcmodel,
                         progressbar=False,
                         threshold=0.8)
    return smcmodel, trace, res

phi = PolynomialBasis(3)
model, trace, res = fit_poly(phi, X, y)
```

    Sample initial stage: ...
    Stage:   0 Beta: 0.000 Steps:  25 Acce: 1.000
    Stage:   1 Beta: 0.000 Steps:  25 Acce: 0.155
    Stage:   2 Beta: 0.000 Steps:  25 Acce: 0.237
    Stage:   3 Beta: 0.000 Steps:  17 Acce: 0.195
    Stage:   4 Beta: 0.000 Steps:  21 Acce: 0.206
    Stage:   5 Beta: 0.001 Steps:  19 Acce: 0.247
    Stage:   6 Beta: 0.001 Steps:  16 Acce: 0.246
    Stage:   7 Beta: 0.003 Steps:  16 Acce: 0.263
    Stage:   8 Beta: 0.004 Steps:  15 Acce: 0.329
    Stage:   9 Beta: 0.007 Steps:  11 Acce: 0.252
    Stage:  10 Beta: 0.013 Steps:  15 Acce: 0.209
    Stage:  11 Beta: 0.020 Steps:  19 Acce: 0.196
    Stage:  12 Beta: 0.029 Steps:  21 Acce: 0.238
    Stage:  13 Beta: 0.042 Steps:  16 Acce: 0.263
    Stage:  14 Beta: 0.057 Steps:  15 Acce: 0.254
    Stage:  15 Beta: 0.075 Steps:  15 Acce: 0.256
    Stage:  16 Beta: 0.095 Steps:  15 Acce: 0.227
    Stage:  17 Beta: 0.118 Steps:  17 Acce: 0.239
    Stage:  18 Beta: 0.147 Steps:  16 Acce: 0.286
    Stage:  19 Beta: 0.182 Steps:  13 Acce: 0.311
    Stage:  20 Beta: 0.230 Steps:  12 Acce: 0.267
    Stage:  21 Beta: 0.299 Steps:  14 Acce: 0.287
    Stage:  22 Beta: 0.389 Steps:  13 Acce: 0.264
    Stage:  23 Beta: 0.511 Steps:  15 Acce: 0.270
    Stage:  24 Beta: 0.645 Steps:  14 Acce: 0.281
    Stage:  25 Beta: 0.825 Steps:  13 Acce: 0.212
    Stage:  26 Beta: 1.000 Steps:  19 Acce: 0.245

### Postprocessing

Once you have the `trace` object for the SMC simulation you can apply all the standard postprocessing tools from `PyMC3` as usual.

Here's the posterior distribution over the weights precision and the the likelihood noise, $\alpha$ and $\sigma^2$ respectively:

```python
_=pm.plot_posterior(trace, var_names=['alpha', 'sigma2'])
```

Here's the posterior predictive mean of the output $y$, i.e., $\mathbb{E}[y|x, w, \sigma]$:

```python
ppsamples = pm.sample_posterior_predictive(model=model,
                               trace=trace, var_names=['ymean'])['ymean']
```

    100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 2785.27it/s]

```python
idx = np.argsort(X)
plt.figure(figsize=(10, 8))
plt.plot(X[idx], ppsamples.mean(0)[idx], linewidth=2.5, label='Posterior Predictive Mean' )
plt.plot(X, y, 'x', markeredgewidth=2.5, markersize=10, label='Observed data')

plt.legend(loc='best', fontsize=20)
plt.tight_layout()
```

SMC does a particle approximation of the posterior distribution. The particles themselves can be obtained from the `trace` object and the particle weights can be obtained from the  `res` object.

Recall that the approximate posterior distribution is of the form $p(\theta|\mathcal{D}) = \sum_{j=1}^{N} w_j \delta(\theta - \theta_j)$.

```python
particles_w = trace.w
particles_alpha = trace.alpha
particle_weights = res.weights   # <- these are the ws from the above equation
```

## Model comparison

Since SMC can approximate the model evidence it provides a principled way of comparing models. Let's compare 5 different polynomial regression models where we change the degree of the polynomial from 1 to 5.

```python
# Evaluate the evidence for the various degrees
log_Zs = []
D = [1, 2, 3, 4, 5]
for d in D:
    phi = PolynomialBasis(d)
    _, _, res = fit_poly(phi, X, y, num_particles=500)
    log_Z = np.log(res.model.marginal_likelihood)
    log_Zs.append(log_Z)
```

    Sample initial stage: ...
    Stage:   0 Beta: 0.000 Steps:  25 Acce: 1.000
    Stage:   1 Beta: 0.000 Steps:  25 Acce: 0.253
    Stage:   2 Beta: 0.000 Steps:  15 Acce: 0.260
    Stage:   3 Beta: 0.000 Steps:  15 Acce: 0.239
    Stage:   4 Beta: 0.001 Steps:  16 Acce: 0.283
    Stage:   5 Beta: 0.001 Steps:  13 Acce: 0.263
    Stage:   6 Beta: 0.002 Steps:  15 Acce: 0.248
    Stage:   7 Beta: 0.004 Steps:  16 Acce: 0.248
    Stage:   8 Beta: 0.006 Steps:  16 Acce: 0.237
    Stage:   9 Beta: 0.010 Steps:  17 Acce: 0.242
    Stage:  10 Beta: 0.016 Steps:  16 Acce: 0.232
    Stage:  11 Beta: 0.026 Steps:  17 Acce: 0.240
    Stage:  12 Beta: 0.043 Steps:  16 Acce: 0.243
    Stage:  13 Beta: 0.069 Steps:  16 Acce: 0.245
    Stage:  14 Beta: 0.108 Steps:  16 Acce: 0.242
    Stage:  15 Beta: 0.177 Steps:  16 Acce: 0.237
    Stage:  16 Beta: 0.280 Steps:  17 Acce: 0.240
    Stage:  17 Beta: 0.448 Steps:  16 Acce: 0.268
    Stage:  18 Beta: 0.700 Steps:  14 Acce: 0.257
    Stage:  19 Beta: 1.000 Steps:  15 Acce: 0.256
    Sample initial stage: ...
    Stage:   0 Beta: 0.000 Steps:  25 Acce: 1.000
    Stage:   1 Beta: 0.000 Steps:  25 Acce: 0.195
    Stage:   2 Beta: 0.000 Steps:  21 Acce: 0.208
    Stage:   3 Beta: 0.000 Steps:  19 Acce: 0.219
    Stage:   4 Beta: 0.000 Steps:  18 Acce: 0.253
    Stage:   5 Beta: 0.001 Steps:  15 Acce: 0.232
    Stage:   6 Beta: 0.002 Steps:  17 Acce: 0.222
    Stage:   7 Beta: 0.003 Steps:  18 Acce: 0.244
    Stage:   8 Beta: 0.005 Steps:  16 Acce: 0.228
    Stage:   9 Beta: 0.008 Steps:  17 Acce: 0.255
    Stage:  10 Beta: 0.013 Steps:  15 Acce: 0.263
    Stage:  11 Beta: 0.021 Steps:  15 Acce: 0.269
    Stage:  12 Beta: 0.033 Steps:  14 Acce: 0.215
    Stage:  13 Beta: 0.050 Steps:  19 Acce: 0.235
    Stage:  14 Beta: 0.076 Steps:  17 Acce: 0.241
    Stage:  15 Beta: 0.113 Steps:  16 Acce: 0.253
    Stage:  16 Beta: 0.171 Steps:  15 Acce: 0.254
    Stage:  17 Beta: 0.254 Steps:  15 Acce: 0.287
    Stage:  18 Beta: 0.379 Steps:  13 Acce: 0.268
    Stage:  19 Beta: 0.556 Steps:  14 Acce: 0.281
    Stage:  20 Beta: 0.830 Steps:  13 Acce: 0.253
    Stage:  21 Beta: 1.000 Steps:  15 Acce: 0.262
    Sample initial stage: ...
    Stage:   0 Beta: 0.000 Steps:  25 Acce: 1.000
    Stage:   1 Beta: 0.000 Steps:  25 Acce: 0.170
    Stage:   2 Beta: 0.000 Steps:  24 Acce: 0.208
    Stage:   3 Beta: 0.000 Steps:  19 Acce: 0.216
    Stage:   4 Beta: 0.000 Steps:  18 Acce: 0.234
    Stage:   5 Beta: 0.001 Steps:  17 Acce: 0.234
    Stage:   6 Beta: 0.001 Steps:  17 Acce: 0.221
    Stage:   7 Beta: 0.002 Steps:  18 Acce: 0.239
    Stage:   8 Beta: 0.004 Steps:  16 Acce: 0.227
    Stage:   9 Beta: 0.007 Steps:  17 Acce: 0.262
    Stage:  10 Beta: 0.011 Steps:  15 Acce: 0.234
    Stage:  11 Beta: 0.016 Steps:  17 Acce: 0.258
    Stage:  12 Beta: 0.024 Steps:  15 Acce: 0.254
    Stage:  13 Beta: 0.034 Steps:  15 Acce: 0.265
    Stage:  14 Beta: 0.046 Steps:  14 Acce: 0.285
    Stage:  15 Beta: 0.060 Steps:  13 Acce: 0.274
    Stage:  16 Beta: 0.077 Steps:  14 Acce: 0.268
    Stage:  17 Beta: 0.097 Steps:  14 Acce: 0.265
    Stage:  18 Beta: 0.123 Steps:  14 Acce: 0.295
    Stage:  19 Beta: 0.154 Steps:  13 Acce: 0.326
    Stage:  20 Beta: 0.192 Steps:  11 Acce: 0.321
    Stage:  21 Beta: 0.244 Steps:  11 Acce: 0.299
    Stage:  22 Beta: 0.311 Steps:  12 Acce: 0.289
    Stage:  23 Beta: 0.395 Steps:  13 Acce: 0.275
    Stage:  24 Beta: 0.508 Steps:  14 Acce: 0.274
    Stage:  25 Beta: 0.670 Steps:  14 Acce: 0.278
    Stage:  26 Beta: 0.899 Steps:  14 Acce: 0.278
    Stage:  27 Beta: 1.000 Steps:  14 Acce: 0.250
    Sample initial stage: ...
    Stage:   0 Beta: 0.000 Steps:  25 Acce: 1.000
    Stage:   1 Beta: 0.000 Steps:  25 Acce: 0.210
    Stage:   2 Beta: 0.000 Steps:  19 Acce: 0.213
    Stage:   3 Beta: 0.000 Steps:  19 Acce: 0.220
    Stage:   4 Beta: 0.000 Steps:  18 Acce: 0.259
    Stage:   5 Beta: 0.001 Steps:  15 Acce: 0.231
    Stage:   6 Beta: 0.001 Steps:  17 Acce: 0.211
    Stage:   7 Beta: 0.002 Steps:  19 Acce: 0.237
    Stage:   8 Beta: 0.004 Steps:  16 Acce: 0.260
    Stage:   9 Beta: 0.006 Steps:  15 Acce: 0.251
    Stage:  10 Beta: 0.010 Steps:  15 Acce: 0.259
    Stage:  11 Beta: 0.015 Steps:  15 Acce: 0.229
    Stage:  12 Beta: 0.023 Steps:  17 Acce: 0.229
    Stage:  13 Beta: 0.033 Steps:  17 Acce: 0.228
    Stage:  14 Beta: 0.046 Steps:  17 Acce: 0.261
    Stage:  15 Beta: 0.060 Steps:  15 Acce: 0.275
    Stage:  16 Beta: 0.077 Steps:  14 Acce: 0.285
    Stage:  17 Beta: 0.098 Steps:  13 Acce: 0.296
    Stage:  18 Beta: 0.121 Steps:  13 Acce: 0.289
    Stage:  19 Beta: 0.150 Steps:  13 Acce: 0.293
    Stage:  20 Beta: 0.182 Steps:  13 Acce: 0.297
    Stage:  21 Beta: 0.223 Steps:  13 Acce: 0.282
    Stage:  22 Beta: 0.271 Steps:  13 Acce: 0.293
    Stage:  23 Beta: 0.331 Steps:  13 Acce: 0.283
    Stage:  24 Beta: 0.412 Steps:  13 Acce: 0.301
    Stage:  25 Beta: 0.523 Steps:  12 Acce: 0.281
    Stage:  26 Beta: 0.667 Steps:  13 Acce: 0.280
    Stage:  27 Beta: 0.855 Steps:  14 Acce: 0.259
    Stage:  28 Beta: 1.000 Steps:  15 Acce: 0.262
    Sample initial stage: ...
    Stage:   0 Beta: 0.000 Steps:  25 Acce: 1.000
    Stage:   1 Beta: 0.000 Steps:  25 Acce: 0.203
    Stage:   2 Beta: 0.000 Steps:  20 Acce: 0.204
    Stage:   3 Beta: 0.000 Steps:  20 Acce: 0.203
    Stage:   4 Beta: 0.000 Steps:  20 Acce: 0.219
    Stage:   5 Beta: 0.001 Steps:  18 Acce: 0.248
    Stage:   6 Beta: 0.001 Steps:  16 Acce: 0.229
    Stage:   7 Beta: 0.003 Steps:  17 Acce: 0.251
    Stage:   8 Beta: 0.004 Steps:  15 Acce: 0.259
    Stage:   9 Beta: 0.007 Steps:  15 Acce: 0.267
    Stage:  10 Beta: 0.011 Steps:  14 Acce: 0.255
    Stage:  11 Beta: 0.017 Steps:  15 Acce: 0.233
    Stage:  12 Beta: 0.026 Steps:  17 Acce: 0.229
    Stage:  13 Beta: 0.037 Steps:  17 Acce: 0.235
    Stage:  14 Beta: 0.052 Steps:  17 Acce: 0.257
    Stage:  15 Beta: 0.068 Steps:  15 Acce: 0.287
    Stage:  16 Beta: 0.085 Steps:  13 Acce: 0.267
    Stage:  17 Beta: 0.107 Steps:  14 Acce: 0.300
    Stage:  18 Beta: 0.131 Steps:  12 Acce: 0.287
    Stage:  19 Beta: 0.159 Steps:  13 Acce: 0.321
    Stage:  20 Beta: 0.193 Steps:  11 Acce: 0.302
    Stage:  21 Beta: 0.234 Steps:  12 Acce: 0.311
    Stage:  22 Beta: 0.281 Steps:  12 Acce: 0.303
    Stage:  23 Beta: 0.340 Steps:  12 Acce: 0.276
    Stage:  24 Beta: 0.411 Steps:  14 Acce: 0.283
    Stage:  25 Beta: 0.501 Steps:  13 Acce: 0.280
    Stage:  26 Beta: 0.621 Steps:  14 Acce: 0.270
    Stage:  27 Beta: 0.783 Steps:  14 Acce: 0.271
    Stage:  28 Beta: 1.000 Steps:  14 Acce: 0.250

```python
for d, log_Z in zip(D, log_Zs):
    print('degree %d gives %.4f'%(d, log_Z))
```

    degree 1 gives 25.0894
    degree 2 gives 40.9726
    degree 3 gives 142.5256
    degree 4 gives 139.2879
    degree 5 gives 137.5308

```python
plt.figure(figsize=(10, 5))
_=plt.bar(D, log_Zs, width=0.3)
_=plt.xticks(D)
plt.xticks(fontsize=20)
plt.yticks(fontsize=20)
plt.xlabel('Polynomial degree', fontsize=20)
plt.ylabel('Model Evidence', fontsize=20)
plt.tight_layout()
```

## Questions
+ The model with degree 3 polynomials has the greatest evidence. However, the degree 4 and 5 seem also very plausible. Is this a problem for the theory of Bayesian model selection? What complicates things here, is that model 3 is included in model 4 which is included in model 5. This requires us to design special priors for the models being right. They have to be consistent in some sense. For example, if model 3 is right then model 4 must be right, etc.

+ Revisit the motorcycle dataset problem. Evaluate the model evidence for a 1) Polynomial basis; 2) a Fourier basis; and 3) a Radial basis function basis.
    - <font color='red'> Need to do this </font>

## Revisiting Challenger Disaster Problem (Model Selection)

```python
challenger_data = np.genfromtxt("challenger_data.csv", skip_header=1,
                                usecols=[1, 2], missing_values="NA",
                                delimiter=",")
# drop the NA values
challenger_data = challenger_data[~np.isnan(challenger_data[:, 1])]

# plot it, as a function of temperature (the first column)
print("Temp (F), O-Ring failure?")
print(challenger_data)
```

    Temp (F), O-Ring failure?
    [[66.  0.]
     [70.  1.]
     [69.  0.]
     [68.  0.]
     [67.  0.]
     [72.  0.]
     [73.  0.]
     [70.  0.]
     [57.  1.]
     [63.  1.]
     [70.  1.]
     [78.  0.]
     [67.  0.]
     [53.  1.]
     [67.  0.]
     [75.  0.]
     [70.  0.]
     [81.  0.]
     [76.  0.]
     [79.  0.]
     [75.  1.]
     [76.  0.]
     [58.  1.]]

```python
# plot it, as a function of temperature (the first column)
plt.figure(figsize=(12, 5))
plt.plot(challenger_data[:, 0], challenger_data[:, 1], 'ro',
         markersize=15)
plt.ylabel("Damage Incident?",fontsize=20)
plt.xlabel("Outside temperature (Fahrenheit)",fontsize=20)
plt.title("Defects of the Space Shuttle O-Rings vs temperature",
          fontsize=20)
plt.yticks([0, 1], fontsize=15)
plt.xticks(fontsize=15)
plt.tight_layout()
```

```python
# gather the data and apply preprocessing if any
temp = challenger_data[:, 0]
temp_scaled = (temp - np.mean(temp))/np.std(temp)
data = challenger_data[:, 1]

# instantiate the pymc3 model
challenger_model = pm.Model()

# define the graph
with challenger_model:
    # define the prior
    alpha = pm.Normal('alpha', mu=0., sigma=10.)
    beta = pm.Normal('beta', mu=0., sigma=10.)

    # get the probabilities of failure at each observed temp
    p = pm.Deterministic('p', 1./(1. + tt.exp(alpha + beta*temp_scaled)))

    # define the likelihood
    x = pm.Bernoulli('x', p=p, observed=data)
print("Challenger space shuttle disaster model:")
challenger_model
```

    Challenger space shuttle disaster model:

$$
            \begin{array}{rcl}
            \text{alpha} &\sim & \text{Normal}(\mathit{mu}=0.0,~\mathit{sigma}=10.0)\\\text{beta} &\sim & \text{Normal}(\mathit{mu}=0.0,~\mathit{sigma}=10.0)\\\text{p} &\sim & \text{Deterministic}(\text{Constant},~\text{Constant},~\text{alpha},~\text{beta},~\text{Constant})\\\text{x} &\sim & \text{Bernoulli}(\mathit{p}=\text{p})
            \end{array}
            $$

```python
num_particles = 500
trace, smc = sample_smc(model=challenger_model,
                           draws=num_particles,
                          threshold=0.8,
                          progressbar=False)
```

    Sample initial stage: ...
    Stage:   0 Beta: 0.003 Steps:  25 Acce: 1.000
    Stage:   1 Beta: 0.016 Steps:  25 Acce: 0.583
    Stage:   2 Beta: 0.035 Steps:   5 Acce: 0.425
    Stage:   3 Beta: 0.068 Steps:   8 Acce: 0.311
    Stage:   4 Beta: 0.123 Steps:  12 Acce: 0.262
    Stage:   5 Beta: 0.216 Steps:  15 Acce: 0.285
    Stage:   6 Beta: 0.380 Steps:  13 Acce: 0.264
    Stage:   7 Beta: 0.633 Steps:  15 Acce: 0.250
    Stage:   8 Beta: 1.000 Steps:  16 Acce: 0.286

```python
ppsamples = pm.sample_posterior_predictive(model=challenger_model,
                                           trace=trace,
                                           var_names=['p'])['p']
```

    100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [00:00<00:00, 2832.21it/s]

```python
ppmean = ppsamples.mean(0)
pp_lower, pp_upper = np.percentile(ppsamples, axis=0, q=[2.5, 97.5])

plt.figure(figsize=(15, 8))
plt.plot(temp, data, 'ro', markersize=12, label='Observed data')
idx=np.argsort(temp)
plt.plot(temp[idx], ppmean[idx], linestyle='--', linewidth=2.5,
         label='Post. pred. mean prob.')
plt.fill_between(temp[idx], pp_lower[idx], pp_upper[idx],
                color='purple', alpha=0.25, label='95% Confidence')
plt.ylabel("Probability estimate",fontsize=20)
plt.xlabel("Outside temperature (Fahrenheit)",fontsize=20)
plt.title("Defects of the Space Shuttle O-Rings vs temperature",
          fontsize=20)
plt.yticks(np.arange(0., 1.01, 0.2), fontsize=20)
plt.xticks(fontsize=20)
plt.legend(loc='best', fontsize=20)
plt.tight_layout()
```

```python
logZ_temp = np.log(smc.model.marginal_likelihood)
print(logZ_temp)
```

    -15.478729006011658

_____
End


# Probabilistic Programming 4: Gaussian dynamical systems
## Monte Carlo sampling

### Preliminaries

- Goal
  - Learn to apply Turing to a dynamical system.
- Materials
  - Mandatory
    - These lecture notes.
  - Optional
      - [Probabilistic Programming notebook](https://github.com/bertdv/BMLIP/tree/master/lessons/notebooks/probprog/Probabilistic-Programming.ipynb)
      - David Blei (https://doi.org/10.1146/annurev-statistics-022513-115657)

```julia
# Package managing
using Pkg
Pkg.activate("workspace")
Pkg.instantiate()

using Logging; disable_logging(LogLevel(0))
using Distributions
using StatsPlots
using MCMCChains
using Turing; Turing.setadbackend(:forward_diff);
using Plots
pyplot();
```

## Data generation

In this notebook, we will be considering continuous state-space models. We will focus on the simplest form: the Gaussian dynamical system:

$$\begin{align}
x_k =&\ A x_{k-1} + q_{k-1} \\
y_k =&\ B x_k + r_k \, ,
\end{align}$$

where $x_k \in \mathbb{R}^{N}$ is the state, $y_k \in \mathbb{R}^{M}$ is the measurement. Process noise and measurement noise are assumed to be zero mean Gaussian with covariance matrices $Q$ and $R$ respectively.

We'll generate the data in the same manner as the HMM setting. We start with an initial state $x_0$, apply the state transition $Ax_0$, and add Gaussian noise $q_0$. We then take the current state $x_1$, apply the emission transformation $B x_1$ and add Gaussian noise again $r_1$. This is repeated until we've reached $T$ observations.

```julia
# Dimensionalities
N = 2
M = 2

# Length of time-series
T = 50

# Lenght of time step
Δt = 1.0

# Transition matrix of latent variables
transition = [1.0 Δt;
              0.0 1.0]

# Emission matrix for observed variables
emission = [1.0 0.;0. 1.0]

# Process noise (latent variables)
process_noise = [0.2*Δt 0.;0. 0.1*Δt]

# Measurement noise (observations)
measurement_noise = [0.1 0.;0. 0.1]

# Preallocate data arrays
states = zeros(2,T)
observations = zeros(2,T)

# Initial state
state_0 = [0., 0.]

# Keep previous state in memory
state_tmin = state_0

# Generate data for entire time-series
for t = 1:T

    # Transition from previous state
    states[:,t] = rand(MvNormal(transition * state_tmin, process_noise), 1)[:,1]

    # Emission of current state
    observations[:,t] = rand(MvNormal(emission * states[:,t], measurement_noise), 1)[:,1]

    # Update previous state
    state_tmin = states[:,t]

end

# Visualization of states
plot(states[1,:], states[2,:], color="red", label="states", grid=false, xlabel="position x-coordinate", ylabel="position y-coordinate")
scatter!(observations[1,:], observations[2,:], color="blue", label="observations")
```

Since we don't have a time-axis anymore, it is hard to see how the process evolves. Animating the path is helpful in this regard.

```julia
"Animation of cart's path"

# Plot initial state
plot([states[1,1]], [states[2,1]],
     color="red",
     label="states",
     grid=false,
     xlabel="position x-coordinate",
     ylabel="position y-coordinate",
     xlims=[-60,60],
     ylims=[-10,10])

# Plot initial observation
scatter!([observations[1,1]], [observations[2,1]], color="blue", label="observations")

anim = @animate for t = 2:T

    title!("t = "*string(t))

    # Plot new state
    plot!([states[1,t-1:t]], [states[2,t-1:t]], color="red", label="")

    # Plot new observation
    scatter!([observations[1,t]], [observations[2,t]], color="blue", label="")

end

gif(anim, "example-GDS.gif", fps = 3)
```

## Model specification

The Bayesian formulation of the LGDS state-space model takes the following form:

$$\begin{align}
p(x_k \mid x_{k-1}) =&\ \mathcal{N}(x_k \mid A x_{k-1}, Q)\\
p(y_k \mid x_k) =&\ \mathcal{N}(y_k \mid B x_k, R) \, .
\end{align}$$

We can easily specify the states and observations as multivariate Gaussian distributions. If we do not know the process and measurement noise matrices, we can add Wishart priors to them. Similarly, if we don't know the transition and emission matrices, we can priors for them as well. Since these can take on negative values, we would specify them as multivariate Gaussians;

$$\begin{align}
p(Q) \sim&\ \mathcal{W}(V_Q, n_Q) \\
p(R) \sim&\ \mathcal{W}(V_R, n_R) \\
p(A) \sim&\ \mathcal{N}(m_A, S_A) \\
p(B) \sim&\ \mathcal{N}(m_B, S_B) \, .
\end{align}$$

These are all known distributions to sample from, and they are locally conjugate. However, a Gibbs sampler can be extremely slow in a time-series setting. Plus, with some component samplers, such as Particle Gibss, errors can propogate forward through the time-series, resulting in drifting estimates. We will run an agnostic MCMC procedure in this situation.

Again, we start simple and slowly improve the model.

### Model 1: estimate states

We will first assume we know the transition and emission matrices, as well as measurement and process noises. Based on these, we purely want to estimate states from observations.

```julia
@model Gaussian_dynamical_system(y, A, Q, B, R) = begin
    "This model implicitly assumes 2-dim states and 2-dim observations"

    # Time-series length
    T = size(y, 2)

    # State sequence.
    x = Vector{Vector}(undef, T)

    # Define initial state
    x_0 ~ MvNormal([0., 0.], [1. 0.;0. 1.])

    # Initialize previous state
    x_tmin = x_0

    # Loop over entire sequence
    for t = 1:T

        # State transition
        x[t] ~ MvNormal(A * x_tmin, Q)

        # Observation emission
        y[:,t] ~ MvNormal(B * x[t], R)

        # Update previous state
        x_tmin = x[t]
    end
end

# Call instance of the model
model1 = Gaussian_dynamical_system(observations, transition, process_noise, emission, measurement_noise);
```

We will use a "No U-Turn Sampler". Its parameters are the number of samples to "burn" and the acceptance ratio of the walk.

```julia
# NUTS logger is cumbersome
Turing.turnprogress(false)

# Length of chain
len_chain = 1000

# Define sampler
sampler1 = NUTS(200, 0.6)

# Call sampler
chain1 = sample(model1, sampler1, len_chain);
```

Let's look at the final state of the chain.

```julia
plot(chain1[Symbol("x["*string(T)*"]")])
```

And plot the inferred states.

```julia
# Extract mean of the chain
x_estimated = reshape(mean(chain1[:x].value.data, dims=[1]), (2,50))

# Extract standard error of the mean
x_sem = reshape(std(chain1[:x].value.data, dims=[1]), (2,50))
x_sem[1,:] ./= sqrt.(range(1, stop=T))
x_sem[2,:] ./= sqrt.(range(1, stop=T));

# Visualization
plot(states[1,:], states[2,:], color="red", label="states", grid=false)
plot!(x_estimated[1,:], x_estimated[2,:], ribbon=[x_sem, x_sem], fillalpha=0.2, color="green", label="inferred")
scatter!(observations[1,:], observations[2,:], color="blue", label="observations")
```

### Model 2: process noise estimation

We can extend the model to include process noise estimation.

```julia
@model Gaussian_dynamical_system(y, A, B, R) = begin
    "This model implicitly assumes 2-dim states and 2-dim observations"

    # Time-series length
    T = size(y, 2)

    # Define prior for process noise
    Q ~ Wishart(2, [1. 0.;0. 1.])

    # State sequence.
    x = Vector{Vector}(undef, T)

    # Define initial state
    x_0 ~ MvNormal([0., 0.], [1. 0.;0. 1.])

    # Initialize previous state
    x_tmin = x_0

    # Loop over entire sequence
    for t = 1:T

        # State transition
        x[t] ~ MvNormal(A*x_tmin, Q)

        # Observation emission
        y[:,t] ~ MvNormal(B*x[t], R)

        # Update previous state
        x_tmin = x[t]
    end
end

# Call instance of the model
model2 = Gaussian_dynamical_system(observations, transition, emission, measurement_noise);
```

Stick to the NUTS sampler.

```julia
# Length of chain
len_chain = 1000

# Define sampler
sampler2 = NUTS(500, 0.6)

# Call sampler
chain2 = sample(model2, sampler2, len_chain);
```

Let's have a look at the state estimates. We expect them to be a worse fit compared to the previous run, because we have to estimate states and parameters jointly.

```julia
# Extract mean of the chain
x_estimated = reshape(mean(chain2[:x].value.data, dims=[1]), (2,50))

# Extract standard error of the mean
x_sem = reshape(std(chain2[:x].value.data, dims=[1]), (2,50))
x_sem[1,:] ./= sqrt.(range(1, stop=T))
x_sem[2,:] ./= sqrt.(range(1, stop=T));

# Visualization
plot(states[1,:], states[2,:], color="red", label="states", grid=false)
plot!(x_estimated[1,:], x_estimated[2,:], ribbon=[x_sem, x_sem], fillalpha=0.2, color="green", label="inferred")
scatter!(observations[1,:], observations[2,:], color="blue", label="observations")
```

Let's also have a look at the estimated process noise matrix.

```julia
plot(chain2[:Q])
```

We'll use a bar plot to visualize the coefficients.

```julia
# Extract state estimates
pnoise_estimate = reshape(mean(chain2[:Q].value.data, dims=[1]), (2,2))

# Visualization
l = @layout [a b]
h1 = bar(0:4, reshape(process_noise, (4,)), label="true", ylims=[0., 1.])
h2 = bar(reshape(pnoise_estimate, (4,)), label="estimated", ylims=[0., 1.])
plot(h1, h2, layout=l, size=(1000,500))
```

### Assignment: transition matrix estimation

The transition matrix can be estimated in a similar fashion as the process noise matrix, except that it does not need to be positive semi-definite. Can you construct a model with states, process noise _and_ the transition matrix as free variables?

```julia
### Your code here
```


# Manual definition of Feynman-Kac models

It is not particularly difficult to define manually your own `FeynmanKac` classes. Consider the following problem: we would like to approximate the probability that $X_t \in [a,b]$ for all $0\leq t < T$, where $(X_t)$ is a random walk: $X_0\sim N(0,1)$, and
$$ X_t | X_{t-1} = x_{t-1} \sim N(x_{t-1}, 1).$$

This probability, at time $t$, equals $L_t$, the normalising constant of the following Feynman-Kac sequence of distributions:

\begin{equation}
\mathbb{Q}_t(dx_{0:t}) = \frac{1}{L_t} M_0(dx_0)\prod_{s=1}^t M_s(x_{s-1},dx_s)
\prod_{s=0}^t G_s(x_{s-1}, x_s)
\end{equation}

where:

* $M_0(dx_0)$ is the $N(0,1)$ distribution;
* $M_s(x_{s-1},dx_s)$ is the $N(x_{s-1}, 1)$ distribution;
* $G_s(x_{s-1}, x_s)= \mathbb{1}_{[0,\epsilon]}(x_s)$

Let's define the corresponding `FeymanKac` object:

```python
%matplotlib inline
import warnings; warnings.simplefilter('ignore')  # hide warnings

from matplotlib import pyplot as plt
import seaborn as sb
import numpy as np
from scipy import stats

import particles

class GaussianProb(particles.FeynmanKac):
    def __init__(self, a=0., b=1., T=10):
        self.a, self.b, self.T = a, b, T

    def M0(self, N):
        return stats.norm.rvs(size=N)

    def M(self, t, xp):
        return stats.norm.rvs(loc=xp, size=xp.shape)

    def logG(self, t, xp, x):
        return np.where((x < self.b) & (x > self.a), 0., -np.inf)
```

The class above defines:

* the initial distribution, $M_0(dx_0)$ and the kernels $M_t(x_{t-1}, dx_t)$, through methods `M0(self, N)` and `M(self, t, xp)`. In fact, these methods simulate $N$ random variables from the corresponding distributions.
* Function `logG(self, t, xp, x)` returns the log of function $G_t(x_{t-1}, x_t)$.

Methods `M0` and `M` also define implicitly how the $N$ particles should be represented internally: as a (N,) numpy array. Indeed, at time $0$, method `M0` generates a (N,) numpy array, and at times $t\geq 1$, method `M` takes as an input (`xp`) and returns as an output arrays of shape (N,). We could use another type of object to represent our $N$ particles; for instance, the `smc_samplers` module defines a `ThetaParticles` class for storing $N$ particles representing $N$ parameter values (and associated information).

Now let's run the corresponding SMC algorithm:

```python
fk_gp = GaussianProb(a=0., b=1., T=30)
alg = particles.SMC(fk=fk_gp, N=100)
alg.run()

plt.style.use('ggplot')
plt.plot(alg.summaries.logLts)
plt.xlabel('t')
plt.ylabel(r'log-probability');
```

That was not so hard. However our implementation suffers from several limitations:

1. The SMC sampler we ran may be quite inefficient when interval $[a,b]$ is small; in that case many particles should get a zero weight at each iteration.
2. We cannot currently run the SQMC algorithm (the quasi Monte Carlo version of SMC); to do so, we need to specify the Markov kernels $M_t$ in a different way: not as simulators, but as deterministic functions that take as inputs uniform variates (see below).

Let's address the second point:

```python
class GaussianProb(particles.FeynmanKac):
    du = 1  # dimension of uniform variates

    def __init__(self, a=0., b=1., T=10):
        self.a, self.b, self.T = a, b, T

    def M0(self, N):
        return stats.norm.rvs(size=N)

    def M(self, t, xp):
        return stats.norm.rvs(loc=xp, size=xp.shape)

    def Gamma0(self, u):
        return stats.norm.ppf(u)

    def Gamma(self, t, xp, u):
        return stats.norm.ppf(u, loc=xp)

    def logG(self, t, xp, x):
        return np.where((x < self.b) & (x > self.a), 0., -np.inf)

fk_gp = GaussianProb(a=0., b=1., T=30)
```

We have added:

* methods `Gamma0` and `Gamma`, which define the deterministic functions $\Gamma_0$ and $\Gamma$ we mentioned above. Mathematically, for $U\sim \mathcal{U}([0,1]^{d_u})$, then $\Gamma_0(U)$ is distributed according to $M_0(dx_0)$, and $\Gamma_t(x_{t-1}, U)$ is distributed according to $M_t(x_{t-1}, dx_t)$.
* class attribute `du`, i.e. $d_u$, the dimension of the $u$-argument of functions $\Gamma_0$ and $\Gamma_t$.

We are now able to run both the SMC and the SQMC algorithms that corresponds to the Feyman-Kac model of interest; let's compare their respective performance.  (Recall that function ``multiSMC`` runs several algorithms multiple times,  possibly with varying parameters; here we vary parameter `qmc`, which determines whether we run SMC or SMQC.)

```python
results = particles.multiSMC(fk=fk_gp, qmc={'smc': False, 'sqmc': True}, N=100, nruns=10)

sb.boxplot(x=[r['qmc'] for r in results], y=[r['output'].logLt for r in results]);
```

We do get some variance reduction, but not so much. Let's see if we can do better by addressing point 1 above.

The considered problem has the structure of a state-space model, where process $(X_t)$ is a random walk, $Y_t = \mathbb{1}_{[a,b]}(X_t)$, and $y_t=1$ for all $t$'s. This remark leads us to define alternative Feynman-Kac models, that would correspond to *guided* and *auxiliary* formalisms of that state-space model. In particular, for the guided filter, the  optimal proposal distribution, i.e. the distribution of $X_t|X_{t-1}, Y_t$, is simply a Gaussian distribution truncated to interval $[a, b]$; let's implement the corresponding Feynman-Kac class.

```python
def logprobint(a, b, x):
    """ returns log probability that X_t\in[a,b] conditional on X_{t-1}=x
    """
    return np.log(stats.norm.cdf(b - x) - stats.norm.cdf(a - x))

class Guided_GP(GaussianProb):

    def Gamma(self, t, xp, u):
        au = stats.norm.cdf(self.a - xp)
        bu = stats.norm.cdf(self.b - xp)
        return xp + stats.norm.ppf(au + u * (bu - au))

    def Gamma0(self, u):
        return self.Gamma(0, 0., u)

    def M(self, t, xp):
        return self.Gamma(t, xp, stats.uniform.rvs(size=xp.shape))

    def M0(self, N):
        return self.Gamma0(stats.uniform.rvs(size=N))

    def logG(self, t, xp, x):
        if t == 0:
            return np.full(x.shape, logprobint(self.a, self.b, 0.))
        else:
            return logprobint(self.a, self.b, xp)

fk_guided = Guided_GP(a=0., b=1., T=30)
```

In this particular case, it is a bit more convenient to define methods `Gamma0` and `Gamma` first, and then define methods `M0` and `M`.

To derive the APF version, we must define the auxiliary functions (functions $\eta_t$ in Chapter 10 of the book) that modify the resampling probabilities; in practice, we define the log of these functions,
as follows:

```python
class APF_GP(Guided_GP):
    def logeta(self, t, x):
        return logprobint(self.a, self.b, x)

fk_apf = APF_GP(a=0., b=1., T=30)
```

Ok, now everything is set! We can do a 3x2 comparison of SMC versus SQMC, for the 3 considered Feynman-Kac models.

```python
results = particles.multiSMC(fk={'boot':fk_gp, 'guided':fk_guided, 'apf': fk_apf},
                          N=100, qmc={'smc': False, 'sqmc': True}, nruns=20)

sb.boxplot(x=['%s-%s'%(r['fk'], r['qmc'])  for r in results], y=[r['output'].logLt for r in results]);
```

Let's discard the bootstrap algorithms to better visualise the results for the other algorithms:

```python
res_noboot = [r for r in results if r['fk']!='boot']
sb.boxplot(x=['%s-%s'%(r['fk'], r['qmc'])  for r in res_noboot], y=[r['output'].logLt for r in res_noboot]);
```

Voilà!


Lea cuidadosamente las siguientes **indicaciones** antes de comenzar el examen de prueba:
- Para resolver el examen edite este mismo archivo y renómbrelo de la siguiente manera: *Examen1_ApellidoNombre*, donde *ApellidoNombre* corresponde a su apellido paterno con la inicial en mayúscula, seguido de su primer nombre con la inicial en mayúscula **sin acentos**.
- Resuelva los puntos en el espacio provisto. Si requiere agregar más celdas para código o escritura, hágalo.
- Recuerde que también se está evaluando su capacidad de interpretar los resultados. Escriba sus interpretaciones/conclusiones en celdas utilizando *Markdown*.
- El formato de presentación de todo el examen en general debe ser adecuado. Use tamaños de letra, colores, etiquetas, etcétera.

## Optimización de funciones escalares usando `sympy`

**Enunciado.** Se desea fabricar un caja sin tapa en forma de paralelepípedo con base cuadrada y área lateral de $432$ unidades cuadradas. ¿Qué dimensiones debe tener la caja de máximo volumen?

*Note que las dimensiones son el lado de la base cuadrada $l$ y la altura de la caja $h$*.

- Encuentre el volumen de la caja (paralelepípedo) en función del lado de la base cuadrada $l$, $V(l)$. También provea el dominio de la función para que tenga sentido.

Realice este punto usando fórmulas en LaTeX, en celdas Markdown.

-  Usando `sympy` maximice la función $V(l)$ en su dominio.

- ¿Cuáles son las dimensiones $l$ y $h$ de la caja de volumen máximo?

##  Programación lineal

**Enunciado.** Una fábrica de carrocerías de automóviles y camiones tiene dos sedes. En la sede 1, para hacer la carrocería de un camión se invierten siete (7) días-operario, mientras que para hacer la carrocería de un coche se precisan dos (2) días-operario. En la sede 2 se invierten tres (3) días operario tanto en carrocerías de camión como de coche. Por limitaciones de mano de obra y maquinaria, la sede 1 dispone de 300 días-operario, y la sede 2 de 270 días-operario. Si las ganancias que se obtienen por cada carrocería de camión son de 6 millones de pesos y por cada carrocería de coche 2 millones de pesos, ¿Cuántas unidades de carrocerías de coche y camión se deben producir en cada sede para maximizar las ganancias?

La siguiente tabla resume toda la información.

```python
import pandas as pd
```

```python
df = pd.DataFrame(columns=['Camion', 'Coche', 'Disponible'], index = ['Sede1_dias-operario', 'Sede2_dias-operario', 'Ganancia por unidad'])
df.loc['Sede1_dias-operario', :] = [7, 2, 300]
df.loc['Sede2_dias-operario', :] = [3, 3, 270]
df.loc['Ganancia por unidad', :] = [6, 2, None]
df
```

      Camion
      Coche
      Disponible

      Sede1_dias-operario
      7
      2
      300

      Sede2_dias-operario
      3
      3
      270

      Ganancia por unidad
      6
      2
      None

- Defina las variables, escriba la función a minimizar junto con las restricciones, explicando detalladamente cada paso (usando fórmulas en LaTeX, en celdas Markdown).

- Escriba el problema en la forma
\begin{equation}
\begin{array}{ll}
\min_{\boldsymbol{x}} & \boldsymbol{c}^T\boldsymbol{x} \\
\text{s. a. }        & \boldsymbol{A}_{eq}\boldsymbol{x}=\boldsymbol{b}_{eq} \\
                     & \boldsymbol{A}\boldsymbol{x}\leq\boldsymbol{b},
\end{array}
\end{equation}

proporcionando $\boldsymbol{c}$, $\boldsymbol{A}$ y $\boldsymbol{b}$ ($\boldsymbol{A}_{eq}$ y $\boldsymbol{b}_{eq}$, de ser necesario) como arreglos de `NumPy` (no es necesario escribir en LaTeX el problema en la forma indicada, solo proporcionar las matrices como arreglos de `NumPy`).

Resuelva el problema utilizando la función `linprog` del módulo `optimize` de la librería `scipy`. ¿Cuántas unidades de carrocerías de coche y camión se deben producir en cada sede para maximizar las ganancias?

## Ajuste de curvas

**Enunciado**. El archivo `forest_land_data.csv` contiene información histórica anual de porcentajes de área forestal de México y Colombia desde el año 1990 hasta el año 2014. La primer columna corresponde a los años, la segunda y tercer columna corresponded al porcentaje de área forestal de México y Colombia, respectivamente.

Tomado de: https://data.worldbank.org/indicator/AG.LND.FRST.ZS?locations=MX&view=chart.

-  Importar los datos en un DataFrame de pandas.

- Usando los años como variable independiente $x$ y el porcentaje de área forestal de México como variable dependiente $y$, ajustar polinomios de grado 1 hasta grado 3.

Mostrar en un solo gráfico los datos de población contra los años, y los polinomios ajustados.

Graficar el error cuadrático acumulado contra el número de términos. ¿Cuál es el polinomio que mejor se ajusta?

-  Con el polinomio que eligió en el punto anterior, estime en qué año el porcentaje de área forestal disminuirá al 30%.

Concluya.

## Clasificador binario

**Enunciado.** Se tienen datos de temperatura (grados Celsius) y presion (kPa) de cien muestras de agua. Además, para cada una de ellas se tiene una clasificación si su estado es líquido (1) o vapor (0). Los datos son los siguientes:

```python
import numpy as np
x1 = 40 + 60*np.random.random((100,))
x2 = 0.6 + 100*np.random.random((100,))
X = np.array([x1, x2]).T
Y = (X[:,1] > 0.6*np.exp(17.625*X[:,0]/(X[:,0]+243.04)))*1
```

```python
import matplotlib.pyplot as plt
%matplotlib inline
plt.figure(figsize = (8,6))
plt.scatter(X[:,0], X[:,1], c=Y)
plt.show()
```

- Diseñe un clasificador binario por regresión logística lineal utilizando únicamente los primeros 80 datos. Muestre, además, el gráfico de la división del clasificador con los puntos de entrenamiento.

- Utilice el clasificador diseñado para clasificar a los 20 datos restantes. ¿Cuántas moléculas de agua se clasifican bien? ¿Cuántas mal?


```
%pylab inline
from sympy.interactive import init_printing
init_printing()
from sympy import sqrt, sin, pi, var, integrate, Symbol, S, Integral
var("x y z")
n = 3 * ((x-1)**2 + (y-1)**2 + (z-1)**2 - 1) / pi
Vh = -((x-1)**4 + (y-1)**4 + (z-1)**4) + 2*(x**2+y**2+z**2)
def laplace(f):
    return f.diff(x, 2) + f.diff(y, 2) + f.diff(z, 2)
print "n ="
n
```

```
print "Vh ="
Vh
```

Let's check charge neutrality:

```
integrate(n, (x, 0, 2), (y, 0, 2), (z, 0, 2))
```

$V_H$ is produced by the charge density $n$:

```
laplace(Vh) / (-4*pi)
```

```
_.simplify()
```

The Hartree energy $E_H$ is:

```
integrate(Vh*n, (x, 0, 2), (y, 0, 2), (z, 0, 2))/2
```

We are solving a Poisson equation:
$$\nabla^2 V_H(x, y, z) = -4\pi n(x, y, z)$$
with
$$n(x, y, z) = {3\over\pi} ((x-1)^2 + (y-1)^2 + (z-1)^2 - 1)$$
on the domain $[0, 2] \times [0, 2] \times [0, 2]$ with periodic boundary condition. This charge density is net neutral. The solution is given by:
$$
V_H({\bf x}) = \int {n({\bf y})\over |{\bf x}-{\bf y}|} d^3 y
$$
where ${\bf x}=(x, y, z)$. In reciprocal space
$$
V_H({\bf G}) = 4\pi {n({\bf G})\over G^2}
$$
where ${\bf G}$ are the reciprocal space vectors. I am interested in the Hartree energy:
$$
E_H = {1\over 2} \int {n({\bf x}) n({\bf y})\over |{\bf x}-{\bf y}|} d^3 x d^3 y
= {1\over 2} \int V_H({\bf x}) n({\bf x}) d^3 x
$$
In reciprocal space this becomes (after discretization):
$$
E_H = 2\pi \sum_{{\bf G}\ne 0} {|n({\bf G})|^2\over G^2}
$$
The ${\bf G}=0$ term is omitted, which effectively makes the charge density net neutral (and since it is already neutral, then everything is consistent).

For the test problem above, this can be evaluated analytically and one gets:
$$
E_H = {128\over 35\pi} = 1.16410...
$$
Now we implement the FFT solver in NumPy below and plot the convergence graphs with respect to this exact energy.

This problem has also been discussed at:

http://scicomp.stackexchange.com/questions/7097/convergence-rate-of-fft-poisson-solver/7169

[Computational Science StackExchange](http://scicomp.stackexchange.com/) is a great site to ask these questions!

```
from numpy import empty, pi, meshgrid, linspace, sum
from numpy.fft import fftn, fftfreq
E_exact = 128/(35*pi)
print "Hartree Energy (exact):      %.15f" % E_exact
f = open("conv.txt", "w")
for N in range(3, 200, 10):
    print "N =", N
    L = 2.
    x1d = linspace(0, L, N+1)[:-1]
    x, y, z = meshgrid(x1d, x1d, x1d)

    nr = 3 * ((x-1)**2 + (y-1)**2 + (z-1)**2 - 1) / pi
    ng = fftn(nr) / N**3

    G1d = N * fftfreq(N) * 2*pi/L
    kx, ky, kz = meshgrid(G1d, G1d, G1d)
    G2 = kx**2+ky**2+kz**2
    G2[0, 0, 0] = 1  # omit the G=0 term

    tmp = 2*pi*abs(ng)**2 / G2
    tmp[0, 0, 0] = 0  # omit the G=0 term
    E = sum(tmp) * L**3
    print "Hartree Energy (calculated): %.15f" % E
    f.write("%d %.15f\n" % (N, E))
f.close()
```

    Hartree Energy (exact):      1.164104726615006
    N = 3
    Hartree Energy (calculated): 2.446338611821947
    N = 13
    Hartree Energy (calculated): 1.213786470434117
    N = 23
    Hartree Energy (calculated): 1.179663010471846
    N = 33
    Hartree Energy (calculated): 1.171624324575022
    N = 43
    Hartree Energy (calculated): 1.168524684513086
    N = 53
    Hartree Energy (calculated): 1.167011254992898
    N = 63
    Hartree Energy (calculated): 1.166160629019379
    N = 73
    Hartree Energy (calculated): 1.165635416812430
    N = 83
    Hartree Energy (calculated): 1.165288523120677
    N = 93
    Hartree Energy (calculated): 1.165047479421731
    N = 103
    Hartree Energy (calculated): 1.164873217880501
    N = 113
    Hartree Energy (calculated): 1.164743164819577
    N = 123
    Hartree Energy (calculated): 1.164643537882899
    N = 133
    Hartree Energy (calculated): 1.164565535654283
    N = 143
    Hartree Energy (calculated): 1.164503323719686
    N = 153
    Hartree Energy (calculated): 1.164452910534660
    N = 163
    Hartree Energy (calculated): 1.164411490567212
    N = 173
    Hartree Energy (calculated): 1.164377045217771
    N = 183
    Hartree Energy (calculated): 1.164348092059736
    N = 193
    Hartree Energy (calculated): 1.164323522587376

```
from pylab import plot, semilogy
from numpy import loadtxt, pi
D = loadtxt("conv.txt")
N = D[:, 0]
E = D[:, 1]
E_exact = 128/(35*pi)
```

```
figure(figsize=(8, 6), dpi=80)
semilogy(N, E-E_exact, "k-")
grid()
title("Convergence of an FFT Poisson solver (semilogy)")
xlabel("N (number of PW in each direction)")
ylabel("E  -  E_exact  [a.u.]")
savefig("fft_convergence_semilogy.png")
```

```
figure(figsize=(8, 6), dpi=80)
calculated = E-E_exact
loglog(N, E-E_exact, "ko", label="calculated")
predicted = 1/N**2
predicted = predicted * (calculated[-1] /  predicted[-1])
loglog(N, predicted, "g-", label="$1/N^2$")
grid()
title("Convergence of an FFT Poisson solver (loglog)")
xlabel("N (number of PW in each direction)")
ylabel("E  -  E_exact  [a.u.]")
legend()
savefig("fft_convergence_loglog.png")
```

# Exercise

Rerun the above FFT calculation with the following potential:

```
nr = 3*pi*exp(sin(pi*x)*sin(pi*y)*sin(pi*z))/4
```

Plot convergence study. Why is the convergence rate different?

## Solution

```
from numpy import empty, pi, meshgrid, linspace, sum, exp, sin
from numpy.fft import fftn, fftfreq
f = open("conv2.txt", "w")
for N in range(3, 30, 2):
    print "N =", N
    L = 2.
    x1d = linspace(0, L, N+1)[:-1]
    x, y, z = meshgrid(x1d, x1d, x1d)

    nr = 3*pi*exp(sin(pi*x)*sin(pi*y)*sin(pi*z))/4
    ng = fftn(nr) / N**3

    G1d = N * fftfreq(N) * 2*pi/L
    kx, ky, kz = meshgrid(G1d, G1d, G1d)
    G2 = kx**2+ky**2+kz**2
    G2[0, 0, 0] = 1  # omit the G=0 term

    tmp = 2*pi*abs(ng)**2 / G2
    tmp[0, 0, 0] = 0  # omit the G=0 term
    E = sum(tmp) * L**3
    print "Hartree Energy (calculated): %.15f" % E
    f.write("%d %.15f\n" % (N, E))
f.close()
```

    N = 3
    Hartree Energy (calculated): 1.580426011908313
    N = 5
    Hartree Energy (calculated): 1.414548280420217
    N = 7
    Hartree Energy (calculated): 1.413994578897476
    N = 9
    Hartree Energy (calculated): 1.413991759887669
    N = 11
    Hartree Energy (calculated): 1.413991748461275
    N = 13
    Hartree Energy (calculated): 1.413991748422897
    N = 15
    Hartree Energy (calculated): 1.413991748422800
    N = 17
    Hartree Energy (calculated): 1.413991748422796
    N = 19
    Hartree Energy (calculated): 1.413991748422795
    N = 21
    Hartree Energy (calculated): 1.413991748422802
    N = 23
    Hartree Energy (calculated): 1.413991748422796
    N = 25
    Hartree Energy (calculated): 1.413991748422800
    N = 27
    Hartree Energy (calculated): 1.413991748422803
    N = 29
    Hartree Energy (calculated): 1.413991748422796

```
from pylab import plot, semilogy
from numpy import loadtxt, pi
D = loadtxt("conv2.txt")
N = D[:, 0]
E = D[:, 1]
E_exact = E[-1]

figure(figsize=(8, 6), dpi=80)
calculated = E-E_exact
semilogy(N, E-E_exact, "ko-", label="calculated")
grid()
title("Convergence of an FFT Poisson solver (loglog)")
xlabel("N (number of PW in each direction)")
ylabel("E  -  E_exact  [a.u.]")
legend()
savefig("fft_convergence_loglog.png")
```

```

```