```python
# %load /Users/facai/Study/book_notes/preconfig.py
%matplotlib inline

import matplotlib.pyplot as plt
import seaborn as sns

from IPython.display import SVG
```

逻辑回归在scikit-learn中的实现简介
==============================

分析用的代码版本信息：

```bash
~/W/g/scikit-learn ❯❯❯ git log -n 1
commit d161bfaa1a42da75f4940464f7f1c524ef53484f
Author: John B Nelson <jnelso11@gmu.edu>
Date:   Thu May 26 18:36:37 2016 -0400

    Add missing double quote (#6831)
```

### 0. 总纲

下面是sklearn中逻辑回归的构成情况:

```python
SVG("./res/sklearn_lr.svg")
```

如[逻辑回归在spark中的实现简介](./spark_ml_lr.ipynb)中分析一样，主要把精力定位到算法代码上，即寻优算子和损失函数。

### 1. 寻优算子

sklearn支持liblinear, sag, lbfgs和newton-cg四种寻优算子，其中lbfgs属于scipy包，liblinear属于LibLinear库，剩下两种由sklearn自己实现。代码很好定位，逻辑也很明了，不多说：

```python
 704         if solver == 'lbfgs':
 705             try:
 706                 w0, loss, info = optimize.fmin_l_bfgs_b(
 707                     func, w0, fprime=None,
 708                     args=(X, target, 1. / C, sample_weight),
 709                     iprint=(verbose > 0) - 1, pgtol=tol, maxiter=max_iter)
 710             except TypeError:
 711                 # old scipy doesn't have maxiter
 712                 w0, loss, info = optimize.fmin_l_bfgs_b(
 713                     func, w0, fprime=None,
 714                     args=(X, target, 1. / C, sample_weight),
 715                     iprint=(verbose > 0) - 1, pgtol=tol)
 716             if info["warnflag"] == 1 and verbose > 0:
 717                 warnings.warn("lbfgs failed to converge. Increase the number "
 718                               "of iterations.")
 719             try:
 720                 n_iter_i = info['nit'] - 1
 721             except:
 722                 n_iter_i = info['funcalls'] - 1
 723         elif solver == 'newton-cg':
 724             args = (X, target, 1. / C, sample_weight)
 725             w0, n_iter_i = newton_cg(hess, func, grad, w0, args=args,
 726                                      maxiter=max_iter, tol=tol)
 727         elif solver == 'liblinear':
 728             coef_, intercept_, n_iter_i, = _fit_liblinear(
 729                 X, target, C, fit_intercept, intercept_scaling, None,
 730                 penalty, dual, verbose, max_iter, tol, random_state,
 731                 sample_weight=sample_weight)
 732             if fit_intercept:
 733                 w0 = np.concatenate([coef_.ravel(), intercept_])
 734             else:
 735                 w0 = coef_.ravel()
 736
 737         elif solver == 'sag':
 738             if multi_class == 'multinomial':
 739                 target = target.astype(np.float64)
 740                 loss = 'multinomial'
 741             else:
 742                 loss = 'log'
 743
 744             w0, n_iter_i, warm_start_sag = sag_solver(
 745                 X, target, sample_weight, loss, 1. / C, max_iter, tol,
 746                 verbose, random_state, False, max_squared_sum, warm_start_sag)
```

### 2. 损失函数

#### 2.1 二分类

二分类的损失函数和导数由`_logistic_loss_and_grad`实现，运算逻辑和[逻辑回归算法简介和Python实现](./demo.ipynb)是相同的，不多说。

#### 2.2 多分类

sklearn的多分类支持ovr (one vs rest，一对多)和multinominal两种方式。

##### 2.2.0 ovr
默认是ovr，它会对毎个标签训练一个二分类的分类器，即总共$K$个。训练代码在

```python
1230         fold_coefs_ = Parallel(n_jobs=self.n_jobs, verbose=self.verbose,
1231                                backend=backend)(
1232             path_func(X, y, pos_class=class_, Cs=[self.C],
1233                       fit_intercept=self.fit_intercept, tol=self.tol,
1234                       verbose=self.verbose, solver=self.solver, copy=False,
1235                       multi_class=self.multi_class, max_iter=self.max_iter,
1236                       class_weight=self.class_weight, check_input=False,
1237                       random_state=self.random_state, coef=warm_start_coef_,
1238                       max_squared_sum=max_squared_sum,
1239                       sample_weight=sample_weight)
1240             for (class_, warm_start_coef_) in zip(classes_, warm_start_coef))
```

注意，1240L的`for class_ in classes`配合1232L的`pos_class=class`，就是逐个取标签来训练的逻辑。

##### 2.2.1 multinominal

前面讲到ovr会遍历标签，逐个训练。为了兼容这段逻辑，真正的二分类问题需要做变化：

```python
1201         if len(self.classes_) == 2:
1202             n_classes = 1
1203             classes_ = classes_[1:]
```

同样地，multinominal需要一次对全部标签做处理，也需要做变化：

```python
1217         # Hack so that we iterate only once for the multinomial case.
1218         if self.multi_class == 'multinomial':
1219             classes_ = [None]
1220             warm_start_coef = [warm_start_coef]
```

好，接下来，我们看multinoinal的损失函数和导数计算代码，它是`_multinomial_loss_grad`这个函数。

sklearn里多分类的代码使用的公式和[逻辑回归算法简介和Python实现](./demo.ipynb)里一致，即：

\begin{align}
  L(\beta) &= \log(\sum_i e^{\beta_{i0} + \beta_i x)}) - (\beta_{k0} + \beta_k x) \\
  \frac{\partial L}{\partial \beta} &= x \left ( \frac{e^{\beta_{k0} + \beta_k x}}{\sum_i e^{\beta_{i0} + \beta_i x}} - I(y = k) \right ) \\
\end{align}

具体到损失函数：

```python
244 def _multinomial_loss(w, X, Y, alpha, sample_weight):
 245 #+-- 37 lines: """Computes multinomial loss and class probabilities.---
 282     n_classes = Y.shape[1]
 283     n_features = X.shape[1]
 284     fit_intercept = w.size == (n_classes * (n_features + 1))
 285     w = w.reshape(n_classes, -1)
 286     sample_weight = sample_weight[:, np.newaxis]
 287     if fit_intercept:
 288         intercept = w[:, -1]
 289         w = w[:, :-1]
 290     else:
 291         intercept = 0
 292     p = safe_sparse_dot(X, w.T)
 293     p += intercept
 294     p -= logsumexp(p, axis=1)[:, np.newaxis]
 295     loss = -(sample_weight * Y * p).sum()
 296     loss += 0.5 * alpha * squared_norm(w)
 297     p = np.exp(p, p)
 298     return loss, p, w
```

+ 292L-293L是计算$\beta_{i0} + \beta_i x$。
+ 294L是计算 $L(\beta)$。注意，这里防止计算溢出，是在`logsumexp`函数里作的，原理和[逻辑回归在spark中的实现简介](./spark_ml_lr.ipynb)一样。
+ 295L是加总（注意，$Y$毎列是单位向量，所以起了选标签对应$k$的作用）。
+ 296L加上L2正则。
+ 注意，297L是p变回了$\frac{e^{\beta_{k0} + \beta_k x}}{\sum_i e^{\beta_{i0} + \beta_i x}}$，为了计算导数时直接用。

好，再看导数的计算：

```python
 301 def _multinomial_loss_grad(w, X, Y, alpha, sample_weight):
 302 #+-- 37 lines: """Computes the multinomial loss, gradient and class probabilities.---
 339     n_classes = Y.shape[1]
 340     n_features = X.shape[1]
 341     fit_intercept = (w.size == n_classes * (n_features + 1))
 342     grad = np.zeros((n_classes, n_features + bool(fit_intercept)))
 343     loss, p, w = _multinomial_loss(w, X, Y, alpha, sample_weight)
 344     sample_weight = sample_weight[:, np.newaxis]
 345     diff = sample_weight * (p - Y)
 346     grad[:, :n_features] = safe_sparse_dot(diff.T, X)
 347     grad[:, :n_features] += alpha * w
 348     if fit_intercept:
 349         grad[:, -1] = diff.sum(axis=0)
 350     return loss, grad.ravel(), p
```

+ 345L-346L，对应了导数的计算式；
+ 347L是加上L2的导数；
+ 348L-349L，是对intercept的计算。

#### 2.3 Hessian

注意，sklearn支持牛顿法，需要用到Hessian阵，定义见维基[Hessian matrix](https://en.wikipedia.org/wiki/Hessian_matrix)，

\begin{equation}
{\mathbf  H}={\begin{bmatrix}{\dfrac  {\partial ^{2}f}{\partial x_{1}^{2}}}&{\dfrac  {\partial ^{2}f}{\partial x_{1}\,\partial x_{2}}}&\cdots &{\dfrac  {\partial ^{2}f}{\partial x_{1}\,\partial x_{n}}}\\[2.2ex]{\dfrac  {\partial ^{2}f}{\partial x_{2}\,\partial x_{1}}}&{\dfrac  {\partial ^{2}f}{\partial x_{2}^{2}}}&\cdots &{\dfrac  {\partial ^{2}f}{\partial x_{2}\,\partial x_{n}}}\\[2.2ex]\vdots &\vdots &\ddots &\vdots \\[2.2ex]{\dfrac  {\partial ^{2}f}{\partial x_{n}\,\partial x_{1}}}&{\dfrac  {\partial ^{2}f}{\partial x_{n}\,\partial x_{2}}}&\cdots &{\dfrac  {\partial ^{2}f}{\partial x_{n}^{2}}}\end{bmatrix}}.
\end{equation}

其实就是各点位的二阶偏导。具体推导就不写了，感兴趣可以看[Logistic Regression - Jia Li](http://sites.stat.psu.edu/~jiali/course/stat597e/notes2/logit.pdf)或[Logistic regression: a simple ANN Nando de Freitas](https://www.cs.ox.ac.uk/people/nando.defreitas/machinelearning/lecture6.pdf)。

基本公式是$\mathbf{H} = \mathbf{X}^T \operatorname{diag}(\pi_i (1 - \pi_i)) \mathbf{X}$，其中$\pi_i = \operatorname{sigm}(x_i \beta)$。

```python
 167 def _logistic_grad_hess(w, X, y, alpha, sample_weight=None):
 168 #+-- 33 lines: """Computes the gradient and the Hessian, in the case of a logistic loss.
 201     w, c, yz = _intercept_dot(w, X, y)
 202 #+--  4 lines: if sample_weight is None:---------
 206     z = expit(yz)
 207 #+--  8 lines: z0 = sample_weight * (z - 1) * y---
 215     # The mat-vec product of the Hessian
 216     d = sample_weight * z * (1 - z)
 217     if sparse.issparse(X):
 218         dX = safe_sparse_dot(sparse.dia_matrix((d, 0),
 219                              shape=(n_samples, n_samples)), X)
 220     else:
 221         # Precompute as much as possible
 222         dX = d[:, np.newaxis] * X
 223
 224     if fit_intercept:
 225         # Calculate the double derivative with respect to intercept
 226         # In the case of sparse matrices this returns a matrix object.
 227         dd_intercept = np.squeeze(np.array(dX.sum(axis=0)))
 228
 229     def Hs(s):
 230         ret = np.empty_like(s)
 231         ret[:n_features] = X.T.dot(dX.dot(s[:n_features]))
 232         ret[:n_features] += alpha * s[:n_features]
 233
 234         # For the fit intercept case.
 235         if fit_intercept:
 236             ret[:n_features] += s[-1] * dd_intercept
 237             ret[-1] = dd_intercept.dot(s[:n_features])
 238             ret[-1] += d.sum() * s[-1]
 239         return ret
 240
 241     return grad, Hs
```

+ 201L, 206L, 和216L是计算中间的$\pi_i (1 - \pi_i)$。
+ 217L-222L，对中间参数变为对角阵后，预算公式后半部份，配合231L就是整个式子了。

这里我也只知其然，以后有时间再深挖下吧。

### 3. 小结

本文简单介绍了sklearn中逻辑回归的实现，包括二分类和多分类的具体代码和公式对应。

```python

```


###### Material desenvolvido para o minicurso: Introdução à solução numérica de EDP's, ministrado no ERMAC/2018 de 5 a 6 de abril de 2018 na Universidade Federal de Lavras, Lavras/MG, Brasil. Autor: [Jonas Laerte Ansoni](http://jonasansoni.blogspot.com.br/).

#  Minicurso: Introdução à solução numérica de EDP's

Já vimos em aulas passadas o método de diferenças finitas para equações diferenciais parciais (EDP's) e o caso envolvendo o problema de convecção. Nesta secção vamos resolver um problema dominado por difução, conforme já apresentamos anteriormente.

Por que separamos a discussão de como resolver problemas dominados por convecção e dominados por difusão, você pode perguntar? É tudo sobre a harmonia entre o modelo matemático e o método numérico. Convecção e difusão são processos físicos inerentemente diferentes.

Vamos a dois exemplos:

* _Riding the wave_ - imagine um surfista em uma onda alta, movendo-se rapidamente para a praia ... a convecção implica transporte, velocidade, direção. A física tem um viés direcional, e descobrimos que os métodos numéricos devem ser compatíveis com isso. É por isso que usamos os métodos _upwind_ para a convecção, e prestamos atenção aos problemas onde as ondas se movem em direções opostas, necessitando de esquemas especiais.

* _Spreading out_ - agora imagine uma gota de corante de comida em um copo de água, espalhando-se lentamente em todas as direções até que todo o líquido tome uma cor uniforme. [Diffusion](http://en.wikipedia.org/wiki/Diffusion) espalha a concentração de algo ao redor (átomos, pessoas, idéias, sujeira, qualquer coisa!). Uma vez que não é um processo direcional, precisamos de métodos numéricos isotrópicos (como diferenças centrais).

##  3. **Difusão**

####  Figura 1. Processo de difusão.

### Observações:

No módulo anterior, discutindo as leis de conservação hiperbólica, aprendemos que as soluções possuem certas características: a informação viaja ao longo do caminho no espaço-tempo, ou seja, qualquer perturbação no tempo $t^n$ irá se propagar para os tempos futuros sem se alterar. Em contraste, as equações parabólicas,  o efeito da pertubação se dissipa, isto é, diminui, comforme aumenta a distância ao ponto de origiem. Isso é típico dos processos de difusão.

Nesta primeira lição do módulo, inicialmente analisamos a equação de difusão 1D e, em seguida, examinamos mais profundamente a questão das condições de fronteira. Em seguida, vamos introduzir a _discretização implícita_ o que permitirá a introdução de novos solucionadores lineares. Na próxima aula, abordaremos os casos em duas dimensões com enfoque nas condições de contorno e estabilidade. Em seguida, estudaremos os métodos implícitos 2D, e entraremos no método Crank-Nicolson: talvez o mais popular de todos.

## Vamos voltar ao problema de dispersão de poluentes

### Modelo simplificado: Equação de transporte

\begin{equation}
\frac{\partial c}{\partial t} = -u \frac{\partial c}{\partial x}+ k \frac{\partial^2 c}{\partial x^2}
\end{equation}

 ** Termo advectivo (convectivo) + Temo difusivo **: __Equação de transporte__

No primeiro experimento desligamos a difusão (colocando $k=0$) e testamos apenas a parte de transporte. A condição inicial, calculada internamente no programa, é um pulso quadrado, cujos dados de entrada são os seus pontos inicial e final. Estamos supondo que a concentração de poluente é $\alpha 1,0$. Neste exemplo, adotamos

### 3.1. Equação de difusão unidimensional

\begin{equation}\frac{\partial c}{\partial t}= k \frac{\partial^2 c}{\partial x^2} \tag{1} \end{equation}

onde $k$ é uma constante conhecida como *coeficiente de difusão*.

A primeira coisa que você deve notar é que esta equação possui uma derivada de segunda ordem.

Relembrando da aula de diferenças finitas

A derivada de segunda ordem pode ser representada geometricamente como a linha tangente à curva dada pela primeira derivada. Vamos discernir a derivada de segunda ordem com um esquema de Diferença Central: uma combinação de diferença progressiva e diferença regressiva da primeira derivada. Considere a expansão Taylor de $c_{i+1}$ and $c_{i-1}$ em torno de $c_i$:

$c_{i+1} = c_i + \Delta x \frac{\partial c}{\partial x}\big|_i + \frac{\Delta x^2}{2!} \frac{\partial ^2 c}{\partial x^2}\big|_i + \frac{\Delta x^3}{3!} \frac{\partial ^3 c}{\partial x^3}\big|_i + {\mathcal O}(\Delta x^4)$

$c_{i-1} = c_i - \Delta x \frac{\partial c}{\partial x}\big|_i + \frac{\Delta x^2}{2!} \frac{\partial ^2 c}{\partial x^2}\big|_i - \frac{\Delta x^3}{3!} \frac{\partial ^3 c}{\partial x^3}\big|_i + {\mathcal O}(\Delta x^4)$

e finalmente:

\begin{equation}\frac{\partial ^2 c}{\partial x^2}=\frac{c_{i+1}-2c_{i}+c_{i-1}}{\Delta x^2} + {\mathcal O}(\Delta x^2) \tag{3} \end{equation}

A aproximação da derivada de 2ª ordem por diferença de diferença central tem precisão de ordem 2.

### Voltando à equação de difusão unidimensional

Agora podemos escrever a versão discretizada da equação de difusão em 1D na sua forma *explícita*:

\begin{equation}\frac{c_{i}^{n+1}-c_{i}^{n}}{\Delta t}=k\frac{c_{i+1}^{n}-2c_{i}^{n}+c_{i-1}^{n}}{\Delta x^2}\tag{4} \end{equation}

Rearranjando obtemos a expressão para obter o termo $c_{i}^{n+1}$ a ser calculado:

\begin{equation}c_{i}^{n+1}=c_{i}^{n}+k\frac{\Delta t}{\Delta x^2}(c_{i+1}^{n}-2c_{i}^{n}+c_{i-1}^{n}) \label{eqExp} \tag{5}\end{equation}

Esta equação discreta nos permite escrever um programa que avança uma solução no tempo, mas precisamos de uma condição inicial.

### _Pergunta:_  Como é o estêncil do método?

### Resposta:

### _Pergunta:_  O método apresentado é explícito ou implícito?

### Estabilidade da equação de difusão

A equação de difusão não está livre de restrições de estabilidade. Assim como as equações de convecção linear e não-linear, há um conjunto de parâmetros de discretização $\Delta x$ e $\Delta t$ que farão explodir a solução numérica. Para a equação de difusão e a discretização utilizada aqui, a condição de estabilidade para difusão é

$$
\begin{equation}
k \frac{\Delta t}{\Delta x^2} \leq \frac{1}{2}
\end{equation}
$$

Logo veremos o por que desse valor.

### Exemplo 1:

Vamos considerar uma função de onda quadrada onde: Em $t=0$, $c=2$ no intervalao $0.5\le x\le 1$ e $c=1$ em qualquer outro lugar.

### Vamos calcular!

As próximas duas células de código inicializam o problema carregando as bibliotecas necessárias, depois definindo os parâmetros da solução e a condição inicial. Desta vez, não deixamos que o usuário escolha *qualquer* $\Delta t$, no entanto; decidimos que isso não é seguro: as pessoas simplesmente gostam de explodir as coisas. Em vez disso, o código calcula um valor de $\Delta t$ que estará no intervalo estável, de acordo com a discretização espacial escolhida! Agora você pode experimentar diferentes parâmetros de solução para ver como a solução numérica muda, mas não explodirá.

```python
import numpy
from matplotlib import pyplot
%matplotlib inline
from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16
```

```python
nx = 41
dx = 2./(nx-1)
nt = 20
k = 0.3   #the value of viscosity
sigma = .2
dt = sigma*dx**2/k

x = numpy.linspace(0,2,nx)
ubound = numpy.where(x >= 0.5)
lbound = numpy.where(x <= 1)

c = numpy.ones(nx)
c[numpy.intersect1d(lbound, ubound)] = 2

cn = numpy.ones(nx)
```

```python
for n in range(nt):
    cn = c.copy()
    c[1:-1] = cn[1:-1] + k*dt/dx**2*(cn[2:] -2*cn[1:-1] +cn[0:-2])

pyplot.plot(x, c, color='#003366', ls='--', lw=3)
pyplot.ylim(0,2.5);
```

### Animação do gráfico

NOTA:  Temos que instalar o video encoder video encoder/decoder chamando `ffmpeg`.

Se você utiliza Linux or OSX, você pode instalar o `ffmpeg` via `conda`:

```console
conda install -c conda-forge ffmpeg
```

Se você utiliza o Windows, sigua as instruções de instalação do link:
http://adaptivesamples.com/how-to-install-ffmpeg-on-windows/

```python
from matplotlib import animation
from IPython.display import HTML
```

Agora precisamos reiniciar nossas condições iniciais!!

```python
nt = 50

c = numpy.ones(nx)
c[numpy.intersect1d(lbound, ubound)] = 2

cn = numpy.ones(nx)
```

```python
fig = pyplot.figure(figsize=(8,5))
ax = pyplot.axes(xlim=(0,2), ylim=(1,2.5))
line = ax.plot([], [], color='#003366', ls='--', lw=3)[0];
```

```python
def diffusion(i):
    line.set_data(x,c)

    cn = c.copy()
    c[1:-1] = cn[1:-1] + k*dt/dx**2*(cn[2:] -2*cn[1:-1] + cn[0:-2])
```

Inicializamos a animação utilizando `animation.FuncAnimation()` e salvando a animação com o nome de `anim`. Os seguintes argumentos da função são:

*  `fig`: nome da figura
*  `diffusion`: nome da função que possui o solver
*  `frames`: número de frames para salvar a figura (igual à variável `nt`)
*  `interval`: número de milissegundos para cada quadro aparecer

```python
anim = animation.FuncAnimation(fig, diffusion, frames=nt, interval=100)
```

Ok! Hora de exibir a animação. Usamos o método de exibição `HTML` que importamos acima e o método `to_html5_video` para torná-lo compatível com a web.

```python
HTML(anim.to_html5_video())
```

### Vamos ver um exemplo clássico de equação parabólica no próximo _notebook_!!

```python
from IPython.core.display import HTML
css_file = '../styles/custom.css'
HTML(open(css_file, "r").read())
```


# Advanced: Extending lambeq

## Creating readers

### Reader example: "Comb" reader

In this example we will create a reader that, given a sentence, it generates the following tensor network:

<center>

</center>

Note that the particular compositional model is not appropriate for classical experiments, since the tensor that implements the layer can become very large for long sentences. However, the model can be implemented without problems on a quantum computer.

```python
from lambeq import AtomicType, Reader
from discopy import Box, Id, Word

N = AtomicType.NOUN

class CombReader(Reader):
    def sentence2diagram(self, sentence):
        words = Id().tensor(*[Word(w, N) for w in sentence.split()])
        layer = Box('LAYER', words.cod, N)
        return words >> layer

diagram = CombReader().sentence2diagram('John gave Mary a flower')
diagram.draw()
```

```python
Id().tensor(*[Word(w, N) for w in ['John', 'gave', 'Mary', 'a', 'flower']]).draw()
```

## Creating rewrite rules

```python
from lambeq import BobcatParser

parser = BobcatParser(verbose='text')
d = parser.sentence2diagram('The food is fresh')
```

### SimpleRewriteRule example: Negation functor

```python
from lambeq import AtomicType, SimpleRewriteRule
from discopy.rigid import Box, Id

N = AtomicType.NOUN
S = AtomicType.SENTENCE
adj = N @ N.l

NOT = Box('NOT', S, S)

negation_rewrite = SimpleRewriteRule(
    cod=N.r @ S @ S.l @ N,
    template=SimpleRewriteRule.placeholder(N.r @ S @ S.l @ N) >> Id(N.r) @ NOT @ Id(S.l @ N),
    words=['is', 'was', 'has', 'have'])
```

```python
from lambeq import Rewriter
from discopy import drawing

not_d = Rewriter([negation_rewrite])(d)
drawing.equation(d, not_d, symbol='->', figsize=(14, 4))
```

### RewriteRule example: "Past" functor

```python
from lambeq import RewriteRule

class PastRewriteRule(RewriteRule):
    mapping = {
        'is': 'was',
        'are': 'were',
        'has': 'had'
    }
    def matches(self, box):
        return box.name in self.mapping

    def rewrite(self, box):
        new_name = self.mapping[box.name]
        return type(box)(name=new_name, dom=box.dom, cod=box.cod)
```

```python
past_d = Rewriter([PastRewriteRule()])(d)
drawing.equation(d, past_d, symbol='->', figsize=(14, 4))
```

## Creating ansätze

```python
d = parser.sentence2diagram('We will go')
```

### CircuitAnsatz example: "Real-valued" ansatz

```python
from discopy.quantum.circuit import Functor, Id
from discopy.quantum.gates import Bra, CX, Ket, Ry
from lambeq import CircuitAnsatz
from lambeq.ansatz import Symbol

class RealAnsatz(CircuitAnsatz):
    def __init__(self, ob_map, n_layers):
        super().__init__(ob_map=ob_map, n_layers=n_layers)

        self.n_layers = n_layers
        self.functor = Functor(ob=self.ob_map, ar=self._ar)

    def _ar(self, box):
        # step 1: obtain label
        label = self._summarise_box(box)

        # step 2: map domain and codomain
        dom, cod = self._ob(box.dom), self._ob(box.cod)

        n_qubits = max(dom, cod)
        n_layers = self.n_layers

        # step 3: construct and return ansatz
        if n_qubits == 1:
            circuit = Ry(Symbol(f'{label}_0'))
        else:
            # this also deals with the n_qubits == 0 case correctly
            circuit = Id(n_qubits)
            for i in range(n_layers):
                offset = i * n_qubits
                syms = [Symbol(f'{label}_{offset + j}') for j in range(n_qubits)]

                # adds a ladder of CNOTs
                for j in range(n_qubits - 1):
                    circuit >>= Id(j) @ CX @ Id(n_qubits - j - 2)

                # adds a layer of Y rotations
                circuit >>= Id().tensor(*[Ry(sym) for sym in syms])

        if cod <= dom:
            circuit >>= Id(cod) @ Bra(*[0]*(dom - cod))
        else:
            circuit <<= Id(dom) @ Ket(*[0]*(cod - dom))
        return circuit
```

```python
real_d = RealAnsatz({N: 1, S: 1}, n_layers=2)(d)
real_d.draw(figsize=(12, 10))
```

### TensorAnsatz example: "Positive" ansatz

```python
from lambeq import TensorAnsatz
from discopy import rigid, tensor
from functools import reduce

class PositiveAnsatz(TensorAnsatz):
    def _ar(self, box):
        # step 1: obtain label
        label = self._summarise_box(box)

        # step 2: map domain and codomain
        dom, cod = self._ob(box.dom), self._ob(box.cod)

        # step 3: construct and return ansatz
        name = self._summarise_box(box)
        n_params = reduce(lambda x, y: x * y, dom @ cod, 1)
        syms = Symbol(name, size=n_params)
        return tensor.Box(box.name, dom, cod, syms ** 2)
```

```python
from discopy import Dim

ansatz = PositiveAnsatz({N: Dim(2), S: Dim(2)})
positive_d = ansatz(d)
positive_d.draw()
```

```python
import numpy as np
from sympy import default_sort_key

syms = sorted(positive_d.free_symbols, key=default_sort_key)
sym_dict = {k: -np.ones(k.size) for k in syms}
subbed_diagram = positive_d.lambdify(*syms)(*sym_dict.values())

subbed_diagram.eval()
```

    Tensor(dom=Dim(1), cod=Dim(2), array=[8., 8.])

## Contributions


```python
%matplotlib inline
from matplotlib.pylab import *
```

# Python Fundamentals

* If you have not used python, I _hope_ you will find it is easy to do easy things with it.
* Experience with any other language will help you get far, but there are some unusual python idioms that you will encounter as you go.
* We use **python 3**, not python 2. Many of the online resources you find will be for python2. The languages are _nearly_ identical but different enought that you will not be able to copy-paste what you find. The main difference is `print` is a function (not a keyword) in python3; also there are differences in the names of modules and the way division works, etc.
* Python has some easter eggs; `import this` is an easter egg that explains python's philosophy succinctly.

```python
import this
```

    The Zen of Python, by Tim Peters

    Beautiful is better than ugly.
    Explicit is better than implicit.
    Simple is better than complex.
    Complex is better than complicated.
    Flat is better than nested.
    Sparse is better than dense.
    Readability counts.
    Special cases aren't special enough to break the rules.
    Although practicality beats purity.
    Errors should never pass silently.
    Unless explicitly silenced.
    In the face of ambiguity, refuse the temptation to guess.
    There should be one-- and preferably only one --obvious way to do it.
    Although that way may not be obvious at first unless you're Dutch.
    Now is better than never.
    Although never is often better than *right* now.
    If the implementation is hard to explain, it's a bad idea.
    If the implementation is easy to explain, it may be a good idea.
    Namespaces are one honking great idea -- let's do more of those!

# The notebook interface

You are using a Jupyter Notebook interface to python.

In a notebook, you execute _cells_ of code.

The output of the last statement is displayed when you execute a cell.

You execute a cell by pressing SHIFT+ENTER

```python
3+3
```

    6

You navigate by pressing ESC and then a to add a call before

## Fundamentals

Primitive types are `int`, `float`, `str`, `list`, `tuple`, `dict`, and `set`.

(There are others -- I am not trying to be comprehensive, ok!)

You don't (usually) need to assign types; python guesses the types of the values you will be using.
_Variables_ refer to objects of _any_ type.

```python
x = 3
print(x)
```

    3

```python
print(type(x))
```

    <class 'int'>

```python
x = '3'
x
```

    '3'

```python
3*x
```

    '333'

As you can see, objects _do_ have types and their behavior depends on their type!!

```python
assert isinstance(x, int), "Python can do typechecking if you ask it to"
```

Most types work the way you _probably_ expect.

```python
x = 4
x**2
```

    16

```python
x ** 0.5
```

    2.0

```python
x % 3
```

    1

```python
-x % 3  # (-6 is the least multiple of 3 < -4)
```

    2

## Tuples

Tuples are a _read-only_ group of values

```python
my_tuple = 1, 2, 3
```

```python
my_tuple
```

    (1, 2, 3)

```python
my_tuple[1]
```

    2

```python
my_tuple[1] = 4  # Expect an error
```

```python
my_other_tuple = 1, 'banana', 3.14159
```

```python
my_other_tuple
```

    (1, 'banana', 3.14159)

```python
x = 1,    # <-- note the trailing comma!
isinstance(x, tuple)
```

    True

```python
x
```

    (1,)

## Loops

In general, loops are to be avoided unless it is awkward to do something without them.

Loops in python are all _foreach_ loops.

```python
for i in 1, 2, 3, 4:  #<-- that is a tuple by the way
    print(i)
```

    1
    2
    3
    4

```python
for x in range(4):
    print(x)
```

    0
    1
    2
    3

```python
for y in range(3, 12, 3):
    print(y, end=' ')
```

    3 6 9

Python also has `break` and `continue` keywords that work as you would expect.

## Lists

```python
my_list = [1, 2, 3, 4]
```

```python
my_list[0]
```

    1

```python
my_list[4]  # Expect 4 is out of bounds
```

    'a'

```python
my_list[-1]  # Indexing in reverse!
```

    4

```python
my_list[2]
```

    3

Slicing is a very powerful way to manipulate lists in python
The fommat is `my_list[start:end:step]` to return a subsequence.

You can think of it as:
```
    for (int i = start; i != end; i += step)
        result.append(a reference to original[i])
```

```python
my_list[:2]
```

    [1, 2]

```python
my_list[2:]
```

    [3, 4]

```python
my_list[1:3]
```

    [2, 3]

```python
my_list[::-1]
```

    [4, 3, 2, 1]

```python
print(my_list.pop())
print(my_list)
```

    4
    [1, 2, 3]

```python
my_list.append(12)
print(my_list)
```

    [1, 2, 3, 12]

```python
my_list += ['a', 'b', 'c']
my_list
```

    [1, 2, 3, 12, 'a', 'b', 'c']

```python
my_list*2
```

    [1, 2, 3, 12, 'a', 'b', 'c', 1, 2, 3, 12, 'a', 'b', 'c']

## Sets

```python
s = {1, 2, 3, 'a'}
```

```python
1 in s
```

    True

```python
4 in s
```

    False

```python
4 not in s
```

    True

```python
t = {2, 3, 4, 5}
```

```python
s.intersection(t)
```

    {2, 3}

```python
s.union(t)
```

    {1, 2, 3, 4, 5, 'a'}

```python
s.add(12)
```

```python
s
```

    {1, 12, 2, 3, 'a'}

## Dictionaries

```python
x = dict(a = 1, b=2, c= 'banana')
```

```python
x
```

    {'a': 1, 'b': 2, 'c': 'banana'}

```python
x['a']
```

    1

```python
print(x.keys())
print(x.values())
print(x.items())
```

    dict_keys(['a', 'b', 'c'])
    dict_values([1, 2, 'banana'])
    dict_items([('a', 1), ('b', 2), ('c', 'banana')])

```python
x.get('a', 123)
```

    1

```python
x.get('notinthedict', 123)
```

    123

```python
x['newkey'] = 321
```

```python
'newkey' in x
```

    True

```python
'newkey' not in x
```

    False

```python
for key in x:  #<-- only iterates ver keys
    print(key)
```

    a
    b
    c
    newkey

```python
x.update(c = 4, d = 5)
x
```

    {'a': 1, 'b': 2, 'c': 4, 'newkey': 321, 'd': 5}

Sometimes it is useful to `pop` (read + remove) a value

```python
print(x)
a = x.pop('a', 0)
print("a=", a)
print(x)
```

    {'a': 1, 'b': 2, 'c': 4, 'newkey': 321, 'd': 5}
    a= 1
    {'b': 2, 'c': 4, 'newkey': 321, 'd': 5}

## Functions

```python
def foo():
    print("Hi, I'm a function")
```

```python
foo()
```

    Hi, I'm a function

```python
def foo(x):
    print("You passed in", x)
```

```python
foo(3)
```

    You passed in 3

```python
def foo(x, y=0, z=0):
    print("You passed in", (x, y, z))
foo(3)
```

    You passed in (3, 0, 0)

```python
foo(3, 4)
```

    You passed in (3, 4, 0)

```python
foo(3, z=12)
```

    You passed in (3, 0, 12)

```python
def foo(*args):
    print("You passed in", args)
foo(1, 2, 3, 4)
```

    You passed in (1, 2, 3, 4)

```python
foo(1, 2, z=7)  # Expect an error
```

```python
def foo(*args, **kwargs):
    print("You passed in", args, "and", kwargs)
```

```python
foo(1, 2, x=6, y=8)
```

    You passed in (1, 2) and {'x': 6, 'y': 8}

```python
def add(x=0, y=0, z=0):
    return x + y + z
```

```python
add(3, 4)
```

    7

```python
args = (1,2, 3)
add(*args)  #Unpacking
```

    6

```python
args = dict(x=3, y=4, z=5)
add(**args) #Unpacking named arguments
```

    12

```python
def add(*args, **kwargs):
    """Adds all of the arguments

    Parameters
    ----------
    args:
        A list of arguments
    kwargs:
        A list of named arguments (names are ignored)

    Returns
    -------
    total: The sum of all arguments.

    Example
    -------
    >>> add(1, 2, x=3, y=4)
    10
    """
    return (sum(args)
            + sum(list(kwargs.values()))) # <-- converting to list...
```

```python
add(1, 2, x=3, y=4)
```

    10

```python
help(add)
```

    Help on function add in module __main__:

    add(*args, **kwargs)
        Adds all of the arguments

        Parameters
        ----------
        args:
            A list of arguments
        kwargs:
            A list of named arguments (names are ignored)

        Returns
        -------
        total: The sum of all arguments.

        Example
        -------
        >>> add(1, 2, x=3, y=4)
        10

Try typing `add` SHIFT TABTABTABTAB

```python
def foo(x:int, y:str)->str:
    """ This function has type annotations.
    They are not used by python, but other tools
    that analyse your code use them.
    """
    return "Blegh"
```

```python
def foo():
    """Generate pseudo random numbers (forever)"""
    while True:
        yield (345 + i * 12345)%(5267) / 5266.
```

```python
foo()
```

    <generator object foo at 0x7fb44058f840>

```python
foo().__next__()
```

    0.4409418913786555

```python
for i,  x in enumerate(foo()):
    print("{:03}: {:.2f}".format(i, x))
    if i > 10:
        break

```

    000: 0.44
    001: 0.07
    002: 0.41
    003: 0.75
    004: 0.10
    005: 0.44
    006: 0.78
    007: 0.13
    008: 0.47
    009: 0.82
    010: 0.16
    011: 0.50

```python

```

## Classes

```python
class Fizz(object):
    def __init__(self, x, y=3): #Constructor
        super().__init__()  # Explicitly call super-class'es constructor
        self.x = x
        self.y = y
        self._private = 0

    def buzz(self):
        print(f"Bzzzz -- also x={self.x} and y={self.y}")
```

```python
f = Fizz(2)
f.buzz()
```

    Bzzzz -- also x=2 and y=3

Try typeing `f.`TAB, notice that `_private` is not suggested.

```python

```

You can _still_ access `f._private`; but you shouldn't unless you are the author of Fizz or you are ready for the repercussins(!)

```python
f._private
```

    0

Many tools will generate a warning if you access pseudo-private members.

```python
class Fizz(object):
    def __init__(self, x, y=3): #Constructor
        super().__init__()  # Explicitly call super-class'es constructor
        self.x = x
        self.y = y
        self._private = 0

    def buzz(self):
        print(f"Bzzzz -- also x={self.x} and y={self.y}")

    @property
    def notsoprivate(self):
        return self._private

    @notsoprivate.setter
    def notsoprivate(self, value):
        if value > 0 and value != self._private:
            self._private = value
            # Maybe do some other stuff here?
        else:
            raise ValueError("notsoprivate must be positive")
```

```python
f = Fizz(2)
```

```python
f.notsoprivate
```

    0

```python
f.notsoprivate = -2  # Expect a ValueError
```

## Special methods

Names with dunders (double undercores) are special

```python
help(dict.__getitem__)
```

    Help on method_descriptor:

    __getitem__(...)
        x.__getitem__(y) <==> x[y]

```python
help(str.__len__)
```

    Help on wrapper_descriptor:

    __len__(self, /)
        Return len(self).

```python
class Foo(object):
    def __repr__(self):
        return "This is how I look in the REPL"

    def __str__(self):
        return "This is what I look like when printed or converted to a string"
```

```python
f = Foo()
```

```python
f
```

    This is how I look in the REPL

```python
print(f)
```

    This is what I look like when printed or converted to a string

```python
class Foo(object):
    def __repr__(self):
        return "This is how I look in the REPL"

    def __str__(self):
        return "This is what I look like when printed or converted to a string"

    def __len__(self):
        return 3

    def __getitem__(self, index):
        return "abc"[index]

    def _repr_html_(self):
        return "This <font color=red>is what Jupyter</font> will show!"
```

```python
f = Foo()
f
```

This is what Jupyter will show!

```python
len(f)
```

    3

```python
f[0]
```

    'a'

# Arrays, Vectors, Matrices

Let $\mathbf{A}$ be a random $2 \times 2$ matrix

```python
A = np.random.randint(0, 10, (2,2)).astype(float);
print(A)
```

    [[9. 1.]
     [2. 0.]]

Let $\mathbf{x}$ be a random vector of two elements.

```python
x = np.random.randint(0, 10, 2).astype(float);
print(x)
```

    [5. 5.]

The _matix product_ $\mathbf{A}\mathbf{x}$ is

```python
y = A.dot(x)
y
```

    array([50., 10.])

You can verify this for the first element

```python
A[0,0]*x[0] + A[0,1]*x[1]
```

    50.0

Is it **NOT** this:

```python
A * x
```

    array([[45.,  5.],
           [10.,  0.]])

Operators on _arrays_ are all _elementwise_ by default.

# Matrix Inversion & Solving

```python
x_ = np.linalg.solve(A, y)
print(x_)
```

    [5. 5.]

```python
A_inv = np.linalg.inv(A)
print(A_inv)
```

    [[ 0.   0.5]
     [ 1.  -4.5]]

```python
print(A_inv.dot(y))
```

    [5. 5.]

# Eigenvalues

```python
evals, evecs = np.linalg.eig(A)
print(np.diag(evals))
print("---- ")
print(evecs)
```

    [[ 9.21699057  0.        ]
     [ 0.         -0.21699057]]
    ----
    [[ 0.97725759 -0.1078623 ]
     [ 0.21205568  0.99416584]]

The _**columns**_ of the eigenvector matrix ($\mathbf{\Lambda}$) are eigenvectors.

```python
v0 = evecs[:,0]
v1 = evecs[:,1]
print("v0:", v0)
print("v1:", v1)
```

    v0: [0.97725759 0.21205568]
    v1: [-0.1078623   0.99416584]

The _**eigenvectors**_ are the vectors that don't change direction when multiplied by $\mathbf{A}$

```python
x, y = np.mgrid[-3:3:20j, -3:3:20j]
u, v = A.dot([x.flatten(), y.flatten()])

quiver(x, y, u, v, color='red')
quiver(x, y, x, y)
arrow(0, 0, *evecs[:,0], width=0.05)
arrow(0, 0, *evecs[:,1], width=0.05)
axis('equal');
```

The eigendecomposition is $\mathbf{A} = \mathbf{Q}\mathbf{\Lambda}\mathbf{Q}^{-1}$

```python
(evecs*evals).dot(np.linalg.inv(evecs))
```

    array([[ 9.00000000e+00,  1.00000000e+00],
           [ 2.00000000e+00, -5.55111512e-17]])

```python
print(A)
```

    [[9. 1.]
     [2. 0.]]

# Singular Value Decomposition

The SVD is $ \mathbf{A} = \mathbf{U} \mathbf{\Sigma} \mathbf{V}^T $.
The matrix $\mathbf{\Sigma} = \texttt{diag}(\pmb{\sigma})$ has the _singular values_.

```python
A = np.random.randint(0, 10, (5, 10)).astype(float)
A
```

    array([[0., 3., 8., 3., 2., 2., 9., 6., 7., 6.],
           [3., 7., 8., 0., 7., 4., 9., 7., 2., 4.],
           [6., 9., 5., 2., 6., 6., 4., 8., 1., 8.],
           [8., 9., 2., 6., 3., 2., 5., 0., 4., 8.],
           [6., 4., 6., 1., 0., 3., 3., 2., 1., 9.]])

```python
U, sigma, VT = np.linalg.svd(A, full_matrices=False)
V = VT.T
print(U.shape)
print(len(sigma))
print(V.shape)
```

    (5, 5)
    5
    (10, 5)

```python
print(sigma)
```

    [35.00648313 11.67178935  8.60357175  6.28330328  3.57968224]

The matrices $\pmb{U}$ and $\pmb{V}$ are _orthonormal_

```python
U.T.dot(U).round()+0
```

    array([[1., 0., 0., 0., 0.],
           [0., 1., 0., 0., 0.],
           [0., 0., 1., 0., 0.],
           [0., 0., 0., 1., 0.],
           [0., 0., 0., 0., 1.]])

```python
V.T.dot(V).round()+0
```

    array([[1., 0., 0., 0., 0.],
           [0., 1., 0., 0., 0.],
           [0., 0., 1., 0., 0.],
           [0., 0., 0., 1., 0.],
           [0., 0., 0., 0., 1.]])

The SVD allows us to _approximate_ the input matrix $\mathbf{A}$ using a small number of vectors.

```python
imshow(A, cmap=cm.gray, vmin=0, vmax=10);
```

Try it out with a different number of singular values; change `t` below to see the effect

```python
t = 2
imshow((U[:,:t]*sigma[:t]).dot(V[:,:t].T), cmap=cm.gray, vmin=0, vmax=10);
```

This can even be used for image compression

```python
import skimage.data, skimage.color
```

```python
A = skimage.data.astronaut()
A = skimage.color.rgb2gray(A)
```

```python
imshow(A, cmap=cm.gray)
```

```python
U, sigma, VT = np.linalg.svd(A, full_matrices=False)
V = VT.T
print(U.shape, V.shape)
```

    (512, 512) (512, 512)

```python
t = 40
imshow((U[:,:t]*sigma[:t]).dot(VT[:t]), cmap=cm.gray)

original_size = A.size
compressed_size = U[:,t].size + t + VT[:t].size
ratio = original_size/compressed_size
print("From", A.size, "to", compressed_size , "values, ",round(ratio), "to 1")
```

## Another useful property

```python
figure(figsize(4,8))

subplot(121)
imshow(U.T.dot(U).round(), cmap=cm.binary)
title('$\mathbf{U}^T\mathbf{U}$');
xticks([]); yticks([])

subplot(122)
imshow(V.T.dot(V).round(), cmap=cm.binary)
title('$\mathbf{V}^T\mathbf{V}$');
xticks([]); yticks([]);

```

We have:
$$ \pmb{U}^{-1} = \pmb{U}^T$$
$$ \pmb{V}^{-1} = \pmb{U}^T$$
$$ \pmb{\Sigma}^{-1} = \texttt{diag}(1/\sigma_i)$$

And a property of matrices
$$(\pmb{U}\pmb{\Sigma}\pmb{V}^T)^{-1} = \pmb{V}^{-T} \pmb{\Sigma}^{-1}\pmb{U}^{-1} = \pmb{V}\pmb{\Sigma}^{-1}\pmb{U}^T$$

So _**this factorization is easy and numerically stable to invert**_

Also we have $\sigma_i^2 = \lambda_i(\pmb{A}^T\pmb{A})$, that is, the ith singular value is the square root of the ith eigenvalue of $\pmb{A}$ squared.

```python
evals, evecs = np.linalg.eig(A.T.dot(A))
print(evals[:5].round())
print((sigma[:5]**2).round())
```

# Other Factorizations

$\pmb{A} = \pmb{Q}\pmb{R}$ -- where $\pmb{Q}^T = \pmb{Q}^{-1}$ and $\pmb{R}$ is upper triangular (easy to solve)

The QR decomposition is useful for computing the SVD, and for solving poorly condition least-squares problems

$\pmb{A} = \pmb{R}^T\pmb{R}$ -- where $\pmb{R}$ is upper triangular. Uppar and lower triangular matrices are easy to solve for $\pmb{x}$.

The _Choleski decomposition_ is useful for solving SPD matrices.

**NOTE:** For the most part, in this class, you will just use `np.solve`.

# Homogenous Coordinates

Often, when representing points, we add an extra '1'. So $(x, y)$ becomes $(x, y, 1)$.
This allows us to represent vectors (differenced between points) as $(x, y, 0)$

# Least Squares

Suppose we have $$ \pmb{y} = \pmb{X}\pmb{w} + \pmb{\delta} $$ where $\pmb{\delta}$ is noise and $\pmb{X}$ is rectangular (more rows than columns).

We would like find $\pmb{w}^*$ to minimize $||\pmb{y}-\pmb{X}\pmb{w}||$

For numerical reasons, we will instead minimize $\frac{1}{2}||\pmb{y}-\pmb{X}\pmb{w}||^2$ (which is minimized for the same $\pmb{w}^*$).

```python
x = np.sort(np.random.rand(100))
X = np.column_stack([x, np.ones_like(x)])
```

```python
X[:5]
```

```python
noise  = 0.1*np.random.randn(len(x))
w_true = np.array([1, 2])
y = X.dot(w_true) + noise
```

```python
scatter(x, y)
plot(x, X.dot(w_true), color='g');
```

We have
$$
\begin{align}
||\pmb{y} - \pmb{X}\pmb{w}||^2 &= (y-Xw)^T(y-Xw) \\
             &= y^Ty-y^TXw -x^TX^Ty-w^TX^TXw \\
             &= y^Ty -2 w^TX^Ty - w^T X^T X w
\end{align}
$$

 *I got tired of making symbols bold... *

Taking the derivative WRT $\pmb{w}$, we get
$$ 2X^Ty - 2X^TXw $$

We want to make the derivative equal zero, so
$$ X^TXw = X^Ty$$
(the factor of 2 does not effect the solution; this is why we put a $\frac{1}{2}$ in front of the error earlier)

So, we can _**solve a system of equations**_ to find $\pmb{w}$

```python
np.linalg.solve(X.T.dot(X), X.T.dot(y)).round(2)
```

```python
w_true
```

There is a more convenient `lstsq` method that used SVD to solve for $\pmb{w}$

```python
w, residual, rank, sigma = np.linalg.lstsq(X, y, rcond=None)
print(w.round(2))
```

The residual is $||y - Xw||^2$

```python
print(sum((y-X.dot(w))**2), float(residual))
```

The singular values are also returned

```python
print(np.linalg.svd(X, compute_uv=False), sigma)
```

The _rank_ is the number of positive singular values (>0).

```python

```


# Calibration of SEIRD model using Approximate Bayesian Computation (ABC)

Based on code of Lander De Visscher

## Background

### Approximate Bayesian Computation (ABC)
ABC methods are part of **Bayesian methods** for parameter inference. In the Bayesian framework, we base our inference on the **posterior distribution** $P(\theta|y_{obs})$ of a parameter $\theta$ given the observed data $y_{obs}$ (and given the model for which we perform our inference). This conditional probability distribution can be derived from Bayes' theorem:

$$
\begin{equation}
P(\theta|y_{obs}) = \frac{P(y_{obs}|\theta)\,P(\theta)}{P(y_{obs})},
\end{equation}
$$

with
- $P(y_{obs}|\theta)$ the likelihood function (i.e. how likely it is to observe the data with our model, given the parameter $\theta$),
- $P(\theta)$ the prior distribution (i.e. the probability distribution that reflects our prior believes/the prior evidence of the parameter distribution)
- $P(y_{obs})$ the marginal likelihood.

Since the marginal likelihood $P(y_{obs})$ is often not (analytically) tractable, most Bayesian methods will use the property that the $posterior \propto likelihood \times prior$ in a **sampling algorithm**, that can generate a (approximate) sample from the posterior distribution.

When the likelihood function $P(y_{obs}|\theta)$ is also unknown/intractable but we can generate simulations $y$ with our model given the parameter $\theta$, **Approximate Bayesian Computation** methods come in. **ABC** will approximate the likelihood function as:

$$
p_{\epsilon}(s_{obs}|\theta) = \int K_\epsilon(\parallel s- s_{obs}\parallel)\,p(s|\theta)\mathrm{d} s
$$

The approximations of these methods are twofold.
1. Instead of using the whole (simulated) data $y$, we work with (lower-dimensional) summary statistics $$s = S(y),$$ in which $S$ is the **summary statistics function**.
2. Instead of conditioning on $y = y_{obs}$ (as in the true likelihood), we condition on $$K_\epsilon(\parallel s- s_{obs}\parallel)$$ in which
    - $K_\epsilon$ is a **smoothing kernel** with **bandwith** $\epsilon$
    - $\parallel . - .\parallel$ is a distance metric.

These methods result in an **approximated partial posterior distribution $p_{\epsilon}(\theta|s_{obs})$**. The ABC method is illustrated in the following image (Beaumont 2009).

### SMC

In the current implementation ABC inference is performed using **Sequential Markov Chain Monte Carlo (SMC MCMC)**, a variant of ABC where a population of so-called **particles** (which are **weighted samples**) are propagated through intermediate proposal distributions, starting from the **prior distribution $ p(\theta)$**.  These proposal distributions are adaptively constructed during the SMC iterations ($i_{SMC}$) and arise from decreasing the bandwith $\epsilon$ of the smoothing kernel function. The decrease is based on the effective sample size (ESS), which is a measure of the particle degeneracy (i.e. when few particles contain most of the particle weight).

This amounts to the following procedure:
>- At $i_{SMC} = 0$: $N$ particles are drawn from the **prior** and all particle weights are set to $1/N$.
>- At $i_{SMC} > 0$:
    1. reweight: $\epsilon$ is reduced so that the ESS is decreased with a fixed (relative) amount $\alpha$
    2. resample: if ESS drops below a specified threshold, $N_{particles}$ particles are drawn from the weighted sample and all particle weights are set to $1/N$.
    3. move: particles $\theta$ with weight > 0 are perturbed with a MCMC kernel and the perturbed particle $\theta'$ is accepted with a probability
    $$\min\left(1,\frac{K_\epsilon(\parallel s(\theta') - s_{obs} \parallel)\,p(\theta')}{K_\epsilon(\parallel s(\theta) - s_{obs} \parallel) \,p(\theta)}\right) $$

**REMARK**: at the moment, code is only usable for **uniform priors**

## Calibration plan

The idea is to perform following calibrations:
- calibration of beta:
    - data: H_in (i.e. daily flow into the hospital)
    - time: first 7 days (i.e. March 15 until 22): period before imposed measures (so when R0 is valid)
- calibration of compliance functions (2 or 3 parameters; indicating how people start adhering to the measures):
    - data: H_in
    - time: first 21 days: period comprising initiation of the measures
- duration of stay in ICU, cohort and recup-after-ICU:
    - data: H_tot
    - time: until some days after the peak

## Initialisations

```python
## For SEIRD model
import numpy as np
import matplotlib.pyplot as plt
from IPython.display import Image
from ipywidgets import interact,fixed,FloatSlider,IntSlider,ToggleButtons
import pandas as pd
import scipy
from scipy.integrate import odeint
import matplotlib.dates as mdates
import matplotlib
import scipy.stats as st
import networkx # to install networkx in your environment: conda install networkx
from covid19model.models import models
from covid19model.data import google, sciensano, polymod, parameters
from covid19model.visualization import output
# OPTIONAL: Load the "autoreload" extension so that package code can change
%load_ext autoreload
# OPTIONAL: always reload modules so that as you change code in src, it gets loaded
%autoreload 2
```

    WARNING (theano.configdefaults): install mkl with `conda install mkl-service`: No module named 'mkl'

```python
## for ABC
import seaborn as sns

import os
from datetime import datetime

from covid19model.optimization.ABC import ABC
from covid19model.optimization.ABC import store_results
from covid19model.optimization.ABC import metadata
%load_ext autoreload
# OPTIONAL: always reload modules so that as you change code in src, it gets loaded
%autoreload 2
```

    The autoreload extension is already loaded. To reload it, use:
      %reload_ext autoreload

```python
modulde_versions_global = metadata.list_module_versions(globals(), return_dict = True)
```

    numpy 1.18.1
    pandas 1.0.3
    scipy 1.4.1
    matplotlib 3.1.3
    networkx 2.4
    seaborn 0.10.1

### Seed
Set seed to ensure reproducibility.

```python
sim_seed = 300494
np.random.seed(sim_seed)
```

### Model parameters

Set SEIRD model parameters.
Following assumptions are made:
- $I_0 = 2$ (first recorded number of infected individuals @ March 1)
- $E_0 = I_0 * 10$ (estimation of initial number of exposed individuals)

```python
params = parameters.get_COVID19_SEIRD_parameters(stratified=False)
```

```python
initN = polymod.get_interaction_matrices()[0]
initN = np.array([initN.sum()])
```

```python
observed_infections = pd.read_csv('../data/raw/sciensano/ObsInf.txt')
```

```python
## deaths and recovered pool is negligible on March 15
## about 50% of cases are asymptomatic (hence not tested), so double the amount
initial_infected = observed_infections.iloc[14].item()
```

```python
levels = initN.size
initial_states = {'S': initN, 'E': np.ones(levels)*initial_infected*9,
                  'I': np.ones(levels)*initial_infected*6, 'M':np.ones(levels)*initial_infected*3}
```

```python
# Make time array for simulation (measured in days)
t_start = 0
t_stop =  7
time = t_stop

# Bundle fixed parameters as dict
fixed_model_parameters = {}
fixed_model_parameters["time"] = time
fixed_model_parameters["initial_states"] = initial_states
```

### Check initial model

**NOTE: momenteel heel weinig inflow in hospital (vgl met bij stratified model)... Vermoedelijk nog iets mis met parameters?**

```python
model = models.COVID19_SEIRD(initial_states, params)
```

```python
# -> user runs a simulation for a defined time period
chk = {'time':  [42],
#        'Nc':    [0.3*Nc_home+(1-0.70)*Nc_work + (1-0.80)*Nc_transport]
       'Nc':    [np.array([1])]

      }
out = model.sim(200)
```

```python
## Sum over stratification only needed for stratified model
(out['H_in'] - out['H_out']).cumsum().sum(dim='stratification').plot()
(out['C']+out['ICU']+out['C_icurec']).sum(dim='stratification').plot()
```

```python
out['H_in'][:, 1:1+time] .plot()
```

### Summary statistic function
For this example the "summary statistics" are the number of infected, recovered and dead individuals of the first 20 days of the outbreak.

```python
def summary_stat_f(y):
    """
    Selects the infected, recovered and dead individuals of the first 20 days.
    And ravels them to a 1D array.
    """
    return y[:t_stop,:].ravel()
```

### Load observed data

```python
sciensano_data = sciensano.get_sciensano_COVID19_data(update=False)

```

```python
sciensano_data.H_in.plot()
```

```python
## For calibration of residence durations: use total number of people present in hospital
y_obs_H = sciensano_data.H_tot
## Of calibration of beta and compliance function: use H_in (daily inflow in hospital)
y_obs_H_in = sciensano_data.H_in
```

```python
y_obs = np.full((y_obs_H_in.shape[0],1),np.nan)
y_obs[:,0] = y_obs_H_in

s_obs = summary_stat_f(y_obs)
```

```python
plt.plot(y_obs_H_in[:t_stop], ".")
```

### ABC hyperparameters
#### Dimensions

```python
n_s = s_obs.shape #shape of summary statistic array
n_particles = 500#1000 # number of particles
```

#### Priors

```python
#distribution of parameter vectors [beta] # estimate one parameter (on only 7 days)
# if you take more parameters, also fill in min, max values and param_bounds
n_param = 1
theta_min = (0)   # minimum values of uniform priors
theta_max = (1)   # maximum value of prior
param_bounds = np.array([[0],       # lower parameter bounds
                         [np.nan]]) # upper parameter bounds
def uniform_priors(N_param, theta_min = theta_min, theta_max = theta_max, n_param = n_param):
    """
    Uniform prior distributions for parameter vectors [BETA, GAMMA, OMEGA, DELTA]

    Parameters
    ----------
    N_param : int
        Number of particles to be drawn.

    theta_min : ndarray
        minimum values of uniform priors.

    theta_max : ndarray
        maximum values of uniform priors.

    n_param : int
        number of parameters to be inferred.

    Returns
    -------
    param_dist_0
        initial parameter vector distribution.

    """
    theta_min = np.array(theta_min)
    theta_max = np.array(theta_max)
    return np.random.uniform(theta_min, theta_max, (N_param,n_param))
```

#### Construct generative function

```python
def GM_dist(params_dist,
       n_draws_per_param,
       summary_stat_f,
       n_s,
       time,
       initial_states):
    """
    Generative model for the SEIRD epidemic model.
    The outputs that are generated are the specified summary statistics of the
    model output.

    Parameters
    ----------
    params_dist : ndarray,
        array containing distribution of parameter vectors [BETA, GAMMA, OMEGA, DELTA] of the SEIRD model
    n_draws_per_param : int,
        number of times the simulated data are generated per parameter vector
    summary_stat_f : JIT-compiled function:
        summary statistic of model outputs
    n_s :
        shape of the summary statistic array
    t_vec : ndarray
        sequence of time points for which to solve for the system of equations
    y_0 : ndarray
        number of generations

    Returns
    -------
    stat_dist : ndarray
        distribution of summary statistics of model outputs, generated with
        the distribution of parameter vectors.
    """
    N = params_dist.shape[0]
    stat_dist = np.empty((N, n_draws_per_param, *n_s))
    for i in range(0,N):
        current_params = params_dist[i]
        params['beta'] = current_params[0]
        model = models.COVID19_SEIRD(initial_states, params)
        out = model.sim(time)
        y = np.full((time,1),np.nan)
        y[:,0] = out['H_in'][:, 1:1+time]
        stat_dist[i,:,:] = summary_stat_f(y)
    return stat_dist

```

#### Create SMC_MCMC object
The ABC module contains the class `SMC_MCMC` which, upon creation, sets all necessary hyperparameters and functions as attributes/methods.
To make an SMC_MCMC object, you have to provide at least the following:
- `generative_model` : a function that accepts following arguments
    - `params_dist` : the particle distribution (i.e. `n_particles` parameter vectors of size length `n_param`
    - `n_draws_per_param` : number of times the simulated data are generated per parameter vector (for deterministic models, use 1)
    - `summary_stat_f` : summary statistic function
    - `n_s` : shape (tuple) of the summary statistic array
    - some fixed model parameters

- `priors` : function that can draw particles from the prior distribution (**REMARK**: at the moment, code is only usable for **uniform priors**)
- `n_param` : number of parameters that are to be inferred
- `y_obs` : observed outputs
- `summary_stat_f` : summary statistic function

Next to that, additional hyperparameters can be set:

- `ESS_min_factor` : ESS threshold for resampling (= fraction of n_particles). If ESS goes under this threshold, the parameters are resampled
- `fixed_model_parameters` : arguments of the `generative_model` that are fixed during calibration
- `max_SMC_iter`: maximum number of SMC iterations
- `distance_f` : distance function to be used (here Euclidean distance), custom functions can be used directly or added to the distances.py module
- `smoothing_kernel_f`: function to be used as smoothing kernel (-> gaussian leads to smoother behaviour than uniform)
- `alpha_ESS`: factor to decrease the ESS at every iteration (keep around 0.9). This factor determines how much the sample size decreases at each iteration. The closer to 1, the slower it goes (but possibly more precise).

Lastly, some words on the MCMC kernel (used in the move step). This is implemented as a separate class, of which the attributes are passed in the dict `MCMC_kernel_specifics`.
To ensure convergence, the scale of the parameter perturbation also needs to decrease during iterations.
When the mode `dynamic_param_dist_range` is selected, the scale is set as a fraction (`r_MCMC`) of the range of particles with weights above 0.
Finally, the range of the parameters can be bounded (see -> priors).

```python
covid_ABC = ABC.SMC_MCMC(GM_dist, #SEIRD.GM_dist
                              priors = uniform_priors,
                              n_param = n_param,
                              y_obs = y_obs,
                              summary_stat_f = summary_stat_f,
                              s_obs = False,
                              ESS_min_factor = 0.5,
                              n_particles = n_particles,
                              n_draws_per_param = 1,
                              fixed_model_parameters = fixed_model_parameters,
                              max_SMC_iter = 151,
                              zero_tol = 5e-4,
                              distance_f = "Euclidean",
                              smoothing_kernel_f = "gaussian",
                              alpha_ESS = 0.9,
                              MCMC_kernel_specifics = {
                                  "kernel_f" : "gaussian",
                                  "scale_mode":"dynamic_param_dist_range",
                                  "r_MCMC" : 0.05,
                                  "bounds" : param_bounds})
```

### Create results folder

```python
obs_group = "COVID19_BE"
timestamp = str(datetime.now()).replace(":","-").split(".")[0]
results_folder = f"SMC_{timestamp}_withoutdiff"
results_path = os.path.join("..","results","COVID19", obs_group, "demo", results_folder)

```

## Run ABC

```python
# =============================================================================
# %% Run algorithm
# =============================================================================
param_distributions, weight_distributions, ABC_bandwiths = covid_ABC.run(results_path,seed = sim_seed)
```

    INFO:root:
    =========================================================================================================================
    Approximate Bayesian Computation

    Version:                                       [try_out_nodiff]-let SEIRD_model output H_in and H_out-ba478789d2ab08a64bf796b36074588234ca3d86
    SMC-MCMC algorithm started at                  2020-06-19 09:00:41.434572
    with seed 300494
    =========================================================================================================================

    INFO:root:iter	 bandwith	ESS
    INFO:root:  0	  133334.62	500
    INFO:root:  1	   49263.44	450:             490 particles moved of the  500 candidates
    INFO:root:  2	   38724.78	405:             442 particles moved of the  448 candidates
    INFO:root:  3	   31935.71	364:             378 particles moved of the  394 candidates
    INFO:root:  4	   26456.35	328:             349 particles moved of the  361 candidates
    INFO:root:  5	   21669.05	295:             320 particles moved of the  329 candidates
    INFO:root:  6	   17298.85	266:             284 particles moved of the  291 candidates
    INFO:root:  7	   13386.54	239: (resample)  492 particles moved of the  500 candidates
    INFO:root:  8	    7843.23	450:             465 particles moved of the  471 candidates
    INFO:root:  9	    5885.20	405:             420 particles moved of the  437 candidates
    INFO:root: 10	    4598.04	364:             386 particles moved of the  398 candidates
    INFO:root: 11	    3615.36	328:             342 particles moved of the  354 candidates
    INFO:root: 12	    2826.59	295:             315 particles moved of the  328 candidates
    INFO:root: 13	    2179.28	266:             275 particles moved of the  292 candidates
    INFO:root: 14	    1672.83	239: (resample)  496 particles moved of the  500 candidates
    INFO:root: 15	     999.47	450:             463 particles moved of the  476 candidates
    INFO:root: 16	     760.28	405:             417 particles moved of the  429 candidates
    INFO:root: 17	     594.47	365:             390 particles moved of the  395 candidates
    INFO:root: 18	     471.15	328:             347 particles moved of the  361 candidates
    INFO:root: 19	     374.39	295:             311 particles moved of the  328 candidates
    INFO:root: 20	     302.53	266:             288 particles moved of the  304 candidates
    INFO:root: 21	     251.20	239: (resample)  482 particles moved of the  500 candidates
    INFO:root: 22	     181.93	450:             459 particles moved of the  491 candidates
    INFO:root: 23	     156.09	405:             405 particles moved of the  443 candidates
    INFO:root: 24	     134.80	365:             370 particles moved of the  402 candidates
    INFO:root: 25	     116.53	328:             337 particles moved of the  368 candidates
    INFO:root: 26	     101.18	295:             311 particles moved of the  347 candidates
    INFO:root: 27	      88.42	266:             280 particles moved of the  325 candidates
    INFO:root: 28	      77.32	239: (resample)  482 particles moved of the  500 candidates
    INFO:root: 29	      57.06	450:             457 particles moved of the  483 candidates
    INFO:root: 30	      48.88	405:             420 particles moved of the  454 candidates
    INFO:root: 31	      42.71	364:             381 particles moved of the  419 candidates
    INFO:root: 32	      37.42	328:             345 particles moved of the  383 candidates
    INFO:root: 33	      32.67	295:             301 particles moved of the  340 candidates
    INFO:root: 34	      28.40	266:             274 particles moved of the  314 candidates
    INFO:root: 35	      24.62	239: (resample)  483 particles moved of the  500 candidates
    INFO:root: 36	      17.64	450:             451 particles moved of the  476 candidates
    INFO:root: 37	      14.85	405:             420 particles moved of the  445 candidates
    INFO:root: 38	      12.92	364:             390 particles moved of the  422 candidates
    INFO:root: 39	       4.93	nan:
    ERROR:root:
    -------------------------------------------------------------------------------------------------------------------------
    Iteration  39:
        all particle weights are zero.

    SMC-MCMC algorithm stopped at                  2020-06-19 09:07:55.637267
    =========================================================================================================================
    NoneType: None

If the algorithm is terminated before the maximum nuber of iterations is reached, this is probably due to issues with the decreasing the bandwith below the (unknown) minimum. Use the particle distribution of the iteration before the last resample as end result.

```python
# =============================================================================
# %% Write results
# =============================================================================
metadata_dict = metadata.get_metadata_from_attributes(covid_ABC,
                                                      skip_attributes = ["param_distributions","weight_distributions","ABC_bandwiths", "logger_header", "logger_sim"],
                                                      custom_classes = {ABC.MCMC_kernel : lambda x : metadata.get_metadata_from_attributes(x,skip_attributes = ["MCMC_object"]),
                                                                        datetime : lambda x : str(x)})
metadata_dict["modulde_versions_global"] = modulde_versions_global

file_path = store_results.write_results_hdf5(
    os.path.join(results_path,results_folder),
                                 metadata_dict,
                                 param_distributions = param_distributions,
                                 weight_distributions = weight_distributions,
                                 ABC_bandwiths = ABC_bandwiths)
```

```python
## read in previous data
# param_distributions, metadata_dict, ABC_bandwiths = store_results.load_results_hdf5(
#     results_path, ["param_distributions","weight_distributions","ABC_bandwiths"])
```

## Plot results

### diagnostic plots of ABC convergence

```python
plt.figure()
plt.semilogy(ABC_bandwiths)
plt.title("Evolution of tolerated distance from observations")
plt.xlabel("iteration")

plt.figure()
plt.title("Evolution of number of moved particles")
plt.xlabel("iteration")
plt.plot(covid_ABC.moved_particles[:covid_ABC.i_SMC])

plt.figure()
plt.title("Evolution of number of move candidates")
plt.xlabel("iteration")
plt.plot(covid_ABC.move_candidates[:covid_ABC.i_SMC])

plt.figure()
plt.title("Evolution of ratio of moved particles/move candidates")
plt.xlabel("iteration")
plt.plot(covid_ABC.moved_particles[:covid_ABC.i_SMC]/covid_ABC.move_candidates[:covid_ABC.i_SMC])
```

### Plots of marginal posterior parameter distributions
First, select SMC iteration. (before break)

```python
i_SMC = param_distributions.shape[0]-3
print(i_SMC)
```

    36

Plot parameter distributions.

```python
parameter_names = ["$\\beta$"]
plt.figure()
for i in range(1):
    plt.subplot(2,2,i+1)
    plt.title(parameter_names[i])
    plt.hist(param_distributions[0,:,i], weights = weight_distributions[0,:]/np.sum( weight_distributions[0,:]), alpha = 0.5)
    plt.hist(param_distributions[i_SMC,:,i], weights = weight_distributions[i_SMC,:]/np.sum( weight_distributions[i_SMC,:]),alpha = 0.75)
plt.tight_layout()
plt.savefig(os.path.join(results_path,f"param_dist_iSMC{i_SMC}.png"),bbox_inches = "tight")
```

### plot simulations with posterior

```python
posterior = param_distributions[i_SMC,:]
norm_weights = weight_distributions[i_SMC,:]/np.sum(weight_distributions[i_SMC,:])
g = sns.pairplot(pd.DataFrame(posterior, columns = parameter_names),
                 diag_kind="hist",
                 diag_kws = {"weights": norm_weights },
                 plot_kws= {"size": norm_weights, "alpha" : 0.05, "markers" : ".", "edgecolor" : "tab:blue"},
                 corner=True)
g.savefig(os.path.join(results_path,f"param_pairplot_iSMC{i_SMC}.png"))
```

```python
t_stop_sim = 7
tStop = 7 # calibration until 7
y_sim = GM_dist(posterior,               # parameters
                      1,                 # n_draws_per_parameter
                      lambda y : y,      # return raw simulation outputs
                      (t_stop_sim,5),
                      t_stop_sim,    # t_vec
                      initial_states)    # inital conditions

for subpop in ["H_in"]:
     fig,ax = plot_full_posterior_sim(y_obs,y_sim, norm_weights/np.max(norm_weights), t_stop_sim, subpop, tStop, alpha_scale = 0.05)
     fig.savefig(os.path.join(results_path,f"postsim_iSMC{i_SMC}_{subpop}.png"))
```

```python
t_stop_sim = 14
tStop = 7 # calibration until 7
y_sim = GM_dist(posterior,               # parameters
                      1,                 # n_draws_per_parameter
                      lambda y : y,      # return raw simulation outputs
                      (t_stop_sim,5),
                      t_stop_sim,    # t_vec
                      initial_states)    # inital conditions
for subpop in ["H_in"]:
     fig,ax = plot_full_posterior_sim(y_obs,y_sim, norm_weights/np.max(norm_weights), t_stop_sim, subpop, tStop, alpha_scale = 0.05)
     fig.savefig(os.path.join(results_path,f"postsim_iSMC{i_SMC}_{subpop}_extrapol.png"))

```

```python
def plot_full_posterior_sim(y_obs, y_sim, norm_weights, t_stop_sim, subpop, t_stop_cal = None, **kwargs):
    """
    Plot simulations of the posterior parameter distribution.

    The results are plotted for a given subpopulation of the results ("susceptible",
    "exposed","infected","recovered", or "dead) and from t_0 = 0 to t_stop_sim

    Parameters
    ----------
    y_obs : ndarray
        observed populations; shape = (n_timesteps, 4)
    y_sim : ndarray
        simulated populations, output from GM_dist
    norm_weights :  normalised weights
        :
    t_stop_sim : int
        time step at which simulations are stopped.
    subpop : str, optional
        "susceptible","exposed","infected","recovered", or "dead. The default is "infected".
    t_stop_cal : int, optional
        if given, this is the last time step used in the calibration
    Returns
    -------
    fig, ax:
        handles to figure and axes

    """

    alpha_vect = norm_weights if "alpha_scale" not in kwargs else norm_weights * kwargs["alpha_scale"]

    i_subpop = {"H_in":0,"H_tot":1}[subpop]
    fig = plt.figure(figsize = (9,6))
    ax = plt.gca()
    n_sim = y_sim.shape[0]
    for i in range(n_sim):
        plt.plot(y_sim[i,0,:,i_subpop],
                 color = "tab:blue",
                 alpha = alpha_vect[i])
    obs_plt, = plt.plot(y_obs[:t_stop_sim,i_subpop],"o", color= "black")
    sim_plt = matplotlib.lines.Line2D([0],[0], color = "tab:blue")
    y_lim = ax.get_ylim()
    if t_stop_cal is not None:
        # plot which data are used for calibration.
        plt.vlines(t_stop_cal-1, y_lim[0], y_lim[1] , "tab:red")
        plt.text((t_stop_cal-1)/2, 0.95*y_lim[1], "calibration", horizontalalignment='center', color = "tab:red")
        plt.text(t_stop_cal-0.5 + (t_stop_sim-t_stop_cal)/2, 0.95*y_lim[1], "extrapolation",horizontalalignment='center', color = "tab:red")
    ax.set_ylim((0,y_lim[1]))
    ax.set_xlim((0,t_stop_sim))
    plt.legend((sim_plt,obs_plt),("simulated", "observed"))
    plt.title("simulations with obtained posterior")
    plt.xlabel("time (days)")
    plt.ylabel(f"number of {subpop} individuals")
    return fig, ax

```

```python

```

```python

```


Created: Tuesday 31 January 2017
github.com/rhyswhitley/fire_limitation

Quantifying the uncertainity of a global fire limitation model using Bayesian inference

Part 2: Bayesian inference

1,* Douglas Kelley,
2 Ioannis Bistinas,
3 Rhys Whitley,
4 Chantelle Burton,
1 Tobias Marthews,
6, 7 Ning Dong

1 Centre for Ecology and Hydrology, Maclean Building, Crowmarsh Gifford, Wallingford, Oxfordshire, United Kingdom

2 Vrije Universiteit Amsterdam, Faculty of Earth and Life Sciences, Amsterdam, Netherlands

3 Natural Perils Pricing, Commercial & Consumer Portfolio & Pricing, Suncorp Group, Sydney, Australia

4 Met Office United Kingdom, Exeter, United Kingdom

5 Centre for Past Climate Change and School of Archaeology, Geography and Environmental Sciences (SAGES), University of Reading, Reading, United Kingdom

6 Department of Biological Sciences, Macquarie University, North Ryde, NSW 2109, Australia

Summary

This notebook aims to quantify the model parameters of a global fire model (defined below). The model is driven by a number of covariates (Xi=1, 2, ... M) that describe: cropland, pasture and urban area footprints; frequency of lightening ignitions, population density, vegetation cover and Alpha, a proxy measure of available soil moisture in the root zone. The model attempts to predict the impact of fire through burnt area and is thus the model target (Y).

Python code and calculations below

```python

```

### Model description
The model considers percentage of burnt area to be the joint product of a set of conditions that modulate fire through fuel load, ignitions, moisture and supression. Each control assumes some equilibrium point that desribes the optimal conditions for fire, that may be proportionally modified through some empirical relationship. These are briefly outlined below for the sake of comprehension in this notebook, but can be referred to in more detail in the model protocol located in the docs/ folder (model protocol).

\begin{eqnarray}
    F_{burn} &=& \prod_{i}S(x_{i}) \\[1em]
\end{eqnarray}

Where $S(x_{i})$ representes some measure of fire conditions by $i =$ fuel, moisture, ignitions and anthropagenic supression, and is describe by a sigmoid:

\begin{equation}
    S(x_{i=fuel, moist, ignite, suppr}) = \frac{1}{1 + \exp\{-b\cdot(x_i-a)\}}
\end{equation}

The $fuel$ sigmoid considers only fractional vegetation cover and therefore have no hyper-parameters. Sigmoids $moist$, $ignite$ and $suppr$ describe an aggregation of other climate and land-use covariates. Because these sigmoids are influenced by an aggregation of different drivers, they are influenced in turn by different sets of hyper-parameters; these are now described below.

#### Fuel load covariate (no hyper-parameters)
\begin{equation}
    x_{fuel} = FPC^{p} \cdot (v_{fuel} \cdot (\alpha_{max}/\alpha -1) + 1)/(1+v_{fuel})
\end{equation}

#### Moisture covariate
\begin{equation}
    x_{moist} = \alpha + v_M \cdot EMC
\end{equation}

where $EMC$ is the equilibrium moisture content.
#### Ignition covariate
\begin{equation}
    x_{ignite} = Lightn + v_p\cdot A_{pasture} + v_{d1}\cdot\rho_{population}
\end{equation}

Where $Lightn$ is the number of cloud-to-ground lightning strikes, modified  as per Kelley et al. 2014.

#### Supression covariate
\begin{equation}
    x_{supress} = A_{urban} + v_C\cdot A_{Crop} + v_{d2}\cdot\rho_{population}
\end{equation}

This leaves 17 free parameters that need to be optimised against observations of burnt area.

### Load libraries

```python
import os
from   io     import StringIO
import numpy  as np
import pandas as pd
import csv

import pymc3  as pm3
from   pymc3.backends import SQLite
from   scipy  import optimize
from   theano import tensor as tt

import matplotlib.pyplot as plt

# setup nice plotting
plt.style.use('ggplot')
%matplotlib inline

# paths and parameters
                    # Full model
sample_pc     = 5 # = 10 # Percentage of grid cells samples (largest = 100%) - training set
nIterations   = 1000 # = 10000 # no. times it samples the posterior
nJobs         = 1   # = 2 # no. times it runs through optimisation proceedure = sample_pc * nJobs
#chains per job
nChains       = 1   # = 3

datPath       = "../data/globfire2.csv"
param_outpath = '../outputs/params-test.csv'
trace_db      = '../data/firemodel_trace.db'
```

## 2.1 Fire limitation model definition

Could possibly contain this in a class object, but I'm not sure theano can instantiate the object to be used by the GPU. If I've made absolutely no sense just then, then I would leave the following as is.

```python
from pdb import set_trace as browser
def fuel_load(vegcover, alphaMax, fuel_pw, fuel_pg):
    """
    Definition to describe fuel load: while return the input; capability to be modified later.
    """

    return (vegcover**fuel_pw) * (fuel_pg * (alphaMax-1) + 1) / (1 + fuel_pg)

def moisture(alpha, emc, treeCover, cM, cMT):
    """
    Definition to describe moisture
    """
    return (alpha + cM*emc + cMT * treeCover) / (1 + cM + cMT)

def ignition(lightning, pasture, population_density, cP, cD1):
    """
    Definition for the measure of ignition
    """
    ignite = lightning + cP*pasture + cD1*population_density

    return ignite

def supression(cropland, population_density, cD2):
    """
    Definition for the measure of fire supression
    """
    return cropland + cD2*population_density

def tt_sigmoid(x, k, x0):
    """
    Sigmoid function to describe limitation using tensor
    """
    return 1.0/(1.0 + tt.exp(-k*(x - x0)))
```

## 2.2 Import data

Load data and do any necessary transformation needed for the Bayesian modelling framework. Rows are defined as a fraction of total data points (above). For full optimiatizaion, we set at 10%, but for testing purposes I've limited the number of rows I'm importing to 2%.

```python
def load_with_buffer(filename, line_select, **kwargs):
    s_buf = StringIO()
    line_select = np.sort(line_select)
    with open(filename) as file:
        count = -1
        lineN = -1
        for line in file:
            lineN += 1
            if lineN == 0 or lineN == line_select[count]:
                s_buf.write(line)
                count += 1
                if count == len(line_select): break

    s_buf.seek(0)
    df = pd.read_csv(s_buf,**kwargs)
    return df

def file_len(fname):
    with open(fname) as f:
        for i, l in enumerate(f): pass
    return i + 1

DATAPATH = os.path.expanduser(datPath)

nlines      = file_len(DATAPATH)
npoints     = round(sample_pc * nlines / 100)
line_select = np.random.choice(range(0, nlines), npoints, False)
fd          = load_with_buffer(DATAPATH, line_select)
```

```python
fd['alphaMax'] = fd['alpha_'] - 1.0 # alpha = alpha_12_months max
```

Do a sanity check to make sure our data has imported correctly.

```python
plt.subplot(1, 2, 1) # 1 = nrows, 2 = ncols, n = location of next plot (can write as 121)
plt.scatter(fd['vegcover'], fd['fire'], alpha = 0.1)
plt.subplot(122)
plt.scatter(fd['alpha'], fd['fire'], alpha = 0.1)
```

```python
fd.info()
print(fd.max())
fd.head(20)
```

    <class 'pandas.core.frame.DataFrame'>
    RangeIndex: 557122 entries, 0 to 557121
    Data columns (total 11 columns):
    emc                    557122 non-null float64
    pasture                557122 non-null float64
    cropland               557122 non-null float64
    population_density     557122 non-null float64
    treecover              557122 non-null float64
    vegcover               557122 non-null float64
    lightning_ignitions    557122 non-null float64
    fire                   557122 non-null float64
    alpha_                 557122 non-null float64
    alpha                  557122 non-null float64
    alphaMax               557122 non-null float64
    dtypes: float64(11)
    memory usage: 46.8 MB
    emc                        1.106945
    pasture                    0.851522
    cropland                   0.817462
    population_density     48377.082000
    treecover                  0.838333
    vegcover                   0.999167
    lightning_ignitions        4.023800
    fire                       0.880313
    alpha_                     3.482101
    alpha                      1.251716
    alphaMax                   2.482101
    dtype: float64

      emc
      pasture
      cropland
      population_density
      treecover
      vegcover
      lightning_ignitions
      fire
      alpha_
      alpha
      alphaMax

      0
      0.194474
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.022735
      0.242084
      0.022735

      1
      0.194517
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.022943
      0.242041
      0.022943

      2
      0.188028
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.023763
      0.242458
      0.023763

      3
      0.195352
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.369517
      0.242256
      0.369517

      4
      0.220651
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.382581
      0.275571
      0.382581

      5
      0.241409
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.633690
      0.324526
      0.633690

      6
      0.227211
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.984901
      0.473039
      0.984901

      7
      0.194740
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.892017
      0.318662
      0.892017

      8
      0.193780
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.957188
      0.308520
      0.957188

      9
      0.221186
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.245285
      0.242737
      0.245285

      10
      0.214060
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.099494
      0.242335
      0.099494

      11
      0.227791
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.782807
      0.357065
      0.782807

      12
      0.151532
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.306412
      0.242607
      0.306412

      13
      0.151208
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.345314
      0.242620
      0.345314

      14
      0.276007
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.561651
      0.392438
      0.561651

      15
      0.231395
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.708487
      0.330781
      0.708487

      16
      0.285923
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.953124
      0.465769
      0.953124

      17
      0.260823
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.783097
      0.416386
      0.783097

      18
      0.199265
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.839537
      0.315524
      0.839537

      19
      0.249006
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0
      1.727342
      0.275443
      0.727342

## 2.3 Baysian framework

A simple explanation of Baye's law is:

\begin{equation}
    P(\beta|X) \propto P(\beta)\cdot P(X|\beta)
\end{equation}

where $X$ is our data (observations of some arbitrary system), and $\beta$ our set of unexplained parameters that describe the reponse of our _proposed understanding_ of this system as it varies with $X$.

### 2.3.1 Prior definitions
Because I have no idea what the uncertainty on the hyper parameters should look like (beyond $\beta> 0$), I've set them all as uniform, where bounds are either physical limits of the variable in question, or set generously beyound what is realistic plausable. Some of them can possibly be describe as exponential or half-normal, due to the physical nature of $\beta$, but we can play around with that later.

\begin{eqnarray}
    P(\beta) &=& \prod_{i=1}^{4}P(a_i)\prod_{i=1}^{4}P(b_i)\cdot P(\sigma)\cdot P(v_c)P(v_p)P(v_{d,1})P(v_{d,2}) \\[1.5em]
    P(a) = P(b) = P(\sigma) &=& \mathcal{N}(0, 1) \\[1em]
    P(v_c) = P(v_p) = P(v_{d,1}) = P(v_{d,2}) &=& \mathcal{U}(\beta_{\min}, \beta_{\max}) \\[1.5em]
\end{eqnarray}

I'm not totally sure about the maths above being right, but it's just to show that _full_ prior is normal. Important, because we'll also describe the error (likelihood) as normal, such that the posterior is therefore normal (conjugate); i.e. $\mathcal{N}\times\mathcal{N}=\mathcal{N}$ (expansion happens in the mean of the exponent).

Back to the code.., `pymc3` is quite funky in that it allows me to create an empty `Model()` object and just add things to it as I need them using a `with` statement. I've called our Bayesian model `fire_error` as that is what we are trying to Quantify.

```python
with pm3.Model() as fire_error:

# first for the sigmoids
    fuel_x0        = pm3.Uniform('fuel_x0'       ,    -1.0,  2.0)
    fuel_k         = pm3.Uniform('fuel_k'        ,     0.0, 1000.0)
    fuel_pw        = pm3.Uniform('fuel_pw'       ,     1.0, 20.0)
    fuel_pg        = pm3.Uniform('fuel_pg'       ,     0.0, 1.0)

    moisture_x0    = pm3.Uniform('moisture_x0'   ,    -1.0,  2.0)
    moisture_k     = pm3.Uniform('moisture_k'    ,     0.0, 1000.0)

    ignition_x0    = pm3.Uniform('ignitions_x0'   ,    -4.0, 4.0)
    ignition_k     = pm3.Uniform('ignitions_k'    ,     0.0, 1000.0)

    suppression_x0 = pm3.Uniform('suppression_x0',  -10.,  2.0)
    suppression_k  = pm3.Uniform('suppression_k' ,     0.0, 10.0)

    max_f          = pm3.Uniform('max_f'         ,     0.0,    1.0)
    #pow_f          = pm3.Uniform('pow_f'         ,     0.0,    1.0)
# now for the hyper-parameters that describe the independent fire condition covariates
    cM  = pm3.Uniform('cM' , 0.0, 1e4)
    cMT = pm3.Uniform('cMT', 0.0, 1e4)
    #cL  = pm3.Uniform('cL' , 0, 1e4)
    cP  = pm3.Uniform('cP' , 0.0, 1e4)
    cD1 = pm3.Uniform('cD1', 0.0, 1e4)
    cD2 = pm3.Uniform('cD2', 0.0, 1e4)
    #cDmax = pm3.Uniform('cDmax', 0, 1e4)
# describe the standard deviation in the error term
    sigma = pm3.HalfNormal('sigma', sd=1)
```

### 2.3.2 Likelihood definition

For the sake of simplicity (and because I don't really know any better), we define the model error as normally distributed (i.i.d.) although it most likely isn't. We could make this more complicated later by defining the error as heteroscedastic, but I wouldn't bother with that until we have some idea of the convergence. We're describing the error (observations minus model predictions) as follows:

\begin{eqnarray}
    P(X|\beta) &=& \mathcal{N}(F_{burn}, \sigma) \\[1em]
    \mathcal{N}(F_{burn}, \sigma) &=& \frac{N}{\sigma\sqrt{2\pi}}\exp\left\{\sum_{i=1}^{N}\left(\frac{y_i - F_{burn, i}}{\sigma_i}\right)^2\right\}
\end{eqnarray}

where $y_i$ is a set of observations we're attempting to optimise on. Below is the code that describes the above:

```python
with fire_error:

    # transform hyper-covariates
    f_fuel        = fuel_load(fd["vegcover"].values, fd["alphaMax"].values, fuel_pw, fuel_pg)

    f_moisture    = moisture(fd["alpha"].values, fd["emc"].values, fd["treecover"].values, cM, cMT)

    f_ignition    = ignition(fd["lightning_ignitions"].values, \
                             fd["pasture"].values, \
                             fd["population_density"].values, \
                             cP, cD1)

    f_suppression = supression(fd["cropland"].values, \
                               fd["population_density"].values, \
                               cD2)

    # burnt area is assumed to be the product of the 4 sigmoids

    prediction = max_f * np.product([tt_sigmoid(f_fuel, fuel_k, fuel_x0),
                             tt_sigmoid(f_moisture, - moisture_k, moisture_x0),
                             tt_sigmoid(f_ignition, ignition_k, ignition_x0),
                             tt_sigmoid(f_suppression, - suppression_k, suppression_x0)])

    # calculate the error between observed and predicted burnt area
    error = pm3.Normal('error', mu=prediction, sd=sigma, observed=fd['fire'].values)
```

### 2.3.3 Posterior sampling

Because it is nigh impossible to determine the posterior solution analytically we will instead sample the information space to **infer** the posterior solutions for each of the model parameters. In this case we are using a Metropolis-Hasting step MCMC.

I've tried using No-U-Turn (NUTS) sampling (which is the new kid on the block), but there are issues with it's current implementation in pymc3 (see github repo issues). Can use it once problems are ironed out - but TBH it doesn't matter if we're getting a reasonable convergence.

```python
nChains = nChains* nJobs
with fire_error:

    # help the sampling out by quickly finding an optimal start position
    start = pm3.find_MAP(model=fire_error.model, fmin=optimize.fmin_powell)

    # set the step-method (criteria algorithm for moving around information space)
    step = pm3.Metropolis()

    # save our sampling to disk so we can access it later
    #db_save = SQLite(trace_db)

    # do the sampling
    mcmc_traces = pm3.sample(nIterations * nChains * nJobs, step=step, start=start, njobs= nJobs, chains = nChains)
```

    /users/global/doukel/anaconda3/lib/python3.6/site-packages/pymc3/tuning/starting.py:92: UserWarning: In future versions, set the optimization algorithm with a string. For example, use `method="L-BFGS-B"` instead of `fmin=sp.optimize.fmin_l_bfgs_b"`.
      warnings.warn('In future versions, set the optimization algorithm with a string. '
    logp = 1.4502e+06:   8%|▊         | 415/5000 [00:51<07:08, 10.71it/s]

    Optimization terminated successfully.
             Current function value: -1450213.518521
             Iterations: 2
             Function evaluations: 416

    logp = 1.4502e+06:   8%|▊         | 416/5000 [01:10<07:08, 10.71it/s]Sequential sampling (1 chains in 1 job)
    CompoundStep
    >Metropolis: [sigma_log__]
    >Metropolis: [cD2_interval__]
    >Metropolis: [cD1_interval__]
    >Metropolis: [cP_interval__]
    >Metropolis: [cMT_interval__]
    >Metropolis: [cM_interval__]
    >Metropolis: [max_f_interval__]
    >Metropolis: [suppression_k_interval__]
    >Metropolis: [suppression_x0_interval__]
    >Metropolis: [ignitions_k_interval__]
    >Metropolis: [ignitions_x0_interval__]
    >Metropolis: [moisture_k_interval__]
    >Metropolis: [moisture_x0_interval__]
    >Metropolis: [fuel_pg_interval__]
    >Metropolis: [fuel_pw_interval__]
    >Metropolis: [fuel_k_interval__]
    >Metropolis: [fuel_x0_interval__]

      0%|          | 0/1500 [00:00<?, ?it/s]
    logp = 1.4502e+06:   8%|▊         | 416/5000 [02:00<22:11,  3.44it/s]
      0%|          | 2/1500 [00:05<1:06:30,  2.66s/it]Exception in thread Thread-4:
    Traceback (most recent call last):
      File "/users/global/doukel/anaconda3/lib/python3.6/threading.py", line 916, in _bootstrap_inner
        self.run()
      File "/users/global/doukel/anaconda3/lib/python3.6/site-packages/tqdm/_tqdm.py", line 144, in run
        for instance in self.tqdm_cls._instances:
      File "/users/global/doukel/anaconda3/lib/python3.6/_weakrefset.py", line 60, in __iter__
        for itemref in self.data:
    RuntimeError: Set changed size during iteration

    100%|██████████| 1500/1500 [2:03:39<00:00,  4.95s/it]
    Only one chain was sampled, this makes it impossible to run some convergence checks

```python
pm3.traceplot(mcmc_traces);
```

## Output paramaters
The iterations at the start are just letting the optimization settle. So we will only sample to last 50% of iterations for futher analysis. We also expor these to csv, which others can use to do their own analysis.

```python
varnames = mcmc_traces.varnames

def cutLastX(varname, mcmc_traces, ncut = 50):
    vals = mcmc_traces.get_values(varname)
    def subcut(vals, r, ncut = 50):
        cut_np = (r+1) * round(len(vals)/nChains)
        ncut = round(len(vals) * ncut / (nChains *100))
        return vals[(cut_np - ncut):cut_np]
    vals = [subcut(vals, r) for r in range(nChains)]
    return np.array(vals).flatten()

vals = [cutLastX(i, mcmc_traces) for i in varnames]
vals = pd.DataFrame(np.array(vals).T, columns=varnames)

vals.to_csv(param_outpath, index=False)
```

## Plotting

Let look at the pdf of the last 50% of iterations for each parameter

```python
variables2Plot = ['fuel_x0'       , 'fuel_k',
                  'moisture_x0'   , 'moisture_k',
                  'ignitions_x0'  , 'ignitions_k',
                  'suppression_x0', 'suppression_k',
                  'fuel_pw'       , 'fuel_pg',
                  'cM'            , 'cMT',
                  'cP',
                  'cD1'           , 'cD2']

nvar = len(variables2Plot)
npcol = 4
nprow = np.ceil(nvar / npcol)

plt.figure(figsize=(20,5 * nprow))
def plotVar(var1, pn):
    plt.subplot(npcol, nprow, pn)
    param = vals[var1]

    hist, bins = np.histogram(param, bins=50)
    hist = 100.0 * hist / np.sum(hist)
    bins = bins[1:] - np.diff(bins)/2
    plt.plot(bins, hist)
    plt.xlabel(var1)

pn = 0
for i in variables2Plot:
    pn = pn + 1
    plotVar(i, pn)
```

And what to the sigmoids look like?

```python
from pdb import set_trace as browser

def pltVsFire(x, xlab, pnt = 'o', *args, **kw):
    plt.plot(x, fd.fire, pnt, alpha = 0.03, *args, **kw)
    plt.xlabel(xlab)

def np_sigmoid(x, k, x0):
    """
    Sigmoid function to describe limitation using tensor
    """
    return 1.0/(1.0 + np.exp(-k*(x - x0)))

def returnSigmoid(x, k, x0):
    return np_sigmoid(x, k, x0)

def meanParam(x, x0, k, kmult = 1.0):
    x0 = np.mean(vals[x0])
    k  = np.mean(vals[k]) * kmult

    return returnSigmoid(x, k, x0)

def randomParam(x, x0, k, kmult = 1.0, size = 100):
    ps = np.random.choice(vals.shape[0], size = size, replace = False)
    return [returnSigmoid(x, vals[k][i] * kmult, vals[x0][i]) for i in ps]
```

```python
plt.figure(figsize=(10, 10))

####################
## Fuel           ##
####################
plt.subplot(2, 2, 1)
## scatter plot
fp = np.mean(vals['fuel_pw'])
fg = np.mean(vals['fuel_pg'])

f_fuel = fuel_load(fd["vegcover"].values, fd["alphaMax"].values, fp, fg)
print(f_fuel.max())
pltVsFire(f_fuel, "Fuel continuity", 'go')

## Line of best fit
Fuel = np.arange(0, f_fuel.max(), 0.01)
r_fuel = randomParam(Fuel, 'fuel_x0', 'fuel_k')
for r in r_fuel: plt.plot(Fuel, r, 'k', alpha=.01)

####################
## Moisture       ##
####################
plt.subplot(2, 2, 2)
## scatter plot
cM = np.mean(vals['cM'])
cMT = np.mean(vals['cMT'])
f_moisture = moisture(fd["alpha"].values, fd["emc"].values, fd["treecover"].values, cM, cMT)
pltVsFire(f_moisture , "Moisture = $\\alpha$ + M $\cdot$ EMC",'bo')

## Line of best fit
mst = np.arange(0.0, f_moisture.max(), 0.05)
r_moisture = randomParam(mst, 'moisture_x0', 'moisture_k', -1)
for r in r_moisture: plt.plot(mst, r, 'k', alpha=.01)

####################
## Ignitions       ##
####################
plt.subplot(2, 2, 3)
## scatter plot
cP  = np.mean(vals['cP' ])
cD1 = np.mean(vals['cD1'])
igniteMax = 10

f_ignite = ignition(fd["lightning_ignitions"].values, \
                    fd["pasture"].values, \
                    fd["population_density"].values, \
                    cP, cD1)

pltVsFire(f_ignite, "Ignitions events = Lightn + P $\cdot$ Pop Dens + D1 $\cdot$ Pasture")

## Line of best fit
Ignite = np.arange(0.0, igniteMax, 0.1)
r_Ignite = randomParam(Ignite, 'ignitions_x0', 'ignitions_k')
for r in r_Ignite: plt.plot(Ignite, r, 'k', alpha=.01)

plt.xlim(0, igniteMax)

####################
## Suppression    ##
####################
plt.subplot(2, 2, 4)
#scatter plot
cD2 = np.mean(vals['cD2'])
f_suppression = supression(fd["cropland"].values, \
                           fd["population_density"].values, \
                           cD2)

pltVsFire(f_suppression, "Suppression = Cropland + D2 $\cdot$ Pop den")

# Line of best fit
Suppress = np.arange(0, 100, 0.01)
r_suppression = randomParam(Suppress, 'suppression_x0', 'suppression_k', -1.0)
for r in r_suppression: plt.plot(Suppress, r, 'k', alpha=.01)

plt.xlim(0, 100)
```

```python

```


```python
from IPython.display import Image
from IPython.core.display import HTML
from sympy import *; x,h,t,y,z   = symbols("x h t y z", real=True)
Image(url= "https://i.imgur.com/jJjPeWl.png")
```

```python
expr  = x**(2/3) + y**(2/3)-25
#print(idiff(expr, y, x, n=1)
circ = x**2 + y**2 - 4     # from idiff??
idiff(circ, y, x)
```

$\displaystyle - \frac{x}{y}$

```python
(-64 + idiff(expr,y,x)*(x--27)).subs(y,-64)
```

$\displaystyle -64 - \frac{4 \sqrt[3]{-1} \left(x + 27\right)}{\sqrt[3]{x}}$

```python
(idiff(expr,y,x).subs(x,-27)).subs(y,-64)
```

$\displaystyle - \frac{4}{3}$

```python
-64 -(4/3)*(x+27)
```

```python
Image(url= "https://i.imgur.com/3OXAbBV.png")
```

```python

```