```python
# Setup
from IPython.display import Markdown, Latex, Math

import sympy
from sympy import pi

# Print units using abbreviations: 'm' instead of 'meter'
sympy.printing.str.StrPrinter._default_settings['abbrev'] = True
```

```python
# INPUTS
from sympy.physics.units import meter, foot, convert_to

input_d_value = 11.1
input_d_unit = foot
input_d_sigfigs = 3

# DESIRED UNITS
# If the input doesn't match this then a conversion will occur
desired_length_unit = meter
```

```python
# Symbol setup and unit conversion
input_d = input_d_value * input_d_unit
sigfigs = input_d_sigfigs

# unit conversion
if input_d_unit != desired_length_unit:
    converted_d = convert_to(input_d, desired_length_unit).n(input_d_sigfigs)
    conversion_latex = Math("d={original}={converted}".format(
        original=sympy.latex(input_d.n(input_d_sigfigs)),
        converted=sympy.latex(converted_d.n(input_d_sigfigs))))
    input_d = converted_d
else:
    conversion_latex=None

d_latex = Math("d={}".format(sympy.latex(input_d.n(input_d_sigfigs))))
```

## Inputs

Circle diameter:

{{ conversion_latex if conversion_latex else d_latex }}

## Area

```python
# Area formula by building expression explicitly
d = sympy.Symbol('d')
area_formula = pi*(d/2)**2

# Substitute values without simplifying
with sympy.evaluate(False):
    area_formula_sub = area_formula.subs(d, input_d)

# Compute answer
area = area_formula_sub.evalf(sigfigs)

# Output the result as an aligned equation.
# The asterisk in `\begin{align*}` tells Latex to not add an equation number on the right hand side.
# Note that str.format() uses curly braces for formatting, so any curly braces that are not part
# of a string replacement must be doubled. Hence `\begin{{align*}}` instead of `\begin{align*}`.
area_block = Latex(r"""
\begin{{align*}}
A & = {area_formula} \\
  & = {area_formula_sub} \\
  & = {area_value}
\end{{align*}}
""".format(area_formula=sympy.latex(area_formula),
           area_formula_sub=sympy.latex(area_formula_sub),
           area_value=sympy.latex(area)))
```

The area of a circle with diameter {{ d_latex }} is:

{{ area_block }}

## Circumference

```python
# circumference formula by using sympify() and displayed all on one line
circ_formula = sympy.sympify("pi*d")

# Substitute values without simplifying
with sympy.evaluate(False):
    circ_formula_sub = circ_formula.subs(d, input_d)

# Compute answer
circ = circ_formula_sub.evalf(sigfigs)

# Simple one-line equation. In regular Markdown this can be done with $$ equation $$ syntax. However,
# Latex formatting currently cannot be done inline in a Markdown cell because of evaluation order issues when
# Latex and Python-markdown Jupyter extensions are used together.
circ_formula_latex = Markdown("$$ C = " + sympy.latex(circ_formula) +
                              " = " + sympy.latex(circ_formula_sub) +
                              " = " + sympy.latex(circ) + "$$")

```

The circumference of a circle with diameter {{ d_latex }} is:

{{ circ_formula_latex }}

```python

```

notebooks/Circle with Units and Sigfigs - SymPy version.ipynb


## Studiamo il calcolo simbolico

```python
#importo la libreria per il calcolo simbolico
from sympy import Symbol
from sympy import symbols
from sympy import factor
from sympy import solve
from sympy.plotting import plot
```

```python
x = Symbol('x')
```

```python
x+x+3
```

    2*x + 3

```python
#il nome della variabile
x.name
```

    'x'

```python
#definizione di più variabile simboliche
y,z = symbols('y,z')
```

```python
#un esempio
p = x*(x+x)
p
```

    2*x**2

```python
f = (x+2)*(x+3)
f
```

    (x + 2)*(x + 3)

```python
#espande se scritto esplicitamente
print(f.expand())
```

    x**2 + 5*x + 6

```python
#esempio di fattorizzazione
factor(x**2-y**2)
```

    (x - y)*(x + y)

```python
#fattorizzazione
factor(x**3+3*x**2*y+3*x*y**2+y**3)
```

    (x + y)**3

```python
#scriviamola un po' meglio
from sympy import pprint
expr = factor(x**3+3*x**2*y+3*x*y**2+y**3)
pprint(expr)
```

           3
    (x + y)

```python
#esempio scomposizione
expre = (x+2)**2-1
```

```python
factor(expre)
```

    (x + 1)*(x + 3)

```python
expre.expand()
```

    x**2 + 4*x + 3

```python
expre2 = x**9 -3*x**6+3*x**3-1
```

```python
factor(expre2)
```

    (x - 1)**3*(x**2 + x + 1)**3

```python
#calcolo simbolico con frazioni - non funziona
#from fractions import Fraction
#factor(Fraction(factor(-4*x**2-6*x+7),factor(x**3-2*x**2-x+2)))
```

## Risolviamo le equazioni

```python
x=Symbol('x')
expr = x**2 + x + 1
solve(expr, dict=True)
```

    [{x: -1/2 - sqrt(3)*I/2}, {x: -1/2 + sqrt(3)*I/2}]

```python
x=Symbol('x')
expr = x**2 - 5*x + 4
solve(expr, dict=True)
```

    [{x: 1}, {x: 4}]

```python
x=Symbol('x')
expr = x**3 - 5*x + 4
solve(expr, dict=True)
```

    [{x: 1}, {x: -1/2 + sqrt(17)/2}, {x: -sqrt(17)/2 - 1/2}]

```python
#sistema di equazioni
x = Symbol('x')
y = Symbol('y')
expr1 = 2*x + 3*y - 6
expr2 = 3*x + 2*y - 12
solve((expr1, expr2), dict=True)
```

    [{x: 24/5, y: -6/5}]

```python
#disegnamo funzioni
x = Symbol('x')
plot(2*x+3)
```

    <sympy.plotting.plot.Plot at 0x7f616b423588>

```python
#disegnamo funzioni
x = Symbol('x')
plot(2*x**2+3)
```

```python

```


# A SEIR Model in Python

## Overview of the SEIR compartmental model

Compartmental modeling allows for numerical reasoning about epidemics. It was proposed in 1927 by Kermack and McKendrick for mathematical modelling of infectious diseases. The SEIR model divides the population into the following compartments:

* Susceptibles $S$: individuals who are susceptible to the disease and can be infected
* Exposed $E$: individuals who have been exposed to the virus
* Infected $I$: individuals who carry an infection
* Recovered $R$: individuals who have been infected and have recovered and possibly have acquired immunity as well.

Individuals move from one compartment to the other. From the stage of recovered they either have acquired immunity or become susceptible again, depending on the disease.

(Image courtesy of the Institute for Disease Modeling)

### Model equations

The model presented here is very simplified. We'll suppose a closed population with no births or deaths and we'll suppose the disease as to be not deadly as well.

\begin{equation}
\frac{d S}{d t} = - \frac{\beta I\cdot S}{N}
\end{equation}

\begin{equation}
\frac{d E}{d t} = \frac{\beta I\cdot S}{N} - \sigma \cdot E
\end{equation}

\begin{equation}
\frac{d I}{d t} = \sigma \cdot E - \gamma \cdot I
\end{equation}

\begin{equation}
\frac{d R}{d t} = \gamma \cdot I
\end{equation}

$N$ = $S$ + $E$ + $I$ + $R$ is the total population.

### Model parameters

* $\beta$ is the infectious rate. It represents the probability of transmitting disease between a susceptible and an infectious individual.
* $\sigma$ is the incubation rate, thus the rate of latent individuals becoming infectious.
* $\gamma$ is the recovery rate

These parameters are expressed as the reciprocal of a time expressed in days. The model implemented here is time based in days, while other implementations may have different time-bases as, for example, years.

The basic reproduction number $R_0$ is the most important derived parameter. It represents the average number of new infections due to an infected individual, assuming the whole population is susceptible: it is given by

\begin{equation}
R_0 = \frac{\beta}{\gamma}
\end{equation}

mesleas, which is a quite infectious disease, has an $R_0$ of $\simeq 11$

Note that:

\begin{equation}
\frac{d E}{d t} + \frac{d I}{d t} = \gamma \left(\frac{\beta S I}{\gamma N} - I\right)
\end{equation}

which tells us that a *growth* of the infection requires a positive term on the right-hand side of the equation, which leads us to define an effective reproduction number.

\begin{equation}
{\cal R}(t) \equiv \frac{\beta S(t)}{\gamma N}
\end{equation}

The effective reproduction number depends on the susceptible population $S$, which gets reduced by the epidemics. After recovery, infected individuals account to the compartment $R$ of recovered individuals, who are immune. Therefore, at a steady state we have  ${\cal R} \simeq 1$.

## Running the model

We are going to run the model over a realistic, albeit hypothetical, scenario. First of all we are going to load the Python module containing the module implementation.

```python
%run SEIR.py
```

Then, we are going to set the parameters. We'll suppose a small community of 1000 individuals sensitive to the virus. At the initial conditions, 100 of them have been exposed to the infection. We'll set $\sigma = \gamma = 0.04762$ which corresponds to an incubation time of 21 days ( $\frac{1}{21} = 0.04762$), same as the infectious time. Recalling that

\begin{equation}
R_0 = \frac{\beta}{\gamma}
\end{equation}

we'll assume $R_0 = 3.8$ (our hypothesis is of a very infectios disease albeit less infectious than measles), then we'll have $\beta = 0.18$. We'll finally set $dt=0.001$ thus small enaugh to achieve a good convergence, and a time frame of $250$ days.

```python
# Set up the model
#
s = Simulation( R0=3.8
               , S=1000
               , E=100
               , I=0
               , R=0
               , days_incubation=21
               , days_recovery=21
               , days_elapsed=250
              )

# Run the simulation
#
s.run_simulation()

# Draw the charts
#
p = Plotter( Simulation = s)

p.plot()

# Show the charts
#
%matplotlib inline
```

The infected's curve reaches its peak about 90 days after day 0, then decreases. Let's now set up a different scenario. Let's suppose absence of containment measures, so 500 people were initially exposed, instead of 100. So E is the only parameter we are going to change.  In this last scenario we have an almost sudden raise of the Infected's curve, reaching its peak about only 50 days from day 0. Also, we'll have many more infected people in a shorter period of time. That's definitely a worst situation in comparison to the one of the first scenario.

This is a new simulation with $R_0=2.5$, $S=35000$ and $E=5000$

```python
s = Simulation( R0=2.5
               , S=35000
               , E=5000
               , I=0
               , R=0
               , days_incubation=21
               , days_recovery=21
               , days_elapsed=250
              )

# Run the simulation
#
s.run_simulation()

# Draw the charts
#
p = Plotter( Simulation_1 = s)

p.plot()

#s.plot_simulation()

# Show the charts
#
%matplotlib inline
```

Now, a comparative scenario with two hypothesis, $R0=2.5$ and $R0=1.5$

```python
sim1 = Simulation(             R0=2.5
                             , S=35000
                             , E=100
                             , I=0
                             , R=0
               , days_incubation=21
               ,   days_recovery=21
               ,    days_elapsed=450
              )

sim2 = Simulation(            R0=1.5
                             , S=35000
                             , E=100
                             , I=0
                             , R=0
               , days_incubation=21
               ,   days_recovery=21
               ,    days_elapsed=450
              )

# Run the simulation
#
sim1.run_simulation()
sim2.run_simulation()

# Draw the charts
#
p = Plotter(   Simulation = sim1
           , Simulation_2 = sim2 )

# Plot the charts
#
p.plot_comparison()

# Show the charts
#
%matplotlib inline
```

# Conclusions

From the chart we may see with ease that the model shows the expected behvior. The effective reproduction number declines progressively and steadily. The epidemics will reach its peak on correspondance of $R(t) = 1$. When $R(t)$ declines below $1$, also the infection declines and progressively fades away.

The SEIR model also shows the effectiveness of isolation measures directed on keeping the number of Exposed individuals as low as possible. This allows a smooth slope on the Infected's curve, allowing more time to take action and an overall number of infected people much lower than the one we may have otherwise.

Please keep in mind that the epidemiological parameters and the subsequent charts presented here are demonstrative and have the only purpose to show the dynamics of epidemiological diseases from a qualitative and numerical perspective and in no way they represent any real world scenario. This document and the Python code presented here have been written solely for study and research purposes.

# References

https://institutefordiseasemodeling.github.io/Documentation/

https://institutefordiseasemodeling.github.io/Documentation/general/model-seir.html

Almut Scherer, Angela McLean - Mathematical models of vaccination

William H. Press, Saul A. Teukolsky, William T. Vetterling, Brian P. Flannery - Numerical Recipes in C: The Art of Scientific Computing - Cambridge University Press

Andrea Vicerè - Corso di Simulazione Numerica


```python
import sympy as sp
from sympy.physics import vector as vec
sp.init_printing()
vec.init_vprinting()
```

```python
R = vec.ReferenceFrame('R')
R
```

    R

```python
R??
```

```python
x, y, z = R[0], R[1], R[2]
# x, y, z, t = sp.symbols('x y z t', real=True)
t = sp.symbols('t', real=True)
c, eps, mu = sp.symbols('c varepsilon_0 mu_0')
c, eps, mu
```

```python
Ex= sp.Function('Ex')(x, t)
Ey= sp.Function('Ey')(x, t)
Ez= sp.Function('Ez')(x, t)
Bx= 0
By= sp.Function('By')(x, t)
Bz= sp.Function('Bz')(x, t)
```

```python
E = Ex*R.x + Ey*R.y + Ez*R.z
B = Bx*R.x + By*R.y + Bz*R.z
vec.curl(E, R)
```

```python
jx= sp.Function('jx')(x, t)
jy= sp.Function('jy')(x, t)
jz= sp.Function('jz')(x, t)
j = jx*R.x + jy*R.y +jz*R.z
```

```python
# FARADAY = sp.Eq(vec.curl(E, R), -B.diff(t,R))
FARADAY = B.diff(t, R)- vec.curl(E, R)
FARADAY
```

```python
AMPERE = vec.curl(B, R) - mu * j - eps * mu * E.diff(t, R)
AMPERE
```

```python
(FARADAY + c * AMPERE).simplify()
```


# PC lab 4: Logistic regression for classification

## Introduction

In a binary classification setting, we are interested in assigning an observation $\mathbf{x}$ to one of two possible classes, denoted by $y$. For example, maybe we would like to tell if a patient has a particular disease (y = 1) or not (y = 0), given certain symptoms $\mathbf{x}$. Generally speaking, we want to predict the probability that the class label $y = 1$, conditional on the data that we have observed, $\mathbf{x}$. This probability is also called the *class posterior* or the *class-membership probability*, which we can denote as follows:

\begin{equation}
Pr(Y=1|X) = P(X) = p(y= 1|\mathbf{x})
\end{equation}

The book uses the statistical notation on the left, but the notation with the feature vector $\mathbf{x}$ is more common in machine learning literature. In any case, both notations mean exactly the same. In this PC lab, we will cover one of the most popular classifiers: logistic regression.

Just like linear regression, logistic regression (LR) is a linear model. However, LR does not model the mean of a continuous outcome, but the logarithm of the [odds](https://en.wikipedia.org/wiki/Odds) of the probability $P(X)$:

\begin{equation}
log \frac{P(X)}{1-P(X)} = w_{0}x_{0} + w_{1}x_{1} + ... + w_{p}x_{p} = \mathbf{w^Tx}
\end{equation}

However, we are really interested in the probability $p$ and not in the odds of p. Therefore, it is common to apply the inverse log-odds transformation on both sides of the equation. This transformation is the **logistic function $\phi(z)$**, hence the name of logistic regression:

\begin{equation}
\phi(z) = \frac{1}{1 + e^{-z}} = \frac{e^{z}}{1+e^{z}}
\end{equation}

Verify for yourself that applying $\phi(z)$ on the log-odds yields $p$.

In other words, we can make predictions for $p$ with logistic regression as follows:

\begin{equation}
p(\mathbf{x}) = \phi(w_{0}x_{0} + w_{1}x_{1} + ... + w_{p}x_{p})
\end{equation}

If we want to classify a data point $\mathbf{x}$, we can calculate $p$ with LR and simply assign it to class 1 if $p$ exceeds a certain probability threshold. A typical threshold is 0.5.

EXERCISE: What would happen to our predictions when we would choose a lower threshold, let's say 0.2? How would this affect the accuracy of our predictions? Can you think of a situation where we would want to do this?

Let's stop for a moment to have a look at what the logistic transformation does:

```python
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
plt.style.use('seaborn-white')

x = np.arange(-8,8,0.01) # Generate a range of x values
y = 1/(1+np.exp(-x)) # Calculate the logistic transformation of these x'es

# Plot them
fig, ax = plt.subplots()
ax.scatter(x,y, marker='.');
ax.set_xlabel('x');
ax.set_ylabel('y');
```

As shown, $\phi$ monotonically maps any number from the real domain to a number in [0,1]. Indeed, this is a desirable property if we want to predict a probability!

## Training a LR model

### Loss function: the cross-entropy loss

Now that we have the logistic regression model to predict the probability of belonging to a certain class, all that remains is the question of how to find the weights of the model on a given set of training data. As always, this is the problem of minimizing a loss function to find an optimal set of weights. Where we used the mean squared error (MSE) for linear regression, we will use the **cross-entropy** loss function for LR. Minimizing the binary cross-entropy loss is equivalent to minimizing the negative log-likelihood of the data under a binomial distribution:

\begin{equation}
l_{log} = \frac{1}{n}\sum\limits_{i=1}^{n}-y_{i}log(p(\mathbf{x}_i))-(1-y_i)log(1-p(\mathbf{x}_i))
\end{equation}

Where $y_i$ is the class of data point $i$ and $p(\mathbf{x}_i)$ is the class-membership probability predicted by logistic regression for the observation $\mathbf{x}_i$. If we look at the cross-entropy loss **for a single data point** $l_{log}^{i}$, we can break it down in two parts:

\begin{equation}
l_{log}^{i} =
\begin{cases}
   -log(p(\mathbf{x}_i)) & \text{if} \ y_i = 1\\
   -log(1-p(\mathbf{x}_i)) & \text{if} \ y_i = 0
\end{cases}
\end{equation}

It should be clear that the cross-entropy loss will be larger for smaller values of $p(\mathbf{x}_i)$ if $y_i = 1$, and vice versa. Let's visualize the cross-entropy loss for these two cases:

```python
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
plt.style.use('seaborn-white')

p = np.arange(0.01,0.99,0.01) # Generate a range of predicted probabilities between zero and 1
l_0 = -np.log(p)  # cross-entropy loss if y = 1
l_1 = -np.log(1-p)

# Plot them
fig, ax = plt.subplots(figsize=(10,6))
ax.scatter(p,l_0, marker='.');
ax.scatter(p, l_1, marker='.');
ax.set_xlabel('Predicted class-membership probability $p$');
ax.set_ylabel('Cross-entropy loss');
ax.legend(['Cross-entropy loss when y is 1', 'Cross-entropy loss when y is 0']);
```

EXERCISE: Make sure you understand the cross-entropy loss. Verify that it correctly penalizes wrong predictions in both cases. Suppose that we have no information about the data at all, what would be the best guess for p to minimize the cross-entropy loss?

### Finding the weights with gradient descent

For linear regression, the solutions to the normal equations provide a convenient analytical solution to obtain the optimal set of model weights $\mathbf{w}$ on a set of training data. There is no such solution to find the optimal weights for a logistic regression model, so instead an optimization algorithm such as **gradient descent** is used to train a LR model.

Gradient descent is an iterative optimization algorithm that searches for the optimum of an objective function by making small changes to a set of optimization variables. Gradient descent (and more complex optimization algorithms, but we offer a separate course for that) are widely used in machine learning to find the optimal set of model weights that minimize a certain loss function. Especially when there is no analytical solution for the weights available like for linear regression.

Generally, gradient descent uses the **gradient** of the loss function with respect to the model weights to perform updates to those weights in each iteration. At iteration $k+1$, the algoritm computes the gradient of a loss function $J(\mathbf{w})$ evaluated in the training data. Then, it performs an update to the current parameter values that is relative to the gradient multiplied with the learning rate $\gamma$, which is a constant:

\begin{equation}
\mathbf{w}_{k+1} = \mathbf{w}_{k} - \gamma\nabla{J(\mathbf{w}_{k})}
\end{equation}

Initially, the weights are often initialized with random draws from some distribution. The algorithm continues to do updates, until it converges or until some stopping criterion is reached.

In order to perform gradient descent to find the weights of a logistic regression model, we need to compute the gradient of the loss function with respect to the model parameters. Recall that, for a single data point, the cross-entropy loss function was as follows:

\begin{equation}
l_{log}^{i}(\mathbf{w}) = -y_{i}log(p(\mathbf{x}_i))-(1-y_i)log(1-p(\mathbf{x}_i))
\end{equation}

Where $p(\mathbf{x}_i)$ is nothing else than the weighted sum of the inputs squashed through the sigmoid function:

\begin{equation}
p(\mathbf{x}_i) = \phi(w_{0}x_{0i} + w_{1}x_{1i} + ... + w_{p}x_{pi})
\end{equation}

Before going on, let's first calculate the partial derivative of the sigmoid function:

\begin{equation}
\frac{\partial}{\partial z} \phi(z) = \frac{\partial}{\partial z} \frac{1}{1+e^{-z}} = \frac{e^{-z}}{(1+e^{-z})^2}
\end{equation}

We can rewrite this as follows:

\begin{equation}
\frac{e^{-z}}{(1+e^{-z})^2} = \frac{1 +e^{-z} -1}{(1+e^{-z})^2} = \frac{1}{1+e^{-z}} \Big( 1 - \frac{1}{1+e^{-z}}\Big) = \phi(z)(1 - \phi(z))
\end{equation}

With this result and by applying the chain rule, we can compute the partial derivative of the loss function with respect to the weight $w_j$. We will use the symbol $z$ to denote the weighted sum of the features (i.e., the input for the logistic function) and drop the superscript $i$ for clarity:

\begin{equation}
\frac{\partial l_{log}(\mathbf{w})}{\partial w_j} = \frac{\partial}{\partial w_j} \Big(-ylog(\phi(z))-(1-y)log(1-\phi(z)) \Big) \\ = \Big(  \frac{-y}{\phi(z)} + \frac{1-y}{1-\phi(z)}  \Big)\frac{\partial}{\partial w_j}\phi(z) \\ = \Big(  \frac{-y}{\phi(z)} + \frac{1-y}{1-\phi(z)}  \Big) \phi(z)(1-\phi(z))\frac{\partial}{\partial w_j}z
\end{equation}

Since $z = w_{0}x_{0} + w_{1}x_{1} + ... + w_{p}x_{p}$, $\frac{\partial}{\partial w_j}z$ is nothing more than $x_j$, so we can rewrite the above as:

\begin{equation}
\frac{\partial l_{log}(\mathbf{w})}{\partial w_j} = \Big( -y(1-\phi(z) + (1-y)\phi(z))\Big)x_j \\ = \big(  -y + \phi(z) \big)x_j = \big( \phi(z) - y \big)x_j
\end{equation}

With this partial derivative of the loss w.r.t $w_j$, we can write the update rule of the gradient descent algorithm for the $j^{th}$ weight:

\begin{equation}
w_{j,k+1} = w_{j,k} - \gamma(\phi(z_k)-y)x_{j}
\end{equation}

In other words, the algorithm will each time perform an update to the weight $w_{j}$ that is in proportion to the difference between the predicted probability of class membership in the previous iteration and the actual class. Makes sense! The entire gradient is simply the vector that contains the partial derivatives with respect to the entire weight vector $\mathbf{w}$, and in reality gradient descent acts on $\mathbf{w}$ and not on an individual weight $w_j$. Also, the gradient is typically not calculated for one data point, but evaluated over the entire training data set.

In practice, software packages such as [scikit-learn](http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html#sklearn.linear_model.LogisticRegressionscikit-learn) do this optimization under the hood, so there is no need to implement it manually each time we want to use logistic regression.

## Application: predicting the status of a breast cancer tumor

In the first application of logistic regression, we will use the [Breast Cancer Wisconsin (Diagnostic) Data Set](https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+%28Diagnostic%29). The dataset contains information on the disease status of 569 breast cancer patients: they were either diagnosed with a malign (status M) or with a benign (status B) tumor.

For each patient, the dataset also contains 30 features that represent statistics of the cell nuclei present in images taken after [fine needle aspirate tissue samples](https://en.wikipedia.org/wiki/Fine-needle_aspiration). These 30 features are the mean, standard deviation and the maximum of 10 measurements on the cell nuclei:
- radius
- texture
- perimeter
- area
- smoothness
- compactness
- concavity
- concave points
- symmetry
- fractal dimension

**Based on these feature of the cell nuclei, we would like to predict whether a patient has a malign or a benign breast cancer tumor.** Let's read in the data:

```python
import pandas as pd
import numpy as np
data = pd.read_csv('./wdbc.data', header=None, index_col=0, names=['Patient ID', 'status'] + list(np.arange(1,31,1)))
status = data['status']
data.head()
```

First, let's look at the distribution of the disease status:

```python
pd.value_counts(data['status']).plot(kind='bar');
```

There are about 350 benign cases and roughly 200 malign cases. This is a fairly balanced dataset.

EXERCISE: Suppose that the dataset was unbalanced, with 525 B cases and only 25 M cases. Can you think of any problems this could give if we would evaluate the accuracy of our logistic regression predicitions? We will come back to this problem in one of the next labs.

In order to perform LR, we will encode the disease status as a binary variable.

```python
from sklearn.preprocessing import LabelEncoder
encoder = LabelEncoder().fit(status)
encoder.classes_  # 'B' will become class 0, 'M' will become class 1
```

```python
y = encoder.transform(status)
x = data.drop('status', axis=1).values # Drop the disease status from the dataframe, convert to numpy array
y
```

EXERCISE: Using scikit-learn, split the data in a 80% training and a 20% test set. Fit a logistic regression model and evaluate trainig and testing accuracy. You should be able to achieve a fairly high accuracy!

Use [this method](http://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html) for train-test splitting and [this implementation](http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html) to perform logistic regression. You can use the [score method](http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html#sklearn.linear_model.LogisticRegression.score) to evaluate the accuracy of your model. This method computes the accuracy as follows:

\begin{equation}
score = \frac{\text{Number of correctly classified instances}}{\text{Total number of instances}}
\end{equation}

```python
# ** solution **
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression

X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.20, random_state=42)

LRmodel = LogisticRegression()
LRmodel.fit(X_train, y_train)

LRmodel.score(X_train, y_train)

```

```python
LRmodel.score(X_test, y_test)
# ** solution **
```

To get an idea of which features are considered important by the LR model, we can visualize the weights it has learned in a bar plot:

```python
fig, ax = plt.subplots(figsize=(10,5))
pd.Series(LRmodel.coef_.flatten()).plot(ax=ax, kind='bar')
```

Use your LR model to predict the class probabilities and the classes for the training data. Use the [```predict_proba()```](http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html#sklearn.linear_model.LogisticRegression.predict_proba) method to generate the predicted probabilities. Use the code below to plot the two against each other. Which data points are most likely to be misclassified?

```python
# ** solution **
predicted_class_probabilities = LRmodel.predict_proba(X_train)[:,1]
predicted_classes = LRmodel.predict(X_train)

#** solution **

misclassified = predicted_classes !=  y_train

colors = ['#b2182b' if wrong else '#2166ac' for wrong in misclassified ]
fig, ax = plt.subplots(figsize=(8,6))
ax.scatter(predicted_class_probabilities, predicted_classes, marker='.', s=100, color=colors)
ax.set_xlabel('Predicted class probabilies').set_fontsize(20)
ax.set_ylabel('Predicted classes').set_fontsize(20)
ax.legend(['Correctly classified'])
```

Clearly, the misclassified points are those points where the predicted probability of class membership is rather close to 0.5.

# Multiclass classification

## One-versus-one classification

One-versus-one classification is another approach to a multiclass classification problem. For a K-class problem, the strategy consists of training $\frac{K(K-1)}{2}$ classifiers. Each of these classifiers much learn to distinguish to classes. One the classifiers are trained, a voting scheme is applied to make a prediction for an unseen data point: each classifier has to decide between two possible classes. The final predicted class is that class that gets the largest number of votes.

## One-versus-all classification

In one-versus-all (OvA) classification, a single classifier is trained per class, with the samples of that class as positive samples and all other samples as negatives. The strategy proceeds as follows for a K-class classification problem:

**Inputs:**
* a classification algorithm L (learner)
* feature matrix $\mathbf{X}$
* label vector y where $y_i \in {1,...,K}$

**Procedure:**
for each k in {1,...,K}:
* construct a new label vector z where $z_i$ is 1 if $y_i$ = k and 0 otherwise
* train L on $\mathbf{X}$ to obtain a classifier $f_k$. The classifier should return class probabilities and not hard labels.

**Returns**
A list of trained classifiers $f_k$ for each  k in {1,...,K}

To make predictions for a new sample $\mathbf{x}$, the $k$ classifiers are applied to $\mathbf{x}$ and the final predicted label is the label that is predicted with the highest confidence (probability):

$\hat{y} = \underset{k \in {1,...,K}}{\mathrm{argmax}} \, f_k(\mathbf{x})$

Let's simulate a toy dataset with three classes and two features, and split it in training and test data:

```python
from sklearn.datasets import make_blobs
from sklearn.model_selection import train_test_split

X, y = make_blobs(n_samples=1000, centers= [[-2.5, 0], [0, 1], [3.5, -1]], random_state=42)

#train-test split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
```

```python
# Make the plot
fig, ax = plt.subplots(figsize=(15,10))
colors=['#66c2a5', '#fc8d62', '#8da0cb']
for i, color in enumerate(colors):
    idx_train = np.where(y_train==i)
    idx_test = np.where(y_test==i)
    plt.scatter(X_train[idx_train,0], X_train[idx_train,1], c=color, edgecolor='black', s=30)
    plt.scatter(X_test[idx_test,0], X_test[idx_test, 1],c='white', edgecolor=color, s=70)

ax.legend(['Class 1 - train',
           'Class 1 - test',
           'Class 2 - train',
           'Class 2 - test',
           'Class 3 - train',
           'Class 3 - test']);

ax.set_xlabel('Feature 1');
ax.set_ylabel('Feature 2');
ax.set_title('Toy dataset for multiclass classification').set_fontsize(20);
```

```python
i = 0
```

```python
z_train = np.zeros(len(y_train))
z_train[np.where(y_train==i)] = 1
```

```python
z_train
```

Implement a one-versus-all loop to tackle this classification problem. Train a list of classifiers on the training data. Make predictions on the test data. You can use the code below to get started.

```python
# ***solution***

L1 = LogisticRegression()
L2 = LogisticRegression()
L3 = LogisticRegression()

L = [L1, L2, L3]

# Train the list of classifiers in one-v-all fashion
for i,l in enumerate(L):
    z_train = (y_train==i)
    l.fit(X_train, z_train)

# Make predictions on the test data

predictions = []

for l in L:
    predictions.append(l.predict_proba(X_test)[:,1])

predicted_classes = np.array([np.argmax([pred[i] for pred in predictions]) for i in range(len(X_test))])

# ***solution***
```

Run the code below to visualize your predictions.

```python
classification_accuracy=np.round(np.mean(y_test == predicted_classes)*100,2)
```

```python
# Visualize the predictions

fig, ax = plt.subplots(figsize=(15,10))
colors=['#66c2a5', '#fc8d62', '#8da0cb']

for i, color in enumerate(colors):
    idx_train = np.where(y_train==i)
    idx_test = np.where(y_test==i)

    plt.scatter(X_train[idx_train,0], X_train[idx_train,1], c=color, edgecolor='black', s=30)
    plt.scatter(X_test[idx_test,0], X_test[idx_test, 1],c='white', edgecolor=color, s=70)

ax.legend(['Class 1 - train',
           'Class 1 - test',
           'Class 2 - train',
           'Class 2 - test',
           'Class 3 - train',
           'Class 3 - test']);

# add predictions
for i, color in enumerate(colors):
    idx_predicted = np.where(predicted_classes==i)
    plt.scatter(X_test[idx_predicted,0], X_test[idx_predicted,1], c=color, marker='s', s=2)

ax.set_xlabel('Feature 1');
ax.set_ylabel('Feature 2');
ax.set_title('Toy dataset for multiclass classification - classification accuracy: {}%'.format(classification_accuracy)).set_fontsize(20);
```

```python

```

```python

```


# Ship Damage Probability
Should I roll the dice? This notebook introduces the probabilities that will help Seafall players make that decision.

```python
%matplotlib inline

import itertools
import matplotlib
import matplotlib.pyplot
import numpy
import os
import pandas
import scipy.misc
import scipy.special
import seaborn
```

```python
# colormap for data visualizations
cmap = seaborn.cubehelix_palette(n_colors = 20, start = 0.8, rot = 0.75,
                                gamma = 0.3, hue = 2.0, dark = 0.005,
                                light = 0.95, reverse = True, as_cmap = True)
```

## A cursed throw during the prologue
I went to explore a region on the first island with my flagship being supported by the second ship in my fleet. With the Woodsman as my advisor I would have 4 dice to roll where I needed 3 successes to avoid damage to my ship. I cast my dice and grimaced after they settled on 3 blanks and merely 1 success. I felt unlucky (even though my endeavor was ultimately successful). Was I cursed?

## Counting the outcomes of dice rolls
Dice rolls can be [framed](http://math.stackexchange.com/questions/900672/how-many-combinations-from-rolling-5-identical-dice) as [stars and bars](http://math.stackexchange.com/questions/208377/combination-with-repetitions) problems. This is a clever way to frame the combinatorics of dice rolling. The combination formula is used to choose the location of the bars, whose position determines how many dice show a particular face. The stars between the bars represent the number of dice showing a particular face; the identity of that face is determined by how many bars are to the right (or left) of the star(s). If two bars are adjacent that means there aren't any dice showing the particular face defined by the position between those bars.

\begin{equation}\text{unique dice combinations} = \binom{n - 1 + k}{n - 1}\end{equation}

\begin{equation}n = \text{the number of faces on a die}\end{equation}

\begin{equation}k = \text{the size of the dice pool}\end{equation}

*Note that all dice in the pool must be identical. Unless otherwise stated, it is assumed that the dice pool consists of identical dice.*

### The combination formula is essential
The combination formula will again prove useful for counting outcomes when considering only two possibilities. In Seafall, often it only matters whether the face of a die is blank or not. In these cases, the combination formula can be used to count the outcomes by enumerating the ways a pool of dice can be divided into two sets. From a given dice pool choose a given number of dice to be part of the *blank* set; since there are only two sets this means the remaining dice are part of the *success* set.

\begin{equation}\text{unique dice combinations} = \binom{n}{k}\end{equation}

\begin{equation}n = \text{the size of the dice pool}\end{equation}

\begin{equation}k = \text{the number of blanks}\end{equation}

Looking ahead, when there are more than two outcomes we'll need the multinomial coefficient formula, which reduces to the combination formula when there is only two outcomes. The multinomial formula comes into play during more complex situations encountered in Seafall. For the most common encounters on the high seas we'll be using [combinations](https://en.wikipedia.org/wiki/Combination) to do the heavy lifting for our calculations.

### Rolling an example
Consider the scenario where 2, 6-sided dice are being rolled. Let's enumerate all of the possible outcomes:

1st die | 2nd die
---: | ---:
1 | 1
1 | 2
1 | 3
1 | 4
1 | 5
1 | 6
2 | 1
2 | 2
2 | 3
2 | 4
2 | 5
2 | 6
3 | 1
3 | 2
3 | 3
3 | 4
3 | 5
3 | 6
4 | 1
4 | 2
4 | 3
4 | 4
4 | 5
4 | 6
5 | 1
5 | 2
5 | 3
5 | 4
5 | 5
5 | 6
6 | 1
6 | 2
6 | 3
6 | 4
6 | 5
6 | 6

***Phew!*** That's a long list.

After counting the rows we find there are 36 unique outcomes if we are keeping track of the identity of the 1st and 2nd die; 6-faces times 6-faces equals 36 pairs.

However, we often don't care about the identity of the dice that we roll, just their values. What if we ignore the identity of the dice and only count unique pairs? For example, given the pair (1,2) we'll want to throw out (2,1).

1st die | 2nd die
---: | ---:
1 | 1
1 | 2
1 | 3
1 | 4
1 | 5
1 | 6
2 | 2
2 | 3
2 | 4
2 | 5
2 | 6
3 | 3
3 | 4
3 | 5
3 | 6
4 | 4
4 | 5
4 | 6
5 | 5
5 | 6
6 | 6

Now we only have 21 rows. This is the number that represents the number of unique outcomes from rolling 2 dice. The stars and bars equation is the formula for finding this number. Stars and bars is easiest to understand through an example. The key is to avoid enumeration, like we did above, and focus on how a group of dice can be distributed into groups that represent the faces on the dice. Consider the 2, 6-sided dice scenario again, but instead of 2 dice imagine we have 2 footballs and instead of 6 faces to a die we have 5 orange cones. We are going to place either a football or a cone along the sideline at every 1 yard hash mark. Let's graphically represent the 2 footballs and 5 cones on the football field:

yardline | 1 | 2 | 3 | 4 | 5 | 6 | 7
--- | --- | --- | --- | --- | --- | --- | --- | ---
 | **football** | **football** | cone | cone | cone | cone | cone

The 5 cones divide our line into 6 groups. In the configuration above, the 2 footballs would be in the first group, because the footballs are placed before the first cone. As dice, this would be the same as rolling (1, 1). Reordering the footballs and cones along the hash marks corresponds to another roll of the dice. For example,

yardline | 1 | 2 | 3 | 4 | 5 | 6 | 7
--- | --- | --- | --- | --- | --- | --- | --- | ---
 | cone | **football** | cone | cone | cone | cone | **football**

This configuration is like rolling (2, 6); the second group exists between the first cone and second cone and the sixth group is after the fifth cone. To enumerate all the configurations of the footballs and the cones we only have to consider where the 5 cones are placed along the hash marks. We don't have to keep track of the footballs, because the footballs are all the same and their location along the hash marks is dictated by the positions of the cones. Therefore, to find the total number of outcomes for rolling 2, 6-sided dice we can calculate where to place 5 cones along the first 7 hash marks of a football field. The combination equation will give the number we are interested in, 7 choose 5.

```python
# FWIW, total number of unique outcomes from rolling 2, 6-sided dice.
# Note that stars-and-bars is also called combination with repition.
print("stars and bars", scipy.misc.comb(6, 2, repetition=True)) # scipy.misc.comb(# of faces, # of dice, repitition=True)
# combination: asks, "Given 5 orange cones, how many different ways are there to roll 2 blanks?"
print("combination", scipy.misc.comb(7,5))
```

    stars and bars 21.0
    combination 21.0

### Where are permutations?
The permutation equation is closely related to the combination equation, but we are not using it at all. Why? It is because we do not care about the order that we are rolling the dice when we are considering the outcome of a dice pool. If it mattered that we rolled a success before we rolled a blank, or vice versa, then the permutation equation would be appropriate. However, a success rolled before a blank or a blank rolled before a success will always add up to 1 success and 1 blank, so the combination formula is correct.

## Weighted dice
Since the possibilities of rolling a blank is weighted, i.e. the chance of rolling a blank does not equal the probability of *not* rolling a blank, counting the outcomes alone will not yield the correct probability. Each outcome for a given combination will however share the same probability, so it is only a matter of finding the probability for a single outcome and multiplying it by the total number of unique outcomes.

```python
cursed_probability = (1.0/3)**3 * (2.0/3) * scipy.misc.comb(4,1) + (1.0/3)**4
print(cursed_probability)
```

    0.111111111111

My roll (or worse) would only happen 11% of the time, so I would consider myself unlucky and stop short of calling myself cursed.

# A simulation
Before we explicitly calculate the probabilites for potential ship damage, simulating the dice rolls to sample the actual probability space is a useful sanity check we can refer to as we code-up the exact probabilities.

We will map a random number from a uniform distribution $\left[0, 1\right]$ to a piecewise function that reflects the probability of a given face resulting from a roll. When we only care about blanks and successes the 6 sides of the dice can be treated as a weighted coin instead. On one "side" of the coin is the blank result shown by 2 of the 6 faces, so it has a $1/3$ chance of happening. That's all we need to know, because any other result will be considered a success. The piecewise function will return a 1 if the random variable is  $<= 1/3$, and a 0 otherwise. The end result will be the total number of blanks found after rolling a dice pool.

```python
p_blank = 1.0/3

def simulate_roll(total_dice):
    x = numpy.random.uniform(size=total_dice)
    conds = [x <= p_blank, (x > p_blank)]
    # each blank counts as 0, other successes count as 1
    roll_array = numpy.piecewise(x, conds, [0, 1])
    return numpy.sum(roll_array)

def success_roll_simulator(total_dice, n):
    sim_list = [simulate_roll(total_dice) for i in range(n)]
    return numpy.array(sim_list)
    #return {"num_success" : num_success_array, "p_success" : p_success_array}

n = 10000
success_count_list = [success_roll_simulator(i, n) for i in numpy.arange(1,16)]
for i, array in enumerate(success_count_list):
    array[array < 0] = 0
    success_count_list[i] = array
success_count_itemfreq = [scipy.stats.itemfreq(a) for a in success_count_list]

sdp_sim = numpy.zeros((15,16))
for i, array in enumerate(success_count_itemfreq):
    for j in array:
        sdp_sim[i, j[0].astype(int)] = j[1]/n

# for the greater than or equal probabilites, sum along the rows
sdp_sim_gte = numpy.zeros((15,15))

for (i, j), _ in numpy.ndenumerate(sdp_sim_gte):
    sdp_sim_gte[i, j] = numpy.sum(sdp_sim[i, j+1:])

row_blank = numpy.arange(1,16) # number of dice showing the blank face
col_dice = numpy.arange(1,16) # number of dice in the pool
sdp_sim_gte_dataframe = pandas.DataFrame(data = sdp_sim_gte, index = row_blank, columns = col_dice)
sdp_sim_gte_dataframe_stack = sdp_sim_gte_dataframe.stack().reset_index().rename(
    columns = {
        "level_0" : "number_success",
        "level_1" : "number_dice_pool",
        0 : "probability_GTE_to_number_success"})

sdp_sim_gte_pivot = sdp_sim_gte_dataframe_stack.pivot("number_success",
                                                      "number_dice_pool",
                                                      "probability_GTE_to_number_success")

sdp_sim_gte_mask = numpy.logical_or(sdp_sim_gte_pivot > 0.999, numpy.triu(numpy.ones((15,15)), 1))
```

```python
seaborn.set(style = "white")
seaborn.set_context("paper")
seaborn.set_style({'font.family': 'Noto Sans'})
matplotlib.pyplot.figure(figsize=(9, 6))
ax = seaborn.heatmap(sdp_sim_gte_pivot, annot = True, cmap = cmap,
                     cbar = False, mask = sdp_sim_gte_mask,
                     annot_kws = {"weight" : "bold", "fontsize" : 12},
                     fmt = ".0%", linewidths = 1, vmin=0.0,
                     vmax=1.0, square=False)
ax.tick_params(labelsize=16)
ax.tick_params(axis='y', rotation=0)

ax.set_title(
    """Ship Damage Probabilty (simulation):
    Probability at least X successes given Y sized dice pool""",
    fontsize=16)
ax.set_xlabel("Number of successes", fontsize=16)
ax.set_ylabel("Size of dice pool", fontsize=16)
fig = ax.get_figure()
fig.savefig(os.path.join("..", "stage1", "charts", "ship_damage_probabilities_1to15_simulation.pdf"))
fig.savefig(os.path.join("..", "stage1", "charts", "ship_damage_probabilities_1to15_simulation.svg"), format="svg", dpi=1200)
```

```python
p_blank = 1.0/3

def prob_success(num_success, total_dice):
    if total_dice < num_success:
        p = 0.0
    else:
        p = (p_blank)**(total_dice-num_success) * \
        (1.0-p_blank)**num_success * \
        scipy.misc.comb(total_dice, num_success)
    return p
```

```python
cursed_probability = prob_success(0,4) + prob_success(1,4)
print(cursed_probability)
```

    0.111111111111

```python
def prob_success_or_better(num_success, total_dice):
    if total_dice < num_success:
        p = 0.0
    else:
        rng_blank = numpy.arange(num_success, total_dice + 1)
        array_blank = [prob_success(n, total_dice) for n in rng_blank]
        p = numpy.sum(array_blank)
    return p
```

```python
cursed_probability = 1 - prob_success_or_better(2,4)
print(cursed_probability)
```

    0.111111111111

# Ship Damage Probability (SDP) matrix
There are 15 dice within Seafall, so a matrix should be large enough to assess the situation where all 15 are rolled. The early game does not use near this many dice, but eventually there is a island to discover at sea with a 30 defense! To cover all these situations several visualizations will be created.

```python
row_success = numpy.arange(1,31) # number of dice showing the blank face
col_dice = numpy.arange(1,31) # number of dice in the pool
sdp = numpy.zeros((numpy.size(row_success),numpy.size(col_dice)))
prob = [prob_success_or_better(*i) for i in itertools.product(row_success, col_dice)]
ind = [tuple(numpy.subtract(i,1)) for i in itertools.product(row_success, col_dice)]
for idx, val in enumerate(ind):
    sdp[val] = prob[idx]
```

```python
# http://stackoverflow.com/questions/41474284/how-to-convert-a-matrix-into-column-array-with-pandas-python
sdp_dataframe = pandas.DataFrame(data = sdp, index = row_success, columns = col_dice)
sdp_dataframe_stack = sdp_dataframe.stack().reset_index().rename(
    columns = {
    "level_0" : "number_successes",
    "level_1" : "number_dice_pool",
    0 : "probability_GTE_to_number_successes"}
)

sdp_dataframe_15 = sdp_dataframe_stack.loc[(sdp_dataframe_stack["number_successes"] < 16) &
                                          (sdp_dataframe_stack["number_dice_pool"] < 16)]

sdp_pivot = sdp_dataframe_15.pivot("number_dice_pool",
                                      "number_successes",
                                      "probability_GTE_to_number_successes")
sdp_mask = numpy.logical_or(sdp_pivot > 0.999, numpy.triu(numpy.ones((15,15)), 1))
```

```python
seaborn.set(style = "white")
seaborn.set_context("paper")
seaborn.set_style({'font.family': 'Noto Sans'})
matplotlib.pyplot.figure(figsize=(9, 6))
ax = seaborn.heatmap(sdp_pivot, annot = True, cmap = cmap,
                     cbar = False, mask = sdp_mask, annot_kws = {"weight" : "bold", "fontsize" : 12},
                     fmt = ".0%", linewidths = 1.5, vmin=0.0,
                     vmax=1.0)
ax.set_title(
    """Ship Damage Probabilty:
    Probability at least X successes given Y sized dice pool""",
    fontsize=16)

ax.tick_params(labelsize=16)
ax.tick_params(axis='y', rotation=0)

ax.set_xlabel("Number of successes", fontsize=16)
ax.set_ylabel("Size of dice pool", fontsize=16)
fig = ax.get_figure()
fig.savefig(os.path.join("..", "stage1", "charts", "ship_damage_probabilities_1to15.pdf"))
fig.savefig(os.path.join("..", "stage1", "charts", "ship_damage_probabilities_1to15.svg"), format="svg", dpi=1200)
```

```python
seaborn.set(style = "white")
seaborn.set_context("paper")
seaborn.set_style({'font.family': 'Noto Sans'})
matplotlib.pyplot.figure(figsize=(9, 6))
sdp_dataframe_7 = sdp_dataframe_stack.loc[(sdp_dataframe_stack["number_successes"] < 8) &
                                          (sdp_dataframe_stack["number_dice_pool"] < 8)]

sdp_pivot_7 = sdp_dataframe_7.pivot("number_dice_pool",
                                    "number_successes",
                                    "probability_GTE_to_number_successes")

sdp_mask_7 = numpy.logical_or(sdp_pivot_7 > 0.999, numpy.triu(numpy.ones((7,7)), 1))

ax = seaborn.heatmap(sdp_pivot_7, annot = True, cmap = cmap,
                     cbar = False, mask = sdp_mask_7, annot_kws = {"weight" : "bold", "fontsize" : 16},
                     fmt = ".0%", linewidths = 2.0, vmin=0.0,
                     vmax=1.0)
ax.set_title(
    """Ship Damage Probabilty:
    Probability at least X successes given Y sized dice pool""",
    fontsize=16)
ax.tick_params(labelsize=16)
ax.tick_params(axis='y', rotation=0)
ax.set_xlabel("Number of successes", fontsize=16)
ax.set_ylabel("Size of dice pool", fontsize=16)
fig = ax.get_figure()
fig.savefig(os.path.join("..", "stage1", "charts","ship_damage_probabilities_1to7.pdf"))
fig.savefig(os.path.join("..", "stage1", "charts","ship_damage_probabilities_1to7.svg"), format="svg", dpi=1200)
```

# A daring raid upon my province
The end of the 2nd game was nigh. I ended my turn with 10 glory and was primed to secure a 2-glory treasure and victory in the next round. However, it was a close game. Mike and Joe both had 9 glory and Mike still had a turn to go in the current round. I had already claimed a 2-glory treasure earlier in the game, and Mike eyed my vault enviously; I had a target on my back. Unfortunately, my ships were both at sea, so my home port was undefended. Mike took advantage and attacked! He had 6 dice to roll and needed 5 successes to raid my treasure room; my next turn would never come. Here is how the numbers break down:

Look at row 6 and column 5. Mike had a 35% chance of 5 or more successes that he needed to plunder my treasure! I hope my luck changes soon. I tip my cap to his bold play.

Eventually, dice pools will exceed the number of dice provided, 15, in the game of SeaFall.

```python
sdp_dataframe_30 = sdp_dataframe_stack.loc[(sdp_dataframe_stack["number_successes"] > 5) &
                                           (sdp_dataframe_stack["number_successes"] < 26) &
                                          (sdp_dataframe_stack["number_dice_pool"] > 15)]

sdp_pivot_30 = sdp_dataframe_30.pivot("number_dice_pool",
                                    "number_successes",
                                    "probability_GTE_to_number_successes")

sdp_mask_30 = numpy.logical_or(sdp_pivot_30 > 0.999, numpy.triu(numpy.ones((15,20)), 11))

seaborn.set(style = "white")
seaborn.set_context("paper")
seaborn.set_style({'font.family': 'Noto Sans'})
matplotlib.pyplot.figure(figsize=(9, 6))
ax = seaborn.heatmap(sdp_pivot_30, annot = True, cmap = cmap,
                     cbar = False, mask = sdp_mask_30, annot_kws = {"weight" : "bold", "fontsize" : 10},
                     fmt = ".0%", linewidths = 1, vmin=0.0,
                     vmax=1.0)
ax.set_title(
    """Ship Damage Probabilty:
    Probability at least X successes given Y sized dice pool""",
    fontsize=16)
ax.tick_params(labelsize=16)
ax.tick_params(axis='y', rotation=0)
ax.set_xlabel("Number of successes", fontsize=16)
ax.set_ylabel("Size of dice pool", fontsize=16)
fig = ax.get_figure()
fig.savefig(os.path.join("..", "stage1", "charts", "ship_damage_probabilities_16to30.pdf"))
fig.savefig(os.path.join("..", "stage1", "charts", "ship_damage_probabilities_16to30.svg"), format="svg", dpi=1200)
```

```python

```


Euler Problem 49
================

The arithmetic sequence, 1487, 4817, 8147, in which each of the terms increases by 3330, is unusual in two ways:
    - each of the three terms are prime, and,
    - each of the 4-digit numbers are permutations of one another.

There are no arithmetic sequences made up of three 1-, 2-, or 3-digit primes, exhibiting this property, but there is one other 4-digit increasing sequence.

What 12-digit number do you form by concatenating the three terms in this sequence?

```python
from sympy import isprime
from collections import defaultdict
from itertools import combinations

primes = defaultdict(list)
for n in range(1001, 10000, 2):
    if isprime(n):
        key = ''.join(sorted(str(n)))
        primes[key].append(n)
for key in primes:
    for p, q in combinations(primes[key], 2):
        r = q + (q - p)
        if r in primes[key]:
            print(f'{p}{q}{r}')
            break
```

    148748178147
    296962999629

```python

```


```python
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
```

```python
# Ensure that we always get the same results
np.random.seed(0)

# Helper function to generate a random data sample
def generate_random_data_sample(sample_size, feature_dim=2, num_classes=2):
    # Create synthetic data using NumPy.
    Y = np.random.randint(size=(sample_size, 1), low=0, high=num_classes)
    # Make sure that the data is separable
    X = (np.random.randn(sample_size, feature_dim)+5) * (Y+1)
    return X, Y

# Helper function to draw decision boundary
def draw_result(x, x_values, y_values, cost=None):
    plt.figure(figsize=(10,5))
    ax = plt.gca()
    ax.grid(color='#b7b7b7', linestyle='-', linewidth=0.5, alpha=0.5)
    plt.scatter(x[:,0], x[:,1], c=colors)
    plt.plot(x_values, y_values, label='Decision Boundary', color='#121212', linewidth=2, alpha=0.9)
    if cost:
        ax.text(0, 0, f'error = {cost:.2f}',fontsize=12,color='#000000')
    plt.show()
```

```python
# Generate random separable data
sample_size = 128
x, labels = generate_random_data_sample(sample_size)
y = np.array([labels.T[0]]).T
colors = ['#333333' if label == 0 else '#b7b7b7' for label in labels]

# generate random decision boundary
theta = np.random.randn(3)
x_values = [np.min(x[:, 0] - 5), np.max(x[:, 1] + 5)]
y_values = -(theta[0] + np.dot(theta[1], x_values))/theta[2]

draw_result(x, x_values, y_values)
```

## Logistic regression

Optimize decision boundary with logistic regression.

### 1. Feedforward
\begin{equation}
\mathbf{z} = XW
\end{equation}

\begin{equation}
\mathbf{\hat{y}} = \sigma(\mathbf{z})
\end{equation}

\begin{equation}
\sigma(z) = \frac{1}{1+e^{-z}}
\end{equation}

### 2. Compute cost function
\begin{equation}
L(\mathbf{y}, \mathbf{\hat{y}}) = -\frac{1}{m}\big(\mathbf{y}^T\log(\mathbf{\hat{y}})+(1-\mathbf{y})^T\log(1-\mathbf{\hat{y}})\big)
\end{equation}

### 3. Backpropagation
\begin{equation}
\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial W}
= \frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial \mathbf{\hat{y}}} \cdot \frac{\partial \mathbf{\hat{y}}}{\partial W}
= \frac{1}{m}X^T(\mathbf{\hat{y}} - \mathbf{y})
\end{equation}

### 4. Gradient descent
\begin{equation}
W = W - \alpha  \frac{\delta Loss(y, \hat{y})}{\delta W}
\end{equation}

```python
class LogisticRegression:
    """ Simple logistic regression """
    def __init__(self):
        pass

    def _init_params(self, x, y, iterations, learning_rate, reg_factor):
        """ Initilize parameters.

        ----------
        W : ndarray, shape (n_features+1,)
            Coefficient vector
        """
        self._X = np.hstack([np.ones((x.shape[0], 1)), x])
        self._y = y
        self._learning_rate = learning_rate
        self._reg_factor = reg_factor
        self.weights_ = np.random.rand(self._X.shape[1],1)
        self.costs_ = np.zeros(iterations)

    def _sigmoid(self, x):
        """ Computes sigmoid function. """
        return 1.0/(1 + np.exp(-x))

    def _feedforward(self):
        """ Computes np.dot(X, W). """
        self._y_hat = self._sigmoid(self._X.dot(self.weights_))

    def _backprop(self):
        """ Update weights. """
        m = len(self._y)
        # update weights with L2 regularization term
        _weights = self.weights_.copy()
        # ignore bias term
        _weights[0, 0] = 0
        self.weights_ -= self._learning_rate * self._X.T.dot(self._y_hat - self._y)/m + self._reg_factor/m*_weights

    def _get_cost(self):
        """ Compute loss. """
        m = len(self._y)
        # cost function with L2 regularization term
        _weights = self.weights_.copy()
        # ignore bias term
        _weights[0, 0] = 0
        return -1/m * (self._y.T.dot(np.log(self._y_hat)) + (1-self._y).T.dot(np.log(1-self._y_hat))) + self._reg_factor/(2*m)*_weights.T.dot(_weights)

    def fit(self, x, y, iterations=1000, learning_rate=0.2, reg_factor=0.5):
        """ Fit model.

        ----------
        x : ndarray, shape (n_samples, n_features)
            Training data
        y : ndarray, shape (n_samples,)
            Target data
        """
        self._init_params(x, y, iterations, learning_rate, reg_factor)

        # train model
        for i in range(iterations):
            self._feedforward()
            self._backprop()
            self.costs_[i] = self._get_cost()
```

```python
lr = LogisticRegression()
lr.fit(x, y)

new_y_values = -(lr.weights_[0] + lr.weights_[1]* x_values)/lr.weights_[2]

draw_result(x, x_values, new_y_values, lr.costs_[-1])
```


### Introduction to ARCH and GARCH models

Various problems such as option pricing in finance have motivated the study of the volatility, or variability, of a time series. ARMA models were used to model the conditional mean of a process when the conditional variance was constant. In many problems, however, the assumption of a constant conditional variance will be violated. Models such as the autoregressive conditionally heteroscedastic or ARCH model were developed to model changes in volatility. These models were later extended to generalized ARCH, or GARCH models.

In these problems, we are concerned with modeling the return or growth rate of a series. For example, if $x_t$ is the value of an asset at time $t$, then the return or relative gain, $r_t$, of the asset at time $t$ is

$$
r_t = \frac{x_t-x_{t-1}}{x_{t-1}}.
$$

If the return represents a small (in magnitude) percentage change then

$$
\log(\frac{x_t}{x_{t-1}}) = \log(1+r_t) \approx r_t.
$$

Typically, for financial series, the return $r_t$, does not have a constant conditional variance, and highly volatile periods tend to be clustered together. In other words, there is a strong dependence of sudden bursts of variability in a return on the series own past.

The simplest ARCH model, the **ARCH(1)**, models the return as
$$
\begin{equation}
r_t = \mu+\epsilon_t\\
\epsilon_t = \sigma_t e_t, \ \ e_t ∼ \ N(0, 1)\\
\sigma_t^2 = \alpha_0 + \alpha_1 r^2_{t-1}.
\end{equation}
$$

The Normal assumption may be relaxed. We must impose some constraints on the model parameters to obtain desirable properties. An obvious constraint is that $\alpha_0, \alpha_1 \geq 0$ because $\sigma^2$ is a variance.

The ARCH(1) models return as a white noise process with nonconstant conditional variance, and that conditional variance depends on the previous return.

The ARCH(1) model can be extended to the general **ARCH(p) model** in an obvious way, by changing the last component in the definition

$$
\sigma_t^2 = \alpha_0 + \alpha_1 r^2_{t-1}+ ... +\alpha_p r^2_{t-p}.
$$

Estimation of the parameters $\alpha_0,\alpha_1,...,\alpha_p$ of the ARCH(p) model is typically accomplished by conditional MLE. The conditional likelihood of the data $r_{p+1},...,r_n$ given $r_1,...,r_p$, is given by

$$
L(\mu,\alpha) = p(r_1,...,r_p \ | \ \mu,\alpha)= 􏱴 \prod_{p+1}^n f_{\alpha}(r_t \ | \ 􏱢􏱢r_{t−1},...,r_{t−p}),
$$
where under the assumption of normality, the conditional densities $f_\alpha(\cdot | \cdot)$ are, for t > p, given by
$$
r_t | 􏱢r_{t-1},...,r_{t-p} ∼ N(0,\alpha_0 +\alpha_1r_{t-1}^2 +···+\alpha_p r_{t-p}^2 ).
$$
Here the constraints are $\alpha_i ≥ 0, i = 1,2,...,p$ and  $(\alpha_1 + · · · + \alpha_p ) < 1$.

The **GARCH(1,1) model** is defined by changing the last component in the following way

$$
\sigma_t^2 = \alpha_0 + \alpha_1 r^2_{t-1} + \beta_1\sigma^2_{t-1} .
$$

*Reference*:

  - *Time series analysis and it's aplications*, fourth edition, Robert H. Shumway and David S. Stoffer (2016)

### Load libraries and Federal Reserve FX Data

```python
# Load packages

import pandas as pd
import numpy as np
import pandas_datareader.data as web
import datetime
import matplotlib.pyplot as plt
import seaborn as sns
import statsmodels.api as sm
from statsmodels.tsa.ar_model import AutoReg
from statsmodels.tsa.arima.model import ARIMA
import scipy.stats as scs
from arch.univariate import GARCH

plt.rc("figure", figsize=(12, 6))
plt.rc("font", family="sans-serif")
plt.rc("font", size=10)
plt.rc("lines", linewidth=1.5)
sns.set_style("darkgrid")

start = datetime.datetime(2005,10,15)
end = datetime.datetime(2015,10,15)

# Federal Reserve FX Data (FRED) from the St. Louis Federal Reserve
# U.S. Dollars to Euro Spot Exchange Rate

fx_useu = web.DataReader("DEXUSEU",'fred', start, end)
```

```python
fx_useu.plot();
```

```python
fx_useu.info()
```

    <class 'pandas.core.frame.DataFrame'>
    DatetimeIndex: 2609 entries, 2005-10-17 to 2015-10-15
    Data columns (total 1 columns):
     #   Column   Non-Null Count  Dtype
    ---  ------   --------------  -----
     0   DEXUSEU  2513 non-null   float64
    dtypes: float64(1)
    memory usage: 40.8 KB

```python
# Check number of missing values
print(fx_useu.isna().sum())
```

    DEXUSEU    96
    dtype: int64

```python
# Perform linear interpolation at missing data points
fx_useu = fx_useu.interpolate()
```

```python
print(fx_useu.isna().sum())
```

    DEXUSEU    0
    dtype: int64

```python
# Compute returns (log returns) of irregular daily time series
rets = np.log(fx_useu / fx_useu.shift(1))
rets.columns = ['Return']
rets = rets.dropna()
```

```python
rets.head().round(3)
```

      Return

      DATE

      2005-10-18
      -0.008

      2005-10-19
      0.004

      2005-10-20
      -0.001

      2005-10-21
      -0.002

      2005-10-24
      0.003

```python
# Plot return
rets.plot(ylabel='Return');
```

Next we present two plots:

- A histogram of the returns is created and the Gaussian/normal density corresponding to the maximum-likelihood fit is drawn.
This graph may appear to display a reasonable fit, but other diagnostics are useful to evaluate whether the distribution is heavier-tailed than a Gaussian distribution.
- A normal qq-plot of the returns is created. The sample of returns is sorted from smallest to largest:
$$
y_{[1]} ≤ y_{[2]} ≤ ··· ≤ y_{[n]}.
$$
Consider a sample of size n from a $N(0,1)$ distribution, $X_1,X_2,...,X_n$.
Define the order statistics as the sorted sample elements:
$$
X_{[1]} ≤X_{[2]} ≤···≤X_{[n]}.
$$
Define $x_1,...,x_n$ such that
$$
x_j = E[X_{[j]} | n], \ \text{the expected value conditional on $n$}.
$$
The qq-plot, plots the points $(x_j , y_j)$. If the sample $\{y_j\}$ is consistent with the Gaussian assumption, then the points will fall close to a straight line (with slope equal to the Gaussian standard deviation, and y intercept equal to the Gaussian mean).

```python
plt.figure(figsize=(10,4))
sns.histplot(rets['Return'], kde=True);
```

```python
# Normal QQ plot of returns
fig, ax =plt.subplots(figsize=(10,4))
sm.graphics.qqplot(rets['Return'], line='s', fit=True, ax=ax)
plt.title('US/EU');
```

Obviously, the sample qantil values do not lie on straight line indicating "nonnormality". On the left and right sides there are many values that lie well below and above the line, respectively. In other words time series exhibits fat tails.

### Geometric Brownian Motion Model

The geometric Brownian motion model for $\{S(t), t\geq 0\}$ is defined by Itô stochastic differential equation,

$$
dS(t) = S(t)[\mu dt+\sigma dB(t)], \ \ S(0)>0.
$$

The solution of this equation satisfies

$$
S(t)=S(0)\exp\Big[(\mu-\frac{\sigma^2}{2})t+\sigma B(t)\Big],
$$

from which it follows at once that $X(t) = log(S(t))$ satisfies

$$
X(t) = X(0) + (\mu-\frac{\sigma^2}{2})t+\sigma B(t),
$$

or equivalently

$$
dX(t) = (\mu-\frac{\sigma^2}{2})dt+\sigma dB(t).
$$

This implies that

gen_path() generates sample Monte Carlo paths for Geometric Brownein Motion.

```python
# This code credit to "Python for Finance: Mastering Data-Driven Finance" by Yves Hilpisch

def gen_paths(S0, r, sigma, T, M, I):
    '''Generate Monte carlo paths for Geometric Brownein Motion.

    Parameters
    ==========
    S0: float
        initial/index level
    r: float
        constant short rate
    sigma: float
        constant vtality
    T: float
        final time horizon
    M: int
        number of time steps
    I: number of path to be stimulated

    Returns
    =======
    paths: ndarrays, shape (M+1,I)
        stimulated paths given the parameters
    '''

    dt = T/M
    paths = np.zeros((M + 1,I))
    paths[0] = S0
    for t in range(1, M + 1):
        rand = np.random.standard_normal(I)
        rand = (rand-rand.mean())/rand.std()
        paths[t] = paths[t - 1]*np.exp((r - 0.5*sigma**2)*dt + sigma*np.sqrt(dt)*rand)
    return paths
```

```python
S0 = 100.
r = 0.05
sigma = 0.2
T = 1.0
M = 50
I = 250000
np.random.seed(1000)
```

```python
paths = gen_paths(S0, r, sigma, T, M, I)
```

```python
S0*np.exp(r*T)
```

    105.12710963760242

```python
paths[-1].mean()
```

    105.12645392478755

```python
paths[:,0].round(4)
```

    array([100.    ,  97.821 ,  98.5573, 106.1546, 105.899 ,  99.8363,
           100.0145, 102.6589, 105.6643, 107.1107, 108.7943, 108.2449,
           106.4105, 101.0575, 102.0197, 102.6052, 109.6419, 109.5725,
           112.9766, 113.0225, 112.5476, 114.5585, 109.942 , 112.6271,
           112.7502, 116.3453, 115.0443, 113.9586, 115.8831, 117.3705,
           117.9185, 110.5539, 109.9687, 104.9957, 108.0679, 105.7822,
           105.1585, 104.3304, 108.4387, 105.5963, 108.866 , 108.3284,
           107.0077, 106.0034, 104.3964, 101.0637,  98.3776,  97.135 ,
            95.4254,  96.4271,  96.3386])

```python
# Ten stimulation paths for Geometric Brownian Motion
plt.figure()
plt.plot(paths[:,:10])
plt.xlabel('time steps')
plt.ylabel('index level');
```

```python
log_returns = np.log(paths[1:]/paths[:-1])
```

```python
fig, ax = plt.subplots(1,2, figsize=(16, 4))
ax[0].hist(log_returns.flatten(), bins=70, density=True, label='frequency', color='b');
ax[0].set_xlabel('log return')
ax[0].set_ylabel('frequency')
ax[0].set_title('Histogram of log returns of GBM and normal density function')
x = np.linspace(ax[0].axis()[0], ax[0].axis()[1])
ax[0].plot(x, scs.norm.pdf(x, loc=r/M, scale=sigma/np.sqrt(M)), 'r', lw=2.0, label='pdf')
ax[0].legend()
sm.graphics.qqplot(log_returns.flatten()[::500], line='s', ax=ax[1])
ax[1].set_title('QQ plot for log returns of geometric Brownian motion');
```

### Time Dependence in Squared-Returns

Nonlinear time dependence in the time series of exchange rate returns is exhibited with the time dependence of the squared returns.
The autocorrelation function (ACF) and the partial autocorrelation function (PACF) are computed for the exchange rate returns and for their squared values. Marginally significant time dependence is present in the returns, while highly significant time dependence is apparent in the squared returns. (The blue lines in the plots are at +/- two standard deviations for the sample correlation coefficients under the null hypothesis of no time-series dependence.)

```python
fig, ax = plt.subplots(1,2, figsize=(14, 4))
sm.graphics.tsa.plot_acf(rets.values.squeeze(), title ='returns', ax=ax[0])
ax[0].set_xlabel('Lag')
ax[0].set_ylabel('ACF')
sm.graphics.tsa.plot_pacf(rets.values.squeeze(), title ='returns', ax=ax[1])
ax[1].set_xlabel('Lag')
ax[1].set_ylabel('Partial ACF')
plt.show()
```

```python
fig, ax = plt.subplots(1,2, figsize=(14, 4))
sm.graphics.tsa.plot_acf(rets.values.squeeze()**2, title ='returns^2', ax=ax[0])
ax[0].set_xlabel('Lag')
ax[0].set_ylabel('ACF')
sm.graphics.tsa.plot_pacf(rets.values.squeeze()**2, title ='returns^2', ax=ax[1])
ax[1].set_xlabel('Lag')
ax[1].set_ylabel('Partial ACF')
plt.show()
```

### Gaussian ARCH and GARCH Models

The following models are fit the EUR/USD exchange rate:

- ARCH(1)
- ARCH(2)
- ARCH(10)
- GARCH(1,1)

The python function arch_model() is fits both ARCH(p) and GARCH(p,q) models by maximum likelihood, assuming Gaussian distributions for the model innovations.

```python
#import sys
#!{sys.executable} -m pip install --user arch
```

```python
from arch import arch_model
import warnings
```

```python
warnings.filterwarnings("ignore")
am = arch_model(100*rets['Return'], p=1, q=0)
res1 = am.fit(update_freq=5)
print(res1.summary())
```

    Iteration:      5,   Func. Count:     38,   Neg. LLF: 2191.6941027200633
    Iteration:     10,   Func. Count:     78,   Neg. LLF: 2183.202659087334
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 2183.086474157681
                Iterations: 14
                Function evaluations: 102
                Gradient evaluations: 14
                         Constant Mean - GARCH Model Results
    ==============================================================================
    Dep. Variable:                 Return   R-squared:                       0.000
    Mean Model:             Constant Mean   Adj. R-squared:                  0.000
    Vol Model:                      GARCH   Log-Likelihood:               -2183.09
    Distribution:                  Normal   AIC:                           4374.17
    Method:            Maximum Likelihood   BIC:                           4397.64
                                            No. Observations:                 2608
    Date:                Mon, Nov 22 2021   Df Residuals:                     2607
    Time:                        12:01:36   Df Model:                            1
                                      Mean Model
    =============================================================================
                     coef    std err          t      P>|t|       95.0% Conf. Int.
    -----------------------------------------------------------------------------
    mu         4.8766e-03  9.866e-03      0.494      0.621 [-1.446e-02,2.421e-02]
                                   Volatility Model
    =============================================================================
                     coef    std err          t      P>|t|       95.0% Conf. Int.
    -----------------------------------------------------------------------------
    omega      1.0609e-03  7.999e-04      1.326      0.185 [-5.068e-04,2.629e-03]
    alpha[1]       0.0325  4.837e-03      6.724  1.764e-11  [2.304e-02,4.200e-02]
    beta[1]        0.9650  5.294e-03    182.286      0.000      [  0.955,  0.975]
    =============================================================================

    Covariance estimator: robust

plot() can be used to quickly visualize the standardized residuals and conditional volatility.

```python
fig = res1.plot(annualize="D")
```

```python
am = arch_model(100*rets['Return'], p=2, q=0)
res2 = am.fit(update_freq=5)
print(res2.summary())
```

    Iteration:      5,   Func. Count:     42,   Neg. LLF: 2374.512902110685
    Iteration:     10,   Func. Count:     73,   Neg. LLF: 2371.830066102596
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 2371.8300661025714
                Iterations: 10
                Function evaluations: 73
                Gradient evaluations: 10
                          Constant Mean - ARCH Model Results
    ==============================================================================
    Dep. Variable:                 Return   R-squared:                       0.000
    Mean Model:             Constant Mean   Adj. R-squared:                  0.000
    Vol Model:                       ARCH   Log-Likelihood:               -2371.83
    Distribution:                  Normal   AIC:                           4751.66
    Method:            Maximum Likelihood   BIC:                           4775.13
                                            No. Observations:                 2608
    Date:                Sun, Nov 21 2021   Df Residuals:                     2607
    Time:                        18:29:34   Df Model:                            1
                                      Mean Model
    =============================================================================
                     coef    std err          t      P>|t|       95.0% Conf. Int.
    -----------------------------------------------------------------------------
    mu         3.9064e-03  1.164e-02      0.336      0.737 [-1.890e-02,2.671e-02]
                                 Volatility Model
    ==========================================================================
                     coef    std err          t      P>|t|    95.0% Conf. Int.
    --------------------------------------------------------------------------
    omega          0.2799  1.656e-02     16.905  4.161e-64   [  0.247,  0.312]
    alpha[1]       0.1056  4.295e-02      2.460  1.391e-02 [2.146e-02,  0.190]
    alpha[2]       0.1580  3.275e-02      4.826  1.396e-06 [9.384e-02,  0.222]
    ==========================================================================

    Covariance estimator: robust

```python
am = arch_model(100*rets['Return'], p=10, q=0)
res10 = am.fit(update_freq=5)
print(res10.summary())
```

    Iteration:      5,   Func. Count:     80,   Neg. LLF: 2303.4474151105883
    Iteration:     10,   Func. Count:    155,   Neg. LLF: 2259.1464364755147
    Iteration:     15,   Func. Count:    233,   Neg. LLF: 2251.63518588464
    Iteration:     20,   Func. Count:    312,   Neg. LLF: 2250.616887357195
    Iteration:     25,   Func. Count:    383,   Neg. LLF: 2250.557696484909
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 2250.5576964850393
                Iterations: 25
                Function evaluations: 383
                Gradient evaluations: 25
                          Constant Mean - ARCH Model Results
    ==============================================================================
    Dep. Variable:                 Return   R-squared:                       0.000
    Mean Model:             Constant Mean   Adj. R-squared:                  0.000
    Vol Model:                       ARCH   Log-Likelihood:               -2250.56
    Distribution:                  Normal   AIC:                           4525.12
    Method:            Maximum Likelihood   BIC:                           4595.51
                                            No. Observations:                 2608
    Date:                Sun, Nov 21 2021   Df Residuals:                     2607
    Time:                        18:29:34   Df Model:                            1
                                      Mean Model
    =============================================================================
                     coef    std err          t      P>|t|       95.0% Conf. Int.
    -----------------------------------------------------------------------------
    mu         6.8632e-05  1.100e-02  6.237e-03      0.995 [-2.150e-02,2.164e-02]
                                   Volatility Model
    =============================================================================
                     coef    std err          t      P>|t|       95.0% Conf. Int.
    -----------------------------------------------------------------------------
    omega          0.1055  1.700e-02      6.207  5.405e-10    [7.221e-02,  0.139]
    alpha[1]       0.0381  2.789e-02      1.367      0.172 [-1.654e-02,9.279e-02]
    alpha[2]       0.0716  2.639e-02      2.715  6.637e-03    [1.992e-02,  0.123]
    alpha[3]       0.0940  3.455e-02      2.720  6.528e-03    [2.626e-02,  0.162]
    alpha[4]       0.0624  3.132e-02      1.993  4.628e-02    [1.029e-03,  0.124]
    alpha[5]       0.1114  3.311e-02      3.366  7.622e-04    [4.656e-02,  0.176]
    alpha[6]       0.1405  3.234e-02      4.344  1.396e-05    [7.712e-02,  0.204]
    alpha[7]       0.0670  3.957e-02      1.693  9.041e-02   [-1.056e-02,  0.145]
    alpha[8]       0.0223  2.758e-02      0.807      0.420 [-3.180e-02,7.630e-02]
    alpha[9]       0.0754  3.327e-02      2.267  2.340e-02    [1.021e-02,  0.141]
    alpha[10]      0.0749  2.467e-02      3.034  2.412e-03    [2.651e-02,  0.123]
    =============================================================================

    Covariance estimator: robust

```python
am = arch_model(100*rets['Return'], p=1, q=1)
res11 = am.fit(update_freq=5)
print(res11.summary())
```

    Iteration:      5,   Func. Count:     38,   Neg. LLF: 2191.6941027200633
    Iteration:     10,   Func. Count:     78,   Neg. LLF: 2183.202659087334
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 2183.086474157681
                Iterations: 14
                Function evaluations: 102
                Gradient evaluations: 14
                         Constant Mean - GARCH Model Results
    ==============================================================================
    Dep. Variable:                 Return   R-squared:                       0.000
    Mean Model:             Constant Mean   Adj. R-squared:                  0.000
    Vol Model:                      GARCH   Log-Likelihood:               -2183.09
    Distribution:                  Normal   AIC:                           4374.17
    Method:            Maximum Likelihood   BIC:                           4397.64
                                            No. Observations:                 2608
    Date:                Sun, Nov 21 2021   Df Residuals:                     2607
    Time:                        18:29:34   Df Model:                            1
                                      Mean Model
    =============================================================================
                     coef    std err          t      P>|t|       95.0% Conf. Int.
    -----------------------------------------------------------------------------
    mu         4.8766e-03  9.866e-03      0.494      0.621 [-1.446e-02,2.421e-02]
                                   Volatility Model
    =============================================================================
                     coef    std err          t      P>|t|       95.0% Conf. Int.
    -----------------------------------------------------------------------------
    omega      1.0609e-03  7.999e-04      1.326      0.185 [-5.068e-04,2.629e-03]
    alpha[1]       0.0325  4.837e-03      6.724  1.764e-11  [2.304e-02,4.200e-02]
    beta[1]        0.9650  5.294e-03    182.286      0.000      [  0.955,  0.975]
    =============================================================================

    Covariance estimator: robust

```python
df = pd.concat([.01*res1.conditional_volatility,
                .01*res2.conditional_volatility,
                .01*res10.conditional_volatility,
                .01*res11.conditional_volatility],axis=1)
df.columns = ['GARCH(1)', 'GARCH(2)', 'GARCH(10)', 'GARCH(1,1)']
subplot = df.plot();
#subplot.set_xlim(xlim)
```

```python

```

The estimated mean is $\hat{\mu} = 0.0049$ and the minimum-AICC GARCH model (with Gaussian noise) for the residuals,  is found to be the GARCH(1,1) with estimated parameter values $\hat{\alpha}_0 = 0.0010, \hat{\alpha}_1 = 0.0325, \hat{\beta}_1 = 0.9650$.

Note:
- The ARCH models have a hard lower bound ($\hat{α}_0$) which gets lower with
higher-order p values.
- The GARCH(1,1) model provides an extremely parsimonious model compared to that of the ARCH(10) model.
- The GARCH(1,1) model is quite smooth when compared to every ARCH
model.
- The GARCH(1,1) model is very close to being non-stationary
$$
\hat{\alpha}_1 + \hat{\beta}_1 = 0.9975
$$
This near-non stationarity is consistent with there being no long-term mean volatility. Instead, the volatility evolves slowly over time (i.e., with high value $\beta_1$) with no tendency to revert toward any specific mean volatility level.

### AR(1)-GARCH(1,1) Models with t Distributions

In this section, we fit AR(1)-GARCH(1,1) model with t distributions for the innovations. This model explicitly incorporates non-linear dependence in the residuals (i.e., volatility) and provides a specific distribution alternative to the Gaussian with excess kurtosis (i.e., heavier tails).

```python
am = arch_model(rets['Return'], mean='AR', lags=1, p=1, q=1, dist='StudentsT')
res11 = am.fit(update_freq=5)
print(res11.summary())
```

    Iteration:      5,   Func. Count:     62,   Neg. LLF: -4044.097391968343
    Iteration:     10,   Func. Count:    133,   Neg. LLF: 41834.76688127982
    Iteration:     15,   Func. Count:    183,   Neg. LLF: 15256.291147191834
    Iteration:     20,   Func. Count:    239,   Neg. LLF: 25101.459066206015
    Iteration:     25,   Func. Count:    283,   Neg. LLF: 25066.252509233804
    Iteration:     30,   Func. Count:    323,   Neg. LLF: 22649.57451713406
    Optimization terminated successfully.    (Exit mode 0)
                Current function value: 10948.938144351581
                Iterations: 36
                Function evaluations: 341
                Gradient evaluations: 32
                                  AR - GARCH Model Results
    ====================================================================================
    Dep. Variable:                       Return   R-squared:                 -502145.785
    Mean Model:                              AR   Adj. R-squared:            -502338.548
    Vol Model:                            GARCH   Log-Likelihood:               -10948.9
    Distribution:      Standardized Student's t   AIC:                           21909.9
    Method:                  Maximum Likelihood   BIC:                           21945.1
                                                  No. Observations:                 2607
    Date:                      Mon, Nov 22 2021   Df Residuals:                     2605
    Time:                              14:06:32   Df Model:                            2
                                     Mean Model
    ============================================================================
                     coef    std err          t      P>|t|      95.0% Conf. Int.
    ----------------------------------------------------------------------------
    Const          0.7666  9.702e-02      7.901  2.762e-15     [  0.576,  0.957]
    Return[1]    697.8144     23.753     29.377 1.069e-189 [6.513e+02,7.444e+02]
                                   Volatility Model
    =============================================================================
                     coef    std err          t      P>|t|       95.0% Conf. Int.
    -----------------------------------------------------------------------------
    omega      1.7089e-09  1.910e-06  8.949e-04      0.999 [-3.741e-06,3.744e-06]
    alpha[1]       0.9714  3.162e-02     30.720 3.063e-207      [  0.909,  1.033]
    beta[1]        0.0252  2.417e-03     10.430  1.817e-25  [2.047e-02,2.994e-02]
                                  Distribution
    ========================================================================
                     coef    std err          t      P>|t|  95.0% Conf. Int.
    ------------------------------------------------------------------------
    nu            11.2058      0.680     16.482  4.903e-61 [  9.873, 12.538]
    ========================================================================

    Covariance estimator: robust

```python
fig, ax =plt.subplots(figsize=(10,4))
sm.graphics.qqplot(res11.resid[1:], line='s', fit=True, ax=ax)
plt.title('Q−Q Plot of Standardized Residuals AR(1)−GARCH(1,1) With t−Dist.');
```

```python
fig, ax = plt.subplots(1,2, figsize=(14, 4))
sm.graphics.tsa.plot_acf(res11.resid[1:]**2, title='residuals^2', ax=ax[0])
ax[0].set_xlabel('Lag')
ax[0].set_ylabel('ACF')
sm.graphics.tsa.plot_pacf(res11.resid[1:]**2, title='residuals^2', ax=ax[1])
ax[1].set_xlabel('Lag')
ax[1].set_ylabel('Partial ACF')
plt.show()
```

Looks like we have achieved a good model fit as there is no obvious autocorrelation in the squared residuals.


```python
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
from matplotlib import cm
import numpy as np
import pandas as pd
from sklearn.datasets import load_iris
from sklearn.linear_model import LogisticRegression

```

##MLE como base para la regresion logística

Considere el problema de MLE (Estimacion de máxima veresimilitud):

\begin{align}
\max \sum_{i=1} \log P(y_i|x_i;h)
\end{align}

Donde:

$y_i$ es la clase o categoría de cada elemento y $x_i$ son los atributos de cada elemento, donde además cada elemento de dataset satisface una distribución de bernulli:

\begin{align}
        \text{P}(y) = \left\{
        \begin{array}{cl}
        p & si\,\,\,\,y = 1, \\
        1 - p & si\,\,\,\,y = 0.
        \end{array}
        \right.
\end{align}

En este caso la veresimilitud esta dada por:

\begin{align}
L = \hat{y}y + (1 - \hat{y})(1 - y)
\end{align}

Esta funcion da probabilidades altas cuando $\hat{y}$ ~ $y$

```python
def likelihood(y, yp):
  return yp*y + (1 - yp) * (1 - y)
```

```python
fig = plt.figure()
ax = fig.gca(projection = '3d')

Y = np.arange(0.0, 1.0, 0.01)
YP = np.arange(0.0, 1.0, 0.01)
Y, YP = np.meshgrid(Y, YP)
Z = likelihood(Y, YP)

surf = ax.plot_surface(Y, YP, Z, cmap = cm.coolwarm)
fig.colorbar(surf, shrink=0.5, aspect=5)
plt.show()
```

Considerando $p \rightarrow \log(p)$, y sumando la verosimilitud para todos los puntos del dataset obtenemos:

$$
max{\sum_i \left(y \log \hat{y} + (1-y) \log (1-\hat{y}) \right)} \\
= \min - \sum_i \left( y \log \hat{y} + (1-y) \log (1-\hat{y}) \right)
$$

que es la conocida función de costo para clasificación conocida como Cross-entropy.

##Regresión logística con Scikit-learn

Recordemos que:
\begin{align}
\hat{y}= \frac{1}{1 + e^\left(-\log-odds \right)}
\end{align}

donde $\log-odds = Β_0 + Β_{1x1} + Β_{nxn}$ y los betas son los parámetros del modelo.

Aplicaremos un ejercicio de clasificación simple con el dataset Iris:

* Flores iris [wikipedia](https://en.wikipedia.org/wiki/Iris_flower_data_set)

* Dataset flores iris [scikit](https://scikit-learn.org/stable/auto_examples/datasets/plot_iris_dataset.html)

```python
atrib_names = ['sepal length', 'sepal width', 'petal length', 'petal width'] #Atributos
X, y =load_iris(return_X_y=True)
```

```python
X[:2]
y[:100]

```

    array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
           0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
           0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
           1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
           1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])

```python
clf = LogisticRegression(random_state = 10, solver = 'liblinear').fit(X[:100], y[:100])#Generados aleatorio, metodo ooptimiaador del modelo
```

Revisamos los parámetros resultantes $Β_i: $

```python
clf.coef_
```

    array([[-0.40247392, -1.46382925,  2.23785648,  1.00009294]])


Author: Suman Shekhar

```python
import sympy as smp
from sympy import *
from sympy.interactive import printing
printing.init_printing(use_latex = True)
from sympy import Eq, solve_linear_system, Matrix
from numpy import linalg
import numpy as np

```

```python
x, y, c = smp.symbols('x y c')
```

# Limits

$$\lim_{x \to \pi} \sin(x/2+ \sin(x))$$

```python
smp.limit(smp.sin(x/2 + smp.sin(x)), x, smp.pi)
```

$\displaystyle 1$

$$ \lim_{x \to 0^+} \frac{2e^{1/x}}{e^{1/x}+1} $$

```python
smp.limit(2*smp.exp(1/x) / (smp.exp(1/x)+1), x, 0, dir='+')
```

$\displaystyle 2$

$$ \lim_{x \to 0^-} \frac{2e^{1/x}}{e^{1/x}+1} $$

```python
smp.limit(2*smp.exp(1/x) / (smp.exp(1/x)+1), x, 0, dir='-')
```

$\displaystyle 0$

$$ \lim_{x \to \infty} \frac{\cos(x)-1}{x} $$

```python
smp.limit((smp.cos(x) - 1)/x, x, smp.oo)
```

$\displaystyle 0$

# Derivatives

$$ \frac{d}{dx} \left( \frac{1+\sin x}{1 - \cos x} \right)^2 $$

```python
smp.diff(((1 + smp.sin(x)) / (1 - smp.cos(x)))**2, x)
```

$\displaystyle \frac{2 \left(\sin{\left(x \right)} + 1\right) \cos{\left(x \right)}}{\left(1 - \cos{\left(x \right)}\right)^{2}} - \frac{2 \left(\sin{\left(x \right)} + 1\right)^{2} \sin{\left(x \right)}}{\left(1 - \cos{\left(x \right)}\right)^{3}}$

$$ \frac{d}{dx} (\log_5 (x))^{x/2}$$

```python
smp.diff(smp.log(x, 5)**(x/2), x)
```

$\displaystyle \left(\frac{\log{\left(x \right)}}{\log{\left(5 \right)}}\right)^{\frac{x}{2}} \left(\frac{\log{\left(\frac{\log{\left(x \right)}}{\log{\left(5 \right)}} \right)}}{2} + \frac{1}{2 \log{\left(x \right)}}\right)$

$$\frac{d}{dx} f(x+g(x)) $$

```python
f, g = smp.symbols('f g', cls=smp.Function)
g = g(x)
f = f(x+g)
```

```python
smp.diff(f,x)
```

$\displaystyle \left(\frac{d}{d x} g{\left(x \right)} + 1\right) \left. \frac{d}{d \xi_{1}} f{\left(\xi_{1} \right)} \right|_{\substack{ \xi_{1}=x + g{\left(x \right)} }}$

# Basic Antiderivatives

$$ \int \csc(x) \cot (x)  dx $$

```python
smp.integrate(smp.csc(x)*smp.cot(x), x)
```

$\displaystyle - \frac{1}{\sin{\left(x \right)}}$

$$ \int 4 \sec (3x) \tan(3x)dx $$

```python
smp.integrate(4*smp.sec(3*x)*smp.tan(3*x), x)
```

$\displaystyle \frac{4}{3 \cos{\left(3 x \right)}}$

$$ \int \left( \frac{2}{\sqrt{1-x^2}} - \frac{1}{x^{1/4}}\right) dx$$

```python
smp.integrate(2/smp.sqrt(1-x**2) - 1/x**smp.Rational(1,4), x)
```

$\displaystyle - \frac{4 x^{\frac{3}{4}}}{3} + 2 \operatorname{asin}{\left(x \right)}$

# Initial Value Problems

1. Given $\frac{dy}{dx} = 8x + \csc^2(x)$ with $y(\pi/2) = -7$  solve for $y(x)$

```python
integral = smp.integrate(8*x + smp.csc(x)**2, x)
```

```python
C = - integral.subs(x, smp.pi/2) - 7
y = integral + C
```

```python
y.subs(x, smp.pi/2)
```

$\displaystyle -7$

```python
y
```

$\displaystyle 4 x^{2} - \pi^{2} - 7 - \frac{\cos{\left(x \right)}}{\sin{\left(x \right)}}$

# More Complicated Expressions

 $$\int \frac{(1+\sqrt{x})^{1/3}}{\sqrt{x}} dx$$

```python
smp.integrate((1+smp.sqrt(x))**smp.Rational(1,3) / smp.sqrt(x), x)
```

$\displaystyle \frac{3 \sqrt{x} \sqrt[3]{\sqrt{x} + 1}}{2} + \frac{3 \sqrt[3]{\sqrt{x} + 1}}{2}$

$$\int x (1-x^2)^{1/4} dx$$

```python
smp.integrate(x*(1-x**2)**smp.Rational(1/4), x)
```

$\displaystyle \frac{2 x^{2} \sqrt[4]{1 - x^{2}}}{5} - \frac{2 \sqrt[4]{1 - x^{2}}}{5}$

$$ \int \frac{(2x-1)\cos(\sqrt{3(2x-1)^2+6})}{\sqrt{3(2x-1)^2+6}} dx $$

```python
smp.integrate((2*x -1)*smp.cos(smp.sqrt(3*(2*x-1)**2 + 6)) / smp.sqrt(3*(2*x-1)**2 + 6), x)
```

$\displaystyle \frac{\sin{\left(\sqrt{3 \left(2 x - 1\right)^{2} + 6} \right)}}{6}$

# Definite Integrals

$$\int_{0}^{\ln(4)}\frac{ce^x dt}{\sqrt{e^{2x}+9}} $$

```python
smp.integrate(smp.exp(x) / smp.sqrt(smp.exp(2*x) + 9), (x, 0, smp.log(4)))
```

$\displaystyle - \operatorname{asinh}{\left(\frac{1}{3} \right)} + \operatorname{asinh}{\left(\frac{4}{3} \right)}$

```python
l= smp.symbols('l')
a = (x**3 - 3*(L**2)*x)
smp.integrate(smp.exp(x) / smp.sqrt(smp.exp(2*x) + 9), (x, 0, smp.log(4)))
```

$$\int_1^t x^{10} e^x dx $$

```python
l= smp.symbols('l')
C2 = smp.symbols('C2')
C3 = smp.symbols('C3')
m = smp.symbols('m')
W2 = (x**3 - 3*(L**2)*x)
W1 = (x**2 - 2*l*x)
Rd = (2*C2 + 6*C3*x) - m+( 270 + C2*W1 + C3*W2)

smp.integrate( W1*Rd, (x,0,0.001))
```

$\displaystyle 1.33333333333333 \cdot 10^{-9} C_{2} l^{2} - 2.000001 \cdot 10^{-6} C_{2} l + 6.66666866666667 \cdot 10^{-10} C_{2} - 3.9999984 \cdot 10^{-9} C_{3} l + 1.49999941666667 \cdot 10^{-12} C_{3} + 1.0 \cdot 10^{-6} l m - 0.00027 l - 3.33333333333333 \cdot 10^{-10} m + 9.0 \cdot 10^{-8}$

```python
smp.integrate( W2*Rd, (x,0,0.001))
```

$\displaystyle 1.6 \cdot 10^{-15} C_{2} l - 2.50000058333333 \cdot 10^{-12} C_{2} - 4.79999805714286 \cdot 10^{-15} C_{3} + 1.25 \cdot 10^{-12} m - 3.375 \cdot 10^{-10}$

```python

```

```python
smp.integrate(x**10*smp.exp(x), (x, 1, t))
```

$\displaystyle \left(t^{10} - 10 t^{9} + 90 t^{8} - 720 t^{7} + 5040 t^{6} - 30240 t^{5} + 151200 t^{4} - 604800 t^{3} + 1814400 t^{2} - 3628800 t + 3628800\right) e^{t} - 1334961 e$

# Improper Integrals

$$ \int_{0}^{\infty} \frac{16 \tan^{-1}(x)}{1+x^2} dx $$

```python
smp.integrate(16*smp.atan(x) / (1+x**2), (x, 0, smp.oo))
```

$\displaystyle 2 \pi^{2}$

# Sequences and Series

```python
n = smp.symbols('n')
```

$$ \sum_{n=0}^\infty \frac{6}{4^n} $$

```python
smp.Sum(6/4**n, (n,0,smp.oo)).doit()
```

$\displaystyle 8$

$$ \sum_{n=0}^\infty \frac{2^{n+1}}{5^n} $$

```python
smp.Sum(2**(n+1) / 5**n, (n,0,smp.oo)).doit()
```

$\displaystyle \frac{10}{3}$

$$ \sum_{n=1}^{\infty} \frac{\tan^{-1}(n)}{n^{1.1}} $$

```python
smp.Sum(smp.atan(n)/n**smp.Rational(11,10), (n,1,smp.oo)).doit()
```

$\displaystyle \sum_{n=1}^{\infty} \frac{\operatorname{atan}{\left(n \right)}}{n^{\frac{11}{10}}}$

$$ \sum_{n=1}^\infty \frac{1 + \cos(n)}{n^2}$$

```python
smp.Sum((1+smp.cos(n))/n**2, (n,1,smp.oo)).n()
```

$\displaystyle 1.969$

```python
m = 400
l= 1

```

```python
2*((l**3)/3 - l**3)
```

    -1.3333333333333335

```python
m*((l**5)/5-3*(l**5)/2 + 4*(l**5)/3)
```

    13.333333333333286

```python
(6/4)*(l**4) -m*((l**6)/6 -3/4*(l**6)) -2*l*(2*l**3 -m*(-4/5*(l**5)))
```

    -409.16666666666663

```python
((l**3)/3-l**3)*270*m
```

    -72000.00000000001

```python

```


# Markov-Chain Monte Carlo (MCMC)

## Lotka-Volterra Model

$u:$ Prey

$v:$ Predator

\begin{equation}
\begin{split}
\frac{du}{dt} =&  \alpha\cdot u-\beta \cdot v\cdot u \\
\frac{dv}{dt} =& -\gamma \cdot  v + \delta \cdot u\cdot v
\end{split}
\end{equation}

```python
import argparse
import os

import matplotlib
import matplotlib.pyplot as plt

from jax.experimental.ode import odeint
import jax.numpy as jnp
from jax.random import PRNGKey

import numpyro
import numpyro.distributions as dist
from numpyro.examples.datasets import LYNXHARE, load_dataset
from numpyro.infer import MCMC, NUTS, Predictive

```

```python

def dz_dt(z, t, theta):
    """
    Lotka–Volterra equations. Real positive parameters `alpha`, `beta`, `gamma`, `delta`
    describes the interaction of two species.
    """
    u = z[0]
    v = z[1]
    alpha, beta, gamma, delta = theta[..., 0], theta[..., 1], theta[..., 2], theta[..., 3]
    du_dt = (alpha - beta * v) * u
    dv_dt = (-gamma + delta * u) * v

    return jnp.stack([du_dt, dv_dt])

def model(N, y=None):
    """
    :param int N: number of measurement times
    :param numpy.ndarray y: measured populations with shape (N, 2)
    """
    # initial population
    z_init = numpyro.sample("z_init", dist.LogNormal(jnp.log(10), 1).expand([2]))
    # measurement times
    ts = jnp.arange(float(N))
    # parameters alpha, beta, gamma, delta of dz_dt
    theta = numpyro.sample(
        "theta",
        dist.TruncatedNormal(low=0., loc=jnp.array([1.0, 0.05, 1.0, 0.05]),
                             scale=jnp.array([0.5, 0.05, 0.5, 0.05])))
    # integrate dz/dt, the result will have shape N x 2
    z = odeint(dz_dt, z_init, ts, theta, rtol=1e-6, atol=1e-5, mxstep=1000)
    # measurement errors
    sigma = numpyro.sample("sigma", dist.LogNormal(-1, 1).expand([2]))
    # measured populations
    numpyro.sample("y", dist.LogNormal(jnp.log(z), sigma), obs=y)

```

```python
device = 'cpu'

num_samples = 1000
num_chains = 2
num_warmup = 200

numpyro.set_platform(device)
numpyro.set_host_device_count(num_chains)

_, fetch = load_dataset(LYNXHARE, shuffle=False)
year, data = fetch()  # data is in hare -> lynx order

# use dense_mass for better mixing rate
mcmc = MCMC(NUTS(model, dense_mass=True),
            num_warmup, num_samples, num_chains=num_chains, progress_bar= True)
mcmc.run(PRNGKey(1), N=data.shape[0], y=jnp.log(data))
mcmc.print_summary()

```

                    mean       std    median      5.0%     95.0%     n_eff     r_hat
      sigma[0]      0.68      0.24      0.74      0.41      0.92      1.01     11.38
      sigma[1]      0.31      0.09      0.34      0.20      0.40      1.01      9.82
      theta[0]      0.64      0.06      0.67      0.52      0.69      2.78      1.21
      theta[1]      3.61      3.41      3.67      0.18      7.02      1.00    233.99
      theta[2]      2.38      1.68      2.55      0.63      4.06      1.00     35.01
      theta[3]      1.71      1.50      1.76      0.19      3.21      1.00     96.89
     z_init[0]      2.41      1.77      2.01      0.66      4.38      1.01     11.20
     z_init[1]      2.29      1.23      1.89      1.07      3.69      1.01      9.09

    Number of divergences: 0

```python
# predict populations
ax = plt.figure(figsize=(15.5, 7.2))
y_pred = Predictive(model, mcmc.get_samples())(PRNGKey(2), data.shape[0])["y"]
pop_pred = jnp.exp(y_pred)
mu, pi = jnp.mean(pop_pred, 0), jnp.percentile(pop_pred, (10, 90), 0)
plt.plot(year, data[:, 0], "ko", mfc="none", ms=4, label="true hare", alpha=0.67)
plt.plot(year, data[:, 1], "bx", label="true lynx")
plt.plot(year, mu[:, 0], "k-.", label="pred hare", lw=1, alpha=0.67)
plt.plot(year, mu[:, 1], "b--", label="pred lynx")
plt.fill_between(year, pi[0, :, 0], pi[1, :, 0], color="k", alpha=0.2)
plt.fill_between(year, pi[0, :, 1], pi[1, :, 1], color="b", alpha=0.3)
plt.gca().set(ylim=(0, 160), xlabel="year", ylabel="population (in thousands)")
plt.title("Posterior predictive (80% CI) with predator-prey pattern.")
plt.legend()
plt.tight_layout()
plt.show()

```

```python
params = mcmc.get_samples()['theta']

fig, ax = plt.subplots(4,2, figsize=(20, 40))

ax[0,0].plot(params[:num_samples,0]), ax[0,1].hist(params[:num_samples,0])
ax[1,0].plot(params[:num_samples,1]), ax[1,1].hist(params[:num_samples,1])
ax[2,0].plot(params[:num_samples,2]), ax[2,1].hist(params[:num_samples,2])
ax[3,0].plot(params[:num_samples,3]), ax[3,1].hist(params[:num_samples,3])

for i in range(4):
    ax[i,0].set_title('Trace plot')
    ax[i,1].set_title('Histogram')
    ax[i,0].set_xlabel('Chain Iter')
    ax[i,1].set_ylabel('Value')
    ax[i,1].set_ylabel('Freq')

plt.show()
```

```python

```