```julia
using DifferentialEquations
using Plots
using LaTeXStrings
pgfplotsx()
```

    Plots.PGFPlotsXBackend()

Consider the simple reaction:

\begin{align}
    A &\longrightarrow B\\
\end{align}

The reaction is elementary, and B acts as a solvent for A and the transition state. We will consider it in a few different solvent environments.

```julia
function batch(du, u, p, t)
    MAR = p["MAR"]
    PAR = p["PAR"]

    k10, K10, NS, V = PAR

    NA   = u[:,1]
    NB   = u[:,2]

    NT     = NA + NB .+ NS

    XA   = NA./NT
    XB   = NB./NT
    XS   = NS./NT

    #For A in solvent
    A12A = MAR[1]

    #For B in solvent
    A12B = MAR[2]

    #For Transition State 1 in solvent
    A12TS1 = MAR[3]

    gammaA   = exp.(XS.^2*A12A)
    gammaB   = exp.(XS.^2*A12B)
    gammaTS1 = exp.(XS.^2*A12TS1)

    z1 = 1/K10*gammaB./gammaA.*XB./XA

    r1 = k10*gammaA./gammaTS1.*XA.*(1 .- z1).*NT/V

    RA  = -r1[1]
    RB  =  r1[1]

    du[1] = RA*V
    du[2] = RB*V
    return du, r1, z1
end
```

    batch (generic function with 1 method)

```julia
k10  = 1
K10  = 1
V    = 1

NTOT = 100
NA0  = 0.1
NB0  = 0.0
NS   = 100 - NA0 - NB0

var0     = [NA0 NB0]
span     = (0.0, 2000.0);
```

```julia
#Solvate transition state relative to reactants
MARSET1 = zeros(3,3)
MARSET1[:,1] = [0.0, 0.0,  0.0] #no solvation
MARSET1[:,2] = [0.0, 0.0,  2.0] #destabilize TS1
MARSET1[:,3] = [0.0, 0.0, -2.0] #stabilize TS1

tfine  = 10 .^(range(-5.0, stop = log10(maximum(span)), length = 1000))
Xout   = zeros(length(tfine), size(MARSET1, 2))
r1out  = zeros(length(tfine), size(MARSET1, 2))
z1out  = zeros(length(tfine), size(MARSET1, 2))
NAout  = zeros(length(tfine), size(MARSET1, 2))
NBout  = zeros(length(tfine), size(MARSET1, 2))

for i = 1:size(MARSET1, 2)
    p0   = Dict("MAR" => MARSET1[:,i], "PAR" => [k10, K10, NS, V])
    prob = ODEProblem(batch, var0, span, p0)
    sol  = solve(prob, Rodas5(), abstol = 1e-10, reltol = 1e-10)
    solf = sol(tfine)
    NA   = solf[1,:]
    NB   = solf[2,:]
    NT   = NA + NB .+ NS
    ext  = NB/NA0
    dut, rt1, zt1 = batch([0., 0.], [NA NB], p0, tfine)

    Xout[:,i]  = ext
    r1out[:,i] = rt1
    z1out[:,i] = zt1
    NAout[:,i] = NA
    NBout[:,i] = NB
end
```

```julia
plt1 = plot(
    xscale  = :log10,
    xlim    = (1e-8, 1),
    xminorticks = 4,
    ylim = (0.0, 1.0),
    yticks = 0.0:0.25:1.0,
    #yticks = (-2.0:1.0:2.0, [@sprintf("%1.1f", x) for x in -2.0:1.0:2.0]),
    yminorticks = 4,
    tick_direction = :out,
    grid = :off,
    guidefontsize = 30,
    tickfontsize = 24,
    xlabel  = "time",
    ylabel  = "Conversion",
    fontfamily = "Times Roman",
    legendfontsize = 22,
    legendfonthalign = :left,
    legend  = :topleft,
    title = "(a)",
    titlelocation = :left,
    titlefontsize = 30
)

plt1 = plot!(plt1, tfine/maximum(tfine), Xout[:,3], ls = :dash,       lc = :blue,  lw = 1.00, label = "Stabilize TS")
plt1 = plot!(plt1, tfine/maximum(tfine), Xout[:,1], ls = :solid,      lc = :black, lw = 0.75, label = "Ideal Solution")
plt1 = plot!(plt1, tfine/maximum(tfine), Xout[:,2], ls = :dashdotdot, lc = :red,   lw = 1.00, label = "Destabilize TS")
display(plt1)
savefig(plt1, "CS1d1.pdf")
```

```julia
idx1 = r1out[:,1] .> 0
idx2 = r1out[:,2] .> 0
idx3 = r1out[:,3] .> 0

plt2 = plot(tfine[idx1]/maximum(tfine), r1out[:,1][idx1], xscale = :log10, yscale = :log10, ls = :solid, lc = :black, xlabel  = "time", ylabel = "rate", labels = "Ideal Solution", legend = :bottomleft, xlim = (1e-7, 1), ylim = (1e-5, 1), grid = :off)
plt2 = plot!(plt2, tfine[idx2]/maximum(tfine), r1out[:,2][idx2], ls = :dot, lc = :black, label = "Destabilize TS")
plt2 = plot!(plt2, tfine[idx3]/maximum(tfine), r1out[:,3][idx3], ls = :dashdotdot, lc = :black, label = "Stabilize TS")
display(plt2)
savefig(plt2, "CS1d2.pdf")
```

```julia
idx1 = r1out[:,1] .> 0
idx2 = r1out[:,2] .> 0
idx3 = r1out[:,3] .> 0

plt3 = plot(
    xlim    = (0.0, 0.5),
    xticks  = 0.0:0.1:0.5,
    xminorticks = 4,
    ylim = (0.0, 1.0),
    yticks = 0.0:0.25:1.0,
    yminorticks = 4,
    tick_direction = :out,
    grid = :off,
    guidefontsize = 30,
    tickfontsize = 24,
    xlabel  = "Conversion",
    ylabel  = "rate",
    fontfamily = "Times Roman",
    legendfontsize = 18,
    legendfonthalign = :left,
    legend  = nothing,
    title = "(b)",
    titlelocation = :left,
    titlefontsize = 30
)

plt3 = plot!(plt3, Xout[:,3][idx3], r1out[:,3][idx3], ls = :dash,       lc = :blue,  lw = 1.00, label = "Stabilize TS")
plt3 = plot!(plt3, Xout[:,1][idx1], r1out[:,1][idx1], ls = :solid,      lc = :black, lw = 0.75, label = "Ideal Solution")
plt3 = plot!(plt3, Xout[:,2][idx2], r1out[:,2][idx2], ls = :dashdotdot, lc = :red,   lw = 1.00, label = "Destabilize TS")

display(plt3)
savefig(plt3, "CS1d3.pdf")
```

```julia
idx1 = z1out[:,1] .< 1.0
idx2 = z1out[:,2] .< 1.0
idx3 = z1out[:,3] .< 1.0

plt4 = plot(tfine[idx1]/maximum(tfine), 1 .- z1out[:,1][idx1], xscale = :log10, ls = :solid, lc = :black, xlabel  = "time", ylabel = "z", label = "Ideal Solution", legend = :topright, grid = :off, minorticks = 4)
plt4 = plot!(plt4, tfine[idx2]/maximum(tfine), 1 .- z1out[:,2][idx2], ls = :dot, lc = :black, label = "Destabilize TS")
plt4 = plot!(plt4, tfine[idx3]/maximum(tfine), 1 .- z1out[:,3][idx3], ls = :dashdotdot, lc = :black, label = "Stabilize TS")
display(plt4)
savefig(plt4, "CS1d4.pdf")
```

```julia
plt5 = plot(
    xlim    = (0.0, 0.5),
    xticks  = 0.0:0.1:0.5,
    xminorticks = 4,
    ylim = (0.0, 1.0),
    yticks = 0.0:0.25:1.0,
    yminorticks = 4,
    tick_direction = :out,
    grid = :off,
    guidefontsize = 30,
    tickfontsize = 24,
    xlabel  = "Conversion",
    ylabel  = "(1 - z)",
    fontfamily = "Times Roman",
    legendfontsize = 18,
    legendfonthalign = :left,
    legend  = nothing,
    title = "(c)",
    titlelocation = :left,
    titlefontsize = 30
)

plt5 = plot!(plt5, Xout[:,2], 1 .- z1out[:,2], ls = :dashdotdot, lc = :red,   lw = 1.00, label = "Destabilize TS")
plt5 = plot!(plt5, Xout[:,1], 1 .- z1out[:,1], ls = :solid,      lc = :black, lw = 0.75, label = "Ideal Solution")
plt5 = plot!(plt5, Xout[:,3], 1 .- z1out[:,3], ls = :dash,       lc = :blue,  lw = 1.00, label = "Stabilize TS")
display(plt5)
savefig(plt5, "CS1d5.pdf")
```

```julia
#Solvate transition state relative to reactants
MARSET2 = zeros(3,3)
MARSET2[:,1] = [0.0, 0.0, 0.0] #no solvation
MARSET2[:,2] = [0.0, 2.0, 0.0] #destabilize TS1
MARSET2[:,3] = [0.0, -2.0, 0.0] #stabilize TS1

tfine  = 10 .^(range(-3.0, stop = log10(maximum(span)), length = 1000))
Xout   = zeros(length(tfine), size(MARSET2, 2))
r1out  = zeros(length(tfine), size(MARSET2, 2))
z1out  = zeros(length(tfine), size(MARSET2, 2))
NAout  = zeros(length(tfine), size(MARSET2, 2))
NBout  = zeros(length(tfine), size(MARSET2, 2))

for i = 1:size(MARSET2, 2)
    p0   = Dict("MAR" => MARSET2[:,i], "PAR" => [k10, K10, NS, V])
    prob = ODEProblem(batch, var0, span, p0)
    sol  = solve(prob, Rodas5(), abstol = 1e-18, reltol = 1e-18)
    solf = sol(tfine)
    NA   = solf[1,:]
    NB   = solf[2,:]
    NT   = NA + NB .+ NS
    ext  = NB/NA0
    dut, rt1, zt1 = batch([0., 0.], [NA NB], p0, tfine)

    Xout[:,i]  = ext
    r1out[:,i] = rt1
    z1out[:,i] = zt1
    NAout[:,i] = NA
    NBout[:,i] = NB
end

```

```julia
plt6 = plot(
    xscale  = :log10,
    xlim    = (1e-8, 1),
    xminorticks = 4,
    ylim = (0.0, 1.0),
    yticks = 0.0:0.25:1.0,
    #yticks = (-2.0:1.0:2.0, [@sprintf("%1.1f", x) for x in -2.0:1.0:2.0]),
    yminorticks = 4,
    tick_direction = :out,
    grid = :off,
    guidefontsize = 30,
    tickfontsize = 24,
    xlabel  = "time",
    ylabel  = "Conversion",
    fontfamily = "Times Roman",
    legendfontsize = 22,
    legendfonthalign = :left,
    legend  = :topleft,
    title = "(d)",
    titlelocation = :left,
    titlefontsize = 30
)

plt6 = plot!(plt6, tfine/maximum(tfine), Xout[:,3], ls = :dash,       lc = :blue,  lw = 1.00, label = "Stabilize B")
plt6 = plot!(plt6, tfine/maximum(tfine), Xout[:,1], ls = :solid,      lc = :black, lw = 0.75, label = "Ideal Solution")
plt6 = plot!(plt6, tfine/maximum(tfine), Xout[:,2], ls = :dashdotdot, lc = :red,   lw = 1.00, label = "Destabilize B")

display(plt6)
savefig(plt6, "CS1d6.pdf")
```

```julia
idx1 = r1out[:,1] .> 0
idx2 = r1out[:,2] .> 0
idx3 = r1out[:,3] .> 0

plt7 = plot(tfine[idx1]/maximum(tfine), r1out[:,1][idx1], xscale = :log10, yscale = :log10, ls = :solid, lc = :black, xlabel  = "time", ylabel = "rate", labels = "Ideal Solution", legend = :bottomleft, xlim = (1e-7, 1), ylim = (1e-5, 1), grid = :off)
plt7 = plot!(plt7, tfine[idx2]/maximum(tfine), r1out[:,2][idx2], ls = :dot, lc = :black, label = "Destabilize B")
plt7 = plot!(plt7, tfine[idx3]/maximum(tfine), r1out[:,3][idx3], ls = :dashdotdot, lc = :black, label = "Stabilize B")
display(plt7)
savefig(plt7, "CS1d7.pdf")
```

```julia
idx1 = r1out[:,1] .> 0
idx2 = r1out[:,2] .> 0
idx3 = r1out[:,3] .> 0

plt8 = plot(
    xlim    = (0.0, 1.0),
    xticks  = 0.0:0.25:1.0,
    xminorticks = 4,
    ylim = (0.0, 0.1),
    yticks = 0.0:0.025:0.1,
    yminorticks = 4,
    tick_direction = :out,
    grid = :off,
    guidefontsize = 30,
    tickfontsize = 24,
    xlabel  = "Conversion",
    ylabel  = "rate",
    fontfamily = "Times Roman",
    legendfontsize = 18,
    legendfonthalign = :left,
    legend  = nothing,
    title = "(e)",
    titlelocation = :left,
    titlefontsize = 30
)

plt8 = plot!(plt8, Xout[:,2][idx2], r1out[:,2][idx2], ls = :dashdotdot, lc = :red,   lw = 1.00, label = "Destabilize B")
plt8 = plot!(plt8, Xout[:,1][idx1], r1out[:,1][idx1], ls = :solid,      lc = :black, lw = 0.75, label = "Ideal Solution")
plt8 = plot!(plt8, Xout[:,3][idx3], r1out[:,3][idx3], ls = :dash,       lc = :blue,  lw = 1.00, label = "Stabilize B")
display(plt8)
savefig(plt8, "CS1d8.pdf")
```

```julia
idx1 = z1out[:,1] .< 1.0
idx2 = z1out[:,2] .< 1.0
idx3 = z1out[:,3] .< 1.0

plt9 = plot(tfine[idx1]/maximum(tfine), 1 .- z1out[:,1][idx1], xscale = :log10, ls = :solid, lc = :black, xlabel  = "time", ylabel = "z", label = "Ideal Solution", legend = :topright, grid = :off)
plt9 = plot!(plt9, tfine[idx2]/maximum(tfine), 1 .- z1out[:,2][idx2], ls = :dot, lc = :black, label = "Destabilize B")
plt9 = plot!(plt9, tfine[idx3]/maximum(tfine), 1 .- z1out[:,3][idx3], ls = :dot, lc = :black, label = "Stabilize B")
display(plt9)
savefig(plt9, "CS1d9.pdf")
```

```julia
plt10 = plot(
    xlim    = (0.0, 1.0),
    xticks  = 0.0:0.2:1.0,
    xminorticks = 4,
    ylim = (0.0, 1.0),
    yticks = 0.0:0.25:1.0,
    yminorticks = 4,
    tick_direction = :out,
    grid = :off,
    guidefontsize = 30,
    tickfontsize = 24,
    xlabel  = "Conversion",
    ylabel  = "(1 - z)",
    fontfamily = "Times Roman",
    legendfontsize = 18,
    legendfonthalign = :left,
    legend  = nothing,
    title = "(f)",
    titlelocation = :left,
    titlefontsize = 30
)

plt10 = plot!(plt10, Xout[:,2], 1 .- z1out[:,2], ls = :dashdotdot, lc = :red,   lw = 1.00, label = "Destabilize B")
plt10 = plot!(plt10, Xout[:,1], 1 .- z1out[:,1], ls = :solid,      lc = :black, lw = 0.75, label = "Ideal Solution")
plt10 = plot!(plt10, Xout[:,3], 1 .- z1out[:,3], ls = :dash,       lc = :blue,  lw = 1.00, label = "Stabilize B")
display(plt10)
savefig(plt10, "CS1d10.pdf")
```


```python
import holoviews as hv
hv.extension('bokeh')
hv.opts.defaults(hv.opts.Curve(width=500),
                 hv.opts.Histogram(width=500))
```

```python
import numpy as np
import scipy.stats
```

# Introduction to linear regression

Many problems can be posed as "finding a relation" between factors/variables

This can be interpreted as predicting and/or explaining a variable given others

Some examples:

- Predicting sales given money spent in advertising
- Predicting chance to rain in Valdivia given temperature, pressure and humidity
- Predicting gasoline consumption of a car given acceleration, weight and number of cylinders
- Predicting chance to get lung cancer given number of smoked cigarettes per day, age and gender

We could ask

- Are these variable related?
- How strong and/or significant is the relationship?
- What is the nature of the relationship?

Answering these helps us **understand the underlying processes behind the data**

## Defining regression

**Regression** refers to a family of statistical methods to find **relationships** between **variables**

In general the relation is modeled as a function $g(\cdot)$ that maps two types of variables

- The input variable $X$ is called **independent variable** or feature
- The output variable $Y$ is called **dependent variable**, response or target

The mapping or function $g$ is called **predictor** or **regressor**

$$
g: X \to Y
$$

The objective is to learn $g$ such that we can predict $Y$ given $X$, *i.e.* $\mathbb{E}[Y|X]$

- **Regression** can be defined from an statistical perspective as a special case of model fitting (parameter estimation)
- In many books **Regression** is defined from a pure-optimization perspective (deterministic)
- **Regression** is considered part of the *supervised learning* paradigm. The difference between **Regression** and *classification* is the nature of the dependent variable (continuous vs categorical)

### Parametric vs non-parametric  regression

Regression methods can be broadly classified as either parametric or non-parametric

In parametric regression

- We know the model of the regressor
- The model has a finite number of parameters
- The parameters of the model are all we need to do predictions
- Simpler but with bigger assumptions (inductive bias)

In nonparametric regression

- There is no functional form for the regressor
- It can have an infinite number of parameters (and a finite number of hyperparameters)
- The regressor is defined from the training data
- More flexible but requires more data to fit it
- Examples: Splines, Support vector regression, Gaussian processes

In this lesson we will focus on parametric regression

### Parametric linear models for regression

Let

- $X$ be a continuous D-dimensional variable (feature) and $Y$ be a continuous unidimensional variable (target)
- $\{x_i, y_i\}$ with $i=1,\ldots,N$ be a set of $N$ *iid* observations of $X$ and $Y$
- $g_\theta$ be a model with a M-dimensional parameter $\theta$

Then we can define parametric regression as finding a value of $\theta$ such that

$$
y_i \approx g_\theta(x_i),\quad i=1,\ldots, N
$$

The simplest parametric model is the **linear model**. A linear model gives rise to **linear regression**

:::{important}

The linear model is linear on $\theta$ but not necessarily on $X$

:::

For example a model with unidimensional input

$$
g_\theta \left(x_i \right) = \theta_0 + \theta_1 x_i  + \theta_2 x_i^2,
$$

is a linear model and

$$
g_\theta(x_i) = \theta_0 + \theta_1 \log(x_i),
$$

is also a linear model but

$$
g_\theta(x_i) = \theta_0 + \log(x_i + \theta_1),
$$

is not a linear model

## The simplest linear model: The line

If we consider a one-dimensional variable $x_i \in \mathbb{R}, i=1,\ldots,N$, then the simplest linear model is

$$
g_\theta(x_i) = \theta_0 + \theta_1 x_i
$$

which has $M=2$ parameters.

This corresponds to a line in $\mathbb{R}^2$ and we recognize

- $\theta_0$ as the intercept
- $\theta_1$ as the slope

If we consider a two-dimensional variable $x_i = (x_{i1}, x_{i2}) \in \mathbb{R}^2, i=1,\ldots,N$ then we obtain

$$
g_\theta(x_i) = \theta_0 + \theta_1 x_{i1} + \theta_2 x_{i2}
$$

which has $M=3$ parameters. This corresponds to a plane in $\mathbb{R}^3$

The most general form assumes a D-dimensional variable $x_i = (x_{i1}, x_{i2}, \ldots, x_{iD}), i=1,\ldots,N$

$$
g_\theta(x_i) = \theta_0 + \sum_{j=1}^D \theta_j x_{ij}
$$

which has $M=D+1$ parameters, which corresponds to an hyperplane in $\mathbb{R}^M$

### Fitting the simplest linear model: Mathematics

Assuming that we have $\{x_i, y_i\}_{i=1,\ldots,N}$ *iid* observations from unidimensional variables X and Y

> How do we find $\theta_0$ and $\theta_1$ such that $y_i \approx \theta_0 + \theta_1 x_i, \forall i$?

Let's start by writing the squared residual (error) as

$$
E_i^2 = (y_i - \theta_0 - \theta_1 x_i)^2,
$$

We can fit (train) the model with

$$
\min_{\theta} L = \sum_{i=1}^N E_i^2 = \sum_{i=1}^N (y_i - \theta_0 - \theta_1 x_i)^2,
$$

where $L$, the sum of squares errors, is a our loss/cost function

:::{note}

Later we will see that this cost function arises when a gaussian likelihood for $Y$ is assumed

:::

Setting the derivative of this expression with respect to the parameters we obtain

$$
\hat \theta_0 = \bar y - \hat \theta_1 \bar x,
$$

where $\bar x = \frac{1}{N} \sum_{i=1}^N x_i$, $\bar y = \frac{1}{N} \sum_{i=1}^N y_i$ and

$$
\hat \theta_1 = \frac{N\sum_i x_i y_i - (\sum_i x_i)(\sum_i y_i)}{N \sum_i x_i^2 - (\sum_i x_i)^2},
$$

:::{dropdown} Proof

With

$$
\begin{align}
\frac{dL}{d\theta_0} &= -2 \sum_{i=1}^N (y_i - \theta_0 - \theta_1 x_i) \nonumber \\
&= -2 \sum_{i=1}^N y_i +  2 N\theta_0 + 2 \theta_1 \sum_{i=1}^N x_i = 0 \nonumber
\end{align}
$$

and

$$
\begin{align}
\frac{dL}{d\theta_1} &= -2 \sum_{i=1}^N (y_i - \theta_0 - \theta_1 x_i) x_i \nonumber \\
&= -2 \sum_{i=1}^N y_i x_i +  2 \theta_0 \sum_{i=1}^N x_i + 2 \theta_1 \sum_{i=1}^N x_i^2 = 0 \nonumber
\end{align}
$$

a system of two equations and two unknowns is obtained

$$
\begin{pmatrix} N & \sum_i x_i \\ \sum_i x_i & \sum_i x_i^2\\\end{pmatrix} \begin{pmatrix} \theta_0 \\ \theta_1 \end{pmatrix}  = \begin{pmatrix} \sum_i y_i \\ \sum_i x_i y_i \end{pmatrix}
$$

whose solution is

$$
\begin{pmatrix} \hat \theta_0 \\ \hat \theta_1 \end{pmatrix}  =
\frac{1}{N\sum_i x_i^2 - \left(\sum_i x_i\right)^2}\begin{pmatrix} \sum_i x_i^2 & -\sum_i x_i \\ -\sum_i x_i & N\\\end{pmatrix}
\begin{pmatrix} \sum_i y_i \\ \sum_i x_i y_i \end{pmatrix}
$$

where we assume that the determinant of the matrix is not zero

:::

### Fitting the simplest linear model: Python

We can fit a line in Python using the `scipy.stats` library

```python
scipy.stats.linregress(x, # N vector or Mx2 matrix (if y is None)
                       y=None, # N vector
                      )
```

This function returns an object, its main attributes are

- `slope`: Equivalent to $\hat \theta_1$
- `intercept`: Equivalent to $\hat \theta_0$
- `rvalue`: The correlation coefficient (more on this later)
- `pvalue`: A p-value for the null hypothesis that $\theta_1 =0$ (more on this later)

Let's create synthetic data to test this function

```python
np.random.seed(12345)
theta, sigma = [0.5 , -1], 0.5
x = np.random.rand(25)*5

def model(x, theta):
    return theta[0] + theta[1]*x

y = model(x, theta) + sigma*np.random.randn(len(x))
```

We fit the data using

```python
res = scipy.stats.linregress(x, y)
hat_theta = np.array([res.intercept, res.slope])

print(f"hat theta0: {hat_theta[0]:0.5f}, hat theta1: {hat_theta[1]:0.5f}")
```

### Predicting with the model and inspecting the results

We can use the fitted model to interpolate/extrapolate on new values $\hat x$

The following plot shows the fitted model on the training samples

```python
hat_x = np.linspace(-1, 6, num=50)
hat_y = model(hat_x, hat_theta)
```

```python
p_fitted = hv.Curve((hat_x, hat_y), label='Fitted model')
p_data = hv.Scatter((x, y), label='Training data').opts(color='k', size=5)

hv.Overlay([p_data, p_fitted])
```

The fitted model (blue) follows the data closely.

To visually assess the quality of the fit we can also plot the residuals, i.e. the distance between each sample of the training set and the fitted line. We can also inspect the histogram of the residuals

```python
residuals = y - model(x, hat_theta)
bins, edges = np.histogram(residuals, density=True)
```

```python
p_residuals = hv.Scatter((y, residuals), 'Target variable', 'Residuals').opts(color='k', size=5, width=350)
p_zero = hv.HLine(0).opts(color='k', line_dash='dashed', alpha=0.5)
p_hist = hv.Histogram((edges, bins), kdims='Residuals', vdims='Density').opts(width=350)

hv.Layout([p_residuals * p_zero, p_hist]).cols(2)
```

Look for residuals that

- concentrate around zero
- are not correlated (white noise like)

Correlation in the residuals is a sign that the choice of the model (line) was not adequate

### Coefficient of determination

We can measure how strong is the linear relation between $y$ and $\hat y = \hat \theta_0 + \hat \theta_1 x$ using the **coefficient of determination** or $r^2$

This is defined as

$$
r^2 = 1 - \frac{\sum_i (y_i - \hat y_i)^2}{\sum_i (y_i - \bar y_i)^2} \in [0, 1]
$$

*i.e.* one minus the sum of residuals divided by the variance of $y$. The $r$ statistic is also known as Pearson's correlation coefficient.

Interpreting $r^2$:

- If $r^2 = 1$, the data points are fitted perfectly by the model. The regressor accounts for all of the variation in y
- If $r^2 = 0$, the regression line is horizontal. The regressor accounts for none of the variation in y

:::{warning}

If the relation is strong but non-linear it will not be detected by $r^2$

:::

Note that $r$ is available in the object returned by `scipy.stats.linregress`

For example in this case:

```python
print(res.rvalue**2)
```


# Simplified ikeda Bilge keel
Comparison between original paper and IMO

```python
import sympy as sp
import dill
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
```

```python
with open('equations.dill', mode='rb') as file:
    equations = dill.load(file)
```

```python
with open('symbols.dill', mode='rb') as file:
    symbols = dill.load(file)
```

## Proposed Formula of the Bilge Keel Roll Damping

```python
equations.B_n_BK
```

$\displaystyle B_{n BK} = A_{BK} x_{5} e^{B_{B2K} x_{3}^{B_{B3K}} + B_{BK}}$

```python
from rolldecayestimators.substitute_dynamic_symbols import lambdify
```

```python
lambdify(equations.B_B3K.rhs)
```

    <function _lambdifygenerated(x_4)>

```python
lambdify(equations.B_n_BK.rhs)
```

    <function _lambdifygenerated(A_BK, B_B2K, B_B3K, B_BK, x_3, x_5)>

```python
lambda_dict = {}
for name, equation in equations._asdict().items():
    lambda_dict[name] = lambdify(equation.rhs)
```

```python
from collections import namedtuple
Lambdas = namedtuple('Lambdas', lambda_dict.keys())
lambdas = Lambdas(**lambda_dict)
```

```python
from inspect import signature
```

```python
def run(function,inputs):
    s = signature(function)
    input_names = set(s.parameters.keys())
    return function(**inputs[input_names])
```

```python
B = 60  # [m]
d = 20  # [m]
Lpp = 300  # [m]
b_BK = 0.03*B
l_BK = 0.3*Lpp
phi_a = 20  # [deg]
w_hat = 1.1
Cb = 0.58
Cm = 0.94

OG_div_d = np.linspace(-1.5,0,8)
OG = OG_div_d*d

inputs = pd.DataFrame()
inputs['OG'] = OG
inputs['OG/d'] = OG/d
inputs['x_1'] = B/d
inputs['x_2'] = Cb
inputs['x_3'] = Cm
inputs['x_4'] = OG/d
inputs['x_5'] = w_hat
inputs['x_6'] = phi_a
inputs['x_7'] = b_BK/B
inputs['x_8'] = l_BK/Lpp

inputs['f_1'] = run(lambdas.f_1, inputs)
inputs['f_2'] = run(lambdas.f_2, inputs)
inputs['f_3'] = run(lambdas.f_3, inputs)

inputs['A_BK'] = run(lambdas.A_BK, inputs)
inputs['B_B2K'] = run(lambdas.B_B2K, inputs)
inputs['B_B3K'] = run(lambdas.B_B3K, inputs)
inputs['B_BK'] = run(lambdas.B_BK, inputs)

inputs['B_n_BK'] = run(lambdas.B_n_BK, inputs)
```

```python
inputs.plot(x='OG/d', y='B_n_BK')
```

```python
equations.f_1
```

$\displaystyle f_{1} = - 2.21 x_{2} + 8.0089 \left(0.3907 - 0.3651 x_{2}\right) \left(0.353356890459364 x_{1} - 1\right)^{2} + 2.632$

```python
equations.f_2
```

$\displaystyle f_{2} = 0.00255 x_{6}^{2} + 0.122 x_{6} + 0.4794$

```python
equations.f_3
```

$\displaystyle f_{3} = x_{8}^{2} \left(- 0.8913 x_{7}^{2} - 0.0733 x_{7}\right) + x_{8} \left(5.2857 x_{7}^{2} - 0.01185 x_{7} + 0.00189\right)$

```python
equations.B_BK
```

$\displaystyle B_{BK} = x_{4} \left(0.3 x_{1} + 0.00125 x_{6}^{2} - 0.0425 x_{6} + 5 x_{7} - 0.2 x_{8} - 1.86\right)$

```python
equations.B_B2K
```

$\displaystyle B_{B2K} = - 0.1 x_{1} + 1.2 x_{2} - 0.0657 x_{4}^{2} + 0.0586 x_{4} - 15 x_{7} + 1.6164$

```python
equations.B_B3K
```

$\displaystyle B_{B3K} = 2.5 x_{4} + 15.75$

## IMO implementation

```python
Q7_str = """
0 1 2 3
1 0 -0.3651 0.3907
2 0 -2.21 2.632
3 0.00255 0.122 0.4794
4 -0.8913 -0.0733 0
5 5.2857 -0.01185 0.00189
6 0.00125 -0.0425 -1.86
7 -0.0657 0.0586 1.6164
"""
```

```python
Q7 = np.fromstring(Q7_str,sep=' ').reshape(8,4)
Q7
```

    array([[ 0.0000e+00,  1.0000e+00,  2.0000e+00,  3.0000e+00],
           [ 1.0000e+00,  0.0000e+00, -3.6510e-01,  3.9070e-01],
           [ 2.0000e+00,  0.0000e+00, -2.2100e+00,  2.6320e+00],
           [ 3.0000e+00,  2.5500e-03,  1.2200e-01,  4.7940e-01],
           [ 4.0000e+00, -8.9130e-01, -7.3300e-02,  0.0000e+00],
           [ 5.0000e+00,  5.2857e+00, -1.1850e-02,  1.8900e-03],
           [ 6.0000e+00,  1.2500e-03, -4.2500e-02, -1.8600e+00],
           [ 7.0000e+00, -6.5700e-02,  5.8600e-02,  1.6164e+00]])

```python
inputs_IMO = pd.DataFrame()
OG = inputs_IMO['OG'] = OG
inputs_IMO['OG/d'] = OG/d
x_1 = inputs_IMO['x_1'] = B/d
x_2 = inputs_IMO['x_2'] = Cb
x_3 = inputs_IMO['x_3'] = Cm
x_4 = inputs_IMO['x_4'] = OG/d
x_5 = inputs_IMO['x_5'] = w_hat
x_6 = inputs_IMO['x_6'] = phi_a
x_7 = inputs_IMO['x_7'] = b_BK/B
x_8 = inputs_IMO['x_8'] = l_BK/Lpp
```

```python
i = np.arange(1,3+1)
sum1 = np.sum(Q7[1,i]*x_2**(3-i))
sum2 = np.sum(Q7[2,i]*x_2**(3-i))

f_1 = inputs_IMO['f_1'] = (x_1-2.83)**2*sum1 + sum2
#f_1 = inputs_IMO['f_1'] = (-0.365*x_2+0.3907)*(x_1-2.83)**2-2.21*x_2+2.632

f_2 = inputs_IMO['f_2'] = np.sum(Q7[3,i]*x_6**(3-i))

rows=2
columns=3
sum3 = np.zeros((rows+1,columns+1))
for i in np.arange(1,rows+1):
    for j in np.arange(1,columns+1):
        sum3[i,j] = Q7[3+i,j]*x_7**(3-j)*x_8**(3-i)

f_3 = inputs_IMO['f_3'] = np.sum(np.sum(sum3))
```

```python
i = np.arange(1,3+1)
sum4 = np.sum(Q7[6,i]*x_6**(3-i))
B_BK = inputs_IMO['B_BK'] = x_4*(5*x_7+0.3*x_1-0.2*x_8 + sum4)
```

## The $B_{BK2}$ equation in IMO is incorrect...

```python
rows=2
sum5 = np.zeros(len(inputs_IMO))
for i in np.arange(1,rows+1):
    sum5+=Q7[7,i]*(OG/d)**(3-i)
#B_B2K = inputs_IMO['B_B2K'] = -15*x_7+1.2*x_2-0.1*x_1 + sum5

B_B2K = inputs_IMO['B_B2K'] = (-15*x_7+1.2*x_2-0.1*x_1
                               -0.0657*x_4**2+0.0586*x_4+1.6164)

B_B3K = inputs_IMO['B_B3K'] = 2.5*OG/d+15.75
A_BK = inputs_IMO['A_BK'] = f_1*f_2*f_3
B_n_BK = inputs_IMO['B_n_BK'] = A_BK*w_hat*np.exp(B_BK+B_B2K*x_3**(B_B3K))
```

```python
fig,ax = plt.subplots()
inputs.plot(x='OG/d', y='B_n_BK', label='Kawahara', ax=ax)
inputs_IMO.plot(x='OG/d', y='B_n_BK', label='IMO', style='--',ax=ax)

```

```python

ys = ['f_1','f_2','f_3','B_BK','B_B2K','B_B3K']

for y in ys:
    fig,ax = plt.subplots()
    inputs.plot(x='OG/d', y=y, label='Kawahara', ax=ax)
    inputs_IMO.plot(x='OG/d', y=y, label='IMO', style='--', ax=ax)
    ax.set_title(y)
```

```python

```


```
from __future__ import division
import sympy
from sympy import *
from sympy import init_printing
init_printing(use_latex='mathjax')
from sympy.utilities.autowrap import ufuncify, autowrap
from copy import deepcopy
import numpy as np
from IPython.display import display
```

# Dielectric constant functions

Let's generate some values for $\theta_\text{1}$ and $\theta_\text{2}$ that we can check by hand.

```
from sympy.abc import D, k, kappa, omega, lamda, eta, alpha, theta
E_s1 = symbols('E_s1')
E_s, E_d, E_eff, h_s, h_d= symbols('E_s E_d E_eff h_s h_d')
k, kappa, omega, D, h = symbols('k kappa omega D h')
```

Intermediates used to calculate $\theta_\text{I}, \theta_\text{II}$:

```
intermediates = {alpha : E_eff / E_d,
eta: sqrt(k**2 + kappa**2 / E_s + omega*1j / D),
lamda: (1 - E_eff / E_s)*(k / eta)}
for key, val in intermediates.viewitems():
    display(Eq(key, val))
```

$$\lambda = \frac{k}{\eta} \left(- \frac{E_{eff}}{E_{s}} + 1\right)$$

$$\alpha = \frac{E_{eff}}{E_{d}}$$

$$\eta = \sqrt{k^{2} + \frac{\kappa^{2}}{E_{s}} + \frac{1.0 i}{D} \omega}$$

```
theta_I_bracket_term = (-lamda / tanh(eta * h_s) +
                        (sinh(k * h_s) * sinh(eta * h_s) + alpha * cosh(k*h_s) * sinh(eta * h_s) -
                         lamda * (cosh(k*h_s) * cosh(eta*h_s) - 2 + lamda * sinh(k*h_s)/sinh(eta *h_s)))/
                        (cosh(k*h_s)*sinh(eta*h_s) + alpha * sinh(k*h_s) * sinh(eta*h_s) - lamda * sinh(k * h_s) * cosh(eta * h_s))
                        )
```

```
theta_I = E_s / E_eff * theta_I_bracket_term
```

```
theta_I
```

$$\frac{E_{s}}{E_{eff}} \left(- \frac{\lambda}{\tanh{\left (\eta h_{s} \right )}} + \frac{\alpha \sinh{\left (\eta h_{s} \right )} \cosh{\left (h_{s} k \right )} - \lambda \left(\frac{\lambda \sinh{\left (h_{s} k \right )}}{\sinh{\left (\eta h_{s} \right )}} + \cosh{\left (\eta h_{s} \right )} \cosh{\left (h_{s} k \right )} - 2\right) + \sinh{\left (\eta h_{s} \right )} \sinh{\left (h_{s} k \right )}}{\alpha \sinh{\left (\eta h_{s} \right )} \sinh{\left (h_{s} k \right )} - \lambda \sinh{\left (h_{s} k \right )} \cosh{\left (\eta h_{s} \right )} + \sinh{\left (\eta h_{s} \right )} \cosh{\left (h_{s} k \right )}}\right)$$

```
theta_I_python = lambdify((k, E_s, E_eff, eta, h_s, alpha, lamda), theta_I)
theta_I_c = autowrap(theta_I, language='C', backend='Cython', args=(k, E_s, E_eff, eta, h_s, alpha, lamda))
theta_I_f = autowrap(theta_I, language='F95', backend='f2py', args=(k, E_s, E_eff, eta, h_s, alpha, lamda))
```

```
theta_II = E_s / E_d * (E_eff + (1-lamda)*E_d/tanh(k*h_d))/(E_eff / tanh(k*h_d) + (1 - lamda) * E_d)
theta_II
```

$$\frac{E_{s} \left(\frac{E_{d} \left(- \lambda + 1\right)}{\tanh{\left (h_{d} k \right )}} + E_{eff}\right)}{E_{d} \left(E_{d} \left(- \lambda + 1\right) + \frac{E_{eff}}{\tanh{\left (h_{d} k \right )}}\right)}$$

```
{k:0.01, E_s:3.5-0.05j, E_d: 3.5-0.05j, E_eff: 3.5 - 0.5j, }
```

    {k: 0.01, E_d: (3.5-0.05j), E_eff: (3.5-0.5j), E_s: (3.5-0.05j)}

```
%timeit theta_I_python(1, 3, 2, 0.2, 4, 1.02, 1.8)
```

    100000 loops, best of 3: 3.06 µs per loop

```
%timeit theta_I_c(1, 3, 2, 0.2, 4, 1.02, 1.8)
```

    1000000 loops, best of 3: 285 ns per loop

```
%timeit theta_I_f(1, 3, 2, 0.2, 4, 1.02, 1.8)
```

    1000000 loops, best of 3: 456 ns per loop

```
((E_s - theta) / (E_s + theta))
```

$$\frac{E_{s} - \theta}{E_{s} + \theta}$$

```
from sympy.functions.elementary.complexes import im
```

```
complex_result = (E_s - theta) / (E_s + theta)
```

```
dielectric_I = complex_result.subs({theta:theta_I}).subs(intermediates).subs(intermediates)
```

```
dielectric_args = (k, omega, kappa, E_s, E_d, E_eff, h_s, D)
dielectric_I_python = lambdify(dielectric_args, dielectric_I, dummify=False, modules="numpy")
```

```
%timeit dielectric_I_python(k=1.1, omega=432, kappa=0.3, E_s=3-0.1j, E_d=4-1j, E_eff=4-5j, h_s=0.07, D=0.0077)
```

    10000 loops, best of 3: 121 µs per loop

```
dielectric_I_python(k=1.1, omega=432, kappa=0.3, E_s=3-0.1j, E_d=4-1j, E_eff=4-5j, h_s=0.07, D=0.0077)
```

    (0.6365044412573243-0.10023319309176819j)

```
lambdify(expr=tanh(h_s * intermediates[eta]), args=[k, kappa, omega, D, h_s, E_s], modules='numpy')(0.01, 0.3, 432, 0.0077, 0.07, 3-1j)
```

    (1.0000000000148801-1.302525298862032e-10j)

```
lambdify(expr=(h_s * intermediates[eta]), args=[k, kappa, omega, D, h_s, E_s], modules='numpy')(0.01, 0.3, 432, 1e-6, 0.07, 3-1j)
```

    (1028.78569201192+1028.7856919473827j)

```
np.tanh(100000)
```

$$1.0$$

```
sympy.printing.lambdarepr.lambdarepr(dielectric_I)
```

    '(E_s - E_s*(-k*(-E_eff/E_s + 1)/(sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D)*tanh(h_s*sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D))) + (-k*(-E_eff/E_s + 1)*(k*(-E_eff/E_s + 1)*sinh(h_s*k)/(sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D)*sinh(h_s*sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D))) + cosh(h_s*k)*cosh(h_s*sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D)) - 2)/sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D) + sinh(h_s*k)*sinh(h_s*sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D)) + E_eff*sinh(h_s*sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D))*cosh(h_s*k)/E_d)/(-k*(-E_eff/E_s + 1)*sinh(h_s*k)*cosh(h_s*sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D))/sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D) + sinh(h_s*sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D))*cosh(h_s*k) + E_eff*sinh(h_s*k)*sinh(h_s*sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D))/E_d))/E_eff)/(E_s + E_s*(-k*(-E_eff/E_s + 1)/(sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D)*tanh(h_s*sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D))) + (-k*(-E_eff/E_s + 1)*(k*(-E_eff/E_s + 1)*sinh(h_s*k)/(sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D)*sinh(h_s*sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D))) + cosh(h_s*k)*cosh(h_s*sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D)) - 2)/sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D) + sinh(h_s*k)*sinh(h_s*sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D)) + E_eff*sinh(h_s*sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D))*cosh(h_s*k)/E_d)/(-k*(-E_eff/E_s + 1)*sinh(h_s*k)*cosh(h_s*sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D))/sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D) + sinh(h_s*sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D))*cosh(h_s*k) + E_eff*sinh(h_s*k)*sinh(h_s*sqrt(k**2 + kappa**2/E_s + 1.0*I*omega/D))/E_d))/E_eff)'

## Small functions

```
_eta = intermediates[eta]
```

```
all_k = [1, 10, 100, 1000, 10000, 100000]
all_eta = [_eta.evalf(subs={k:_k, kappa:3500, E_s:3 - 0.001j, D: 0.005, omega:300})
           for _k in all_k]
```

```
print(str(all_eta).replace('*I', 'j').replace(",", ",\n"))
```

    [2020.78311260126 + 15.182507854811j,
     2020.80760652432 + 15.182323829782j,
     2023.25550170076 + 15.163955048756j,
     2254.66583909462 + 13.607584302718j,
     10202.1243828582 + 3.007271263178j,
     100020.414581093 + 0.30674293451j]

```
_lamda = intermediates[lamda]
all_lamda = [_lamda.evalf(subs=
                {k:_k, E_s:3-0.001j, D:0.005,
                 E_eff:3 - 100j, eta: _eta_}) for _k, _eta_ in zip(all_k, all_eta)]

```

```
print(str(all_lamda).replace('*I', 'j').replace(",", ",\n"))
```

    [0.0001184255261724 + 0.0164941987549306j,
     0.00118421087011718 + 0.164939988752172j,
     0.0117978533636026 + 1.64740475175451j,
     0.0842948437214929 + 14.7834985873234j,
     -0.00125999301746353 + 32.672603689536j,
     -0.0110065260871034 + 33.3261929274151j]

```
all_theta_I = [theta_I.evalf(subs={k:_k, E_s:3-0.001j, D:0.005,
                                   E_eff:3 - 100j, eta: _eta_,
                                   lamda:_lamda_, h_s:0.1, alpha: 0.65-0.0002j}) for _k, _eta, _lamda_ in zip(all_k, all_eta, all_lamda)]
```

```
print(str(all_theta_I).replace('*I', 'j').replace(",", ",\n"))
```

    [0.00157126996626562 + 0.0210682675809495j,
     0.00672782406000677 + 0.0281575198774334j,
     0.050275664263775 + 0.0281213204722464j,
     0.443934273416263 + 0.0140052914999941j,
     0.980197277948465 + 0.000305155415174606j,
     0.999795989512753 + 3.05416795636227e-6j]

```
sympy.printing.lambdarepr.lambdarepr(theta_I)
```

    'E_s*(-lamda/tanh(eta*h_s) + (alpha*sinh(eta*h_s)*cosh(h_s*k) - lamda*(lamda*sinh(h_s*k)/sinh(eta*h_s) + cosh(eta*h_s)*cosh(h_s*k) - 2) + sinh(eta*h_s)*sinh(h_s*k))/(alpha*sinh(eta*h_s)*sinh(h_s*k) - lamda*sinh(h_s*k)*cosh(eta*h_s) + sinh(eta*h_s)*cosh(h_s*k)))/E_eff'

```
theta_I_python = lambdify((k, E_s, D, E_eff, eta, lamda, h_s, alpha), theta_I, dummify=False, modules='numpy')
```

```
def copy_update_dict(dictionary, **kwargs):
    copied_dictionary = deepcopy(dictionary)
    copied_dictionary.update(kwargs)
    return copied_dictionary
```

```
theta_subs = {k: 1, E_s:3-0.001j, D:0.005, E_eff:3 - 100j, eta: 2020.78311260126 + 15.182507854811j, lamda:0.0001184255261724 + 0.0164941987549306j, h_s:0.1, alpha: 0.65-0.0002j}
theta_I_kwargs = {str(key): np.complex128(val) for key, val in theta_subs.items()}
theta_I_all_kwargs = [copy_update_dict(theta_I_kwargs, k=np.float(_k), eta=np.complex128(_eta_), lamda=np.complex128(_lamda_))
                          for _k, _eta_, _lamda_ in zip(all_k, all_eta, all_lamda)]
all_theta_I_lambda = [theta_I_python(**kwargs) for kwargs in theta_I_all_kwargs]
```

    /Users/ryandwyer/anaconda/envs/jittermodel/lib/python2.7/site-packages/numpy/__init__.py:1: RuntimeWarning: overflow encountered in tanh
      """
    /Users/ryandwyer/anaconda/envs/jittermodel/lib/python2.7/site-packages/numpy/__init__.py:1: RuntimeWarning: invalid value encountered in tanh
      """
    /Users/ryandwyer/anaconda/envs/jittermodel/lib/python2.7/site-packages/numpy/__init__.py:1: RuntimeWarning: overflow encountered in sinh
      """
    /Users/ryandwyer/anaconda/envs/jittermodel/lib/python2.7/site-packages/numpy/__init__.py:1: RuntimeWarning: overflow encountered in cosh
      """
    /Users/ryandwyer/anaconda/envs/jittermodel/lib/python2.7/site-packages/numpy/__init__.py:1: RuntimeWarning: invalid value encountered in cosh
      """
    /Users/ryandwyer/anaconda/envs/jittermodel/lib/python2.7/site-packages/numpy/__init__.py:1: RuntimeWarning: invalid value encountered in sinh
      """

```
all_theta_I_lambda
```

    [(0.0015712699662656167+0.021068267580949523j),
     (0.0067278240600067672+0.028157519877433347j),
     (0.050275664263774951+0.028121320472246414j),
     (0.44393427341626346+0.01400529149999408j),
     (nan+nan*j),
     (nan+nan*j)]

So for some reason the sympy version of this is implemented better.

```
theta_subs = {k: 1, E_s:3-0.001j, D:0.005, E_eff:3 - 100j, E_d: 3 - 0.001j, eta: 2020.78311260126 + 15.182507854811j, lamda:0.0001184255261724 + 0.0164941987549306j, h_s:0.1, alpha: 0.65-0.0002j}
all_theta_II = [theta_II.evalf(subs={k: _k, E_s:3-0.001j, E_eff:3 - 100j,
                                     E_d: 3 - 0.001j, lamda:_lamda_, h_d:0.1})
                for _k, _lamda_ in zip(all_k, all_lamda)]
```

```
print(str(all_theta_II).replace('*I', 'j').replace(",", ",\n"))
```

    [0.101145810077246 + 0.0296480635666554j,
     0.764320753451023 + 0.0123928030520502j,
     0.999999996277978 + 2.1003332939236e-10j,
     1.0 + 8.470329472543e-22j,
     1.00000000000000,
     1.00000000000000]

## Avoiding numerical errors in calculating $\theta_I$

To avoid numerical errors in calculating $\theta_I$ (associated with numpy's problems with large values of hyperbolic trigonometry functions for large values of $\theta$ (see Eric Moore's github issue). We have $\theta_I$ is

$$\frac{\epsilon_{\text{s}}}{\epsilon_{\text{eff}}} \left(- \frac{\lambda}{\tanh{\left (\eta h_{\text{s}} \right )}} + \frac{\alpha \sinh{\left (\eta h_{\text{s}} \right )} \cosh{\left (h_{\text{s}} k \right )} - \lambda \left(\frac{\lambda \sinh{\left (h_{\text{s}} k \right )}}{\sinh{\left (\eta h_{\text{s}} \right )}} + \cosh{\left (\eta h_{\text{s}} \right )} \cosh{\left (h_{\text{s}} k \right )} - 2\right) + \sinh{\left (\eta h_{\text{s}} \right )} \sinh{\left (h_{\text{s}} k \right )}}{\alpha \sinh{\left (\eta h_{\text{s}} \right )} \sinh{\left (h_{\text{s}} k \right )} - \lambda \sinh{\left (h_{\text{s}} k \right )} \cosh{\left (\eta h_{\text{s}} \right )} + \sinh{\left (\eta h_{\text{s}} \right )} \cosh{\left (h_{\text{s}} k \right )}}\right)$$

```
theta_I_numexpr = lambdify((k, E_s, D, E_eff, eta, lamda, h_s, alpha), theta_I, dummify=False, modules='numexpr')
theta_subs = {k: 1, E_s:3-0.001j, D:0.005, E_eff:3 - 100j, eta: 2020.78311260126 + 15.182507854811j, lamda:0.0001184255261724 + 0.0164941987549306j, h_s:0.1, alpha: 0.65-0.0002j}
theta_I_kwargs = {str(key): np.complex128(val) for key, val in theta_subs.items()}
theta_I_all_kwargs = [copy_update_dict(theta_I_kwargs, k=np.float(_k), eta=np.complex128(_eta_), lamda=np.complex128(_lamda_))
                          for _k, _eta_, _lamda_ in zip(all_k, all_eta, all_lamda)]
all_theta_I_numexpr = [theta_I_numexpr(**kwargs) for kwargs in theta_I_all_kwargs]
```

In the limit,

$$e^{k h_{\text{s}}} \gg 1$$

we can approximate,

$$\coth \eta h_\text{s} = \tanh \eta h_\text{s} = 1$$,

$$\sinh k h_\text{s} = \cosh k h_\text{s} = \exp(k h_\text{s})/2$$,

$$\sinh \eta h_\text{s} = \cosh \eta h_\text{s} = \exp(\eta h_\text{s})/2$$

```
large_k = {sinh(k*h_s): exp(k*h_s)/2, cosh(k*h_s): exp(k*h_s)/2,
           sinh(eta*h_s): exp(eta*h_s)/2, cosh(eta*h_s): exp(eta*h_s)/2,
           coth(eta*h_s): 1, tanh(eta*h_s): 1}

theta_I.subs(large_k)
```

$$\frac{E_{s}}{E_{eff}} \left(- \lambda + \frac{\frac{\alpha}{4} e^{\eta h_{s}} e^{h_{s} k} - \lambda \left(\lambda e^{- \eta h_{s}} e^{h_{s} k} + \frac{e^{\eta h_{s}}}{4} e^{h_{s} k} - 2\right) + \frac{e^{\eta h_{s}}}{4} e^{h_{s} k}}{\frac{\alpha}{4} e^{\eta h_{s}} e^{h_{s} k} - \frac{\lambda}{4} e^{\eta h_{s}} e^{h_{s} k} + \frac{e^{\eta h_{s}}}{4} e^{h_{s} k}}\right)$$

```
theta_full = theta_I.subs(intermediates).subs(intermediates)

def theta_k(_k):
    return {k: _k, E_s:3-0.001j, D:0.005, E_d:4.65, E_eff:3 - 100j, h_s:0.1, kappa:3500, E_s:3 - 0.001j, D: 0.005, omega:300}

simplified_pre_sub = (E_s/E_eff*(-lamda + (1+alpha-lamda*(1 - 8*exp(-h_s*(k+eta)) + 4*lamda*exp(-2*eta*h_s)))/(1 + alpha - lamda)))
simplified = simplified_pre_sub.subs(intermediates).subs(intermediates)
```

```
print(theta_full.subs(theta_k(60)).evalf())
print(simplified.subs(theta_k(60)).evalf())
print(theta_I.subs(large_k).subs(intermediates).subs(intermediates).subs(theta_k(60)).evalf())
```

    0.0305522794210152 + 0.0288610299476901*I
    0.0305522357456196 + 0.0288606649881057*I
    0.0305522357456196 + 0.0288606649881057*I

```
eta.subs(intermediates).subs(intermediates).subs(theta_k(80)).evalf()
```

$$2022.36570074502 + 15.170626889408 i$$

```
def theta_k2(_k, _D, _kappa):
    return {k: _k, E_s:3-0.001j, D:_D, E_d:4.65, E_eff:3 - 100j, h_s:0.1, kappa:_kappa, E_s:3 - 0.001j, D: 0.005, omega:300}
```

```
print(theta_full.subs(theta_k2(60, 0.005/1e6, 3500*1e6)).evalf())
print(simplified.subs(theta_k2(60, 0.005/1e6, 3500*1e6)).evalf())
```

    0.000909256549701857 + 0.0299730883346336*I
    0.000909211401593154 + 0.0299727236530035*I


```python
%matplotlib inline

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

import torch
from torch.autograd import Variable

from tqdm import tqdm, trange

import copy

from sklearn.metrics import mean_squared_error
```

```python
torch.__version__
```

    '1.6.0+cu101'

```python
sns.set_context("paper")
sns.set_style("ticks")

np.random.seed(101217209)
torch.manual_seed(101217209)
```

    <torch._C.Generator at 0x7f734e3adfd8>

```python
def get_data(N, min_x, max_x):
    w, b = np.random.randn(2)
    def true_model(X):
        lin_factor = w * X + b
        y = np.sin(lin_factor*10)*(lin_factor**2)
        return y
    X_true = np.arange(min_x, max_x, 0.01)
    y_true = true_model(X_true)
    span = (max_x - min_x)
    scale = 0.2
    X_obs = min_x + span*scale + np.random.rand(N)*(span - 2*scale*span)
    y_obs = true_model(X_obs) + np.random.randn(N)*0.1

    y_mean, y_std = y_obs.mean(), y_obs.std()
    y_obs = (y_obs - y_mean)/y_std
    y_true = (y_true - y_mean)/y_std
    return (X_obs, y_obs, X_true, y_true), (w, b, true_model)
```

```python
N = 40
min_x, max_x = -1, 1
(X_obs, y_obs, X_true, y_true), (w, b, true_model) = get_data(N, min_x, max_x)
```

```python
class SimpleModel(torch.nn.Module):
    def __init__(self, p=0.05, decay=0.001, non_linearity=torch.nn.ReLU):
        super(SimpleModel, self).__init__()
        self.dropout_p = p
        self.decay = decay
        self.f = torch.nn.Sequential(
            torch.nn.Linear(1,20),
            torch.nn.ReLU(),
            torch.nn.Dropout(p=self.dropout_p),
            torch.nn.Linear(20, 20),
            non_linearity(),
            torch.nn.Dropout(p=self.dropout_p),
            torch.nn.Linear(20,1)
        )
    def forward(self, X):
        X = Variable(torch.Tensor(X), requires_grad=False)
        return self.f(X)

def ewc_criterion(model, fisher_matrix, prior_params, alpha=1):
    ewc_loss = Variable(torch.zeros(1))
    for p, fm, prior in zip(model.parameters(), fisher_matrix, prior_params):
        param_loss = torch.pow(p - Variable(prior, requires_grad=False), 2)
        fisher_scaled_loss = alpha*Variable(fm, requires_grad=False).mul(param_loss)
        ewc_loss += fisher_scaled_loss.sum()
    return ewc_loss[0] # Scalar

def get_criterion_optimizer(model):
    criterion = torch.nn.MSELoss()
    optimizer = torch.optim.Adam(
        model.parameters(),
        #lr=0.001,
        #momentum=0.5,
        weight_decay=model.decay)
    return criterion, optimizer

def plot_predictions(model, model_simple, ax=None, uncertainty=0, N=0, l2=0.005):
    if ax is None:
        fig, ax = plt.subplots(1,1)
    ax.plot(X_obs[:10], y_obs[:10], ls="none", marker="o", color="b", label="task 1")
    ax.plot(X_obs[10:20], y_obs[10:20], ls="none", marker="d", color="cyan", label="task 2")
    ax.plot(X_obs[20:], y_obs[20:], ls="none", marker="s", color="magenta", label="task 3")
    ax.plot(X_true, y_true, ls="-", lw=1, color="0.5", label="true")

    for model_t, model_name, color in [
        (model, "ewc", "r"),
        (model_simple, "simple", "g")
    ]:
        if uncertainty > 0:
            model_t.train()
            y_preds = []
            for i in trange(uncertainty):
                y_pred = model_t(X_true[:, np.newaxis]).data.numpy()
                y_preds.append(y_pred)
            y_preds = np.hstack(y_preds)
            y_pred = y_preds.mean(axis=1)
            y_var = y_preds.var(axis=1)
            tau = l2 * (model_t.dropout_p) / (2*N*model_t.decay)
            y_var += (1/tau)
            y_std = np.sqrt(y_var) #+ (1/tau)
            for i in range(4):
                ax.fill_between(
                    X_true,
                    y_pred - y_std * ((i+1)/2),
                    y_pred + y_std * ((i+1)/2),
                    color=color,
                    alpha=0.1**(i+1)
                )
        else:
            model_t.eval()
            y_pred = model_t(X_true[:, np.newaxis]).data.numpy()
        mse = mean_squared_error(y_true, y_pred)
        ax.plot(X_true, y_pred, ls="--", lw=1, color=color,
                label="{} ({:.3f})".format(model_name, mse))
    ax.legend()
    ax.set_xlabel("X")
    ax.set_ylabel("Y")
    sns.despine(offset=10)
    return ax

def plot_losses(losses, losses_simple, ax=None):
    if ax is None:
        fig, ax = plt.subplots(1,1,figsize=(10,4))
    ax.plot(losses, lw=0.1, color='r', label='ewc')
    ax.plot(losses_simple, lw=0.1, color='0.5', label='simple')
    ax.legend()
    ax.set_xlabel("Epochs")
    ax.set_ylabel("Loss")
    sns.despine(offset=10)
    return ax

def get_fisher_matrix(model, criterion, X, Y, old_fisher_matrix=None):
    fisher_matrix = [torch.zeros(p.data.shape) for p in model.parameters()]
    counts = 0
    #print(X.shape, Y.shape)
    for x,y in tqdm(zip(X[:, np.newaxis, np.newaxis], Y[:, np.newaxis])):
        criterion.zero_grad()
        #print(x.shape, y.shape)
        y_pred = model(x)
        #print(y_pred.data.shape)
        loss = criterion(y_pred, Variable(torch.Tensor(y), requires_grad=False))
        loss.backward()
        for i, p in enumerate(model.parameters()):
            grad_squared = torch.pow(p.grad.data, 2)
            #print(grad_squared.shape, fisher_matrix[i].shape)
            fisher_matrix[i] += grad_squared
        counts += 1
    #print(grad_squared.shape, fisher_matrix[i].shape)
    ## Calculate the means
    fisher_matrix = [k/counts for k in fisher_matrix]
    if old_fisher_matrix:
        fisher_matrix = [k + o for k,o in zip(fisher_matrix, old_fisher_matrix)]
    return fisher_matrix

def train_step(
    model, X, Y, criterion, optimizer,
    fisher_matrix=None, model_params=None,
    ecw_alpha=0, clip=10):
    y_pred = model(X[:, np.newaxis])
    loss = criterion(y_pred, Variable(torch.Tensor(Y), requires_grad=False))
    train_loss = loss.detach().numpy().item()
    if ecw_alpha > 0:
        ewc_loss = ewc_criterion(model, fisher_matrix, model_params, alpha=ecw_alpha)
        loss += ewc_loss
    loss.backward()
    torch.nn.utils.clip_grad_norm(model.parameters(),clip)
    optimizer.step()
    return train_loss
```

```python
def train_plot_compare(X_obs, y_obs, prefix="IID", clip=10, uncertainty=0, l2=0.01, **kwargs):
    model = SimpleModel(**kwargs)
    model_simple = copy.deepcopy(model)

    fig, ax = plt.subplots(1,3, sharex=True, sharey=True, figsize=(18, 6))
    fig_loss, ax_loss = plt.subplots(3,1, sharex=True, sharey=False, figsize=(10, 12))

    ## Task 1

    fisher_matrix = None
    model_params = None
    N = 0
    for i, X, Y, alpha in zip(
        range(3),
        [X_obs[:10], X_obs[10:20], X_obs[20:]],
        [y_obs[:10], y_obs[10:20], y_obs[20:]],
        [0, 1e6, 1e6] # Alpha should increase with tasks
    ):
        title = "Task {}".format(" + ".join(str(k+1) for k in range(i+1)))
        model.train()
        criterion, optimizer = get_criterion_optimizer(model)

        model_simple.train()
        criterion_simple, optimizer_simple = get_criterion_optimizer(model_simple)

        losses = []
        losses_simple = []
        for epoch in trange(10000):
            loss = train_step(
                model, X, Y,
                criterion, optimizer,
                fisher_matrix=fisher_matrix,
                model_params=model_params,
                ecw_alpha=alpha, clip=clip
            )
            losses.append(loss)

            loss = train_step(
                model_simple, X, Y, criterion_simple, optimizer_simple, clip=clip)
            losses_simple.append(loss)
        plot_losses(losses, losses_simple, ax=ax_loss[i])
        N = X.shape[0]
        plot_predictions(model, model_simple, ax=ax[i], uncertainty=uncertainty, N=N, l2=l2)
        ax_loss[i].set_title(title)
        ax[i].set_title(title)
        print(title)

        model.eval()
        fisher_matrix = get_fisher_matrix(model, criterion, X, Y, old_fisher_matrix=fisher_matrix)
        model_params = [copy.deepcopy(p.data) for p in model.parameters()]

    fig.tight_layout()
    fig_loss.tight_layout()

    sns.despine(offset=10, fig=fig)
    sns.despine(offset=10, fig=fig_loss)

    fig.savefig("{}_EWC_predictions.pdf".format(prefix), bbox_inches="tight")
    fig.savefig("{}_EWC_losses.pdf".format(prefix), bbox_inches="tight")

    return model, model_simple
```

## IID samples

```python
plt.plot(X_obs[:10], y_obs[:10], ls="none", marker="o", color="b", label="task 1")
plt.plot(X_obs[10:20], y_obs[10:20], ls="none", marker="d", color="cyan", label="task 2")
plt.plot(X_obs[20:], y_obs[20:], ls="none", marker="s", color="magenta", label="task 3")
plt.plot(X_true, y_true, ls="-", lw=1, color="0.5", label="true")
plt.legend()
sns.despine(offset=10)
```

```python
model, model_simple = train_plot_compare(
    X_obs, y_obs, prefix="IID",
    p=0.1, decay=1e-6, non_linearity=torch.nn.ReLU)
```

```python
"""
model.eval()
fisher_matrix = get_fisher_matrix(model, criterion, X_obs[:10], y_obs[:10])
model_params = [copy.deepcopy(p.data) for p in model.parameters()]

fig = plt.figure(0)
ax = fig.add_subplot(111)

el = plt.matplotlib.patches.Ellipse(
    xy=[
        model_params[0].squeeze().numpy()[0],
        model_params[1].numpy()[0]
    ],
    width=2./fisher_matrix[0].squeeze().numpy()[0],
    height=2./fisher_matrix[1].numpy()[0])
ax.add_artist(el)
#el.set_clip_box(ax.bbox)
el.set_color('0.5')
el.set_alpha(0.1)
ax.scatter(model_params[0].squeeze().numpy(), model_params[1].numpy(), s=1)
"""
```

    "\nmodel.eval()\nfisher_matrix = get_fisher_matrix(model, criterion, X_obs[:10], y_obs[:10])\nmodel_params = [copy.deepcopy(p.data) for p in model.parameters()]\n\nfig = plt.figure(0)\nax = fig.add_subplot(111)\n\n\nel = plt.matplotlib.patches.Ellipse(\n    xy=[\n        model_params[0].squeeze().numpy()[0],\n        model_params[1].numpy()[0]\n    ],\n    width=2./fisher_matrix[0].squeeze().numpy()[0],\n    height=2./fisher_matrix[1].numpy()[0])\nax.add_artist(el)\n#el.set_clip_box(ax.bbox)\nel.set_color('0.5')\nel.set_alpha(0.1)\nax.scatter(model_params[0].squeeze().numpy(), model_params[1].numpy(), s=1)\n"

## Sequential samples

```python
sorted_idx = np.argsort(X_obs)
X_obs = X_obs[sorted_idx]
y_obs = y_obs[sorted_idx]
```

```python
plt.plot(X_obs[:10], y_obs[:10], ls="none", marker="o", color="b", label="task 1")
plt.plot(X_obs[10:20], y_obs[10:20], ls="none", marker="d", color="cyan", label="task 2")
plt.plot(X_obs[20:], y_obs[20:], ls="none", marker="s", color="magenta", label="task 3")
plt.plot(X_true, y_true, ls="-", lw=1, color="0.5", label="true")
plt.legend()
sns.despine(offset=10)
```

```python
model, model_simple = train_plot_compare(
    X_obs, y_obs, prefix="Seq",
    p=0.1, decay=1e-6, non_linearity=torch.nn.ReLU)
```

## Improvements

Before each forward pass

$$
\begin{equation}
\sigma_{\theta} = F_{\theta}^{-1/2}\\
\epsilon \sim \mathcal{N}(0, 1)\\
\theta = \theta + \sigma*\epsilon
\end{equation}
$$

```python
model, model_simple = train_plot_compare(
    X_obs, y_obs, prefix="Seq", uncertainty=100, l2=0.01,
    p=0.1, decay=1e-6, non_linearity=torch.nn.ReLU
)
```

```python

```


# Import Problem Instance
We start by importing a simple problem instance to demonstrate the tsplib reader.

```python
from tsplib95 import tsplib95
import itertools
import networkx as nx

instance = tsplib95.load_problem('./tsplib/ulysses16.tsp')

instance.comment
```

    'Odyssey of Ulysses (Groetschel/Padberg)'

Remember, this repository contains a small selection of TSP instances that you can use to test your algorithms.

| name | nodes | description |
|------|-------|-------------|
| ulysses16.tsp | 16 | Odyssey of Ulysses |
| ulysses7.tsp | 7 | subset of ulysses16 for testing purposes |
| bayg29.tsp | 29 | 29 Cities in Bavaria |
| bier127.tsp | 127 | 127 Biergaerten in Augsburg |
| bier20.tsp | 20 | subset of bier127 |
| brazil58.tsp | 58 | 58 cities in Brazil |
| ali535.tsp | 535 | 535 Airports around the globe |
| d18512.tsp | 18512 | 18512 places in Germany |

The following calls show the dimension = number of nodes of the problem, its node set and the edge weights. The functions `instance.get_nodes()` and `instance.get_edges()` are implemented as iterators, so you can loop over the nodes or edges. To get a list of nodes or edges, you have to explicitly construct one using `list(instance.get_nodes())`. Note that node counting may start at 1 for some instances while others use 0 as starting point. For convenience, we store the index of the first node as `first_node`.

```python
instance.dimension

instance.get_nodes()
print("List of nodes: ", list(instance.get_nodes()))

first_node = min(instance.get_nodes())
first_node

for i,j in instance.get_edges():
    if i >= j:
        continue
    print(f"edge {{ {i:2},{j:2} }} has weight {instance.wfunc(i,j):3}.")

```

    List of nodes:  [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
    edge {  1, 2 } has weight   6.
    edge {  1, 3 } has weight   5.
    edge {  1, 4 } has weight   3.
    edge {  1, 5 } has weight  11.
    edge {  1, 6 } has weight   8.
    edge {  1, 7 } has weight   7.
    edge {  1, 8 } has weight   1.
    edge {  1, 9 } has weight  12.
    edge {  1,10 } has weight   8.
    edge {  1,11 } has weight  26.
    edge {  1,12 } has weight   5.
    edge {  1,13 } has weight   5.
    edge {  1,14 } has weight   5.
    edge {  1,15 } has weight   7.
    edge {  1,16 } has weight   1.
    edge {  2, 3 } has weight   1.
    edge {  2, 4 } has weight   4.
    edge {  2, 5 } has weight  17.
    edge {  2, 6 } has weight  14.
    edge {  2, 7 } has weight  13.
    edge {  2, 8 } has weight   6.
    edge {  2, 9 } has weight  17.
    edge {  2,10 } has weight  13.
    edge {  2,11 } has weight  32.
    edge {  2,12 } has weight  11.
    edge {  2,13 } has weight  11.
    edge {  2,14 } has weight  11.
    edge {  2,15 } has weight  13.
    edge {  2,16 } has weight   7.
    edge {  3, 4 } has weight   5.
    edge {  3, 5 } has weight  16.
    edge {  3, 6 } has weight  13.
    edge {  3, 7 } has weight  12.
    edge {  3, 8 } has weight   6.
    edge {  3, 9 } has weight  16.
    edge {  3,10 } has weight  12.
    edge {  3,11 } has weight  31.
    edge {  3,12 } has weight  10.
    edge {  3,13 } has weight  10.
    edge {  3,14 } has weight  11.
    edge {  3,15 } has weight  12.
    edge {  3,16 } has weight   6.
    edge {  4, 5 } has weight  13.
    edge {  4, 6 } has weight  11.
    edge {  4, 7 } has weight  10.
    edge {  4, 8 } has weight   3.
    edge {  4, 9 } has weight  15.
    edge {  4,10 } has weight  11.
    edge {  4,11 } has weight  28.
    edge {  4,12 } has weight   8.
    edge {  4,13 } has weight   8.
    edge {  4,14 } has weight   8.
    edge {  4,15 } has weight   9.
    edge {  4,16 } has weight   5.
    edge {  5, 6 } has weight   4.
    edge {  5, 7 } has weight   6.
    edge {  5, 8 } has weight  11.
    edge {  5, 9 } has weight   8.
    edge {  5,10 } has weight   8.
    edge {  5,11 } has weight  16.
    edge {  5,12 } has weight   7.
    edge {  5,13 } has weight   7.
    edge {  5,14 } has weight   6.
    edge {  5,15 } has weight   4.
    edge {  5,16 } has weight  11.
    edge {  6, 7 } has weight   1.
    edge {  6, 8 } has weight   8.
    edge {  6, 9 } has weight   5.
    edge {  6,10 } has weight   4.
    edge {  6,11 } has weight  17.
    edge {  6,12 } has weight   3.
    edge {  6,13 } has weight   3.
    edge {  6,14 } has weight   3.
    edge {  6,15 } has weight   3.
    edge {  6,16 } has weight   8.
    edge {  7, 8 } has weight   7.
    edge {  7, 9 } has weight   5.
    edge {  7,10 } has weight   3.
    edge {  7,11 } has weight  18.
    edge {  7,12 } has weight   2.
    edge {  7,13 } has weight   2.
    edge {  7,14 } has weight   2.
    edge {  7,15 } has weight   3.
    edge {  7,16 } has weight   7.
    edge {  8, 9 } has weight  12.
    edge {  8,10 } has weight   8.
    edge {  8,11 } has weight  26.
    edge {  8,12 } has weight   5.
    edge {  8,13 } has weight   5.
    edge {  8,14 } has weight   5.
    edge {  8,15 } has weight   6.
    edge {  8,16 } has weight   2.
    edge {  9,10 } has weight   4.
    edge {  9,11 } has weight  15.
    edge {  9,12 } has weight   7.
    edge {  9,13 } has weight   7.
    edge {  9,14 } has weight   7.
    edge {  9,15 } has weight   8.
    edge {  9,16 } has weight  11.
    edge { 10,11 } has weight  19.
    edge { 10,12 } has weight   3.
    edge { 10,13 } has weight   4.
    edge { 10,14 } has weight   4.
    edge { 10,15 } has weight   6.
    edge { 10,16 } has weight   7.
    edge { 11,12 } has weight  20.
    edge { 11,13 } has weight  21.
    edge { 11,14 } has weight  20.
    edge { 11,15 } has weight  20.
    edge { 11,16 } has weight  25.
    edge { 12,13 } has weight   0.
    edge { 12,14 } has weight   1.
    edge { 12,15 } has weight   3.
    edge { 12,16 } has weight   5.
    edge { 13,14 } has weight   1.
    edge { 13,15 } has weight   3.
    edge { 13,16 } has weight   4.
    edge { 14,15 } has weight   2.
    edge { 14,16 } has weight   5.
    edge { 15,16 } has weight   7.

You have already seen how to draw a graph, here is the relevant code again.

```python
G = instance.get_graph()
if instance.is_depictable():
    pos = {i: instance.get_display(i) for i in instance.get_nodes()}
else:
    pos = nx.drawing.layout.spring_layout(G)
nx.draw_networkx_nodes(G, pos, node_color='#66a3ff', node_size=200)
nx.draw_networkx_labels(G, pos, font_weight='bold' )
nx.draw_networkx_edges(G, pos,  edge_color='#e6e6e6')
```

    /Users/ritter/Library/Caches/pypoetry/virtualenvs/assignments-ma4502-S2019-py3.7/lib/python3.7/site-packages/networkx/drawing/nx_pylab.py:579: MatplotlibDeprecationWarning:
    The iterable function was deprecated in Matplotlib 3.1 and will be removed in 3.3. Use np.iterable instead.
      if not cb.iterable(width):

    <matplotlib.collections.LineCollection at 0x1204a71d0>

# Implementing a multi commodity flow model in Gurobi
We will implement a multi commodity flow model using binary variables $x_{ij} \in \{0,1\}$ to indicate whether edge $\{i,j\}$ is being used in the tour. For the flow we use variables $y_{i,j,k}$ indicating the flow of commodity $k$ from node $v_0$ into node $k$ along arc $(i,j)$. Flow is only allowed on arcs that are used in the tour and the capacity of each such arc is $1$, we send $2$ units of flow from $v_0$ to $k$. For the formulation, we employ the condition that $i < j$ for any edge $\{i,j\}$ for the $x$-variables. The formulation looks like this:

\begin{align}
\min\;&\sum_{i,j \in E} c_{i,j} \cdot x_{i,j}\\
&\sum_{j \ne i} x_{i,j} = 2 \quad \text{for all nodes $i$}\\
&\sum_{j \ne k} y_{v_0,j,k} = 2 \quad \text{for all nodes $k \ne v_0$}\\
&\sum_{j \ne k} y_{j,v_0,k} = 0 \quad \text{for all nodes $k \ne v_0$}\\
&\sum_{j \ne i} y_{j,i,k} - \sum_{j \ne i} y_{i,j,k} = 0 \quad \text{for all nodes $k \ne v_0$ and all nodes $i \notin \{v_0,k\}$}\\
&y_{i,j,k} \le x_{i,j} \quad \text{for all arcs $(i,j)$ and all $k \ne v_0$}\\
&y_{j,i,k} \le x_{i,j} \quad \text{for all arcs $(i,j)$ and all $k \ne v_0$}\\
&x_{i,j} \in \{0,1\}\\
&y_{i,j,k} \ge 0 \quad \text{for all $k \ne 1$}
\end{align}

## Creating the variables
We start by creating the model and the variables and setting the objective.

```python
import gurobipy as grb

model = grb.Model(name="Multi Commodity Flow TSP Formulation")
model.reset()

x = model.addVars(filter(lambda e: e[0] < e[1], instance.get_edges()), vtype=grb.GRB.BINARY, name="x")

y = model.addVars(filter(lambda e: e[0] != e[1], instance.get_edges()), range(first_node+1, instance.dimension+1), name="y")

model.setObjective(sum(x[i,j]*instance.wfunc(i,j) for i,j in x.keys()))

```

    Academic license - for non-commercial use only

## Degree Constraints
We first add the degree constraints for the $x$-variables:
\begin{align}
&\sum_{j \ne i} x_{i,j} = 2 \quad \text{for all nodes $i$}
\end{align}

```python
for i in instance.get_nodes():
    model.addConstr(x.sum(i,'*') + x.sum('*',i) == 2, name=f"degree[{i}]")
```

## Flow Constraints
**Task 1:** Add the flow conservations constraints as well as the in- and out-flow constraints for $v_0$ for each commodity k:

```python
for k in itertools.islice(instance.get_nodes(), 1, None):
    model.addConstr(y.sum(first_node,'*',k) == 2, name=f"v0_outflow_for_commodity_{k}")
    model.addConstr(y.sum('*',first_node,k) == 0, name=f"v0_inflow_for_commodity_{k}")
    for i in filter(lambda e: e not in {first_node,k}, instance.get_nodes()):
        model.addConstr(y.sum('*',i,k) - y.sum(i,'*',k) == 0, name=f"{k}_flow_conservation_at_node_{i}")
```

## Capacity Constraints
**Task 2:** Add the capacity constraints:
\begin{align}
&y_{i,j,k} \le x_{i,j} \quad \text{for all arcs $(i,j)$ and all $k \ne v_0$}\\
&y_{j,i,k} \le x_{i,j} \quad \text{for all arcs $(i,j)$ and all $k \ne v_0$}
\end{align}

```python
for k in itertools.islice(instance.get_nodes(), 1, None):
    model.addConstrs( (y[i,j,k] <= x[i,j] for i,j in x.keys()), name=f"flow_bound_for_commodity_{k}_on_")
    model.addConstrs( (y[j,i,k] <= x[i,j] for i,j in x.keys()), name=f"reverse_flow_bound_for_commodity_{k}_on_")
```

## Starting the Optimization Process
Finally, we set the objective to minimization and call the optimizer.

```python
model.ModelSense = grb.GRB.MINIMIZE
model.reset()
model.optimize()
```

    Optimize a model with 3856 rows, 3720 columns and 14190 nonzeros
    Variable types: 3600 continuous, 120 integer (120 binary)
    Coefficient statistics:
      Matrix range     [1e+00, 1e+00]
      Objective range  [1e+00, 3e+01]
      Bounds range     [1e+00, 1e+00]
      RHS range        [2e+00, 2e+00]
    Presolve removed 240 rows and 225 columns
    Presolve time: 0.02s
    Presolved: 3616 rows, 3495 columns, 13305 nonzeros
    Variable types: 3375 continuous, 120 integer (120 binary)
    Found heuristic solution: objective 124.0000000

    Root relaxation: objective 7.050000e+01, 2402 iterations, 0.07 seconds

        Nodes    |    Current Node    |     Objective Bounds      |     Work
     Expl Unexpl |  Obj  Depth IntInf | Incumbent    BestBd   Gap | It/Node Time

         0     0   70.50000    0   10  124.00000   70.50000  43.1%     -    0s
    H    0     0                      71.0000000   70.50000  0.70%     -    0s

    Explored 1 nodes (2512 simplex iterations) in 0.17 seconds
    Thread count was 8 (of 8 available processors)

    Solution count 2: 71 124

    Optimal solution found (tolerance 1.00e-04)
    Best objective 7.100000000000e+01, best bound 7.100000000000e+01, gap 0.0000%

## Querying and Visualizing the Solution
Before we visualize our result, let us look at a few key figures of our solution.

```python
model.ObjVal

solution_edges = [(i,j) for i,j in x.keys() if x[i,j].x > 0.9]
solution_edges

flow_edges = [f"({i},{j},{k}): {y[i,j,k].x}" for i,j,k in y.keys() if y[i,j,k].x > 0]
flow_edges
```

    ['(1,3,2): 1.0',
     '(1,3,3): 1.0',
     '(1,3,4): 1.0',
     '(1,3,5): 1.0',
     '(1,3,6): 1.0',
     '(1,3,7): 1.0',
     '(1,3,8): 1.0',
     '(1,3,9): 1.0',
     '(1,3,10): 1.0',
     '(1,3,11): 1.0',
     '(1,3,12): 1.0',
     '(1,3,13): 1.0',
     '(1,3,14): 1.0',
     '(1,3,15): 1.0',
     '(1,3,16): 1.0',
     '(1,16,2): 1.0',
     '(1,16,3): 1.0',
     '(1,16,4): 1.0',
     '(1,16,5): 1.0',
     '(1,16,6): 1.0',
     '(1,16,7): 1.0',
     '(1,16,8): 1.0',
     '(1,16,9): 1.0',
     '(1,16,10): 1.0',
     '(1,16,11): 1.0',
     '(1,16,12): 1.0',
     '(1,16,13): 1.0',
     '(1,16,14): 1.0',
     '(1,16,15): 1.0',
     '(1,16,16): 1.0',
     '(2,3,3): 1.0',
     '(2,4,4): 1.0',
     '(2,4,5): 1.0',
     '(2,4,6): 1.0',
     '(2,4,7): 1.0',
     '(2,4,8): 1.0',
     '(2,4,9): 1.0',
     '(2,4,10): 1.0',
     '(2,4,11): 1.0',
     '(2,4,12): 1.0',
     '(2,4,13): 1.0',
     '(2,4,14): 1.0',
     '(2,4,15): 1.0',
     '(2,4,16): 1.0',
     '(3,2,2): 1.0',
     '(3,2,4): 1.0',
     '(3,2,5): 1.0',
     '(3,2,6): 1.0',
     '(3,2,7): 1.0',
     '(3,2,8): 1.0',
     '(3,2,9): 1.0',
     '(3,2,10): 1.0',
     '(3,2,11): 1.0',
     '(3,2,12): 1.0',
     '(3,2,13): 1.0',
     '(3,2,14): 1.0',
     '(3,2,15): 1.0',
     '(3,2,16): 1.0',
     '(4,2,2): 1.0',
     '(4,2,3): 1.0',
     '(4,8,5): 1.0',
     '(4,8,6): 1.0',
     '(4,8,7): 1.0',
     '(4,8,8): 1.0',
     '(4,8,9): 1.0',
     '(4,8,10): 1.0',
     '(4,8,11): 1.0',
     '(4,8,12): 1.0',
     '(4,8,13): 1.0',
     '(4,8,14): 1.0',
     '(4,8,15): 1.0',
     '(4,8,16): 1.0',
     '(5,6,2): 1.0',
     '(5,6,3): 1.0',
     '(5,6,4): 1.0',
     '(5,6,6): 1.0',
     '(5,6,7): 1.0',
     '(5,6,8): 1.0',
     '(5,6,14): 1.0',
     '(5,6,15): 1.0',
     '(5,11,9): 1.0',
     '(5,11,10): 1.0',
     '(5,11,11): 1.0',
     '(5,11,12): 1.0',
     '(5,11,13): 1.0',
     '(5,11,16): 1.0',
     '(6,5,5): 1.0',
     '(6,5,9): 1.0',
     '(6,5,10): 1.0',
     '(6,5,11): 1.0',
     '(6,5,12): 1.0',
     '(6,5,13): 1.0',
     '(6,5,16): 1.0',
     '(6,7,2): 1.0',
     '(6,7,3): 1.0',
     '(6,7,4): 1.0',
     '(6,7,7): 1.0',
     '(6,7,8): 1.0',
     '(6,7,14): 1.0',
     '(6,7,15): 1.0',
     '(7,6,5): 1.0',
     '(7,6,6): 1.0',
     '(7,6,9): 1.0',
     '(7,6,10): 1.0',
     '(7,6,11): 1.0',
     '(7,6,12): 1.0',
     '(7,6,13): 1.0',
     '(7,6,16): 1.0',
     '(7,14,2): 1.0',
     '(7,14,3): 1.0',
     '(7,14,4): 1.0',
     '(7,14,8): 1.0',
     '(7,14,14): 1.0',
     '(7,14,15): 1.0',
     '(8,4,2): 1.0',
     '(8,4,3): 1.0',
     '(8,4,4): 1.0',
     '(8,15,5): 1.0',
     '(8,15,6): 1.0',
     '(8,15,7): 1.0',
     '(8,15,9): 1.0',
     '(8,15,10): 1.0',
     '(8,15,11): 1.0',
     '(8,15,12): 1.0',
     '(8,15,13): 1.0',
     '(8,15,14): 1.0',
     '(8,15,15): 1.0',
     '(8,15,16): 1.0',
     '(9,10,10): 1.0',
     '(9,10,12): 1.0',
     '(9,10,13): 1.0',
     '(9,10,16): 1.0',
     '(9,11,2): 1.0',
     '(9,11,3): 1.0',
     '(9,11,4): 1.0',
     '(9,11,5): 1.0',
     '(9,11,6): 1.0',
     '(9,11,7): 1.0',
     '(9,11,8): 1.0',
     '(9,11,11): 1.0',
     '(9,11,14): 1.0',
     '(9,11,15): 1.0',
     '(10,9,2): 1.0',
     '(10,9,3): 1.0',
     '(10,9,4): 1.0',
     '(10,9,5): 1.0',
     '(10,9,6): 1.0',
     '(10,9,7): 1.0',
     '(10,9,8): 1.0',
     '(10,9,9): 1.0',
     '(10,9,11): 1.0',
     '(10,9,14): 1.0',
     '(10,9,15): 1.0',
     '(10,12,12): 1.0',
     '(10,12,13): 1.0',
     '(10,12,16): 1.0',
     '(11,5,2): 1.0',
     '(11,5,3): 1.0',
     '(11,5,4): 1.0',
     '(11,5,5): 1.0',
     '(11,5,6): 1.0',
     '(11,5,7): 1.0',
     '(11,5,8): 1.0',
     '(11,5,14): 1.0',
     '(11,5,15): 1.0',
     '(11,9,9): 1.0',
     '(11,9,10): 1.0',
     '(11,9,12): 1.0',
     '(11,9,13): 1.0',
     '(11,9,16): 1.0',
     '(12,10,2): 1.0',
     '(12,10,3): 1.0',
     '(12,10,4): 1.0',
     '(12,10,5): 1.0',
     '(12,10,6): 1.0',
     '(12,10,7): 1.0',
     '(12,10,8): 1.0',
     '(12,10,9): 1.0',
     '(12,10,10): 1.0',
     '(12,10,11): 1.0',
     '(12,10,14): 1.0',
     '(12,10,15): 1.0',
     '(12,13,13): 1.0',
     '(12,13,16): 1.0',
     '(13,12,2): 1.0',
     '(13,12,3): 1.0',
     '(13,12,4): 1.0',
     '(13,12,5): 1.0',
     '(13,12,6): 1.0',
     '(13,12,7): 1.0',
     '(13,12,8): 1.0',
     '(13,12,9): 1.0',
     '(13,12,10): 1.0',
     '(13,12,11): 1.0',
     '(13,12,12): 1.0',
     '(13,12,14): 1.0',
     '(13,12,15): 1.0',
     '(13,16,16): 1.0',
     '(14,7,5): 1.0',
     '(14,7,6): 1.0',
     '(14,7,7): 1.0',
     '(14,7,9): 1.0',
     '(14,7,10): 1.0',
     '(14,7,11): 1.0',
     '(14,7,12): 1.0',
     '(14,7,13): 1.0',
     '(14,7,16): 1.0',
     '(14,15,2): 1.0',
     '(14,15,3): 1.0',
     '(14,15,4): 1.0',
     '(14,15,8): 1.0',
     '(14,15,15): 1.0',
     '(15,8,2): 1.0',
     '(15,8,3): 1.0',
     '(15,8,4): 1.0',
     '(15,8,8): 1.0',
     '(15,14,5): 1.0',
     '(15,14,6): 1.0',
     '(15,14,7): 1.0',
     '(15,14,9): 1.0',
     '(15,14,10): 1.0',
     '(15,14,11): 1.0',
     '(15,14,12): 1.0',
     '(15,14,13): 1.0',
     '(15,14,14): 1.0',
     '(15,14,16): 1.0',
     '(16,13,2): 1.0',
     '(16,13,3): 1.0',
     '(16,13,4): 1.0',
     '(16,13,5): 1.0',
     '(16,13,6): 1.0',
     '(16,13,7): 1.0',
     '(16,13,8): 1.0',
     '(16,13,9): 1.0',
     '(16,13,10): 1.0',
     '(16,13,11): 1.0',
     '(16,13,12): 1.0',
     '(16,13,13): 1.0',
     '(16,13,14): 1.0',
     '(16,13,15): 1.0']

For debugging purposes, it might be helpful to export the model held by Gurobi into a human-readable format:

```python
model.write('test.lp')
```

Finally, let us visualize the solution using NetworkX. In this case, we need to prescribe positions and draw the nodes and two layers of edges separately.

```python
if instance.is_depictable():
    pos = {i: instance.get_display(i) for i in instance.get_nodes()}
else:
    pos = nx.drawing.layout.spring_layout(G)
nx.draw_networkx_nodes(G, pos, node_color='#66a3ff', node_size=500)
nx.draw_networkx_labels(G, pos, font_weight='bold' )
nx.draw_networkx_edges(G, pos,  edge_color='#e6e6e6')
nx.draw_networkx_edges(G, pos,  edgelist=solution_edges, edge_color='#ffa31a', width=4)
```

**Task 3:** Test the model for different instances. How does it compare to the MTZ formulation?

```python

```


 ILI285 - Computación Científica I  / INF285 - Computación Científica

 Roots of 1D equations

  [S]cientific [C]omputing [T]eam

 Version: 1.32

## Table of Contents
* [Introduction](#intro)
* [Bisection Method](#bisection)
* [Cobweb Plot](#cobweb)
* [Fixed Point Iteration](#fpi)
* [Newton Method](#nm)
* [Wilkinson Polynomial](#wilkinson)
* [Acknowledgements](#acknowledgements)

```python
import numpy as np
import matplotlib.pyplot as plt
import sympy as sym
%matplotlib inline
from ipywidgets import interact
from ipywidgets import widgets
sym.init_printing()
from scipy import optimize
```

## Introduction

Hello again! In this document we're going to learn how to find a 1D equation's solution using numerical methods. First, let's start with the definition of a root:

Definition: The function $f(x)$ has a root in $x = r$ if $f(r) = 0$.

An example: Let's say we want to solve the equation $x + \log(x) = 3$. We can rearrange the equation: $x + \log(x) - 3 = 0$. That way, to find its solution we can find the root of $f(x) = x + \log(x) - 3$. Now let's study some numerical methods to solve these kinds of problems.

Defining a function $f(x)$

```python
f = lambda x: x+np.log(x)-3
```

Finding $r$ using sympy

```python
y = sym.Symbol('y')
fsym = lambda y: y+sym.log(y)-3
r_all=sym.solve(sym.Eq(fsym(y), 0), y)
r=r_all[0].evalf()
print(r)
print(r_all)
```

    2.20794003156932
    [LambertW(exp(3))]

```python
def find_root_manually(r=2.0):
    x = np.linspace(1,3,1000)
    plt.figure(figsize=(8,8))
    plt.plot(x,f(x),'b-')
    plt.grid()
    plt.ylabel('$f(x)$',fontsize=16)
    plt.xlabel('$x$',fontsize=16)
    plt.title('What is r such that $f(r)='+str(f(r))+'$? $r='+str(r)+'$',fontsize=16)
    plt.plot(r,f(r),'k.',markersize=20)
    plt.show()
interact(find_root_manually,r=(1e-5,3,1e-3))
```

    interactive(children=(FloatSlider(value=2.0, description='r', max=3.0, min=1e-05, step=0.001), Output()), _dom…

    <function __main__.find_root_manually(r=2.0)>

## Bisection Method

The bisection method finds the root of a function $f$, where $f$ is a **continuous** function.
If we want to know if this has a root, we have to check if there is an interval $[a,b]$ for which
$f(a)\cdot f(b) < 0$. When these 2 conditions are satisfied, it means that there is a value $r$, between $a$ and $b$, for which $f(r) = 0$. To summarize how this method works, start with the aforementioned interval (checking that there's a root in it), and split it into two smaller intervals $[a,c]$ and $[c,b]$. Then, check which of the two intervals contains a root. Keep splitting each "eligible" interval until the algorithm converges or the tolerance is surpassed.

```python
def bisect(f, a, b, tol=1e-8):
    fa = f(a)
    fb = f(b)
    i = 0
    # Just checking if the sign is not negative => not root  necessarily
    if np.sign(f(a)*f(b)) >= 0:
        print('f(a)f(b)<0 not satisfied!')
        return None

    #Printing the evolution of the computation of the root
    print(' i |     a     |     c     |     b     |     fa    |     fc     |     fb     |   b-a')
    print('----------------------------------------------------------------------------------------')

    while(b-a)/2 > tol:
        c = (a+b)/2.
        fc = f(c)
        print('%2d | %.7f | %.7f | %.7f | %.7f | %.7f | %.7f | %.7f' %
              (i+1, a, c, b, fa, fc, fb, b-a))
        # Did we find the root?
        if fc == 0:
            print('f(c)==0')
            break
        elif np.sign(fa*fc) < 0:
            b = c
            fb = fc
        else:
            a = c
            fa = fc
        i += 1

    xc = (a+b)/2.
    return xc
```

```python
## Finding a root of cos(x). What about if you change the interval?
#f = lambda x: np.cos(x)
## Another function
#f = lambda x: x**3-2*x**2+(4/3)*x-(8/27)
## Computing the cubic root of 7.
#f = lambda x: x**3-7
#bisect(f,0,2)
f = lambda x: x*np.exp(x)-3
#f2 = lambda x: np.cos(x)-x
bisect(f,0,3,tol=1e-13)
```

It's very important to define a concept called **convergence rate**.
This rate shows how fast the convergence of a method is at a specified point.

The convergence rate for the bisection is always 0.5 because this method uses the half of the interval for each iteration.

## Cobweb Plot

```python
def cobweb(x,g=None):
    min_x = np.amin(x)
    max_x = np.amax(x)

    plt.figure(figsize=(10,10))
    ax = plt.axes()
    plt.plot(np.array([min_x,max_x]),np.array([min_x,max_x]),'b-')
    for i in np.arange(x.size-1):
        delta_x = x[i+1]-x[i]
        head_length =  np.abs(delta_x)*0.04
        arrow_length = delta_x-np.sign(delta_x)*head_length
        ax.arrow(x[i], x[i], 0, arrow_length, head_width=1.5*head_length, head_length=head_length, fc='k', ec='k')
        ax.arrow(x[i], x[i+1], arrow_length, 0, head_width=1.5*head_length, head_length=head_length, fc='k', ec='k')

    if g!=None:
        y = np.linspace(min_x,max_x,1000)
        plt.plot(y,g(y),'r')

    plt.title('Cobweb diagram')
    plt.grid(True)
    plt.show()
```

## Fixed Point Iteration

To learn about the Fixed-Point Iteration we will first learn about the concept of Fixed Point.

A Fixed Point of a function $g$ is a real number $r$, where $g(r) = r$

The Fixed-Point Iteration is based in the Fixed Point concept and works like this to find the root of a function:

\begin{equation} x_{0} = initial\_guess \\ x_{i+1} = g(x_{i})\end{equation}

To find an equation's solution using this method you'll have to move around some things to rearrange the equation in the form $x = g(x)$. That way, you'll be iterating over the funcion $g(x)$, but you will **not** find $g$'s root, but $f(x) = g(x) - x$ (or $f(x) = x - g(x)$)'s root. In our following example, we'll find the solution to $f(x) = x - \cos(x)$ by iterating over the funcion $g(x) = \cos(x)$.

```python
def fpi(g, x0, k, flag_cobweb=False):
    x = np.empty(k+1)
    x[0] = x0
    error_i = np.nan

    print(' i |     x(i)     |    x(i+1)    ||x(i+1)-x(i)| | e_i/e_{i-1}')
    print('--------------------------------------------------------------')

    for i in range(k):
        x[i+1] = g(x[i])
        error_iminus1 = error_i
        error_i = abs(x[i+1]-x[i])
        print('%2d | %.10f | %.10f | %.10f | %.10f' %
              (i,x[i],x[i+1],error_i,error_i/error_iminus1))
    if flag_cobweb:
        cobweb(x,g)
    return x[-1]
```

```python
g = lambda x: np.cos(x)
fpi2(g, 2, 20, True)
```

Let's quickly explain the Cobweb Diagram we have here. The blue line is the function $x$ and the red is the function $g(x)$. The point in which they meet is $g$'s fixed point. In this particular example, we start at $y = x = 1.5$ (the top right corner) and then we "jump" **vertically** to $y = \cos(1.5) \approx 0.07$. After this, we jump **horizontally** to $x = \cos(1.5) \approx 0.07$. Then, we jump again **vertically** to $y = \cos\left(\cos(1.5)\right) \approx 0.997$ and so on. See the pattern here? We're just iterating over $x = \cos(x)$, getting closer to the center of the diagram where the fixed point resides, in $x \approx 0.739$.

It's very important to mention that the algorithm will converge only if the rate of convergence $S < 1$, where $S = \left| g'(r) \right|$. If you want to use this method, you'll have to construct $g(x)$ starting from $f(x)$ accordingly. In this example, $g(x) = \cos(x) \Rightarrow g'(x) = -\sin(x)$ and $|-\sin(0.739)| \approx 0.67$.

### Another example. Source: https://divisbyzero.com/2008/12/18/sharkovskys-theorem/amp/?__twitter_impression=true

```python
g = lambda x: -(3/2)*x**2+(11/2)*x-2
gp = lambda x: -3*x+11/2

a=-1/2.7
g2 = lambda x: x+a*(x-g(x))

#x=np.linspace(2,3,100)
#plt.plot(x,gp(x),'-')
#plt.plot(x,gp(x)*0+1,'r-')
#plt.plot(x,gp(x)*0-1,'g-')
#plt.grid(True)
#plt.show()
fpi(g2, 2.45, 12, True)
```

## Newton's Method

For this method, we want to iteratively find some function $f(x)$'s root, that is, the number $r$ for which $f(r) = 0$. The algorithm is as follows:

\begin{equation} x_0 = initial\_guess \end{equation}
\begin{equation} x_{i+1} = x_i - \cfrac{f(x_i)}{f'(x_i)} \end{equation}

which means that you won't be able to find $f$'s root if $f'(r) = 0$. In this case, you would have to use the modified version of this method, but for now let's focus on the unmodified version first. Newton's (unmodified) method is said to have quadratic convergence.

```python
def newton_method(f, fp, x0, rel_error=1e-8, m=1):
    #Initialization of hybrid error and absolute
    hybrid_error = 100
    error_i = np.inf

    print('i |     x(i)     |     x(i+1)   |      |x(i+1)-x(i)|     |  e_i/e_{i-1} | e_i/e_{i-1}^2')
    print('----------------------------------------------------------------------------------------')

    #Iteration counter
    i = 1
    while (hybrid_error > rel_error and hybrid_error < 1e12 and i < 1e4):
        #Newton's iteration
        x1 = x0-m*f(x0)/fp(x0)

        #Checking if root was found
        if f(x1) == 0.0:
            hybrid_error = 0.0
            break

        #Computation of hybrid error
        hybrid_error = abs(x1-x0)/np.max([abs(x1),1e-12])

        #Computation of absolute error
        error_iminus1 = error_i
        error_i = abs(x1-x0)

        #Increasing counter
        i += 1

        #Showing some info
        print("%d | %.10f | %.10f | %.20f | %.10f | %.10f" %
              (i, x0, x1, error_i, error_i/error_iminus1, error_i/(error_iminus1**2)))
        #Updating solution
        x0 = x1

    #Checking if solution was obtained
    if hybrid_error < rel_error:
        return x1
    elif i>=1e4:
        print('Newton''s Method diverged. Too many iterations!!')
        return None
    else:
        print('Newton''s Method diverged!')
        return None
```

```python
f = lambda x: np.sin(x)
fp = lambda x: np.cos(x) # the derivative of f
newton_method(f, fp, 3.1,rel_error=1e-14)
```

```python
f = lambda x: x**2
fp = lambda x: 2*x # the derivative of f
newton_method(f, fp, 3.1,rel_error=1e-2, m=2)
```

## Wilkinson Polynomial
https://en.wikipedia.org/wiki/Wilkinson%27s_polynomial

**Final question: Why is the root far far away from $16$?**

```python
x = sym.symbols('x', reals=True)
W=1
for i in np.arange(1,21):
        W*=(x-i)
W # Printing W nicely
```

```python
# Expanding the Wilkinson polynomial
We=sym.expand(W)
We
```

```python
# Just computiong the derivative
Wep=sym.diff(We,x)
Wep
```

```python
# Lamdifying the polynomial to be used with sympy
P=sym.lambdify(x,We)
Pp=sym.lambdify(x,Wep)
```

```python
# Using scipy function to compute a root
root = optimize.newton(P,16)
print(root)
```

# Acknowledgements
* _Material created by professor Claudio Torres_ (`ctorres@inf.utfsm.cl`) _and assistants: Laura Bermeo, Alvaro Salinas, Axel Simonsen and Martín Villanueva. DI UTFSM. March 2016._ v1.1.
* _Update April 2020 - v1.32 - C.Torres_ : Re-ordering the notebook.

# Propose Classwork
Build a FPI such that given $x$ computes $\displaystyle \frac{1}{x}$. Write down your solution below or go and see the [solution](#sol1)

```python
print('Please try to think and solve before you see the solution!!!')
```

# In class

### From the textbook

```python
g1 = lambda x: 1-x**3
g2 = lambda x: (1-x)**(1/3)
g3 = lambda x: (1+2*x**3)/(1+3*x**2)
fpi(g3, 0.5, 10, True)
```

```python
g1p = lambda x: -3*x**2
g2p = lambda x: -(1/3)*(1-x)**(-2/3)
g3p = lambda x: ((1+3*x**2)*(6*x**2)-(1+2*x**3)*6*x)/((1+3*x**2)**2)
r=0.6823278038280194
print(g3p(r))
```

### Adding another implementation of FPI including and extra column for analyzing quadratic convergence

```python
def fpi2(g, x0, k, flag_cobweb=False):
    x = np.empty(k+1)
    x[0] = x0
    error_i = np.inf

    print(' i |     x(i)     |    x(i+1)    ||x(i+1)-x(i)| | e_i/e_{i-1} | e_i/e_{i-1}^2')
    print('-----------------------------------------------------------------------------')

    for i in range(k):
        x[i+1] = g(x[i])
        error_iminus1 = error_i
        error_i = abs(x[i+1]-x[i])
        print('%2d | %.10f | %.10f | %.10f | %.10f | %.10f' %
              (i,x[i],x[i+1],error_i,error_i/error_iminus1,error_i/(error_iminus1**2)))
    if flag_cobweb:
        cobweb(x,g)
    return x[-1]
```

Which function shows quadratic convergence? Why?

```python
g1 = lambda x: (4./5.)*x+1./x
g2 = lambda x: x/2.+5./(2*x)
g3 = lambda x: (x+5.)/(x+1)
fpi2(g1, 3.0, 30, True)
```

### Building a FPI to compute the cubic root of 7

```python
# What is 'a'? Can we find another 'a'?
a = -3*(1.7**2)
print(a)
```

```python
f = lambda x: x**3-7
g =  lambda x: f(x)/a+x
r=fpi(g, 1.7, 14, True)
print(f(r))
```

### Playing with some roots

```python
f = lambda x: 8*x**4-12*x**3+6*x**2-x
fp = lambda x: 32*x**3-36*x**2+12*x-1

x = np.linspace(-1,1,1000)
plt.figure(figsize=(10,10))

plt.title('What are we seeing with the semiloigy plot? Is this function differentiable?')
plt.semilogy(x,np.abs(f(x)),'b-')
plt.semilogy(x,np.abs(fp(x)),'r-')
plt.grid()
plt.ylabel('$f(x)$',fontsize=16)
plt.xlabel('$x$',fontsize=16)
plt.show()
```

```python
r=newton_method(f, fp, 0.3, rel_error=1e-8, m=1)
print([r,f(r)])
# Is this showing quadratic convergence? If not, can you fix it?
```

# Solutions

Problem: Build a FPI such that given $x$ computes $\displaystyle \frac{1}{x}$

```python
# We are finding the 1/a
# Solution code:
a = 2.1
g = lambda x: 2*x-a*x**2
gp = lambda x: 2-2*a*x
r=fpi2(g, 0.7, 7, flag_cobweb=True)
print([r,1/a])
# Are we seeing quadratic convergence?
```

### What is this plot telling us?

```python
xx=np.linspace(0.2,0.8,1000)
plt.figure(figsize=(10,10))
plt.plot(xx,g(xx),'-',label=r'$g(x)$')
plt.plot(xx,gp(xx),'r-',label=r'$gp(x)$')
plt.plot(xx,xx,'g-',label=r'$x$')
plt.plot(xx,0*xx+1,'k--')
plt.plot(xx,0*xx-1,'k--')
plt.legend(loc='best')
plt.grid()
plt.show()
```

# In-Class 20200420

```python
g = lambda x: (x**2-1.)/2.
gh = lambda x: x+0.7*(x-g(x))
#fpi(g, 2, 15, True)
fpi2(gh, 0, 15, True)
```

# In-Class 20200423

```python
g1 = lambda x: np.cos(x)
g2 = lambda x: np.cos(x)**2
g3 = lambda x: np.sin(x)
g4 = lambda x: 1-5*x+(15/2)*x**2-(5/2)*x**3 # (0,1) y (1,2)

interact(lambda x0, n: fpi2(g4, x0, n, True),x0=widgets.FloatSlider(min=0, max=3, step=0.01, value=0), n=widgets.IntSlider(min=10, max=100, step=1, value=10))
```

```python

```


# Synchronize homework files

```
! rm -rf sample_data; curl -s https://raw.githubusercontent.com/mposa/MEAM517/master/sync_hw.py > sync_hw.py
files = ["find_throwing_trajectory.py", "dynamics_constraints.py", "kinematic_constraints.py", "planar_arm.urdf"]

from sync_hw import sync_hw
sync_hw(5, files, False)
```

# Install Drake and set up a server for the visualizer

```
# Install drake. We are using the lqr controller in drake as the final stabilizing controller.
# The installation process will take about 2 minutes but it's only required in the start of the Colab's virtual machine.
!curl -s https://raw.githubusercontent.com/mposa/MEAM517/master/colab_drake_setup.py > colab_drake_setup.py
from colab_drake_setup import setup
setup()

# Install pyngrok.
server_args = []
!pip install pyngrok
server_args = ['--ngrok_http_tunnel']

# Start a single meshcat server instance to use for the remainder of this notebook.
from meshcat.servers.zmqserver import start_zmq_server_as_subprocess
proc, zmq_url, web_url = start_zmq_server_as_subprocess(server_args=server_args)
```

# Problem 4 - Trajectory Optimization for a planar arm

## Compute the end effector (the ball) position and velocity using symbolic variables

This is how we get the expression in `kinematic_constraints.py`

```
# Computes the end effector velocity as xdot, ydot J_q (pos) * qdot

from sympy import *
import numpy as np

l, q0, q1, q2, qdot0, qdot1, qdot2 = symbols('l q0 q1 q2 qdot0 qdot1 qdot2', real=True)
pos = l * Matrix([-sin(q0), -cos(q0)]) + \
      l * Matrix([-sin(q0 + q1), -cos(q0 + q1)]) + \
      l * Matrix([-sin(q0 + q1 + q2), -cos(q0 + q1 + q2)])
vel = Matrix(pos).jacobian([q0, q1, q2]) @ np.reshape(np.array([qdot0, qdot1, qdot2]), (3,1))

print('pos (x_ball, z_ball):\n', pos)
print('')
print('vel (x_ball dot, z_ball dot):\n', vel)
```

## Run trajectory optimization to find the optimal trajectory

```
import importlib
import find_throwing_trajectory
importlib.reload(find_throwing_trajectory)
from find_throwing_trajectory import find_throwing_trajectory

N = 7
initial_state = np.zeros(6)
tf = 5.0
distance = 25.0
x_traj, u_traj, prog = find_throwing_trajectory(N, initial_state, distance, tf)
```

## Visualize the optimal trajectory
Open the visualizer by clicking on the url in the output of this cell.

```
%matplotlib notebook
import matplotlib.pyplot as plt
import numpy as np
import time
from pydrake.systems.framework import DiagramBuilder
from pydrake.systems.analysis import Simulator
from pydrake.systems.drawing import plot_system_graphviz

from pydrake.all import (
  ConnectMeshcatVisualizer, DiagramBuilder,
  Simulator
)

from pydrake.geometry import (
  SceneGraph, ConnectDrakeVisualizer
)

from pydrake.common import FindResourceOrThrow
from pydrake.multibody.plant import MultibodyPlant
from pydrake.multibody.parsing import Parser
from pydrake.systems.rendering import MultibodyPositionToGeometryPose
from pydrake.systems.primitives import (
  TrajectorySource,
  Demultiplexer,
  ConstantVectorSource
)

# Create a MultibodyPlant for the arm
file_name = "planar_arm.urdf"
builder = DiagramBuilder()
scene_graph = builder.AddSystem(SceneGraph())
planar_arm = builder.AddSystem(MultibodyPlant(0.0))
planar_arm.RegisterAsSourceForSceneGraph(scene_graph)
Parser(plant=planar_arm).AddModelFromFile(file_name)
planar_arm.Finalize()

# Create meshcat
visualizer = ConnectMeshcatVisualizer(
    builder,
    scene_graph,
    scene_graph.get_pose_bundle_output_port(),
    zmq_url=zmq_url,
    server_args=server_args)

x_traj_source = builder.AddSystem(TrajectorySource(x_traj))
u_traj_source = builder.AddSystem(TrajectorySource(u_traj))

demux = builder.AddSystem(Demultiplexer(np.array([3, 3])))
to_pose = builder.AddSystem(MultibodyPositionToGeometryPose(planar_arm))
zero_inputs = builder.AddSystem(ConstantVectorSource(np.zeros(3)))

builder.Connect(zero_inputs.get_output_port(), planar_arm.get_actuation_input_port())
builder.Connect(x_traj_source.get_output_port(), demux.get_input_port())
builder.Connect(demux.get_output_port(0), to_pose.get_input_port())
builder.Connect(to_pose.get_output_port(), scene_graph.get_source_pose_port(planar_arm.get_source_id()))

ConnectDrakeVisualizer(builder, scene_graph);

diagram = builder.Build()
diagram.set_name("diagram")

visualizer.load()
print("\n!!!Open the visualizer by clicking on the URL above!!!")

# Visualize the motion for `n_playback` times
n_playback = 3
for i in range(n_playback):
  # Set up a simulator to run this diagram.
  simulator = Simulator(diagram)
  simulator.Initialize()
  simulator.set_target_realtime_rate(1)
  simulator.AdvanceTo(tf);
  time.sleep(2)
```