# Simplification Solutions

```python
from sympy import *
x, y, z = symbols('x y z')
init_printing()
```

For each exercise, fill in the function according to its docstring.

## Polynomial/Rational Function Simplification

In each exercise, apply specific simplification functions to get the desired result.

```python
def polysimp1(expr):
    """
    >>> polysimp1(cos(x)*sin(x) + cos(x))
    (sin(x) + 1)*cos(x)
    >>> polysimp1(cos(x)*sin(x) + cos(x) + 1)
    (sin(x) + 1)*cos(x) + 1
    """
    return collect(expr, cos(x))
```

```python
polysimp1(cos(x)*sin(x) + cos(x))
```

```python
polysimp1(cos(x)*sin(x) + cos(x) + 1)
```

```python
def polysimp2(expr):
    """
    >>> polysimp2((2*x + 1)/(x**2 + x))
    1/(x + 1) + 1/x
    >>> polysimp2((x**2 + 3*x + 1)/(x**3 + 2*x**2 + x))
    1/(x**2 + 2*x + 1) + 1/x
    """
    return expand(apart(expr))
```

```python
polysimp2((2*x + 1)/(x**2 + x))
```

```python
polysimp2((x**2 + 3*x + 1)/(x**3 + 2*x**2 + x))
```

## Powers

In each exercise, apply specific simplification functions to get the desired result.

```python
def powersimp1(expr):
    """
    >>> powersimp1(exp(x)*(exp(y) + 1))
    exp(x) + exp(x + y)
    """
    return powsimp(expand(expr))
```

```python
powersimp1(exp(x)*(exp(y) + 1))
```

```python
def powersimp2(expr):
    """
    >>> powersimp2(2**x*x**x)
    (2*x)**x
    >>> powersimp2(x**x*x**x)
    (x**2)**x
    """
    return powsimp(expr, force=True)
```

```python
powersimp2(2**x*x**x)
```

```python
powersimp2(x**x*x**x)
```

```python
def powersimp3(expr):
    """
    >>> a, b, c = symbols('a b c')
    >>> powersimp3((a**b)**c)
    a**(b*c)
    >>> powersimp3((a**b)**(c + 1))
    a**(b*c + b)
    """
    return powdenest(expand_power_exp(expr), force=True)
```

```python
a, b, c = symbols('a b c')
```

```python
powersimp3((a**b)**c)
```

```python
powersimp3((a**b)**(c + 1))
```

## Logs

```python
def logsimp1(expr):
    """
    >>> a, b = symbols('a b', positive=True)
    >>> logsimp1(log(x**y*a**b))
    y*log(x) + log(a**b)
    >>> logsimp1(log(x*y*a*b))
    log(x) + log(y) + log(a*b)
    """
    return logcombine(expand_log(expr, force=True))
```

```python
a, b = symbols('a b', positive=True)
```

```python
logsimp1(log(x**y*a**b))
```

```python
logsimp1(log(x*y*a*b))
```

## Miscellaneous

```python
def miscsimp1(expr):
    """
    >>> miscsimp1(sin(x + y))
    2*(-tan(x/2)**2 + 1)*tan(y/2)/((tan(x/2)**2 + 1)*(tan(y/2)**2 + 1)) + 2*(-tan(y/2)**2 + 1)*tan(x/2)/((tan(x/2)**2 + 1)*(tan(y/2)**2 + 1))
    """
    return expand_trig(expr).rewrite(tan)
```

```python
miscsimp1(sin(x + y))
```

```python
def miscsimp2(expr):
    """
    >>> miscsimp2(gamma(x + 4))
    x**4*gamma(x) + 6*x**3*gamma(x) + 11*x**2*gamma(x) + 6*x*gamma(x)
    """
    return expand(expand_func(expr))
```

```python
miscsimp2(gamma(x + 4))
```

## Continued Fractions

If we do not cover this, see http://asmeurer.github.io/scipy-2014-tutorial/html/tutorial/simplification.html#example-continued-fractions.

```python
def list_to_frac(l):
    expr = Integer(0)
    for i in reversed(l[1:]):
        expr += i
        expr = 1/expr
    return l[0] + expr
```

```python
a0, a1, a2, a3, a4 = symbols('a0:5')
```

Determine the list used to create the continued fraction $$\frac{a_{0} a_{1} a_{2} a_{3} a_{4} + a_{0} a_{1} a_{2} + a_{0} a_{3} a_{4} + a_{0} + a_{1} a_{2} a_{3} + a_{1} a_{3} a_{4} + a_{1} + a_{3}}{a_{0} a_{1} a_{2} a_{4} + a_{0} a_{4} + a_{1} a_{2} + a_{1} a_{4} + 1}.$$

```python
def continued_frac():
    """
    Determine the original list used to create the fraction.

    Return the original list from this function.

    >>> orig_frac = (a0*a1*a2*a3*a4 + a0*a1*a2 + a0*a3*a4 + a0 + a1*a2*a3 + a1*a3*a4 + a1 + a3)/(a0*a1*a2*a4 + a0*a4 + a1*a2 + a1*a4 + 1)
    >>> pprint(orig_frac)
    a₀⋅a₁⋅a₂⋅a₃⋅a₄ + a₀⋅a₁⋅a₂ + a₀⋅a₃⋅a₄ + a₀ + a₁⋅a₂⋅a₃ + a₁⋅a₃⋅a₄ + a₁ + a₃
    ─────────────────────────────────────────────────────────────────────────
                     a₀⋅a₁⋅a₂⋅a₄ + a₀⋅a₄ + a₁⋅a₂ + a₁⋅a₄ + 1
    >>> cancel(list_to_frac(continued_frac())) == orig_frac
    True
    """
    return [a3, a4, a0, a2, a1]
```

```python
orig_frac = (a0*a1*a2*a3*a4 + a0*a1*a2 + a0*a3*a4 + a0 + a1*a2*a3 + a1*a3*a4 + a1 + a3)/(a0*a1*a2*a4 + a0*a4 + a1*a2 + a1*a4 + 1)
```

```python
orig_frac
```

```python
cancel(list_to_frac(continued_frac())) == orig_frac
```

```python

```


# Small noise approximation - mRNA level

(c) 2017 Manuel Razo. This work is licensed under a [Creative Commons Attribution License CC-BY 4.0](https://creativecommons.org/licenses/by/4.0/). All code contained herein is licensed under an [MIT license](https://opensource.org/licenses/MIT).

```python
import glob
import pickle
import dill
# Our numerical workhorses
import numpy as np
import scipy.optimize
import scipy.special
import scipy.integrate
import mpmath

# To compute symbolic expressions
import sympy

# Import matplotlib stuff for plotting
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import matplotlib as mpl

# Seaborn, useful for graphics
import seaborn as sns

# Import the utils for this project
import chann_cap_utils as chann_cap

# Set PBoC plotting style
chann_cap.set_plotting_style()

# Magic function to make matplotlib inline; other style specs must come AFTER
%matplotlib inline

# This enables SVG graphics inline (only use with static plots (non-Bokeh))
%config InlineBackend.figure_format = 'svg'

# figdir = '../../fig/chemical_master_analytic_protein/'
tmpdir = '../../tmp/'
```

### $\LaTeX$ macros

$\newcommand{kpon}{k^p_{\text{on}}}$
$\newcommand{kpoff}{k^p_{\text{off}}}$
$\newcommand{kron}{k^r_{\text{on}}}$
$\newcommand{kroff}{k^r_{\text{off}}}$
$\newcommand{rm}{r _m}$
$\newcommand{rp}{r _p}$
$\newcommand{gm}{\gamma _m}$
$\newcommand{gp}{\gamma _p}$
$\newcommand{mm}{\left\langle m \right\rangle}$
$\newcommand{foldchange}{\text{fold-change}}$

## The small noise approximation.

In this notebook we will explore the use of the small noise approximation in the context of the information processing capacity of the simple repression motif at steady state focusing on the mRNA level.

As derived by Tkacik, Callan and Bialek in their [2008 Phys Rev E paper](https://journals.aps.org/pre/abstract/10.1103/PhysRevE.78.011910), if one assumes the small noise approximation is valid, computing the channel capacity of a genetic regulatory system becomes analytically tractable.
The calculation requires a mapping between the input (in our case $c$ the inducer concentration) and the **mean output** (for this particular notebook the mean mRNA copy number $\mm$). Since these means are deterministic functions of the input we can use the change of variables
$$
P_c(c) dc = P_{\mm} (\mm) d \mm.
$$
In that case the optimal distribution $P^*_c(c)$ can be mapped to the optimal mean output distribution $P^*_{\mm}(\mm)$.

Using this change of variables and the small noise approximation (SMA) the optimal mean output distribution is given by
$$
P^*(\mm) = {1 \over \mathcal{Z}} {1 \over \sigma_{\mm}(\mm)},
$$
where $\mathcal{Z}$ is a normalization constant of the form
$$
\mathcal{Z} = \int_{\mm _\min}^{\mm _\max}
              {d\mm \over \sigma_{\mm}\left( \mm \right)}.
$$

Giventhis optimal distribution it can be shown that the channel capacity
$I^*(c;\mm)$ is given by
$$
I^*(c;\mm) = \log_2 \left( {\mathcal{Z} \over \sqrt{2\pi e}} \right).
$$

So with this amazing and simple result all we need is an expression for the noise $\sigma_{\mm}$ as a function of the mean output $\mm$ in order to compute the channel capacity of our system of interest!

This can be done numerically if we sample over a fine grid of inducer concentration and we compute both the mean $\mm(c)$ and the standard deviation $\sigma(c)$, and given these two results we perform a numerical integration of the resulting one-to-one mapping between these two functions.

## Import `lambdify` functions using `dill`.

When the analytical results were derived we generated `lambdify` functions to explore these results numerically and export them into a file using the `dill` library. Let's now import these functions into this notebook to make use of them.

```python
# Import two-state mRNA moments
# Parameters are feed in the following order:
# (kp_on, kp_off, rm, gm)
with open(tmpdir + 'two_state_mRNA_lambdify.dill', 'rb') as file:
    first_unreg_m = dill.load(file)
    second_unreg_m = dill.load(file)
    third_unreg_m = dill.load(file)

# Import two-state mRNA moments
# Parameters are feed in the following order:
# (kr_on, kr_off, kp_on, kp_off, rm, gm)
with open(tmpdir + 'three_state_mRNA_lambdify.dill', 'rb') as file:
    first_reg_m = dill.load(file)
    second_reg_m = dill.load(file)
    third_reg_m = dill.load(file)
```

## Integrating the number of repressors into the picture

The way that the repressor copy number enters the kinetic scheme is by assuming that the $\kron$ rate is a diffusion limited rate of the form
$$
\kron = k_o \left[ R \right].
$$
The constant $k_o$ was indirectly inferred by the Sunney Xie lab and quoted by Jones and Brewster as $k_o = 2.7 \times 10^{-3} (\text{s nM})^{-1}$.

The value of $\kroff$ is then constrained by the value of the binding energy as inferred using equilibrium statistical mechanics. For the stat. mech. framework we have that the fold-change in gene expression is given by
$$
\foldchange = \frac{1}{1 + \frac{R \cdot p_{act}}{N_{NS}} e^{-\beta \Delta \epsilon_r}},
$$
where $R$ is the absolute copy number of repressors per cell, $N_{NS}$ is the number of non-specific binding sites for the repressors (in our case the entire *E. coli genome* $4.6 \times 10^6$), $\beta$ is the inverse temperature times the Boltzmann constant and $\Delta \epsilon_r$ is the binding energy of the repressor to the operator binding site.

In the case of the three-state promoter the fold-change is given by
$$
\foldchange = {1 \over 1 + {\kron \over \kroff}
               \left( {\kpoff \over \kpoff + \kpon} \right)}
            = {1 \over 1 + {k_o [R] p_{act} \over \kroff}
               \left( {\kpoff \over \kpoff + \kpon} \right)}.
$$

Given that these two equation have the same functional form we can conclude that
$$
\frac{R \cdot p_{act}}{N_{NS}} e^{-\beta \Delta \epsilon_r} =
{k_o [R] p_{act} \over \kroff}
               \left( {\kpoff \over \kpoff + \kpon} \right).
$$
If we solve for $\kroff$ and convert the absolute number or repressors $R$ into a concentration $[R]$ we find that
$$
\kroff = 1.66 \cdot k_0 \cdot N_{NS} e^{\beta \Delta \epsilon_r}
          \left( {\kpoff \over \kpoff + \kpon} \right).
$$

Given this result let's define a function that computes $\kroff$ as a function of the binding energies and the RNAP rates.

```python
def kr_off_fun(eRA, k0, kp_on, kp_off, Nns=4.6E6):
    '''
    Returns the off rate of the repressor as a function of the stat. mech.
    binding energy and the RNAP on and off rates
    Parameters
    ----------
    eRA : float.
        Repressor binding energies [kbT]
    k0 : float.
        Diffusion limited constant [s**-1 nM**-1]
    kp_on : float.
        RNAP on rate. [time**-1]
    kp_off : float.
        RNAP off rate. [time**-1]
    Nns : float.
        Number of non-specific binding sites
    Returns
    -------
    Repressor off rate
    '''
    return 1.66 * k0 * Nns * np.exp(eRA) * kp_off / (kp_off + kp_on)
```

Let's now define a function that computes the **mean** $\mm(c)$ and **standard deviation** $\sigma_\mm(c)$ for the regulated promoter as a function of repressor copy number and inducer concentration. For this we will assume that the number of active repressors is given by the equilibrium value as determiend by $R_{act} = R \cdot p_{act}$, where $p_{act}$ is given by the MWC model.

```python
def std_reg_m(C, rep, eRA, rm, gm, k0, kp_on, kp_off,
                  Nns=4.6E6, ka=139, ki=0.53, epsilon=4.5):
    '''
    Computes the inverse standard deviation in mRNA copy number
    as a function  of all the parameters that go into the chemical master
    equation.

    Parameters
    ----------
    C : array-like.
        Concentration at which evaluate the probability.
    rep: float.
        repressor copy number per cell.
    eRA : float.
        Repressor binding energy [kBT]
    rm : float.
        transcription initiation rate. [time**-1]
    gm : float.
        mRNA degradation rate. [time**-1]
    rp : float.
        translation initiation rate. [time**-1]
    gp : float.
        protein degradation rate. [time**-1]
    k0 : float.
        diffusion limited rate of a repressor binding the promoter
    kp_on : float.
        RNAP on rate. [time**-1]
    kp_off : float.
        RNAP off rate. [time**-1]
    Nns : float.
        Number of non-specific binding sites
    ki, ka : float.
        dissociation constants for the inactive and active states respectively
        in the MWC model of the lac repressor.
    epsilon : float.
        energetic barrier between the inactive and the active state.

    Returns
    -------
    standard deviation in the repressor copy number
    '''
    # Convert C into np.array
    C = np.array(C)

    # Calculate the repressor on rate including the MWC model
    kr_on = k0 * rep * chann_cap.p_act(C, ka, ki, epsilon)
    # Compute the repressor off-rate based on the on-rate and the binding energy
    kr_off = kr_off_fun(eRA, k0, kp_on, kp_off, Nns)

    return np.sqrt(second_reg_m(kr_on, kr_off, kp_on, kp_off, rm, gm) - \
                   first_reg_m(kr_on, kr_off, kp_on, kp_off, rm, gm)**2)

def mean_reg_m(C, rep, eRA, rm, gm, k0, kp_on, kp_off,
                  Nns=4.6E6, ka=139, ki=0.53, epsilon=4.5):
    '''
    Computes the mean mRNA copy number  as a function  of all the
    parameters that go into the chemical master equation.

    Parameters
    ----------
    C : array-like.
        Concentration at which evaluate the probability.
    rep: float.
        repressor copy number per cell.
    eRA : float.
        Repressor binding energy [kBT]
    rm : float.
        transcription initiation rate. [time**-1]
    gm : float.
        mRNA degradation rate. [time**-1]
    rp : float.
        translation initiation rate. [time**-1]
    gp : float.
        protein degradation rate. [time**-1]
    k0 : float.
        diffusion limited rate of a repressor binding the promoter
    kp_on : float.
        RNAP on rate. [time**-1]
    kp_off : float.
        RNAP off rate. [time**-1]
    Nns : float.
        Number of non-specific binding sites
    ki, ka : float.
        dissociation constants for the inactive and active states respectively
        in the MWC model of the lac repressor.
    epsilon : float.
        energetic barrier between the inactive and the active state.

    Returns
    -------
    mean mRNA copy number
    '''
    # Convert C into np.array
    C = np.array(C)

    # Calculate the repressor on rate including the MWC model
    kr_on = k0 * rep * chann_cap.p_act(C, ka, ki, epsilon)
    # Compute the repressor off-rate based on the on-rate and the binding energy
    kr_off = kr_off_fun(eRA, k0, kp_on, kp_off, Nns)

    return first_reg_m(kr_on, kr_off, kp_on, kp_off, rm, gm)
```

Let's now evaluate both functions for a grid of inducer concentrations and plot mean vs inverse standard deviation.

First let's define the parameters for the *lacUV5* promoter.

```python
# List the parameters fit for the lacUV5 promoter
par_UV5 = dict(kp_on=5.5, kp_off=28.9, rm=87.6, gm=1)
# Define the k0 parameters in units of the mRNA degradation time
k0_norm = 2.7E-3 / 0.00284
par_UV5['k0'] = k0_norm
```

Now let's define the parameters for different operators with 260 repressors per cell.

```python
# Define grid of inducer concentrations
IPTG = np.logspace(-1, 8, 1000)
# Copy parameter list
par = par_UV5.copy()
# Add binding energy and repressor copy number
par['rep'] = 260

# Define energies of different operators
operators = ['O3', 'O2', 'O1']
energies = [-9.7, -13.9, -15.7]

# Loop through energies
for i, op in enumerate(operators):
    par['eRA'] = energies[i]

    mean_m = mean_reg_m(IPTG, **par)
    std_m = std_reg_m(IPTG, **par)

    plt.plot(mean_m, 1 / std_m, label=op)

plt.xlabel(r'$\left\langle m \right\rangle$')
plt.ylabel(r'$1 / \sigma_{\left\langle m \right\rangle}$')
plt.title('repressors / cell = 260')
plt.legend(title='operators')
plt.ylim(bottom=0)
```

    (0, 2.084576322110383)

Let's do it instead for varying number of repressors

```python
# Define grid of inducer concentrations
IPTG = np.logspace(-1, 8, 1000)
# Copy parameter list
par = par_UV5.copy()
# Add binding energy and repressor copy number
par['eRA'] = -13.9

# Define the repressor copy numbers to use
repressors = [22, 60, 124, 260, 1220, 1740]

# Loop through repressors
for rep in repressors:
    par['rep'] = rep

    mean_m = mean_reg_m(IPTG, **par)
    std_m = std_reg_m(IPTG, **par)

    plt.plot(mean_m, 1 / std_m, label=rep)

plt.xlabel(r'$\left\langle m \right\rangle$')
plt.ylabel(r'$1 / \sigma_{\left\langle m \right\rangle}$')
plt.title('operator : O2')
plt.legend(title='rep./cell')
plt.ylim(bottom=0)
```

    (0, 2.2066751494393233)

# Compute the channel capacity from this sampling.

Now that we can numerically evaluate the noise in gene expression as a function of the mean expression level, let's use numerical integration to compute $\mathcal{Z}$ defined as
$$
\mathcal{Z} = \int_{\mm _\min}^{\mm _\max}
              {d\mm \over \sigma_{\mm}\left( \mm \right)}.
$$
To then compute the channel capacity defined as
$I^*(c;\mm)$ is given by
$$
I^*(c;\mm) = \log_2 \left( {\mathcal{Z} \over \sqrt{2\pi e}} \right).
$$

Let's define a function to compute this.

```python
def chann_cap_SMA_m(C, rep, eRA, rm, gm, k0, kp_on, kp_off,
                    Nns=4.6E6, ka=139, ki=0.53, epsilon=4.5):
    '''
    Computes the channel capacity (in bits) using the
    small noise approximation as a function  of all the parameters
    that go into the chemical master equation.

    Parameters
    ----------
    C : array-like.
        Concentration at which evaluate the probability.
    rep: float.
        repressor copy number per cell.
    eRA : float.
        Repressor binding energy [kBT]
    rm : float.
        transcription initiation rate. [time**-1]
    gm : float.
        mRNA degradation rate. [time**-1]
    rp : float.
        translation initiation rate. [time**-1]
    gp : float.
        protein degradation rate. [time**-1]
    k0 : float.
        diffusion limited rate of a repressor binding the promoter
    kp_on : float.
        RNAP on rate. [time**-1]
    kp_off : float.
        RNAP off rate. [time**-1]
    Nns : float.
        Number of non-specific binding sites
    ki, ka : float.
        dissociation constants for the inactive and active states respectively
        in the MWC model of the lac repressor.
    epsilon : float.
        energetic barrier between the inactive and the active state.

    Returns
    -------
    maximum number of bits that the system can process
    '''
    # Compute the mean number of mRNA for the inducer grid
    mean_m = mean_reg_m(C, rep, eRA, rm, gm, k0, kp_on, kp_off,
                  Nns, ka, ki, epsilon)

    # Compute the std of mRNA for the inducer grid
    std_m = std_reg_m(C, rep, eRA, rm, gm, k0, kp_on, kp_off,
                      Nns, ka, ki, epsilon)

    # Compute Z
    Z = scipy.integrate.simps(1 / std_m, mean_m)

    return np.log2(Z / np.sqrt(2 * np.pi * np.e))
```

Let's now look at the channel capacity as a function of repressor copy number

```python
# Define grid of repressor copy number
rep_array = np.logspace(0, 3, 200)

# Define grid of inducer concentrations
IPTG = np.logspace(-1, 8, 2000)
# Copy parameter list
par = par_UV5.copy()

# Define energies of different operators
operators = ['O3', 'O2', 'O1']
energies = [-9.7, -13.9, -15.7]

# Loop through energies
for j, eRA in enumerate(energies):
    # Add binding energy and repressor copy number
    par['eRA'] = eRA
    # Initialize array to save channel capacity
    channel_cap = np.zeros_like(rep_array)
    # Loop through repressor copy number computing the channel capacity
    for i, rep in enumerate(rep_array):
        par['rep'] = rep
        channel_cap[i] = chann_cap_SMA_m(IPTG, **par)

    # Plot repressors vs channel capacity
    plt.plot(rep_array, channel_cap, label=operators[j])

plt.xlabel('repressors / cell')
plt.ylabel('channel capacity (bits)')
plt.xscale('log')
plt.legend(title='operators')
```

    <matplotlib.legend.Legend at 0x1a20099400>


```python
import numpy as np
import pandas as pd
def cost(theta):

  l_theta=theta**2
  return l_theta
```

```python
def cost_diff(theta_curr):

  return 2*theta_curr
```

```python
def step(x_new,x_old,learning_rate):

  while abs(x_new-x_old)>0.00001:
    x_old=x_new
    d_x=-cost_diff(x_old)
    x_new=x_old+learning_rate*d_x

  print(x_new)

```

```python
step(0,1,1)

```

    0

```python
def cost2_diff(theta_curr):
  return 2*theta_curr
```

```python
theta0_old=1;
theta1_old=1;
learning_rate=0.01

def step2(theta0_new,theta0_old,theta1_new,theta1_old,learning_rate):

  while abs(theta0_new-theta0_old)>0.00001 and abs(theta1_new-theta1_old)>0.00001:
    theta0_old=theta0_new
    d_1=-cost2_diff(theta0_old)
    theta1_new=theta0_old+learning_rate*d_1

    theta1_old=theta1_new
    d_2=-cost2_diff(theta1_old)
    theta1_new=theta1_old+learning_rate*d_2

  print(theta0_new)
  print(theta1_new)

```

```python
step2(0,1,0,1,0.01)
```

    0
    0.0

```python
import numpy as np

t0=1
alpha=0.1
while 1:
  t1=t0-2*(t0-1)*alpha
  if(t1-t0)**2<0.00001:
    break
  t0=t1

print(t1)
```

    1.0

```python
import numpy as np

t00 = 1
t10 = 1
alpha = 0.1
while 1:
  t01 = t00 - 4*alpha*(t00-1)
  t11 = t10 - 2*alpha*(t10-1)

  if (t01-t00)**2 + (t11 - t10)**2 < 0.01:
    break
  t00 = t01
  t11 = t10

print(t01, t10)
```

    1.0 1

```python
def  cal_cost(theta,X,y):

    m = len(y)

    predictions = X.dot(theta)
    cost = (1/2*m) * np.sum(np.square(predictions-y))
    return cost
```

```python
def gradient_descent(X,Y,theta,learning_rate=0.01,iterations=100):

    m = len(Y)
    Y = Y.to_numpy().reshape(m,1)
    #cost_history = np.zeros(iterations)
    #theta_history = np.zeros((iterations,2))
    for it in range(iterations):

        prediction = np.dot(X,theta)

        theta = theta -(1/m)*learning_rate*( X.T.dot((prediction - Y)))

        #cost_history[it]  = cal_cost(theta,X,y)

    return theta
```

```python
df = pd.read_csv("data.csv")
X = df.x
Y = df.y

X = (X-np.mean(X))/np.std(X)
lr =0.01
n_iter = 1000

theta = np.random.randn(2,1)

X_b = np.c_[np.ones((len(X),1)),X]
theta = gradient_descent(X_b,Y,theta,lr,n_iter)

print(theta)
```

    [[23.71706365]
     [-6.86672194]]

```python
#question 2

def stochastic_gradient_descent(X, Y, theta, lrate=0.01, n_iter=100):
  m = len(Y)
  cost_history = np.zeros(n_iter)
  Y = Y.to_numpy().reshape(m,1)
  #print(Y.shape)
  #Y = Y.reshape(m,1)
  for it in range(n_iter):
    cost = 0.0
    for i in range(m):
      rand_ind = np.random.randint(0,m)
      X_i = X[rand_ind, :].reshape(1, X.shape[1])

      Y_i = Y[rand_ind, :].reshape(1,1)

      prediction = np.dot(X_i, theta)
      theta = theta - lrate*(X_i.T.dot(prediction - Y_i))
      cost += cal_cost(theta, X_i, Y_i)
    cost_history[it] = cost
  return theta, cost_history
```

```python
df = pd.read_csv("data.csv")
X = df.x
Y = df.y

X = (X-np.mean(X))/np.std(X)

n_iterations = 1000;
lrate = 0.01

theta = np.random.rand(2,1)*100
#print(theta)
X_b = np.c_[np.ones((len(X),1)), X]
theta, cost_history = stochastic_gradient_descent(X_b, Y, theta, lrate, n_iterations)

print(theta, cost_history[n_iterations-1])
```

    [[23.74522185]
     [-6.79031234]] 840.8423792848323

```python
def J(X, Y, theta):
  cost = np.sum(np.square(Y-(X.dot(theta))))
  return cost
```

```python
def J1(theta):
  return theta*theta
```

```python
def delta_J(X, Y, theta):
    prediction = np.dot(X,theta)
    return X.T.dot((prediction-Y))
```

```python
def delta_J1(theta):
  return 2*theta

```

```python
#wrong answer check
alpha0=0.01
alpha_prev=0.01
precision=0.000001
theta_0=0
df =lambda x:2*x
prev=theta_0
count =0
while count<100000:

  alpha_curr=alpha_prev-df(alpha_prev)/2
  curr=prev-alpha_curr*df(prev)
  count=count+1
  alpha_prev=alpha_curr
  prev=curr

print(prev)
```

    0.0

```python
#question 3
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import pandas as pd
from sympy import *

def steepest_descent(X, Y, theta, lrate=0.01, n_iter=1000):
  cost_history = np.zeros(n_iter)
  alpha_history = np.zeros(n_iter)
  m = len(Y)
  Y = Y.to_numpy().reshape(m,1)
  #lrate_history = np.zeros(n_iter)

  alpha = Symbol('alpha')
  phi = J(X, Y, theta - alpha*delta_J(X,Y,theta))

  old_a = np.random.randn()
  #print(phi)
  for it in range(n_iter):
    diff_phi = phi.diff(alpha)
    new_a = old_a - (diff_phi.subs(alpha, old_a)/diff_phi.diff(alpha).subs(alpha, old_a))
    prediction = np.dot(X, theta)
    theta = theta - new_a*(X.T.dot(prediction - Y))
    cost_history[it] = cal_cost(theta, X, Y)
    alpha_history[it] = old_a
    old_a = new_a

  return theta, cost_history, alpha_history
```

```python

```

```python
df = pd.read_csv("data.csv")
X = df.x
Y = df.y

X = (X-np.mean(X))/np.std(X)

n_iterations = 1000;
lrate = 0.01

theta = np.random.rand(2,1)*100
#print(theta)
X_b = np.c_[np.ones((len(X),1)), X]
theta, cost_history, alpha_h = steepest_descent(X_b, Y, theta, lrate, n_iterations)

print(theta, cost_history[n_iterations-1], alpha_h[n_iterations-1])
```

    [[23.7180851063830]
     [-6.86699544725538]] 73914.56724393774 0.010638297872340422

```python

```

```python

```

```python
def L(theta):
  return theta**2
alpha = Symbol('alpha')
theta = Symbol('theta')
#L = theta**2
delta_L = L(theta).diff(theta)
n_iter = 1000

t0 = np.random.randn()*100
old_a = np.random.randn()

phi = L(t0-alpha*delta_L.subs(theta, t0))

for it in range(n_iter):
  diff_phi = phi.diff(alpha)
  new_a = old_a - (diff_phi.subs(alpha, old_a)/diff_phi.diff(alpha).subs(alpha, old_a))
  t0 = t0 - new_a*delta_L.subs(theta, t0)
  old_a = new_a

print(t0, new_a)
```

    0 0.500000000000000

```python
def L1(theta1, theta2):
  return theta1**2 + theta2**2
alpha = Symbol('alpha')
theta1 = Symbol('theta1')
theta2 = Symbol('theta2')
#L = theta**2
delta_L1 = L1(theta1, theta2).diff(theta1)
delta_L2 = L1(theta1, theta2).diff(theta2)
n_iter = 1000

t0 = np.random.randn()*100
t1 = np.random.randn()*100
old_a = np.random.randn()

phi_0 = L1(t0 - alpha*delta_L1.subs(theta1, t0), t1)
phi_1 = L1(t0, t1 - alpha*delta_L2.subs(theta2, t1))

for it in range(n_iter):
  diff_phi = phi_0.diff(alpha)
  new_a = old_a - (diff_phi.subs(alpha, old_a)/diff_phi.diff(alpha).subs(alpha, old_a))
  t0 = t0 - new_a*delta_L1.subs(theta1, t0)
  t1 = t1 - new_a*delta_L2.subs(theta2, t1)
  old_a = new_a

print(t0, t1, new_a)
```

    0 0 0.500000000000000


# Classical Euler methods

### Table of contents
 1. Chapter 2: Classical Methods
    1. [Section 1.1: Explicit Euler](#explicitEuler)
    1. [Section 1.2: Implicit Euler](#implicitEuler)

```python
# If you do not have numpy, matplotlib, scipy or nodepy, run this cell
!pip install numpy
# This is the basic package in python with all the numerical functions

!pip install scipy
# This package has some functions to deal with polynomials

!pip install matplotlib
# This package allows to  plot

!pip install nodepy
# This package has some interesting features for RK methods
```

```python
# We need a couple of packages in this chapter
import numpy as np
# This is the basic package in python with all the numerical functions

import matplotlib.pyplot as plt
# This package allows to  plot

from scipy import optimize
# This module has optimization tools

from nodepy import *
#This package already implemented some functions for Runge Kutta and multistep methods
```

We want to **approximate** the ODE on $I:=[t_0, t_{end}]\subset \mathbb{R}$ for the unknown variable $y:I\to \mathbb{R}^{S}$ with *continuous* function $F:I\times \mathbb{R}^S\to\mathbb{R}^S$

$$
\begin{equation}
\begin{cases}\label{eq:IVP}
\frac{dy}{dt} = F(t,y(t)),\\
y(0)=y_0.
\end{cases}
\end{equation}
$$

* Why approximate? Solution may be unknown, or too complicated to be solved analytically.
* How we want to approximate? We want to be **accurate** and we want to preserve the physical properties we have seen before.

## Explicit Euler

Consider the time interval $I=[t_0,t_{end}]$ and let us subdivide it into $N$ subintervals,

$$t_0=t^0<t^1< \dots <t^n < \dots <t^N=t_{end}.$$

We approximate naïvely the integral form

$$ y(t^{n+1})=y(t^n) +\int_{t^n}^{t^{n+1}} F(s,y(s))ds \approx y(t^n) -\underbrace{(t^{n+1}-t^n)}_{\Delta t^n} F(t^n,y(t^n)) $$

leading to the method (forward Euler/explicit Euler), where we use $y^n$ to approximate $y(t^n)$

$$
\begin{cases}
y^0=y_0,\\
y^{n+1}=y^n+\Delta t^n F(t^n,y^n), \qquad n=0,\dots, N-1.
\end{cases}
$$

```python
def explicitEuler(func, tspan, y_0):
    '''
    Simple implementation of the explicit Euler method
    Input are
    func the function F of the ODE which takes as input y and t F(y,t)
    tspan is the vector of all timesteps (t^0,...,t^N)
    y_0 is the initial condition
    '''
    N_time=len(tspan)  # N+1
    dim=len(y_0)          # S
    y=np.zeros((dim,N_time))    # initializing the variable of solutions
    y[:,0]=y_0                 # first timestep
    for n in range(N_time-1):    # n=0,..., N-1
        y[:,n+1]=y[:,n]+ FILL IN THE EVOLUTION ## Code the explicit euler step
    return tspan, y
```

Test on $$
\begin{equation}\label{eq:linear_test}
\begin{aligned}
& \begin{cases}
c_1'(t)=c_2(t)-5c_1(t),\\
c_2'(t)=5c_1(t)-c_2(t),
\end{cases}\\
 &c_1(0)=c_1^0=0.9, \quad  &c_2(0)=c_2^0=0.1 \, ,\\
 &t\in [0,3].
\end{aligned}
\end{equation}
$$

```python
# Define the function F
def linSysF(y,t=0):
    # evolution function
    F=np.zeros(np.shape(y))
    F[0] = FILL IN THE DEFINITION
    F[1] = FILL IN THE DEFINITION
    return F

## Now we plot the solution with different number of timesteps
for N in [100,30,10]:
    tspan=np.linspace(0,3,N)
    y0=np.array([0.9,0.1])

    tt,yy=explicitEuler(linSysF,tspan,y0)

    A=np.array([[-5,1],[5,-1]])

    y_exact=np.zeros((len(y0),len(tt)))
    for it, t in enumerate(tt):
        y_exact[:,it]=y0+(1-np.exp(-6*t))/6*np.dot(A,y0)

    plt.figure()
    plt.plot(tt,y_exact[0,:],":", label="c1 ex")
    plt.plot(tt,y_exact[1,:],":", label="c2 ex")

    plt.plot(tt,yy[0,:],label="c1")
    plt.plot(tt,yy[1,:],label="c2")
    plt.title("N=%d"%N)
    plt.legend()
```

Preliminary, we can observe that
1. The more the points we put in the time discretization the better the solution gets
1. Explicit Euler does not preserve **unconditionally** the positivity of the solution ($N=10$)
1. The total mass is conserved

$$ c_1^{n+1}+c_2^{n+1}=c_1^{n}+c_2^{n}+ \Delta t\left( -5c_1^{n}+c_2^{n}+5c_1^{n}-c_2^{n} \right) = c_1^{n}+c_2^{n} $$

### Error analysis
The error that we observe
$$e_n=y(t^n)-y^n$$

is composed of several parts that we can divide and study separately.

#### Consistency error
Given the exact solution $y(t)$, we define the consistency error to be

$$
\varepsilon_n = y(t^{n+1})-y(t^n) - \Delta t F(t^n,y(t^n)) = \int_{t^n}^{t^{n+1}} y'(t) -y'(t^n)\, dt.
$$

Notice that $|\varepsilon_n|\leq \Delta t \omega (y',\Delta t)$, where $\omega$ is the modulus of continuity of a bounded function, i.e.,
$$
\omega(f,\Delta t):= \max_{t,t': |t-t'|\leq \Delta t} |f(t)-f(t')|.
$$

Essentially, this is the error that we obtain by substituting the exact solution inside the method. It is one of the 2 ingredients that leads the error of a method.

Going back to the error, we observe that

$$
e_{n+1}=y(t^{n+1})-y^{n+1}=e_n +\varepsilon_n +\Delta t \big(f(t^n,y(t^n))-f(t^n,y^n)\big)
$$

using the Lipschitz continuity of $f$, we have

$$
|e_{n+1}|\leq |e_n| +|\varepsilon_n| +\Delta t L|y(t^n)-y^n| =(1+L\Delta t)|e_n| +|\varepsilon_n|.
$$

Using the **Discrete Gronwall Lemma** we obtain that

$$
|e_{n}|\leq e^{L|t^n-t^0|}|e_0| + \sum_{i=0}^{n-1} e^{L(t^n-t^{i+1})}|\varepsilon_i|.
$$

This tells us that, except for the initial error (that usually we can bound accurately or know its error), the consistency error leads this sum. So, if we keep $\varepsilon_n$ small enough, the final error will be small enough.

Using the estimation for $\varepsilon_i$ and suppose $\Delta t^n=\Delta t$, we can collect
$$
\begin{align}
|e_{n}|&\leq e^{L|t^n-t^0|}|e_0| + \Delta t \omega(y',\Delta t) \sum_{i=0}^{n-1} e^{L(t^n-t^{i+1})}\\
&\leq e^{L|t^n-t^0|}|y^0-y(t^0)| + \Delta t \omega(y',\Delta t) \frac{e^{L(t^n-t^{0})}-1}{L}.
\end{align}
$$

This shows that the solution converges to the exact one as $\Delta t \to 0$, if the initial datum is correct.

If we know more on the regularity of the solution ($y\in \mathcal C^2$), we can say that

$$
|y(t^n)-y^n|\leq e^{L(t^n-t^0)}|y^0-y(t^0)| + \Delta t \int_{t^0}^{t^n} e^{L(t^n-s)} |y''(s)| ds.
$$

#### Local vs Global Error
A small remark must be done in order to understand how the global error generates from the local one.
The local truncation error is the one given for one time step, i.e., using Taylor expansion and supposing $y^0=y(t^0)$,

$$
e_1=|y^1-y(t^1)|=|y^0 +\Delta t F(t^0,y^0) - \left(y(t^0) + \Delta t y'(t^0) + \frac{1}{2} \Delta t^2 y''(t^0)\right) + \mathcal{O}(\Delta t^3)| = \frac{1}{2} \Delta t^2 |y''(t^0)| +  \mathcal{O}(\Delta t^3).
$$

In one step we see an error of $\mathcal O (\Delta t^2)$, when integrating of the whole time interval $[t^0,t^N]$ one obtains an $\mathcal O (\Delta t)$ as we have seen before.

Naïvely one can see it as if in every step we are adding an $\mathcal O (\Delta t^2)$ to the global error

$$
e_N\approx \sum_{i=1}^N |y(t^i)-y^i| \approx N \Delta t^2 \max_{t\in [t^0,t^N]} |y''(t)| = \frac{t^N-t^0}{\Delta t} \Delta t^2\max_{t\in [t^0,t^N]} |y''(t)|= (t^N-t^0) \Delta t\max_{t\in [t^0,t^N]} |y''(t)|
$$

The **order of accuracy** of a method is the largest *integer* $p$, such that the error can be bounded by

$$
|e_N| \leq C \Delta t^p, \qquad \forall \Delta t \in \mathbb R ^+.
$$

This definition is of course meant to be verified in the limit for $\Delta t\to 0$ (in realistic cases we stop at $\approx 10^{-14}$).

The explicit Euler method is of order (at least) 1. (one can check that it is not 2 with numerical tests)

#### Roundoff effects
We should always keep in mind that the error we studied before is not the only one that a computer produces.
Indeed, at each operation (initial value, evaluation of $F$, sums, products) we introduce a roundoff error due to the machine precision.
One can get similar estimations to what we have seen before, knowing that the error can be controlled.

We ignore this error in this course.

### Stability
Question: going with $\Delta t \to 0$ should produce nice results, but, what can we say about $\Delta t >>0$?
Can we give a qualitatively say when a method could be stable/reliable in particular when *stiff* problems are considered. *Stiff* problems are the ones for which a *normal* discretization can not produce decent results.

This rough description can be made more precise and studied only in limited cases. In particular, we restrict to linear problems with constant coefficients.

$$
y'(t)=My(t)
$$

with $M$ constant matrix. We fix the timestep $\Delta t$. The explicit Euler method gives the update

$$
y^{n}=y^{n-1}+\Delta t M y^{n-1} =(I+\Delta t M) y^{n-1} = (I+\Delta t M)^{n} y^0.
$$

Doing a change of basis given by the nonsingular matrix $S$ so that  $\hat{M}=S^{-1} M S$ is in the Jordan canonical form, and defining $\hat{y}^n=S^{-1} y^n$, we have that

$$
\hat y^{n}=\hat y^{n-1}+\Delta t \hat M \hat{y}^{n-1} =(I+\Delta t \hat M) \hat{y}^{n-1} = (I+\Delta t \hat M)^{n} \hat{y}^0.
$$

This means that for each distinct eigenvalue $q$ we can study the linear scalar equation

$$
y'(t)= q  y(t).
$$

The other components that correspond to the same Jordan block will depend on this solution, but will not contribute to its behaviour.

The final question is whether $(1+\Delta t q)^N$ is an *acceptable* approximation of $e^{N\Delta t q }$.

We are interested in bounded behaviors for $N\to \infty$ , this implies that $|1+\Delta t q|\leq 1$, or that $Re(q)\leq 0$. ($q$ could be complex as it is an eigenvalue of $M$).

Rewriting the problem with $z=q\Delta t$, one can see that

$$
y^n=y^{n-1}+z y^{n-1}=(1+z)y^{n-1}
$$

and the method will be stable if
$|1+z|\leq 1$, which is the region in the complex plane denoted by a circle of radius 1 with center in $(-1,0)$.

The function $R(z):=1+z$ for the explicit Euler is the *stability function*.
For a general method for the Dahlquist's equation

$$
y'(t)=qy(t)
$$

denoting by $z=\Delta t q$, the method can be written as

$$
y^{n+1}=R(z) y^n.
$$

```python
## We will see soon how to write a RK method
## This is the explicit Euler method written into the RK formalism
## and we plot the stability region using the nodepy module
A=np.array([[0]])
b=np.array([1])
exEuler=runge_kutta_method.ExplicitRungeKuttaMethod(A,b)
p,q=exEuler.stability_function()
print(p)
exEuler.plot_stability_region();
```

#### How can we ensure the belonging to the stability region?
We want $z=q\Delta t$ to stay in the stability region. On $q$ we do not have control, hence, we can only modify $\Delta t$.
In particular, denoting $q=p+ir$ with $p,r \in \mathbb R $ and $p\leq 0$, the stability relation we have seen before leads to at least check that the real part verifies the relation

$$
|1+\Delta t p + i \Delta t r|\leq 1 \\
1-\Delta t |p|  \geq 1\\
\Delta t \leq \frac{2}{|p|}
$$

where $|p|$ is for sure bounded by the Lipschitz constant $L$ of the function $F$.

So, it is necessary to check that

$$
\Delta t \leq \frac{2}{L}.
$$

This can be generalized also for nonlinear problems.

#### Imaginary eigenvalues
If the problem we are considering contains only imaginary eigenvalues, then we cannot solve it with explicit Euler method.
An example is

$$
u''=-u
$$

Consider the exact solution

$$
u=\sin(t)
$$

So, we can put it into a system of first order ODEs with initial conditions

$$
\begin{cases}
u'=v,\\
v'=-u,\\
u(0) = 0,\\
v(0) = 1.
\end{cases}
$$

```python
# Define the function F
def linSysF(y,t=0):
    # evolution function
    F=np.zeros(np.shape(y))
    F[0] = y[1]
    F[1] = -y[0]
    return F

## Now we plot the solution with different number of timesteps
dt = 1
T_end = 100

tspan=np.linspace(0,T_end,np.int(T_end/dt)+1)
y0=np.array([0,1])

tt,yy=explicitEuler(linSysF,tspan,y0)

plt.figure()
plt.plot(tt,np.sin(tt),":", label="c1 ex")

plt.plot(tt,yy[0,:],label="c1")
plt.title("N=%d"%N)
plt.legend()
plt.show()

plt.plot(tt,yy[0,:]-np.sin(tt))
plt.title("Error")
```

## Implicit Euler

The implicit Euler method approximates our problem with the following strategy

$$
y^{n+1}=y^n +\Delta t f(t^{n+1},y^{n+1}).
$$

1. It is not always easy to find the solution of such method, for example when $f$ is nonlinear, one may need nonlinear solvers to find the solution (e.g. Newton method, Broyden, and so on)
1. We can compute the error estimate similarly to explicit Euler, obtaining that also implicit Euler is a *first* order method
1. More interesting are the **stability** property of this scheme.

Consider again the Dahlquist's equation

$$y'=qy$$

and the implicit Euler method

$$
\begin{align}
y^{n+1}=y^n+ \Delta t q y^{n+1},\\
(1-\Delta t q) y^{n+1}=y^n,\\
y^{n+1}=\frac{1}{1-\Delta t q} y^n.
\end{align}
$$

So the stability function is $R(z)=\frac{1}{1-z}$ and the stability region $\mathcal S := \lbrace z \in \mathbb C : |R(z)|\leq 1 \rbrace$ contains the whole left complex semiplane. Indeed, if Re$(z)\leq 0$, then Re$(1-z)\geq 1$ and $|1-z|\geq 1$.
So, $R(z)\leq 1$.

```python
## We will see soon how to write a RK method
## This is the implicit Euler and we plot the stability region
A=np.array([[1]])
b=np.array([1])
imEuler=runge_kutta_method.RungeKuttaMethod(A,b)
p,q=imEuler.stability_function()
print(p) ## Numerator
print(q) ## Denominator
imEuler.plot_stability_region(bounds=[-10,4, -5,5]);
```

#### Unconditionally TVD/positivity preserving
For some class of linear problems, it can be shown that for positive systems it is **positivity preserving**, or more in general for finite difference method in hyperbolic conservation laws, it is total variation diminishing (TVD).

The clou is that these properties are true independently on the size of $\Delta t$.

For TVD one can read the SSPRK article by Gottlieb, Shu and Tadmor [link](https://www.researchgate.net/publication/2365594_Strong_Stability-Preserving_High-Order_Time_Discretization_Methods).

##### TVD for incremental form problem
The implicit Euler method for incremental problems, i.e.,
$$
U^{n+1}_j=U^{n}_j +\Delta t \left [ C_{j+1/2}(U_{j+1}^{n+1}-U_{j}^{n+1})-D_{j-1/2}(U_{j}^{n+1}-U_{j-1}^{n+1}) \right]
$$

where $C_{j+1/2},D_{j+1/2}\geq 0$ is TVD independently on $\Delta t$.

###### Proof (Harten)
Define

$$
TV(U^n) = \sum_j |U^n_{j+1}-U^n_j|.
$$

We can compute
$$
U^{n+1}_j=U^{n}_j +\Delta t \left [ C_{j+1/2}(U_{j+1}^{n+1}-U_{j}^{n+1})-D_{j-1/2}(U_{j}^{n+1}-U_{j-1}^{n+1}) \right]\\
[1+\Delta t (C_{j+1/2}+D_{j+1/2})](U^{n+1}_{j+1}-U_j^{n+1})=U^{n}_{j+1}-U_j^{n} +\Delta t \left [ C_{j+3/2}(U_{j+2}^{n+1}-U_{j+1}^{n+1})+D_{j-1/2}(U_{j}^{n+1}-U_{j-1}^{n+1}) \right]\\
[1+\Delta t (C_{j+1/2}+D_{j+1/2})]|U^{n+1}_{j+1}-U_j^{n+1}|\leq|U^{n}_{j+1}-U_j^{n}| +\Delta t \left [ C_{j+3/2}|U_{j+2}^{n+1}-U_{j+1}^{n+1}|+D_{j-1/2}|U_{j}^{n+1}-U_{j-1}^{n+1}| \right]\\
TV(U^{n+1}) +\Delta t \sum_j(C_{j+1/2}+D_{j+1/2})|U^{n+1}_{j+1}-U^{n+1}_j| \leq TV(U^{n}) +\Delta t \sum_j(C_{j+1/2}+D_{j+1/2})|U^{n+1}_{j+1}-U^{n+1}_j| \\
TV(U^{n+1}) \leq TV(U^n).
$$

Reminder: Total variation diminishing means for conservation laws, that if the initial solution is positive, it stays positive.

##### Positivity for production destruction system
We will see the positivity for a specific case: a production-destruction system with constant coefficient. It can be written as

$$
y'=My
$$

with

$$
M_{ii}<0,\qquad M_{ij}\geq 0,\, i\neq j, \qquad \sum_{i}M_{ij}=0.
$$

The linear system at the beginning of this chapter falls in this example.

This system is positive if $y_i^0\geq 0$. The implicit Euler is also positive.

$$
(I-\Delta t M)y^{n+1}= y^{n}
$$

##### Theorem
Defining with $A:=I-\Delta t M$, we can prove that $A$ is non singular and that $A^{-1}\geq 0$, i.e., every entry of the matrix is nonnegative.

##### Proof
1. $A$ is strictly diagonally dominant by columns
Indeed,

$$
0< A_{ii} = 1+\Delta t |M_{ii}| > \Delta t \sum_{j:j\neq i} |M_{ji}| = \sum_{j:j\neq i} |A_{ji}|
$$

Hence, $A$ is nonsingular.

2. The Jacobi method converge and the Jacobi Matrix is positive [Jacobi method](https://en.wikipedia.org/wiki/Jacobi_method)

Define the Jacobi matrix $B=D^{-1}(D-A)$, with $D=\text{diag}(A)$.
The diagonal of $B$ is 0 and each element on the off diagonal terms are

$$
B_{ji}=\frac{-A_{ji}}{A_{ii}}, \quad j\neq i.
$$

So, the spectral radius of $B$ is bounded by

$$
\rho(B)\leq ||B||_{\infty} =\max_{i}\sum_{j\neq i} \frac{|A_{ji}|}{|A_{ii}|} \leq 1.
$$

The iterative Jacobi method is convergent to the solution of $Ay^{n+1}=y^n$.

The method reads

$$
w^{(k+1)}=D^{-1}(y^n- (D-A)w^{(k)})
$$

which is a linear combination of positive matrices and vectors. Hence, the solutions $w^{(k)}$ stay positive if $y^n$ is positive. Knowing that $Dy^{n+1}=(D-A)y^{n+1} +y^n$, the error at each iteration reads

$$
e^{(k+1)}:=w^{(k+1)}-y^{n+1} = D^{-1}(y^n- (D-A)w^{(k)})-D^{-1}(y^n- (D-A)y^{n+1})=D^{-1}(D-A)(w^{(k)}-y^{n+1})= B e^{(k)}.
$$

Knowing that $B$ has norm smaller than 1, we know that the iteration process converges to the solution of the system.

```python
def implicitEulerLinear(M, tspan, y_0):
    '''
    Simple implementation of the implicit Euler for Linear systems y'=My with M constant matrix
    Input are
    M the ODE constant matrix
    tspan vector of timesteps (t^0,...,t^N)
    y_0 initial value
    '''
    N_time=len(tspan)  # N+1
    dim=len(y_0)        # S
    y=np.zeros((dim,N_time))    # initializing the variable of solutions
    y[:,0]=y_0                 # first timestep
    for n in range(N_time-1):    # n=0,..., N-1
        A= FILL IN # define the matrix that need to be inverted
        y[:,n+1]=np.linalg.solve(A,y[:,n])
    return tspan, y
```

```python
# Test implicit Euler on the linear systems: first the production destruction system with matrix
linSysM=np.array([[-5,1],[5,-1]])

def linSysF(y,t=0):
    # evolution function
    F=np.zeros(np.shape(y))
    F[0] = FILL IN THE EVOLUTION
    F[1] = FILL IN THE EVOLUTION
    return F

for N in [100,30,10]:
    tspan=np.linspace(0,3,N)
    y0=np.array([0.9,0.1])

    tt,yy=implicitEulerLinear(linSysM,tspan,y0)

    A=np.array([[-5,1],[5,-1]])

    y_exact=np.zeros((len(y0),len(tt)))
    for it, t in enumerate(tt):
        y_exact[:,it]=y0+(1-np.exp(-6*t))/6*np.dot(A,y0)

    plt.figure()
    plt.plot(tt,y_exact[0,:],":", label="c1 ex")
    plt.plot(tt,y_exact[1,:],":", label="c2 ex")

    plt.plot(tt,yy[0,:],label="c1")
    plt.plot(tt,yy[1,:],label="c2")
    plt.title("N=%d"%N)
    plt.legend()
```

#### Let's check the order of accuracy of the implicit and explicit Euler!

```python
# Convergence error
def linSysF(y,t=0):
    # evolution function
    F=np.zeros(np.shape(y))
    F[0] = y[1]-5*y[0]
    F[1] = -F[0]
    return F

linSysM=np.array([[-5,1],[5,-1]])

y0=np.array([0.9,0.1])

def exact_sol(t):
    return y0+(1-np.exp(-6*t))/6*np.dot(linSysM,y0)

def error(tt,yy):
    '''
    Compute the average error over the whole time domain,
    in norm 2 on the components of the system
    '''
    errors=np.zeros(len(tt))
    for it, t in enumerate(tt):
        errors[it]=np.linalg.norm(yy[:,it]-exact_sol(t))
    return np.mean(errors)

Ns=[2**k for k in range(1,12)]
errorEx=np.zeros(len(Ns))
errorIm=np.zeros(len(Ns))
dts=    np.zeros(len(Ns))

for iN, N in enumerate(Ns):
    tspan=np.linspace(0,3,N)
    dts[iN]=tspan[1]-tspan[0]

    tt,yy=explicitEuler(linSysF,tspan,y0)
    errorEx[iN]=error(tt,yy)
    tt,yy=implicitEulerLinear(linSysM,tspan,y0)
    errorIm[iN]=error(tt,yy)

plt.figure()
plt.loglog(dts,errorEx,label="ex Euler")
plt.loglog(dts,errorIm, label="im Euler")
plt.loglog(dts,0.1*dts,":", label="order 1")
plt.loglog(dts,0.1*dts**2., ":", label="order 2")
plt.legend()
```

```python
### Test the implicit Euler method with the linear system with purely imaginary eigenvalues

```

## Extra exercise: code implicit Euler for nonlinear fluxes
* Use a nonlinear solver to solve $y^{n+1}-\Delta t F(y^{n+1},t^{n+1})=y^n$ (**scipy.optimize.newton**, scipy.optimize.broyden1)
* Use lambda function to define the nonlinear function
* Search the documentation on Google

```python
def implicitEuler(func, tspan, y_0):
    '''
    Implicit Euler method with a nonlinear solver
    Input:
    func (nonlinear) function fo the ODE
    tspan vector of timesteps (t^0,...,t^N)
    y_0 initial value
    '''
    N_time=len(tspan)  # N+1
    dim=len(y_0)          # S
    y=np.zeros((dim,N_time))    # initializing the variable of solutions
    y[:,0]=y_0                 # first timestep
    for n in range(N_time-1):    # n=0,..., N-1
        FILL IN WITH THE OPERATIONS THAT LEAD
        TO THE SOLUTION OF THE NONLIENEAR FUNCTION
        y[:,n+1] = ###
    return tspan, y
```

```python
## Nonlinear 3x3 system production destruction
def nonlinear_system3_flux(u,t=0):
    ff=np.zeros(len(u))
    ff[0]= -u[0]*u[1]/(u[0]+1)
    ff[1]= u[0]*u[1]/(u[0]+1) -0.3*u[1]
    ff[2]= 0.3*u[1]
    return ff

y_0 = np.array([9.98,0.01,0.01])
T_fin = 30
```

```python
## Run implicit Euler method and plot the solution
tt=np.linspace(0,T_fin, 100)
tt,yy = implicitEuler(nonlinear_system3_flux, tt, y_0)
plt.figure(figsize=(10,4))
plt.subplot(121)
plt.title("implicit Euler")
plt.plot(tt,yy[0,:])
plt.plot(tt,yy[1,:])
plt.plot(tt,yy[2,:])

tt,yy = explicitEuler(nonlinear_system3_flux, tt, y_0)
plt.subplot(122)
plt.title("explicit Euler")
plt.plot(tt,yy[0,:])
plt.plot(tt,yy[1,:])
plt.plot(tt,yy[2,:])
```

```python
## Nonlinear stiff problem: Robertson

def Robertson_flux(u,t=0,alpha=10**4,beta=0.04, gamma=3*10**7):
    ff=np.zeros(np.shape(u))
    ff[0] = alpha*u[1]*u[2]-beta*u[0]
    ff[1] = beta*u[0]-alpha*u[1]*u[2] - gamma*u[1]**2
    ff[2] = gamma*u[1]**2
    return ff

NN=10000
tt = np.array([10**k for k in np.linspace(-7,11,NN)])
y_0 = np.array([1.,10**-20,10**-20])
```

```python
tt,yy = implicitEuler(Robertson_flux, tt, y_0)
plt.semilogx(tt,yy[0,:])
plt.semilogx(tt,yy[1,:]*10**4)
plt.semilogx(tt,yy[2,:])
plt.ylim([-0.05, 1.05])
```

```python
tt,yy = explicitEuler(Robertson_flux, tt, y_0)
plt.semilogx(tt,yy[0,:])
plt.semilogx(tt,yy[1,:]*10**4)
plt.semilogx(tt,yy[2,:])
plt.ylim([-0.05, 1.05])
```


---
## 01. Integradores Simplécticos

Eduard Larrañaga (ealarranaga@unal.edu.co)

---

### Resumen

Se presentan los integradores LeapFrog y Verlet para resolver sistemas de ODEs.

`A. Garcia. Numerical Methods for Physics. (1999). Chapter 3 `

---

## Integrador LeapFrog

Considere el siguiente sistema de ODEs,

\begin{equation}
\frac{d^2 x}{dt^2} = a(x),
\end{equation}

que puede ser escrito como el sistema diferencial de primer orden

\begin{align}
\begin{cases}
\frac{d x}{dt} = &v \\
\frac{dv}{dt} = & a(x).
\end{cases}
\end{align}

**Note la dependencia de la función $a = a(x(t))$ el el lado derecho de las ecuaciones.**

Discretización de la derivada temporal (utilizando diferencias centradas) en la segunda ecuación da como resultado

\begin{equation}
\frac{v(t+\Delta t) - v(t-\Delta t)}{2\Delta t} + \mathcal{O}(\Delta t^2)= a(x).
\end{equation}

En la primera ecuación diferencial también se utilizará una discretización centrada de la derivada, pero en este caso se centrará en el punto $t+\Delta t$ (esto quiere decir que allí se utilizaran los puntos $t+2\Delta t$ y $t$ en la diferencia finita). Con ello se obtiene

\begin{equation}
\frac{x(t+2\Delta t) - x(t)}{2\Delta t} + \mathcal{O}(\Delta t^2)= v(t+\Delta t).
\end{equation}

Estas dos ecuaciones de diferencias se escriben en forma iterativa asi,

\begin{align}
\begin{cases}
\frac{x_{n+2} - x_n}{2\Delta t} + \mathcal{O}(\Delta t^2) &=  v_{n+1} \\
\frac{v_{n+1} - v_{n-1}}{2\Delta t} + \mathcal{O}(\Delta t^2) &= a_n.
\end{cases}
\end{align}

Re-acomodando estas ecuaciones, se obtiene el método iteratio conocido como **Leap-Frog**,

\begin{align}
\begin{cases}
v_{n+1} &=  v_{n-1} + 2 a_n \Delta t  + \mathcal{O}(\Delta t^3)\\
x_{n+2} &=  x_n + 2 v_{n+1}\Delta t   + \mathcal{O}(\Delta t^3)
\end{cases}
\end{align}

Este es un método de orden $\mathcal{O}(\Delta t^3)$ y su nombre se debe a que la solución se construye con pasos de tamaño $2\Delta t$, con la función de posición $x$ avanzando en valores pares, $x_0, x_2, x_4, ...$, mientras que la función velocidad $v$ avanza en valores impares $v_1, v_3, v_5, ...$.

En algunas referencias, las ecuaciones del método LeapFrog se escriben en la siguiente forma (equivalente al método de *Euler-Cromer*):

\begin{align}
\begin{cases}
v_{n+\frac{1}{2}} &=  v_{n-\frac{1}{2}} +  a_n \Delta t  + \mathcal{O}(\Delta t^3)\\
x_{n+1} &=  x_n +  v_{n+\frac{1}{2}}\Delta t   + \mathcal{O}(\Delta t^3)
\end{cases}
\end{align}

El origen del nombre de este método está en la dependencia de la función $a=a(x)$, la cual implica que su valor debe ser evaluado en un instante de tiempo centrado entre los valores antiguo y nuevo de la función $v$.

<center>
</a>
</center>

El método LeapFrog es computacionalmente mas eficiente que el método Predictor-Corrector  y requiere menos almacenamiento de información. Al ser uno de los algoritmo **simplecticos** (i.e. que preservan el área en el espacio de fase §), la conservación de la energía es respetada, aún con pasos de tiempo grandes.De esta forma, este tipo de algoritmos puede lograr una disminución apreciable en el tiempo de computo. Sin embargo, en algunos casos en los que se requieren valores de posición y velocidad mas precisos, se suelen preferir algoritmos como el Predictor-Corrector o los métodos RK.

§ Se puede demostrar que el algoritmo Leapfrog preserva las areas encerradas por las trayectorias en el espacio de fase (p vs. x) a lo largo de la evolución del sistema. Esto implica que la energía es conservada durante la evolución.

### Iniciación del Método LeapFrog

El método Leap-Frog no se *auto-inicia* porque, además de los valores iniciales ,$(x_0, v_0)$, es necesario conocer el valore $v_{-1}$. Una de las posibilidades para inciar el algoritmo es realizar un paso de Euler hacia atrás para obtener la información requerida,
\begin{align}
v_{-1} = &v_0 -  a_0\Delta t,
\end{align}

o medio paso hacia atras si se escriben las ecuaciones en la segunda forma presentada,
\begin{align}
v_{-\frac{1}{2}} = &v_0 - \frac{1}{2} a_0 \Delta t.
\end{align}

Se puede mostrar que el método LeapFrog es mucho más rápido y más estable que el método de Euler.

---
## Método Velocidad-Verlet

Una modificación del método LeapFrog consiste en agregar una ecuación extra para poder calcular la velocidad en los mismos instantes que la posición. Este se conoce como método **velocidad-Verlet** y las ecuaciones que lo definen se obtienen a partir de las de LeapFrog, modificando la ecuación de velocidades y agregando una ecuación que incrementa en medio paso la velocidad utilizando Euler,

\begin{equation}
\begin{cases}
v_{n+\frac{1}{2}} = & v_n + \frac{1}{2} a_n \Delta t\\
x_{n+1} = & x_n + v_{n+\frac{1}{2}} \Delta t \\
v_{n+1} = & v_{n+\frac{1}{2}} + \frac{1}{2} a_{n+1} \Delta t
\end{cases}
\end{equation}

---
## Método Posición-Verlet

De forma similar,tmabién se puede modificar el método LeapFrog para obtener ls posiciones en los mismos instantes que la velocidad. Este se conoce como método **posición-Verlet** y las ecuaciones que lo definen se obtienen a partir de las de LeapFrog, modificando la ecuación de velocidades y agregando una ecuación que incrementa en medio paso la velocidad utilizando Euler,

\begin{equation}
\begin{cases}
x_{n+\frac{1}{2}} = & x_n + \frac{1}{2}v_{n} \Delta t\\
v_{n+1} = & v_n + a_{n+\frac{1}{2}} \Delta t\\
x_{n+1} = & x_{n+\frac{1}{2}} + \frac{1}{2}v_{n+1} \Delta t \\
\end{cases}
\end{equation}

---
## Integrador Verlet

Considere de nuevo el sistema de ODEs

\begin{equation}
\frac{d^2 x}{dt^2} = a(x)
\end{equation}

o reescrito, introduciendo el concepto de velocidad, en la forma

\begin{align}
\begin{cases}
\frac{d x}{dt} &= v \\
\frac{d^2 x}{dt^2} &=  a(x).
\end{cases}
\end{align}

Utilizaremos diferencias centradas para la primera y segunda derivada y con ello obtenemos las formulas iterativas

\begin{align}
\begin{cases}
\frac{x_{n+1} - x_{n-1}}{2\Delta t}  + \mathcal{O} (\Delta t^2) &= v_n \\
\frac{x_{n+1} - 2x_n + x_{n-1}}{\Delta t^2} + \mathcal{O} (\Delta t^2)  &= a_n.
\end{cases}
\end{align}

Re-acomodando estas ecuaciones da como resultado

\begin{align}
\begin{cases}
v_n = &\frac{x_{n+1} - x_{n-1}}{2\Delta t}  + \mathcal{O} (\Delta t^2) \\
x_{n+1} = &2x_n - x_{n-1} +  a_n\Delta t^2 + \mathcal{O} (\Delta t^4) .
\end{cases}
\end{align}

Este esquema se conoce como el método de **Verlet**. Nótese que, de nuevo, el método no es auto-iniciable debido a que se necesita los valores de $x_n$ y $x_{n-1}$ para obtener $x_{n+1}$. Además, es importante resaltar que este método puede obtener la evolución de la posición de forma independiente de los valores de la velocidad.

---

## Ventajas de los Métodos Simplécticos

Los métodos simplécticos presentados poseen las siguientes ventajas sobre otros algoritmos:

### 1. Invarianza bajo inversión temporal

Al igual que la mecánica Newtoniana, los algoritmos simplécticos son invariantes bajo inversión temporal. Para comprender esta característica, considere que se incia el algoritmo en un tiempo $t_0$ con una condición $(x_0,v_0) = (X,V)$ y se avanza un paso para encontrar los valores $(x_1,v_1)$ mediante alguno de los métodos (e.g. velocidad-Verlet). Al reemplazar en las ecuaciones se tiene

\begin{equation}
\begin{cases}
v_{\frac{1}{2}} = & V + \frac{1}{2} a_0 \Delta t\\
x_{1} = & X + v_{\frac{1}{2}} \Delta t \\
v_{1} = & v_{\frac{1}{2}} + \frac{1}{2} a_{1} \Delta t
\end{cases}
\end{equation}

y por ello

\begin{equation}
\begin{cases}
x_{1} = & x_0 + v_0 \Delta t + \frac{1}{2} a_0 \Delta t^2 \\
v_{1} = & v_0 + \frac{1}{2} (a_0 + a_{1}) \Delta t.
\end{cases}
\end{equation}

Si ahora se inicia el algoritmo con la condición inicial $(x_0^r, v_0^r) = (x_1, -v_1)$ obtenidos arriba, las ecuaciones del método resultan en

\begin{equation}
\begin{cases}
v^r_{\frac{1}{2}} = & -v_1 + \frac{1}{2} a^r_0 \Delta t\\
x^r_{1} = & x_1 + v^r_{\frac{1}{2}} \Delta t \\
v^r_{1} = & v^r_{\frac{1}{2}} + \frac{1}{2} a^r_{1} \Delta t
\end{cases}
\end{equation}

con lo que se obtiene

\begin{equation}
\begin{cases}
x^r_{1} = & x_1 -v_1 \Delta t + \frac{1}{2} a^r_0 \Delta t^2 \\
v^r_{1} = & -v_1 + \frac{1}{2} a^r_0 \Delta t + \frac{1}{2} a^r_{1} \Delta t
\end{cases}
\end{equation}

Al reemplazar los valores obtenidos en la primera parte,

\begin{equation}
\begin{cases}
x^r_{1} = & x_0 + v_0 \Delta t + \frac{1}{2} a_0 \Delta t^2 - \left( v_0 + \frac{1}{2} (a_0 + a_{1}) \Delta t \right) \Delta t + \frac{1}{2} a^r_0 \Delta t^2 \\
v^r_{1} = & -v_0 - \frac{1}{2} (a_0 + a_{1}) \Delta t   + \frac{1}{2} a^r_0 \Delta t + \frac{1}{2} a^r_{1} \Delta t
\end{cases}
\end{equation}

\begin{equation}
\begin{cases}
x^r_{1} = & x_0   - \frac{1}{2}  a_{1} \Delta t^2 + \frac{1}{2} a^r_0 \Delta t^2 \\
v^r_{1} = & -v_0 - \frac{1}{2} (a_0 + a_{1} - a^r_0 - a^r_{1}) \Delta t  .
\end{cases}
\end{equation}

Finalmente, identificando que $a^r_0 \equiv a_1$ y $a^r_1 \equiv a_0$ se tiene el resultado buscado

\begin{equation}
\begin{cases}
x^r_{1} = & x_0    \\
v^r_{1} = & -v_0.
\end{cases}
\end{equation}

## Conservación del Área en el Espacio de Fase (Conservación de la Energía)

El adjetivo *Simpléctico* hace referencia a la conservación del área en el espacio de fase del sistema físico.

Considere los cuatro vértices de un cuadrado en el espacio de fase en un instante de tiempo $t$: $(x,p)$, $(x+dx,p)$, $(x,p+dp)$ y $(x+dx,p+dp)$, los cuales encierran un área $dA=dxdp$. Para un tiempo posterior, $t'$, los cuatro puntos cambiarán de ubicación, pero si la evolución es conservativa (de acuerdo con el teorema de Liouville), la nueva área encerrada será igual a la inicial, $dA' = dA$.

La evolución temporal se puede representar como una transformación de coordenadas

\begin{equation}
\begin{cases}
x' =& x'(x,p)\\
p' =& p'(x,p)
\end{cases}
\end{equation}

Bajo la transformación de coordenadas, el área encerrada en el espacio de fase cambia de acuerdo con

\begin{equation}
dA' = \det{J} dA
\end{equation}

donde $\det{J}$ es el *Jacobiano* de la transformación y

\begin{equation}
J = \begin{pmatrix}
\frac{\partial x'}{\partial x} & \frac{\partial x'}{\partial p} \\
\frac{\partial p'}{\partial x} & \frac{\partial p'}{\partial p}
\end{pmatrix}.
\end{equation}

Si la transformación (i.e. la evolución) es simpléctica, se tiene $\det{J} = 1$.

Para comprobar que los algoritmos presentados son simplécticos, calcularemos el Jacobiano. Por ejemplo, tomaremos las ecuaciones del método velocidad-Verlet. Al aplicar la primera de ellas a los puntos $(x_0, v_0)$ y $(x_0+\delta x_0, v_0 + \delta v_0)$, se obtienen las ecuaciones

\begin{equation}
v_{\frac{1}{2}} =  v_0 + \frac{1}{2} a_0 \Delta t \hspace{2cm}; \hspace{2cm}
v_{\frac{1}{2}} + \delta v_{\frac{1}{2}} =  v_0 +\delta v_0 + \frac{1}{2} a(x_0 + \delta x_0) \Delta t
\end{equation}

Restando estas relaciones se tiene

\begin{align}
\delta v_{\frac{1}{2}} = & \delta v_0 + \frac{1}{2} a(x_0 + \delta x_0) \Delta t - \frac{1}{2} a_0 \Delta t\\
\delta v_{\frac{1}{2}} = & \delta v_0 + \frac{1}{2} \left(a(x_0 + \delta x_0)  - a_0\right) \Delta t\\
\delta v_{\frac{1}{2}} = & \delta v_0 + \frac{1}{2} \frac{\left(a(x_0 + \delta x_0)  - a_0\right)}{\delta x_0} \delta x_0 \Delta t.
\end{align}

Tomando el límite cuando $\delta x_0 \rightarrow 0$, se puede escribir

\begin{align}
\delta v_{\frac{1}{2}} = & \delta v_0 + \frac{1}{2} \left. \frac{da}{dx} \right|_{x_0} \delta x_0 \Delta t.
\end{align}

De esta forma, se puede escribir la transformación en forma matricial,

\begin{equation}
\begin{pmatrix}
\delta x_0 \\
\delta v_{\frac{1}{2}}
\end{pmatrix} = B
\begin{pmatrix}
\delta x_0 \\
\delta v_0
\end{pmatrix}
\end{equation}

donde

\begin{equation}
B =
\begin{pmatrix}
1 & 0 \\
\frac{1}{2}a'(x_0) \Delta t & 1
\end{pmatrix}.
\end{equation}

En la misma forma, pero utilizando la segunda y tercera ecuación del método de velocidad-Verlet, se obtienen las relaciones

\begin{equation}
\begin{pmatrix}
\delta x_1 \\
\delta v_{\frac{1}{2}}
\end{pmatrix} = C
\begin{pmatrix}
\delta x_0 \\
\delta v_{\frac{1}{2}}
\end{pmatrix}
\hspace{2cm} ; \hspace{2cm}
\begin{pmatrix}
\delta x_1 \\
\delta v_1
\end{pmatrix} = D
\begin{pmatrix}
\delta x_1 \\
\delta v_{\frac{1}{2}}
\end{pmatrix}
\end{equation}

donde

\begin{equation}
C =
\begin{pmatrix}
1 & \Delta t \\
0 & 1
\end{pmatrix}
\hspace{2cm} y \hspace{2cm}
D =
\begin{pmatrix}
1 & 0 \\
\frac{1}{2}a'(x_1) \Delta t & 1
\end{pmatrix}
.
\end{equation}

De estas realciones se obtiene la transformación

\begin{equation}
\begin{pmatrix}
\delta x_1 \\
\delta v_1
\end{pmatrix} = J
\begin{pmatrix}
\delta x_0 \\
\delta v_0
\end{pmatrix}
\end{equation}

con $J = DCB$. El determinante de esta matriz (Jacobiano) es

\begin{equation}
\det J = \det D \det C \det B.
\end{equation}

Ya que cada una de las matrices construidas satisface $\det B = \det C = \det D = 1$, se tiene que $\det J = 1$ y por lo tanto el método de itnegración es *simpléctico*. El mismo proceso puede ser aplicado a los metodos LeapFrog, posición-Verlet y Verlet.

```python

```

07. ODEs II. Integradores Simplecticos/01. Integradores Simplecticos.ipynb


Introduction
-------------------

Brzezniak (2000) is a great book because it approaches conditional expectation through a sequence of exercises, which is what we are trying to do here. The main difference is that Brzezniak takes a more abstract measure-theoretic approach to the same problems. Note that you *do* need to grasp the measure-theoretic to move into more advanced areas in stochastic processes, but for what we have covered so far, working the same problems in his text using our methods is illuminating. It always helps to have more than one way to solve *any* problem.  I urge you to get a copy of his book or at least look at some pages on Google Books. I have numbered the examples corresponding to the book.

Examples
-------------

This is Example 2.1 from Brzezniak:

> Three coins, 10p, 20p and 50p are tossed. The values of those coins that land heads up are added to work out the total amount. What is the expected total amount given that two coins have landed heads up?

In this case we have we want to compute $\mathbb{E}(\xi|\eta)$ where

$$ \xi = 10 X_{10} + 20 X_{20} +50 X_{50} $$

where $X_i \in \{ 0,1\} $. This represents the sum total value of the heads-up coins. The $\eta$ represents the fact that only two of the three coins are heads-up. Note

$$\eta = X_{10} X_{20} (1-X_{50})+ (1-X_{10}) X_{20} X_{50}+ X_{10} (1-X_{20}) X_{50} $$

is a function that is only non-zero when two of the three coins is heads. Each triple term catches each of these three possibilities. For example, the first term is when the 10p and 20p are heads up  and the 50p is heads down.

To compute the conditional expectation, we want to find a function $h$ of $\eta$ that minimizes the MSE

$$ \sum_{X\in\{0,1\}^3} \frac{1}{8} (\xi - h( \eta ))^2 $$

where the sum is taken over all possible triples of outcomes for $ \{X_{10} , X_{20} ,X_{50}\}$ and the $\frac{1}{8} = \frac{1}{2^3} $ since each coin has a $\frac{1}{2}$ chance of coming up heads.

Now, the question boils down to what function $h(\eta)$ should we try? Note that $\eta \in \{0,1\}$ so $h$ takes on only two values. Thus, we only have to try $h(\eta)=\alpha \eta$  and find $\alpha$. Writing this out gives,

$$ \sum_{X\in\{0,1\}^3} \frac{1}{8} (\xi - \alpha( \eta ))^2 $$

which boils down to solving for $\alpha$,

$$\langle \xi , \eta \rangle = \alpha \langle \eta,\eta \rangle$$

where

$$ \langle \xi , \eta \rangle =\sum_{X\in\{0,1\}^3} \frac{1}{8} (\xi  \eta ) $$

This is tedious and a perfect job for `sympy`.

```
import sympy as S

eta = S.Symbol('eta')
xi = S.Symbol('xi')
X10 = S.Symbol('X10')
X20 = S.Symbol('X20')
X50 = S.Symbol('X50')

eta = X10 * X20 *(1-X50 )+ X10 * (1-X20) *(X50 )+ (1-X10) * X20 *(X50 )
xi = 10*X10 +20* X20+ 50*X50

num=S.summation(xi*eta,(X10,0,1),(X20,0,1),(X50,0,1))
den=S.summation(eta*eta,(X10,0,1),(X20,0,1),(X50,0,1))
alpha=num/den
print alpha
```

    160/3

This means that

$$ \mathbb{E}(\xi|\eta) = \frac{160}{3} \eta $$

which we can check with a quick simulation

```
import numpy as np
from numpy import array

x=np.random.randint(0,2,(3,5000))
print (160/3.,np.dot(x[:,x.sum(axis=0)==2].T,array([10,20,50])).mean())

```

    (53.333333333333336, 53.243528790279981)

Example
--------

This is example 2.2:

> Three coins, 10p, 20p and 50p are tossed as before. What is the conditional expectation of the total amount shown by the three coins given the total amount shown by the 10p and 20p coins only?

For this problem,

$$\eta = 30 X_{10} X_{20} + 20 (1-X_{10}) X_{20}  + 10 X_{10} (1-X_{20})  $$

which takes on three values (10,20,30) and only considers the 10p and 20p coins. Here, we'll look for affine functions, $h(\eta) = a \eta + b $.

```
from sympy.abc import a,b

eta = X10 * X20 * 30 + X10 * (1-X20) *(10 )+ (1-X10) * X20 *(20 )
h = a*eta + b

J=S.summation((xi - h)**2 * S.Rational(1,8),(X10,0,1),(X20,0,1),(X50,0,1))

sol=S.solve( [S.diff(J,a), S.diff(J,b)],(a,b) )
print sol
```

    {b: 25, a: 1}

This means that

$$ \mathbb{E}(\xi|\eta) = 25+ \eta $$

since $\eta$ takes on only four values, $\{0,10,20,30\}$, we can write this out as

$$ \mathbb{E}(\xi|\eta=0) = 25  $$
$$ \mathbb{E}(\xi|\eta=10) = 35  $$
$$ \mathbb{E}(\xi|\eta=20) = 45  $$
$$ \mathbb{E}(\xi|\eta=30) = 55  $$

The following is  a quick simulation to demonstrate this.

```
x=np.random.randint(0,2,(3,5000))  # random samples for 3 coins tossed
eta=np.dot(x[:2,:].T,array([10,20])) # sum of 10p and 20p

print np.dot(x[:,eta==0].T,array([10,20,50])).mean() # E(xi|eta=0)
print np.dot(x[:,eta==10].T,array([10,20,50])).mean()# E(xi|eta=10)
print np.dot(x[:,eta==20].T,array([10,20,50])).mean()# E(xi|eta=20)
print np.dot(x[:,eta==30].T,array([10,20,50])).mean()# E(xi|eta=30)
```

    25.1587301587
    34.3410852713
    43.7323358271
    56.6238973536

Example
-----------

This is Example 2.3

Note that "Lebesgue measure" on $[0,1]$ just means uniformly distributed on that interval. Also, note the the `Piecewise` object in `sympy` is not complete at this point in its development, so we'll have to work around that in the following.

```
%pylab inline
```

    Populating the interactive namespace from numpy and matplotlib

```
x=S.Symbol('x')
c=S.Symbol('c')

xi = 2*x**2

eta=S.Piecewise((1,S.And(S.Gt(x,0),S.Lt(x,S.Rational(1,3)))),  #  0 < x < 1/3
                (2,S.And(S.Gt(x,S.Rational(1,3)),S.Lt(x,S.Rational(2,3)))), # 1/3 < x < 2/3,
                (0,S.And(S.Gt(x,S.Rational(2,3)),S.Lt(x,1))),
                )

h = a + b*eta + c*eta**2
J=S.integrate((xi - h)**2 ,(x,0,1))

sol=S.solve( [S.diff(J,a),
              S.diff(J,b),
              S.diff(J,c),
              ],
              (a,b,c) )
print sol

print S.piecewise_fold(h.subs(sol))
```

    {c: 8/9, b: -20/9, a: 38/27}
    Piecewise((2/27, And(x < 1/3, x > 0)), (14/27, And(x < 2/3, x > 1/3)), (38/27, And(x < 1, x > 2/3)))

Thus, collecting this result gives:

$$ \mathbb{E}(\xi|\eta) = \frac{38}{27} - \frac{20}{9}\eta + \frac{8}{9} \eta^2$$

which can be re-written as a piecewise function as

$$\mathbb{E}(\xi|\eta) =\begin{cases} \frac{2}{27} & \text{for}\: 0 < x < \frac{1}{3} \\\frac{14}{27} & \text{for}\: \frac{1}{3} < x < \frac{2}{3} \\\frac{38}{27} & \text{for}\: \frac{2}{3}<x < 1 \end{cases}
$$

The following is a quick simulation to demonstrate this.

```
x = np.random.rand(1000)
f,ax= subplots()
ax.hist(2*x**2,bins=array([0,1/3.,2/3.,1])**2*2,normed=True,alpha=.5)
ax.vlines([2/27.,14/27.,38/27.],0,ax.get_ylim()[1],linestyles='--')
ax.set_xlabel(r'$2 x^2$',fontsize=18);
```

This plot shows the intervals that correspond to the respective domains of $\eta$ with the vertical dotted lines showing the $\mathbb{E}(\xi|\eta) $ for that piece.

Example
-----------

This is Example 2.4

```
x,a=S.symbols('x,a')

xi = 2*x**2

half = S.Rational(1,2)

eta_0=S.Piecewise((2, S.And(S.Ge(x,0), S.Lt(x,half))),
                  (0, S.And(S.Ge(x,half), S.Le(x,1))))

eta_1=S.Piecewise((0, S.Lt(x,half)),
                  (x, S.And(S.Ge(x,half),S.Le(x,1))))

v=S.var('b:3') # coefficients for quadratic function of eta
h = a*eta_0 + (eta_1**np.arange(len(v))*v).sum()
J=S.integrate((xi - h)**2 ,(x,0,1))
sol=S.solve([J.diff(i) for i in v+(a,)],v+(a,))
hsol = h.subs(sol)
f=S.lambdify(x,hsol,'numpy')
print S.piecewise_fold(h.subs(sol))
t = np.linspace(0,1,51,endpoint=False)

fig,ax = subplots()
ax.plot(t, 2*t**2,label=r'$\xi=2 x^2$')
ax.plot(t,[f(i) for i in t],'-x',label=r'$\mathbb{E}(\xi|\eta)$')
ax.plot(t,map(S.lambdify(x,eta_0+eta_1),t),label=r'$\eta(x)$')
ax.set_ylim(ymax = 2.3)
ax.grid()
ax.legend(loc=0);
#ax.plot(t,map(S.lambdify(x,eta),t))
```

The figure shows the $\mathbb{E}(\xi|\eta)$ against $\xi$ and $\eta$. Note that $\xi= \mathbb{E}(\xi|\eta)= 2 x^2$ when $x\in[0,\frac{1}{2}]$ . Assembling the solution gives,

$$\mathbb{E}(\xi|\eta) =\begin{cases} \frac{1}{6} & \text{for}\: 0 \le x < \frac{1}{2} \\ 2 x^2 & \text{for}\: \frac{1}{2} < x \le 1 \end{cases}$$

This example warrants more a more detailed explanation since $\eta$ is more complicated. The first question is why did we choose $h(\eta)$ as a quadratic function? Since $\xi$ is a squared function of $x$ and since $x$ is part of $\eta$, we chose a quadratic function so that $h(\eta)$ would contain a $x^2$ in the domain where $\eta=x$. The motivation is that we are asking for a function $h(x)$ that most closely approximates $2x^2$. Well, obviously, the exact function is $h(x)=2 x^2$! Thus, we want $h(x)=2 x^2$ over the domain where $\eta=x$, which is $x\in[\frac{1}{2},1]$ and that is exactly what we have.

We could have used our inner product by considering two separate functions,

$\eta_1 (x) = 2$

where $x\in [0,\frac{1}{2}]$ and

$$\eta_2 (x) = x$$

where $x\in [\frac{1}{2},1]$. Thus, at the point of projection, we have

$$ \mathbb{E}((2 x^2 - 2 c) \cdot 2) = 0$$

which leads to

$$\int_0^{\frac{1}{2}} 2 x^2 \cdot 2 dx = \int_0^{\frac{1}{2}} c 2 \cdot 2   dx $$

and a solution for $c$,

$$ c = \frac{1}{12} $$

Assembling the solution for $x\in[0,\frac{1}{2}]$ gives

$$ \mathbb{E}(\xi|\eta) =  \frac{2}{12}$$

We can do the same thing for the other piece, $\eta_2$,

$$ \mathbb{E}((2 x^2 - c x^2) \cdot x) = 0$$

which, by inspection, gives $c=2$. Thus, for $x\in[\frac{1}{2},1]$ , we have

$$ \mathbb{E}(\xi|\eta)= 2 x^2$$

which is what we had before.

Example
-----------

This is Exercise 2.6

```
x,a=S.symbols('x,a')

xi = 2*x**2
eta = 1 - abs(2*x-1)

half = S.Rational(1,2)

eta=S.Piecewise((1+(2*x-1), S.And(S.Ge(x,0),S.Lt(x,half))),
                (1-(2*x-1), S.And(S.Ge(x,half),S.Lt(x,1))))

v=S.var('b:3') # assume h is quadratic in eta

h = (eta**np.arange(len(v))*v).sum()

J=S.integrate((xi - h)**2 ,(x,0,1))
sol=S.solve([J.diff(i) for i in v],v)
hsol = h.subs(sol)

print S.piecewise_fold(h.subs(sol))

t = np.linspace(0,1,51,endpoint=False)

fig,ax = subplots()
fig.set_size_inches(5,5)

ax.plot(t, 2*t**2,label=r'$\xi=2 x^2$')
ax.plot(t,[hsol.subs(x,i) for i in t],'-x',label=r'$\mathbb{E}(\xi|\eta)$')
ax.plot(t,map(S.lambdify(x,eta),t),label=r'$\eta(x)$')
ax.legend(loc=0)
ax.grid()

```

The figure shows that the $\mathbb{E}(\xi|\eta)$ is continuous over the entire domain. The code above solves for the conditional expectation using optimization assuming that $h$ is a quadratic function of $\eta$, but we can also do it by using the inner product. Thus,

$$ \mathbb{E}\left((2 x^2 - h(\eta_1(x)) )\eta_1(x)\right)= \int_0^{\frac{1}{2}} (2 x^2 - h(\eta_1(x)) )\eta_1(x)  dx = 0$$

where $\eta_1 = 2x $ for $x\in [0,\frac{1}{2}]$. We can re-write this in terms of $\eta_1$ as

$$ \int_0^1 \left(\frac{\eta_1^2}{2}-h(\eta_1)\right)\eta_1 d\eta_1$$

and the solution jumps right out as $h(\eta_1)=\frac{\eta_1^2}{2}$. Note that $\eta_1\in[0,1]$. Doing the same thing for the other piece,

$$ \eta_2  = 2 - 2 x, \hspace{1em} \forall x\in[\frac{1}{2},1]$$

gives,

$$ \int_0^1 \left(\frac{(2-\eta_2)^2}{2}-h(\eta_2)\right)\eta_2 d\eta_2$$

and again, the optimal $h(\eta_2)$ jumps right out as

$$  h(\eta_2) = \frac{(2-\eta_2)^2}{2} , \hspace{1em} \forall \eta_2\in[0,1]$$

and since $\eta_2$ and $\eta_2$ represent the same variable over the same domain we can just add these up to get the full solution:

$$ h(\eta) = \frac{1}{2} \left( 2 - 2 \eta + \eta^2\right) $$

and then back-substituting each piece for $x$ produces the same solution as `sympy`.

Example
-----------

This is Exercise 2.14

```
x,a=S.symbols('x,a')
half = S.Rational(1,2)

xi = 2*x**2
eta=S.Piecewise((2*x,    S.And(S.Ge(x,0),S.Lt(x,half))),
                ((2*x-1),S.Ge(x,half)),
               )

v=S.var('b:3')

h = (eta**np.arange(len(v))*v).sum()

J=S.integrate((xi - h)**2 ,(x,0,1))
sol=S.solve([J.diff(i) for i in v],v)
hsol = h.subs(sol)

print S.piecewise_fold(h.subs(sol))

t = np.linspace(0,1,51,endpoint=False)

fig,ax = subplots()
fig.set_size_inches(5,5)

ax.plot(t, 2*t**2,label=r'$\xi=2 x^2$')
ax.plot(t,[hsol.subs(x,i) for i in t],'-x',label=r'$\mathbb{E}(\xi|\eta)$')
ax.plot(t,map(S.lambdify(x,eta),t),label=r'$\eta(x)$')
ax.legend(loc=0)
ax.grid()
```

As before, using the inner product for this problem, gives:

$$ \int_0^1 \left(\frac{\eta_1^2}{2}-h(\eta_1)\right)\eta_1 d\eta_1=0$$

and the solution jumps right out as
$$h(\eta_1)=\frac{\eta_1^2}{2}  , \hspace{1em} \forall \eta_1\in[0,1$$

where $\eta_1(x)=2x$. Doing the same thing for $\eta_2=2x-1$ gives,

$$ \int_0^1 \left(\frac{(1+\eta_2)^2}{2}-h(\eta_2)\right)\eta_1 d\eta_2=0$$

with

$$h(\eta_2)=\frac{(1+\eta_2)^2}{2} , \hspace{1em} \forall \eta_2\in[0,1]$$

and then adding these up as before gives the full solution:

$$ h(\eta)= \frac{1}{2} +\eta + \eta^2$$

Back-substituting each piece for $x$ produces the same solution as `sympy`.

```
xs = np.random.rand(100)
print np.mean([(2*i**2-hsol.subs(x,i))**2 for i in xs])
print S.integrate((2*x**2-hsol)**2,(x,0,1)).evalf()

```

    0.282679822464177
    0.270833333333333

## Summary

We worked out some of the great examples in Brzezniak's book using our methods as a way to show multiple ways to solve the same problem. In particular, comparing Brzezniak's more measure-theoretic methods to our less abstract techniques is a great way to get a handle on those concepts which you will need for more advanced study in stochastic process.

As usual, the corresponding [IPython Notebook](www.ipython.org) notebook for this post  is available for download [here](https://github.com/unpingco/Python-for-Signal-Processing/blob/master/Conditional_expectation_MSE_Ex.ipynb)

Comments and corrections welcome!

References
---------------

* Brzezniak, Zdzislaw, and Tomasz Zastawniak. Basic stochastic processes: a course through exercises. Springer, 2000.

```

```


# Xhistogram Tutorial

Histograms are the foundation of many forms of data analysis.
The goal of xhistogram is to make it easy to calculate weighted histograms in multiple dimensions over n-dimensional arrays, with control over the axes.
Xhistogram builds on top of xarray, for automatic coordiantes and labels, and dask, for parallel scalability.

## Toy Data

We start by showing an example with toy data. First we use xarray to create some random, normally distributed data.

### 1D Histogram

```python
import xarray as xr
import numpy as np
%matplotlib inline

nt, nx = 100, 30
da = xr.DataArray(np.random.randn(nt, nx), dims=['time', 'x'],
                  name='foo') # all inputs need a name
display(da)
da.plot()
```

By default xhistogram operates on all dimensions of an array, just like numpy. However, it operates on xarray DataArrays, taking labels into account.

```python
from xhistogram.xarray import histogram

bins = np.linspace(-4, 4, 20)
h = histogram(da, bins=[bins])
display(h)
h.plot()
```

**TODO:**
- Bins needs to be a list; this is annoying, would be good to accept single items
- The `foo_bin` coordinate is the estimated bin center, not the bounds. We need to add the bounds to the coordinates, but we can as long as we are returning DataArray and not Dataset.

Both of the above need GitHub Issues

### Histogram over a single axis

```python
h_x = histogram(da, bins=[bins], dim=['time'])
h_x.plot()
```

**TODO:**
  - Relax / explain requirement that dims is always a list

```python
h_x.mean(dim='x').plot()
```

### Weighted Histogram

Weights can be the same shape as the input:

```python
weights = 0.4 * xr.ones_like(da)
histogram(da, bins=[bins], weights=weights)
```

Or can use Xarray broadcasting:

```python
weights = 0.2 * xr.ones_like(da.x)
histogram(da, bins=[bins], weights=weights)
```

## 2D Histogram

Now let's say we have multiple input arrays. We can calculate their joint distribution:

```python
db = xr.DataArray(np.random.randn(nt, nx), dims=['time', 'x'],
                  name='bar') - 2

histogram(da, db, bins=[bins, bins]).plot()
```

## Real Data Example

### Ocean Volume Census in TS Space

Here we show how to use xhistogram to do a volume census of the ocean in Temperature-Salinity Space

First we open the World Ocean Atlas dataset from the opendap dataset (http://apdrc.soest.hawaii.edu/dods/public_data/WOA/WOA13/1_deg/annual).

Here we read the annual mean Temparature, Salinity and Oxygen on a 5 degree grid.

```python
# Read WOA using opendap
Temp_url = 'http://apdrc.soest.hawaii.edu:80/dods/public_data/WOA/WOA13/5_deg/annual/temp'
Salt_url = 'http://apdrc.soest.hawaii.edu:80/dods/public_data/WOA/WOA13/5_deg/annual/salt'
Oxy_url = 'http://apdrc.soest.hawaii.edu:80/dods/public_data/WOA/WOA13/5_deg/annual/doxy'

ds = xr.merge([
    xr.open_dataset(Temp_url).tmn.load(),
    xr.open_dataset(Salt_url).smn.load(),
    xr.open_dataset(Oxy_url).omn.load()])
ds
```

Use histogram to bin data points. Use canonical ocean T/S ranges, and bin size of $0.1^0C$, and $0.025psu$. Similar ranges and bin size as this review paper on Mode Waters: https://doi.org/10.1016/B978-0-12-391851-2.00009-X .

```python
sbins = np.arange(31,38, 0.025)
tbins = np.arange(-2, 32, 0.1)
```

```python
# histogram of number of data points
# histogram of number of data points
hTS = histogram(ds.smn, ds.tmn, bins=[sbins, tbins])
np.log10(hTS.T).plot(levels=31)
```

However, we would like to do a volume census, which requires the data points to be weighted by volume of the grid box.

\begin{equation}
dV = dz*dx*dy
\end{equation}

```python
# histogram of number of data points weighted by volume resolution
# Note that depth is a non-uniform axis

# Create a dz variable
dz = np.diff(ds.lev)
dz =np.insert(dz, 0, dz[0])
dz = xr.DataArray(dz, coords= {'lev':ds.lev}, dims='lev')

# weight by volume of grid cell (resolution = 5degree, 1degree=110km)
dVol = dz * (5*110e3) * (5*110e3*np.cos(ds.lat*np.pi/180))

# Note: The weights are automatically broadcast to the right size
hTSw = histogram(ds.smn, ds.tmn, bins=[sbins, tbins], weights=dVol)
np.log10(hTSw.T).plot(levels=31, vmin=11.5, vmax=16, cmap='brg')
```

The ridges of this above plot are indicative of T/S classes with a lot of volume, and some of them are indicative of Mode Waters (example Eighteen Degree water with T$\sim18^oC$, and S$\sim36.5psu$.

#### Averaging a variable

Next we calculate the mean oxygen value in each TS bin.

\begin{equation}
\overline{A} (m,n) = \frac{\sum_{T(x,y,z)=m, S(x,y,z)=n} (A(x,y,z) dV)}{\sum_{T(x,y,z)=m, S(x,y,z)=n}dV}.
\end{equation}

```python
hTSO2 = (histogram(ds.smn.where(~np.isnan(ds.omn)),
                   ds.tmn.where(~np.isnan(ds.omn)),
                   bins=[sbins, tbins],
                   weights=ds.omn.where(~np.isnan(ds.omn))*dVol)/
                histogram(ds.smn.where(~np.isnan(ds.omn)),
                          ds.tmn.where(~np.isnan(ds.omn)),
                          bins=[sbins, tbins],
                          weights=dVol))

(hTSO2.T).plot(vmin=1, vmax=8)
```

Some interesting patterns in average oxygen emerge. Convectively ventilated cold water have the highest oxygen and mode waters have relatively high oxygen. Oxygen minimum zones are interspersed in the middle of volumetic ridges (high volume waters).

TODO: NaNs in weights are not handled properly, and a few nanvalues make the weighted sum as nan. Maybe use nansum.

## Dask Integration

Should just work, but need examples.


# NAUTILUS basic example

Suppose we have the problem defined in the analytical problem example:

\begin{equation}
\begin{aligned}
& \underset{\mathbf x}{\text{min}}
& & x_1^2 - x_2; x_2^2 - 3x_1 \\
& \text{s.t.} & &  x_1 + x_2 \leq 10 \\
& & &  \mathbf{x} \; \in S, \\
\end{aligned}
\end{equation}

where the feasible region is

\begin{equation}
x_i \in \left[-5, 5\right] \; \forall i \;\in \left[1,2\right].
\end{equation}

```python
from desdeov2.problem.Problem import ScalarMOProblem
from desdeov2.problem.Objective import ScalarObjective
from desdeov2.problem.Variable import Variable
from desdeov2.problem.Constraint import ScalarConstraint

# Args: name, starting value, lower bound, upper bound
x1 = Variable("x_1", 0, -0.5, 0.5)
x2 = Variable("x_2", 0, -0.5, 0.5)

# Args: name, callable
obj1 = ScalarObjective("f_1", lambda x: x[0]**2 - x[1])
obj2 = ScalarObjective("f_2", lambda x: x[1]**2 - 3*x[0])

# Args: name, n of variables, n of objectives, callable
cons1 = ScalarConstraint("c_1", 2, 2, lambda x, _: 10 - (x[0] + x[1]))

# Args: list of objevtives, variables and constraints
problem = ScalarMOProblem([obj1, obj2]
                         ,[x1, x2]
                         ,[cons1])
```

To solve it using ENAUTILUS is simple. Begin by importing and defining the method.

```python
from desdeov2.methods.Nautilus import Nautilus

method = Nautilus(problem)
```

Next, the method must be initialized by supplying it with a desired number of iterations. Let's choose 5 iterations for the sake of the example.

```python
(fs, bounds, d) = method.initialize(5)
print("Current objective values", fs)
print("Lower (worse) and upper (better) bounds of the reachable solutions from the current objective values", bounds)
print("Distance to the Pareto front from the current iteration (0: farthest, 100: closest):", d)
```

Preferences can be expressed in two way. The first way is to simply divide 100 points between all objectives, the more points, the more important it is to improve that objective.

```python
import numpy as np

points = np.array([30, 70])
remaining_iters = method.interact(percentages=points)

(fs, bounds, d) = method.iterate()

print("Objectives: ", fs)
```

Notice how the objectives have both improved. The other way to indicate preference, is to use indexing. Say, we deem objective 1 to be more important, then

```python
index_set = np.array([2, 1])
remaining_iters = method.interact(index_set=index_set)

(fs, bounds, d) = method.iterate()

print("Objectives: ", fs)
```

Again, the objectives have decreased. This is the basic usage of the algorithm. It can support more advanced stuff. like taking steps back or taking shorter steps. Refer to the documentation and other examples.


```python
import numpy
import sympy
from matplotlib import pyplot
%matplotlib inline
from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16
from sympy import init_printing
init_printing()
```

```python
Vmax = 80
L = 11
rho_max = 250
nx = 51
dt = .001 #[hours]
dx = L/nx
```

```python
#Initial Conditions
x = numpy.linspace(0, L, nx)
rho0 = numpy.ones(nx)*10
rho0[10:20] = 50

#Boundary Condition
rho0[0] = 10
```

```python
F, V, t = sympy.symbols('F V t')
#F = Vmax*rho*(1-rho/rho_max)
rho = rho0
```

```python
v0 = Vmax*(1-rho0/rho_max)
vmin = min(v0)/3.6 #units in m/s
print("Min. velocity at t=0 is: {} m/s".format(vmin))
```

```python
#test code, can do one time step at a time.
#rho_n = rho0.copy()
#test = rho_n
#test[0] = 10
#test[1:] = -dt/dx*(Vmax*rho_n[1:]*(1-rho_n[1:]/rho_max) - Vmax*rho_n[0:-1]*(1-rho_n[0:-1]/rho_max)) + rho_n[1:]
#print(test[0])

#pyplot.figure(figsize=(8,5), dpi=100)
#pyplot.plot(x, test, ls='--', lw=3)
#pyplot.xlim([0,20])
#pyplot.ylim([0,60]);
```

```python
T = 3/60
nt = int(T/dt)

for n in range(nt):
    rho_n = rho.copy()

    rho[0] = 10
    rho[1:] = -dt/dx*Vmax*(rho_n[1:]*(1-rho_n[1:]/rho_max) - \
                     rho_n[0:-1]*(1-rho_n[0:-1]/rho_max)) + rho_n[1:]
```

```python
pyplot.figure(figsize=(8,5), dpi=100)
pyplot.plot(x, rho, ls='--', lw=3)
pyplot.xlim([0,20])
pyplot.ylim([0,60]);
```

```python
#find average velocity at t= 3 minutes
V = Vmax*(1- rho/rho_max)
V_ave = numpy.mean(V)/3.6 #units in m/s
print("Average velocity at t=3min. is {}".format(V_ave))

```

```python
Vmax = 80
L = 11
rho_max = 250
nx = 101
dt = .001 #[hours]
dx = L/nx

#Initial Conditions
x = numpy.linspace(0, L, nx)
rho0 = numpy.ones(nx)*10
rho0[10:20] = 50

#Boundary Condition
rho0[0] = 10

rho = rho0 #reset rho to initial condition

#find minimum velocity at t=6 minutes
T = 0.1 # 6 minutes
nt = int(T/dt)

for n in range(nt):
    rho_n = rho.copy()
    rho[0] = 10
    rho[1:] = -dt/dx*Vmax*(rho_n[1:]*(1-rho_n[1:]/rho_max) - \
                     rho_n[0:-1]*(1-rho_n[0:-1]/rho_max)) + rho_n[1:]

V = Vmax*(1-rho0/rho_max)
vmin = min(V)/3.6 #units in m/s
#print(V)
print("Min. velocity at t=6 is: {} m/s".format(vmin))
pyplot.figure(figsize=(8,5), dpi=100)
pyplot.plot(x, rho, ls='--', lw=3)
pyplot.xlim([0,15])
pyplot.ylim([0,60]);
```

```python
Vmax = 136
L = 11
rho_max = 250
nx = 51
dt = .001 #[hours]
dx = L/nx

#Initial Conditions
x = numpy.linspace(0, L, nx)
rho0 = numpy.ones(nx)*10
rho0[10:20] = 50

#Boundary Condition
rho0[0] = 20
rho = rho0
```

```python
v0 = Vmax*(1-rho0/rho_max)
vmin = min(v0)/3.6 #units in m/s
print("Min. velocity at t=0 is: {} m/s".format(vmin))
```

    Min. velocity at t=0 is: 30.222222222222225 m/s

```python
T = 3/60
nt = int(T/dt)

for n in range(nt):
    rho_n = rho.copy()

    rho[0] = 20
    rho[1:] = -dt/dx*Vmax*(rho_n[1:]*(1-rho_n[1:]/rho_max) - \
                     rho_n[0:-1]*(1-rho_n[0:-1]/rho_max)) + rho_n[1:]

pyplot.figure(figsize=(8,5), dpi=100)
pyplot.plot(x, rho, ls='--', lw=3)
pyplot.xlim([0,20])
pyplot.ylim([0,60]);

#find average velocity at t= 3 minutes
V = Vmax*(1- rho/rho_max)
V_ave = numpy.mean(V)/3.6 #units in m/s
print("Average velocity at t=3min. is {}".format(V_ave))
vmin = min(V)/3.6 #units in m/s
print("Minimum velocity at t=3min. is {}".format(vmin))
```

```python
Vmax = 136
L = 11
rho_max = 250
nx = 51
dt = .001 #[hours]
dx = L/nx

#Initial Conditions
x = numpy.linspace(0, L, nx)
rho0 = numpy.ones(nx)*20
rho0[10:20] = 50

#Boundary Condition
rho0[0] = 20

rho = rho0 #reset rho to initial condition

#find minimum velocity at t=6 minutes
T = 0.1 # 6 minutes
nt = int(T/dt)

for n in range(nt):
    rho_n = rho.copy()
    rho[0] = 20
    rho[1:] = -dt/dx*Vmax*(rho_n[1:]*(1-rho_n[1:]/rho_max) - \
                     rho_n[0:-1]*(1-rho_n[0:-1]/rho_max)) + rho_n[1:]

V = Vmax*(1-rho0/rho_max)
vmin = min(V)/3.6 #units in m/s
print("Min. velocity at t=6 is: {} m/s".format(vmin))
pyplot.figure(figsize=(8,5), dpi=100)
pyplot.plot(x, rho, ls='--', lw=3)
pyplot.xlim([0,15])
pyplot.ylim([0,60]);
```

```python

```


# Introduction

This tutorial will guide you through estimating normal vectors and principal curvatures from 3D point clouds using classic PCA and Jet fitting methods as well as our recent DeepFit method.

Surface normals and curvatures are a very important properties in shape analysis and are widely used in different fields like computer graphics, computer vision and more.

### The math
3D point clouds are represented as a matrix of $(x, y, z)$ coordinates of points in 3D space.
If you are unfamiliar with plane fitting and jets, this section provides a short background. For more in-depth information see the refrences at the bottom.

The following methods share the following:
* Input: 3D point cloud  + querry point $q_i$
* Find $q_i$'s k nearest neighbors.
* Do something fancey :)
* Output: Normal vector at  query point $N_{q_i}$

For estimating the normal at each point we use each point in the point cloud as the query point.

#### PCA
In this method we estimate the tangent plane to the underlying surface at the query point. This boils down to  solving the eigenvalue and eigenvector decomposition of the covariance matrix created from the points nearest neighbors:

\begin{equation}
C = \frac{1}{k}\sum_{i=1}^{k}(p_i-\hat{p})(p_i-\hat{p})^T
\end{equation}

Here $p_i$ are the neighboring points and $\hat{p}$ is the neighbours centroid.
The normal vector is the eugenvector associated with the smallest eigenvalue.

Using this method we cannot directly estimate the principal curvatures (the principal curvatures of a plane are 0).

#### Jet fitting

 An $n$-jet of the height function over a surface is given by:

\begin{equation}
    f(x,y)=J_{\beta,n}(x,y)= \sum_{k=0}^{n}\sum_{j=0}^{k}\beta_{k-j,j}x^{k-j}y^j
\end{equation}

Here $\beta$ is the jet coefficients vector that consists of $N_n=(n+1)(n+2)/2$ terms.

We require that every point satisfy the equation above, yielding the system of linear equations:
\begin{equation}
    M\beta = B
\end{equation}

It is well known that the solution can be expressed in closed-form as:
\begin{equation}
    \beta = (M^TM)^{-1}M^TB
\end{equation}

Here $M=(1, x_i, y_i, ..., x_i y_i^{n-1}, y_i^n)_{i=1,...,N_p}\in \mathbb{R}^{N_p \times N_n}$ is the Vandermonde matrix and the height function vector $B=(z_1, z_2,...z_{N_p})^T \in \mathbb{R}^N_p$. Both represent the sampled points.

### The code

```python
import sys
sys.path.insert(0, '../utils')
sys.path.insert(0, '../models')
sys.path.insert(0, '../trained_models')
import DeepFit
import tutorial_utils as tu
import torch
import ipyvolume as ipv
import ipywidgets as widgets
from IPython.display import display
import functools
```

### Generating data
3D point cloud data can be obtained from 3D sensors like LiDAR or RGBD cameras. For this tutorial we will generate a synthetic example. This way we have true ground truth parametric surface as reference for evaluating our results.

```python
jet_order_data = 3
n_points = 4096
point_cloud_dataset = tu.SyntheticPointCloudDataset(n_points, jet_order_data, points_per_patch=128)
dataloader = torch.utils.data.DataLoader(point_cloud_dataset, batch_size=256, num_workers=8)

points = point_cloud_dataset.points
gt_normals = point_cloud_dataset.gt_normals
```

### Normal estimation

```python
jet_order_fit = 3
for batchind, points in enumerate(dataloader):
    pca_beta, n_est, _ = DeepFit.fit_Wjet(points, torch.ones_like(points[:, 0]), order=1,
                               compute_neighbor_normals=False)
    pca_normals = n_est if batchind==0 else torch.cat([pca_normals, n_est], 0)

    jet_beta, n_est, _ = DeepFit.fit_Wjet(points, torch.ones_like(points[:, 0]), order=jet_order_fit,
                               compute_neighbor_normals=False)
    jet_normals = n_est if batchind==0 else torch.cat([jet_normals, n_est], 0)
```

The normals are unoriented so we will now flip them upwards (assuming that the positive z axis points up)

```python
n_sign = torch.sign(torch.sum(pca_normals*
                              torch.tensor([0., 0., 1.]).repeat([n_points, 1]), dim=1)).unsqueeze(-1)
pca_normals = n_sign * pca_normals
pca_normals = pca_normals.detach().cpu().numpy()

n_sign = torch.sign(torch.sum(jet_normals*
                              torch.tensor([0., 0., 1.]).repeat([n_points, 1]), dim=1)).unsqueeze(-1)
jet_normals = n_sign * jet_normals
jet_normals = jet_normals.detach().cpu().numpy()
```

### Visualization

```python
color_n_pca = tu.normal2rgb(pca_normals) #convert normal vectors to RGB
color_n_jet = tu.normal2rgb(jet_normals)
color_n_gt = tu.normal2rgb(gt_normals)

# make some buttons to toggle between the colors
btn_pca = widgets.Button(description='PCA')
btn_pca.style.button_color='lightgray'
btn_jet = widgets.Button(description='Jet')
btn_jet.style.button_color='lightgray'
btn_solid = widgets.Button(description='solid')
btn_solid.style.button_color='lightgray'
btn_gt = widgets.Button(description='GT')
btn_gt.style.button_color='lightgray'

def update_pc_normal_color(b, scatter_h):
    """
        this function is linked to the buttons and updates the the point cloud color
    """
    if b.description == 'PCA':
        scatter_h.color = color_n_pca
    elif b.description == 'Jet':
        scatter_h.color = color_n_pca
    elif b.description == 'solid':
        scatter_h.color = 'red'
    elif b.description == 'GT':
        scatter_h.color = color_n_gt

#plot
fig_h = ipv.figure()
scatter_h = ipv.pylab.scatter(point_cloud_dataset.points[:, 0],
                              point_cloud_dataset.points[:, 1],
                              point_cloud_dataset.points[:, 2], size=1, marker="sphere", color='red')
ipv.pylab.xyzlim(-1, 1)
ipv.style.use('minimal')
# ipv.show()

btn_pca.on_click(functools.partial(update_pc_normal_color, scatter_h=scatter_h))
btn_jet.on_click(functools.partial(update_pc_normal_color, scatter_h=scatter_h))
btn_gt.on_click(functools.partial(update_pc_normal_color, scatter_h=scatter_h))
btn_solid.on_click(functools.partial(update_pc_normal_color, scatter_h=scatter_h))

display(widgets.VBox((widgets.HBox((btn_pca, btn_jet, btn_gt, btn_solid)), fig_h)))
```

    VBox(children=(HBox(children=(Button(description='PCA', style=ButtonStyle(button_color='lightgray')), Button(d…

### Loading the data

```python
point_cloud_dataset = tu.SinglePointCloudDataset('./Boxy_smooth100k.xyz', points_per_patch=128)
print(point_cloud_dataset.points)

```

### Fit an n-jet (and compute normal vector)
This may take a while, depending on the number of points

```python
jet_order =3
dataloader = torch.utils.data.DataLoader(point_cloud_dataset, batch_size=256, num_workers=8)

for batchind, data in enumerate(dataloader, 0):
    points = data[0]
    data_trans = data[1]
    beta, n_est, neighbors_n_est = DeepFit.fit_Wjet(points, torch.ones_like(points[:, 0]), order=jet_order,
                               compute_neighbor_normals=False)
    n_est = torch.bmm(n_est.unsqueeze(1), data_trans.transpose(2, 1)).squeeze(dim=1) # cancel out pca
    normals = n_est if batchind==0 else torch.cat([normals, n_est], 0)
```

The normals are unoriented so we will now flip them outwards (assuming that the origin is an internal point and the shape is simple)

```python
n_sign = torch.sign(torch.sum(normals*torch.tensor(point_cloud_dataset.points), dim=1)).unsqueeze(-1)
normals = n_sign * normals
normals = normals.detach().cpu().numpy()
```

### Visualize the point cloud
We plot the point cloud and allow for 3 color overlays:
* Solid - (all points have the same color)
* Normals - We map the normal vectors to the RGB cube and use these values for coloring the point cloud.
* Curvatures - We map the principal curvatures to the XXX colormap to RGB.

```python
color_n = tu.normal2rgb(normals) #convert normal vectors to RGB

# make some buttons to toggle between the colors
btn_pc = widgets.Button(description='Solid')
btn_pc.style.button_color='lightgray'
btn_n = widgets.Button(description='normals')
btn_n.style.button_color='lightgray'
btn_c = widgets.Button(description='Curvatures TBD')
btn_c.style.button_color='lightgray'

def update_pc_color_to_solid(b, scatter_h):
    """
        this function is linked to the buttons and updates the the point cloud color
    """
    scatter_h.color = 'red'

def update_pc_color_to_normal(b, scatter_h):
    """
        this function is linked to the buttons and updates the the point cloud color
    """
    scatter_h.color = color_n

#plot
fig_h = ipv.figure()
scatter_h = ipv.pylab.scatter(point_cloud_dataset.points[:, 0],
                              point_cloud_dataset.points[:, 1],
                              point_cloud_dataset.points[:, 2], size=1, marker="sphere", color='red')
ipv.pylab.xyzlim(-1, 1)
ipv.style.use('minimal')
# ipv.show()

btn_pc.on_click(functools.partial(update_pc_color_to_solid, scatter_h=scatter_h))
btn_n.on_click(functools.partial(update_pc_color_to_normal, scatter_h=scatter_h))

display(widgets.VBox((widgets.HBox((btn_pc, btn_n)), fig_h)))
```

### References
* [PCL normal estimation using PCA](http://pointclouds.org/documentation/tutorials/normal_estimation.php)
* [CGAL normal estimation using Jets](https://doc.cgal.org/latest/Jet_fitting_3/index.html#Jet_fitting_3Mathematical)
* [DeepFit paper](https://arxiv.org/pdf/2003.10826.pdf)
* [Jet fitting paper](https://graphics.stanford.edu/courses/cs468-03-fall/Papers/cazals_jets.pdf)

```python

```