```python
from sympy import Matrix, symbols, pi, BlockDiagMatrix
from sympy.abc import x, y, z
from numpy import array
from numpy.testing import assert_array_equal

from lmi_sdp import LMI_PSD, LMI_NSD, lmi_to_coeffs, objective_to_coeffs, \
    get_variables, to_cvxopt, to_sdpa_sparse, to_sdpa_dense
```

### test_lmi_to_coeffs()

```python
vars = [x, y, z]
m1 = Matrix([[x, y], [y, z+1]])
c1 = Matrix([[0, 1], [1, 2]])
lmi1 = LMI_PSD(m1, c1)
m2 = Matrix([[y, 0], [0, 2*x]])
c2 = Matrix([[30, 0], [0, 40]])
lmi2 = LMI_NSD(m2, c2)
expected = [([array([[1., 0.],
                     [0., 0.]]),
              array([[0., 1.],
                     [1., 0.]]),
              array([[0., 0.],
                     [0., 1.]])],
             array([[0., -1.],
                    [-1., -1.]])),
            ([array([[0., 0.], [0., -2]]),
              array([[-1., 0.], [0., 0.]]),
              array([[0., 0.], [0., 0.]])],
             array([[30., 0.], [0., 40.]]))]

coeffs = lmi_to_coeffs([lmi1, lmi2], vars)
for i in range(len(coeffs)):
    assert_array_equal(coeffs[i][0], expected[i][0])
    assert_array_equal(coeffs[i][1], expected[i][1])

m3 = BlockDiagMatrix(m1-c1, -m2+c2)
lmi3 = LMI_PSD(m3)
coeffs = lmi_to_coeffs(lmi3, vars,
                       split_blocks='BlockDiagMatrix')
for i in range(len(coeffs)):
    assert_array_equal(coeffs[i][0], expected[i][0])
    assert_array_equal(coeffs[i][1], expected[i][1])
```

### test_lmi_to_coeffs_split()

```python
vars = [x, y, z]
m1 = Matrix([[x, y], [y, z+1]])
c1 = Matrix([[0, 1], [1, 2]])
lmi1 = LMI_PSD(m1, c1)
m2 = Matrix([[y, 0], [0, 2*x]])
c2 = Matrix([[30, 0], [0, 40]])
lmi2 = LMI_NSD(m2, c2)
expected = [([array([[1., 0.],
                     [0., 0.]]),
              array([[0., 1.],
                     [1., 0.]]),
              array([[0., 0.],
                     [0., 1.]])],
             array([[0., -1.],
                    [-1., -1.]])),
            ([array([[0.]]),
              array([[-1.]]),
              array([[0.]])],
             array([[30.]])),
            ([array([[-2.]]),
              array([[0.]]),
              array([[0.]])],
             array([[40.]]))]

coeffs = lmi_to_coeffs([lmi1, lmi2], vars,
                       split_blocks=True)
for i in range(len(coeffs)):
    assert_array_equal(coeffs[i][0], expected[i][0])
    assert_array_equal(coeffs[i][1], expected[i][1])

m3 = BlockDiagMatrix(m1-c1, -m2+c2)
lmi3 = LMI_PSD(m3)
coeffs = lmi_to_coeffs(lmi3, vars,
                       split_blocks=True)
for i in range(len(coeffs)):
    assert_array_equal(coeffs[i][0], expected[i][0])
    assert_array_equal(coeffs[i][1], expected[i][1])
```

### test_objective_to_coeffs()

```python
vars = [x, y, z]
assert_array_equal(objective_to_coeffs(1.2 + x - 3.4*y, vars, 'max'),
                   array([-1.0, 3.4, 0.0]))

except_ok = False
try:
    objective_to_coeffs(1.2 + x*y, vars)
except ValueError:
    except_ok = True
assert except_ok
```

### test_get_variables()

```python
x1, x2, x3 = symbols('x1 x2 x3')
variables = [x1, x2, x3]

obj = 1.2 + pi*x3
lmis = [Matrix([x2]), LMI_PSD(Matrix([1.4*x2 + x1]))]

assert variables == get_variables(obj, lmis)
```

### test_to_sdpa_sparse()

```python
x1, x2 = symbols('x1 x2')
variables = x1, x2
min_obj = 10*x1 + 20*x2
lmi_1 = LMI_PSD(
    -Matrix([[1, 0, 0, 0], [0, 2, 0, 0], [0, 0, 3, 0], [0, 0, 0, 4]]) +
    Matrix([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]])*x1 +
    Matrix([[0, 0, 0, 0], [0, 1, 0, 0], [0, 0, 5, 2], [0, 0, 2, 6]])*x2)

dat = to_sdpa_sparse(min_obj, lmi_1, variables, comment='test sparse')

ok_dat = ('"test sparse"\n'
          '2 = ndim\n'
          '3 = nblocks\n'
          '1 1 2 = blockstruct\n'
          '10.0, 20.0 = objcoeffs\n'
          '0 1 1 1 1.0\n'
          '0 2 1 1 2.0\n'
          '0 3 1 1 3.0\n'
          '0 3 2 2 4.0\n'
          '1 1 1 1 1.0\n'
          '1 2 1 1 1.0\n'
          '2 2 1 1 1.0\n'
          '2 3 1 1 5.0\n'
          '2 3 1 2 2.0\n'
          '2 3 2 2 6.0\n')

assert ok_dat == dat
```

### test_to_sdpa_dense()

```python
x1, x2 = symbols('x1 x2')
variables = x1, x2
min_obj = 10*x1 + 20*x2
lmi_1 = LMI_PSD(
    -Matrix([[1, 0, 0, 0], [0, 2, 0, 0], [0, 0, 3, 0], [0, 0, 0, 4]]) +
    Matrix([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]])*x1 +
    Matrix([[0, 0, 0, 0], [0, 1, 0, 0], [0, 0, 5, 2], [0, 0, 2, 6]])*x2)

dat = to_sdpa_dense(min_obj, lmi_1, variables, comment='test dense')

ok_dat = ('"test dense"\n'
          '2 = ndim\n'
          '3 = nblocks\n'
          '1 1 2 = blockstruct\n'
          '10.0, 20.0 = objcoeffs\n'
          '{\n'
          ' {\n'
          '  { 1.0 }\n'
          ' }\n'
          ' {\n'
          '  { 2.0 }\n'
          ' }\n'
          ' {\n'
          '  { 3.0, -0.0 },\n'
          '  { -0.0, 4.0 }\n'
          ' }\n'
          '}\n'
          '{\n'
          ' {\n'
          '  { 1.0 }\n'
          ' }\n'
          ' {\n'
          '  { 1.0 }\n'
          ' }\n'
          ' {\n'
          '  { 0.0, 0.0 },\n'
          '  { 0.0, 0.0 }\n'
          ' }\n'
          '}\n'
          '{\n'
          ' {\n'
          '  { 0.0 }\n'
          ' }\n'
          ' {\n'
          '  { 1.0 }\n'
          ' }\n'
          ' {\n'
          '  { 5.0, 2.0 },\n'
          '  { 2.0, 6.0 }\n'
          ' }\n'
          '}\n')

assert ok_dat == dat
```

### test_to_cvxopt()

```python
try:
    from cvxopt import matrix
except ImportError:  # pragma: no cover
    pass
else:
    variables = symbols('x1 x2 x3')
    x1, x2, x3 = variables

    min_obj = x1 - x2 + x3

    LMI_1 = LMI_NSD(
        x1*Matrix([[-7, -11], [-11, 3]]) +
        x2*Matrix([[7, -18], [-18, 8]]) +
        x3*Matrix([[-2, -8], [-8, 1]]),
        Matrix([[33, -9], [-9, 26]]))

    LMI_2 = LMI_NSD(
        x1*Matrix([[-21, -11, 0], [-11, 10, 8], [0, 8, 5]]) +
        x2*Matrix([[0, 10, 16], [10, -10, -10], [16, -10, 3]]) +
        x3*Matrix([[-5, 2, -17], [2, -6, 8], [-17, 8, 6]]),
        Matrix([[14, 9, 40], [9, 91, 10], [40, 10, 15]]))

    ok_c = matrix([1., -1., 1.])
    ok_Gs = [matrix([[-7., -11., -11., 3.],
                    [7., -18., -18., 8.],
                    [-2., -8., -8., 1.]])]
    ok_Gs += [matrix([[-21., -11., 0., -11., 10., 8., 0., 8., 5.],
                      [0., 10., 16., 10., -10., -10., 16., -10., 3.],
                      [-5., 2., -17., 2., -6., 8., -17., 8., 6.]])]
    ok_hs = [matrix([[33., -9.], [-9., 26.]])]
    ok_hs += [matrix([[14., 9., 40.], [9., 91., 10.], [40., 10., 15.]])]

    c, Gs, hs = to_cvxopt(min_obj, [LMI_1, LMI_2], variables)

    assert not any(ok_c - c)
    for i in range(len(ok_Gs)):
        assert not any(ok_Gs[i] - Gs[i])
    for i in range(len(ok_hs)):
        assert not any(ok_hs[i] - hs[i])
```


```python
%matplotlib inline
```

# Body-oriented mill

This model is introduced in

P. Degond, A. Diez, M. Na, Bulk topological states in a new collective dynamics model,  arXiv:2101.10864, 2021

First of all, some standard imports.

```python
import os
import sys
import time
import math
import torch
import numpy as np
from matplotlib import pyplot as plt
import sisyphe.models as models
from sisyphe.display import save

use_cuda = torch.cuda.is_available()
dtype = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor
```

## Body-oriented particles with initial perpendicular twist

The system is composed of body-oriented particles which are initially uniformly scattered in a periodic box but their body-orientations are "twisted". The body orientation of a particle at position $(x,y,z)$ is initially:

\begin{align}\left(\begin{array}{ccc} 1 & 0 & 0 \\ 0 & \cos(2\pi z) & -\sin(2\pi z) \\ 0 & \sin(2\pi z) & \cos(2\pi z)\end{array}\right).\end{align}

```python
from sisyphe.initial import cyclotron_twist_z

N = 1500000
L = 1
R = .025
nu = 40
c = 1
kappa = 10

pos, bo = cyclotron_twist_z(N,L,1,kappa,dtype)

simu = models.BOAsynchronousVicsek(pos=pos,bo=bo,
                 v=c,
                 jump_rate=nu,kappa=kappa,
                 interaction_radius=R,
                 box_size=L,
                 boundary_conditions='periodic',
                 variant = {"name" : "normalised", "parameters" : {}},
                 options = {},
                 sampling_method='vonmises',
                 block_sparse_reduction=True,
                 number_of_cells=15**3)
```

Run the simulation over 5 units of time and save the azimuthal angle of the mean direction of motion defined by:

\begin{align}\varphi = \mathrm{arg}(\Omega^1+i\Omega^2) \in [0,2\pi],\end{align}

where $\Omega = (\Omega^1,\Omega^2,\Omega^3)$ is the mean direction of motion of the particles with velocities $(\Omega_i)_{1\leq i\leq N}$ :

\begin{align}\Omega := \frac{\sum_{i=1}^N \Omega_i}{|\sum_{i=1}^N \Omega_i|}\end{align}

```python
frames = [5.]

s = time.time()
data = save(simu,frames,[],["phi"],save_file=False)
e = time.time()
```

Print the total simulation time and the average time per iteration.

```python
print('Total time: '+str(e-s)+' seconds')
print('Average time per iteration: '+str((e-s)/simu.iteration)+' seconds')
```

Plot the azimuthal angle $\varphi$.

```python
plt.plot(data["time"],data["phi"])
plt.xlabel("time")
plt.ylabel("Azimuthal angle")
```


# Musculoskeletal modeling and simulation

> Marcos Duarte
> Laboratory of Biomechanics and Motor Control ([http://demotu.org/](http://demotu.org/))
> Federal University of ABC, Brazil

We will now simulate the dynamics of the musculoskeletal system with muscle dynamics. You should have read [Muscle modeling](http://nbviewer.ipython.org/github/demotu/BMC/blob/master/notebooks/MuscleModeling.ipynb) and [Muscle simulation](http://nbviewer.ipython.org/github/demotu/BMC/blob/master/notebooks/MuscleSimulation.ipynb). In addition, chapter 4 of Nigg and Herzog (2006) is a good introduction to this topic.

Let's start with the simplest case, a one-link rigid body with one degree of freedom and one muscle.

## One-link system with one DoF and one muscle

Let's simulate the extension movement of the knee. This problem is based on an example from chapter 4 of Nigg and Herzog (2006). We will model the task as a planar movement, the leg and foot as one rigid segment and the rest of the body fixed, the knee as a revolute joint with fixed axis of rotation and one degree of freedom, and only one muscle as knee extensor, as illustrated in the figure below.

Figure. Planar model of the knee with one degree of freedom, one extensor muscle, and the corresponding free-body diagram for the leg. $\theta$: knee angle with horizontal; $F_M$ force of the quadriceps muscle; $F_J$: force at the knee joint; $r_M$: moment arm of the quadriceps; $r_{cm}$: distance to the knee joint of the center of mass of the leg+foot segment; $mg$: weight of the leg+foot; $L_{MT0}$: muscle-tendon unit length at $\theta=90^o$.

From figure above, the Newton-Euler equation for the sum of moment of forces around the knee joint is:

\begin{equation}
r_M F_M + r_{cm}\cos(\theta)mg = I\dfrac{\mathrm{d}^2 \theta}{\mathrm{d}t^2}
\label{}
\end{equation}

Where $I$ is the rotational inertia of the leg+foot segment around the knee joint.
In our convention, the first term on the left-hand side is a positive (extensor) moment of force and the second term is a negative (flexor) moment of force when $\theta$ is greater than $90^o$. The Newton-Euler equation above is a second-order nonlinear differential equation.

In a typical movement analysis using inverse dynamics, we would measure the leg and thigh positions, calculate the leg and knee linear and angular position and acceleration, and then estimate the net moment of force at the knee joint and force of the quadriceps muscle using the equation above. Now, we want to perform forward dynamics, i.e., muscle force will be the input and the movement of the leg will be the output. The figure below compares the forward (direct) and inverse dynamics approaches.

Figure. Inverse dynamics and Forward (or Direct) dynamics approaches for movement analysis (adapted from Zajac and Gordon, 1989).

The dependence of the length of the quadriceps muscle-tendon unit with the knee angle is given by:

\begin{equation}
L_{MT} = L_{MT0} - r_m (\theta - \pi/2)
\label{}
\end{equation}

Examining the equations above, if we excite the muscle, it will generate force which will be transmitted to the leg segment and accelerate it, which in turn will vary the muscle length and velocity, affecting the muscle force. We already have seen that muscle force is related to muscle length and velocity by a first-order ODE (see [Muscle simulation](http://nbviewer.ipython.org/github/demotu/BMC/blob/master/notebooks/MuscleSimulation.ipynb)). So, muscle force and joint angle are described by two coupled ODEs. Mathematically, this means that the musculoskeletal system we modeled has as state variables: $L_M$, $\dot{L}_M$, and $\theta$ (and muscle activation $a$ could also be considered as a state variable in certain situations, but not for the present problem, where we consider it as an input parameter). Let's rewrite the Newton-Euler equation in a more clear form:

\begin{equation}
\dfrac{\mathrm{d}^2 \theta}{\mathrm{d}t^2} = I^{-1} \left[ r_M F_M + r_{cm}\cos(\theta)mg \right]
\label{}
\end{equation}

$F_M$ will be determined when $L_M$ is calculated, which according to the muscle model we used in [Muscle simulation](http://nbviewer.ipython.org/github/demotu/BMC/blob/master/notebooks/MuscleSimulation.ipynb) is the solution of the following first-order ODE:

\begin{equation}
\dfrac{\mathrm{d} L_M}{\mathrm{d}t} = f_v^{-1}\left(\dfrac{F_{SE}(L_{MT}-L_M\cos\alpha)/\cos\alpha - F_{PE}(L_M)}{a f_l(L_M)}\right)
\label{}
\end{equation}

And:

\begin{equation}
F_M = F_{SE}(L_{MT}-L_M\cos\alpha)/\cos\alpha
\label{}
\end{equation}

To apply numerical methods to solve these coupled equations, we have to express them as a system of first-order differential equations (see [Ordinary differential equation](http://nbviewer.ipython.org/github/demotu/BMC/blob/master/notebooks/OrdinaryDifferentialEquation.ipynb)):

\begin{equation}
\left\{
\begin{array}{l l}
\dfrac{\mathrm{d} L_M}{\mathrm{d}t} = & f^{-1}_{v}(L_M(\theta)), \quad &L_M(t_0) = L_{M0}
\\
\dfrac{\mathrm{d} \theta}{\mathrm{d}t} = &\omega, \quad &\theta(t_0) = \theta
\\
\dfrac{\mathrm{d} \omega}{\mathrm{d}t} = &I^{-1} \left[ r_M F_M(L_M(\theta)) + r_{cm}\cos(\theta)mg \right], \quad &\omega(t_0) = \omega_0
\end{array}
\right.
\label{}
\end{equation}

Let's solve this problem in Python.
First, let's import the necessary Python libraries and customize the environment:

```python
import numpy as np
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
matplotlib.rcParams['lines.linewidth'] = 3
matplotlib.rcParams['font.size'] = 13
matplotlib.rcParams['lines.markersize'] = 5
matplotlib.rc('axes', grid=False, labelsize=14, titlesize=16, ymargin=0.05)
matplotlib.rc('legend', numpoints=1, fontsize=11)
from scipy.integrate import odeint, ode
# import the muscles.py module
import sys
sys.path.insert(1, r'./../functions')
import muscles
```

Let's define a function with the system of first-order ODEs:

```python
def onelink_eq(lmthetaomega, t):
    """equation for one-link system."""

    # parameters from Nigg and Herzog (2006)
    lmt0 = 0.310
    rm   = 0.033
    rcm  = 0.264
    m    = 10
    g    = 9.8
    I    = 0.1832

    lm      = lmthetaomega[0]  # muscle length
    theta   = lmthetaomega[1]  # knee position
    omega   = lmthetaomega[2]  # knee velocity
    lmt     = lmt0 - rm_fun(theta)*(theta - np.pi/2)
    lmd     = fv_1_eq(t, lm, lmt)
    thetadd = (rm_fun(theta)*fm_eq(lm, lmd, lmt) + rcm*np.cos(theta)*m*g)/I  # acceleration

    return [lmd, omega, thetadd]
```

```python
# define a function for the moment arm
def rm_fun(theta):
    """Vastus intermedius moment arm."""

    rm = 0.033

    return rm
```

Functions to calculate the force-velocity relationship:

```python
def fv_1_eq(t, lm, lmt):

    lmopt = ms.P['lmopt']
    ltslack = ms.P['ltslack']
    a = 1
    alpha = 0
    if lm < 0.1*lmopt:
        lm = 0.1*lmopt

    lt    = lmt - lm*np.cos(alpha)
    fse   = ms.force_se(lt=lt, ltslack=ltslack)
    fpe   = ms.force_pe(lm=lm/lmopt)
    fl    = ms.force_l(lm=lm/lmopt)
    fce_t = fse/np.cos(alpha) - fpe
    vm    = ms.velo_fm(fm=fce_t, a=a, fl=fl)

    return vm

def fm_eq(lm, lmd, lmt):

    fm0  = ms.P['fm0']
    lmopt = ms.P['lmopt']
    ltslack = ms.P['ltslack']
    a = 1
    alpha = 0

    lt = lmt - lm*np.cos(alpha)
    fl = ms.force_l(lm=lm/lmopt)
    fpe = ms.force_pe(lm=lm/lmopt)
    fm = ms.force_vm(vm=lmd, fl=fl, lmopt=lmopt, a=a) + fpe
    fm = fm*fm0

    return fm
```

Let's instantiate the `muscles` class:

```python
ms = muscles.Thelen2003()
ms.set_parameters()
ms.set_states()
ms.muscle_plot();
```

Let's input the initial states and use `scipy.odeint` to solve the ODEs:

```python
lmthetaomega = [0.087, np.pi/2, 0]
time = np.arange(0, .12, 0.001)
data  = odeint(onelink_eq, lmthetaomega, time)
```

Plots of the simulated angular position and velocity:

```python
def sim_plot(time, data, ms):

    datam = []
    for i, t in enumerate(time):
        fm0  = ms.P['fm0']
        lmopt = ms.P['lmopt']
        ltslack = ms.P['ltslack']
        a = 1
        alpha = 0
        lmt0 = 0.310

        lm = data[i, 0]
        theta = data[i, 1]
        lmt = lmt0 - rm_fun(theta)*(theta - np.pi/2)
        lt = lmt - lm*np.cos(alpha)
        fl = ms.force_l(lm=lm/lmopt)
        fpe = ms.force_pe(lm=lm/lmopt)
        fse = ms.force_se(lt=lt, ltslack=ltslack)
        fce_t = fse/np.cos(alpha) - fpe
        vm = ms.velo_fm(fm=fce_t, a=a, fl=fl, lmopt=lmopt)
        fm = ms.force_vm(vm=vm, fl=fl, lmopt=lmopt, a=a) + fpe
        d = [t, lmt, lm, lt, vm, fm*fm0, fse*fm0, a*fl*fm0, fpe*fm0, alpha]
        datam.append(d)

    datam = np.array(datam)

    fig, axs = plt.subplots(nrows=1, ncols=2, sharex = True, figsize=(10, 3))
    axs[0].plot(time, data[:, 1]*180/np.pi)
    axs[0].set_xlabel('Time (s)')
    axs[0].set_ylabel('Knee angle $(^o)$')
    axs[1].plot(time, data[:, 2]*180/np.pi)
    axs[1].set_xlabel('Time (s)')
    axs[1].set_ylabel('Knee velocity $(^o/s)$')
    plt.tight_layout()
    plt.show()

    return axs, datam

axs, data2 = sim_plot(time, data, ms)
```

And the plots for the simulated muscle mechanics:

```python
axs_lm = ms.lm_plot(data2)
```

### Moment arm is not constant

In fact, knee moment arms of the knee extensors are not constant and change with the knee angle. We can import these data from [OpenSim](http://web.stanford.edu/group/opensim/index.html). Here is the data for the vastus intermedius moment arm:

```python
vas_int_r = np.loadtxt('./../data/vas_int_r.mot', skiprows=7)
print('Shape of vas_int_r:', vas_int_r.shape)
```

    Shape of vas_int_r: (100, 3)

We need to be able to calculate the moment arm for any angle, not only for the 100 values in these data. Let's fit a spline to these data, which will return a function we can use to calculate the moment arm at any angle:

```python
from scipy import interpolate
S = interpolate.InterpolatedUnivariateSpline(vas_int_r[:, 1], vas_int_r[:, 2])
```

```python
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(6, 3))
ax.plot(vas_int_r[:, 1], vas_int_r[:, 2], label='Raw data')
ax.plot(vas_int_r[:, 1], S(vas_int_r[:, 1]), 'r--', label='Spline')
ax.set_xlabel('Knee angle $(^o)$')
ax.set_ylabel('Moment arm (m)')
plt.legend(loc='best')
plt.tight_layout()
plt.show()
```

It works as expected, we just have to handle the fact that the angle range is different.
Let's define a new function for the moment arm for our simulation:

```python
# define a function for the moment arm
def rm_fun(theta, data=S):
    """Vastus intermedius moment arm."""

    rm = S(theta*np.pi/180 - np.pi)

    return rm
```

```python
data = odeint(onelink_eq, lmthetaomega, time)
```

```python
axs, data2 = sim_plot(time, data, ms)
```

```python
axs_lm2 = ms.lm_plot(data2)  # axs_lm
```

## References

- Nigg BM and Herzog W (2006) [Biomechanics of the Musculo-skeletal System](https://books.google.com.br/books?id=hOIeAQAAIAAJ&dq=editions:ISBN0470017678). 3rd Edition. Wiley.
- Zajac FE, Gordon ME (1989) [Determining muscle's force and action in multi-articular movement](https://drive.google.com/open?id=0BxbW72zV7WmUcC1zSGpEOUxhWXM&authuser=0). Exercise and Sport Sciences Reviews, 17, 187-230.


####

 Text provided under a Creative Commons Attribution license, CC-BY. All code is made available under the FSF-approved MIT license. (c) Kyle T. Mandli

```python
%matplotlib inline
import numpy
import matplotlib.pyplot as plt
```

# Sources of Error

Error can come from many sources when using applying a numerical method:
 - Model/Data Error
 - Truncation Error
 - Floating Point Error

**Goal:** Categorize and understand each type of error and explore some simple approaches to analyzing error.

## Model and Data Error

Errors in fundamental formulation
 - Lotka-Volterra - fractional rabbits, no extinctions, etc.
 - Data Error - Inaccuracy in measurement or uncertainties in parameters

Unfortunatley we cannot control model and data error directly but we can use methods that may be more robust in the presence of these types of errors.

## Truncation Error

Errors arising from approximating a function with a simpler function (e.g. $sin(x) \approx x$ for $|x| \approx 0 $ ).

## Floating Point Error

Errors arising from approximating real numbers with finite-precision numbers and arithmetic.

## Basic Definitions

Given a true value of a function $f$ and an approximate solution $\hat{f}$ define:

Absolute Error:  $e = |f - \hat{f}|$

Relative Error:  $r = \frac{e}{|f|} = \frac{|f - \hat{f}|}{|f|}$

Decimal precision $p$ is defined as the minimum value that satisfies

$$x = \text{round}(10^{-n} \cdot x) \cdot 10^n$$

where

$$n = \text{floor}(\log_{10} x) + 1 - p$$

Note that if we are asking the decimal precision of the approximation $\hat{f}$ of $f$ then we need to use the absolute error to determine the precision.  To find the decimal precision in this case look at the magnitude of the absolute error and deterimine the place of the first error.  Combine this with the number of "correct" digits and you will get the decimal precision of the approximation.

## Truncation Error and Taylor's Theorem

**Taylor's Theorem:**  Let $f(x) \in C^{m+1}[a,b]$ and $x_0 \in [a,b]$, then for all $x \in (a,b)$ there exists a number $c = c(x)$ that lies between $x_0$ and $x$ such that

$$ f(x) = T_N(x) + R_N(x)$$

where $T_N(x)$ is the Taylor polynomial approximation

$$T_N(x) = \sum^N_{n=0} \frac{f^{(n)}(x_0)\cdot(x-x_0)^n}{n!}$$

and $R_N(x)$ is the residual (the part of the series we left off)

$$R_N(x) = \frac{f^{(n+1)}(c) \cdot (x - x_0)^{n+1}}{(n+1)!}$$

Another way to think about these results involves replacing $x - x_0$ with $\Delta x$.  The primary idea here is that the residual $R_N(x)$ becomes smaller as $\Delta x \rightarrow 0$.

$$T_N(x) = \sum^N_{n=0} \frac{f^{(n)}(x_0)\cdot\Delta x^n}{n!}$$

and $R_N(x)$ is the residual (the part of the series we left off)

$$R_N(x) = \frac{f^{(n+1)}(c) \cdot \Delta x^{n+1}}{(n+1)!} \leq M \Delta x^{n+1} = O(\Delta x^{n+1})$$

#### Example 1

$f(x) = e^x$ on $[0,1]$ with $x_0 = 0$

Using this we can find expressions for the relative and absolute error as a function of $x$ assuming $N=2$.

$$f'(x) = e^x, ~~~ f''(x) = e^x, ~~~ f^{(n)}(x) = e^x ~~ \forall n\in\mathbb N$$

$$T_N(x) = \sum^N_{n=0} e^0 \frac{x^n}{n!} ~~~~\Rightarrow ~~~~ T_2(x) = 1 + x + \frac{x^2}{2}$$

$$R_N(x) = e^c \frac{x^{n+1}}{(n+1)!} = e^c \cdot \frac{x^3}{6} ~~~~ \Rightarrow ~~~~ R_2(x) \leq \frac{e^1}{6} \approx 0.5$$

$$e^1 = 2.718\ldots$$

$$T_2(1) = 2.5 \Rightarrow e_{max} = e(1) = |R_2(1)| \approx 0.2, ~~ r \approx 0.1$$

We can also use the package sympy which has the ability to calculate Taylor polynomials built-in!

```python
import sympy
x = sympy.symbols('x')
f = sympy.symbols('f', cls=sympy.Function)

f = sympy.exp(x)
f.series(x0=0, n=6)
```

Lets plot this numerically for a section of $x$.

```python
x = numpy.linspace(-1, 1, 100)
T_N = 1.0 + x + x**2 / 2.0
R_N = numpy.exp(1) * x**3 / 6.0

plt.plot(x, T_N, 'r', x, numpy.exp(x), 'k', x, R_N, 'b')
plt.title("Taylor Polynomial and Residual for $N=2$")
plt.xlabel("x")
plt.ylabel("$f(x)$, $T_N(x)$, $R_N(x)$")
plt.legend(["$T_N(x)$", "$f(x)$", "$R_N(x)$"], loc=2)
plt.show()
```

#### Example 2

$f(x) = \frac{1}{x} ~~~~~~ x_0  = 1$, approximate with $\hat{f}(x) = T_2(x)$

$$f'(x) = -\frac{1}{x^2} ~~~~~~~ f''(x) = \frac{2}{x^3} ~~~~~~~ f^{(n)}(x) = \frac{(-1)^n n!}{x^{n+1}}$$

$$T_N(x) = \sum^N_{n=0} (-1)^n (x-1)^n ~~~~ \Rightarrow ~~~~ T_2(x) = 1 - (x - 1) + (x - 1)^2$$

$$R_N(x) = \frac{(-1)^{n+1}(x - 1)^{n+1}}{c^{n+2}} ~~~~ \Rightarrow ~~~~ R_2(x) = \frac{-(x - 1)^{3}}{c^{4}}$$

```python
x = numpy.linspace(0.8, 2, 100)
T_N = 1.0 - (x-1) + (x-1)**2
R_N = -(x-1.0)**3 / (1.1**4)

plt.plot(x, T_N, 'r', x, 1.0 / x, 'k', x, R_N, 'b')
plt.title("Taylor Polynomial and Residual for $N=2$")
plt.xlabel("x")
plt.ylabel("$f(x)$, $T_N(x)$, $R_N(x)$")

plt.legend(["$T_N(x)$", "$f(x)$", "$R_N(x)$"], loc=8)
plt.show()
```

### Symbols and Definitions

Big-O notation: $f(x) = \text{O}(g(x))$ as $x \rightarrow a$ if and only if $|f(x)| \leq M |g(x)|$ as $|x - a| < \delta$ for $M$ and $a$ positive.

In practice we use Big-O notation to say something about how the terms we may have left out of a series might behave.  We saw an example earlier of this with the Taylor's series approximations:

#### Example:
$f(x) = \sin x$ with $x_0 = 0$ then

$$T_N(x) = \sum^N_{n=0} (-1)^{n} \frac{x^{2n+1}}{(2n+1)!}$$

We can actually write $f(x)$ then as

$$f(x) = x - \frac{x^3}{6} + \frac{x^5}{120} + O(x^7)$$

This becomes more useful when we look at this as we did before with $\Delta x$:

$$f(x) = \Delta x - \frac{\Delta x^3}{6} + \frac{\Delta x^5}{120} + O(\Delta x^7)$$

**We can also develop rules for error propagation based on Big-O notation:**

In general, there are two theorems that do not need proof and hold when the value of x is large:

Let
$$f(x) = p(x) + O(x^n)$$
$$g(x) = q(x) + O(x^m)$$
$$k = \max(n, m)$$ then
$$f+g = p + q + O(x^k)$$
$$f \cdot g = p \cdot q + O(x^{n\cdot m})$$

On the other hand, if we are interested in small values of x, say Δx, the above expressions can be modified as follows:

$$f(\Delta x) = p(\Delta x) + O(\Delta x^n)$$
$$g(\Delta x) = q(\Delta x) + O(\Delta x^m)$$
$$r = \min(n, m)$$ then

$$f+g = p + q + O(\Delta x^r)$$

$$f \cdot g = p \cdot q + p \cdot O(\Delta x^m) + q \cdot O(\Delta x^n) + O(\Delta x^{n+m}) = p \cdot q + O(\Delta x^r)$$

**Note 1:** In this case we suppose that at least the polynomial with $k = \max(n, m)$ has the following form:
$$p(\Delta x) = 1 + p_1x + p_2x^2 + ...$$
or $$q(\Delta x) = 1 + q_1x + q_2x^2 + ...$$
so that there is an O(1) term that guarantees the existence of $O(\Delta x^r)$ in the final product.

To get a sense of why we care most about the power on $\Delta x$ when considering convergence the following figure shows how different powers on the convergence rate can effect how quickly we converge to our solution.  Note that here we are plotting the same data two different ways.  Plotting the error as a function of $\Delta x$ is a common way to show that a numerical method is doing what we expect and exhbits the correct convergence behavior.  Since errors can get small quickly it is very common to plot these sorts of plots on a log-log scale to easily visualize the results.  Note that if a method was truly of the order $n$ that they will be a linear function in log-log space with slope $n$.

```python
dx = numpy.linspace(1.0, 1e-4, 100)

fig = plt.figure()
fig.set_figwidth(fig.get_figwidth() * 2.0)

axes = []
axes.append(fig.add_subplot(1, 2, 1))
axes.append(fig.add_subplot(1, 2, 2))

for n in xrange(1, 5):
    axes[0].plot(dx, dx**n, label="$\Delta x^%s$" % n)
    axes[1].loglog(dx, dx**n, label="$\Delta x^%s$" % n)

axes[0].legend(loc=2)
axes[1].set_xticks([10.0**(-n) for n in xrange(5)])
axes[1].set_yticks([10.0**(-n) for n in xrange(16)])
axes[1].legend(loc=4)

axes[0].set_title("Growth of Error vs. $\Delta x^n$")
axes[0].set_xlabel("$\Delta x$")
axes[0].set_ylabel("Estimated Error")
axes[1].set_title("Growth of Error vs. $\Delta x^n$ on a log-scale")
axes[1].set_xlabel("$\log(\Delta x)$")
axes[1].set_ylabel("$\log$(Estimated Error)")

plt.show()
```

## Horner's Method for Evaluating Polynomials

Given

$$P_N(x) = p_1 x^N + p_2 x^{N-1} + p_3 x^{N-2} + \ldots + p_{N+1}$$

want to find best way to evaluate $P_N(x)$.  First consider two ways to write $P_3$:

$$ P_3(x) = p_1 x^3 + p_2 x^2 + p_3 x + p_4$$

and using nested multiplication:

$$ P_3(x) = ((p_1 x + p_2) x + p_3) x + p_4$$

Consider how many operations it takes for each...

$$ P_3(x) = p_1 x^3 + p_2 x^2 + p_3 x + p_4$$

$$P_3(x) = \overbrace{p_1 \cdot x \cdot x \cdot x}^3 + \overbrace{p_2 \cdot x \cdot x}^2 + \overbrace{p_3 \cdot x}^1 + p_4$$

Adding up all the operations we can in general think of this as a pyramid

We can estimate this way that the algorithm written this way will take approximately $O(N^2 / 2)$ operations to complete.

Looking at our other means of evaluation:

$$ P_3(x) = ((p_1 x + p_2) x + p_3) x + p_4$$

Here we find that the method is $O(N)$ (the 2 is usually ignored in these cases).  The important thing is that the first evaluation is $O(N^2)$ and the second $O(N)$!

### Algorithm

Fill in the function and implement Horner's method:
```python
def eval_poly(p, x):
    """Evaluates polynomial given coefficients p at x

    Function to evaluate a polynomial in order N operations.  The polynomial is defined as

    P(x) = p[0] x**n + p[1] x**(n-1) + ... + p[n-1] x + p[n]

    The value x should be a float.
    """
    pass
```

```python
def eval_poly(p, x):
    """Evaluates polynomial given coefficients p at x

    Function to evaluate a polynomial in order N operations.  The polynomial is defined as

    P(x) = p[0] x**n + p[1] x**(n-1) + ... + p[n-1] x + p[n]

    The value x should be a float.
    """

    y = p[0]
    for coefficient in p[1:]:
        y = y * x + coefficient

    return y
```

or an alternative version that allows `x` to be a vector of values:
```python
def eval_poly(p, x):
    """Evaluates polynomial given coefficients p at x

    Function to evaluate a polynomial in order N operations.  The polynomial is defined as

    P(x) = p[0] x**n + p[1] x**(n-1) + ... + p[n-1] x + p[n]

    The value x can by a NumPy ndarray.
    """

    y = numpy.ones(x.shape) * p[0]
    for coefficient in p[1:]:
        y = y * x + coefficient

    return y
```
This version calculates each `y` value simultaneously making for much faster code!

```python
def eval_poly(p, x):
    """Evaluates polynomial given coefficients p at x

    Function to evaluate a polynomial in order N operations.  The polynomial is defined as

    P(x) = p[0] x**n + p[1] x**(n-1) + ... + p[n-1] x + p[n]

    The value x can by a NumPy ndarray.
    """

    y = numpy.ones(x.shape) * p[0]
    for coefficient in p[1:]:
        y = y * x + coefficient

    return y

p = [1, -3, 10, 4, 5, 5]
x = numpy.linspace(-10, 10, 100)
plt.plot(x, eval_poly(p, x))
plt.title("Polynomial with $p_0$=%s,$p_1$=%s,$p_2$=%s,$p_3$=%s,$p_4$=%s,$p_5$=%s "%(p[0],p[1],p[2],p[3],p[4],p[5]))
plt.xlabel("x")
plt.ylabel("p(x)")
plt.show()
```

## Truncation Error vs. Floating Point Error

Truncation error:  Errors arising from approximation of a function, truncation of a series...

$$\sin x \approx x - \frac{x^3}{3!} + \frac{x^5}{5!} + O(x^7)$$

Floating-point Error:  Errors arising from approximating real numbers with finite-precision numbers

$$\pi \approx 3.14$$

or $\frac{1}{3} \approx 0.333333333$ in decimal, results form finitely number of registers to represent each number.

## Floating Point Systems

Numbers in floating point systems are represented as a series of bits that represent different pieces of a number.  In *nomalized floating point systems* there are some standard conventions for what these bits are used for.  In general the numbers are stored by breaking them down into the form

$$\hat{f} = \pm d_1 . d_2 d_3 d_4 \ldots d_p \times \beta^E$$

where
1. $\pm$ is a single bit and of course represents the sign of the number
2. $d_1 . d_2 d_3 d_4 \ldots d_p$ is called the *mantissa*.  Note that technically the decimal could be moved but generally, using scientific notation, the decimal can always be placed at this location.  The digits $d_2 d_3 d_4 \ldots d_p$ are called the *fraction* with $p$ digits of precision.  Normalized systems specifically put the decimal point in the front like we have and assume $d_1 \neq 0$ unless the number is exactly $0$.
3. $\beta$ is the *base*.  For binary $\beta = 2$, for decimal $\beta = 10$, etc.
4. $E$ is the *exponent*, an integer in the range $[E_{\min}, E_{\max}]$

The important points on any floating point system is that
1. There exist a discrete and finite set of representable numbers
2. These representable numbers are not evenly distributed on the real line
3. Arithmetic in floating point systems yields different results from infinite precision arithmetic (i.e. "real" math)

### Example:  Toy System
Consider the toy 2-digit precision decimal system (normalized)
$$f = \pm d_1 . d_2 \times 10^E$$
with $E \in [-2, 0]$.

#### Number and distribution of numbers
1. How many numbers can we represent with this system?

2. What is the distribution on the real line?

3. What is the underflow and overflow limits?

How many numbers can we represent with this system?

$f = \pm d_1 . d_2 \times 10^E$ with $E \in [-2, 0]$.

$$ 2 \times 9 \times 10 \times 3 + 1 = 541$$

What is the distribution on the real line?

```python
d_1_values = [1, 2, 3, 4, 5, 6, 7, 8, 9]
d_2_values = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
E_values = [0, -1, -2]

fig = plt.figure(figsize=(10.0, 1.0))
axes = fig.add_subplot(1, 1, 1)

for E in E_values:
    for d1 in d_1_values:
        for d2 in d_2_values:
            axes.plot( (d1 + d2 * 0.1) * 10**E, 0.0, 'r+', markersize=20)
            axes.plot(-(d1 + d2 * 0.1) * 10**E, 0.0, 'r+', markersize=20)

axes.plot(0.0, 0.0, '+', markersize=20)
axes.plot([-10.0, 10.0], [0.0, 0.0], 'k')

axes.set_title("Distribution of Values")
axes.set_yticks([])
axes.set_xlabel("x")
axes.set_ylabel("")
axes.set_xlim([-0.1, 0.1])
plt.show()
```

What is the underflow and overflow limits?

Smallest number that can be represented is the underflow:  $1.0 \times 10^{-2} = 0.01$
Largest number that can be represented is the overflow:  $9.9 \times 10^0 = 9.9$

## Properties of Floating Point Systems
All floating-point systems are characterized by several important numbers
 - Smalled normalized number (underflow if below)
 - Largest normalized number (overflow if above)
 - Zero
 - Machine $\epsilon$ or $\epsilon_{\text{machine}}$
 - `inf` and `nan`, infinity and **N**ot **a** **N**umber respectively
 - Subnormal numbers

## Binary Systems
Consider the 2-digit precision base 2 system:

$$f=\pm d_1 . d_2 \times 2^E ~~~~ \text{with} ~~~~ E \in [-1, 1]$$

#### Number and distribution of numbers
1. How many numbers can we represent with this system?

2. What is the distribution on the real line?

3. What is the underflow and overflow limits?

How many numbers can we represent with this system?

$$f=\pm d_1 . d_2 \times 2^E ~~~~ \text{with} ~~~~ E \in [-1, 1]$$

$$ 2 \times 1 \times 2 \times 3 + 1 = 13$$

What is the distribution on the real line?

```python
d_1_values = [1]
d_2_values = [0, 1]
E_values = [1, 0, -1]

fig = plt.figure(figsize=(10.0, 1.0))
axes = fig.add_subplot(1, 1, 1)

for E in E_values:
    for d1 in d_1_values:
        for d2 in d_2_values:
            axes.plot( (d1 + d2 * 0.1) * 2**E, 0.0, 'r+', markersize=20)
            axes.plot(-(d1 + d2 * 0.1) * 2**E, 0.0, 'r+', markersize=20)

axes.plot(0.0, 0.0, 'r+', markersize=20)
axes.plot([-4.5, 4.5], [0.0, 0.0], 'k')

axes.set_title("Distribution of Values")
axes.set_yticks([])
axes.set_xlabel("x")
axes.set_ylabel("")
axes.set_xlim([-3, 3])
plt.show()
```

Smallest number that can be represented is the underflow:  $1.0 \times 2^{-1} = 0.25$
Largest number that can be represented is the overflow:  $1.1 \times 2^1 = 2.2$

## Real Systems - IEEE 754 Binary Floating Point Systems

### Single Precision
 - Total storage alloted is 32 bits
 - Exponent is 8 bits $\Rightarrow E \in [-126, 127]$
 - Fraction 23 bits ($p = 24$)

```
s EEEEEEEE FFFFFFFFFFFFFFFFFFFFFFF
0 1      8 9                     31
```
Overflow $= 2^{127} \approx 3.4 \times 10^{38}$

Underflow $= 2^{-126} \approx 1.2 \times 10^{-38}$

$\epsilon_{\text{machine}} = 2^{-23} \approx 1.2 \times 10^7$

### Double Precision
 - Total storage alloted is 64 bits
 - Exponent is 11 bits $\Rightarrow E \in [-1022, 1024]$
 - Fraction 52 bits ($p = 53$)

```
s EEEEEEEEEE FFFFFFFFFF FFFFFFFFFF FFFFFFFFFF FFFFFFFFFF FFFFFFFFFF FF
0 1       11 12                                                      63
```
Overflow $= 2^{1024} \approx 1.8 \times 10^{308}$

Underflow $= 2^{-1022} \approx 2.2 \times 10^{-308}$

$\epsilon_{\text{machine}} = 2^{-52} \approx 2.2 \times 10^{-16}$

### Python Access to IEEE Numbers

Access many important parameters, such as machine epsilon:

```python
import numpy
numpy.finfo(float).eps
```

```python
print numpy.finfo(float).eps
print numpy.finfo(float).max
print numpy.finfo(float).min
print numpy.finfo(float).nmant
print numpy.finfo(float).precision
print numpy.finfo(float).tiny
```

## Why should we care about this?

 - Floating point arithmetic is not commutative or associative
 - Floating point errors compound, do not assume even double precision is enough!
 - Mixing precision is very dangerous

 ### Examples

Simple arithmetic $\delta < \epsilon_{\text{machine}}$

   $$(1+\delta) - 1 = 1 - 1 = 0$$

   $$1 - 1 + \delta = \delta$$

Catastrophic cancelation (assume $x + y \neq 0$):

$$x(1 + \epsilon_x) + y (1 + \epsilon_y) = x + y + x \epsilon_x + y \epsilon_y =(x+y) \left( 1 + \frac{x \epsilon_x + y \epsilon_y}{x+y} \right ) \Rightarrow$$

$$\epsilon_{x+y} = \frac{x}{x + y} \epsilon_x  + \frac{y}{x+y} \epsilon_y$$

This implies that the error made from doing the actual sum can be arbitrarily large!  This is not too bad unless $x$ and $y$ have opposite sign, then if $x+y \approx 0$ then $\frac{1}{x+y}$ can be huge.  This is refered to as *catastrophic* cancelation.

Matrix vector multiplication

   $$ \begin{bmatrix} \epsilon & 1 \\ 1 & 1 \end{bmatrix} \cdot \begin{bmatrix} 1 \\ 1 \end{bmatrix}
= \begin{bmatrix} \epsilon + 1 \\ 1 + 1 \end{bmatrix} = \begin{bmatrix} 1 \\ 2 \end{bmatrix}$$

Solve $A x = b$ where

   $$A = \begin{bmatrix} 10 & 1 \\ 3 & 0.3 \end{bmatrix}~~~~~~\text{and}~~~~~~b=\begin{bmatrix} 11 \\ 3.3 \end{bmatrix}$$

   and we know that

   $$x = \begin{bmatrix} 1 \\ 1 \end{bmatrix}$$

   When used in a solver we get

   $$x = \begin{bmatrix}-0.5 \\ 16 \end{bmatrix}$$

Evaluation of a polynomial

   $$f(x) = x^7 - 7x^6 + 21 x^5 - 35 x^4 + 35x^3-21x^2 + 7x - 1$$

```python
x = numpy.linspace(0.988, 1.012, 1000)
y = x**7 - 7.0 * x**6 + 21.0 * x**5 - 35.0 * x**4 + 35.0 * x**3 - 21.0 * x**2 + 7.0 * x - 1.0

fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(x, y, 'r')
axes.set_xlabel("x")
axes.set_ylabel("y")
plt.show()
```

For more examples see the "[100-digit Challenge](http://bookstore.siam.org/ot86/)" book.

## Combination of Error

In general we need to concern ourselves with the combination of both truncation error and floating point error.

### Example

Consider the finite difference approximation where $f(x) = e^x$ and we are evaluating at $x=1$?

$$f'(x) \approx \frac{f(x + \Delta x) - f(x)}{\Delta x}$$

Compare the error between decreasing $\Delta x$ and the true solution $f'(1) = e$.

```python
delta_x = numpy.linspace(1e-20, 5.0, 1000)
x = 1.0
f_hat = (numpy.exp(x + delta_x) - numpy.exp(x)) / (delta_x)

fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.loglog(delta_x, numpy.abs(f_hat - numpy.exp(1)) / numpy.exp(1))
axes.set_xlabel("$\Delta x$")
axes.set_ylabel("Relative Error")
plt.show()
```

### Example 2

Evaluate $e^x$ with its Taylor series.

$$e^x = \sum^\infty_{n=0} \frac{x^n}{n!}$$

Can we pick $N < \infty$ that can approximate $e^x$ over a give range $x \in [a,b]$ such that the relative error $E$ satisfies $E < 8 \cdot \varepsilon_{\text{machine}}$?

What might be a better way than simply evaluating the Taylor polynomial directly for various $N$?

```python
import scipy.misc

def my_exp(x, N=10):
    value = 0.0
    for n in xrange(N + 1):
        value += x**n / scipy.misc.factorial(n)

    return value

x = numpy.linspace(-2, 2, 100)
for N in range(1, 50):
    error = numpy.abs((numpy.exp(x) - my_exp(x, N=N)) / numpy.exp(x))
    if numpy.all(error < 100.0 * numpy.finfo(float).eps):
        break

print N

fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(x, error)
axes.set_xlabel("x")
axes.set_ylabel("Relative Error")
plt.show()
```

Some other links that might be helpful regarding IEEE Floating Point:
 - [What Every Computer Scientist Should Know About Floating-Point Arithmetic](http://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html)
 - [IEEE 754 Floating Point Calculator](http://babbage.cs.qc.edu/courses/cs341/IEEE-754.html)
 - [Numerical Computing with IEEE Floating Point Arithmetic](http://epubs.siam.org/doi/book/10.1137/1.9780898718072)


# Tutorial 1: Geometric view of data
**Week 1, Day 5: Dimensionality Reduction**

**By Neuromatch Academy**

__Content creators:__ Alex Cayco Gajic, John Murray

__Content reviewers:__ Roozbeh Farhoudi, Matt Krause, Spiros Chavlis, Richard Gao, Michael Waskom, Siddharth Suresh, Natalie Schaworonkow, Ella Batty

**Our 2021 Sponsors, including Presenting Sponsor Facebook Reality Labs**

---
# Tutorial Objectives

*Estimated timing of tutorial: 50 minutes*

In this notebook we'll explore how multivariate data can be represented in different orthonormal bases. This will help us build intuition that will be helpful in understanding PCA in the following tutorial.

Overview:
 - Generate correlated multivariate data.
 - Define an arbitrary orthonormal basis.
 - Project the data onto the new basis.

```python
# @title Tutorial slides

# @markdown These are the slides for the videos in all tutorials today
from IPython.display import IFrame
IFrame(src=f"https://mfr.ca-1.osf.io/render?url=https://osf.io/kaq2x/?direct%26mode=render%26action=download%26mode=render", width=854, height=480)
```

```python
# @title Video 1: Geometric view of data
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
      def __init__(self, id, page=1, width=400, height=300, **kwargs):
          self.id=id
          src = 'https://player.bilibili.com/player.html?bvid={0}&page={1}'.format(id, page)
          super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id="BV1Af4y1R78w", width=854, height=480, fs=1)
  print('Video available at https://www.bilibili.com/video/{0}'.format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id="THu9yHnpq9I", width=854, height=480, fs=1, rel=0)
  print('Video available at https://youtube.com/watch?v=' + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)
```

    Tab(children=(Output(), Output()), _titles={'0': 'Youtube', '1': 'Bilibili'})

---
# Setup

```python
# Imports

import numpy as np
import matplotlib.pyplot as plt
```

```python
# @title Figure Settings
import ipywidgets as widgets  # interactive display
%config InlineBackend.figure_format = 'retina'
plt.style.use("https://raw.githubusercontent.com/NeuromatchAcademy/course-content/master/nma.mplstyle")
```

```python
# @title Plotting Functions

def plot_data(X):
  """
  Plots bivariate data. Includes a plot of each random variable, and a scatter
  plot of their joint activity. The title indicates the sample correlation
  calculated from the data.

  Args:
    X (numpy array of floats) :   Data matrix each column corresponds to a
                                  different random variable

  Returns:
    Nothing.
  """

  fig = plt.figure(figsize=[8, 4])
  gs = fig.add_gridspec(2, 2)
  ax1 = fig.add_subplot(gs[0, 0])
  ax1.plot(X[:, 0], color='k')
  plt.ylabel('Neuron 1')
  plt.title('Sample var 1: {:.1f}'.format(np.var(X[:, 0])))
  ax1.set_xticklabels([])
  ax2 = fig.add_subplot(gs[1, 0])
  ax2.plot(X[:, 1], color='k')
  plt.xlabel('Sample Number')
  plt.ylabel('Neuron 2')
  plt.title('Sample var 2: {:.1f}'.format(np.var(X[:, 1])))
  ax3 = fig.add_subplot(gs[:, 1])
  ax3.plot(X[:, 0], X[:, 1], '.', markerfacecolor=[.5, .5, .5],
           markeredgewidth=0)
  ax3.axis('equal')
  plt.xlabel('Neuron 1 activity')
  plt.ylabel('Neuron 2 activity')
  plt.title('Sample corr: {:.1f}'.format(np.corrcoef(X[:, 0], X[:, 1])[0, 1]))
  plt.show()

def plot_basis_vectors(X, W):
  """
  Plots bivariate data as well as new basis vectors.

  Args:
    X (numpy array of floats) :   Data matrix each column corresponds to a
                                  different random variable
    W (numpy array of floats) :   Square matrix representing new orthonormal
                                  basis each column represents a basis vector

  Returns:
    Nothing.
  """

  plt.figure(figsize=[4, 4])
  plt.plot(X[:, 0], X[:, 1], '.', color=[.5, .5, .5], label='Data')
  plt.axis('equal')
  plt.xlabel('Neuron 1 activity')
  plt.ylabel('Neuron 2 activity')
  plt.plot([0, W[0, 0]], [0, W[1, 0]], color='r', linewidth=3,
           label='Basis vector 1')
  plt.plot([0, W[0, 1]], [0, W[1, 1]], color='b', linewidth=3,
           label='Basis vector 2')
  plt.legend()
  plt.show()

def plot_data_new_basis(Y):
  """
  Plots bivariate data after transformation to new bases.
  Similar to plot_data but with colors corresponding to projections onto
  basis 1 (red) and basis 2 (blue). The title indicates the sample correlation
  calculated from the data.

  Note that samples are re-sorted in ascending order for the first
  random variable.

  Args:
    Y (numpy array of floats):   Data matrix in new basis each column
                                 corresponds to a different random variable

  Returns:
    Nothing.
  """
  fig = plt.figure(figsize=[8, 4])
  gs = fig.add_gridspec(2, 2)
  ax1 = fig.add_subplot(gs[0, 0])
  ax1.plot(Y[:, 0], 'r')
  plt.xlabel
  plt.ylabel('Projection \n basis vector 1')
  plt.title('Sample var 1: {:.1f}'.format(np.var(Y[:, 0])))
  ax1.set_xticklabels([])
  ax2 = fig.add_subplot(gs[1, 0])
  ax2.plot(Y[:, 1], 'b')
  plt.xlabel('Sample number')
  plt.ylabel('Projection \n basis vector 2')
  plt.title('Sample var 2: {:.1f}'.format(np.var(Y[:, 1])))
  ax3 = fig.add_subplot(gs[:, 1])
  ax3.plot(Y[:, 0], Y[:, 1], '.', color=[.5, .5, .5])
  ax3.axis('equal')
  plt.xlabel('Projection basis vector 1')
  plt.ylabel('Projection basis vector 2')
  plt.title('Sample corr: {:.1f}'.format(np.corrcoef(Y[:, 0], Y[:, 1])[0, 1]))
  plt.show()
```

---
# Section 1: Generate correlated multivariate data

```python
# @title Video 2: Multivariate data
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
      def __init__(self, id, page=1, width=400, height=300, **kwargs):
          self.id=id
          src = 'https://player.bilibili.com/player.html?bvid={0}&page={1}'.format(id, page)
          super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id="BV1xz4y1D7ES", width=854, height=480, fs=1)
  print('Video available at https://www.bilibili.com/video/{0}'.format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id="jcTq2PgU5Vw", width=854, height=480, fs=1, rel=0)
  print('Video available at https://youtube.com/watch?v=' + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)
```

    Tab(children=(Output(), Output()), _titles={'0': 'Youtube', '1': 'Bilibili'})

This video describes the covariance matrix and the multivariate normal distribution.

 Click here for text recap of video

To gain intuition, we will first use a simple model to generate multivariate data. Specifically, we will draw random samples from a *bivariate normal distribution*. This is an extension of the one-dimensional normal distribution to two dimensions, in which each $x_i$ is marginally normal with mean $\mu_i$ and variance $\sigma_i^2$:

\begin{align}
x_i \sim \mathcal{N}(\mu_i,\sigma_i^2).
\end{align}

Additionally, the joint distribution for $x_1$ and $x_2$ has a specified correlation coefficient $\rho$. Recall that the correlation coefficient is a normalized version of the covariance, and ranges between -1 and +1:

\begin{align}
\rho = \frac{\text{cov}(x_1,x_2)}{\sqrt{\sigma_1^2 \sigma_2^2}}.
\end{align}

For simplicity, we will assume that the mean of each variable has already been subtracted, so that $\mu_i=0$ for both $i=1$ and $i=2$. The remaining parameters can be summarized in the covariance matrix, which for two dimensions has the following form:

\begin{align}
{\bf \Sigma} =
\begin{pmatrix}
 \text{var}(x_1) & \text{cov}(x_1,x_2) \\
 \text{cov}(x_1,x_2) &\text{var}(x_2)
\end{pmatrix}.
\end{align}

In general, $\bf \Sigma$ is a symmetric matrix with the variances $\text{var}(x_i) = \sigma_i^2$ on the diagonal, and the covariances on the off-diagonal. Later, we will see that the covariance matrix plays a key role in PCA.

## Coding Exercise 1: Draw samples from a distribution

We have provided code to draw random samples from a zero-mean bivariate normal distribution with a specified covariance matrix (`get_data`). Throughout this tutorial, we'll imagine these samples represent the activity (firing rates) of two recorded neurons on different trials. Fill in the function below to calculate the covariance matrix given the desired variances and correlation coefficient. The covariance can be found by rearranging the equation above:

\begin{align}
\text{cov}(x_1,x_2) = \rho \sqrt{\sigma_1^2 \sigma_2^2}.
\end{align}

```python
# @markdown Execute this cell to get helper function `get_data`

def get_data(cov_matrix):
  """
  Returns a matrix of 1000 samples from a bivariate, zero-mean Gaussian.

  Note that samples are sorted in ascending order for the first random variable

  Args:
    cov_matrix (numpy array of floats): desired covariance matrix

  Returns:
    (numpy array of floats) : samples from the bivariate Gaussian, with each
                              column corresponding to a different random
                              variable
  """

  mean = np.array([0, 0])
  X = np.random.multivariate_normal(mean, cov_matrix, size=1000)
  indices_for_sorting = np.argsort(X[:, 0])
  X = X[indices_for_sorting, :]

  return X

help(get_data)
```

    Help on function get_data in module __main__:

    get_data(cov_matrix)
        Returns a matrix of 1000 samples from a bivariate, zero-mean Gaussian.

        Note that samples are sorted in ascending order for the first random variable

        Args:
          cov_matrix (numpy array of floats): desired covariance matrix

        Returns:
          (numpy array of floats) : samples from the bivariate Gaussian, with each
                                    column corresponding to a different random
                                    variable

```python
def calculate_cov_matrix(var_1, var_2, corr_coef):
  """
  Calculates the covariance matrix based on the variances and correlation
  coefficient.

  Args:
    var_1 (scalar)          : variance of the first random variable
    var_2 (scalar)          : variance of the second random variable
    corr_coef (scalar)      : correlation coefficient

  Returns:
    (numpy array of floats) : covariance matrix
  """

  #################################################
  ## TODO for students: calculate the covariance matrix
  # Fill out function and remove
  # raise NotImplementedError("Student excercise: calculate the covariance matrix!")
  #################################################

  # Calculate the covariance from the variances and correlation
  cov = corr_coef * np.sqrt(var_1*var_2)

  cov_matrix = np.array([[var_1, cov], [cov, var_2]])

  return cov_matrix

# Set parameters
np.random.seed(2020)  # set random seed
variance_1 = 1
variance_2 = 1
corr_coef = 0.8

# Compute covariance matrix
cov_matrix = calculate_cov_matrix(variance_1, variance_2, corr_coef)

# Generate data with this covariance matrix
X = get_data(cov_matrix)

# Visualize
plot_data(X)
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D5_DimensionalityReduction/solutions/W1D5_Tutorial1_Solution_85104841.py)

*Example output:*

## Interactive Demo 1: Correlation effect on data

We'll use the function you just completed but now we can change the correlation coefficient via slider. You should get a feel for how changing the correlation coefficient affects the geometry of the simulated data.

1.   What effect do negative correlation coefficient values have?
2.   What correlation coefficient results in a circular data cloud?

Note that we sort the samples according to neuron 1's firing rate, meaning the plot of neuron 1 firing rate over sample number looks clean and pretty unchanging when compared to neuron 2.

```python
# @markdown Execute this cell to enable widget

def _calculate_cov_matrix(var_1, var_2, corr_coef):

  # Calculate the covariance from the variances and correlation
  cov = corr_coef * np.sqrt(var_1 * var_2)

  cov_matrix = np.array([[var_1, cov], [cov, var_2]])

  return cov_matrix

@widgets.interact(corr_coef = widgets.FloatSlider(value=.2, min=-1, max=1, step=0.1))
def visualize_correlated_data(corr_coef=0):
  variance_1 = 1
  variance_2 = 1

  # Compute covariance matrix
  cov_matrix = _calculate_cov_matrix(variance_1, variance_2, corr_coef)

  # Generate data with this covariance matrix
  X = get_data(cov_matrix)

  # Visualize
  plot_data(X)
```

    interactive(children=(FloatSlider(value=0.2, description='corr_coef', max=1.0, min=-1.0), Output()), _dom_clas…

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D5_DimensionalityReduction/solutions/W1D5_Tutorial1_Solution_4413477e.py)

---
# Section 2: Define a new orthonormal basis

*Estimated timing to here from start of tutorial: 20 min*

```python
# @title Video 3: Orthonormal bases
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
      def __init__(self, id, page=1, width=400, height=300, **kwargs):
          self.id=id
          src = 'https://player.bilibili.com/player.html?bvid={0}&page={1}'.format(id, page)
          super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id="BV1wT4y1E71g", width=854, height=480, fs=1)
  print('Video available at https://www.bilibili.com/video/{0}'.format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id="PC1RZELnrIg", width=854, height=480, fs=1, rel=0)
  print('Video available at https://youtube.com/watch?v=' + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)
```

    Tab(children=(Output(), Output()), _titles={'0': 'Youtube', '1': 'Bilibili'})

This video shows that data can be represented in many ways using different bases. It also explains how to check if your favorite basis is orthonormal.

 Click here for text recap of video

Next, we will define a new orthonormal basis of vectors ${\bf u} = [u_1,u_2]$ and ${\bf w} = [w_1,w_2]$. As we learned in the video, two vectors are orthonormal if:

1.   They are orthogonal (i.e., their dot product is zero):
\begin{align}
{\bf u\cdot w} = u_1 w_1 + u_2 w_2 = 0
\end{align}
2.   They have unit length:
\begin{align}
||{\bf u} || = ||{\bf w} || = 1
\end{align}

In two dimensions, it is easy to make an arbitrary orthonormal basis. All we need is a random vector ${\bf u}$, which we have normalized. If we now define the second basis vector to be ${\bf w} = [-u_2,u_1]$, we can check that both conditions are satisfied:
\begin{align}
{\bf u\cdot w} = - u_1 u_2 + u_2 u_1 = 0
\end{align}
and
\begin{align}
{|| {\bf w} ||} = \sqrt{(-u_2)^2 + u_1^2} = \sqrt{u_1^2 + u_2^2} = 1,
\end{align}
where we used the fact that ${\bf u}$ is normalized. So, with an arbitrary input vector, we can define an orthonormal basis, which we will write in matrix by stacking the basis vectors horizontally:

\begin{align}
{{\bf W} } =
\begin{pmatrix}
 u_1 & w_1 \\
 u_2 & w_2
\end{pmatrix}.
\end{align}

## Coding Exercise 2: Find an orthonormal basis

In this exercise you will fill in the function below to define an orthonormal basis, given a single arbitrary 2-dimensional vector as an input.

**Steps**
* Modify the function `define_orthonormal_basis` to first normalize the first basis vector $\bf u$.
* Then complete the function by finding a basis vector $\bf w$ that is orthogonal to $\bf u$.
* Test the function using initial basis vector ${\bf u} = [3,1]$. Plot the resulting basis vectors on top of the data scatter plot using the function `plot_basis_vectors`. (For the data, use  $\sigma_1^2 =1$, $\sigma_2^2 =1$, and $\rho = .8$).

```python
def define_orthonormal_basis(u):
  """
  Calculates an orthonormal basis given an arbitrary vector u.

  Args:
    u (numpy array of floats) : arbitrary 2-dimensional vector used for new
                                basis

  Returns:
    (numpy array of floats)   : new orthonormal basis
                                columns correspond to basis vectors
  """

  #################################################
  ## TODO for students: calculate the orthonormal basis
  # Fill out function and remove
  # raise NotImplementedError("Student excercise: implement the orthonormal basis function")
  #################################################

  # Normalize vector u
  u = u / np.sqrt(u[0] ** 2 + u[1] ** 2)

  # Calculate vector w that is orthogonal to w
  w = np.array([-u[1], u[0]])

  # Put in matrix form
  W = np.column_stack([u, w])

  return W

print('u = W[0] =', W[0])
print('w = W[1] =', W[1])
print('u*w=', W[0]*W[1])
# Set up parameters
np.random.seed(2020)  # set random seed
variance_1 = 1
variance_2 = 1
corr_coef = 0.8
u = np.array([3, 1])

# Compute covariance matrix
cov_matrix = calculate_cov_matrix(variance_1, variance_2, corr_coef)

# Generate data
X = get_data(cov_matrix)

# Get orthonomal basis
W = define_orthonormal_basis(u)

# Visualize
plot_basis_vectors(X, W)
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D5_DimensionalityReduction/solutions/W1D5_Tutorial1_Solution_25e1d102.py)

*Example output:*

---
# Section 3: Project data onto new basis

*Estimated timing to here from start of tutorial: 35 min*

```python
# @title Video 4: Change of basis
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
      def __init__(self, id, page=1, width=400, height=300, **kwargs):
          self.id=id
          src = 'https://player.bilibili.com/player.html?bvid={0}&page={1}'.format(id, page)
          super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id="BV1LK411J7NQ", width=854, height=480, fs=1)
  print('Video available at https://www.bilibili.com/video/{0}'.format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id="Mj6BRQPKKUc", width=854, height=480, fs=1, rel=0)
  print('Video available at https://youtube.com/watch?v=' + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)
```

    Tab(children=(Output(), Output()), _titles={'0': 'Youtube', '1': 'Bilibili'})

Finally, we will express our data in the new basis that we have just found. Since $\bf W$ is orthonormal, we can project the data into our new basis using simple matrix multiplication :

\begin{align}
{\bf Y = X W}.
\end{align}

We will explore the geometry of the transformed data $\bf Y$ as we vary the choice of basis.

## Coding Exercise 3: Change to orthonormal basis
In this exercise you will fill in the function below to change data to a orthonormal basis.

**Steps**
* Complete the function `change_of_basis` to project the data onto the new basis.
* Plot the projected data using the function `plot_data_new_basis`.
* What happens to the correlation coefficient in the new basis? Does it increase or decrease?
* What happens to variance?

```python
def change_of_basis(X, W):
  """
  Projects data onto new basis W.

  Args:
    X (numpy array of floats) : Data matrix each column corresponding to a
                                different random variable
    W (numpy array of floats) : new orthonormal basis columns correspond to
                                basis vectors

  Returns:
    (numpy array of floats)    : Data matrix expressed in new basis
  """

  #################################################
  ## TODO for students: project the data onto o new basis W
  # Fill out function and remove
  # raise NotImplementedError("Student excercise: implement change of basis")
  #################################################

  # Project data onto new basis described by W
  Y = X @ W

  return Y

# Project data to new basis
Y = change_of_basis(X, W)

# Visualize
plot_data_new_basis(Y)
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D5_DimensionalityReduction/solutions/W1D5_Tutorial1_Solution_80a5f41b.py)

*Example output:*

## Interactive Demo 3: Play with the basis vectors
To see what happens to the correlation as we change the basis vectors, run the cell below. The parameter $\theta$ controls the angle of $\bf u$ in degrees. Use the slider to rotate the basis vectors.

1.   What happens to the projected data as you rotate the basis?
2.   How does the correlation coefficient change? How does the variance of the projection onto each basis vector change?
3.   Are you able to find a basis in which the projected data is **uncorrelated**?

```python
# @markdown Make sure you execute this cell to enable the widget!

def refresh(theta=0):
  u = [1, np.tan(theta * np.pi / 180)]
  W = define_orthonormal_basis(u)
  Y = change_of_basis(X, W)
  plot_basis_vectors(X, W)
  plot_data_new_basis(Y)

_ = widgets.interact(refresh, theta=(0, 90, 5))
```

    interactive(children=(IntSlider(value=0, description='theta', max=90, step=5), Output()), _dom_classes=('widge…

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D5_DimensionalityReduction/solutions/W1D5_Tutorial1_Solution_993c2f00.py)

---
# Summary

*Estimated timing of tutorial: 50 minutes*

- In this tutorial, we learned that multivariate data can be visualized as a cloud of points in a high-dimensional vector space. The geometry of this cloud is shaped by the covariance matrix.

- Multivariate data can be represented in a new orthonormal basis using the dot product. These new basis vectors correspond to specific mixtures of the original variables - for example, in neuroscience, they could represent different ratios of activation  across a population of neurons.

- The projected data (after transforming into the new basis) will generally have a different geometry from the original data. In particular, taking basis vectors that are aligned with the spread of cloud of points decorrelates the data.

* These concepts - covariance, projections, and orthonormal bases - are key for understanding PCA, which we be our focus in the next tutorial.

---
# Notation

\begin{align}
x_i &\quad \text{data point for dimension } i\\
\mu_i &\quad \text{mean along dimension } i\\
\sigma_i^2 &\quad \text{variance along dimension } i \\
\bf u, \bf w &\quad \text{orthonormal basis vectors}\\
\rho &\quad \text{correlation coefficient}\\
\bf \Sigma &\quad \text{covariance matrix}\\
\bf X &\quad \text{original data matrix}\\
\bf W &\quad \text{projection matrix}\\
\bf Y &\quad \text{transformed data}\\
\end{align}


Code for generating results for the graph with two instrumental variables (Figure 2(b) and Figure 3(b)).

```python
import numpy
import sympy
import pandas
import numpy as np
import pandas as pd
import datetime
import copy
import attr
import time
import logging
import itertools
import pickle
import sys
import os
import functools

import matplotlib.pyplot as plt
from matplotlib import rcParams
import collections

from scipy.optimize import minimize
import warnings
from IPython.display import clear_output

HOME_DIR = ""
warnings.simplefilter(action='ignore', category=UserWarning)
```

```python
class ModelParams:
  def __init__(self, b, c, d, a1, a2, z1, z2, w, x, y):
    self.b = b
    self.c = c
    self.d = d
    self.a1 = a1
    self.a2 = a2
    self.z1 = z1
    self.z2 = z2
    self.w = w
    self.x = x
    self.y = y

  def get_true_param(self):
    return self.b

  def __str__(self):
    return "b=%f, c=%f, d=%f, a1=%f, a2=%f, z1=%f, z2=%f, w=%f, x=%f, y=%f" % (self.b, self.c, self.d, self.a1, self.a2, self.z1, self.z2, self.w, self.x, self.y)

truth = ModelParams(b=1, c=1, d=1, a1=1, a2=1, z1=1, z2=4, w=1, x=1, y=1)
```

```python
def generate_data_samples(num_samples, model, k=0.5):
  np = numpy
  pd = pandas

  u_z1 = np.random.normal(scale=np.sqrt(model.z1), size=(num_samples,))
  u_z2 = np.random.normal(scale=np.sqrt(model.z2), size=(num_samples,))
  u_w = np.random.normal(scale=np.sqrt(model.w), size=(num_samples,))
  u_x = np.random.normal(scale=np.sqrt(model.x), size=(num_samples,))
  u_y = np.random.normal(scale=np.sqrt(model.y), size=(num_samples,))

  def get_Z1(u):
    return u

  def get_Z2(u):
    return u

  def get_W(u):
    return u

  def get_X(Z1, Z2, W, u):
    return model.a1 * Z1 + model.a2 * Z2 + model.c * W + u

  def get_Y(X, W, u):
    return model.b * X + model.d * W + u

  Z1 = get_Z1(u_z1)
  Z2 = get_Z2(u_z2)
  W = get_W(u_w)
  X = get_X(Z1, Z2, W, u_x)
  Y = get_Y(X, W, u_y)
  SEL = np.zeros_like(Y, np.int32)
  SEL[:int(SEL.shape[0] * k)] = 1

  df = pd.DataFrame({
      "Z1": Z1,
      "Z2": Z2,
      "X": X,
      "Y": Y,
      "SEL": SEL
  })
  return df
```

```python
num_samples = 1000
df = generate_data_samples(num_samples, truth)
df.head()
```

      Z1
      Z2
      X
      Y
      SEL

      0
      -2.071566
      -1.133387
      -3.962130
      -3.581768
      1

      1
      -0.419037
      3.996725
      3.317942
      5.427126
      1

      2
      -0.099292
      -0.049934
      -1.036306
      -1.210658
      1

      3
      -1.907402
      -2.215544
      -5.649825
      -5.745939
      1

      4
      -1.250415
      2.165945
      -2.899494
      -3.417869
      1

```python
class GMM:

  def __init__(self, df):
    np = numpy
    self.df_dict = {k: df[k].values for k in df.columns}
    self.df = self.df_dict
    self.num_samples = len(df)
    self.num_moments = 2

  def momconds(self, params):
    """Returns the emprical moment vector (equivalent of \hat{g}_n(\theta))."""
    np = numpy
    n = self.num_samples
    return np.sum(self._momconds_arr(params), axis=-1) / n

  def _compute_moment_covariance(self, params):
    """Estimate the optimal weight matrix using the emprical moments."""
    n = self.num_samples
    moms = self._momconds_arr(params)
    moment_covariance = (moms @ moms.T) / n
    return moment_covariance

  def _get_objective_fn(self, weight_matrix_inv):
    np = numpy

    # print(weight_matrix_inv)
    def objective(params):
      moms = self.momconds(params)
      w_inv_mom = np.linalg.solve(weight_matrix_inv, moms)
      obj = moms.T @ w_inv_mom
      return obj

    return objective

  def _momconds_arr(self, params):
    np = numpy

    p = params
    df = self.df
    n = self.num_samples
    moment_1_arr = df["SEL"] * df["Z1"] * (df["Y"] - df["X"] * p[0])
    moment_2_arr = (1 - df["SEL"]) * df["Z2"] * (df["Y"] - df["X"] * p[0])

    return np.array([moment_1_arr, moment_2_arr])

  def _get_asymptotic_variance_matrix(self, k, moment_covariance, params):
    np = numpy

    current_k = np.mean(self.df["SEL"])

    df = self.df
    n = self.num_samples
    jacobian_1_1 = (k / current_k) * np.sum(-df["SEL"] * df["Z1"] * df["X"]) / n
    jacobian_2_1 = ((1 - k) / (1 - current_k)) * np.sum(-(1 - df["SEL"]) * df["Z2"] * df["X"]) / n

    jacobian = np.array([[jacobian_1_1], [jacobian_2_1]])
    moment_covariance_reweight = np.array([[k/current_k, 0],
                                       [0, (1-k)/(1-current_k)]])

    moment_covariance = moment_covariance * moment_covariance_reweight
    mom_cov_inv_jac = np.linalg.solve(moment_covariance, jacobian)
    return np.linalg.inv(jacobian.T @ mom_cov_inv_jac)

  def _get_asymptotic_variance(self, k, moment_covariance, params):
    return self._get_asymptotic_variance_matrix(k, moment_covariance, params)[0][0]

  def _optimize_find_parameters(self, weight_matrix_inv, initial_guess=None):
    np = numpy

    initial_guess = np.array([truth.b]) * 1.5 if initial_guess is None else initial_guess
    res = minimize(self._get_objective_fn(weight_matrix_inv), initial_guess)
    return res.x

  def find_parameters(self, num_iters=2, weight_matrix_reg=None):
    np = numpy

    weight_matrix_inv = np.eye(self.num_moments)
    for i in range(num_iters):
      params = self._optimize_find_parameters(weight_matrix_inv, initial_guess=params if i > 0 else None)
      weight_matrix_inv = self._compute_moment_covariance(params)

      if weight_matrix_reg is not None:
        weight_matrix_inv += weight_matrix_reg * np.eye(weight_matrix_inv.shape[0])

    return params, weight_matrix_inv

  def find_optimal_k(self, moment_covariance, params):
    np = numpy

    initial_guess = np.array([0.5])
    lower_bound = 0.2
    upper_bound = 0.8
    res = minimize(lambda x: self._get_asymptotic_variance(x[0], moment_covariance, params),
                   initial_guess, bounds=[(lower_bound, upper_bound)])

    return 0 if res.x[0] <= 0.5 else 1
```

```python
gmm = GMM(df)
params, moment_covariance = gmm.find_parameters(num_iters=2)

print("True params:")
print(truth)
print("Estimated params:")
print(params)

optimal_k = gmm.find_optimal_k(moment_covariance, params)
print("Estimated optimal selection ratio k: %f" % optimal_k)

del gmm, params, moment_covariance, optimal_k
```

    True params:
    b=1.000000, c=1.000000, d=1.000000, a1=1.000000, a2=1.000000, z1=1.000000, z2=4.000000, w=1.000000, x=1.000000, y=1.000000
    Estimated params:
    [0.96418235]
    Estimated optimal selection ratio k: 0.000000

```python
class SampleRevealer:

  def __init__(self, budget, df):
    self.initial_budget = budget
    self.budget = budget
    self.buffer_size = len(df)
    self.df = df
    self.counter = 0
    self.cost_per_reveal = 1

  def reset(self):
    self.counter = 0
    self.budget = self.initial_budget

  def is_budget_left(self, samples_to_reveal):
    return self.budget >= (self.cost_per_reveal * samples_to_reveal)

  def reveal(self, reveal_k, samples_to_reveal):
    if self.counter + samples_to_reveal >= self.buffer_size:
      raise ValueError("no buffer")

    observe_count = int(reveal_k * samples_to_reveal)
    self.df["SEL"].values[self.counter:self.counter+observe_count] = 1
    self.df["SEL"].values[self.counter+observe_count:self.counter+samples_to_reveal] = 0

    self.counter += samples_to_reveal
    self.budget -= (self.cost_per_reveal * samples_to_reveal)

  def get_dataset(self):
    return self.df[:self.counter]
```

```python
def batch_fractions_to_sizes(horizon, batch_fractions):
  batch_sizes = []

  for i in range(len(batch_fractions) + 1):
    prev_batch_end = 0 if i == 0 else int(horizon * batch_fractions[i - 1])
    curr_batch_end = horizon if i == len(batch_fractions) else int(horizon * batch_fractions[i])

    batch_sizes.append(curr_batch_end - prev_batch_end)

  return batch_sizes
```

```python
def compute_reveal_k(current_samples, samples_to_reveal, current_k, target_k):
  reveal_k = (target_k * (current_samples + samples_to_reveal) - current_k * current_samples) / (samples_to_reveal)
  return min(1, (max(0, reveal_k)))
```

```python
class StrategyRunResult:

  def __init__(self):
    self.budgets_used = []
    self.squared_errors = []
    self.optimal_ks = []
    self.current_ks = []

  def append(self, budget_used, squared_error, optimal_k, current_k):
    self.budgets_used.append(budget_used)
    self.squared_errors.append(squared_error)
    self.optimal_ks.append(optimal_k)
    self.current_ks.append(current_k)
```

```python
class OracleStrategy:

  def __init__(self, sample_revealer, optimal_k, horizon, batch_fractions):
    self.sample_revealer = sample_revealer
    self.name = 'oracle'
    self.batch_sizes = batch_fractions_to_sizes(horizon, batch_fractions)
    self.optimal_k = optimal_k

  def get_current_df_vals(self):
    np = numpy

    dataset = self.sample_revealer.get_dataset()

    len_left_corner = np.sum(dataset["SEL"])
    len_right_corner = len(dataset) - len_left_corner
    total = (len_left_corner + len_right_corner)
    return len_left_corner / total, total

  def can_step(self):
    return self.sample_revealer.is_budget_left()

  def get_and_store_params(self):
    dataset = self.sample_revealer.get_dataset()
    self.gmm = GMM(dataset)
    self.params, self.moment_covariance = self.gmm.find_parameters(num_iters=2)
    return self.params

  def get_squared_error(self):
    error = (truth.get_true_param() - self.params[0])**2
    return error

  def execute_run(self):
    np = numpy

    result = StrategyRunResult()
    for i, batch_size in enumerate(self.batch_sizes):
      if i == 0:
        current_k, current_samples = 0, 0
      else:
        current_k, current_samples = self.get_current_df_vals()
      reveal_k = compute_reveal_k(current_samples, batch_size, current_k, self.optimal_k)

      self.sample_revealer.reveal(reveal_k=reveal_k, samples_to_reveal=batch_size)
      _ = self.get_and_store_params()

      current_k, _ = self.get_current_df_vals()

      result.append(
          self.sample_revealer.initial_budget - self.sample_revealer.budget,
          self.get_squared_error(), self.optimal_k, current_k,
      )

    return result
```

```python
class ETCStrategy:

  def __init__(self, sample_revealer, horizon, batch_fractions):
    self.sample_revealer = sample_revealer
    self.name = 'etc'
    # We assume that the first batch size is exploration.
    self.batch_sizes = batch_fractions_to_sizes(horizon, batch_fractions)

  def get_current_df_vals(self):
    np = numpy

    dataset = self.sample_revealer.get_dataset()

    len_left_corner = np.sum(dataset["SEL"])
    len_right_corner = len(dataset) - len_left_corner
    total = (len_left_corner + len_right_corner)
    return len_left_corner / total, total

  def can_step(self):
    return self.sample_revealer.is_budget_left()

  def get_and_store_params(self):
    dataset = self.sample_revealer.get_dataset()
    self.gmm = GMM(dataset)
    self.params, self.moment_covariance = self.gmm.find_parameters(num_iters=2)
    return self.params

  def get_squared_error(self):
    error = (truth.get_true_param() - self.params[0])**2
    return error

  def execute_run(self):
    np = numpy

    result = StrategyRunResult()
    for i, batch_size in enumerate(self.batch_sizes):
      if i == 0:
        reveal_k = 0.5
      else:
        current_k, current_samples = self.get_current_df_vals()
        reveal_k = compute_reveal_k(current_samples, batch_size, current_k, self.optimal_k)

      self.sample_revealer.reveal(reveal_k=reveal_k, samples_to_reveal=batch_size)
      params = self.get_and_store_params()

      if i == 0:
        self.optimal_k = self.gmm.find_optimal_k(self.moment_covariance, params)

      current_k, _ = self.get_current_df_vals()

      result.append(
          self.sample_revealer.initial_budget - self.sample_revealer.budget,
          self.get_squared_error(), self.optimal_k, current_k,
      )

    return result
```

```python
class ETGreedyStrategy:

  def __init__(self, sample_revealer, horizon, batch_fractions):
    self.sample_revealer = sample_revealer
    self.name = 'et_greedy'
    # We assume that the first batch size is exploration.
    self.batch_sizes = batch_fractions_to_sizes(horizon, batch_fractions)
    self.weight_matrix_reg = 0.01

  def get_current_df_vals(self):
    np = numpy

    dataset = self.sample_revealer.get_dataset()

    len_left_corner = np.sum(dataset["SEL"])
    len_right_corner = len(dataset) - len_left_corner
    total = (len_left_corner + len_right_corner)
    return len_left_corner / total, total

  def can_step(self):
    return self.sample_revealer.is_budget_left()

  def get_and_store_params(self, is_last_step):
    dataset = self.sample_revealer.get_dataset()
    self.gmm = GMM(dataset)
    if is_last_step:
      self.params, self.moment_covariance = self.gmm.find_parameters(num_iters=2)
    else:
      self.params, self.moment_covariance = (
          self.gmm.find_parameters(num_iters=2, weight_matrix_reg=self.weight_matrix_reg)
      )
    return self.params

  def get_squared_error(self):
    error = (truth.get_true_param() - self.params[0])**2
    return error

  def execute_run(self):
    np = numpy

    result = StrategyRunResult()
    for i, batch_size in enumerate(self.batch_sizes):
      if i == 0:
        reveal_k = 0.5
      else:
        current_k, current_samples = self.get_current_df_vals()
        reveal_k = compute_reveal_k(current_samples, batch_size, current_k, self.optimal_k)

      self.sample_revealer.reveal(reveal_k=reveal_k, samples_to_reveal=batch_size)
      is_last_step = (i==len(self.batch_sizes)-1)
      params = self.get_and_store_params(is_last_step)

      self.optimal_k = self.gmm.find_optimal_k(self.moment_covariance, params)

      current_k, _ = self.get_current_df_vals()

      result.append(
          self.sample_revealer.initial_budget - self.sample_revealer.budget,
          self.get_squared_error(), self.optimal_k, current_k,
      )

    return result
```

```python
def execute_strategy_iteration(strategy_name, iteration_num, horizon):
  np = numpy

  # Uncomment the next two lines to replicate the exact runs used in the paper.
  # random_seed = 232281293 + iteration_num
  # np.random.seed(random_seed)
  df = generate_data_samples(num_samples=horizon * 5, model=truth)
  np.random.seed(None)

  sample_revealer = SampleRevealer(budget=horizon, df=df)

  if strategy_name == "etc_0.2":
    strategy = ETCStrategy(sample_revealer,
                           horizon=horizon,
                           batch_fractions=[0.2, 0.9])
  elif strategy_name == "etc_0.1":
    strategy = ETCStrategy(sample_revealer,
                           horizon=horizon,
                           batch_fractions=[0.1, 0.9])
  elif strategy_name == "etc_0.4":
    strategy = ETCStrategy(sample_revealer,
                           horizon=horizon,
                           batch_fractions=[0.4, 0.9])
  elif strategy_name == "oracle":
    strategy = OracleStrategy(sample_revealer, optimal_k=0.05,
                              horizon=horizon, batch_fractions=[0.1, 0.9])
  elif strategy_name == "etg_0.1":
    strategy = ETGreedyStrategy(sample_revealer,
                        horizon=horizon,
                        batch_fractions=[0.1, 0.2, 0.3, 0.4, 0.6, 0.8])
  elif strategy_name == "etg_0.2":
    strategy = ETGreedyStrategy(sample_revealer,
                        horizon=horizon,
                        batch_fractions=[0.2, 0.4, 0.6, 0.8])
  else:
    raise ValueError("invalid strategy_name")

  return strategy.execute_run()
```

```python
# Test a strategy.

result = execute_strategy_iteration("etc_0.4", 1, horizon=1000)
print(result.squared_errors)
```

    [0.00010819794897878681, 0.0005009086364382247, 0.0005204006336571472]

**Executing the runs for each strategy in parallel.**

For the paper, we execute 12,000 runs for each strategy. We execute the runs
in parallel using
[`ipyparallel`](https://ipyparallel.readthedocs.io/en/latest/).

The following command starts the `ipyparallel` engines:
```
ipcluster start -n <num_engines>
```

```python
import ipyparallel as ipp
```

```python
# Verify that ipcluster is running and import the necessary Python packages.

parallel_client = ipp.Client(debug=False)
dview = parallel_client[:]
# Execute an identity map in parallel.
ar = dview.map(lambda x: x, (i for i in range(0, 2000000, 2)))
assert ar.get()[0] == 0

# Import the required Python packages.
with dview.sync_imports():
  from abc import ABC, abstractmethod
  import numpy
  import sympy
  import pandas
  import sympy
  import datetime
  import copy
  import attr
  import time
  import logging
  import itertools
  import pickle
  import os
  import functools
  import ipyparallel

  import collections

  from scipy.optimize import minimize
  import warnings

  try:
    from cPickle import dumps, loads, HIGHEST_PROTOCOL as PICKLE_PROTOCOL
  except ImportError:
    from pickle import dumps, loads, HIGHEST_PROTOCOL as PICKLE_PROTOCOL

# Make sure ipyparallel is still able to execute functions.
dview = parallel_client[:]
ar = dview.map(lambda x: x, (i for i in range(0, 2000000, 2)))
assert ar.get()[0] == 0
```

    importing ABC,abstractmethod from abc on engine(s)
    importing numpy on engine(s)
    importing sympy on engine(s)
    importing pandas on engine(s)
    importing datetime on engine(s)
    importing copy on engine(s)
    importing attr on engine(s)
    importing time on engine(s)
    importing logging on engine(s)
    importing itertools on engine(s)
    importing pickle on engine(s)
    importing os on engine(s)
    importing functools on engine(s)
    importing ipyparallel on engine(s)
    importing collections on engine(s)
    importing minimize from scipy.optimize on engine(s)
    importing warnings on engine(s)

```python
def combine_parallel_results(async_result):
  budgets = async_result.get()[0].budgets_used
  errors = np.vstack([res.squared_errors for res in async_result.get()])
  optimal_ks = np.vstack([res.optimal_ks for res in async_result.get()])
  current_ks = np.vstack([res.current_ks for res in async_result.get()])

  return budgets, errors, optimal_ks, current_ks

def execute_strategy_in_parallel(strategy_name, horizon, iterations):
  num_threads = len(parallel_client.ids)
  dview = parallel_client[:]

  print("Executing %s over %d iterations across %d engines" % (strategy_name, iterations, num_threads))

  dview["batch_fractions_to_sizes"] = batch_fractions_to_sizes
  dview["compute_reveal_k"] = compute_reveal_k
  dview["ModelParams"] = ModelParams
  dview["truth"] = truth
  dview["generate_data_samples"] = generate_data_samples
  dview["StrategyRunResult"] = StrategyRunResult
  dview["GMM"] = GMM
  dview["ETCStrategy"] = ETCStrategy
  dview["OracleStrategy"] = OracleStrategy
  dview["ETGreedyStrategy"] = ETGreedyStrategy
  dview["SampleRevealer"] = SampleRevealer
  dview["execute_strategy_iteration"] = execute_strategy_iteration

  def execute_iteration(i):
    return execute_strategy_iteration(strategy_name, i, horizon)

  return dview.map(execute_iteration, range(iterations))
```

```python
def get_timeseries_for(strategy_names, horizons, iterations, results_dict):
  results_dict["truth"] = truth

  for strategy_name in strategy_names:
    for horizon in horizons:
      print("Timestamp start: %s, Strategy: %s, Horizon: %d, Iters: %d" % (
          datetime.datetime.now(), strategy_name, horizon, iterations))
      async_result = execute_strategy_in_parallel(strategy_name, horizon, iterations)
      budgets, errors, optimal_ks, current_ks = combine_parallel_results(async_result)
      print("Timestamp end: %s" % (datetime.datetime.now()))

      if strategy_name not in results_dict:
        results_dict[strategy_name] = {}

      results_dict[strategy_name][horizon] = {
          "budgets": budgets,
          "errors": errors * budgets,
          "optimal_ks": optimal_ks,
          "current_ks": current_ks,
      }
```

```python
results_dict = {}
get_timeseries_for([
                    "oracle",
                    "etc_0.1", "etc_0.2", "etc_0.4",
                    "etg_0.1", "etg_0.2",
                    ],
                   horizons=[200, 300, 400, 500, 600, 800, 1000, 1300],
                   iterations=12,
                   results_dict=results_dict)
```

```python
# Save results to file.
# pickle.dump(results_dict,
#             open(os.path.join(HOME_DIR,
#                               "%s_two_ivs.pkl" % datetime.datetime.now()),
#                  "wb"))
```

### Plot the results

```python
# For the color map:
# https://gist.github.com/AndiH/c957b4d769e628f506bd

# Tableau 20 Colors
tableau20 = [(31, 119, 180), (174, 199, 232), (255, 127, 14), (255, 187, 120),
             (44, 160, 44), (152, 223, 138), (214, 39, 40), (255, 152, 150),
             (148, 103, 189), (197, 176, 213), (140, 86, 75), (196, 156, 148),
             (227, 119, 194), (247, 182, 210), (127, 127, 127), (199, 199, 199),
             (188, 189, 34), (219, 219, 141), (23, 190, 207), (158, 218, 229)]

# Tableau Color Blind 10
tableau20blind = [(0, 107, 164), (255, 128, 14), (171, 171, 171), (89, 89, 89),
             (95, 158, 209), (200, 82, 0), (137, 137, 137), (163, 200, 236),
             (255, 188, 121), (207, 207, 207)]

# Rescale to values between 0 and 1
for i in range(len(tableau20)):
    r, g, b = tableau20[i]
    tableau20[i] = (r / 255., g / 255., b / 255.)
for i in range(len(tableau20blind)):
    r, g, b = tableau20blind[i]
    tableau20blind[i] = (r / 255., g / 255., b / 255.)
```

```python
def plot_regret_curve(results_dict):

  clist = rcParams['axes.prop_cycle']
  cgen = itertools.cycle(clist)

  oracle_mses = {}
  for horizon, timeseries in results_dict["oracle"].items():
    oracle_mses[timeseries["budgets"][-1]] = np.mean(timeseries["errors"][:, -1])

  def plot(axs, name, info, result, with_var=False):
    x_vals = []
    y_mean = []
    y_std = []

    current_k_mean = []
    optimal_k_mean = []

    for horizon, timeseries in result.items():
      x_vals.append(timeseries["budgets"][-1])
      y_scaled = ( timeseries["errors"][:, -1] - oracle_mses[x_vals[-1]] )  / oracle_mses[x_vals[-1]] * 100
      y_mean.append(np.mean(y_scaled))
      y_std.append(np.std(y_scaled))

    x_vals = np.array(x_vals)
    y_mean = np.array(y_mean)
    y_std = np.array(y_std)

    x_sort_idx = np.argsort(x_vals)
    x_vals = x_vals[x_sort_idx]
    y_mean = y_mean[x_sort_idx]
    y_std = y_std[x_sort_idx]

    color = next(cgen)["color"]

    plt.plot(x_vals, y_mean, label=name, color=info[1], linestyle=info[0],  marker=info[2])
    plt.legend()

    if with_var:
      ci = 1.96 * y_std / np.sqrt(timeseries["errors"].shape[0])
      axs.errorbar(x_vals, y_mean, yerr=ci, ls="none", color=info[1])

  name_to_linestyle_color = {
      "etc_0.1": ["dashed", tableau20blind[0], "o"],
      "etc_0.2": ["dashdot", tableau20blind[1], "s"],
      "etc_0.4": ["solid", tableau20blind[2], "v"],
      "etg_0.1": ["dotted", tableau20blind[4], "^"],
      "etg_0.2": ["dashed", tableau20blind[6], "D"],
  }
  plt.title("Relative regret vs horizon")
  plt.xlabel("Total samples collected (Horizon)")
  plt.ylabel("Relative regret (%)")
  for name, info in name_to_linestyle_color.items():
    plot(plt, name, info, results_dict[name], with_var=True)

  # plt.savefig(os.path.join(HOME_DIR, "figures/two_ivs_regret_curve.eps"), bbox_inches='tight', pad_inches=0.0)
```

```python
SMALL_SIZE = 12
MEDIUM_SIZE = 12
BIGGER_SIZE = 12

plt.rc('font', size=SMALL_SIZE)          # controls default text sizes
plt.rc('axes', titlesize=SMALL_SIZE+4)     # fontsize of the axes title
plt.rc('axes', labelsize=MEDIUM_SIZE+4)    # fontsize of the x and y labels
plt.rc('xtick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
plt.rc('ytick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
plt.rc('legend', fontsize=SMALL_SIZE)    # legend fontsize
plt.rc('figure', titlesize=BIGGER_SIZE+20)  # fontsize of the figure title
```

```python
plot_regret_curve(results_dict)
```