```python
# %load imports.py
## Local packages:

%matplotlib inline
%load_ext autoreload
%autoreload 2
%config Completer.use_jedi = False  ## (To fix autocomplete)

## External packages:
import pandas as pd
pd.options.display.max_rows = 999
pd.options.display.max_columns = 999
pd.set_option("display.max_columns", None)
import numpy as np
np.set_printoptions(linewidth=150)

import numpy as np
import os
import matplotlib.pyplot as plt
#if os.name == 'nt':
#    plt.style.use('presentation.mplstyle')  # Windows

import plotly.express as px
import plotly.graph_objects as go

import seaborn as sns
import sympy as sp
from sympy.physics.mechanics import (dynamicsymbols, ReferenceFrame,
                                      Particle, Point)
from sympy.physics.vector.printing import vpprint, vlatex
from IPython.display import display, Math, Latex
from src.substitute_dynamic_symbols import run, lambdify

import pyro

import sklearn
import pykalman
from statsmodels.sandbox.regression.predstd import wls_prediction_std
import statsmodels.api as sm

from scipy.integrate import solve_ivp

## Local packages:
from src.data import mdl

from src.symbols import *
from src.parameters import *
import src.symbols as symbols
from src import prime_system
from src.models import regression
from src.visualization.regression import show_pred
from src.visualization.plot import track_plot

## Load models:
# (Uncomment these for faster loading):
import src.models.vmm_martin  as vmm
from src.models.vmm import ModelSimulator
from example_ship2 import ship_parameters, df_parameters

if os.name == 'nt':
    plt.style.use('../docs/book/book.mplstyle')  # Windows

from src.visualization.plot import track_plot, plot
```

```python
from src.extended_kalman_vmm import ExtendedKalman
from src.extended_kalman_filter import loglikelihood
```

## Load test

```python
id=22773
df, units, meta_data = mdl.load(dir_path = '../data/raw', id=id)
df.index = df.index.total_seconds()
df.index-=df.index[0]
df['x0']-=df.iloc[0]['x0']
df['y0']-=df.iloc[0]['y0']
df['psi']-=df.iloc[0]['psi']

```

```python
fig,ax=plt.subplots()
fig.set_size_inches(10,10)
track_plot(df=df, lpp=meta_data.lpp, x_dataset='x0', y_dataset='y0',  psi_dataset='psi', beam=meta_data.beam, ax=ax);
```

## Filtering

### Lowpass filter

```python
from numpy import cos as cos
from numpy import sin as sin
from src.data.lowpass_filter import lowpass_filter

df_lowpass = df.copy()
t = df_lowpass.index
ts = np.mean(np.diff(t))
fs = 1/ts

position_keys = ['x0','y0','psi']
for key in position_keys:
    df_lowpass[key] = lowpass_filter(data=df_lowpass[key], fs=fs, cutoff=1, order=1)

df_lowpass['x01d_gradient'] = x1d_ = np.gradient(df_lowpass['x0'], t)
df_lowpass['y01d_gradient'] = y1d_ = np.gradient(df_lowpass['y0'], t)
df_lowpass['r'] = r_ = np.gradient(df_lowpass['psi'], t)

psi_ = df_lowpass['psi']

df_lowpass['u'] = x1d_*cos(psi_) + y1d_*sin(psi_)
df_lowpass['v'] = -x1d_*sin(psi_) + y1d_*cos(psi_)

velocity_keys = ['u','v','r']
for key in velocity_keys:
    df_lowpass[key] = lowpass_filter(data=df_lowpass[key], fs=fs, cutoff=1, order=1)
```

```python
data = df.copy()
data['thrust'] = data['Prop/PS/Thrust'] + data['Prop/SB/Thrust']

data['u'] = df_lowpass['u']
data['v'] = df_lowpass['v']
data['r'] = df_lowpass['r']
data=data.iloc[200:-100]
data.index-=data.index[0]
```

### Extended Kalman filter

```python
parameters = df_parameters["prime"].copy()
ek = ExtendedKalman(vmm=vmm,
                    parameters=parameters,
                    ship_parameters=ship_parameters)

```

#### Simulate with system model

```python
E = np.array([
    [0,0,0],
    [0,0,0],
    [0,0,0],
    [1,0,0],
    [0,1,0],
    [0,0,1],
    ],
)

t = np.linspace(0,10,100)
data_ = pd.DataFrame(index=t)
data_['delta'] = 0.0
data_['thrust'] = 20
data_['x0'] = 0
data_['y0'] = 0
data_['psi'] = 0
data_['u'] = 2
data_['v'] = 0
data_['r'] = 0

ek.simulate(data=data, input_columns=['delta','thrust'],
            E=E).tail()
```

```python
data_frames = {'data':data, 'sim':ek.df_simulation}

fig,ax=plt.subplots()
styles = {
    'Mesurement' : {
        'linestyle' : '',
        'marker' : '.',
        'ms' : 1,
        'zorder':-10,
    },

    'Kalman filter' : {
        'lw' : 2,
    },

}

for label,df_ in data_frames.items():
    track_plot(
            df=df_,
            lpp=ship_parameters["L"],
            beam=ship_parameters["B"],
            ax=ax,
            label=label,
            plot_boats=True,
            **styles.get(label,{})
        );
ax.legend()

plot(data_frames, keys=ek.df_simulation.columns);
```

### Extended Kalman filter and RTS smoother

```python
Cd = np.array([
    [1, 0, 0, 0, 0, 0],
    [0, 1, 0, 0, 0, 0],
    [0, 0, 1, 0, 0, 0],
])

E = np.array([
    [0,0,0],
    [0,0,0],
    [0,0,0],
    [1,0,0],
    [0,1,0],
    [0,0,1],
    ],
)

P_prd = np.diag([0.1, 0.1, np.deg2rad(0.01), 0.01, 0.01, np.deg2rad(0.01)])
Qd = np.diag([0.01, 0.01, np.deg2rad(0.01)])  #process variances: u,v,r

error_max_pos = 0.1
sigma_pos = error_max_pos/3
variance_pos = sigma_pos**2

error_max_psi = np.deg2rad(0.1)
sigma_psi = error_max_psi/3
variance_psi = sigma_psi**2

Rd = np.diag([variance_pos, variance_pos, variance_psi])

ek.filter(
        data=data, P_prd=P_prd, Qd=Qd, Rd=Rd, E=E, Cd=Cd,
        input_columns=['delta','thrust'],
    )

ek.smoother();
```

```python
dataframes = {
    'Mesurement' : data,
    #'Kalman filter' : ek.df_kalman,
    'RTS': ek.df_smooth,
}

styles = {
    'Mesurement' : {
        'linestyle' : '',
        'marker' : '.',
        'ms' : 1,
        'zorder':-10,
    },

    'Kalman filter' : {
        'lw' : 2,
    },

}

plot(dataframes = dataframes,
     fig_size=(10,10),
     styles = ['r-','g-','b-'],
     keys=['x0','y0','psi','u','v','r','u1d','v1d','r1d']);
```

```python
ek2 = ek.copy()
```

```python
loglikelihood(ek2.time_steps_smooth)
```

```python
ek2.parameters
```

```python
def vary(parameters):

    ek2.parameters = parameters

    ek2.filter(
        data=data, P_prd=P_prd, Qd=Qd, Rd=Rd, E=E, Cd=Cd,
        input_columns=['delta','thrust'],
    )

    ek2.smoother();

    return ek2

```

```python
from scipy.optimize import minimize

def _vary(x, keys:list):
    parameters = deepcopy(ek.parameters)
    parameters[keys] = x

    ek_ = vary(parameters)
    return ek_

def fun(x,keys:list):
    ek_ = _vary(x, keys)
    return -loglikelihood(ek_.time_steps_smooth)

```

```python
from copy import deepcopy

eks = []
key = 'Ndelta'
keys = [key]
values = np.linspace(0.5*ek.parameters[key], 1.5*ek.parameters[key], 5)
for value in values:

    ek_ = _vary(value, keys)
    eks.append(ek_.copy())

```

```python
loglikelihoods = []

for ek_ in eks:

    loglikelihoods.append(loglikelihood(ek_.time_steps_smooth))

```

```python
fig,ax=plt.subplots()

ax.plot(values, loglikelihoods)
ax.set_xlabel(key)
ax.set_ylabel('loglikelihood')
```

```python
ek.parameters.values
```

```python
%%time
#keys = ek.parameters.keys()
#keys = ['Ndelta']

keys = [
'Xu',
'Ndelta',
'Nv',
'Nr',
]

parameters = ek.parameters[keys]
x0 = parameters.values
#bounds = [(-np.inf,0.0001) if value < 0 else (0.0001,np.inf) for value in parameters.values]

small=0.5
large=3
bounds = [(large*value,small*value) if value < 0 else (small*value,large*value) for value in parameters.values]

#res = minimize(fun, x0=x0, bounds=bounds, tol=10**6, args=(keys,))

res = minimize(fun, x0=x0, tol=10**6, args=(keys,))
```

```python
res
```

```python
x0
```

```python
ek_ = _vary(res.x, keys=keys)
ek_.simulate(data=data, input_columns=['delta','thrust'],
            E=E).tail()
```

```python
#keys = [
#'Xu',
#'Xvr',
#'Ydelta',
#'Yv',
#'Yur',
#'Ndelta',
#'Nv',
#]
#
#x = [-0.003     , -0.006     ,  0.003     , -0.01231641,  0.00413222, -0.0015    , -0.00318395]
#ek_ = _vary(x, keys=keys)
#ek_.simulate(data=data, input_columns=['delta','thrust'],
#            E=E, x0=data.iloc[0][["x0", "y0", "psi", "u", "v", "r"]].values).tail()
```

```python
data_frames = {'data':data, 'sim':ek_.df_simulation}

fig,ax=plt.subplots()
styles = {
    'Mesurement' : {
        'linestyle' : '',
        'marker' : '.',
        'ms' : 1,
        'zorder':-10,
    },

    'Kalman filter' : {
        'lw' : 2,
    },

}

for label,df_ in data_frames.items():
    track_plot(
            df=df_,
            lpp=ship_parameters["L"],
            beam=ship_parameters["B"],
            ax=ax,
            label=label,
            plot_boats=True,
            **styles.get(label,{})
        );
ax.legend()

plot(data_frames, keys=ek.df_simulation.columns);
```

```python
%%time
input = {'delta':0,'thrust':1.0}
psi = 0
u = 2
v = 0
r = 0
x_dot = run(
            ek._lambda_f,
            **ek.parameters,
            **ek.ship_parameters,
            **input,
            psi=psi,
            u=u,
            v=v,
            r=r,
        )
```

```python
ek._lambda_f
```

```python
parameters = deepcopy(ek.parameters)
ship_parameters = deepcopy(ek.ship_parameters)

from inspect import signature
s = signature(ek._lambda_f)
keys = list(set(ek.ship_parameters) & set(s.parameters.keys()))
ship_parameters = {key:value for key,value in ek.ship_parameters.items() if key in keys}

```

```python
%%time

input = {'delta':0,'thrust':1.0}
psi = 0
u = 2
v = 0
r = 0

ek._lambda_f(**parameters,
             **input,
             **ship_parameters,
            psi=psi,
            u=u,
            v=v,
            r=r,)
```

```python

```

```python

```

```python

```


# Tutorial 1 Part 1 - Sequential Probability Ratio Test

Please execute the cell below to initialize the notebook environment

```python
import numpy as np                 # import numpy
import scipy as sp                 # import scipy
from scipy import stats

import matplotlib.pyplot as plt    # import matplotlib
```

```python
#@title Figure Settings
#@ Figure Settings
%matplotlib inline
fig_w, fig_h = (8, 6)
plt.rcParams.update({'figure.figsize': (fig_w, fig_h)})
# %config InlineBackend.figure_format = 'retina'
```

```python
#@title Helper functions
def simulate_and_plot_SPRT_fixedtime(sigma, stop_time, num_samples):
  """Helper function for Exercise 1"""

  evidence_history_list = []
  print("#Trial\tTotal_Evidence\tDecision")
  for i in range(num_sample):
    evidence_history, decision, data = simulate_SPRT_fixedtime(sigma, stop_time)
    print("{}\t{:f}\t{}".format(i,evidence_history[-1], decision))
    evidence_history_list.append(evidence_history)

  fig, ax = plt.subplots()
  maxlen_evidence = np.max(list(map(len,evidence_history_list)))
  ax.plot(np.zeros(maxlen_evidence),'--',c='red',alpha=1.0)
  for evidences in evidence_history_list:
    ax.plot(np.arange(len(evidences)), evidences)
    ax.set_xlabel("Time")
    ax.set_ylabel("Cumulated log likelihood ratio")
    ax.set_title("Log likelihood ratio trajectories under the fixed-time stopping rule")

  plt.show(fig)

def simulate_and_plot_accuracy_vs_stoptime(sigma, stop_time_list, num_sample):
  """Helper function for Exercise 2"""
  accuracy_list, _ = simulate_accuracy_vs_stoptime(sigma, stop_time_list, num_sample=num_sample)

  fig, ax = plt.subplots()
  ax.plot(stop_time_list,accuracy_list)
  ax.set_xlabel('Stop Time')
  ax.set_ylabel('Average Accuracy')

  plt.show(fig)

def simulate_and_plot_SPRT_fixedthreshold(sigma, num_sample, alpha):
  """Helper function for Exercise 3"""
  # calculate evidence threshold from error rate
  threshold = threshold_from_errorrate(alpha)

  # run simulation
  evidence_history_list = []

  print("#Trial\tTime\tCumulated Evidence\tDecision")
  for i in range(num_sample):
    evidence_history, decision, data = simulate_SPRT_threshold(sigma, threshold, batch_size=10)
    print("{}\t{}\t{:f}\t{}".format(i,len(data), evidence_history[-1], decision))
    evidence_history_list.append(evidence_history)

  fig, ax = plt.subplots()
  maxlen_evidence = np.max(list(map(len,evidence_history_list)))
  ax.plot(np.repeat(threshold,maxlen_evidence + 1),c="red")
  ax.plot(-np.repeat(threshold,maxlen_evidence + 1),c="red")
  ax.plot(np.zeros(maxlen_evidence + 1),'--',c='red',alpha=0.5)

  for evidences in evidence_history_list:
      ax.plot(np.arange(len(evidences) + 1), np.concatenate([[0],evidences]))

  ax.set_xlabel("Time")
  ax.set_ylabel("Cumulated log likelihood ratio")
  ax.set_title("Log likelihood ratio trajectories under the threshold rule")

  plt.show(fig)

def simulate_and_plot_speed_vs_accuracy(sigma, threshold_list, num_sample):
  """Helper function for Exercise 4"""
  accuracy_list, decision_speed_list = simulate_accuracy_vs_speed(sigma, threshold_list, num_sample=num_sample)

  fig, ax = plt.subplots()
  ax.plot(decision_speed_list, accuracy_list, linestyle="--", marker="o")
  ax.plot([0.0, 1.0], [0.5, 0.5], c='red')  # plot baseline for random choice
  ax.set_xlabel("Average Decision Speed")
  ax.set_ylabel('Average Accuracy')
  ax.set_title("Speed/Accuracy Tradeoff")
  ax.set_ylim(bottom=0.45)

  plt.show(fig)
```

# Introduction

```python
#@title Video 1
# Insert the ID of the corresponding youtube video
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="3CG92WsTk4Q", width=854, height=480, fs=1)
print("Video available at https://youtu.be/" + video.id)
video
```

    Video available at https://youtu.be/3CG92WsTk4Q

### Tutorial objectives
In this tutorial, we will consider a simplified random dot motion task. On each trial $i$, we are shown a dot moving at velocity $v_i$, either in either a leftward ($v<0$) or rightward ($v>0$) direction. Although the dots' velocity varies from trial to trial, the set of all $v_i$ are  generated by a fixed probability distribution, which we know to be either:
$$
\\
\begin{eqnarray}
p_L &=& \mathcal{N}(-1,\; \sigma^2) \\
&&\textrm{or} \\
p_R &=& \mathcal{N}(+1,\; \sigma^2) \\
\end{eqnarray}
\\
$$
We want to determine which of these two possibilities is the true data generating distribution.

In W2D1, we learned how to choose between possibilities based on the relative weight of the evidence. Depending on the sensory evidence and our prior experience, we learned to choose between these *two* options: accepting hypothesis $H_L$, that the data comes from the $p_L$ distribution, or accepting $H_R$, that it comes from $p_R$.

Here, we add a *third* option: choose to collect more evidence before making a decision.

---

In this notebook we will perform a *Sequential Probability Ratio Test* between two hypotheses $H_L$ and $H_R$ by running simulations of a *Drift Diffusion Model (DDM)*.

As independent and identically distributed(*i.i.d*) samples from the true data-generating distribution coming in, we accumulate our evidence linearly until a certain criterion is met before deciding which hypothesis to accept.  Two types of stopping criterion/stopping rule will be implemented: after seeing a fixed amount of data, and after the likelihood ratio passes a pre-defined threshold. Due to the noisy nature of observations, there will be a *drifting* term governed by expected mean output and a *diffusion* term governed by observation noise.

In this tutorial, you will

* Simulate Drift-Diffusion Model with different stopping rules.
* Observe the relation between accuracy and reaction time, get an intuition about the speed/accuracy tradeoff.

---

### Sequential Probability Ratio Test(SPRT)

Suppose we receive a sequence of independent samples from distribution $p$. We know that $p$ is from $\{p_0,p_1\}$ determined by a binary latent variable $x$ and need to test between the two hyptheses:

$H_L: p=p_L \text{ or } x=0$

$H_R: p=p_R \text{ or } x=1$

When we see $n$ samples $\{x_{1}...x_n\}$, we want to calculate the total log likelihood ratio as our evidence for decision:

$$S_n = \log \frac{\prod_{i=1}^n p_R(x_i)}{\prod_{i=1}^n p_L(x_i)} = \sum_{i=1}^n \log p_R(x_i) - \sum_{i=}^n \log p_L(x_i) \tag{1}$$

Due to the independence of samples, this can be calculated in a incremental way when new data points come in sequentially:

$$ S_n =  S_{n-1} + \log \frac{p_R(x_n)}{p_L(x_n)} = S_{n-1} + \log \Lambda_n \tag{2}$$

The stopping rule can be implemented in two ways:

1. Fixed time

Make a decision based on $S_n$ immediately when we have collected $n$ samples. That is, accept $H_R$ if $S_n > 0$, accept $H_L$ if $S_n < 0$, and accept $H_R$ with probability $\frac{1}{2}$ if $S_n = 0$. The significance level or desired error rate $\alpha$ can then be determined as

$$\alpha = \frac{1}{1+\exp(|S_n|)} \tag{4}$$

2. Thresholding

We assume equal probability to make a false positive decision and to make a false negative decision, and denote it with $\alpha$. Then we accept hypothesis $H_R$ if $S_n \ge b$ or accept hypothesis $H_L$ if $S_n \le a$ where the thresholds are determined by

$$a=\log \frac{\alpha}{1-\alpha},b=\log \frac{1-\alpha}{\alpha} \tag{3}$$

# SPRT as a Drift Diffusion Model (DDM)

```python
#@title Video 2
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="a-O5n_d5tOA", width=854, height=480, fs=1, start=0, end=40)
print("Video available at https://youtu.be/" + video.id)
video
```

    Video available at https://youtu.be/a-O5n_d5tOA

SPRT as a Drift Diffusion Model (DDM)

Let's assume two different Gaussian observation models conditioned on discrete latent variable $z$

$$p_L(x|z=0) = \mathcal{N}(\mu_L,\sigma_L^2)$$

$$p_R(x|z=1) = \mathcal{N}(\mu_R,\sigma_R^2)$$

Then the log likelihood ratio for a single data point $x_i$ is

$$ \log \Lambda_i = \log \frac{\sigma_L}{\sigma_R} -0.5 \biggr[\frac{(x_i-\mu_1)^2}{\sigma_R^2} - \frac{(x_i-\mu_L)^2}{\sigma_L^2}\biggl] \tag{5}$$

Without loss of generality, let's further assume the true data generating distribution is $p_R$. In this case $x_i$ can be expressed as $x_i = \mu_R + \sigma_R \epsilon$ where $\epsilon$ comes from a standard Gaussian. The foregoing formula can then be rewritten as

$$
\log \Lambda_i = \bigg( \log \frac{\sigma_L}{\sigma_R} + 0.5 \frac{(\mu_R-\mu_L)^2}{\sigma_L^2} \bigg) + \bigg( \frac{\mu_R-\mu_L}{\sigma_L^2}\epsilon -0.5\big[1-(\frac{\sigma_R}{\sigma_L})^2\big]\epsilon^2  \bigg) \tag{6}
$$

where the first two constant terms serve as the *drifting* part and the last terms are the *diffusion* part. If we further let $\sigma_L=\sigma_R$, we can get rid of the quadratic term and this reduces to the classical discrete drift-diffusion equation where we have analytical solutions for mean and expected auto-covariance:

$$
\log \Lambda_i = 0.5 \frac{(\mu_R-\mu_L)^2}{\sigma_L^2}  + \frac{\mu_R-\mu_L}{\sigma_L^2}\epsilon, \text{where } \epsilon \sim \mathcal{N}(0,1)
$$

# 1. Simulating DDM with fixed-time stopping rule

```python
#@title Video 3a
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="TpqMzI0iZJ8", width=854, height=480, fs=1, start=0, end=40)
print("Video available at https://youtu.be/" + video.id)
video
```

    Video available at https://youtu.be/TpqMzI0iZJ8

Exercise 1
---

Assume we are performing a random dot motion task and at each time we see a moving dot with velocity $x_t$. All data points are sampled from the same distribution $p$, which is either $p_L=\mathcal{N}(-\mu,\sigma^2)$ or $p_R=\mathcal{N}(\mu,\sigma^2)$. Let's now generate some simulated data under this setting and perform SPRT using the fixed time stopping rule.

In this exercise, without loss of generality, we assume the true data-generating model is $p_R$.

* Complete the code in function `simulate_SPRT_fixedtime` to create two Gaussian random variables to represent our observation models.
* Complete the function `log_likelihood_ratio` to calculate log likelihood ratio for a sequence of data points
* Complete the code in function `simulate_SPRT_fixedtime` to calculate cumulated evidence given a list of individual evidences
* Run 10 simulations and plot the DDM traces by commenting out our provided code

```python
def log_likelihood_ratio(xvec,p0,p1):
    """
    Given a sequence(vector) of observed data, calculate the log of likelihood ratio of p1 and p0

    Args:
        xvec (numpy vector): a vector of scalar measurements
        p0 (Gaussian random variable): a normal random variable with `logpdf` method
        p1 (Gaussian random variable): a normal random variable with `logpdf` method

    Returns:
        llvec: a vector of log likelihood ratios for each input data point
    """
    ################################################################################
    ## Insert your code here to:
    ##      Calculate the log of likelihood ratios of `p1` and `p0` given a vector of measurements
    ##      Hint: using `p1.logpdf`
    ################################################################################
    raise NotImplementedError("Function `log_likelihood_ratio` incomplete")

def simulate_SPRT_fixedtime(sigma, stop_time, true_dist=1):
    """
    Simulate a Sequential Probability Ratio Test with fixed time stopping rule.
    Two observation models are 1D Gaussian distributions N(1,sigma^2) and N(-1,sigma^2).

    Args:
        sigma (float): standard deviation
        stop_time (int): number of samples to take before stopping

    Returns:
        evidence_history (numpy vector): the history of cumulated evidence given generated data
        decision (int): 1 for pR, 0 for pL
        data (numpy vector): the generated sequences of data in this trial
    """
    muL = -1.0
    muR = 1.0
    ################################################################################
    ## Insert your code here to:
    ##      Create two Gaussian variables `pL` and `pR` with mean `muL` and `muR` respectively and same std. `sigma`
    ##      Hint: using `stats.norm(...)` to construct an instance of 1D Gaussian distribution
    ################################################################################

    if 'pL' not in locals() or 'pR' not in locals():
        raise NotImplementedError("Function `simulate_SPRT_fixedtime` incomplete")

    # Generate a random sequence of data
    if true_dist == 1:
        data = pR.rvs(size=stop_time)
    else:
        data = pL.rvs(size=stop_time)

    # Calculate cumulated evidence
    ll_vec = log_likelihood_ratio(data, pL, pR)
    ################################################################################
    ## Insert your code here to:
    ##      Calculate cumulated evidence given a vector of individual evidences
    ##      Hint: use `np.cumsum`
    ################################################################################
    evidence_history = None

    if evidence_history is None:
        raise NotImplementedError("Function `simulate_SPRT_fixedtime` incomplete")

    # Make decision
    if evidence_history[-1] > 0:
        decision = 1
    elif evidence_history[-1] < 0:
        decision = 0
    else:
        decision = np.random.randint(2)

    return evidence_history, decision, data

sigma = 3.5  # standard deviation for pL and pR
num_sample = 10  # number of simulations to run for each stopping time
stop_time = 150 # stopping time

#######p#########################################################################
## Un-comment the following code block after completing this exercise
################################################################################
#plot_and_simulate_SPRT_fixedtime(sigma, stop_time, num_sample)
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W2D3_DecisionMaking/solutions/W2D3_Tutorial1_Solution_a06a0721.py)

*Example output:*

Now let's look at how the dynamics change if you change the noise level and stopping time.

* Play with different noise levels and stopping times and observe the corresponding trajectories. Once you have completed exercise 1, check the box to enable the demo.

*Hint*: you can click `...`>`Form`>`Hide code` to hide the code section if the slider is too far from the figure

```python
#@title # Interactive Demo: Noise level and stopping time { run : "auto" }

sigma = 4  #@param {type:"slider", min:0.05, max:10.0, step:0.05}
num_sample = 10  # number of simulations to run for each stopping time
stop_time = 394 #@param {type:"slider", min:5, max:500, step:1}
################################################################################
## Un-comment the following code block after complete the exercise above
################################################################################
try:
  simulate_and_plot_SPRT_fixedtime(sigma, stop_time, num_sample)
except:
  print("Complete Exercise 1 first")
```

# 2. Accuracy vs. Stopping time

```python
#@title Video 3b
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="TpqMzI0iZJ8", width=854, height=480, fs=1, start=41)
print("Video available at https://youtu.be/" + video.id)
video
```

    Video available at https://youtu.be/TpqMzI0iZJ8

Exercise 2
---

If you stop taking samples too early, e.g., make a decision after only seeing 5 samples, or there's a huge amount of observation noise that buries the signal, you are likely to be driven by observation noise to a negative cumulated log likelihood ratio and thus make a wrong decision. You could get a sense of this by increasing noise level or decreasing stopping time in the last exercise.

Now let's look at how decision accuracy varies with the number of samples we see quantitatively. First we'll fix our observation noise level. In this exercise you will run several repeated simulations for a certain stopping time to calculate the average decision accuracy. Do this for a range of stopping times and plot the relation between average decision accuracy and stopping time. You should get a positive correlation between these two quantities.

* Choose a noise level. For example, $\sigma=3$
* Complete the function `simulate_accuracy_vs_stoptime` to simulate and compute corresponding average accuracies for a list of stopping times.
* Plot accuracy versus stopping time using the pre-written codes

```python
sigma = 4.65  # standard deviation for observation noise
num_sample = 200  # number of simulations to run for each stopping time
# list of stopping times to play with
stop_time_list = list(range(1,150,10))

def simulate_accuracy_vs_stoptime(sigma, stop_time_list, num_sample=100):
    """
    Calculate the average decision accuracy vs. stopping time by running repeated SPRT simulations
    for each stop time

    Args:
        sigma (float): standard deviation for observation model
        stop_list_list (list-like object): a list of stopping times to run over
        num_sample (int): number of simulations to run per stopping time

    Returns:
        accuracy_list: a list of average accuracies corresponding to input `stop_time_list`
        decisions_list: a list of decisions made in all trials
    """
    accuracy_list = []
    decisions_list = []
    for stop_time in stop_time_list:
        decision_list = []
        ################################################################################
        ## Insert your code here to:
        ##      * Run `num_sample` repeated simulations, collect decision into `decision_list`
        ##      * Calculate average decision accuracy as `accuracy`
        ################################################################################
        for i in range(num_sample):
            # _, decision,_= ...
            # append to container
            pass
        # accuracy = ...

        if not 'accuracy' in locals():
            raise NotImplementedError("function `simulate_accuracy_vs_stoptime` incomplete")

        accuracy_list.append(accuracy)
        decisions_list.append(decision_list)

    return accuracy_list, decisions_list

################################################################################
## Un-comment the following code  after completing this exercise
################################################################################
# simulate_and_plot_accuracy_vs_stoptime(sigma, stop_time_list, num_sample)
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W2D3_DecisionMaking/solutions/W2D3_Tutorial1_Solution_a5a89cc2.py)

*Example output:*

**Suggestions**

* Play with difference values of  noise level `sigma` and observe how that affects the curve. What does that mean?    *Hint*: you can click `...`>`Form`>`Hide code` to hide the code section if the slider is too far from the figure

```python
#@title Interactive Demo: Accuracy vs. Stopping Time { run : "auto"}

sigma = 5.5  #@param {type:"slider", min:0.05, max:10.0, step:0.05}
num_sample = 200  # number of simulations to run for each stopping time
# list of stopping times to play with
stop_time_list = list(range(1,150,10))

try:
  simulate_and_plot_accuracy_vs_stoptime(sigma, stop_time_list, num_sample)
except NotImplementedError:
  print("Complete Exercise 2 first")
```

# Simulate DDM with fixed thresholds

Exercise 3
---

In this exercise, we will use thresholding as our stopping rule and observe the behavior of the DDM.

With thresholding stopping rule, we define a desired error rate and will continue making measurements until that error rate is reached. Experimental evidence suggested that evidence accumulation and thresholding stopping strategy happens at neuronal level (see [this article](https://www.annualreviews.org/doi/full/10.1146/annurev.neuro.29.051605.113038) for further reading).

* Complete the function `threshold_from_errorrate` to calculate the evidence threshold from desired error rate $\alpha$ as described in the formulas below. The evidence thresholds $th_L$ and $th_R$ for $p_L$ and $p_R$ are opposite of each other as shown below, so you can just return the absolute value.
$$
\begin{align}
 th_{L} &= \log \frac{\alpha}{1-\alpha} &= -th_{R} \\
 th_{R} &= \log \frac{1-\alpha}{\alpha} &= -th{_L}\\
 \end{align}
 $$

* Complete the function `simulate_SPRT_threshold` to simulate an SPRT with thresholding stopping rule given noise level and desired threshold

* Run repeated simulations for a given noise level and a desired error rate visualize the DDM traces using our provided code

```python
def threshold_from_errorrate(alpha):
    """
    Calculate log likelihood ratio threshold from desired error rate `alpha`

    Args:
        alpha (float): in (0,1), the desired error rate

    Return:
        threshold: corresponding evidence threshold
    """
    ################################################################################
    ## Insert your code here to:
    ##      * calculate the evidence threshold from desired error rate
    ################################################################################
    raise NotImplementedError("function `threshold_from_errorrate` incomplete")

def simulate_SPRT_threshold(sigma, threshold , true_dist=1, batch_size=100):
    """
    Simulate a Sequential Probability Ratio Test with thresholding stopping rule.
    Two observation models are 1D Gaussian distributions N(1,sigma^2) and N(-1,sigma^2).

    Args:
        sigma (float): standard deviation
        threshold (float): desired log likelihood ratio threshold to achieve before making decision
        batch_size (int): generate and process data in batches instead of serially for speed. The size of each batch

    Returns:
        evidence_history (numpy vector): the history of cumulated evidence given generated data
        decision (int): 1 for pR, 0 for pL
        data (numpy vector): the generated sequences of data in this trial
    """
    muL = -1.0
    muR = 1.0

    pL = stats.norm(muL, sigma)
    pR = stats.norm(muR, sigma)

    has_enough_data = False

    data = np.zeros(0)
    evidence_history = np.zeros(0)
    current_evidence = 0.0
    while not has_enough_data:
        # generate a batch of data
        if true_dist == 1:
            data_batch = pR.rvs(size=batch_size)
        else:
            data_batch = pL.rvs(size=batch_size)

        # individual log likelihood ratios
        ll_batch = log_likelihood_ratio(data_batch, pL, pR)
        # cumulated evidence for this batch
        evidence_history_batch = np.cumsum(ll_batch) + current_evidence
        # update the collection of all data
        data = np.concatenate([ data, data_batch])
        # update the collection of all cumulated evidence history
        evidence_history = np.concatenate([evidence_history, evidence_history_batch])
        current_evidence = evidence_history[-1]

        # check if we've got enough data
        if np.abs(current_evidence) > threshold:
            has_enough_data = True

    # find earliest time to cross threshold
    ################################################################################
    ## Insert your code here to:
    ##      * Find the index of first data that crosses one of the thresholds
    ##      and assign that to `id_crossing`
    ##      Hint: use `np.argmax` on a vector of Bool numbers will give you the index of the
    ##          first non-zero value
    ################################################################################

    if id_crossing not in locals():
        raise NotImplementedError("function `simulate_SPRT_threshold` incomplete")

    # discard redundant data because of  batching
    if len(data)-1 > id_crossing:
        data = data[:id_crossing + 1]
        evidence_history = evidence_history[:id_crossing+1]

    # Make decision
    if evidence_history[-1] >0:
        decision = 1
    elif evidence_history[-1] <0:
        decision = 0
    else:
        decision = np.random.randint(2)

    return evidence_history, decision, data

sigma = 2.8
num_sample = 10
log10_alpha = -6.5 #log10(alpha)
alpha = np.power(10.0,log10_alpha)

# Uncomment the line below to run the simulation and visualize the results
# simulate_and_plot_SPRT_fixedthreshold(sigma, num_sample, alpha)
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W2D3_DecisionMaking/solutions/W2D3_Tutorial1_Solution_490660b2.py)

*Example output:*

**Suggestion**

* Play with difference values of `alpha` and `sigma` and observe how that affects the dynamics of Drift-Diffusion Model. *Hint*: you can click `...`>`Form`>`Hide code` to hide the code section if the slider is too far from the figure

```python
#@title Interactive Demo: Noise levels and thresholds { run : "auto"}
sigma = 5  #@param {type:"slider", min:0.05, max:5.0, step:0.05}
num_sample = 10
log10_alpha = -5.9 #@param {type:"slider", min:-8, max:-0.1, step:0.1}
alpha = np.power(10.0, log10_alpha)
threshold = threshold_from_errorrate(alpha)
# construct a simulation model
num_sample = 10

################################################################################
## Un-comment the following code block after completing the exercise above
################################################################################
try:
  simulate_and_plot_SPRT_fixedthreshold(sigma, num_sample, alpha)
except NotImplementedError:
  print("Finish exercise 3 first")
```

# Speed/Accuracy Tradeoff

---
### EXERCISE 4:

The faster you make a decision (by allowing higher error rate $\alpha$), the lower your accuracy will be. This is known as the speed/accuracy tradeoff. To illustrate the speed/accuracy under thresholding stopping rule, let's run some simulations under differenct thresholds and look at how average decision speed changes with average decision accuracy.

* Complete the function `simulate_accuracy_vs_speed` to simulate and compute average accuracies vs. average decision speeds for a list of error rates.

* We've set up a list of error rates. Run repeated simulations and collect average accuracy with average speed for each error rate in this list, and use our provided code to visualize the speed/accuracy tradeoff. You should see a negative correlation between speed and accuracy.

```python
sigma = 3.95
num_repeats = 500  # number of simulations to run for each error rate
alpha_list = np.power(10,list(range(-5,0)) + np.linspace(-0.9,-0.1,9).tolist())   # list of error rates

threshold_list = list(map(threshold_from_errorrate, alpha_list))

def simulate_accuracy_vs_speed(sigma, threshold_list, num_sample=100):
    """
    Calculate the average decision accuracy vs. average decision speed by running repeated SPRT simulations
    with thresholding stopping rule for each threshold

    Args:
        sigma (float): standard deviation for observation model
        threshold_list (list-like object): a list of evidence thresholds to run over
        num_sample (int): number of simulations to run per stopping time

    Returns:
        accuracy_list: a list of average accuracies corresponding to input `stop_time_list`
        decision_speed_list: a list of average decision speeds
    """
    decision_speed_list = []  # container for average decision speed for each alpha
    accuracy_list = [] # container for decision accuracy for each alpha
    for threshold in threshold_list:
        decision_time_list = [] # container for decision time for each simulation
        decision_list = [] # container for decision for every simulation
        for i in range(num_repeats):
            # run simulation and get decision of current simulation
            ################################################################################
            ## Insert your code here to:
            ##      * run a simulation of SPRT with thresholding using function `simulate_SPRT_threshold`
            ##      * calculate decision time(number of data seen) from return values and assign that to `decision_time`
            ################################################################################

            if 'decision_time' not in locals():
                raise NotImplementedError("function `simulate_accuracy_vs_speed` incomplete")
            decision_list.append(decision)
            decision_time_list.append(decision_time)
        ################################################################################
        ## Insert your code here to:
        ##      * Calculate decision speed given a list of decision times
        ##      and assign that to variable `decision_speed`
        ################################################################################

        if 'decision_speed' not in locals():
            raise NotImplementedError("function `simulate_accuracy_vs_speed` incomplete")
        decision_accuracy = sum(decision_list) / len(decision_list)
        decision_speed_list.append(decision_speed)
        accuracy_list.append(decision_accuracy)

    return accuracy_list, decision_speed_list

################################################################################
## Un-comment the following code block after completing this exercise
################################################################################
# simulate_and_plot_speed_vs_accuracy(sigma, threshold_list, num_sample)
```

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W2D3_DecisionMaking/solutions/W2D3_Tutorial1_Solution_be044152.py)

*Example output:*

**Suggestions**

* Play with difference values of  noise level `sigma` and observe how that affects the speed/accuracy tradeoff. *Hint*: you can click `...`>`Form`>`Hide code` to hide the code section if the slider is too far from the figure

```python
#@title Interactive Cell { run : "auto" }
sigma = 3.75  #@param {type:"slider", min:0.05, max:5.0, step:0.05}
num_repeats = 500  # number of simulations to run for each error rate
alpha_list = np.power(10,list(range(-5,0)) + np.linspace(-0.9,-0.1,9).tolist())   # list of error rates

################################################################################
## Un-comment the following code block after completing the exercise above
################################################################################
try:
  threshold_list = list(map(threshold_from_errorrate, alpha_list))
  accuracy_list, decision_speed_list = simulate_accuracy_vs_speed(sigma, threshold_list, num_sample=num_sample)
# # Plotting

  fig, ax = plt.subplots()
  ax.plot(decision_speed_list, accuracy_list, linestyle="--", marker="o")
  ax.plot([0.0, 1.0], [0.5, 0.5], c='red')  # plot baseline for random choice
  ax.set_xlabel("Average Decision Speed")
  ax.set_ylabel('Average Accuracy')
  ax.set_title("Speed/Accuracy Tradeoff")
  ax.set_ylim(bottom=0.45)
  plt.show()

except NotImplementedError:
  print("Finish the functions above first.")

```

# Summary

Good job! By simulating Drift Diffusion Models to perform decision making, you have learnt how to

1. Calculate individual sample evidence as the log likelihood ratio of two candidate models, accumulate evidence from new data points, and make decision based on current evidence in `Exercise 1`
2. Run repeated simulations to get an estimate of decision accuraries in `Exercise 2`
3. Implement the thresholding stopping rule where we can control our error rate by taking adequate amounts of data, and calculate the evidence threshold from desired error rate in `Exercise 3`
4. Explore and gain intuition about the speed/accuracy tradeoff for perceptual decision making in `Exercise 4`


```python
import sys
import os
import PIL
import numpy as np
from numpy.linalg import norm
from math import *
from scipy import ndimage
from scipy import misc
import skimage

from sympy import *

%matplotlib inline
import matplotlib.pyplot as plt

```

```python
def estimateModelAttempt1():
    r,g = symbols('r g')
    a0,r0,g0 = symbols('a0 r0 g0')
    a1,r1,g1 = symbols('a1 r1 g1')

    print solveset(Eq(r0, a0*r + (1-a0)*255), r)

    # r as a function of a0, r0
    # Eq(r, rFromA0R0)
    rFromA0R0 = (255*a0 + r0 - 255)/a0

    print Eq(r1, a1*r + (1-a1)*255).subs(r, rFromA0R0)
    # r1 as a function of a0, r0, a1
    # Eq(r1, r1FromA0R0A1)
    r1FromA0R0A1 = -255*a1 + 255 + a1*(255*a0 + r0 - 255)/a0

    # Similarly, g1 as a function of a0, g0, a1
    # Eq(g1, g1FromA0R0A1)
    g1FromA0G0A1 = -255*a1 + 255 + a1*(255*a0 + g0 - 255)/a0

    print solveset(Eq(r1, r1FromA0R0A1), a1)
    a1FromA0R0R1 = a0*(r1 - 255)/(r0 - 255)

    g1FromA0G0R0R1 = simplify(g1FromA0G0A1.subs(a1, a1FromA0R0R1))
    print g1FromA0G0R0R1
    # a0 disappears!
    g1FromA0G0R0R1 = (g0*r1 - 255*g0 + 255*r0 - 255*r1)/(r0 - 255)

    print '==='

estimateModelAttempt1()

```

     {(255*a0 + r0 - 255)/a0}
    Eq(r1, -255*a1 + 255 + a1*(255*a0 + r0 - 255)/a0)
    {a0*(r1 - 255)/(r0 - 255)}
    (g0*r1 - 255*g0 + 255*r0 - 255*r1)/(r0 - 255)
    ===

```python
def testModel1():
    r0,g0,b0 = (157.,212.,244.)
    r1,g1,b1 = (218.,239.,251.)
    print 'g1FromR = ', (g0*r1 - 255*g0 + 255*r0 - 255*r1)/(r0 - 255)
    print 'g1FromB = ', (g0*b1 - 255*g0 + 255*b0 - 255*b1)/(b0 - 255)
    print 'b1FromR = ', (b0*r1 - 255*b0 + 255*r0 - 255*r1)/(r0 - 255)
    print 'b1FromG = ', (b0*g1 - 255*b0 + 255*g0 - 255*g1)/(g0 - 255)
    print 'r1FromG = ', (r0*g1 - 255*r0 + 255*g0 - 255*g1)/(g0 - 255)
    print 'r1FromB = ', (r0*b1 - 255*r0 + 255*b0 - 255*b1)/(b0 - 255)

testModel1()
```

    g1FromR =  238.765306122
    g1FromB =  239.363636364
    b1FromR =  250.846938776
    b1FromG =  250.906976744
    r1FromG =  218.534883721
    r1FromB =  219.363636364

```python
def testModel1_2():
    r0,g0,b0 = (127.,255.,255.)
    r1,g1,b1 = (255.,255.,255.)
    print 'g1FromR = ', (g0*r1 - 255*g0 + 255*r0 - 255*r1)/(r0 - 255)
    print 'b1FromR = ', (b0*r1 - 255*b0 + 255*r0 - 255*r1)/(r0 - 255)
    print 'b1FromG = ', (b0*g1 - 255*b0 + 255*g0 - 255*g1)/(g0 - 255)
    print 'r1FromG = ', (r0*g1 - 255*r0 + 255*g0 - 255*g1)/(g0 - 255)

testModel1_2()
```

```python
def estimateSimplerModel():
    r,g = symbols('r g')
    a0,r0,g0 = symbols('a0 r0 g0')
    a1,r1,g1 = symbols('a1 r1 g1')

    r0def = a0*r + (1-a0)*255
    g0def = a0*g + (1-a0)*255

    # Does not depend on a0, getting (r - 255)/(g - 255)
    print simplify((r0def-255)/(g0def-255))

    # Conclusion 1: (r0-255)/(g0-255) = (r1-255)/(g1-255) if they come from the same color
    # If the background was black, we'd get r0/g0 = r1/g1

    # Conclusion 2: we can't recover the true color and alpha values if the background
    # is grayscale.

estimateSimplerModel()

```

    (r - 255)/(g - 255)

```python
def testSimpleModel():
    r0,g0,b0 = (157.,212.,244.)
    r1,g1,b1 = (218.,239.,251.)

    # Should be the same
    print (r0-255.)/(g0-255.)
    print (r1-255.)/(g1-255.)

    # Should be the same
    print (r0-255.)/(b0-255.)
    print (r1-255.)/(b1-255.)

testSimpleModel()
```

    2.27906976744
    2.3125
    8.90909090909
    9.25

```python
def testColor0():
    ref = np.array([81.,179.,235.])
    v0 = np.array([157.,212.,244.])
    v1 = np.array([218.,239.,251.])
    v2 = np.array([110.,191.,238.])

    alphas0 = (v0 - 255) / (ref - 255)
    alphas1 = (v1 - 255) / (ref - 255)
    alphas2 = (v2 - 255) / (ref - 255)
    print 'alphas0', alphas0
    print 'alphas1', alphas1
    print 'alphas2', alphas2

testColor0()

```

    alphas0 [ 0.56321839  0.56578947  0.55      ]
    alphas1 [ 0.21264368  0.21052632  0.2       ]
    alphas2 [ 0.83333333  0.84210526  0.85      ]

```python
def linearToSrgb(L):
    if L <= 0.0031308:
        return L * 12.92 * 255.0
    else:
        return 255.0 * ((1.055 * L**0.41667) - 0.055)
linearToSrgb = np.vectorize(linearToSrgb)

def sRgbToLinearRgb(S):
    S = S/255.0
    if (S <= 0.04045):
        return S/12.92
    else:
        return ((S+0.055)/1.055)**2.4
sRgbToLinearRgb = np.vectorize(sRgbToLinearRgb)

def testSrgbToLinear():
    srgb0 = np.array([157.,212.,244.])
    linear0 = sRgbToLinearRgb(srgb0)
    srgba1 = linearToSrgb(linear0)

testSrgbToLinear()

def testWithGamma1():
    ref = np.array([241.,230.,42.])
    v0 = np.array([243.,232.,66.])
    v1 = np.array([245.,239.,127.])

    ref = v0

    #ref = sRgbToLinearRgb(ref)
    #v0 = sRgbToLinearRgb(v0)
    #v1 = sRgbToLinearRgb(v1)

    alphas0 = (v0 - 255) / (ref - 255)
    alphas1 = (v1 - 255) / (ref - 255)
    print 'alphas0', alphas0
    print 'alphas1', alphas1

    r,g,b = ref
    r0,g0,b0 = v0
    r1,g1,b1 = v1
    print 'g1FromR = ', (g*r1 - 255*g + 255*r - 255*r1)/(r - 255)
    print 'g1FromB = ', (g*b1 - 255*g + 255*b - 255*b1)/(b - 255)
    print 'b1FromR = ', (b*r1 - 255*b + 255*r - 255*r1)/(r - 255)
    print 'b1FromG = ', (b*g1 - 255*b + 255*g - 255*g1)/(g - 255)
    print 'r1FromG = ', (r*g1 - 255*r + 255*g - 255*g1)/(g - 255)
    print 'r1FromB = ', (r*b1 - 255*r + 255*b - 255*b1)/(b - 255)

testWithGamma1()

```

    alphas0 [ 1.  1.  1.]
    alphas1 [ 0.83333333  0.69565217  0.67724868]
    g1FromR =  235.833333333
    g1FromB =  239.423280423
    b1FromR =  97.5
    b1FromG =  123.52173913
    r1FromG =  246.652173913
    r1FromB =  246.873015873

```python
def testWithGamma2():
    ref = np.array([81.,179.,235.])
    v0 = np.array([126.,198.,241.])
    v1 = np.array([84.,180.,235.])

    ref = sRgbToLinearRgb(ref)
    v0 = sRgbToLinearRgb(v0)
    v1 = sRgbToLinearRgb(v1)

    alphas0 = (v0 - 255) / (ref - 255)
    alphas1 = (v1 - 255) / (ref - 255)
    print 'alphas0', alphas0
    print 'alphas1', alphas1

    r,g,b = ref
    r0,g0,b0 = v0
    r1,g1,b1 = v1
    print 'g1FromR = ', (g*r1 - 255*g + 255*r - 255*r1)/(r - 255)
    print 'g1FromB = ', (g*b1 - 255*g + 255*b - 255*b1)/(b - 255)
    print 'b1FromR = ', (b*r1 - 255*b + 255*r - 255*r1)/(r - 255)
    print 'b1FromG = ', (b*g1 - 255*b + 255*g - 255*g1)/(g - 255)
    print 'r1FromG = ', (r*g1 - 255*r + 255*g - 255*g1)/(g - 255)
    print 'r1FromB = ', (r*b1 - 255*r + 255*b - 255*b1)/(b - 255)

testWithGamma2()

```

    alphas0 [ 0.99950433  0.99955244  0.9998078 ]
    alphas1 [ 0.999975   0.9999779  1.       ]
    g1FromR =  0.457149449509
    g1FromB =  0.450785782838
    b1FromR =  0.837124043939
    b1FromG =  0.836386719912
    r1FromG =  0.0879160909596
    r1FromB =  0.0822827071298

```python
def testPlotImage():
    # image = ndimage.io.imread('../DaltonLensTests/gnuplotLt5CairoCropped.png')
    # image = ndimage.io.imread('../DaltonLensTests/gnuplotLt5Cropped.png')
    # image = ndimage.io.imread('../DaltonLensTests/gnuplotLt5ScreenCaptureCropped.png')
    # image = ndimage.io.imread('../DaltonLensTests/ComplexPlotCropped2.png')
    image = ndimage.io.imread('../DaltonLensTests/RandomPlotsCropped.png')
    # image = ndimage.io.imread('../DaltonLensTests/XcodeBackground.png')
    float_image = skimage.util.dtype.img_as_float(image[:,:,0:3])
    npix = float_image.shape[0]*float_image.shape[1]
    float_image = float_image.reshape((npix,3))
    print float_image.shape
    plt.plot (255.0 - float_image[:,0]*255., 255.0 - float_image[:,1]*255., '.')
    plt.axis([0, 255, 0, 255])
    plt.figure()

    plt.plot (255.0 - float_image[:,0]*255., 255.0 - float_image[:,2]*255., '.')
    plt.axis([0, 255, 0, 255])
    plt.figure()

    plt.plot (255.0 - float_image[:,1]*255., 255.0 - float_image[:,2]*255., '.')
    plt.axis([0, 255, 0, 255])
    plt.figure()

    # Algorithm IDEA:
    # compute points (R-255, G-255), (R-255, B-255), (G-255, B-255)
    # see if they are in the same line as the reference points (fit a line, distance to line < k)
    # count the number of discriminant points (max 3). Values very close to 255 for every channel
    # are not informative. Informative if min(R,G), min(R,B) or min(G,B) < e.g. 100
    # if compatible and informative, mark as definitely a match
    # if compatible and one neighbor is a match, accept it too

    # (b0*r1 - 255*b0 + 255*r0 - 255*r1)/(r0 - 255)
    # ratios = (float_image[:,0]*255. - 255.0001)/(float_image[:,2]*255. - 255.0001)
    # plt.plot (np.arange(0, npix, 1), ratios, '.')
    # plt.figure()

testPlotImage()

```

```python
from mpl_toolkits.mplot3d import Axes3D
def plotUncertainty():
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')

    R = np.arange(0.0, 254.0, 1.0)
    G = np.arange(0.0, 254.0, 1.0)

    R, G = np.meshgrid(R, G)
    gamma = ((256.0-R)/(255.0-G)) - ((255.0-R)/(256.0-G))
    ax.set_xlabel('R')
    ax.set_ylabel('G')
    ax.plot_surface(R, G, gamma, color='b')

plotUncertainty()
```

```python
def gammaOf(R,G):
    return ((256.0-R)/(255.0-G)) - ((255.0-R)/(256.0-G))
gammaOf_ufunc = np.frompyfunc(gammaOf, 2, 1)

def distFromOrigin(R,G):
    return (255.0-R)/(255.0-G)**2

def plotUncertaintyFromRatio():
    R = np.arange(1.0, 254.0, 1.0)
    G = np.arange(1.0, 254.0, 1.0)

    gamma = gammaOf_ufunc.outer(R,G).flatten()
    print np.shape(gamma)

    rOverGValues = np.frompyfunc(distFromOrigin, 2, 1).outer(R,G).flatten()
    print rOverGValues
    print gamma

    plt.axis('auto')
    plt.plot(rOverGValues.flatten(), gamma.flatten())
    plt.figure()

plotUncertaintyFromRatio()
```