# Neuromatch Academy: Week 3, Day 1, Tutorial 1
# Real Neurons: The Leaky Integrate-and-Fire (LIF) Neuron Model
__Content creators:__ Qinglong Gu, Songtin Li, John Murray, Richard Naud, Arvind Kumar

__Content reviewers:__  Maryam Vaziri-Pashkam, Ella Batty, Lorenzo Fontolan, Richard Gao, Matthew Krause, Spiros Chavlis, Michael Waskom

---
# Tutorial Objectives
This is Tutorial 1 of a series on implementing realistic neuron models. In this tutorial, we will build up a leaky integrate-and-fire (LIF) neuron model and study its dynamics in response to various types of inputs. In particular, we are going to write a few lines of code to:

- simulate the LIF neuron model

- drive the LIF neuron with external inputs, such as direct currents, Gaussian white noise, and Poisson spike trains, etc.

- study how different inputs affect the  LIF neuron's output (firing rate and spike time irregularity)

Here, we will especially emphasize identifying conditions (input statistics) under which a neuron can spike at low firing rates and in an irregular manner. The reason for focusing on this is that in most cases, neocortical neurons spike in an irregular manner.

---
# Setup

```python
# Imports
import numpy as np
import matplotlib.pyplot as plt
```

```python
# @title Figure Settings
import ipywidgets as widgets  # interactive display
%config InlineBackend.figure_format = 'retina'
# use NMA plot style
plt.style.use("https://raw.githubusercontent.com/NeuromatchAcademy/course-content/master/nma.mplstyle")
my_layout = widgets.Layout()
```

```python
# @title Helper functions

def plot_volt_trace(pars, v, sp):
  """
  Plot trajetory of membrane potential for a single neuron

  Expects:
  pars   : parameter dictionary
  v      : volt trajetory
  sp     : spike train

  Returns:
  figure of the membrane potential trajetory for a single neuron
  """

  V_th = pars['V_th']
  dt, range_t = pars['dt'], pars['range_t']
  if sp.size:
    sp_num = (sp / dt).astype(int) - 1
    v[sp_num] += 20  # draw nicer spikes

  plt.plot(pars['range_t'], v, 'b')
  plt.axhline(V_th, 0, 1, color='k', ls='--')
  plt.xlabel('Time (ms)')
  plt.ylabel('V (mV)')
  plt.legend(['Membrane\npotential', r'Threshold V$_{\mathrm{th}}$'],
             loc=[1.05, 0.75])
  plt.ylim([-80, -40])

def plot_GWN(pars, I_GWN):
  """
  Args:
    pars  : parameter dictionary
    I_GWN : Gaussian white noise input

  Returns:
    figure of the gaussian white noise input
  """

  plt.figure(figsize=(12, 4))
  plt.subplot(121)
  plt.plot(pars['range_t'][::3], I_GWN[::3], 'b')
  plt.xlabel('Time (ms)')
  plt.ylabel(r'$I_{GWN}$ (pA)')
  plt.subplot(122)
  plot_volt_trace(pars, v, sp)
  plt.tight_layout()

def my_hists(isi1, isi2, cv1, cv2, sigma1, sigma2):
  """
  Args:
    isi1 : vector with inter-spike intervals
    isi2 : vector with inter-spike intervals
    cv1  : coefficient of variation for isi1
    cv2  : coefficient of variation for isi2

  Returns:
    figure with two histograms, isi1, isi2

  """
  plt.figure(figsize=(11, 4))
  my_bins = np.linspace(10, 30, 20)
  plt.subplot(121)
  plt.hist(isi1, bins=my_bins, color='b', alpha=0.5)
  plt.xlabel('ISI (ms)')
  plt.ylabel('count')
  plt.title(r'$\sigma_{GWN}=$%.1f, CV$_{\mathrm{isi}}$=%.3f' % (sigma1, cv1))

  plt.subplot(122)
  plt.hist(isi2, bins=my_bins, color='b', alpha=0.5)
  plt.xlabel('ISI (ms)')
  plt.ylabel('count')
  plt.title(r'$\sigma_{GWN}=$%.1f, CV$_{\mathrm{isi}}$=%.3f' % (sigma2, cv2))
  plt.tight_layout()
  plt.show()

# the function plot the raster of the Poisson spike train
def my_raster_Poisson(range_t, spike_train, n):
  """
  Generates poisson trains

  Args:
    range_t     : time sequence
    spike_train : binary spike trains, with shape (N, Lt)
    n           : number of Poisson trains plot

  Returns:
    Raster plot of the spike train
  """

  # find the number of all the spike trains
  N = spike_train.shape[0]

  # n should smaller than N:
  if n > N:
    print('The number n exceeds the size of spike trains')
    print('The number n is set to be the size of spike trains')
    n = N

  # plot rater
  i = 0
  while i < n:
    if spike_train[i, :].sum() > 0.:
      t_sp = range_t[spike_train[i, :] > 0.5]  # spike times
      plt.plot(t_sp, i * np.ones(len(t_sp)), 'k|', ms=10, markeredgewidth=2)
    i += 1
  plt.xlim([range_t[0], range_t[-1]])
  plt.ylim([-0.5, n + 0.5])
  plt.xlabel('Time (ms)', fontsize=12)
  plt.ylabel('Neuron ID', fontsize=12)
```

---
# Section 1: The Leaky Integrate-and-Fire (LIF) model

```python
#@title Video 1: Reduced Neuron Models
from IPython.display import YouTubeVideo
video = YouTubeVideo(id='rSExvwCVRYg', width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
video
```

    Video available at https://youtube.com/watch?v=rSExvwCVRYg

## Implementation of an LIF neuron model

Now, it's your turn to implement one of the simplest mathematical model of a neuron: the leaky integrate-and-fire (LIF) model. The basic idea of LIF neuron was proposed in 1907 by Louis Édouard Lapicque, long before we understood the electrophysiology of a neuron (see a translation of [Lapicque's paper](https://pubmed.ncbi.nlm.nih.gov/17968583/) ). More details of the model can be found in the book [**Theoretical neuroscience**](http://www.gatsby.ucl.ac.uk/~dayan/book/) by Peter Dayan and Laurence F. Abbott.

The subthreshold membrane potential dynamics of a LIF neuron is described by

\begin{eqnarray}
C_m\frac{dV}{dt} = -g_L(V-E_L) + I,\quad (1)
\end{eqnarray}

where $C_m$ is the membrane capacitance, $V$ is the membrane potential, $g_L$ is the leak conductance ($g_L = 1/R$, the inverse of the leak resistance $R$ mentioned in previous tutorials), $E_L$ is the resting potential, and $I$ is the external input current.

Dividing both sides of the above equation by $g_L$ gives

\begin{align}
\tau_m\frac{dV}{dt} = -(V-E_L) + \frac{I}{g_L}\,,\quad (2)
\end{align}

where the $\tau_m$ is membrane time constant and is defined as $\tau_m=C_m/g_L$.

You might wonder why dividing capacitance by conductance gives units of time! Find out yourself why this is true.

Below, we will use Eqn.(2) to simulate LIF neuron dynamics.

If $I$ is sufficiently strong such that $V$ reaches a certain threshold value $V_{\rm th}$, $V$ is reset to a reset potential $V_{\rm reset}< V_{\rm th}$, and voltage is clamped to $V_{\rm reset}$ for $\tau_{\rm ref}$ ms, mimicking the refractoriness of the neuron during an action potential:

\begin{eqnarray}
\mathrm{if}\quad V(t_{\text{sp}})\geq V_{\rm th}&:& V(t)=V_{\rm reset} \text{  for } t\in(t_{\text{sp}}, t_{\text{sp}} + \tau_{\text{ref}}]
\end{eqnarray}
where $t_{\rm sp}$ is the spike time when $V(t)$ just exceeded $V_{\rm th}$.

(__Note__: in the lecture slides, $\theta$ corresponds to the threshold voltage $V_{th}$, and $\Delta$ corresponds to the refractory time $\tau_{\rm ref}$.)

Thus, the LIF model captures the facts that a neuron:
- performs spatial and temporal integration of synaptic inputs
- generates a spike when the voltage reaches a certain threshold
- goes refractory during the action potential
- has a leaky membrane

The LIF model assumes that the spatial and temporal integration of inputs is linear. Also, membrane potential dynamics close to the spike threshold are much slower in LIF neurons than in real neurons.

## Exercise 1: Python code to simulate the LIF neuron

We now write Python code to calculate Eqn. (2) and simulate the LIF neuron dynamics. We will use the Euler method, which you saw in the linear systems case last week,  to numerically integrate Eq 2.

The cell below initializes a dictionary that stores parameters of the LIF neuron model and the simulation scheme. You can use `pars=default_pars(T=simulation_time, dt=time_step)` to get the parameters (you can try to print the dictionary `pars`). Note that, `simulation_time` and `time_step` have the unit `ms`. In addition, you can add the value to a new parameter by `pars['New_param'] = value`.

```python
# @title

# @markdown Execute this code to initialize the default parameters

def default_pars(**kwargs):
  pars = {}

  # typical neuron parameters#
  pars['V_th'] = -55.     # spike threshold [mV]
  pars['V_reset'] = -75.  # reset potential [mV]
  pars['tau_m'] = 10.     # membrane time constant [ms]
  pars['g_L'] = 10.       # leak conductance [nS]
  pars['V_init'] = -75.   # initial potential [mV]
  pars['E_L'] = -75.      # leak reversal potential [mV]
  pars['tref'] = 2.       # refractory time (ms)

  # simulation parameters #
  pars['T'] = 400.  # Total duration of simulation [ms]
  pars['dt'] = .1   # Simulation time step [ms]

  # external parameters if any #
  for k in kwargs:
    pars[k] = kwargs[k]

  pars['range_t'] = np.arange(0, pars['T'], pars['dt'])  # Vector of discretized time points [ms]

  return pars

pars = default_pars()
```

```python
print(pars)
```

    {'V_th': -55.0, 'V_reset': -75.0, 'tau_m': 10.0, 'g_L': 10.0, 'V_init': -75.0, 'E_L': -75.0, 'tref': 2.0, 'T': 400.0, 'dt': 0.1, 'range_t': array([0.000e+00, 1.000e-01, 2.000e-01, ..., 3.997e+02, 3.998e+02,
           3.999e+02])}

The cell below defines the function to simulate the LIF neuron when receiving external current inputs. You can use `v, sp = run_LIF(pars, Iinj)` to get the membrane potential (`v`) and spike train (`sp`) given the dictionary `pars` and input current `Iinj`.

```python
def run_LIF(pars, Iinj, stop=False):
  """
  Simulate the LIF dynamics with external input current

  Args:
    pars       : parameter dictionary
    Iinj       : input current [pA]. The injected current here can be a value
                 or an array
    stop       : boolean. If True, use a current pulse

  Returns:
    rec_v      : membrane potential
    rec_sp     : spike times
  """

  # Set parameters
  V_th, V_reset = pars['V_th'], pars['V_reset']
  tau_m, g_L = pars['tau_m'], pars['g_L']
  V_init, E_L = pars['V_init'], pars['E_L']
  dt, range_t = pars['dt'], pars['range_t']
  Lt = range_t.size
  tref = pars['tref']

  # Initialize voltage and current
  v = np.zeros(Lt)
  v[0] = V_init
  Iinj = Iinj * np.ones(Lt)
  if stop:  # set end of current to 0 if current pulse
    Iinj[:int(len(Iinj) / 2) - 1000] = 0
    Iinj[int(len(Iinj) / 2) + 1000:] = 0
  tr = 0.  # the count for refractory duration

  # Simulate the LIF dynamics
  rec_spikes = []  # record spike times
  for it in range(Lt - 1):
    if tr > 0:  # check for refractoriness
      v[it] = V_reset
      tr = tr - 1
    elif v[it] >= V_th:  # reset voltage and record spike event
      rec_spikes.append(it)
      v[it] = V_reset
      tr = tref / dt

    ########################################################################
    ## TODO for students: compute the membrane potential v, spike train sp #
    # Fill out function and remove
    # raise NotImplementedError('Student Exercise: calculate the dv/dt and the update step!')
    ########################################################################
    # calculate the increment of the membrane potential
    dv = (dt / tau_m) * (E_L - v[it] + Iinj[it] / g_L)

    # update the membrane potential
    v[it+1] = v[it] + dv

  rec_spikes = np.array(rec_spikes) * dt

  return v, rec_spikes

pars = default_pars(T=500)
# Uncomment below to test your function
v, sp = run_LIF(pars, Iinj=100, stop=True)
plot_volt_trace(pars, v, sp)
plt.ylim([-80, -60])
plt.show()
```

```python
# to_remove solution
def run_LIF(pars, Iinj, stop=False):
  """
  Simulate the LIF dynamics with external input current

  Args:
    pars       : parameter dictionary
    Iinj       : input current [pA]. The injected current here can be a value
                 or an array
    stop       : boolean. If True, use a current pulse

  Returns:
    rec_v      : membrane potential
    rec_sp     : spike times
  """
  # Set parameters
  V_th, V_reset = pars['V_th'], pars['V_reset']
  tau_m, g_L = pars['tau_m'], pars['g_L']
  V_init, E_L = pars['V_init'], pars['E_L']
  dt, range_t = pars['dt'], pars['range_t']
  Lt = range_t.size
  tref = pars['tref']

  # Initialize voltage and current
  v = np.zeros(Lt)
  v[0] = V_init
  Iinj = Iinj * np.ones(Lt)
  if stop:  # set end of current to 0 if current pulse
    Iinj[:int(len(Iinj) / 2) - 1000] = 0
    Iinj[int(len(Iinj) / 2) + 1000:] = 0
  tr = 0.  # the count for refractory duration

  # Simulate the LIF dynamics
  rec_spikes = []  # record spike times
  for it in range(Lt - 1):
    if tr > 0:  # check for refractoriness
      v[it] = V_reset
      tr = tr - 1
    elif v[it] >= V_th:  # reset voltage and record spike event
      rec_spikes.append(it)
      v[it] = V_reset
      tr = tref / dt

    # calculate the increment of the membrane potential
    dv = (-(v[it] - E_L) + Iinj[it] / g_L) * (dt / tau_m)

    # update the membrane potential
    v[it+1] = v[it] + dv

  rec_spikes = np.array(rec_spikes) * dt

  return v, rec_spikes

pars = default_pars(T=500)
# Uncomment below to test your function
v, sp = run_LIF(pars, Iinj=100, stop=True)
with plt.xkcd():
  plot_volt_trace(pars, v, sp)
  plt.ylim([-80, -60])
  plt.show()
```

---
# Section 2: Response of an LIF model to different types of input currents
In the following section, we will learn how to inject direct current and white noise to study the response of an LIF neuron.

```python
#@title Video 2: Response of the LIF neuron to different inputs
from IPython.display import YouTubeVideo
video = YouTubeVideo(id='preNGdab7Kk', width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
video
```

    Video available at https://youtube.com/watch?v=preNGdab7Kk

## Section 2.1: Direct current (DC)

### LIF neuron driven by constant current
Investigate the voltage response of the LIF neuron when receiving a DC input of 300 pA by `run_LIF` function.

```python
pars = default_pars(T=100)  # get the parameters

# Run the model to obtain v and sp
v, sp = run_LIF(pars, Iinj=300)

plot_volt_trace(pars, v, sp)
plt.show()
```

In the plot above, you see the membrane potential of an LIF neuron. You may notice that the neuron generates a spike. But this is just a cosmetic spike only for illustration purposes. In an LIF neuron, we only need to keep track of times when the neuron hit the threshold so the postsynaptic neurons can be informed of the spike.

### Interactive Demo: Parameter exploration of DC input amplitude
Here's an interactive demo that shows how the LIF neuron behavior changes for DC input with different amplitudes.

How much DC is needed to reach the threshold (rheobase current)? How does the membrane time constant affect the frequency of the neuron?

```python
# @title
# @markdown Make sure you execute this cell to enable the widget!
my_layout.width = '450px'
@widgets.interact(
    I_dc=widgets.FloatSlider(50., min=0., max=300., step=10.,
                               layout=my_layout),
    tau_m=widgets.FloatSlider(10., min=2., max=20., step=2.,
                                layout=my_layout)
)

def diff_DC(I_dc=200., tau_m=10.):
  pars = default_pars(T=100.)
  pars['tau_m'] = tau_m
  v, sp = run_LIF(pars, Iinj=I_dc)
  plot_volt_trace(pars, v, sp)
  plt.show()
```

    interactive(children=(FloatSlider(value=50.0, description='I_dc', layout=Layout(width='450px'), max=300.0, ste…

```python
# to_remove explanation

"""
1. As we increase the current, we observe that at 210 pA we cross the threshold.

2. As we increase the membrane time constant (slower membrane), the firing rate
is decreased because the membrane needs more time to reach the threshold after
the reset.
""";
```

## Section 2.2: Gaussian white noise (GWN) current

Given the noisy nature of neuronal activity _in vivo_, neurons usually receive complex, time-varying inputs.

To mimic this, we will now investigate the neuronal response when the LIF neuron receives Gaussian white noise $\xi(t)$ with mean

\begin{eqnarray}
E[\xi(t)]=\mu=0,
\end{eqnarray}

and autocovariance
\begin{eqnarray}
E[\xi(t)\xi(t+\tau)]=\sigma_\xi^2 \delta(\tau)
\end{eqnarray}

Note that the GWN has zero mean, that is, it describes only the fluctuations of the input received by a neuron. We can thus modify our definition of GWN to have a nonzero mean value $\mu$ that equals the DC input, since this is the average input into the cell. The cell below defines the modified gaussian white noise currents with nonzero mean $\mu$.

```python
#@title
#@markdown Execute this cell to get function to generate GWN

def my_GWN(pars, mu, sig, myseed=False):
  """
  Function that generates Gaussian white noise input

  Args:
    pars       : parameter dictionary
    mu         : noise baseline (mean)
    sig        : noise amplitute (standard deviation)
    myseed     : random seed. int or boolean
                 the same seed will give the same
                 random number sequence

  Returns:
    I          : Gaussian white noise input
  """

  # Retrieve simulation parameters
  dt, range_t = pars['dt'], pars['range_t']
  Lt = range_t.size

  # set random seed
  # you can fix the seed of the random number generator so that the results
  # are reliable however, when you want to generate multiple realization
  # make sure that you change the seed for each new realization.
  if myseed:
      np.random.seed(seed=myseed)
  else:
      np.random.seed()

  # generate GWN
  # we divide here by 1000 to convert units to sec.
  I_gwn = mu + sig * np.random.randn(Lt) / np.sqrt(dt / 1000.)

  return I_gwn
```

### Exercise 2: LIF neuron driven by GWN

You can generate a noisy input with `my_GWN(pars, mu, sig, myseed=False)`. Here, $\mu=250$ and $\sigma=5$. Note that fixing the value of the random seed (e.g., `myseed=2020`) will allow you to obtain the same result every time you run this.

```python
pars = default_pars(T=100.)
sig_gwn = 5.
mu_gwn = 250.
#######################################################
## TODO for students: compute the noisy input I_GWN   #
# and then compute v, sp with inject I_GWN into LIF   #
#######################################################

# Calculate the GWN current
I_GWN = my_GWN(pars, mu=mu_gwn, sig=sig_gwn, myseed=2020)

# Run the model and calculate the v and the sp
v, sp = run_LIF(pars, Iinj = I_GWN)

# Uncomment these lines after obtaining v, sp
plot_GWN(pars, I_GWN)
plt.show()
```

```python
# to_remove solution
pars = default_pars(T=100.)
sig_gwn = 5.
mu_gwn = 250.

# Calculate the GWN current
I_GWN = my_GWN(pars, mu=mu_gwn, sig=sig_gwn, myseed=2020)

# Run the model and calculate the v and the sp
v, sp = run_LIF(pars, Iinj=I_GWN)

with plt.xkcd():
  plot_GWN(pars, I_GWN)
  plt.show()
```

### Interactive Demo: LIF neuron Explorer for noisy input

The mean of the GWN is the amplitude of DC. Indeed, when $\sigma = 0$, GWN is just a DC.

So the question arises how does $\sigma$ of the GWN affect the spiking behavior of the neuron. For instance we may want to know
- how does the minimum input (i.e. $\mu$) needed to make a neuron spike change with increase in $\sigma$
- how does the spike regularity change with increase in $\sigma$

To get an intuition about these questions you can use the following interactive demo that shows how the LIF neuron behavior changes for noisy input with different amplitudes (the mean $\mu$) and fluctuation sizes ($\sigma$).

```python
# @title

# @markdown Make sure you execute this cell to enable the widget!

my_layout.width = '450px'
@widgets.interact(
    mu_gwn=widgets.FloatSlider(200., min=100., max=300., step=5.,
                               layout=my_layout),
    sig_gwn=widgets.FloatSlider(2.5, min=0., max=5., step=.5,
                                layout=my_layout)
)

def diff_GWN_to_LIF(mu_gwn, sig_gwn):
  pars = default_pars(T=100.)
  I_GWN = my_GWN(pars, mu=mu_gwn, sig=sig_gwn)
  v, sp = run_LIF(pars, Iinj=I_GWN)
  plt.figure(figsize=(12, 4))
  plt.subplot(121)
  plt.plot(pars['range_t'][::3], I_GWN[::3], 'b')
  plt.xlabel('Time (ms)')
  plt.ylabel(r'$I_{GWN}$ (pA)')
  plt.subplot(122)
  plot_volt_trace(pars, v, sp)
  plt.tight_layout()
  plt.show()
```

    interactive(children=(FloatSlider(value=200.0, description='mu_gwn', layout=Layout(width='450px'), max=300.0, …

```python
# to_remove explanation

"""
If we have bigger current fluctuations (increased sigma), the minimum input needed
to make a neuron spike is smaller as the fluctuations can help push the voltage above
threshold.

The standard deviation (or size of current fluctuations) dictates the level of
irregularity of the spikes; the higher the sigma the more irregular the observed
spikes.

""";
```

## Think!
- As we increase the input average ($\mu$) or the input fluctuation ($\sigma$), the spike count changes. How much can we increase the spike count, and what might be the relationship between GWN mean/std or DC value and spike count?

- We have seen above that when we inject DC, the neuron spikes in a regular manner (clock like), and this regularity is reduced when GWN is injected. The question is, how irregular can we make the neurons spiking by changing the parameters of the GWN?

---
# Section 3: Firing rate and spike time irregularity
When we plot the output firing rate as a function of GWN mean or DC value, it is called the input-output transfer function of the neuron (so simply F-I curve).

Spike regularity can be quantified as the **coefficient of variance (CV) of the inter-spike-interval (ISI)**:
\begin{align}
\text{CV}_{\text{ISI}} = \frac{std(\text{ISI})}{mean(\text{ISI})}
\end{align}

A Poisson train is an example of high irregularity, in which $\textbf{CV}_{\textbf{ISI}} \textbf{= 1}$. And for a clocklike (regular) process we have $\textbf{CV}_{\textbf{ISI}} \textbf{= 0}$ because of **std(ISI)=0**.

## Interactive Demo: F-I Explorer for different `sig_gwn`

How does the F-I curve of the LIF neuron change as we increase the $\sigma$ of the GWN? We can already expect that the F-I curve will be stochastic and the results will vary from one trial to another. But will there be any other change compared to the F-I curved measured using DC?

Here's an interactive demo that shows how the F-I curve of a LIF neuron changes for different levels of fluctuation $\sigma$.

```python
# @title

# @markdown Make sure you execute this cell to enable the widget!

my_layout.width = '450px'
@widgets.interact(
    sig_gwn=widgets.FloatSlider(3.0, min=0., max=6., step=0.5,
                                layout=my_layout)
)

def diff_std_affect_fI(sig_gwn):
  pars = default_pars(T=1000.)
  I_mean = np.arange(100., 400., 10.)
  spk_count = np.zeros(len(I_mean))
  spk_count_dc = np.zeros(len(I_mean))

  for idx in range(len(I_mean)):
      I_GWN = my_GWN(pars, mu=I_mean[idx], sig=sig_gwn, myseed=2020)
      v, rec_spikes = run_LIF(pars, Iinj=I_GWN)
      v_dc, rec_sp_dc = run_LIF(pars, Iinj=I_mean[idx])
      spk_count[idx] = len(rec_spikes)
      spk_count_dc[idx] = len(rec_sp_dc)

  # Plot the F-I curve i.e. Output firing rate as a function of input mean.
  plt.figure()
  plt.plot(I_mean, spk_count, 'k',
           label=r'$\sigma_{\mathrm{GWN}}=%.2f$' % sig_gwn)
  plt.plot(I_mean, spk_count_dc, 'k--', alpha=0.5, lw=4, dashes=(2, 2),
           label='DC input')
  plt.ylabel('Spike count')
  plt.xlabel('Average injected current (pA)')
  plt.legend(loc='best')
  plt.show()
```

    interactive(children=(FloatSlider(value=3.0, description='sig_gwn', layout=Layout(width='450px'), max=6.0, ste…

```python
# to_remove explanation

"""
Discussion: If we use a DC input, the F-I curve is deterministic, and we can
found its shape by solving the membrane equation of the neuron. If we have GWN,
as we increase the sigma, the F-I curve has a more linear shape, and the neuron
reaches its threshold using less average injected current.
"""
```

    '\nDiscussion: If we use a DC input, the F-I curve is deterministic, and we can \nfound its shape by solving the membrane equation of the neuron. If we have GWN, \nas we increase the sigma, the F-I curve has a more linear shape, and the neuron \nreaches its threshold using less average injected current.\n'

### Exercise 3: Compute $CV_{ISI}$ values

As shown above, the F-I curve becomes smoother while increasing the amplitude of the fluctuation ($\sigma$). In addition, the fluctuation can also change the irregularity of the spikes. Let's investigate the effect of $\mu=250$ with $\sigma=0.5$ vs $\sigma=3$.

Fill in the code below to compute ISI, then plot the histogram of the ISI and compute the $CV_{ISI}$. Note that, you can use `np.diff` to calculate ISI.

```python
def isi_cv_LIF(spike_times):
  """
  Calculates the inter-spike intervals (isi) and
  the coefficient of variation (cv) for a given spike_train

  Args:
    spike_times : (n, ) vector with the spike times (ndarray)

  Returns:
    isi         : (n-1,) vector with the inter-spike intervals (ms)
    cv          : coefficient of variation of isi (float)

  """
  ########################################################################
  ## TODO for students: compute the membrane potential v, spike train sp #
  # Fill out function and remove
  # raise NotImplementedError('Student Exercise: calculate the isi and the cv!')
  ########################################################################
  if len(spike_times) >= 2:
    # Compute isi
    isi = np.diff(spike_times)
    # Compute cv
    cv = np.std(isi) / np.mean(isi)
  else:
    isi = np.nan
    cv = np.nan

  return isi, cv

pars = default_pars(T=1000.)
mu_gwn = 250
sig_gwn1 = 0.5
sig_gwn2 = 3.0

I_GWN1 = my_GWN(pars, mu=mu_gwn, sig=sig_gwn1, myseed=2020)
_, sp1 = run_LIF(pars, Iinj=I_GWN1)

I_GWN2 = my_GWN(pars, mu=mu_gwn, sig=sig_gwn2, myseed=2020)
_, sp2 = run_LIF(pars, Iinj=I_GWN2)

# Uncomment to check your function
isi1, cv1 = isi_cv_LIF(sp1)
isi2, cv2 = isi_cv_LIF(sp2)

my_hists(isi1, isi2, cv1, cv2, sig_gwn1, sig_gwn2)
```

```python
# to_remove solution
def isi_cv_LIF(spike_times):
  """
  Calculates the inter-spike intervals (isi) and
  the coefficient of variation (cv) for a given spike_train

  Args:
    spike_times : (n, ) vector with the spike times (ndarray)

  Returns:
    isi         : (n-1,) vector with the inter-spike intervals (ms)
    cv          : coefficient of variation of isi (float)

  """

  if len(spike_times) >= 2:
    # Compute isi
    isi = np.diff(spike_times)
    # Compute cv
    cv = isi.std()/isi.mean()
  else:
    isi = np.nan
    cv = np.nan

  return isi, cv

pars = default_pars(T=1000.)
mu_gwn = 250
sig_gwn1 = 0.5
sig_gwn2 = 3.0

I_GWN1 = my_GWN(pars, mu=mu_gwn, sig=sig_gwn1, myseed=2020)
_, sp1 = run_LIF(pars, Iinj=I_GWN1)

I_GWN2 = my_GWN(pars, mu=mu_gwn, sig=sig_gwn2, myseed=2020)
_, sp2 = run_LIF(pars, Iinj=I_GWN2)

# Uncomment to check your function
isi1, cv1 = isi_cv_LIF(sp1)
isi2, cv2 = isi_cv_LIF(sp2)

with plt.xkcd():
  my_hists(isi1, isi2, cv1, cv2, sig_gwn1, sig_gwn2)
```

## Interactive Demo: Spike irregularity explorer for different `sig_gwn`

In the above illustration, we see that the CV of inter-spike-interval (ISI) distribution depends on $\sigma$ of GWN. What about the mean of GWN, should that also affect the CV$_{\rm ISI}$? If yes, how? Does the efficacy of $\sigma$ in increasing the CV$_{\rm ISI}$ depend on $\mu$?

In the following interactive demo, you will examine how different levels of fluctuation $\sigma$ affect the CVs for different average injected currents ($\mu$).

```python
#@title

#@markdown Make sure you execute this cell to enable the widget!

my_layout.width = '450px'
@widgets.interact(
    sig_gwn=widgets.FloatSlider(0.0, min=0., max=10.,
                                step=0.5, layout=my_layout)
)

def diff_std_affect_fI(sig_gwn):
  pars = default_pars(T=1000.)
  I_mean = np.arange(100., 400., 20)
  spk_count = np.zeros(len(I_mean))
  cv_isi = np.empty(len(I_mean))

  for idx in range(len(I_mean)):
    I_GWN = my_GWN(pars, mu=I_mean[idx], sig=sig_gwn)
    v, rec_spikes = run_LIF(pars, Iinj=I_GWN)
    spk_count[idx] = len(rec_spikes)
    if len(rec_spikes) > 3:
      isi = np.diff(rec_spikes)
      cv_isi[idx] = np.std(isi) / np.mean(isi)

  # Plot the F-I curve i.e. Output firing rate as a function of input mean.
  plt.figure()
  plt.plot(I_mean[spk_count > 5], cv_isi[spk_count > 5], 'bo', alpha=0.5)
  plt.xlabel('Average injected current (pA)')
  plt.ylabel(r'Spike irregularity ($\mathrm{CV}_\mathrm{ISI}$)')
  plt.ylim(-0.1, 1.5)
  plt.grid(True)
  plt.show()
```

    interactive(children=(FloatSlider(value=0.0, description='sig_gwn', layout=Layout(width='450px'), max=10.0, st…

### Try to answer the following:
- Does the standard deviation of the injected current affect the F-I curve in any qualitative manner?
- Why does increasing the mean of GWN reduce the $CV_{ISI}$?
- If you plot spike count (or rate) vs. $CV_{ISI}$, should there be a relationship between the two? Try out yourself.

```python
# to_remove explanation

"""
1. Yes, it does. With DC input the F-I curve has a strong non-linearity but when
a neuron is driven with GWN, as we increase the $\sigma$ the non-linearity is
smoothened out. Essentially, in this case noise is acting to suppress the
non-linearities and render a neuron as a linear system.

2. (here is a short answer) When we increase the mean of the GWN, at some point
effective input mean is above the spike threshold and then the neuron operates
in the so called mean-driven regime -- as the input is so high all the neuron
is does is charge up to the spike threshold and reset. This essentially gives
almost regular spiking.

3. In an LIF, high firing rates are achieved for high GWN mean. Higher the mean,
higher the firing rate and lower the CV_ISI. So you will expect that as firing rate
increases, spike irregularity decreases. This is because of the spike threshold.
FOr a Poisson process there is no relationship between spike rate and spike
irregularity.
""";
```

---
# Section 4: Generation of Poisson type spike trains

*In the next tutorials, we will often use Poisson type spike train to explore properties of neurons and synapses. Therefore, it is good to know how to generate Poisson type spike trains.*

Mathematically, a spike train is a Point Process. One of the simplest models of a sequence of presynaptic pulse inputs is the Poisson process. We know that given temporal integration and refractoriness, neurons cannot behave as a Poisson Process, and Gamma Process gives a better approximation (*find out what might be the difference in the two processes*).
Here, however, we will assume that the incoming spikes are following Poisson statistics. A question arises about how to simulate a Poisson process. The generation of the Poisson process can be realized by at least two following ways:

- By definition, for a Poisson process with rate $\lambda$, the probability of finding one event in the time window with a sufficiently small length $\Delta t$ is $P(N = 1) = \lambda \Delta t$. Therefore, in each time window, we generate a uniformly distributed random variable $r \in [0,1]$ and generate a Poisson event when $r <\lambda \Delta t$. This method allows us to generate Poisson distributed spikes in an online manner.

- The interval $t_{k+1}-t_{k}$ between two Poisson events with rate $\lambda$ follows the exponential distribution, i.e., $P(t_{k+1}-t_{k}<t) = 1 - e^{\lambda t}$. Therefore, we only need to generate a set of exponentially distributed variables $\{s_k\}$ to obtain the timing of Poisson events $t_{k+1}=t_{k}+s_{k}$. In this method, we need to generate all future spikes at once.

Below, we use the first method in a function `Poisson_generator`, which takes arguments `(pars, rate, n, myseed)`.

```python
# @title
# @markdown Execute this cell to get a Poisson_generator function

def Poisson_generator(pars, rate, n, myseed=False):
  """
  Generates poisson trains

  Args:
    pars            : parameter dictionary
    rate            : noise amplitute [Hz]
    n               : number of Poisson trains
    myseed          : random seed. int or boolean

  Returns:
    pre_spike_train : spike train matrix, ith row represents whether
                      there is a spike in ith spike train over time
                      (1 if spike, 0 otherwise)
  """

  # Retrieve simulation parameters
  dt, range_t = pars['dt'], pars['range_t']
  Lt = range_t.size

  # set random seed
  if myseed:
      np.random.seed(seed=myseed)
  else:
      np.random.seed()

  # generate uniformly distributed random variables
  u_rand = np.random.rand(n, Lt)

  # generate Poisson train
  poisson_train = 1. * (u_rand < rate * (dt / 1000.))

  return poisson_train

```

```python
# we can use Poisson_generator to mimic presynaptic spike trains
pars = default_pars()
pre_spike_train = Poisson_generator(pars, rate=10, n=100, myseed=2020)
my_raster_Poisson(pars['range_t'], pre_spike_train, 100)

```

How do we make sure that the above spike trains are following Poisson statistics?
A Poisson process must have the following properties:
- The ratio of the mean and variance of spike count is 1
- Inter-spike-intervals are exponentially distributed
- Spike times are irregular i.e. $CV_{\rm ISI} = 1$
- Adjacent spike intervals are independent of each other.

---
# Summary

Congratulations! You've just built a leaky integrate-and-fire (LIF) neuron model from scratch, and studied its dynamics in response to various types of inputs, having:

- simulated the LIF neuron model

- driven the LIF neuron with external inputs, such as direct current, Gaussian white noise, and Poisson spike trains, etc.

- studied how different inputs affect the  LIF neuron's output (firing rate and spike time irregularity),

with a special focus on low rate and irregular firing regime to mimc real cortical neurons. The next tutorial will look at how spiking statistics may be influenced by a neuron's input statistics.

However, if you have extra time, follow the section below to explore a different type of noise input.

---
# Bonus 1: Orenstein-Uhlenbeck Process

When a neuron receives spiking input, the synaptic current is Shot Noise -- which is a kind of colored noise and the spectrum of the noise determined by the synaptic kernel time constant. That is, a neuron is driven by **colored noise** and not GWN.

We can model colored noise using the Ohrenstein-Uhlenbeck process - filtered white noise.

## Ornstein-Uhlenbeck (OU) current

We next study if the input current is temporally correlated and is modeled as an Ornstein-Uhlenbeck process $\eta(t)$, i.e., low-pass filtered GWN with a time constant $\tau_{\eta}$:

$$\tau_\eta \frac{d}{dt}\eta(t) = \mu-\eta(t) + \sigma_\eta\sqrt{2\tau_\eta}\xi(t).$$

**Hint:** An OU process as defined above has

$$E[\eta(t)]=\mu$$

and autocovariance

$$[\eta(t)\eta(t+\tau)]=\sigma_\eta^2e^{-|t-\tau|/\tau_\eta},$$

which can be used to check your code.

```python
# @title `my_OU(pars, mu, sig, myseed)`
# @markdown Ececute this cell to enable the OU process

def my_OU(pars, mu, sig, myseed=False):
  """
  Function that produces Ornstein-Uhlenbeck input

  Args:
    pars       : parameter dictionary
    sig        : noise amplitute
    myseed     : random seed. int or boolean

  Returns:
    I_ou       : Ornstein-Uhlenbeck input current
  """

  # Retrieve simulation parameters
  dt, range_t = pars['dt'], pars['range_t']
  Lt = range_t.size
  tau_ou = pars['tau_ou']  # [ms]

  # set random seed
  if myseed:
    np.random.seed(seed=myseed)
  else:
    np.random.seed()

  # Initialize
  noise = np.random.randn(Lt)
  I_ou = np.zeros(Lt)
  I_ou[0] = noise[0] * sig

  # generate OU
  for it in range(Lt-1):
    I_ou[it+1] = I_ou[it] + (dt / tau_ou) * (mu - I_ou[it]) + np.sqrt(2 * dt / tau_ou) * sig * noise[it + 1]

  return I_ou
```

### Interactive Demo: LIF Explorer with OU input
In the following, we will check how a neuron responds to a noisy current that follows the statistics of an OU process.

```python
# @title

# @markdown Remember to enable the widget by running the cell!

my_layout.width = '450px'
@widgets.interact(
  tau_ou=widgets.FloatSlider(10.0, min=5., max=20.,
                             step=2.5, layout=my_layout),
  sig_ou=widgets.FloatSlider(10.0, min=5., max=40.,
                             step=2.5, layout=my_layout),
  mu_ou=widgets.FloatSlider(190.0, min=180., max=220.,
                            step=2.5, layout=my_layout)
)

def LIF_with_OU(tau_ou=10., sig_ou=40., mu_ou=200.):
  pars = default_pars(T=1000.)
  pars['tau_ou'] = tau_ou  # [ms]

  I_ou = my_OU(pars, mu_ou, sig_ou)

  v, sp = run_LIF(pars, Iinj=I_ou)

  plt.figure(figsize=(12, 4))
  plt.subplot(121)
  plt.plot(pars['range_t'], I_ou, 'b', lw=1.0)
  plt.xlabel('Time (ms)')
  plt.ylabel(r'$I_{\mathrm{OU}}$ (pA)')
  plt.subplot(122)
  plot_volt_trace(pars, v, sp)
  plt.tight_layout()
  plt.show()
```

    interactive(children=(FloatSlider(value=10.0, description='tau_ou', layout=Layout(width='450px'), max=20.0, mi…

## Think!
- How does the OU type input change neuron responsiveness?
- What do you think will happen to the spike pattern and rate if you increased or decreased the time constant of the OU process?

```python
# to_remove explanation

"""
Discussion:

In a limiting case, when the time constant of the OU process is very long and the
input current is almost flat, we expect the firing rate to decrease and neuron
will spike more regularly. So as the OU process time constant increases, we expect
firing rate and CV_ISI to decrease, if all other parameters are kept constant. We
can also relate the OU process time constant to the membrane time constant as the
neuron membrane does the same operation. This way we can link to the very first
interactive demo.
""";
```

---
# Bonus 2: Generalized Integrate-and-Fire models

LIF model is not the only abstraction of real neurons. If you want to learn about more realistic types of neuronal models, watch the Bonus Video!

```python
#@title Video 3 (Bonus): Extensions to Integrate-and-Fire models
from IPython.display import YouTubeVideo
video = YouTubeVideo(id='G0b6wLhuQxE', width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
video
```

    Video available at https://youtube.com/watch?v=G0b6wLhuQxE


[Yee Whye Teh et al](http://www.cs.berkeley.edu/~jordan/papers/hdp.pdf)'s 2005 paper _Hierarchical Dirichlet Processes_ describes a nonparametric prior for grouped clustering problems. For example, the HDP helps in generalizing the [latent Dirichlet allocation](https://en.wikipedia.org/wiki/Latent_Dirichlet_allocation) model to the case the number of topics in the data are discovered by the inference algorithm instead of being specified as a parameter of the model.

The authors describe three MCMC-based algorithms for fitting HDP based models. Unfortunately, the algorithms are described somewhat vaguely and in general terms. A fair bit of mathematical leg work is required before the HDP algorithms can be applied to the specific case of nonparametric latent Dirichlet allocation.

Here are some notes I've compiled in my effort to understand these algorithms.

## HDP-LDA Generative Model

The generative model for Hierarchical Dirichlet Process Latent Dirichlet Allocation is as follows:

\begin{equation}
\begin{aligned}
  H & \sim \text{Dirichlet}(\beta) \\
  G_0 \,|\, \gamma, H & \sim \text{DP}(\gamma, H) \\
  G_j \,|\, \alpha_0, G_0 & \sim \text{DP}(\alpha_0, G_0) \\
  \theta_{ji} \,|\, G_j & \sim G_j \\
  x_{ij} \,|\, \theta_{ji} & \sim \text{Categorical}(\theta_{ji}) \\
\end{aligned}
\end{equation}

* $H$ is Dirichlet distribution whose dimension is the size of the vocabulary, i.e. it is distribution over an uncountably-number of term distributions (topics).
* $G_0$ is a distribution over a countably-infinite number of categorical term distributions, i.e. topics.
* For each document $j$, $G_j$ is a is a distribution over a countably-infinite number of categorical term distributions, i.e. topics.
* $\theta_{ji}$ is a categorical distribution over terms, i.e. a topic.
* $x_{ij}$ is a term.

To see code for sampling from this generative model, see [this post](/nonparametric-lda/).

### Chinese Restaurant Franchise Approach

Instead of the above Dirichlet process model, we can think of an identical "Chinese Restaurant Franchise" model.

Each $\theta_{ji}$ is a customer in restaurant $j$. Each customer is sitting at a table, and each table has multiple customers.

There is a global menu of $K$ dishes that the restaurants serve, $\phi_1,\ldots,\phi_K$.

Some other definitions:

* $\psi_{jt}$ is the dish served at table $t$ in restaurant $j$; i.e. each $\psi_{jt}$ corresponds to some $\phi_k$.
* $t_{ji}$ is the index of the $\psi_{jt}$ associated with $\theta_{ji}$.
* $k_{jt}$ is the index of $\phi_k$ associated with $\psi_{jt}$.

*Customer $i$ in restaurant $j$ sits at table $t_{ji}$ while table $t$ in restaurant $j$ serves dish $k_{jt}$.*

There are two arrays of count variables we will want to track:

* $n_{jtk}$ is the number of customers in restaurant $j$ at table $t$ eating dish $k$.
* $m_{jk}$ is the number of tables in restaurant $j$ serving dish $k$ (multiple tables may serve the same dish).

To summarize:

$x_{ij}$ are observed data (words). We assume $x_{ij}\sim F(\theta_{ij})$. Further, we assume $\theta_{ji}$ is associated with table $t_{ji}$, that is $\theta_{ji}=\psi_{jt_{ji}}$. Further, we assume the topic for table $j$ is indexed by $k_{jt}$, i.e. $\psi_{jt}=\phi_{k_{jt}}$. Thus, if we know $t_{ji}$ (the table assignment for $x_{ij}$) and $k_{jt}$ (the dish assignment for table $t$) for all $i, j, t$, we could determine the remaining parameters by sampling.

## Gibbs Sampling

[Teh et al](http://www.cs.berkeley.edu/~jordan/papers/hdp.pdf) describe three Gibbs samplers for this model. The first and third are most applicable to the LDA application. The section helps with more complication applications of the LDA algorithm (e.g. the hidden Markov model).

### 5.3 Posterior sampling by direct assignment

Section 5.3 describes a direct assignment Gibbs sampler that directly assigns words to topics by augmenting the model with an assignment variable $z_{ji}$ that is equivalent to $k_{jt_{ji}}$.  This also requires a count variable $m_{jk}$: the number of tables in document/franchise $j$ assigned to dish/topic $k$. This sampler requires less "bookkeeping" than the algorithm from 5.1, however it requires expensive simulation or computation of recursively computed Stirling numbers of the first kind.

My notes below are derive the necessary equations for Gibbs sampling for the algorithm in section 5.1. However, they also provide most of the derivations needed for 5.3.

### 5.1 Posterior sampling in the Chinese restaurant franchise

Section 5.1 describes "Posterior sampling in the Chinese restaurant franchise". Given observed data $\mathbf{x}$ (i.e. documents), we sample over the index variables $t_{ji}$ (associating tables with customers/words) and $k_{jt}$ (associating tables with dishes/topics). Given these variables, we can reconstruct the distribution over topics for each document and distribution over words for each topic.

#### Notes on Implementing Algorithm 5.1

Teh et al's original HDP paper is sparse on details with regard to applying these samplers to the specific case of nonparametric LDA. For example, both samplers require computing the conditional distribution of word $x_{ji}$ under topic $k$ given all data items except $x_{ji}$: $f_k^{x_{ji}}(x_{ji})$) (eq. 30).

[A Blogger Formerly Known As Shuyo](https://shuyo.wordpress.com/2012/08/15/hdp-lda-updates/) has a brief post where he states (with little-to-no derivation) the equations specific to the LDA case. Below I attempt provide some of those derivations in pedantic detail.

As stated above, in the case of topic modeling, $H$ is a Dirichlet distribution over terms distributions and $F$ is a multinomial distribution over terms.

By definition,

\begin{equation} h(\phi_k)=\frac{1}{Z}\prod_v[\phi_k]_v^{\beta-1} \end{equation}

and

\begin{equation}
    f(x_{ji}=v \,|\, \phi_k)=\phi_{kv}.
\end{equation}

##### Equation (30): $f_k^{x_{ji}}(x_{ji})$

For convenience, define $v=x_{ji}$ (word $i$ in document $j$), define $k=k_{jt_{ji}}$ (topic assignment for the table in document $j$ containing word $i$), and

\begin{equation}
    n_{kv}^{-ji}=\left|\left\{
        x_{mn} \,|\, k_{mt_{mn}}=k,\, x_{mn}=v,\, (m,n)\neq(j,i)
    \right\}\right|
\end{equation}

(the number of times the term $x_{ji}$, besides $x_{ji}$ itself, is generated by the same topic as was $x_{ji}$).

First look at the term (for fixed $k$):

\begin{equation}
   \prod_{j'i'\neq ji, z_{j'i'=k}}
        f(x_{j'i'} \,|\, \phi_k)=
    \prod_{j'}
    \prod_{i'\neq i, z_{j'i'}=k}
        [\phi_{k}]_{x_{j'i'}}
\end{equation}

$[\phi_k]_v$ is the probability that term $v$ is generated by topic $k$. The double sums run over every word generated by topic $k$ in every document. Since $[\phi_{k}]_{x_{j'i'}}$ is fixed for a given word $w$, we could instead do a product over the each word of the vocabulary:

\begin{equation}
\prod_{j'i'\neq ji, z_{j'i'=k}}
        f(x_{j'i'} \,|\, \phi_k)
    =\prod_{w\in\mathcal{V}}[\phi_k]_w^{n_{kw}^{-ji}}
\end{equation}

We use this early on in the big derivation below.

Also, note that

\begin{equation}
        \int
            \phi_{kv}^{n_{kv}^{-ji}+\beta}
            \prod_{w\neq v}
                \phi_{kw}^{n_{kw}^{-ji}+\beta-1}
        d\phi_k  \text{   and    }
                    \int
                \prod_w
                    \phi_{kw}^{n_{kw}^{-ji}+\beta-1}
            d\phi_k
\end{equation}

are the normalizing coefficients for Dirichlet distributions.

Equation (30) in Teh's paper is:

\begin{equation}
\begin{aligned}
    f_k^{-x_{ji}}(x_{ji})
    &=\frac{
            \int
                f(x_{ji} \,|\, \phi_k)
                   \left[
                   \prod_{j'i'\neq ji, z_{j'i'}=k}
                        f(x_{j'i'} \,|\, \phi_k)
                   \right]
                        h(\phi_k)
                        d\phi_k
        }
        {
            \int
                \left[
                    \prod_{j'i'\neq ji, z_{j'i'}=k}
                    f(x_{j'i'}|\phi_k)
                \right]
                h(\phi_k)d\phi_k
        } \\
    &=\frac{
            \int
                f(x_{ji} \,|\, \phi_k)
                    \left[
                    \prod_{j'}
                    \prod_{i'\neq i, z_{j'i'}=k}
                        \phi_{kv}
                    \right]\cdot
                        h(\phi_k)
                        d\phi_k
        }
        {
            \int
                \left[
                    \prod_{j'i'\neq ji, z_{j'i'}=k}
                    f(x_{j'i'}|\phi_k)
                \right]h(\phi_k)d\phi_k
        } \\
    &\propto\frac{
            \int
                \phi_{kv}
                \prod_w
                    \phi_{kw}^{n_{kw}^{-ji}}
                \prod_{w}
                    \phi_{kw}^{\beta-1}
            d\phi_k
        }{
            \int
                \prod_w
                    \phi_{kw}^{n_{kw}^{-ji}}
                \prod_w
                    \phi_{kw}^{\beta-1}
            d\phi_k
        }\\
    &=\frac{
            \int
                \phi_{kv}\cdot
                \phi_{kv}^{n_{kv}^{-ji}}\cdot
                \prod_{w\neq v}
                    \phi_{kw}^{n_{kw}^{-ji}}\cdot
                \phi_{kv}^{\beta-1}\cdot
                \prod_{w\neq v}
                    \phi_{kw}^{\beta-1}
            d\phi_k
        }{
            \int
                \prod_w
                    \phi_{kw}^{n_{kw}^{-ji}}
                \prod_w
                    \phi_{kw}^{\beta-1}
            d\phi_k
        }\\
    &=
        \int
            \phi_{kv}^{n_{kv}^{-ji}+\beta}
            \prod_{w\neq v}
                \phi_{kw}^{n_{kw}^{-ji}+\beta-1}
        d\phi_k
        \cdot
        \frac{
            1
        }{
            \int
                \prod_w
                    \phi_{kw}^{n_{kw}^{-ji}+\beta-1}
            d\phi_k.
        }
\end{aligned}
\end{equation}

Recognizing these integrals as those that occur in the Dirichlet distribution, we have,

\begin{equation}
\begin{aligned}
    f_k^{-x_{ji}}(x_{ji})
    &=\frac{
            \Gamma\left(\beta+n_{kv}^{-ji}+1\right)
            \prod_{w\neq v} \Gamma\left(\beta+n_{kw}^{-ji}\right)
        }{
            \Gamma\left(
                \sum_{w\neq v}
                    \left[
                         \beta+n_{kw}^{-ji}
                    \right]+
                (\beta+n_{kv}^{-ji}+1)\right)
        }\cdot
        \frac{
            1
        }{
            \int\prod_w
                \left(\phi_{kw}\right)^{n_{kw}^{-ji}+\beta-1}
            d\phi_k
        }\\
    &=\frac{
            \Gamma\left(\beta+n_{kv}^{-ji}+1\right)
            \prod_{w\neq v} \Gamma\left(\beta+n_{kw}^{-ji}\right)
        }{
            \Gamma\left(
                \sum_{w\in\mathcal{V}}
                    \left[
                         \beta+n_{kw}^{-ji}
                    \right]
                +1\right)
        }\cdot
        \frac{
            1
        }{
            \int\prod_w
                \left(\phi_{kw}\right)^{n_{kw}^{-ji}+\beta-1}
            d\phi_k
        }\\
    &=\frac{
            \Gamma\left(\beta+n_{kv}^{-ji}+1\right)
            \prod_{w\neq v} \Gamma\left(\beta+n_{kw}^{-ji}\right)
        }{
            \Gamma\left(
                \sum_{w\in\mathcal{V}}
                    \left[
                         \beta+n_{kw}^{-ji}
                    \right]
                +1\right)
        }\cdot
        \frac{
            \Gamma\left(
            \sum_{w\in\mathcal{V}}
                \left[
                    \beta+n_{kw}^{-ji}
                \right]
            \right)
        }{
            \prod_{w}\Gamma\left(\beta+n_{kw}^{-ji}\right)
        }\\
    &=\frac{
            \Gamma\left(\beta+n_{kv}^{-ji}+1\right)
            \prod_{w\neq v} \Gamma\left(\beta+n_{kw}^{-ji}\right)
        }{
            \Gamma\left(V\beta+n_{k\cdot}^{-ji}+1\right)
        }\cdot
        \frac{
            \Gamma\left(V\beta+n_{k\cdot}^{-ji}\right)
        }{
            \prod_{w}\Gamma\left(\beta+n_{kw}^{-ji}\right)
        }\\
    &=\frac{
            \Gamma\left(\beta+n_{kv}^{-ji}+1\right)
            \Gamma\left(V\beta+n_{k\cdot}^{-ji}\right)
        }{
            \Gamma\left(V\beta+n_{k\cdot}^{-ji}+1\right)
        }\cdot
        \frac{
            \prod_{w\neq v} \Gamma\left(\beta+n_{kw}^{-ji}\right)
        }{
            \prod_{w}\Gamma\left(\beta+n_{kw}^{-ji}\right)
        }\\
    &=\frac{
            \Gamma\left(\beta+n_{kn}^{-ji}+1\right)
            \Gamma\left(V\beta+n_{k\cdot}^{-ji}\right)
        }{
            \Gamma\left(V\beta+n_{k\cdot}^{-ji} + 1\right)
            \Gamma\left(\beta+n_{kv}^{-ji}\right)
        }\\
    &=\frac{
            \Gamma\left(\beta+n_{kn}^{-ji}+1\right)
        }{
            \Gamma\left(\beta+n_{kv}^{-ji}\right)
        }\cdot
        \frac{
            \Gamma\left(V\beta+n_{k\cdot}^{-ji}\right)
        }{
            \Gamma\left(V\beta+n_{k\cdot}^{-ji} + 1\right)
        }\\
    &=\frac{\beta+n_{kv}^{-ji}}{V\beta+n_{k\cdot}^{-ji}}
\end{aligned}
\end{equation}

This is validated in the [appendix of this paper](http://arxiv.org/pdf/1201.1657.pdf).

##### Equation: $f_{k^\text{new}}^{x_{ji}}(x_{ji})$

We also need the prior density of $x_{ji}$ to compute the likelihood that $x_{ji}$ will be seated at a new table.

\begin{equation}
\begin{aligned}
    f_{k^{\text{new}}}^{-x_{ji}}(x_{ji})
    &=
        \int
            f(x_{ji} \,|\, \phi)
            h(\phi)d\phi_k
        \\
    &=\int
        \phi_v \cdot
        \frac{
                \Gamma(V\beta)
            }{
                \prod_w \Gamma(\beta)
        }
        \prod_w \phi_w^{\beta-1}
        d\phi \\
    &=\frac{
                \Gamma(V\beta)
            }{
                \prod_w \Gamma(\beta)
        }
        \int
            \phi_v \cdot
            \prod_w \phi_w^{\beta-1}
        d\phi\\
    &=\frac{
                \Gamma(V\beta)
            }{
                \prod_w \Gamma(\beta)
        }
        \int
            \phi_v^\beta \cdot
            \prod_{w\neq v} \phi_w^{\beta-1}
        d\phi\\
    &=\frac{
                \Gamma(V\beta)
            }{
                \prod_w \Gamma(\beta)
        }\cdot
        \frac{
            \Gamma(\beta+1)\prod_{w\neq v}\Gamma(\beta)
            }{
            \Gamma(V\beta+1)
        }\\
    &=\frac{
                \Gamma(V\beta)
            }{
                \Gamma(V \beta+1)
        }\cdot
        \frac{
            \beta\prod_{w}\Gamma(\beta)
            }{
            \prod_w \Gamma(\beta)
        }\\
    &=\frac{
                1
            }{
                V \beta
        }\cdot
        \beta\\
    &= \frac{1}{V}
\end{aligned}
\end{equation}

##### Equation (31): $p(x_{ji} \,|\, {\bf t}^{-ji}, t_{ji}=t^{new}, {\bf k})$

These last two derivations give us Equation (31), the likelihood that $t_{ji}=t^{new}$:

\begin{equation}
\begin{aligned}
    p(x_{ji} \,|\, {\bf t}^{-ji}, t_{ji}=t^{new}, {\bf k})
    &=\sum_{k=1}^K \left[
        \frac{
            m_{\cdot k}
            }{
            m_{\cdot\cdot}+\gamma
        }\cdot
        \frac{\beta+n_{kv}^{-ji}}{V\beta+n_{k\cdot}^{-ji}}
   \right] \\
   &\phantom{=}+ \frac{
       \gamma
       }{
       m_{\cdot\cdot}+\gamma
   }
   \cdot \frac{1}{V}
\end{aligned}
\end{equation}

##### Equation (32): $p(t_{ji}=t \,|\, {\bf t}^{-ji}, {\bf k})$

From this, we know the conditional distribution of $t_{ji}$ is:

\begin{equation}
    p(t_{ji}=t \,|\, {\bf t}^{-ji}, {\bf k}) \propto
    \begin{cases}
        n_{jt\cdot}^{-ji} \cdot \frac{\beta+n_{k_{jt}v}^{-ji}}{V\beta+n_{k_{jt}\cdot}^{-ji}}
        & {\tiny \text{if } t \text{ previously used,}}\\
        {\tiny
            \alpha_0 \cdot
            p(x_{ji} \,|\, {\bf t}^{-ji}, t_{ji}=t^{new}, {\bf k})}
        & {\tiny \text{if } t=t^{\text{new}}}.
    \end{cases}
\end{equation}

##### Equation (33): $p(k_{jt^\text{new}}=k \,|\, {\bf t}, {\bf k}^{-jt^\text{new}})$

If the sampled value of $t_{ji}$ is $t^{\text{new}}$, we sample a dish $k_{jt^\text{new}}$ for the table with:

\begin{equation}
    p(k_{jt^\text{new}}=k \,|\, {\bf t}, {\bf k}^{-jt^\text{new}}) \propto
    \begin{cases}
        m_{\cdot k}\cdot\frac{\beta+n_{kv}^{-ji}}{V\beta+n_{k\cdot}^{-ji}}
        & {\tiny \text{if } k \text{ previously used,}}\\
        \frac{
            \gamma
        }{V}
        & {\tiny \text{if } t=k^{\text{new}}}.
    \end{cases}
\end{equation}

##### Equation (34): $p(k_{jt}=k \,|\, {\bf t}, {\bf k}^{-jt})$

We need to sample $k_{jt}$ (the dish/topic for table $t$ in restaurant $j$):

\begin{equation}\displaystyle
    p(k_{jt}=k \,|\, {\bf t}, {\bf k}^{-jt}) \propto
    \begin{cases}
        m_{\cdot k}^{-jt}\cdot f_k^{-{\bf x}_{jt}}({\bf x}_{jt})
            & {\tiny \text{if } k \text{ previously used,}}\\
        \gamma\cdot f_{k^\text{new}}^{-{\bf x}_{jt}}({\bf x}_{jt})
            & {\tiny \text{if } t=k^{\text{new}}}.
    \end{cases}
\end{equation}

where $f_k^{-{\bf x}_{jt}}({\bf x}_{jt})$ is the "conditional density of ${\bf x}_{jt}$ given all data items associated with mixture component $k$ leaving out ${\bf x}_{jt}$" (Teh, et al). (${\bf x}_{jt}$ is every customer in restaurant $j$ seated at table $t$). $m_{\cdot k}^{-jt}$ is the number of tables (in all franchises) serving dish $k$ when we remove table $jt$.

This requires $f_k^{-{\bf x}_{jt}}({\bf x}_{jt})$; this is different from Equation (30), though they look quite similar.

\begin{equation}
\begin{aligned}
    f_k^{-{\bf x}_{jt}}({\bf x}_{jt})
    &=\frac{\displaystyle
            \int
                {\prod_{x_{ji}\in {\bf x}_{jt}}}
                f(x_{ji} \,|\, \phi_k)
                   \left[
                   \prod_{x_{j'i'}\not\in {\bf x}_{jt}, z_{j'i'}=k}
                        f(x_{j'i'} \,|\, \phi_k)
                   \right]
                        h(\phi_k)
                        d\phi_k
        }
        {\displaystyle
            \int
                \left[
                    \displaystyle\prod_{x_{j'i'}\not\in {\bf x}_{jt}, z_{j'i'}=k}
                    f(x_{j'i'}|\phi_k)
                \right]
                h(\phi_k)d\phi_k
        } \\
    &=\frac{\displaystyle
            \int
                {\prod_{x_{ji}\in {\bf x}_{jt}}}
                \phi_{k x_{ji}}
                   \left[
                   \prod_{x_{j'i'}\not\in {\bf x}_{jt}, z_{j'i'}=k}
                        \phi_{k x_{j'i'}}
                   \right]
                        \prod_{w}
                    \phi_{kw}^{\beta-1}
                        d\phi_k
        }
        {\displaystyle
            \int
                \left[
                    \displaystyle\prod_{x_{j'i'}\not\in {\bf x}_{jt}, z_{j'i'}=k}
                    f(x_{j'i'}|\phi_k)
                \right]
                \prod_{w}
                    \phi_{kw}^{\beta-1}
                    d\phi_k
        } \\
    &=\frac{\displaystyle
            \int
                {\prod_{x_{ji}\in {\bf x}_{jt}}}
                \phi_{k x_{ji}}
                   \left[
                   \prod_{x_{j'i'}\not\in {\bf x}_{jt}, z_{j'i'}=k}
                        \phi_{k x_{j'i'}}
                   \right]
                        \prod_{w}
                    \phi_{kw}^{\beta-1}
                        d\phi_k
        }
        {\displaystyle
            \int
                \left[
                    \displaystyle\prod_{x_{j'i'}\not\in {\bf x_{jt}}, z_{j'i'}=k}
                    \phi_{k x_{j'i'}}
                \right]
                \prod_{w}
                    \phi_{kw}^{\beta-1}
                    d\phi_k
        }
\end{aligned}
\end{equation}

The denominator is

\begin{equation}
\begin{aligned}
    \text{denominator}&=
    \int\left[
        \prod_{x_{j'i'}\not\in {\bf x_{jt}}, z_{j'i'}=k}
        \phi_{k x_{j'i'}}
    \right]
    \prod_{w}
        \phi_{kw}^{\beta-1}
        d\phi_k \\
    &=\int\left[
        \prod_w \phi_{kw}^{n_{kw}^{-jt}} \prod_w \phi_{kw}^{\beta-1}
    \right]
        d\phi_k \\
    &=\int\left[
        \prod_w \phi_{kw}^{n_{kw}^{-jt}+\beta-1}
    \right]
        d\phi_k \\
    &=\frac{
        \prod_w \Gamma\left(
            n_{kw}^{-jt} + \beta
        \right)
    }{
        \Gamma\left( \sum_w
            n_{kw}^{-jt}+\beta
        \right)
    } \\
    &=\frac{
        \prod_w \Gamma\left(
            n_{kw}^{-jt} + \beta
        \right)
    }{
        \Gamma\left(
            n_{k\cdot}^{-jt}+V\beta
        \right)
    }
\end{aligned}
\end{equation}

The numerator is

\begin{equation}
\begin{aligned}
    \text{numerator}
    &=\int
    {\prod_{x_{ji}\in {\bf x}_{jt}}}
    \phi_{k x_{ji}}
       \left[
       \prod_{x_{j'i'}\not\in {\bf x}_{jt}, z_{j'i'}=k}
            \phi_{k x_{j'i'}}
       \right]
            \prod_{w}
        \phi_{kw}^{\beta-1}
            d\phi_k \\
    &=\int
    \prod_{w}
        \phi_{kw}^{
            n_{kw}^{-jt} +
            n_{\cdot w}^{jt} +
            \beta + 1
            }
        d\phi_k \\
    &=\frac{
       \prod_w \Gamma\left(
           n_{kw}^{-jt} + n_{\cdot w}^{jt} + \beta
       \right)
    }{
      \Gamma \left(
        \sum_w
        n_{kw}^{-jt} + n_{\cdot w}^{jt} + \beta
      \right)
    }\\
    &=\frac{
       \prod_w \Gamma\left(
           n_{kw}^{-jt} + n_{\cdot w}^{jt} + \beta
       \right)
    }{
      \Gamma \left(
        n_{k\cdot}^{-jt} + n_{\cdot \cdot}^{jt} + \beta
      \right)
    }
\end{aligned}
\end{equation}

This gives us a closed form version of this conditional distribution:

\begin{equation}
\begin{aligned}
    f_k^{-{\bf x_{jt}}}({\bf x}_{jt})
    &= \displaystyle\frac{
       \prod_w \Gamma\left(
           n_{kw}^{-jt} + n_{\cdot w}^{jt} + \beta
       \right)
    }{
            \prod_w \Gamma\left(
                n_{kw}^{-jt} + \beta
            \right)
        }
    \frac{
            \Gamma\left(
                n_{k\cdot}^{-jt}+V\beta
            \right)
        }{
      \Gamma \left(
        n_{k\cdot}^{-jt} + n_{\cdot \cdot}^{jt} + \beta
      \right)
    }.
\end{aligned}
\end{equation}

We also need the conditional distribution of $k$ is a new dish: $f_{k^\text{new}}^{-{\bf x}_{jt}}({\bf x}_{jt})$. Shuyo provides without derivation:

\begin{equation}
\begin{aligned}
f_{k^\text{new}}^{-{\bf x}_{jt}}({\bf x}_{jt})
    &=\int
        \left[
        \prod_{x_{ji}\in \mathbf{x_{jt}}}
            f(x_{ji} \,|\, \phi)
        \right]
        h(\phi)d\phi_k
        \\
    &=\int
        \prod_{x_{ji}\in \mathbf{x_{jt}}}
            \phi_{x_{ji}}
        \cdot
        \frac{
                \Gamma(V\beta)
            }{
                \prod_w \Gamma(\beta)
        }
        \prod_w \phi_w^{\beta-1}
        d\phi \\
    &=\frac{
                \Gamma(V\beta)
            }{
                \prod_w \Gamma(\beta)
        }
        \int
            \prod_{x_{ji}\in \mathbf{x_{jt}}}
            \phi_{x_{ji}}
            \cdot
            \prod_w \phi_w^{\beta-1}
        d\phi\\
    &=\frac{
                \Gamma(V\beta)
            }{
                \prod_w \Gamma(\beta)
        }
        \int
            \prod_{x_{ji}\in \mathbf{x_{jt}}}
                \phi_{x_{ji}}^{\left(\beta+1\right)-1}
            \cdot
            \prod_{x_{jt}\not\in \mathbf{x_{jt}}} \phi_{x_{jt}}^{\beta-1}
        d\phi\\
    &=\frac{
                \Gamma(V\beta)
            }{
                \prod_w \Gamma(\beta)
        }\cdot
        \frac{
            \prod_{x_{ji}\in \mathbf{x_{jt}}}
                \Gamma(\beta+1)
            \prod_{x_{ji}\not\in \mathbf{x_{jt}}}\Gamma(\beta)
            }{
            \Gamma(V\beta+\sum_{x_{ji}\in \mathbf{x_{jt}}} 1)
        }\\
    &=\frac{
        \Gamma(V\beta)
        \prod_w \Gamma(\beta+n_{\cdot w}^{jt})
    }{
        \Gamma(V\beta+n_{\cdot\cdot}^{jt})
        \prod_w \Gamma(\beta)
    }.
\end{aligned}
\end{equation}

Given these equations for $f_{k}^{-{\bf x}_{jt}}({\bf x}_{jt})$ and $f_{k^\text{new}}^{-{\bf x}_{jt}}({\bf x}_{jt})$, we can draw samples from $p(k_{jt}=k \,|\, {\bf t}, {\bf k}^{-jt})$ by enumeration over topics. We now have a complete Gibbs sampler for the [Posterior sampling in the Chinese restaurant franchise in Teh, et al.](http://www.cs.berkeley.edu/~jordan/papers/hdp.pdf)


# Loogbook

```python
# %load imports.py
"""
These is the standard setup for the notebooks.
"""

%matplotlib inline
%load_ext autoreload
%autoreload 2

from jupyterthemes import jtplot
jtplot.style(theme='onedork', context='notebook', ticks=True, grid=False)

import pandas as pd
pd.options.display.max_rows = 999
pd.options.display.max_columns = 999
pd.set_option("display.max_columns", None)
import numpy as np
import os
import matplotlib.pyplot as plt
#plt.style.use('paper')

import copy

import numpy as np
import os

from src.data import database
from mdldb import mdl_to_evaluation
from mdldb.tables import Run
import src.data
import os.path

from sklearn.pipeline import Pipeline

import sympy as sp
from sklearn.metrics import r2_score
import src.reporting.paper_writing as paper_writing

from src.equations import equations
from src.equations import symbols
```

## 2020-11-10
* Made an overview of the manoeuvring tests in the old system [01.01_db_overview](01.01_db_overview.ipynb)
* Analyzed mean values from reverse spiral tests and the Nomoto model : [02.01_db_nomoto](02.01_db_nomoto.ipynb)
   * Found out that higher order in yaw rate r is needed to reproduce the spiral.

```python

```


```python
%matplotlib inline
```

Sequence-to-Sequence Modeling with nn.Transformer and TorchText
===============================================================

This is a tutorial on how to train a sequence-to-sequence model
that uses the
`nn.Transformer <https://pytorch.org/docs/master/nn.html?highlight=nn%20transformer#torch.nn.Transformer>`__ module.

PyTorch 1.2 release includes a standard transformer module based on the
paper `Attention is All You
Need <https://arxiv.org/pdf/1706.03762.pdf>`__. The transformer model
has been proved to be superior in quality for many sequence-to-sequence
problems while being more parallelizable. The ``nn.Transformer`` module
relies entirely on an attention mechanism (another module recently
implemented as `nn.MultiheadAttention <https://pytorch.org/docs/master/nn.html?highlight=multiheadattention#torch.nn.MultiheadAttention>`__) to draw global dependencies
between input and output. The ``nn.Transformer`` module is now highly
modularized such that a single component (like `nn.TransformerEncoder <https://pytorch.org/docs/master/nn.html?highlight=nn%20transformerencoder#torch.nn.TransformerEncoder>`__
in this tutorial) can be easily adapted/composed.

Define the model
----------------

In this tutorial, we train ``nn.TransformerEncoder`` model on a
language modeling task. The language modeling task is to assign a
probability for the likelihood of a given word (or a sequence of words)
to follow a sequence of words. A sequence of tokens are passed to the embedding
layer first, followed by a positional encoding layer to account for the order
of the word (see the next paragraph for more details). The
``nn.TransformerEncoder`` consists of multiple layers of
`nn.TransformerEncoderLayer <https://pytorch.org/docs/master/nn.html?highlight=transformerencoderlayer#torch.nn.TransformerEncoderLayer>`__. Along with the input sequence, a square
attention mask is required because the self-attention layers in
``nn.TransformerEncoder`` are only allowed to attend the earlier positions in
the sequence. For the language modeling task, any tokens on the future
positions should be masked. To have the actual words, the output
of ``nn.TransformerEncoder`` model is sent to the final Linear
layer, which is followed by a log-Softmax function.

```python
import math
import torch
import torch.nn as nn
import torch.nn.functional as F

class TransformerModel(nn.Module):

    def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5):
        super(TransformerModel, self).__init__()
        from torch.nn import TransformerEncoder, TransformerEncoderLayer
        self.model_type = 'Transformer'
        self.pos_encoder = PositionalEncoding(ninp, dropout)
        encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
        self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
        self.encoder = nn.Embedding(ntoken, ninp)
        self.ninp = ninp
        self.decoder = nn.Linear(ninp, ntoken)

        self.init_weights()

    def generate_square_subsequent_mask(self, sz):
        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)
        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
        return mask

    def init_weights(self):
        initrange = 0.1
        self.encoder.weight.data.uniform_(-initrange, initrange)
        self.decoder.bias.data.zero_()
        self.decoder.weight.data.uniform_(-initrange, initrange)

    def forward(self, src, src_mask):
        src = self.encoder(src) * math.sqrt(self.ninp)
        src = self.pos_encoder(src)
        output = self.transformer_encoder(src, src_mask)
        output = self.decoder(output)
        return output
```

``PositionalEncoding`` module injects some information about the
relative or absolute position of the tokens in the sequence. The
positional encodings have the same dimension as the embeddings so that
the two can be summed. Here, we use ``sine`` and ``cosine`` functions of
different frequencies.

```python
class PositionalEncoding(nn.Module):

    def __init__(self, d_model, dropout=0.1, max_len=5000):
        super(PositionalEncoding, self).__init__()
        self.dropout = nn.Dropout(p=dropout)

        pe = torch.zeros(max_len, d_model)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        pe = pe.unsqueeze(0).transpose(0, 1)
        self.register_buffer('pe', pe)

    def forward(self, x):
        x = x + self.pe[:x.size(0), :]
        return self.dropout(x)
```

Load and batch data
-------------------

This tutorial uses ``torchtext`` to generate Wikitext-2 dataset. The
vocab object is built based on the train dataset and is used to numericalize
tokens into tensors. Starting from sequential data, the ``batchify()``
function arranges the dataset into columns, trimming off any tokens remaining
after the data has been divided into batches of size ``batch_size``.
For instance, with the alphabet as the sequence (total length of 26)
and a batch size of 4, we would divide the alphabet into 4 sequences of
length 6:

\begin{align}\begin{bmatrix}
  \text{A} & \text{B} & \text{C} & \ldots & \text{X} & \text{Y} & \text{Z}
  \end{bmatrix}
  \Rightarrow
  \begin{bmatrix}
  \begin{bmatrix}\text{A} \\ \text{B} \\ \text{C} \\ \text{D} \\ \text{E} \\ \text{F}\end{bmatrix} &
  \begin{bmatrix}\text{G} \\ \text{H} \\ \text{I} \\ \text{J} \\ \text{K} \\ \text{L}\end{bmatrix} &
  \begin{bmatrix}\text{M} \\ \text{N} \\ \text{O} \\ \text{P} \\ \text{Q} \\ \text{R}\end{bmatrix} &
  \begin{bmatrix}\text{S} \\ \text{T} \\ \text{U} \\ \text{V} \\ \text{W} \\ \text{X}\end{bmatrix}
  \end{bmatrix}\end{align}

These columns are treated as independent by the model, which means that
the dependence of ``G`` and ``F`` can not be learned, but allows more
efficient batch processing.

```python
import io
import torch
from torchtext.datasets import WikiText2
from torchtext.data.utils import get_tokenizer
from collections import Counter
from torchtext.vocab import Vocab

train_iter = WikiText2(split='train')
tokenizer = get_tokenizer('basic_english')
counter = Counter()
for line in train_iter:
    counter.update(tokenizer(line))
vocab = Vocab(counter)

def data_process(raw_text_iter):
  data = [torch.tensor([vocab[token] for token in tokenizer(item)],
                       dtype=torch.long) for item in raw_text_iter]
  return torch.cat(tuple(filter(lambda t: t.numel() > 0, data)))

train_iter, val_iter, test_iter = WikiText2()
train_data = data_process(train_iter)
val_data = data_process(val_iter)
test_data = data_process(test_iter)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

def batchify(data, bsz):
    # Divide the dataset into bsz parts.
    nbatch = data.size(0) // bsz
    # Trim off any extra elements that wouldn't cleanly fit (remainders).
    data = data.narrow(0, 0, nbatch * bsz)
    # Evenly divide the data across the bsz batches.
    data = data.view(bsz, -1).t().contiguous()
    return data.to(device)

batch_size = 20
eval_batch_size = 10
train_data = batchify(train_data, batch_size)
val_data = batchify(val_data, eval_batch_size)
test_data = batchify(test_data, eval_batch_size)
```

    wikitext-2-v1.zip: 100%|██████████| 4.48M/4.48M [00:01<00:00, 3.06MB/s]

```python

```

Functions to generate input and target sequence
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

``get_batch()`` function generates the input and target sequence for
the transformer model. It subdivides the source data into chunks of
length ``bptt``. For the language modeling task, the model needs the
following words as ``Target``. For example, with a ``bptt`` value of 2,
we’d get the following two Variables for ``i`` = 0:

It should be noted that the chunks are along dimension 0, consistent
with the ``S`` dimension in the Transformer model. The batch dimension
``N`` is along dimension 1.

```python
bptt = 35
def get_batch(source, i):
    seq_len = min(bptt, len(source) - 1 - i)
    data = source[i:i+seq_len]
    target = source[i+1:i+1+seq_len].reshape(-1)
    return data, target
```

Initiate an instance
--------------------

The model is set up with the hyperparameter below. The vocab size is
equal to the length of the vocab object.

```python
ntokens = len(vocab.stoi) # the size of vocabulary
emsize = 200 # embedding dimension
nhid = 200 # the dimension of the feedforward network model in nn.TransformerEncoder
nlayers = 2 # the number of nn.TransformerEncoderLayer in nn.TransformerEncoder
nhead = 2 # the number of heads in the multiheadattention models
dropout = 0.2 # the dropout value
model = TransformerModel(ntokens, emsize, nhead, nhid, nlayers, dropout).to(device)
```

Run the model
-------------

`CrossEntropyLoss <https://pytorch.org/docs/master/nn.html?highlight=crossentropyloss#torch.nn.CrossEntropyLoss>`__
is applied to track the loss and
`SGD <https://pytorch.org/docs/master/optim.html?highlight=sgd#torch.optim.SGD>`__
implements stochastic gradient descent method as the optimizer. The initial
learning rate is set to 5.0. `StepLR <https://pytorch.org/docs/master/optim.html?highlight=steplr#torch.optim.lr_scheduler.StepLR>`__ is
applied to adjust the learn rate through epochs. During the
training, we use
`nn.utils.clip_grad_norm\_ <https://pytorch.org/docs/master/nn.html?highlight=nn%20utils%20clip_grad_norm#torch.nn.utils.clip_grad_norm_>`__
function to scale all the gradient together to prevent exploding.

```python
criterion = nn.CrossEntropyLoss()
lr = 5.0 # learning rate
optimizer = torch.optim.SGD(model.parameters(), lr=lr)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)

import time
def train():
    model.train() # Turn on the train mode
    total_loss = 0.
    start_time = time.time()
    src_mask = model.generate_square_subsequent_mask(bptt).to(device)
    for batch, i in enumerate(range(0, train_data.size(0) - 1, bptt)):
        data, targets = get_batch(train_data, i)
        optimizer.zero_grad()
        if data.size(0) != bptt:
            src_mask = model.generate_square_subsequent_mask(data.size(0)).to(device)
        output = model(data, src_mask)
        loss = criterion(output.view(-1, ntokens), targets)
        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)
        optimizer.step()

        total_loss += loss.item()
        log_interval = 200
        if batch % log_interval == 0 and batch > 0:
            cur_loss = total_loss / log_interval
            elapsed = time.time() - start_time
            print('| epoch {:3d} | {:5d}/{:5d} batches | '
                  'lr {:02.2f} | ms/batch {:5.2f} | '
                  'loss {:5.2f} | ppl {:8.2f}'.format(
                    epoch, batch, len(train_data) // bptt, scheduler.get_last_lr()[0],
                    elapsed * 1000 / log_interval,
                    cur_loss, math.exp(cur_loss)))
            total_loss = 0
            start_time = time.time()

def evaluate(eval_model, data_source):
    eval_model.eval() # Turn on the evaluation mode
    total_loss = 0.
    src_mask = model.generate_square_subsequent_mask(bptt).to(device)
    with torch.no_grad():
        for i in range(0, data_source.size(0) - 1, bptt):
            data, targets = get_batch(data_source, i)
            if data.size(0) != bptt:
                src_mask = model.generate_square_subsequent_mask(data.size(0)).to(device)
            output = eval_model(data, src_mask)
            output_flat = output.view(-1, ntokens)
            total_loss += len(data) * criterion(output_flat, targets).item()
    return total_loss / (len(data_source) - 1)
```

Loop over epochs. Save the model if the validation loss is the best
we've seen so far. Adjust the learning rate after each epoch.

```python
best_val_loss = float("inf")
epochs = 3 # The number of epochs
best_model = None

for epoch in range(1, epochs + 1):
    epoch_start_time = time.time()
    train()
    val_loss = evaluate(model, val_data)
    print('-' * 89)
    print('| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | '
          'valid ppl {:8.2f}'.format(epoch, (time.time() - epoch_start_time),
                                     val_loss, math.exp(val_loss)))
    print('-' * 89)

    if val_loss < best_val_loss:
        best_val_loss = val_loss
        best_model = model

    scheduler.step()
```

Evaluate the model with the test dataset
-------------------------------------

Apply the best model to check the result with the test dataset.

```python
test_loss = evaluate(best_model, test_data)
print('=' * 89)
print('| End of training | test loss {:5.2f} | test ppl {:8.2f}'.format(
    test_loss, math.exp(test_loss)))
print('=' * 89)
```


# [HW11] Multiple Linear Regression

지난시간에는 하나의 독립 변수 $x$에 대해서 하나의 종속 변수 $y$ 사이의 관계를 알아보는 simple linear regression에 대해서 실습했습니다. 하지만 실제로 예측을 하고자 할 떄 보통 하나 이상의 변수들을 고려해야 합니다. 예를 들어서 집 값을 예측을 하고자 한다면, 집의 크기, 주변의 편의 시설, 위치, 화장실의 개수, 건축 년도 등등 고려해야 할 것들이 매우 많습니다. 이번 과제에서는 이렇게 다양한 입력 변수들을 다루는 **multiple linear regression** 에 대해서 알아보겠습니다.

##  1.1 Dataset

Multiple linear regression을 진행하기 위해서 데이터셋을 가져오겠습니다. 이번 과제에서 사용할 데이터셋은 자동차의 여러 기술적인 사양들을 고려하여 연비를 예측하는 auto miles per gallon(MPG) dataset입니다.

데이터를 불러오겠습니다.

```python
import pandas
import seaborn
seaborn.set()
```

```python
from urllib.request import urlretrieve
URL = 'https://go.gwu.edu/engcomp6data3'
urlretrieve(URL, 'auto_mpg.csv')
```

    ('auto_mpg.csv', <http.client.HTTPMessage at 0x7fe8cb2fb850>)

```python
mpg_data = pandas.read_csv('/content/auto_mpg.csv')
mpg_data.head()
```

      mpg
      cylinders
      displacement
      horsepower
      weight
      acceleration
      model year
      origin
      car name

      0
      18.0
      8
      307.0
      130.0
      3504.0
      12.0
      70
      1
      chevrolet chevelle malibu

      1
      15.0
      8
      350.0
      165.0
      3693.0
      11.5
      70
      1
      buick skylark 320

      2
      18.0
      8
      318.0
      150.0
      3436.0
      11.0
      70
      1
      plymouth satellite

      3
      16.0
      8
      304.0
      150.0
      3433.0
      12.0
      70
      1
      amc rebel sst

      4
      17.0
      8
      302.0
      140.0
      3449.0
      10.5
      70
      1
      ford torino

      <button class="colab-df-convert" onclick="convertToInteractive('df-e86060db-a5d4-49c1-b659-89f664fc80c3')"
              title="Convert this dataframe to an interactive table."
              style="display:none;">

  <svg xmlns="http://www.w3.org/2000/svg" height="24px"viewBox="0 0 24 24"
       width="24px">
    <path d="M0 0h24v24H0V0z" fill="none"/>
    <path d="M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z"/><path d="M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z"/>
  </svg>
      </button>

mpg_data.info()를 통해서 Data에 대한 정보를 살펴보겠습니다.

```python
mpg_data.info()
```

    <class 'pandas.core.frame.DataFrame'>
    RangeIndex: 392 entries, 0 to 391
    Data columns (total 9 columns):
     #   Column        Non-Null Count  Dtype
    ---  ------        --------------  -----
     0   mpg           392 non-null    float64
     1   cylinders     392 non-null    int64
     2   displacement  392 non-null    float64
     3   horsepower    392 non-null    float64
     4   weight        392 non-null    float64
     5   acceleration  392 non-null    float64
     6   model year    392 non-null    int64
     7   origin        392 non-null    int64
     8   car name      392 non-null    object
    dtypes: float64(5), int64(3), object(1)
    memory usage: 27.7+ KB

총 392개의 데이터가 있고 9개의 정보들이 있습니다.

여기서 car name 은 object로 자동차의 이름을 담고 있습니다.

그리고 origin은 int로 정수 형태이지만 이것이 만들어진 도시로 categorical 한 값입니다(ex. 서울 : 1, 경기 : 2, ... ).

그렇기 때문에 이번에 linear regression을 할 때는 car name, origin 값은 제외하고 생각하도록 하겠습니다.

```python
y_col = 'mpg'
x_cols = mpg_data.columns.drop(['car name', 'origin', 'mpg'])  # also drop mpg column

print(x_cols)
```

    Index(['cylinders', 'displacement', 'horsepower', 'weight', 'acceleration',
           'model year'],
          dtype='object')

## 1.2 Data exploration

먼저 linear regression을 진행하기 전에 자동차의 정보들과 연비와의 1대1 상관관계를 알아보겠습니다.

시각화해서 보는 것이 가장 직관적으로 이해하기 쉽습니다.

```python
seaborn.pairplot(data=mpg_data, height=5, aspect=1,
             x_vars=x_cols,
             y_vars=y_col);
```

Accerlation과 model_year 의 정보는 양의 상관관계에 있고 나머지는 음의 상관관계에 있습니다.

이러한 상관관계를 통해서 linear model 이 연비를 예측하는데 충분하다는 것을 알 수 있습니다.

## 1.3 Linear model in matrix form

Multiple linear regression 에서 입력 변수가 $d$개라고 하면 우리가 찾아야 하는 model은 다음과 같습니다.
$$\hat{y} = w_0 + w_1x_1 + w_2x_2 + ... + w_dx_d$$
여기서 $\hat{y}$는 우리가 예측한 값입니다. 여기서 편의를 위해 $x_0=1$ 를 추가해서 위 식을 정리하면 다음과 같게 됩니다.
$$\hat{y} = \sum_{i=0}^d w_ix_i = \mathbf{x}^\top \mathbf{w}$$
여기서 $\mathbf{x} = (x_0, x_1, ... , x_d)$로 입력 변수들로 이루어진 벡터이고  $\mathbf{w} = (w_0, w_1, ... , w_d)$로 입력 변수에 곱해지는 계수들로 이루어진 벡터입니다.

이제 우리는 392개의 데이터를 가지고 있습니다. 이를 $N$개라고 생각하고 식을 나열해보면 다음과 같습니다.
$$\hat{y}^{(1)} = w_0x_0^{(1)} + w_1x_1^{(1)} + w_2x_2^{(1)} + ... + w_dx_d^{(1)}$$
$$\hat{y}^{(2)} = w_0x_0^{(2)} + w_1x_1^{(2)} + w_2x_2^{(2)} + ... + w_dx_d^{(2)}$$
$$\vdots$$
$$\hat{y}^{(N)} = w_0x_0^{(N)} + w_1x_1^{(N)} + w_2x_2^{(N)} + ... + w_dx_d^{(N)}$$
이제 최종적으로 위의 식을 한번에 행렬의 형태로 표현하면 다음과 같습니다.

$$\hat{\mathbf{y}} = \begin{bmatrix}
\hat{y}^{(1)}  \\
\hat{y}^{(2)}\\
\vdots  \\
\hat{y}^{(N)}
\end{bmatrix} =
\begin{bmatrix}
x_0^{(1)} &  x_1^{(1)} & \cdots & x_d^{(1)} \\
x_0^{(2)} &  x_1^{(2)} & \cdots & x_d^{(2)} \\
\vdots & \vdots & \ddots & \vdots \\
x_0^{(N)} &  x_1^{(N)} & \cdots & x_d^{(N)}
\end{bmatrix} \begin{bmatrix}
w_0  \\
w_1\\
\vdots  \\
w_d
\end{bmatrix}
= \mathbf{X}\mathbf{w}$$

여기서 $\mathbf{x}$ 는 입력 변수들을 행렬의 형태로 나타낸 것입니다. 그리고 맨 앞에 $x_0 = 1$을 추가했었기 때문에 $\mathbf{X}$의 맨 왼쪽 행렬은 1로 이루어져 있습니다.

그리고 $\hat{\mathbf{y}}$는 예측한 변수들을 모아놓은 벡터이고, $\mathbf{y}$는 실제 연비들을 모아놓은 벡터입니다.

이제 이것들을 코드로 표현해 보겠습니다.

```python
from autograd import numpy
from autograd import grad
```

```python
X = mpg_data[x_cols].values
X = numpy.hstack((numpy.ones((X.shape[0], 1)), X))  # pad 1s to the left of input matrix
y = mpg_data[y_col].values

print("X.shape = {}, y.shape = {}".format(X.shape, y.shape))
```

    X.shape = (392, 7), y.shape = (392,)

이제 cost function을 정의해보겠습니다. [HW10]에서 했던 것처럼 mean squared error 를 사용하겠습니다.

$$L(\mathbf{y}, \hat{\mathbf{y}}) = \frac{1}{N} \sum_{i=1}^N (y^{(i)} - \hat{y}^{(i)})^2$$

Cost function과 우리의 linear regression model을 코드로 나타내면 다음과 같습니다.

```python
def linear_regression(params, X):
    '''
    The linear regression model in matrix form.
    Arguments:
      params: 1D array of weights for the linear model
      X     : 2D array of input values
    Returns:
      1D array of predicted values
    '''
    return numpy.dot(X, params)

def cost_function(params, model, X, y):
    '''
    The mean squared error loss function.
    Arguments:
      params: 1D array of weights for the linear model
      model : function for the linear regression model
      X     : 2D array of input values
      y     : 1D array of predicted values
    Returns:
      float, mean squared error
    '''
    y_pred = model(params, X)
    return numpy.mean( numpy.sum((y-y_pred)**2) )
```

## 1.4 Find the weights using gradient descent

이제 Gradient descent로 cost function을 최소로 해주는 계수를 찾아보겠습니다. autograd.grad()함수로 기울기를 구해서 사용하겠습니다.

```python
gradient = grad(cost_function)
```

기울기값이 잘 구해지는지 랜덤한 값을 통해 알아보겠습니다.

```python
gradient(numpy.random.rand(X.shape[1]), linear_regression, X, y)
```

    array([5.83724933e+05, 3.50080698e+06, 1.33120950e+08, 6.76064890e+07,
           1.90019610e+09, 8.82082454e+06, 4.41098533e+07])

기울기 값이 매우 크게 나오는 것을 알 수 있습니다. 한번 gradient descent를 진행해보겠습니다.

```python
max_iter = 30
alpha = 0.001
params = numpy.zeros(X.shape[1])

for i in range(max_iter):
    descent = gradient(params, linear_regression, X, y)
    params = params - descent * alpha
    loss = cost_function(params, linear_regression, X, y)
    if i%5 == 0:
        print("iteration {}, loss = {}".format(i, loss))
```

    iteration 0, loss = 9.668521029386564e+18
    iteration 5, loss = 5.933187473385046e+87
    iteration 10, loss = 3.64096169342092e+156
    iteration 15, loss = 2.2343136994110975e+225
    iteration 20, loss = 1.3711096484197387e+294
    iteration 25, loss = inf

    /usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:24: RuntimeWarning: overflow encountered in square
    /usr/local/lib/python3.7/dist-packages/autograd/tracer.py:48: RuntimeWarning: overflow encountered in power
      return f_raw(*args, **kwargs)

loss가 무한대를 넘어가서 오류가 발생한 것을 알 수 있습니다. 이런 일이 왜 일어난 것일까요?

## 1.5 Feature scaling

Gradient descent를 진행했더니 loss가 무한대로 발산했습니다. 이것은 입력 변수들 중에 특정 값들이 너무 커서 일어난 일입니다. 입력 데이터들의 max 와 min 값을 한번 출력해보겠습니다.

```python
mpg_data[x_cols].describe().loc[['max', 'min']]
```

      cylinders
      displacement
      horsepower
      weight
      acceleration
      model year

      max
      8.0
      455.0
      230.0
      5140.0
      24.8
      82.0

      min
      3.0
      68.0
      46.0
      1613.0
      8.0
      70.0

      <button class="colab-df-convert" onclick="convertToInteractive('df-2f3752d5-acc8-4971-80cd-4b85de262f0f')"
              title="Convert this dataframe to an interactive table."
              style="display:none;">

  <svg xmlns="http://www.w3.org/2000/svg" height="24px"viewBox="0 0 24 24"
       width="24px">
    <path d="M0 0h24v24H0V0z" fill="none"/>
    <path d="M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z"/><path d="M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z"/>
  </svg>
      </button>

weight 값을 보면 다른 값들에 비해 매우 큰 것을 알 수 있습니다. 그래서 우리는 값들을 비슷한 크기를 가지도록 바꿔줄 필요가 있습니다. 이번에 사용할 방법은 min-max scaling으로 모든 데이터의 범위를 최대 1, 최소 0으로 맞춰줍니다. 변환하는 식은 다음과 같습니다.

$$x' = \frac{x - \min(x)}{\max(x)-\min(x)}$$

우리는 이 normalization을 scikit-learn이라는 패키지를 이용해서 하도록 하겠습니다.

```python
from sklearn.preprocessing import MinMaxScaler

min_max_scaler = MinMaxScaler()
X_scaled = min_max_scaler.fit_transform(mpg_data[x_cols])
X_scaled = numpy.hstack((numpy.ones((X_scaled.shape[0], 1)), X_scaled))

pandas.DataFrame(X_scaled).describe().loc[['max', 'min']]
```

      0
      1
      2
      3
      4
      5
      6

      max
      1.0
      1.0
      1.0
      1.0
      1.0
      1.0
      1.0

      min
      1.0
      0.0
      0.0
      0.0
      0.0
      0.0
      0.0

      <button class="colab-df-convert" onclick="convertToInteractive('df-f44ebcb1-bd0a-48d1-b8c9-a8e392696e6a')"
              title="Convert this dataframe to an interactive table."
              style="display:none;">

  <svg xmlns="http://www.w3.org/2000/svg" height="24px"viewBox="0 0 24 24"
       width="24px">
    <path d="M0 0h24v24H0V0z" fill="none"/>
    <path d="M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z"/><path d="M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z"/>
  </svg>
      </button>

0번째 행은 처음에 1을 추가해준 행이므로 1로 유지되는게 맞습니다. 이제 변화된 데이터로 다시 gradient descent를 진행해 보겠습니다.

```python
max_iter = 1000
alpha = 0.001
params = numpy.zeros(X.shape[1])

for i in range(max_iter):
    descent = gradient(params, linear_regression, X_scaled, y)
    params = params - descent * alpha
    loss = cost_function(params, linear_regression, X_scaled, y)
    if i%100 == 0:
        print("iteration {}, loss = {}".format(i, loss))
```

    iteration 0, loss = 116081.11124253299
    iteration 100, loss = 5035.737208606848
    iteration 200, loss = 4694.653830594688
    iteration 300, loss = 4613.631473760375
    iteration 400, loss = 4583.327630964672
    iteration 500, loss = 4568.155268087917
    iteration 600, loss = 4559.268593877387
    iteration 700, loss = 4553.692765631068
    iteration 800, loss = 4550.09985725416
    iteration 900, loss = 4547.761957132873

params가 학습 완료되었습니다. 학습된 params는 다음과 같고, 우리는 예측 값을 params와 $\mathbf{X}$의 곱으로 나타낼 수 있습니다.

```python
params
```

    array([ 27.56342896,  -1.24000064,   1.70706502,  -0.2853682 ,
           -23.08358717,   1.06664226,   8.99271696])

```python
y_pred_gd = X_scaled @ params
```

## 1.6 How accurate is the model?

이제 우리가 만든 모델이 얼마나 정확한지 알아보도록 하겠습니다. Regression 문제에서는 주로 두개의 기본 지표가 있습니다. Mean absolute error(MAE)와 root mean squared error(RMSE)입니다. 두개의 식은 다음과 같습니다.

$$\text{MAE}(\mathbf{y}, \hat{\mathbf{y}}) = \frac{1}{N} \sum_{i=1}^N \left\vert y^{(i)} - \hat{y}^{(i)}\right\vert$$
$$\text{RMSE}(\mathbf{y}, \hat{\mathbf{y}}) =\sqrt{ \frac{1}{N} \sum_{i=1}^N (y^{(i)} - \hat{y}^{(i)})^2}$$

이 지표들도 scikit-learn 패키지를 통해서 사용해보겠습니다.

```python
from sklearn.metrics import mean_absolute_error, mean_squared_error

mae = mean_absolute_error(y, y_pred_gd)
rmse = mean_squared_error(y, y_pred_gd, squared=False)
print("mae  = {}".format(mae))
print("rmse = {}".format(rmse))
```

    mae  = 2.613991601156043
    rmse = 3.40552056741184

## 2. Gradient descent land global temperature anomaly dataset

먼저 어제 사용한 Dataset을 다시 불러오겠습니다.

```python
from urllib.request import urlretrieve
URL = 'http://go.gwu.edu/engcomp1data5?accessType=DOWNLOAD'
urlretrieve(URL, 'land_global_temperature_anomaly-1880-2016.csv')
```

    ('land_global_temperature_anomaly-1880-2016.csv',
     <http.client.HTTPMessage at 0x7fe8bf43e3d0>)

```python
import numpy
import sympy
```

```python
fname = '/content/land_global_temperature_anomaly-1880-2016.csv'

year, temp_anomaly = numpy.loadtxt(fname, delimiter=',', skiprows=5, unpack=True)
```

```python
from matplotlib import pyplot
%matplotlib inline

pyplot.rc('font', family='serif', size='18')

#You can set the size of the figure by doing:
pyplot.figure(figsize=(10,5))

#Plotting
pyplot.plot(year, temp_anomaly, color='#2929a3', linestyle='-', linewidth=1)
pyplot.title('Land global temperature anomalies. \n')
pyplot.xlabel('Year')
pyplot.ylabel('Land temperature anomaly [°C]')
pyplot.grid();
```

어제는 직접 계산을 하는 방법으로 linear regression을 진행하였습니다. 이번에는 gradient descent 방법으로 해보겠습니다.

Linear regression 를 하기 위해서 먼저 직선을 정의합니다.

$$f(x_i) = wx + b$$

그 다음 수업 시간에 배운 cost function을 정의하도록 하겠습니다. 우리가 최소화 해야 할 cost function은 다음과 같습니다.

$$\frac{1}{n} \sum_{i=1}^n (y_i - f(x_i))^2 = \frac{1}{n} \sum_{i=1}^n (y_i - (wx_i + b))^2$$

이제 코드를 통해 접근해 보도록 하겠습니다.

먼저 cost function을 정의해보겠습니다.

```python
w, b, x, y = sympy.symbols('w b x y')

cost_function = (w*x + b - y)**2
cost_function
```

$\displaystyle \left(b + w x - y\right)^{2}$

```python
grad_b = sympy.lambdify([w,b,x,y], cost_function.diff(b), 'numpy')
grad_w = sympy.lambdify([w,b,x,y], cost_function.diff(w), 'numpy')
```

```python
w = 0
b = 0

for i in range(1000):
    descent_b = numpy.sum(grad_b(w,b,year,temp_anomaly))/len(year)
    descent_w = numpy.sum(grad_w(w,b,year,temp_anomaly))/len(year)
    w = w - descent_w*0.001 # with 0.01 the step size
    b = b - descent_b*0.001

print(w)
print(b)
```

    nan
    nan

    <string>:2: RuntimeWarning: overflow encountered in multiply
    /usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:7: RuntimeWarning: invalid value encountered in double_scalars
      import sys

이번에도 발산하였습니다. 이것도 위의 예시와 같이 우리가 예측해야할 값의 범위와 우리의 입력 변수의 값의 범위가 많이 차이나서 일어난 일입니다. 그래서 normalization을 진행하여 다시 해보겠습니다. 이번에는 Z-Score normalization을 사용하겠습니다. 식은 다음과 같습니다.

$$x' = \frac{x - \text{mean}(x)}{\text{std}(x)}$$

여기서 mean 은 평균이고 std는 표준 편차입니다. 코드로 표현해보겠습니다.

```python
year_scaled = (year - year.mean())/year.std()
temp_anomaly_scaled = (temp_anomaly - temp_anomaly.mean()) / temp_anomaly.std()
```

이게 다시 gradient descent 를 진행해보겠습니다.

```python
w = 0
b = 0

for i in range(1000):
    descent_b = numpy.sum(grad_b(w,b,year_scaled,temp_anomaly_scaled))/len(year_scaled)
    descent_w = numpy.sum(grad_w(w,b,year_scaled,temp_anomaly_scaled))/len(year_scaled)
    w = w - descent_w*0.001 # with 0.01 the step size
    b = b - descent_b*0.001

print(w)
print(b)
```

    0.756766964479979
    5.551763428541569e-17

$w$ 와 $b$ 를 모두 구했습니다. 이제 그래프로 표현해보겠습니다.

```python
reg = b + w * year_scaled
```

```python
pyplot.figure(figsize=(10, 5))

pyplot.plot(year_scaled, temp_anomaly_scaled, color='#2929a3', linestyle='-', linewidth=1, alpha=0.5)
pyplot.plot(year_scaled, reg, 'k--', linewidth=2, label='Linear regression')
pyplot.xlabel('Year')
pyplot.ylabel('Land temperature anomaly [°C]')
pyplot.legend(loc='best', fontsize=15)
pyplot.grid();
```

오늘은 multiple linear regression을 실습을 통해서 배워보았습니다. Feature scaling(normalization)의 중요성을 실습을 진행하면서 알아보고 개선해보았습니다.

수고하셨습니다~ 질문 있으면 편하게 해주세요~

```python

```


```python
import numpy as np
import sympy as sp
import scipy.linalg as la
from time import time
from numba import jit
```

```python
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import axes3d
```

```python
a = sp.Symbol('a')
lat = np.array([
    [0, a, a],
    [a, 0, a],
    [a, a, 0]])
nlat = (lat/a).astype(int)
lat
```

```python
# inverse lattice
nilat = la.inv(nlat)
ilat = 1/a * nilat
```

```python
sm = 2*np.array([
    [-1, 2, 2],
    [2, -1, 2],
    [2, 2, -1]
])
print(sm)
Ntarget = int(la.det(sm))
print(Ntarget)
```

```python
# superlattice
slat = sm.dot(lat)
nslat = (slat / a).astype(int)
print(slat[:, :])
```

```python
# inverse superlattice
nislat = la.inv(nslat)
islat = nislat / a
islat, nislat
```

#### Fractional coordinates

```python
rf1 = np.array([0, 0, 0])
rf2 = np.array([1/4, 1/4, 1/4])
```

#### Cartesian coordinates

```python
rc1 = rf1.dot(lat)
rc2 = rf2.dot(lat)
rc2
```

```python
sm.dot(lat[:, 0])
```

```python
sm.dot(rc2)
```

### Algorithm
1. Construct lattice points
2. Check if lattice point lies within supercell
3. Check if lattice point was found. before, if yes, discard.

#### Criterion for inside/outside the cell
1. Find fractional coord.s of atoms in new basis vectors,
2. discard atoms that have fractional components with
$$ \tilde r_i < 0 ,$$
or
$$ \tilde r_i \geq 1 $$

```python
# collect atomic positions:
R = [rc1, rc2]
```

```python
new_atoms = []
new_atoms_frac = []
```

```python
# generate new atoms and append if they lie in the supercell
eps = 1e-5

@jit(nopython=True)
def inlist(elem, lis, tol=1e-5):
    for ii in range(len(lis)):
        ll = lis[ii]
        # Check length
        na = np.linalg.norm(elem)
        nb = np.linalg.norm(ll)
        if abs(na - nb)  > tol:
            continue
        # Check for zero vector
        if abs(na) + abs(nb) < tol:
            return True
        # Check angle
        if abs(np.dot(ll, elem) / na / nb - 1) < tol:
            return True
    #
    # if nothing found:
    return False

def bruteforce(basis, lat, smatrix, checklist=False, verbose=False, tol=1e-5):
    # first find lattice points
    # List of lattice points:

    slat = np.dot(smatrix, lat)
    islat = la.inv(slat).T

    max1 = abs(smatrix[0, :]).max()*2 + 1
    max2 = abs(smatrix[1, :]).max()*2 + 1
    max3 = abs(smatrix[2, :]).max()*2 + 1

    # maximal distance = diagonal
    dmax = np.linalg.norm(slat[:, 0] + slat[:, 1] + slat[:, 2]) + 2
    print(dmax)

    lps = -np.ones((Ntarget, 3), dtype=float)
    flps = -np.ones((Ntarget, 3), dtype=float)

    counter = 0
    for n1 in range(-max1, max1):
        if verbose: print('Progress = {}'.format((n1 + max1) / (2*max1)))
        for n2 in range(-max2, max2):
            for n3 in range(-max3, max3):
                # lattice point
                # print(n1, n2, n3)
                lp = (n1*lat[0] + n2*lat[1] + n3*lat[2]).astype(float)

                # Check if this one is relevant
                if np.linalg.norm(lp) > dmax:
                    continue
                #
                # Check if is in list
                if checklist:
                    if counter > 1:
                        if inlist(lp, lps[:counter]):
                            print('lp: {} already in list'.format(lp))
                            continue

                # fractional lattice point
                flp = (islat.dot(lp)).astype(float)
                # Check if is inside supercell:
                if any(np.array(flp) < -tol):
                    # print(f'{flp} discarded < 0')
                    continue
                if any(np.array(flp) > 1 - tol):
                    #print(f'{flp} discarded > 1')
                    continue
                #
                # if passed:
                # print('{} accepted'.format(flp))
                lps[counter] = lp
                flps[counter] = flp
                counter += 1
    #
    print('{} lattice points found'.format(counter))
    #print('{} of them within supercell'.format(len(flps)))
    return lps, flps, counter
```

```python
print(nlat, nslat)
stime = time()
lps, flps, counter = bruteforce(R, nlat, sm)
print(f'Finished in {time()-stime:7.2f}s')
```

```python
lps
```

```python
#new_atoms_frac
```

```python
# check if number of atom is correct
print(f'Number of atoms in the supercell is')
print(f'  {int(la.det(sm))} = {len(lps)}')
if int(la.det(sm) == len(lps)):
    print(f'..correct.')
else:
    print(f'..wrong!')
```

## Plot
Plot the old and new cells with positions

```python
# old positions as number
nR = [(pos / a).astype(float) for pos in R]
```

```python
# new positions as numbers
nsR = [(pos).astype(float) for pos in lps]
```

```python
%matplotlib notebook
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
for ii in range(3):
    X = np.zeros(3)
    Y = np.zeros(3)
    Z = np.zeros(3)
    V1 = nlat[ii,0]
    V2 = nlat[ii,1]
    V3 = nlat[ii,2]
    ax.quiver(X,Y,Z,V1,V2,V3)

# Plot old positions:
#xcpos = [pos[0] for pos in nR]
#ycpos = [pos[1] for pos in nR]
#zcpos = [pos[2] for pos in nR]
#ax.scatter(xcpos, ycpos, zcpos)

# Draw supercell boundary
for ii in range(3):
    X = np.zeros(3)
    Y = np.zeros(3)
    Z = np.zeros(3)
    V1 = nslat[ii,0]
    V2 = nslat[ii,1]
    V3 = nslat[ii,2]
    ax.quiver(X,Y,Z,V1,V2,V3, color='k', arrow_length_ratio=0)
    for jj in range(2):
        X2 = nslat[ii,0]
        Y2 = nslat[ii,1]
        Z2 = nslat[ii,2]
        V21 = nslat[(ii+1+jj) % 3,0]
        V22 = nslat[(ii+1+jj) % 3,1]
        V23 = nslat[(ii+1+jj) % 3,2]
        ax.quiver(X2,Y2,Z2,V21,V22,V23, color='k', arrow_length_ratio=0)
    for jj in range(1):
        X3 = nslat[ii,0] + nslat[(ii+1+jj)%3,0]
        Y3 = nslat[ii,1] + nslat[(ii+1+jj)%3,1]
        Z3 = nslat[ii,2] + nslat[(ii+1+jj)%3,2]
        V31 = nslat[(ii+2+jj) % 3,0]
        V32 = nslat[(ii+2+jj) % 3,1]
        V33 = nslat[(ii+2+jj) % 3,2]
        ax.quiver(X3,Y3,Z3,V31,V32,V33, color='k', arrow_length_ratio=0)

# Plot new positions:
xcspos = [pos[0] for pos in nsR]
ycspos = [pos[1] for pos in nsR]
zcspos = [pos[2] for pos in nsR]
ax.scatter(xcspos, ycspos, zcspos)

# Limits
lims = nslat[0, :] + nslat[1, :] + nslat[2, :]
ax.set_xlim([0, lims[0]])
ax.set_ylim([0, lims[1]])
ax.set_zlim([0, lims[2]])
plt.show()
#for angle in range(0, 360):
#    ax.view_init(30, angle)
#    plt.draw()
#    plt.pause(.001)
```

```python

```

```python

```

```python

```