# Neuromatch Academy: Week 3, Day 1, Tutorial 2
# Real Neurons: Effects of Input Correlation
__Content creators:__ Qinglong Gu, Songtin Li, John Murray, Richard Naud, Arvind Kumar

__Content reviewers:__ Maryam Vaziri-Pashkam, Ella Batty, Lorenzo Fontolan, Richard Gao, Matthew Krause, Spiros Chavlis, Michael Waskom

---
# Tutorial Objectives
In this tutorial, we will use the leaky integrate-and-fire (LIF) neuron model (see Tutorial 1) to study how they transform input correlations to output properties (transfer of correlations). In particular, we are going to write a few lines of code to:

- inject correlated GWN in a pair of neurons

- measure correlations between the spiking activity of the two neurons

- study how the transfer of correlation depends on the statistics of the input, i.e. mean and standard deviation.

---
# Setup

```python
# Import libraries
import matplotlib.pyplot as plt
import numpy as np
import time
```

```python
# @title Figure Settings
import ipywidgets as widgets       # interactive display
%config InlineBackend.figure_format = 'retina'
# use NMA plot style
plt.style.use("/share/dataset/COMMON/nma.mplstyle.txt")
my_layout = widgets.Layout()
```

```python
# @title Helper functions
def default_pars(**kwargs):
  pars = {}

  ### typical neuron parameters###
  pars['V_th'] = -55.     # spike threshold [mV]
  pars['V_reset'] = -75.  # reset potential [mV]
  pars['tau_m'] = 10.     # membrane time constant [ms]
  pars['g_L'] = 10.       # leak conductance [nS]
  pars['V_init'] = -75.   # initial potential [mV]
  pars['V_L'] = -75.      # leak reversal potential [mV]
  pars['tref'] = 2.       # refractory time (ms)

  ### simulation parameters ###
  pars['T'] = 400. # Total duration of simulation [ms]
  pars['dt'] = .1  # Simulation time step [ms]

  ### external parameters if any ###
  for k in kwargs:
    pars[k] = kwargs[k]

  pars['range_t'] = np.arange(0, pars['T'], pars['dt'])  # Vector of discretized
                                                         # time points [ms]
  return pars

def run_LIF(pars, Iinj):
  """
  Simulate the LIF dynamics with external input current

  Args:
    pars       : parameter dictionary
    Iinj       : input current [pA]. The injected current here can be a value or an array

  Returns:
    rec_spikes : spike times
    rec_v      : mebrane potential
  """

  # Set parameters
  V_th, V_reset = pars['V_th'], pars['V_reset']
  tau_m, g_L = pars['tau_m'], pars['g_L']
  V_init, V_L = pars['V_init'], pars['V_L']
  dt, range_t = pars['dt'], pars['range_t']
  Lt = range_t.size
  tref = pars['tref']

  # Initialize voltage and current
  v = np.zeros(Lt)
  v[0] = V_init
  Iinj = Iinj * np.ones(Lt)
  tr = 0.

  # simulate the LIF dynamics
  rec_spikes = []   # record spike times
  for it in range(Lt - 1):
    if tr > 0:
      v[it] = V_reset
      tr = tr - 1
    elif v[it] >= V_th:  # reset voltage and record spike event
      rec_spikes.append(it)
      v[it] = V_reset
      tr = tref / dt

    # calculate the increment of the membrane potential
    dv = (-(v[it] - V_L) + Iinj[it] / g_L) * (dt / tau_m)

    # update the membrane potential
    v[it + 1] = v[it] + dv

  rec_spikes = np.array(rec_spikes) * dt

  return v, rec_spikes

def my_GWN(pars, sig, myseed=False):
  """
  Function that calculates Gaussian white noise inputs

  Args:
    pars       : parameter dictionary
    mu         : noise baseline (mean)
    sig        : noise amplitute (standard deviation)
    myseed     : random seed. int or boolean
                 the same seed will give the same random number sequence

  Returns:
    I          : Gaussian white noise input
  """

  # Retrieve simulation parameters
  dt, range_t = pars['dt'], pars['range_t']
  Lt = range_t.size

  # Set random seed. You can fix the seed of the random number generator so
  # that the results are reliable however, when you want to generate multiple
  # realization make sure that you change the seed for each new realization
  if myseed:
      np.random.seed(seed=myseed)
  else:
      np.random.seed()

  # generate GWN
  # we divide here by 1000 to convert units to sec.
  I_GWN = sig * np.random.randn(Lt) * np.sqrt(pars['tau_m'] / dt)

  return I_GWN

def Poisson_generator(pars, rate, n, myseed=False):
  """
  Generates poisson trains

  Args:
    pars       : parameter dictionary
    rate       : noise amplitute [Hz]
    n          : number of Poisson trains
    myseed     : random seed. int or boolean

  Returns:
    pre_spike_train : spike train matrix, ith row represents whether
                      there is a spike in ith spike train over time
                      (1 if spike, 0 otherwise)
  """

  # Retrieve simulation parameters
  dt, range_t = pars['dt'], pars['range_t']
  Lt = range_t.size

  # set random seed
  if myseed:
      np.random.seed(seed=myseed)
  else:
      np.random.seed()

  # generate uniformly distributed random variables
  u_rand = np.random.rand(n, Lt)

  # generate Poisson train
  poisson_train = 1. * (u_rand < rate * (dt / 1000.))

  return poisson_train

def example_plot_myCC():
  pars = default_pars(T=50000, dt=.1)

  c = np.arange(10) * 0.1
  r12 = np.zeros(10)
  for i in range(10):
    I1gL, I2gL = correlate_input(pars, mu=20.0, sig=7.5, c=c[i])
    r12[i] = my_CC(I1gL, I2gL)

  plt.figure()
  plt.plot(c, r12, 'bo', alpha=0.7, label='Simulation', zorder=2)
  plt.plot([-0.05, 0.95], [-0.05, 0.95], 'k--', label='y=x',
           dashes=(2, 2), zorder=1)
  plt.xlabel('True CC')
  plt.ylabel('Sample CC')
  plt.legend(loc='best')

def LIF_output_cc(pars, mu, sig, c, bin_size, n_trials=20):
  """ Simulates two LIF neurons with correlated input and computes output correlation

  Args:
  pars       : parameter dictionary
  mu         : noise baseline (mean)
  sig        : noise amplitute (standard deviation)
  c          : correlation coefficient ~[0, 1]
  bin_size   : bin size used for time series
  n_trials   : total simulation trials

  Returns:
  r          : output corr. coe.
  sp_rate    : spike rate
  sp1        : spike times of neuron 1 in the last trial
  sp2        : spike times of neuron 2 in the last trial
  """

  r12 = np.zeros(n_trials)
  sp_rate = np.zeros(n_trials)
  for i_trial in range(n_trials):
    I1gL, I2gL = correlate_input(pars, mu, sig, c)
    _, sp1 = run_LIF(pars, pars['g_L'] * I1gL)
    _, sp2 = run_LIF(pars, pars['g_L'] * I2gL)

    my_bin = np.arange(0, pars['T'], bin_size)

    sp1_count, _ = np.histogram(sp1, bins=my_bin)
    sp2_count, _ = np.histogram(sp2, bins=my_bin)

    r12[i_trial] = my_CC(sp1_count[::20], sp2_count[::20])
    sp_rate[i_trial] = len(sp1) / pars['T'] * 1000.

  return r12.mean(), sp_rate.mean(), sp1, sp2

def plot_c_r_LIF(c, r, mycolor, mylabel):
  z = np.polyfit(c, r, deg=1)
  c_range = np.array([c.min() - 0.05, c.max() + 0.05])
  plt.plot(c, r, 'o', color=mycolor, alpha=0.7, label=mylabel, zorder=2)
  plt.plot(c_range, z[0] * c_range + z[1], color=mycolor, zorder=1)

```

The helper function contains the:

- Parameter dictionary: `default_pars( **kwargs)`
- LIF simulator: `run_LIF`
- Gaussian white noise generator: `my_GWN(pars, sig, myseed=False)`
- Poisson type spike train generator: `Poisson_generator(pars, rate, n, myseed=False)`
- Two LIF neurons with correlated inputs simulator: `LIF_output_cc(pars, mu, sig, c, bin_size, n_trials=20)`
- Some additional plotting utilities

---
# Section 1: Correlations (Synchrony)
Correlation or synchrony in neuronal activity can be described for any readout of brain activity. Here, we are concerned with the spiking activity of neurons.

In the simplest way, correlation/synchrony refers to coincident spiking of neurons, i.e., when two neurons spike together, they are firing in **synchrony** or are **correlated**. Neurons can be synchronous in their instantaneous activity, i.e., they spike together with some probability. However, it is also possible that spiking of a neuron at time $t$ is correlated with the spikes of another neuron with a delay (time-delayed synchrony).

## Origin of synchronous neuronal activity:
- Common inputs, i.e., two neurons are receiving input from the same sources. The degree of correlation of the shared inputs is proportional to their output correlation.
- Pooling from the same sources. Neurons do not share the same input neurons but are receiving inputs from neurons which themselves are correlated.
- Neurons are connected to each other (uni- or bi-directionally): This will only give rise to time-delayed synchrony. Neurons could also be connected via gap-junctions.
- Neurons have similar parameters and initial conditions.

## Implications of synchrony
When neurons spike together, they can have a stronger impact on downstream neurons. Synapses in the brain are sensitive to the temporal correlations (i.e., delay) between pre- and postsynaptic activity, and this, in turn, can lead to the formation of functional neuronal networks - the basis of unsupervised learning (we will study some of these concepts in a forthcoming tutorial).

Synchrony implies a reduction in the dimensionality of the system. In addition, correlations, in many cases, can impair the decoding of neuronal activity.

```python
# @title Video 1: Input & output correlations
from IPython.display import IFrame
class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
        self.id=id
        src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
        super(BiliVideo, self).__init__(src, width, height, **kwargs)

video = BiliVideo(id='BV1Bh411o7eV', width=854, height=480, fs=1)
print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
video
```

    Video available at https://www.bilibili.com/video/BV1Bh411o7eV

## How to study the emergence of correlations

A simple model to study the emergence of correlations is to inject common inputs to a pair of neurons and measure the output correlation as a function of the fraction of common inputs.

Here, we are going to investigate the transfer of correlations by computing the correlation coefficient of spike trains recorded from two unconnected LIF neurons, which received correlated inputs.

The input current to LIF neuron $i$ $(i=1,2)$ is:

\begin{equation}
\frac{I_i}{g_L} =\mu_i + \sigma_i (\sqrt{1-c}\xi_i + \sqrt{c}\xi_c) \quad (1)
\end{equation}

where $\mu_i$ is the temporal average of the current. The Gaussian white noise $\xi_i$ is independent for each neuron, while $\xi_c$ is common to all neurons. The variable $c$ ($0\le c\le1$) controls the fraction of common and independent inputs. $\sigma_i$ shows the variance of the total input.

So, first, we will generate correlated inputs.

```python
# @title
#@markdown Execute this cell to get a function for generating correlated GWN inputs
def correlate_input(pars, mu=20., sig=7.5, c=0.3):
  """
  Args:
    pars       : parameter dictionary
    mu         : noise baseline (mean)
    sig        : noise amplitute (standard deviation)
    c.         : correlation coefficient ~[0, 1]

  Returns:
    I1gL, I2gL : two correlated inputs with corr. coe. c
  """

  # generate Gaussian whute noise xi_1, xi_2, xi_c
  xi_1 = my_GWN(pars, sig)
  xi_2 = my_GWN(pars, sig)
  xi_c = my_GWN(pars, sig)

  # Generate two correlated inputs by Equation. (1)
  I1gL = mu + np.sqrt(1. - c) * xi_1 + np.sqrt(c) * xi_c
  I2gL = mu + np.sqrt(1. - c) * xi_2 + np.sqrt(c) * xi_c

  return I1gL, I2gL

print(help(correlate_input))
```

    Help on function correlate_input in module __main__:

    correlate_input(pars, mu=20.0, sig=7.5, c=0.3)
        Args:
          pars       : parameter dictionary
          mu         : noise baseline (mean)
          sig        : noise amplitute (standard deviation)
          c.         : correlation coefficient ~[0, 1]

        Returns:
          I1gL, I2gL : two correlated inputs with corr. coe. c

    None

### Exercise 1: Compute the correlation

The _sample correlation coefficient_ between two input currents $I_i$ and $I_j$ is defined as the sample covariance of $I_i$ and $I_j$ divided by the square root of the sample variance of $I_i$ multiplied with the square root of the sample variance of $I_j$. In equation form:

\begin{align}
r_{ij} &= \frac{cov(I_i, I_j)}{\sqrt{var(I_i)} \sqrt{var(I_j)}}\\
cov(I_i, I_j) &= \sum_{k=1}^L (I_i^k -\bar{I}_i)(I_j^k -\bar{I}_j) \\
var(I_i) &= \sum_{k=1}^L (I_i^k -\bar{I}_i)^2
\end{align}

where $\bar{I}_i$ is the sample mean, k is the time bin, and L is the length of $I$.  This means that $I_i^k$ is current i at time $k\cdot dt$. Note that the equations above are not accurate for sample covariances and variances as they should be additionally divided by L-1 - we have dropped this term because it cancels out in the sample correlation coefficient formula.

The _sample correlation coefficient_ may also be referred to as the _sample Pearson correlation coefficient_. Here, is a beautiful paper that explains multiple ways to calculate and understand correlations [Rodgers and Nicewander 1988](https://www.stat.berkeley.edu/~rabbee/correlation.pdf).

In this exercise, we will create a function, `my_CC` to compute the sample correlation coefficient between two time series. Note that while we introduced this computation here in the context of input currents, the sample correlation coefficient is used to compute the correlation between any two time series - we will use it later on binned spike trains.

```python
def my_CC(i, j):
  """
  Args:
    i, j  : two time series with the same length

  Returns:
    rij   : correlation coefficient
  """
  ########################################################################
  ## TODO for students: compute rxy, then remove the NotImplementedError #
  # Tip1: array([a1, a2, a3])*array([b1, b2, b3]) = array([a1*b1, a2*b2, a3*b3])
  # Tip2: np.sum(array([a1, a2, a3])) = a1+a2+a3
  # Tip3: square root, np.sqrt()
  # Fill out function and remove
  raise NotImplementedError("Student exercise: compute the sample correlation coefficient")
  ########################################################################
  # Calculate the covariance of i and j
  cov = ...
  # Calculate the variance of i
  var_i = ...
  # Calculate the variance of j
  var_j = ...
  # Calculate the correlation coefficient
  rij = ...

  return rij

# Uncomment the line after completing the my_CC function
# example_plot_myCC()
```

[*Click for solution*](https://github.com/erlichlab/course-content/tree/master//tutorials/W3D1_RealNeurons/solutions/W3D1_Tutorial2_Solution_03e44bdc.py)

*Example output:*

### Exercise 2: Measure the correlation between spike trains

After recording the spike times of the two neurons, how can we estimate their correlation coefficient?

In order to find this, we need to bin the spike times and obtain two time series. Each data point in the time series is the number of spikes in the corresponding time bin. You can use `np.histogram()` to bin the spike times.

Complete the code below to bin the spike times and calculate the correlation coefficient for two Poisson spike trains. Note that `c` here is the ground-truth correlation coefficient that we define.

```python
# @title

# @markdown Execute this cell to get a function for generating correlated Poisson inputs (generate_corr_Poisson)

def generate_corr_Poisson(pars, poi_rate, c, myseed=False):
  """
  function to generate correlated Poisson type spike trains
  Args:
    pars       : parameter dictionary
    poi_rate   : rate of the Poisson train
    c.         : correlation coefficient ~[0, 1]

  Returns:
    sp1, sp2   : two correlated spike time trains with corr. coe. c
  """

  range_t = pars['range_t']

  mother_rate = poi_rate / c
  mother_spike_train = Poisson_generator(pars, rate=mother_rate,
                                         n=1, myseed=myseed)[0]
  sp_mother = range_t[mother_spike_train > 0]

  L_sp_mother = len(sp_mother)
  sp_mother_id = np.arange(L_sp_mother)
  L_sp_corr = int(L_sp_mother * c)

  np.random.shuffle(sp_mother_id)
  sp1 = np.sort(sp_mother[sp_mother_id[:L_sp_corr]])

  np.random.shuffle(sp_mother_id)
  sp2 = np.sort(sp_mother[sp_mother_id[:L_sp_corr]])

  return sp1, sp2

print(help(generate_corr_Poisson))
```

    Help on function generate_corr_Poisson in module __main__:

    generate_corr_Poisson(pars, poi_rate, c, myseed=False)
        function to generate correlated Poisson type spike trains
        Args:
          pars       : parameter dictionary
          poi_rate   : rate of the Poisson train
          c.         : correlation coefficient ~[0, 1]

        Returns:
          sp1, sp2   : two correlated spike time trains with corr. coe. c

    None

```python
def corr_coeff_pairs(pars, rate, c, trials, bins):
  """
  Calculate the correlation coefficient of two spike trains, for different
  realizations

  Args:
      pars   : parameter dictionary
      rate   : rate of poisson inputs
      c      : correlation coefficient ~ [0, 1]
      trials  : number of realizations
      bins   : vector with bins for time discretization

  Returns:
    r12      : correlation coefficient of a pair of inputs
  """

  r12 = np.zeros(n_trials)

  for i in range(n_trials):
    ##############################################################
    ## TODO for students: Use np.histogram to bin the spike time #
    ##   e.g., sp1_count, _= np.histogram(...)
    # Use my_CC() compute corr coe, compare with c
    # Note that you can run multiple realizations and compute their r_12(diff_trials)
    # with the defined function above. The average r_12 over trials can get close to c.
    # Note: change seed to generate different input per trial
    # Fill out function and remove
    raise NotImplementedError("Student exercise: compute the correlation coefficient")
    ##############################################################

    # Generate correlated Poisson inputs
    sp1, sp2 = generate_corr_Poisson(pars, ..., ..., myseed=2020+i)

    # Bin the spike times of the first input
    sp1_count, _ = np.histogram(..., bins=...)

    # Bin the spike times of the second input
    sp2_count, _ = np.histogram(..., bins=...)

    # Calculate the correlation coefficient
    r12[i] = my_CC(..., ...)

  return r12

poi_rate = 20.
c = 0.2  # set true correlation
pars = default_pars(T=10000)

# bin the spike time
bin_size = 20  # [ms]
my_bin = np.arange(0, pars['T'], bin_size)
n_trials = 100  # 100 realizations

# Uncomment to test your function
# r12 = corr_coeff_pairs(pars, rate=poi_rate, c=c, trials=n_trials, bins=my_bin)
# print(f'True corr coe = {c:.3f}')
# print(f'Simu corr coe = {r12.mean():.3f}')
```

Sample output

```
True corr coe = 0.200
Simu corr coe = 0.197
```

[*Click for solution*](https://github.com/erlichlab/course-content/tree/master//tutorials/W3D1_RealNeurons/solutions/W3D1_Tutorial2_Solution_38477225.py)

---
# Section 2: Investigate the effect of input correlation on the output correlation

Now let's combine the aforementioned two procedures. We first generate the correlated inputs by Equation (1). Then we inject the correlated inputs $I_1, I_2$ into a pair of neurons and record their output spike times. We continue measuring the correlation between the output and
investigate the relationship between the input correlation and the output correlation.

## Drive a neuron with correlated inputs and visualize its output
In the following, you will inject correlated GWN in two neurons. You need to define the mean (`gwn_mean`), standard deviation (`gwn_std`), and input correlations (`c_in`).

We will simulate $10$ trials to get a better estimate of the output correlation. Change the values in the following cell for the above variables (and then run the next cell) to explore how they impact the output correlation.

```python
# Play around with these parameters

pars = default_pars(T=80000, dt=1.)  # get the parameters
c_in = 0.3  # set input correlation value
gwn_mean = 10.
gwn_std = 10.

```

```python
# @title

# @markdown Do not forget to execute this cell to simulate the LIF

bin_size = 10.  # ms

starttime = time.perf_counter()  # time clock

r12_ss, sp_ss, sp1, sp2 = LIF_output_cc(pars, mu=gwn_mean, sig=gwn_std, c=c_in,
                                        bin_size=bin_size, n_trials=10)

# just the time counter
endtime = time.perf_counter()
timecost = (endtime - starttime) / 60.
print(f"Simulation time = {timecost:.2f} min")

print(f"Input correlation = {c_in}")
print(f"Output correlation = {r12_ss}")

plt.figure(figsize=(12, 6))
plt.plot(sp1, np.ones(len(sp1)) * 1, '|', ms=20, label='neuron 1')
plt.plot(sp2, np.ones(len(sp2)) * 1.1, '|', ms=20, label='neuron 2')
plt.xlabel('time (ms)')
plt.ylabel('neuron id.')
plt.xlim(1000, 8000)
plt.ylim(0.9, 1.2)
plt.legend()
plt.show()
```

## Think!
- Is the output correlation always smaller than the input correlation? If yes, why?
- Should there be a systematic relationship between input and output correlations?

You will explore these questions in the next figure but try to develop your own intuitions first!

Lets vary `c_in` and plot the relationship between the `c_in` and output correlation. This might take some time depending on the number of trials.

```python
#@title

#@markdown Don't forget to execute this cell!

pars = default_pars(T=80000, dt=1.)  # get the parameters
bin_size = 10.
c_in = np.arange(0, 1.0, 0.1)  # set the range for input CC
r12_ss = np.zeros(len(c_in))  # small mu, small sigma

starttime = time.perf_counter() # time clock
for ic in range(len(c_in)):
  r12_ss[ic], sp_ss, sp1, sp2 = LIF_output_cc(pars, mu=10.0, sig=10.,
                                              c=c_in[ic], bin_size=bin_size,
                                              n_trials=10)

endtime = time.perf_counter()
timecost = (endtime - starttime) / 60.
print(f"Simulation time = {timecost:.2f} min")

plt.figure(figsize=(7, 6))
plot_c_r_LIF(c_in, r12_ss, mycolor='b', mylabel='Output CC')
plt.plot([c_in.min() - 0.05, c_in.max() + 0.05],
         [c_in.min() - 0.05, c_in.max() + 0.05],
         'k--', dashes=(2, 2), label='y=x')

plt.xlabel('Input CC')
plt.ylabel('Output CC')
plt.legend(loc='best', fontsize=16)
plt.show()
```

[*Click for solution*](https://github.com/erlichlab/course-content/tree/master//tutorials/W3D1_RealNeurons/solutions/W3D1_Tutorial2_Solution_ecc2167a.py)

---
# Section 3: Correlation transfer function
The above plot of input correlation vs. output correlation is called the __correlation transfer function__ of the neurons.

## Section 3.1: How do the mean and standard deviation of the GWN affect the correlation transfer function?

The correlations transfer function appears to be linear. The above can be taken as the input/output transfer function of LIF neurons for correlations, instead of the transfer function for input/output firing rates as we had discussed in the previous tutorial (i.e., F-I curve).

What would you expect to happen to the slope of the correlation transfer function if you vary the mean and/or the standard deviation of the GWN?

```python
#@markdown Execute this cell to visualize correlation transfer functions

pars = default_pars(T=80000, dt=1.) # get the parameters
no_trial = 10
bin_size = 10.
c_in = np.arange(0., 1., 0.2)  # set the range for input CC
r12_ss = np.zeros(len(c_in))   # small mu, small sigma
r12_ls = np.zeros(len(c_in))   # large mu, small sigma
r12_sl = np.zeros(len(c_in))   # small mu, large sigma

starttime = time.perf_counter()  # time clock
for ic in range(len(c_in)):
  r12_ss[ic], sp_ss, sp1, sp2 = LIF_output_cc(pars, mu=10.0, sig=10.,
                                              c=c_in[ic], bin_size=bin_size,
                                              n_trials=no_trial)
  r12_ls[ic], sp_ls, sp1, sp2 = LIF_output_cc(pars, mu=18.0, sig=10.,
                                              c=c_in[ic], bin_size=bin_size,
                                              n_trials=no_trial)
  r12_sl[ic], sp_sl, sp1, sp2 = LIF_output_cc(pars, mu=10.0, sig=20.,
                                              c=c_in[ic], bin_size=bin_size,
                                              n_trials=no_trial)
endtime = time.perf_counter()
timecost = (endtime - starttime) / 60.
print(f"Simulation time = {timecost:.2f} min")

plt.figure(figsize=(7, 6))
plot_c_r_LIF(c_in, r12_ss, mycolor='b', mylabel=r'Small $\mu$, small $\sigma$')
plot_c_r_LIF(c_in, r12_ls, mycolor='y', mylabel=r'Large $\mu$, small $\sigma$')
plot_c_r_LIF(c_in, r12_sl, mycolor='r', mylabel=r'Small $\mu$, large $\sigma$')
plt.plot([c_in.min() - 0.05, c_in.max() + 0.05],
         [c_in.min() - 0.05, c_in.max() + 0.05],
         'k--', dashes=(2, 2), label='y=x')
plt.xlabel('Input CC')
plt.ylabel('Output CC')
plt.legend(loc='best', fontsize=14)
plt.show()
```

### Think!
Why do both the mean and the standard deviation of the GWN affect the slope of the correlation transfer function?

[*Click for solution*](https://github.com/erlichlab/course-content/tree/master//tutorials/W3D1_RealNeurons/solutions/W3D1_Tutorial2_Solution_bb194a8f.py)

## Section 3.2: What is the rationale behind varying $\mu$ and $\sigma$?
The mean and the variance of the synaptic current depends on the spike rate of a Poisson process. We can use [Campbell's theorem](https://en.wikipedia.org/wiki/Campbell%27s_theorem_(probability)) to estimate the mean and the variance of the synaptic current:

\begin{align}
\mu_{\rm syn} = \lambda J \int P(t) \\
\sigma_{\rm syn} = \lambda J \int P(t)^2 dt\\
\end{align}

where $\lambda$ is the firing rate of the Poisson input, $J$ the amplitude of the postsynaptic current and $P(t)$ is the shape of the postsynaptic current as a function of time.

Therefore, when we varied $\mu$ and/or $\sigma$ of the GWN, we mimicked a change in the input firing rate. Note that, if we change the firing rate, both $\mu$ and $\sigma$ will change simultaneously, not independently.

Here, since we observe an effect of $\mu$ and $\sigma$ on correlation transfer, this implies that the input rate has an impact on the correlation transfer function.

### Think!

- What are the factors that would make output correlations smaller than input correlations? (Notice that the colored lines are below the black dashed line)
- What does it mean for the correlation in the network?
- Here we have studied the transfer of correlations by injecting GWN. But in the previous tutorial, we mentioned that GWN is unphysiological. Indeed, neurons receive colored noise (i.e., Shot noise or OU process). How do these results obtained from injection of GWN apply to the case where correlated spiking inputs are injected in the two LIFs? Will the results be the same or different?

Reference
- De La Rocha, Jaime, et al. "Correlation between neural spike trains increases with firing rate." Nature (2007) (https://www.nature.com/articles/nature06028/)

- Bujan AF, Aertsen A, Kumar A. Role of input correlations in shaping the variability and noise correlations of evoked activity in the neocortex. Journal of Neuroscience. 2015 Jun 3;35(22):8611-25. (https://www.jneurosci.org/content/35/22/8611)

[*Click for solution*](https://github.com/erlichlab/course-content/tree/master//tutorials/W3D1_RealNeurons/solutions/W3D1_Tutorial2_Solution_4575a736.py)

---
# Summary

In this tutorial, we studied how the input correlation of two LIF neurons is mapped to their output correlation. Specifically, we:

- injected correlated GWN in a pair of neurons,

- measured correlations between the spiking activity of the two neurons, and

- studied how the transfer of correlation depends on the statistics of the input, i.e., mean and standard deviation.

Here, we were concerned with zero time lag correlation. For this reason, we restricted estimation of correlation to instantaneous correlations. If you are interested in time-lagged correlation, then we should estimate the cross-correlogram of the spike trains and find out the dominant peak and area under the peak to get an estimate of output correlations.

We leave this as a future to-do for you if you are interested.

---
# Bonus 1: Example of a conductance-based LIF model
Above, we have written code to generate correlated Poisson spike trains. You can write code to stimulate the LIF neuron with such correlated spike trains and study the correlation transfer function for spiking input and compare it to the correlation transfer function obtained by injecting correlated GWNs.

```python
# @title Function to simulate conductance-based LIF

def run_LIF_cond(pars, I_inj, pre_spike_train_ex, pre_spike_train_in):
  """
  conductance-based LIF dynamics

  Args:
    pars               : parameter dictionary
    I_inj              : injected current [pA]. The injected current here can
                         be a value or an array
    pre_spike_train_ex : spike train input from presynaptic excitatory neuron
    pre_spike_train_in : spike train input from presynaptic inhibitory neuron

  Returns:
    rec_spikes         : spike times
    rec_v              : mebrane potential
    gE                 : postsynaptic excitatory conductance
    gI                 : postsynaptic inhibitory conductance
  """

  # Retrieve parameters
  V_th, V_reset = pars['V_th'], pars['V_reset']
  tau_m, g_L = pars['tau_m'], pars['g_L']
  V_init, E_L = pars['V_init'], pars['E_L']
  gE_bar, gI_bar = pars['gE_bar'], pars['gI_bar']
  VE, VI = pars['VE'], pars['VI']
  tau_syn_E, tau_syn_I = pars['tau_syn_E'], pars['tau_syn_I']
  tref = pars['tref']
  dt, range_t = pars['dt'], pars['range_t']
  Lt = range_t.size

  # Initialize
  tr = 0.
  v = np.zeros(Lt)
  v[0] = V_init
  gE = np.zeros(Lt)
  gI = np.zeros(Lt)
  Iinj = I_inj * np.ones(Lt)  # ensure I has length Lt

  if pre_spike_train_ex.max() == 0:
    pre_spike_train_ex_total = np.zeros(Lt)
  else:
    pre_spike_train_ex_total = pre_spike_train_ex * np.ones(Lt)

  if pre_spike_train_in.max() == 0:
    pre_spike_train_in_total = np.zeros(Lt)
  else:
    pre_spike_train_in_total = pre_spike_train_in * np.ones(Lt)

  # simulation
  rec_spikes = []  # recording spike times
  for it in range(Lt - 1):
    if tr > 0:
      v[it] = V_reset
      tr = tr - 1
    elif v[it] >= V_th:   # reset voltage and record spike event
      rec_spikes.append(it)
      v[it] = V_reset
      tr = tref / dt
    # update the synaptic conductance
    gE[it+1] = gE[it] - (dt / tau_syn_E) * gE[it] + gE_bar * pre_spike_train_ex_total[it + 1]
    gI[it+1] = gI[it] - (dt / tau_syn_I) * gI[it] + gI_bar * pre_spike_train_in_total[it + 1]

    # calculate the increment of the membrane potential
    dv = (-(v[it] - E_L) - (gE[it + 1] / g_L) * (v[it] - VE) - \
            (gI[it + 1] / g_L) * (v[it] - VI) + Iinj[it] / g_L) * (dt / tau_m)

    # update membrane potential
    v[it + 1] = v[it] + dv

  rec_spikes = np.array(rec_spikes) * dt

  return v, rec_spikes, gE, gI

print(help(run_LIF_cond))
```

    Help on function run_LIF_cond in module __main__:

    run_LIF_cond(pars, I_inj, pre_spike_train_ex, pre_spike_train_in)
        conductance-based LIF dynamics

        Args:
          pars               : parameter dictionary
          I_inj              : injected current [pA]. The injected current here can
                               be a value or an array
          pre_spike_train_ex : spike train input from presynaptic excitatory neuron
          pre_spike_train_in : spike train input from presynaptic inhibitory neuron

        Returns:
          rec_spikes         : spike times
          rec_v              : mebrane potential
          gE                 : postsynaptic excitatory conductance
          gI                 : postsynaptic inhibitory conductance

    None

## Interactive Demo: Correlated spike input to an LIF neuron

In the following you can explore what happens when the neurons receive correlated spiking input.

You can vary the correlation between excitatory input spike trains. For simplicity, the correlation between inhibitory spike trains is set to 0.01.

Vary both excitatory rate and correlation and see how the output correlation changes. Check if the results are qualitatively similar to what you observed previously when you varied the $\mu$ and $\sigma$.

```python
# @title

# @markdown Make sure you execute this cell to enable the widget!

my_layout.width = '450px'
@widgets.interact(
  pwc_ee=widgets.FloatSlider(0.3, min=0.05, max=0.99, step=0.01,
                             layout=my_layout),
  exc_rate=widgets.FloatSlider(1e3, min=500., max=5e3, step=50.,
                               layout=my_layout),
  inh_rate=widgets.FloatSlider(500., min=300., max=5e3, step=5.,
                               layout=my_layout),
)

def EI_isi_regularity(pwc_ee, exc_rate, inh_rate):
  pars = default_pars(T=1000.)
  # Add parameters
  pars['V_th'] = -55.     # spike threshold [mV]
  pars['V_reset'] = -75.  # reset potential [mV]
  pars['tau_m'] = 10.     # membrane time constant [ms]
  pars['g_L'] = 10.       # leak conductance [nS]
  pars['V_init'] = -65.   # initial potential [mV]
  pars['E_L'] = -75.      # leak reversal potential [mV]
  pars['tref'] = 2.       # refractory time (ms)

  pars['gE_bar'] = 4.0    # [nS]
  pars['VE'] = 0.         # [mV] excitatory reversal potential
  pars['tau_syn_E'] = 2.  # [ms]
  pars['gI_bar'] = 2.4    # [nS]
  pars['VI'] = -80.       # [mV] inhibitory reversal potential
  pars['tau_syn_I'] = 5.  # [ms]

  my_bin = np.arange(0, pars['T']+pars['dt'], .1)  # 20 [ms] bin-size

  # exc_rate = 1e3
  # inh_rate = 0.4e3
  # pwc_ee = 0.3
  pwc_ii = 0.01
  # generate two correlated spike trains for excitatory input
  sp1e, sp2e = generate_corr_Poisson(pars, exc_rate, pwc_ee)

  sp1_spike_train_ex, _ = np.histogram(sp1e, bins=my_bin)
  sp2_spike_train_ex, _ = np.histogram(sp2e, bins=my_bin)

  # generate two uncorrelated spike trains for inhibitory input
  sp1i, sp2i = generate_corr_Poisson(pars, inh_rate, pwc_ii)
  sp1_spike_train_in, _ = np.histogram(sp1e, bins=my_bin)
  sp2_spike_train_in, _ = np.histogram(sp2e, bins=my_bin)

  v1, rec_spikes1, gE, gI = run_LIF_cond(pars, 0, sp1_spike_train_ex, sp1_spike_train_in)
  v2, rec_spikes2, gE, gI = run_LIF_cond(pars, 0, sp2_spike_train_ex, sp2_spike_train_in)

  # bin the spike time
  bin_size = 20  # [ms]
  my_bin = np.arange(0, pars['T'], bin_size)
  spk_1, _ = np.histogram(rec_spikes1, bins=my_bin)
  spk_2, _ = np.histogram(rec_spikes2, bins=my_bin)

  r12 = my_CC(spk_1, spk_2)
  print(f"Input correlation = {pwc_ee}")
  print(f"Output correlation = {r12}")

  plt.figure(figsize=(14, 7))
  plt.subplot(211)
  plt.plot(sp1e, np.ones(len(sp1e)) * 1, '|', ms=20,
           label='Exc. input 1')
  plt.plot(sp2e, np.ones(len(sp2e)) * 1.1, '|', ms=20,
           label='Exc. input 2')
  plt.plot(sp1e, np.ones(len(sp1e)) * 1.3, '|k', ms=20,
           label='Inh. input 1')
  plt.plot(sp2e, np.ones(len(sp2e)) * 1.4, '|k', ms=20,
           label='Inh. input 2')

  plt.ylim(0.9, 1.5)
  plt.legend()
  plt.ylabel('neuron id.')

  plt.subplot(212)
  plt.plot(pars['range_t'], v1, label='neuron 1')
  plt.plot(pars['range_t'], v2, label='neuron 2')
  plt.xlabel('time (ms)')
  plt.ylabel('membrane voltage $V_{m}$')
  plt.tight_layout()
  plt.show()
```

Above, we are estimating the output correlation for one trial. You can modify the code to get a trial average of output correlations.

---
# Bonus 2: Ensemble Response

Finally, there is a short BONUS lecture video on the firing response of an ensemble of neurons to time-varying input. There are no associated coding exercises - just enjoy.

```python
#@title Video 2 (Bonus): Response of ensemble of neurons to time-varying input
from IPython.display import IFrame
class BiliVideo(IFrame):
    def __init__(self, id, page=1, width=400, height=300, **kwargs):
        self.id=id
        src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
        super(BiliVideo, self).__init__(src, width, height, **kwargs)

video = BiliVideo(id='BV18K4y1x7Pt', width=854, height=480, fs=1)
print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
video
```

    Video available at https://www.bilibili.com/video/BV18K4y1x7Pt


# (Step 4) Model Evaluation
<!-- # - file: defect-prediction/model-validation.ipynb
# - file: defect-prediction/evaluation-measures.ipynb -->

## Always select evaluation measures according to the goals of studies

Different studies often have different goals of developing defect models.
For example, one study may want its defect models that yield the highest discriminatory power (i.e., optimise based on Area Under the receiver operator characteristic Curve (AUC) (AUC) {cite}`Hanley1982-uo`).
Another study may want its defect models to discover an actual defective module as early as possible (i.e., optimise based on initial false alarm {cite}`huang2017supervised`).
Thus, several evaluation measures have been proposed to serve different goals of developing defect models.
<!-- Examples are predictive accuracy measures (e.g., Area Under the ROC Curve (AUC) {cite}`Hanley1982-uo` and Matthews Correlation Coefficients (MCC) {cite}`matthews1975comparison`) and effort-aware measures (e.g., Distance-to-heaven {cite}`agrawal2018better` and Top k% LOC precision {cite}`huang2017supervised`). -->
Below, we describe *5* predictive accuracy and *3* effort-aware measures in detail with interactive tutorials.
We also provide a code snippet for setting up the environment for interactive tutorials below, i.e., data preparation and model construction.

```python
## Load Data and preparing datasets

# Import for Load Data
from os import listdir
from os.path import isfile, join
import pandas as pd
import numpy as np

# Import for Split Data into Training and Testing Samples
from sklearn.model_selection import train_test_split

# Import for Construct Defect Models (Classification)
from sklearn.ensemble import RandomForestClassifier # Random Forests

train_dataset = pd.read_csv(("../../datasets/lucene-2.9.0.csv"), index_col = 'File')
test_dataset = pd.read_csv(("../../datasets/lucene-3.0.0.csv"), index_col = 'File')

outcome = 'RealBug'
features = ['OWN_COMMIT', 'Added_lines', 'CountClassCoupled', 'AvgLine', 'RatioCommentToCode']

# process outcome to 0 and 1
train_dataset[outcome] = pd.Categorical(train_dataset[outcome])
train_dataset[outcome] = train_dataset[outcome].cat.codes

test_dataset[outcome] = pd.Categorical(test_dataset[outcome])
test_dataset[outcome] = test_dataset[outcome].cat.codes

X_train = train_dataset.loc[:, features]
X_test = test_dataset.loc[:, features]

y_train = train_dataset.loc[:, outcome]
y_test = test_dataset.loc[:, outcome]

# commits - # of commits that modify the file of interest
# Added lines - # of added lines of code
# Count class coupled - # of classes that interact or couple with the class of interest
# LOC - # of lines of code
# RatioCommentToCode - The ratio of lines of comments to lines of code
features = ['nCommit', 'AddedLOC', 'nCoupledClass', 'LOC', 'CommentToCodeRatio']

X_train.columns = features
X_test.columns = features
training_data = pd.concat([X_train, y_train], axis=1)
testing_data = pd.concat([X_test, y_test], axis=1)

## Construct defect models
# Random Forests
rf_model = RandomForestClassifier(random_state=1234, n_jobs = 10)
rf_model.fit(X_train, y_train)
```

    RandomForestClassifier(n_jobs=10, random_state=1234)

### Predictive Accuracy Measures

#### Precision
Precision measures the proportion between the number of lines that are correctly identified as defective and the number of lines that are identified by the models.
Particularly, precision can be computed using a calculation of $\frac{TP}{(TP+FP)}$, where $TP$ is the number of actual defective lines that are predicted as defective and $FP$ is the number of clean lines that are predicted as defective.
A high precision value indicates that the models can correctly identify high number of defective lines.

```python
from sklearn.metrics import confusion_matrix
from sklearn.metrics import precision_score

# construct a confusion matrix
print('Construct a confusion matrix:')
tn, fp, fn, tp = confusion_matrix(y_test,rf_model.predict(X_test)).ravel()
print('(True Positive, False Positive) = (' + str(tp) +','+str(fp)+')')
print('(False Negative, True Negative) = (' + str(fn) +','+str(tn)+')\n')

# calculate precision manually
rf_precision_manual = tp/(tp + fp)

# calculate precision with a function
rf_precision_function = precision_score(y_test, rf_model.predict(X_test))

print('Precision (manual calculation)\t\t:', rf_precision_manual)
print('Precision (precision_score function)\t:', rf_precision_function)
```

    Construct a confusion matrix:
    (True Positive, False Positive) = (99,147)
    (False Negative, True Negative) = (56,1035)

    Precision (manual calculation)		: 0.4024390243902439
    Precision (precision_score function)	: 0.4024390243902439

#### Recall
Recall measures the proportion between the number of lines that are correctly identified as defective and the number of actual defective lines.
More specifically, we compute recall using a calculation of $\frac{TP}{(TP+FN)}$, where $TP$ is the number of actual defective lines that are predicted as defective and $FN$ is the number of actual defective lines that are predicted as clean.
A high recall value indicates that the approach can identify more defective lines.

```python
from sklearn.metrics import recall_score

# calculate recall manually
rf_recall_manual = tp/(tp + fn)

# calculate recall with a function
rf_recall_function = recall_score(y_test, rf_model.predict(X_test))

print('Recall (manual calculation)\t\t:', rf_recall_manual)
print('Recall (recall_score function)\t:', rf_recall_function)
```

    Recall (manual calculation)		: 0.6387096774193548
    Recall (recall_score function)	: 0.6387096774193548

#### False Alarm Rate (FAR) or False Positive Rate (FPR)
FAR measures a proportion between the number of clean lines that are identified as defective and the number of actual clean lines.
More specifically, FAR can be calculated with a calculation of $\frac{FP}{(FP+TN)}$, where $FP$ is the number of actual clean lines that are predicted as defective and $TN$ is the number of actual clean lines that are predicted as clean.
The lower the FAR value is, the fewer the clean lines that are identified as defective.
In other words, a low FAR value indicates that developers spend less effort when inspecting defect-prone lines identified by the an approach.

```python
# calculate FAR manually
rf_FAR_manual = fp / (fp + tn)

print('FAR (manual calculation)\t\t:', rf_FAR_manual)
```

    FAR (manual calculation)		: 0.12436548223350254

#### Area Under the receiver operator characteristic Curve (AUC)
AUC measures the discriminatory power of predictive models and is widely suggested by recent research {cite}`lessmann2008benchmarking`{cite}`ghotra2015revisiting`{cite}`rahman2013and`.
The axes of the curve of the AUC measure are the coverage of non-defective modules (true negative rate) for the x-axis and the coverage of defective modules (true positive rate) for the y-axis.
The AUC measure is a threshold-independent performance measure that evaluates the ability of models in discriminating between defective and clean instances.
The values of AUC range between 0 (worst), 0.5 (no better than random guessing), and 1 (best) {cite}`Hanley1982-uo`.

```python
from sklearn.metrics import roc_auc_score

# calculate AUC with a function
rf_AUC_function = roc_auc_score(y_test, rf_model.predict_proba(X_test)[:,1])

print('AUC (roc_auc_score function)\t\t:', rf_AUC_function)
```

    AUC (roc_auc_score function)		: 0.8589023524916761

#### Matthews Correlation Coefficients (MCC)
MCC measures a correlation coefficients between actual and predicted outcomes using the following calculation:
\begin{equation}
\frac{TP \times TN - FP \times FN}{\sqrt{(TP+FP)(TP+FN)(TN+FP)(TN+FN)}}
\end{equation}
An MCC value ranges from -1 to +1, where an MCC value of 1 indicates a perfect prediction, and -1 indicates total disagreement between the prediction.

```python
from sklearn.metrics import matthews_corrcoef

# calculate MCC manually
rf_MCC_manual = ((tp * tn) - (fp * fn)) / (((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)) ** (1/2))

# calculate MCC with a function
rf_MCC_function = matthews_corrcoef(y_test, rf_model.predict(X_test))

print('MCC (manual calculation)\t\t:', rf_MCC_manual)
print('MCC (matthews_corrcoef function)\t:', rf_MCC_function)
```

    MCC (manual calculation)		: 0.4249604383453459
    MCC (matthews_corrcoef function)	: 0.4249604383453459

### Effort-Aware Measures

#### Initial False Alarm (IFA)
IFA measures the number of clean lines on which developers spend SQA effort until the first defective line is found when lines are ranked by their defect-proneness {cite}`huang2017supervised`.
A low IFA value indicates that few clean lines are ranked at the top, while a high IFA value indicates that developers will spend unnecessary effort on clean lines.
The intuition behinds this measure is that developers may stop inspecting if they could not get promising results (i.e., find defective lines) within the first few inspected lines {cite}`parnin2011automated`.

```python
# Generate a defect-proneness ranking of testing instances
X_test_df = X_test.copy()
X_test_df['predicted_prob'] = rf_model.predict_proba(X_test)[:, 1]
X_test_df = X_test_df.sort_values(by = ['predicted_prob'], ascending = False)

# Determine the Initial False Alarm (IFA)
IFA = 0
for test_index in X_test_df.index:
    IFA += 1
    if y_test.loc[test_index] == 1:
        break

print('Initial False Alarm (IFA)\t\t:', IFA)
```

    Initial False Alarm (IFA)		: 1

#### Distance-to-Heaven (D2H)

D2H is a combination of recall and FAR proposed by Agrawal and Menzies {cite}`agrawal2018better` {cite}`agrawal2019dodge`.
The calculation of D2H is the root mean square of the recall and false alarm values (i.e., $\sqrt{\frac{(1-Recall)^2 + (0-FAR)^2}{2}}$).
A d2h value of 0 indicates that an approach achieves a perfect identification, i.e., an approach can identify all defective lines (Recall $= 1$) without any false positives (FAR $= 0$).
A high d2h value indicates that the performance of an approach is far from perfect, e.g., achieving a high recall value but also have high a FAR value and vice versa.

```python
from sklearn.metrics import recall_score

# calculate recall with a function
rf_recall_function = recall_score(y_test, rf_model.predict(X_test))

# calculate FAR manually
rf_FAR_manual = fp / (fp + tn)

# calculate D2H manually
rf_D2H_numerator = ((1 - rf_recall_function)**2) + ((0 - rf_FAR_manual)**2)
rf_D2H_denominator = 2
rf_D2H_manual = (rf_D2H_numerator / rf_D2H_denominator)**(1/2)

print('D2H (manual calculation)\t\t:', rf_D2H_manual)
```

    D2H (manual calculation)		: 0.27018278105904375

#### Top k% LOC Precision
Top k% LOC Precision measures how many defective lines found when inspecting the top k% of lines ranked by the defect-proneness estimated by the models {cite}`huang2017supervised`.
A high value of Top k% LOC precision indicates that the model can rank many defective lines at the top and many defective lines can be found given the fixed amount of effort (i.e., k% of LOC).
On the other hand, the low value of Top k% LOC precision indicates many clean lines are in the top k% LOC and developers need to inspect more lines to identify defects.
Similar to prior studies {cite}`mende2010effort` {cite}`kamei2010revisiting` {cite}`rahman2014comparing` {cite}`ray2016naturalness`, we use 20% of LOC as a fixed cutoff for an effort in this interactive tutorial.

```python
from sklearn.metrics import precision_score

# Generate a defect-proneness ranking of testing instances
X_test_df = X_test.copy()
X_test_df['predicted_prob'] = rf_model.predict_proba(X_test)[:, 1]
X_test_df = X_test_df.sort_values(by = ['predicted_prob'], ascending = False)

# calculate the value of k% LOC, where k = 20
k_percent = 20.0
total_LOC = np.sum(X_test_df['LOC'])
p20_LOC = total_LOC * k_percent / 100

# find Top k% LOC according to the defect-proneness ranking of testing instances
cumsum_LOC = 0
last_index = -1
for i in range(len(X_test)):
    cumsum_LOC += X_test_df['LOC'].iloc[i]
    last_index = i

    if cumsum_LOC > p20_LOC:
        print('Cumsum_LOC =', cumsum_LOC, 'Index', last_index)
        break
p20_LOC_X_test_df = X_test_df.iloc[:last_index, :]
p20_LOC_y_test_df = y_test[p20_LOC_X_test_df.index]

# calculate precision of Top k% LOC
rf_p20_precision_function = precision_score(p20_LOC_y_test_df, rf_model.predict(p20_LOC_X_test_df.loc[:, X_test.columns]))

print('Top 20% LOC Precision (precision_score function)\t:', rf_p20_precision_function)
```

    Cumsum_LOC = 3736 Index 231
    Top 20% LOC Precision (precision_score function)	: 0.4025974025974026

#### Top k% LOC Recall
Top k%LOC recall measures how many actual defective lines found given a fixed amount of effort, i.e., the top k% of lines ranked by their defect-proneness {cite}`huang2017supervised`.
A high value of top k% LOC recall indicates that an approach can rank many actual defective lines at the top and many actual defective lines can be found given a fixed amount of effort.
On the other hand, a low value of top k% LOC recall indicates many clean lines are in the top k% LOC and developers need to spend more effort to identify defective lines.
Similarly, we use 20% of LOC as a fixed cutoff for an effort.

```python
from sklearn.metrics import recall_score

# Generate a defect-proneness ranking of testing instances
X_test_df = X_test.copy()
X_test_df['predicted_prob'] = rf_model.predict_proba(X_test)[:, 1]
X_test_df = X_test_df.sort_values(by = ['predicted_prob'], ascending = False)

# calculate the value of k% LOC, where k = 20
k_percent = 20.0
total_LOC = np.sum(X_test_df['LOC'])
p20_LOC = total_LOC * k_percent / 100

# find Top k% LOC according to the defect-proneness ranking of testing instances
cumsum_LOC = 0
last_index = -1
for i in range(len(X_test)):
    cumsum_LOC += X_test_df['LOC'].iloc[i]
    last_index = i

    if cumsum_LOC > p20_LOC:
        print('Cumsum_LOC =', cumsum_LOC, 'Index', last_index)
        break
p20_LOC_X_test_df = X_test_df.iloc[:last_index, :]
p20_LOC_y_test_df = y_test[p20_LOC_X_test_df.index]

# calculate recall of Top k% LOC
rf_p20_recall_function = recall_score(p20_LOC_y_test_df, rf_model.predict(p20_LOC_X_test_df.loc[:, X_test.columns]))

print('Top 20% LOC Recall (recall_score function)\t:', rf_p20_recall_function)
```

    Cumsum_LOC = 3736 Index 231
    Top 20% LOC Recall (recall_score function)	: 1.0

```{note}
Parts of this chapter have been published by Supatsara Wattanakriengkrai, Patanamon Thongtanunam, Chakkrit Tantithamthavorn, Hideaki Hata, Kenichi Matsumoto: Predicting Defective Lines Using a Model-Agnostic Technique. CoRR abs/2009.03612 (2020).
```

## Suggested Readings

[1] Amritanshu Agrawal, Tim Menzies: Is "better data" better than "better data miners"?: on the benefits of tuning SMOTE for defect prediction. ICSE 2018: 1050-1061.

[2] Amritanshu Agrawal, Wei Fu, Di Chen, Xipeng Shen, Tim Menzies: How to "DODGE" Complex Software Analytics? CoRR abs/1902.01838 (2019).

[3] Chris Parnin, Alessandro Orso: Are automated debugging techniques actually helping programmers? ISSTA 2011: 199-209.

[4] Qiao Huang, Xin Xia, David Lo: Supervised vs Unsupervised Models: A Holistic Look at Effort-Aware Just-in-Time Defect Prediction. ICSME 2017: 159-170.

```python

```


# P2: Segunda Avaliação de Conhecimentos

Grupo: B2

Brian Luís Coimbra Maia

Mathews Edwirds Gomes Almeida

Raíssa Guimarães de Castro

Daniel Possato Juliaci

Responsável pela entrega:

Brian Luís Coimbra Maia

Data de formação da equipe:

19/07/21

# Condições Gerais

Essa tarefa tem como objetivo avaliar os conhecimentos adquiridos durante a disciplina de Fundamentos de Mecânica das Estruturas.

As condicões abaixo devem ser observadas:

1. Serão formadas equipes e cada uma delas com até e 4 integrantes.

2. A avaliação será realizada por meio da entrega de um vídeo explicativo e de
códigos desenvolvidos até a data estipulada.

3. Os documentos necessários para a entrega do trabalho são (1) um vídeo com
a descricão da solução e (2) os códigos desenvolvidos pela equipe. Os vídeos podem ser separados por questão, porém o tempo somado de todos os vídeos não deve ultrapassar **12** minutos. A equipe deve usar este modelo de notebook para desenvolver os códigos.

4. A distribuição da pontuacão referente a cada ítem aparece no enunciado de
cada proposicão.

4. Um dos integrantes será indentificado como responsável pela entrega do documentos necessários.

6. As equipes devem ser formadas até às **18 horas o dia 23/07/2021** por meio do preenchimento da planilha [[MAC019] Formação das Equipes](https://docs.google.com/spreadsheets/d/1XhCWfpfDqR951V5RHR1aS_dBhocfzFNLdZe8NpCO-Ng/edit#gid=153704268).

7. A formação das equipes pode ser acompanhada arquivo [[MAC019] Formação das Equipes](https://docs.google.com/spreadsheets/d/1XhCWfpfDqR951V5RHR1aS_dBhocfzFNLdZe8NpCO-Ng/edit#gid=153704268). Cada equipe será indentificada por uma letra em ordem alfabética seguida do número 2 (A2, B2, C2, e assim por diante). O arquivo está aberto para edição e pode ser alterado pelos alunos até a data estipulada.

8. Equipes formadas após a data estabelecida para a formação das equipes terão a nota da avaliação multiplicada por um coeficiente de **0.80**.

9. A equipe deve indicar no arquivo [[MAC019] Formação das Equipes](https://docs.google.com/spreadsheets/d/1XhCWfpfDqR951V5RHR1aS_dBhocfzFNLdZe8NpCO-Ng/edit#gid=153704268) um responsável pela entrega do projeto.

10. A entrega dos projetos (videos explicativos e códigos) deve ocorrer até às **23:59 do dia 06/08/2021** na plataforma da disciplina pelo responsável pela entrega. **Os arquivos devem ser compartilhados com leonardo.goliatt@ufjf.br**

11. Quaiquer dúvidas ou esclarecimentos devem ser encaminhadas pela sala de aula virtual.

## Instalação dos pacotes

Nesta aula vamos usar o pacote sympy/Beam (computação simbólica) e compará-lo com o pacote beambending

```python
!pip install beambending
!pip install anastruct
!pip install sympy
```

```python
# importando os pacotes necessários
import numpy as np
import matplotlib.pyplot as plt
from anastruct import SystemElements
from termcolor import colored
```

# Questão 1 (20 pontos)

Apresente um modelo computacional para o pórtico abaixo.
A carga distribuída tem o valor de 12 kN/m. A distância entre os pontos $A$ e $B$ é de 3 metros a a distância horizontal entre os apoios é de 4 metros.

```python
#-------------------------------------------------------------------------------
# Informações da estrutura
#-------------------------------------------------------------------------------
node={"M":(0, 9), "N":(4, 9), "O":(8, 9), "P":(12, 9),
      "Q":(0, 6), "R":(4, 6), "S":(8, 6), "T":(12, 6),
      "U":(0, 3), "B":(4, 3), "V":(8, 3), "W":(12, 3),
      "X":(0, 0), "A":(4, 0), "Y":(8, 0), "Z":(12, 0)}

# Define as conectividades entre os nós
conec=[('M', 'N'), ('N', 'O'), ('Q', 'R'), ('R', 'S'), ('U', 'B'), ('B', 'V'),
       ('M', 'Q'), ('N', 'R'), ('O', 'P'), ('O', 'S'), ('P', 'T'),
       ('Q', 'U'), ('R', 'B'), ('S', 'T'), ('S', 'V'), ('T', 'W'),
       ('U', 'X'),  ('B', 'A'), ('V', 'W'), ('V', 'Y'), ('W', 'Z')]

#-------------------------------------------------------------------------------
# Montagem do modelo estrutural
#-------------------------------------------------------------------------------
#  Instancia o sistema estrutural
ss = SystemElements()

# Implementa a conectividade entre os nós gerando os elementos ou "membros"
for e in conec:
  element = (node[e[0]], node[e[1]])
  ss.add_element(location = element)    # Add um elemento comum entre cada nó conectado

# carregamentos distribuídos
ss.q_load(element_id=[1,2], q=-12, direction='y')
ss.q_load(element_id=3, q=-12, direction='y')
ss.q_load(element_id=4, q=-12, direction='y')
ss.q_load(element_id=5, q=-12, direction='y')
ss.q_load(element_id=6, q=-12, direction='y')

# apoios
node_id = ss.find_node_id(node['X'])
ss.add_support_hinged(node_id=node_id)
node_id = ss.find_node_id(node['A'])
ss.add_support_hinged(node_id=node_id)
node_id = ss.find_node_id(node['Y'])
ss.add_support_hinged(node_id=node_id)
node_id = ss.find_node_id(node['Z'])
ss.add_support_hinged(node_id=node_id)

#-------------------------------------------------------------------------------
# Solução e resultados do sistema estrutural
#-------------------------------------------------------------------------------
ss.solve()

ss.show_structure(scale=0.7, figsize=(10,7), offset=(0,0))
ss.show_reaction_force(scale=0.9, figsize=(10,7), offset=(0,1))
ss.show_axial_force(scale=0.7, figsize=(10,7), offset=(0,0))
ss.show_shear_force(scale=0.7, figsize=(10,7), offset=(0,0))
ss.show_bending_moment(scale=0.7, figsize=(10,7), offset=(0,0))
ss.show_displacement(scale=0.7, figsize=(10,7), offset=(0,0))
```

# Questão 2 (20 pontos)

Apresente os diagramas de esforços para a estrutura abaixo.

```python
#-------------------------------------------------------------------------------
# Informações da estrutura
#-------------------------------------------------------------------------------
node={"A":(0, 4), 'E':(3,4), "B":(6, 4), "C":(6, 0), "D":(12, 0)}

# Define as conectividades entre os nós
conec=[('A', 'E'), ('E','B'), ('B', 'C'), ('C', 'D')]

concentrated_load = {'E':(0,0,-30)}

#-------------------------------------------------------------------------------
# Montagem do modelo estrutural
#-------------------------------------------------------------------------------
#  Instancia o sistema estrutural
ss = SystemElements()

# Implementa a conectividade entre os nós gerando os elementos ou "membros"
ss.add_element(location=[node['A'], node['E']])
ss.add_element(location=[node['E'], node['B']])
ss.add_element(location=[node['B'], node['C']], spring={1:0, 2:0})  # Rótulas
ss.add_element(location=[node['C'], node['D']], spring={1:0}) # Rótulas

# Carregamentos concentrados
for l in concentrated_load:
  fx, fy, mz = concentrated_load[l]
  node_id = ss.find_node_id(node[l])
  if fx!=0 or fy!=0:
    ss.point_load(node_id=node_id, Fx=fx, Fy=fy)
  if mz!=0:
    ss.moment_load(node_id=node_id, Ty=mz)

# Carregamentos distribuídos
ss.q_load(element_id=4, q=-5, direction='y')

# Apoios
node_idA = ss.find_node_id(node['A'])
ss.add_support_fixed(node_id=node_idA)   #engaste
node_idD = ss.find_node_id(node['D'])
ss.add_support_hinged(node_id=node_idD)

#-------------------------------------------------------------------------------
# Solução e resultados do sistema estrutural
#-------------------------------------------------------------------------------
ss.solve()

ss.show_structure(scale=0.7, figsize=(8,4), offset=(0,0))
ss.show_reaction_force(scale=0.9, figsize=(8,4), offset=(0,1))
ss.show_axial_force(scale=0.7, figsize=(8,4), offset=(0,0))
ss.show_shear_force(scale=0.7, figsize=(8,4), offset=(0,0))
ss.show_bending_moment(scale=0.7, figsize=(8,4), offset=(0,0))
ss.show_displacement(scale=0.7, figsize=(8,4), offset=(0,0))
```

# Questão 3 (20 pontos)

A viga abaixo suporta o carregamento indicado. Determine o valor de $x$ que minimize o momento máximo $M_{max}$ na viga. Especifique o valor de $M_{max}$.

```python
#-------------------------------------------------------------------------------
# Informações da estrutura
#-------------------------------------------------------------------------------
comprimento = 8
salto = 0.1
intervalo = np.arange(0.5, comprimento/2, salto)
carregamento = 20

lista_momentos = []
#-------------------------------------------------------------------------------
# Montagem do modelo estrutural
#-------------------------------------------------------------------------------
for i in intervalo:
  node={'A':[0,0], 'B':[i,0], 'C':[comprimento-i,0], 'D':[comprimento,0]}

  # instancia o sistema estrutural
  ss = SystemElements()

  # conexão dos nós
  ss.add_element(location=[node['A'], node['B'], ], spring={})
  ss.add_element(location=[node['B'], node['C'], ], spring={})
  ss.add_element(location=[node['C'], node['D'], ], spring={})

  # carregamentos distribuídos
  ss.q_load(element_id=1, q=-carregamento/2, direction='y')
  ss.q_load(element_id=2, q=-carregamento, direction='y')
  ss.q_load(element_id=3, q=-carregamento, direction='y')

  # apoios
  node_id = ss.find_node_id(node['B'])
  ss.add_support_hinged(node_id=node_id)
  node_id = ss.find_node_id(node['C'])
  ss.add_support_hinged(node_id=node_id)

  #-------------------------------------------------------------------------------
  # Solução do sistema estrutural
  #-------------------------------------------------------------------------------
  ss.solve()
  print("intervalo =", i)
  lista_momentos.append(ss.get_element_result_range('moment')[2])
  print("Momentos: " + str(ss.get_element_result_range('moment')))
  #print(ss.get_element_results())
  #-------------------------------------------------------------------------------
  #ss.show_structure()
  #ss.show_reaction_force()
  ss.show_bending_moment(scale=0.8, figsize=(8,3), offset=(0,0))

print(colored("\n\nO valor de x que gera o menor momento maximo é: " + str(intervalo[min(lista_momentos).argmin()]),'green'))
print(colored("O menor momento máximo é: " + str(min(lista_momentos)),'green'))
```

# Questão 4 (40 pontos)

Determine o valor a posição do momento máximo _positivo_ na ponte causado pela passagem do veiculo.

Durante o processo de modelagem justifique as escolhas realizadas para encontrar a solução.

```python
from sympy.physics.continuum_mechanics.beam import Beam
from sympy import symbols
import numpy as np

movimento = np.arange(0, 13, 1.7)#(0, 13, 1.7) # Arranjo feito convenientemente para fazer a aplicação das forças ao longo da ponte
max_momentos = [] # Lista para salvar todos os momentos máximos encontrados a cada iteração
max_posicoes = [] # Lista para salvar todos as posições que geram momentos máximos

# Laço para movimentar as forças aplicadas pelo carrinho ao longo da ponte
for m in movimento:
  A, B = symbols('A, B')
  b = Beam(10, A, B)

  # Aplicando os apoios
  b.apply_support(loc=0, type='roller')
  b.apply_support(loc=10, type='pin')

  # Aplicando as forças
  b.apply_load(value=8, start=m, order=-1)
  b.apply_load(value=6, start=m-2, order=-1)
  b.apply_load(value=4, start=m-3.5, order=-1)

  # Exibindo a figura com a barra, apoios e forças aplicadas
  p = b.draw()
  p.show()

  # Utilizando função para resolver as forças de reação
  R_0, R_10 = symbols('R_0, R_10')
  b.solve_for_reaction_loads(R_0, R_10)

  # Utilizando a função max_bmoment() que retorna o momento máximo gerado e a coordenada em que isso ocorre na ponte
  x = b.max_bmoment()
  max_momentos.append(x[1])
  max_posicoes.append(x[0])

  # Printa momento máximo e coordenadas em que as forças estão sendo aplicadas a cada iteração
  print("Momento máximo: " + str(x))
  print("Coordenadas: " + str(m) + " " + str(m-2) + " " + str(m-3.5))

  # Plota gráfico de momentos
  b.plot_bending_moment()

print("Momento máximo encontrado durante o deslocamento do carrinho na ponte: " + str(np.amax(max_momentos)) + " kN.m")
print("Posição em que o momento máximo foi encontrado: " + str(max_posicoes[max_momentos.index(np.amax(max_momentos))]))
```


---
layout: page
title: Regressão Linear
nav_order: 15
---

[](https://colab.research.google.com/github/icd-ufmg/icd-ufmg.github.io/blob/master/_lessons/15-linear.ipynb)

# Regressão Linear Simples

{: .no_toc .mb-2 }

Entendimento de relação entre dados.
{: .fs-6 .fw-300 }

{: .no_toc .text-delta }
Resultados Esperados

1. Revisitar os mínimos quadrados (ALC)
1. Entender a regressão linear do ponto de vista estatístico (diferente de ALC)
1. Entender o coeficiente de pearson e o R2.
1. Executar regressões em python

---
**Sumário**
1. TOC
{:toc}
---

```python
# -*- coding: utf8

from scipy import stats as ss

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
```

```python
plt.style.use('seaborn-colorblind')
plt.rcParams['figure.figsize']  = (16, 10)
plt.rcParams['axes.labelsize']  = 20
plt.rcParams['axes.titlesize']  = 20
plt.rcParams['legend.fontsize'] = 20
plt.rcParams['xtick.labelsize'] = 20
plt.rcParams['ytick.labelsize'] = 20
plt.rcParams['lines.linewidth'] = 4
```

```python
plt.ion()
```

```python
def despine(ax=None):
    if ax is None:
        ax = plt.gca()
    # Hide the right and top spines
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)

    # Only show ticks on the left and bottom spines
    ax.yaxis.set_ticks_position('left')
    ax.xaxis.set_ticks_position('bottom')
```

## Introdução

Antes de iniciar o nosso estudo da regressão, vamos pensar em um modelo que tenta capturar o valor médio no eixo y. Para uma base de dados unidimensional (apenas Y), a média ($\bar{y}$) é um bom estimador. Isto é, imagine que você está em uma sala de aula e entra um novo aluno na sala. Posso prever alguma característica deste aluno (por exemplo, o seu rendimento escolar) usando a média dos outros alunos em sala de aula.

Agora, como podemos proceder quando temos duas variáveis X e Y? Em uma aula anterior, usamos a função de correlação para medir a força da relação linear entre duas variáveis. Para a maioria das aplicações, saber que esse relacionamento linear existe não é suficiente. Queremos ser capazes de entender a natureza do relacionamento. É aqui que vamos usar a regressão linear simples.

Vamos olhar para uma bas bases dados que ajudaram a criar a regressão linear.

## Poor Person's KNN  (Vizinhos Próximos Implementado de Forma Simples)

Vamos iniciar com a base de dados original analisada por [Galton](https://en.wikipedia.org/wiki/Francis_Galton). Tal base captura altura de um filho, após alguns anos, com base na altura média dos pais. O problema sendo estudado era prever qual vai ser a altura futura de uma criança usando apenas a idade dos pais como entrada. Como temos um pai e uma mãe, usamos a média das alturas entre os dois. Então temos que:

1. X é a altura média dos pais
1. Y é a altura da criança após alguns anos (o mesmo número de anos para toda criança)

```python
df = pd.read_csv('https://media.githubusercontent.com/media/icd-ufmg/material/master/aulas/16-RegressaoLinear/galton.csv')
df.head()
```

      family
      father
      mother
      midparentHeight
      children
      childNum
      gender
      childHeight

      0
      1
      78.5
      67.0
      75.43
      4
      1
      male
      73.2

      1
      1
      78.5
      67.0
      75.43
      4
      2
      female
      69.2

      2
      1
      78.5
      67.0
      75.43
      4
      3
      female
      69.0

      3
      1
      78.5
      67.0
      75.43
      4
      4
      female
      69.0

      4
      2
      75.5
      66.5
      73.66
      4
      1
      male
      73.5

Abaixo temos a dispersão dos dados

```python
x = df['midparentHeight'].values
y = df['childHeight'].values

plt.scatter(x, y, alpha=0.8, edgecolors='k', s=80)
plt.xlabel('Altura média dos pais')
plt.ylabel('Altura dos filhos')
plt.title('Regressão linear simples')
despine()
```

Uma forma de capturar a correlação é fazer uma regressão de vizinhos próximos (nearest neighbors). Para tal, precisamos agrupar, no eixo-x, pontos próximos um dos outros. É possível fazer o mesmo nos eixos x e y usando uma distância euclideana. Por simplicidade vamos agrupar apenas em x. No caso particular destes dados onde os valores são bem espalhados no eixo x, podemos simplesmente arredondar os números

```python
copy = df[['midparentHeight', 'childHeight']].copy()
copy['midparentHeight'] = copy['midparentHeight'].round()
```

Agrupar por x e tirar a média de y.

```python
model = copy.groupby('midparentHeight').mean()
model.head()
```

      childHeight

      midparentHeight

      64.0
      63.000000

      65.0
      64.229630

      66.0
      64.350000

      67.0
      65.535632

      68.0
      66.240458

Observando o modelo.

```python
x_nn = model.index
y_nn = model.values
plt.scatter(x, y, alpha=0.8, edgecolors='k', s=80)
plt.plot(x_nn, y_nn, color='magenta')
plt.xlabel('Altura média dos pais')
plt.ylabel('Altura dos filhos')
plt.title('Regressão linear simples')
```

Previsão: Para um novo ponto, arredondar seu peso. Ver valor de y_nn

```python
idade_media_pais = 71
model.loc[71.0]
```

    childHeight    67.740594
    Name: 71.0, dtype: float64

Observe que o modelo acima é muito perto de uma reta. Este é um dos pontos e partidas para entender a correlação linear. Precisamos de alguma forma:

1. Capturar a média de valores próximos um dos outros em x
1. Capturar a média de valores próximos um dos outros em y

Resolvemos o (1) problema nos dados acima com um round. No caso (2) com um group-by e mean. Observe como o modelo é quase o mesmo quando z-normalizamos os dados.

```python
znormed = df[['midparentHeight', 'childHeight']].copy()
znormed = (znormed - znormed.mean()) / znormed.std(ddof=1)
znormed.head()
```

      midparentHeight
      childHeight

      0
      3.452802
      1.803190

      1
      3.452802
      0.685637

      2
      3.452802
      0.629760

      3
      3.452802
      0.629760

      4
      2.470762
      1.887006

```python
x = znormed['midparentHeight'].copy()
y = znormed['childHeight'].copy()
```

```python
znormed['midparentHeight'] = znormed['midparentHeight'].round()
model = znormed.groupby('midparentHeight').mean()
```

```python
x_nn = model.index
y_nn = model.values
plt.scatter(x, y, alpha=0.8, edgecolors='k', s=80)
plt.plot(x_nn, y_nn, color='magenta')
plt.xlabel('Altura média dos pais')
plt.ylabel('Altura dos filhos')
plt.title('Regressão linear simples')
despine()
```

# Regressão Linear Simples

Nos slides da aula e no material do [](inferentialthinking.org), leia a discussão sobre como uma regressão linear tenta capturar a mesma ideia do nosso KNN simples acima. De forma simples, queremos uma reta possível de explicar tando a dispersão em x quanto em y. Tal reta deve se aproximar da média dos dois valores quando vistos em janelas (estilo o NN acima). Em particular, você supõe que existem constantes $\alpha$ (alfa) e $\beta$ (beta) tais que:

$$\hat{y}_i = \beta x_i + \alpha + \epsilon_i$$

Idealmente, vamos conseguir capturar o valor médio de $y_i$ para grupos de valores próximos em $x_i$. $\epsilon_i$ é um termo de erro (esperançosamente pequeno) que representa o fato de que existem outros fatores não explicados por este modelo simples.

Como escolhemos alfa e beta? Bem, qualquer escolha de alfa e beta nos dá uma saída prevista para cada entrada $x_i$. Como sabemos a saída real $y_i$, podemos calcular o erro de cada par. Computa o valor $\epsilon_i$.

```python
def error(alpha, beta, x, y):
    model = beta * x + alpha
    return y - model
```

Observe o erro médio para um primeiro modelo

```python
x = df['midparentHeight'].values
y = df['childHeight'].values

alpha = 1
beta = 0.5
e = error(alpha, beta, x, y)
e.mean()
```

    31.142544967880085

```python
x = df['midparentHeight'].values
y = df['childHeight'].values

y_model = beta * x + alpha

plt.scatter(x, y, alpha=0.8, edgecolors='k', s=80)
plt.plot(x, y_model, color='magenta')
plt.xlabel('Altura média dos pais')
plt.ylabel('Altura dos filhos')
plt.title('Regressão linear simples')
```

Agora para um segundo.

```python
alpha = 22
beta = 0.5
e = error(alpha, beta, x, y)
e.mean()
```

    10.142544967880088

```python
x = df['midparentHeight'].values
y = df['childHeight'].values

y_model = beta * x + alpha
plt.scatter(x, y, alpha=0.8, edgecolors='k', s=80)
plt.plot(x, y_model, color='magenta')
plt.xlabel('Altura média dos pais')
plt.ylabel('Altura dos filhos')
plt.title('Regressão linear simples')
```

O segundo modelo é bem melhor. Porém os dois são péssimos. Para facilitar a nossa vida, vamos fazer uso da soma dos erros ao quadrado. Isto vai nos ajudar nas derivadas.

Somamos os erros quadrados:

```python
def sum_of_squared_errors(alpha, beta, x, y):
    return (error(alpha, beta, x, y) ** 2).sum()
```

A *solução de mínimos quadrados (least squares solution)*  é escolher o alfa e o beta que tornam o `sum_of_squared_errors` o menor possível. Para chegar em tais valores, podemos ver o erro como uma função de $\alpha$, $\beta$. Vamos iniciar com o caso normalizado. Aqui $\alpha=0$.

### Dados Normalizados

Lembre-se que na z-normalização, os dados de X e Y ficam com média 0 e desvio 1. Em particular: $x'_i = \frac{x_i - \bar{x}}{s_x}$ e $y'_i = \frac{y_i - \bar{y}}{s_y}$. Note como a média dos dados tanto em X quanto em Y é zero:

$$\bar{x'} = n^{-1} \sum_i \frac{x_i - \bar{x}}{s_x}$$

$$\bar{x'} = n^{-1} s^{-1}_x (\sum_i x_i - \bar{x} \sum_i 1)$$

$$\bar{x'} = n^{-1} s^{-1}_x (\sum_i x_i - \bar{x} n)$$

$$\bar{x'} = n^{-1} s^{-1}_x (\sum_i x_i - x_i)$$

$$\bar{x'} = n^{-1} s^{-1}_x (0) = 0$$

Além do mais, o desvio padrão é 1. Provando:

$$s_{x'} = n^{-1} \sum_i (\frac{x_i - \bar{x}}{s_x} - 0)^2$$
$$s_{x'} = s^{-1}_x n^{-1} \sum_i (x_i - \bar{x})^2$$
$$s_{x'} = s^{-1}_x s_x = 1$$

Como a média dos pontos é (0, 0), a nossa melhor reta vai passar pela origem. Ou seja, $\alpha=0$. Além do mais,  cada ponto dos seus dados é uma reta entre (0, 0) e o ponto $(x'_i, y'_i)$. Neste caso, resolver o problema abaixo para chegar na equação da reta:

$$L(\beta) = \sum_i (y'_i - \hat{y}'_i)^2$$

$$L(\beta) = \sum_i (y'_i - \beta x'_i)^2$$

$$L(\beta) = \sum_i y'^2_i - 2 \beta x'_i y'_i + \beta^2 x'^2_i$$

Derivando a função em $\beta$:

$$\frac{d}{d\beta} L(\Theta) = -\sum_i 2 x'_i y'_i + 2 \beta x'^2_i$$

Fazendo $\frac{d}{d\beta}=0$:

$$\beta = \frac{\sum_i x'_i y'_i}{x'^2_i}$$

Nos slides discutimos como tal solução captura é uma função da covariância dos dados.

### Dados Originais

Agora vamos resolver sem normalizar os dados. Vamos definir $\Theta = [\alpha, \beta]$, isto é, um vetor com alfa e beta.

$$L(\Theta) = \sum_i (y_i - \hat{y}_i)^2$$

$$L(\Theta) = \sum_i (y_i - \beta x_i + \alpha)^2$$

Resolvendo o quadrado dentro do somatório temos:

$$L(\Theta) = \sum_i (y_i^2 + \alpha^2 + x_i^2\beta^2 - 2 \beta \alpha x_i - 2 \beta x_i y_i + 2\alpha y_i)$$

Derivando em relação a $\alpha$:

$$\frac{d}{d\alpha} L(\Theta)= \sum_i (2\alpha - 2 \beta x_i + 2\beta y_i)$$

Derivando em relação a $\beta$:

$$\frac{d}{d\beta} L(\Theta) = \sum_i (2x_i^2\beta - 2 \alpha x_i + 2 x_i y_i)$$

Para resolver otimizar esta função precisamos setar as duas derivadas para zero.

$$\frac{d}{d\alpha} L(\Theta) = 0$$

$$\frac{d}{d\beta} L(\Theta) = 0$$

Após isto, podemos resolver os valores ótimos de $\alpha$ e $\beta$. Note que esta é uma otimização um pouco chata de resolve.

\begin{align}
 \alpha & = \bar{y} - \beta\,\bar{x}, \\[5pt]
  \beta &= \frac{ \sum_{i=1}^n (x_i - \bar{x})(y_i - \bar{y}) }{ \sum_{i=1}^n (x_i - \bar{x})^2 } \\[6pt]
            &= \frac{ \operatorname{Cov}(x, y) }{ \operatorname{Var}(x) } \\[5pt]
            &= r_{xy} \frac{s_y}{s_x}. \\[6pt]
\end{align}

```python
def least_squares_fit(x, y):
    correlationxy = ss.pearsonr(x, y)[0]
    beta = correlationxy * np.std(y, ddof=1) / np.std(x, ddof=1)
    alpha = np.mean(y) - beta * np.mean(x)
    return alpha, beta
```

Sem passar pela matemática exata, vamos pensar em por que isso pode ser uma solução razoável. A escolha do alfa simplesmente diz que quando vemos o valor médio da variável independente $x$, predizemos o valor médio da variável dependente $y$.

A escolha de beta significa que quando o valor de entrada aumenta pelo desvio padrão de $x$, a previsão de $y$ aumenta pela correlação entre $x$ e $y$ multiplicada pelo desvio padrão de $y$ (`correlationxy * np.std(y)`). No caso em que $x$ e $y$ estão perfeitamente correlacionados, um aumento de um desvio padrão em $x$ resulta em um aumento de um desvio padrão de $y$ na predição. Quando eles são perfeitamente anti-correlacionados, o aumento em $x$ resulta em uma *diminuição* no valor da previsão. E quando a correlação é *zero*, o beta é *zero*, o que significa que as alterações em $x$ não afetam a previsão.

Para um pouco mais sobre a intuição dessa fórmula, veja este [vídeo](https://www.khanacademy.org/math/ap-statistics/bivariate-data-ap/least-squares-regression/v/calculating-the-equation-of-a-regression-line).

```python
x = df['midparentHeight'].values
y = df['childHeight'].values
alpha, beta = least_squares_fit(x, y)
y_model = beta * x + alpha
plt.scatter(x, y, alpha=0.8, edgecolors='k', s=80)
plt.plot(x, y_model, color='magenta')
plt.xlabel('Altura média dos pais')
plt.ylabel('Altura dos filhos')
plt.title('Regressão linear simples')
```

```python
alpha, beta
```

    (22.636240549589736, 0.6373608969694791)

Isso fornece valores de alfa = 22,63 e beta = 0,63. Portanto, podemos dizer que crianças vão ser aproximadamente 22 polegadas maiores do que a 63\% da média da altura dos seus pais.

É claro que precisamos de uma maneira melhor de descobrir o quão bem nós ajustamos os dados do que simplesmente encarar o gráfico da regressão. Uma medida comum é o coeficiente de determinação (ou R-quadrado, ou $R^2$, ou R-dois), que mede a fração da variação total na variável dependente (o $y$) que é capturada pelo modelo:

```python
def total_sum_of_squares(y):
    return ((y - y.mean()) ** 2).sum()

def r_squared(alpha, beta, x, y):
    return 1.0 - (sum_of_squared_errors(alpha, beta, x, y) /
                  total_sum_of_squares(y))

```

```python
r_squared(alpha, beta, x, y)
```

    0.10300883578346631

Agora, escolhemos o alfa e o beta que minimizaram a soma dos quadrados dos erros de previsão. Um modelo linear que poderíamos ter escolhido é "sempre prever a média de $y$" (correspondendo a `alpha = np.mean(y)` e `beta = 0`), cuja soma dos erros quadrados é exatamente igual a sua soma total de quadrados. Isso significa um $R^2$ de zero, que indica um modelo que (obviamente, neste caso) não funciona melhor do que apenas prever a média.

Claramente, o modelo de mínimos quadrados deve ser pelo menos tão bom quanto esse, o que significa que a soma dos erros quadrados é no máximo a soma total de quadrados, o que significa que o $R^2$ deve ser pelo menos zero. E a soma dos erros quadrados deve ser pelo menos $0$, o que significa que o $R^2$ pode ser no máximo $1$.

Quanto maior for o $R^2$, melhor o nosso modelo se ajusta aos dados. Aqui nós calculamos um $R^2$ de $0.10$, o que nos diz que o nosso modelo está fraco ajustar os dados, e que claramente existem outros fatores em jogo.

Observe como o modelo para, visualmente, bem pior ao plotar em outra escala. Existe muita variabilidade no eixo-y para ser capturada.

```python
x = df['midparentHeight'].values
y = df['childHeight'].values
alpha, beta = least_squares_fit(x, y)
y_model = beta * x + alpha
plt.scatter(x, y, alpha=0.8, edgecolors='k', s=80)
plt.plot(x, y_model, color='magenta')
plt.xlabel('Altura média dos pais')
plt.ylabel('Altura dos filhos')
plt.title('Regressão linear simples')
plt.xlim((55, 80))
plt.ylim((55, 80))
```

Por fim, a raiz de $R^2$ é a correlação.

```python
r_squared(alpha, beta, x, y) ** 0.5
```

    0.32094989606395935

```python
ss.pearsonr(x, y)[0]
```

    0.3209498960639593

Formas de interpretar o $R^2$.

1. Quão melhor é o meu modelo de um que retorna apenas a média de y.
1. Quanto da variancia de y é explicada por x. Apenas 10% acima.

Para entender o segundo ponto, o `total_sum_of_squares` é proporcional a variância. Na verdade é $s_y$ * (n-1). Ou seja, sem tirar a média dos desvios quadrados. Sendo $\hat{y}_i$ a nossa previsão.

$$R^2 = \sum_i \frac{(y_i - \bar{y})^2 - (y_i - \hat{y}_i)^2}{(y_i - \bar{y})^2}$$

$(y_i - \bar{y})^2$ define nossa variância. Quanto dela está em $(y_i - \hat{y}_i)^2$. Esta é outra forma de pensar no $R^2$.

## Bootstrap na Regressão

Lembre-se que nossa regressão foi realizada em uma amostra. Portanto, qualque estatística que temos vêm com erros. Os coeficientes da nossa regressão: $\alpha$ e $\beta$ têm erros. Além da correlação $r$ e do $R^2$. Como saber a variablidade de tais estimativas? Bootstrap. O código abaixo faz um Bootstrap de: $\alpha$, $\beta$ e $R^2$.

```python
def bootstrap_regression(df, n=10000, size=None):
    if size is None:
        size = len(df)

    alphas = np.zeros(n)
    betas = np.zeros(n)
    rs = np.zeros(n)

    for i in range(n):
        sample = df.sample(size, replace=True)
        x = sample['midparentHeight'].values
        y = sample['childHeight'].values
        alpha, beta = least_squares_fit(x, y)

        alphas[i] = alpha
        betas[i] = beta
        rs[i] = ss.pearsonr(x, y)[0] ** 2
    return alphas, betas, rs
```

```python
alphas, betas, rs = bootstrap_regression(df)
```

Plotando para alfa

```python
plt.hist(alphas, edgecolor='k')
lower = np.percentile(alphas, 2.5)
upper = np.percentile(alphas, 97.5)
despine()
print(alphas.mean())
print(lower, upper)
```

Beta

```python
plt.hist(betas, edgecolor='k')
lower = np.percentile(betas, 2.5)
upper = np.percentile(betas, 95)
despine()
print(betas.mean())
print(lower, upper)
```

R2

```python
plt.hist(rs, edgecolor='k')
lower = np.percentile(rs, 2.5)
upper = np.percentile(rs, 97.5)
despine()
print(rs.mean())
print(lower, upper)
```

## Statsmodels

Observe como entender tal ideia é interessante. Quanto usamos bibliotecas mais avançadas, tipo o statsmodels, os resultados descrevem: ICs para cada parâmetro e valores-p. No notebook anterior computamos um valor-p da correlação com permutação. Podemos fazer o mesmo para alfa e beta. Classicamente, existem estatísticas para tais valores como a $F$ measure que permitem ics e valores-p teóricos.

```python
import statsmodels.api as sm

stocks = {'Year': [2017,2017,2017,2017,2017,2017,2017,2017,2017,2017,2017,2017,2016,2016,2016,2016,2016,2016,2016,2016,2016,2016,2016,2016],
          'Month': [12, 11,10,9,8,7,6,5,4,3,2,1,12,11,10,9,8,7,6,5,4,3,2,1],
          'Interest_Rate': [2.75,2.5,2.5,2.5,2.5,2.5,2.5,2.25,2.25,2.25,2,2,2,1.75,1.75,1.75,1.75,1.75,1.75,1.75,1.75,1.75,1.75,1.75],
          'Unemployment_Rate': [5.3,5.3,5.3,5.3,5.4,5.6,5.5,5.5,5.5,5.6,5.7,5.9,6,5.9,5.8,6.1,6.2,6.1,6.1,6.1,5.9,6.2,6.2,6.1],
          'Stock_Index_Price': [1464,1394,1357,1293,1256,1254,1234,1195,1159,1167,1130,1075,1047,965,943,958,971,949,884,866,876,822,704,719]
         }

df = pd.DataFrame(stocks, columns=['Year','Month', 'Interest_Rate', 'Unemployment_Rate', 'Stock_Index_Price'])
```

Oberver o IC da constante (alfa) abaixo. Note que a mesma não é siginificativa. Como o IC abrange números negativos e positivos, não podemos descartar um zero como sendo a média.

```python
X = df['Interest_Rate']
Y = df['Stock_Index_Price']

X = sm.add_constant(X) # adding a constant

model = sm.OLS(Y, X).fit()
predictions = model.predict(X)

print_model = model.summary()
print(print_model)
```

                                OLS Regression Results
    ==============================================================================
    Dep. Variable:      Stock_Index_Price   R-squared:                       0.876
    Model:                            OLS   Adj. R-squared:                  0.870
    Method:                 Least Squares   F-statistic:                     155.0
    Date:                Wed, 26 Aug 2020   Prob (F-statistic):           1.95e-11
    Time:                        21:25:27   Log-Likelihood:                -136.94
    No. Observations:                  24   AIC:                             277.9
    Df Residuals:                      22   BIC:                             280.2
    Df Model:                           1
    Covariance Type:            nonrobust
    =================================================================================
                        coef    std err          t      P>|t|      [0.025      0.975]
    ---------------------------------------------------------------------------------
    const           -99.4643     95.210     -1.045      0.308    -296.918      97.990
    Interest_Rate   564.2039     45.317     12.450      0.000     470.221     658.186
    ==============================================================================
    Omnibus:                        5.361   Durbin-Watson:                   0.514
    Prob(Omnibus):                  0.069   Jarque-Bera (JB):                3.647
    Skew:                          -0.927   Prob(JB):                        0.161
    Kurtosis:                       3.459   Cond. No.                         15.8
    ==============================================================================

    Warnings:
    [1] Standard Errors assume that the covariance matrix of the errors is correctly specified.