# NMA Refresher Course: Week 0, Day 4, Tutorial 1
## Calculus: Basics of Differential and Integral Calculus
__Content creators:__ John S Butler, Arvind Kumar with help from Ella Batty

__Content reviewers:__  ??

__Production editors:__ Matthew McCann, Manisha Sinha

---
# Tutorial Objectives

In this tutorial, we will cover aspects of differential calculus that will be frequently used in the main NMA course. We assume that you  have some familiarty with differential calculus, but may be a bit rusty or may not have done much practice.  Specifically the objectives of this tutorial are

*   Get an intuitive understanding of derivative and integration operations
*   Learn to calculate the derivatives of 1- and 2-dimensional functions/signals numerically
*   Familiarize with the concept of neuron transfer function in 1- and 2-dimensions.
*   Familiarize with the idea of numerical integration using Riemann sum
*   Learn about the notion of eigenfunction

```python
#@title Video 1: Why do we care about calculus?
# Insert the ID of the corresponding youtube video
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="781o_1hRtpk", width=854, height=480, fs=1)
print("Video available at https://youtu.be/" + video.id)
video
```

---
# Setup

```python
# Imports
!pip install sympy --quiet

import numpy as np
import scipy.optimize as opt       # import root-finding algorithm
import sympy as sp                 # Python toolbox for symbolic maths
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D # Toolbox for rendring 3D figures
from mpl_toolkits import mplot3d   # Toolbox for rendring 3D figures
```

```python
# @title Figure Settings
import ipywidgets as widgets  # interactive display
from ipywidgets import interact
%config InlineBackend.figure_format = 'retina'
# use NMA plot style
plt.style.use("https://raw.githubusercontent.com/NeuromatchAcademy/course-content/master/nma.mplstyle")
my_layout = widgets.Layout()

fig_w, fig_h = 12, 4.5
my_fontsize = 16
my_params = {'axes.labelsize': my_fontsize,
          'axes.titlesize': my_fontsize,
          'figure.figsize': [fig_w, fig_h],
          'font.size': my_fontsize,
          'legend.fontsize': my_fontsize-4,
          'lines.markersize': 8.,
          'lines.linewidth': 2.,
          'xtick.labelsize': my_fontsize-2,
          'ytick.labelsize': my_fontsize-2}

plt.rcParams.update(my_params)
```

```python
# @title Plotting Functions
def move_sympyplot_to_axes(p, ax):
    backend = p.backend(p)
    backend.ax = ax
    backend.process_series()
    backend.ax.spines['right'].set_color('none')
    backend.ax.spines['bottom'].set_position('zero')
    backend.ax.spines['top'].set_color('none')
    plt.close(backend.fig)

def plot_functions(function, show_derivative, show_integral):

  # For sympy we first define our symbolic variable
  x, y, z, t, f = sp.symbols('x y z t f')

  # We define our function
  if function == 'Linear':
    f = -2*t
    name = r'$-2t$'
  elif function == 'Parabolic':
    f =  t**2
    name = r'$t^2$'
  elif function == 'Exponential':
    f =  sp.exp(t)
    name = r'$e^t$'
  elif function == 'Sine':
    f =  sp.sin(t)
    name = r'$sin(t)$'
  elif function == 'Sigmoid':
    f = 1/(1 + sp.exp(-(t-5)))
    name = r'$\frac{1}{1+e^{-(t-5)}}$'

  if show_derivative and not show_integral:
    # Calculate the derivative of sin(t) as a function of t
    diff_f = sp.diff(f)
    print('Derivative of', f, 'is ', diff_f)

    p1 = sp.plot(f, diff_f, show=False)
    p1[0].line_color='r'
    p1[1].line_color='b'
    p1[0].label='Function'
    p1[1].label='Derivative'
    p1.legend=True
    p1.title = 'Function = ' + name + '\n'
    p1.show()
  elif show_integral and not show_derivative:

    int_f = sp.integrate(f)
    int_f = int_f - int_f.subs(t, -10)
    print('Integral of', f, 'is ', int_f)

    p1 = sp.plot(f, int_f, show=False)
    p1[0].line_color='r'
    p1[1].line_color='g'
    p1[0].label='Function'
    p1[1].label='Integral'
    p1.legend=True
    p1.title = 'Function = ' + name + '\n'
    p1.show()

  elif show_integral and show_derivative:

    diff_f = sp.diff(f)
    print('Derivative of', f, 'is ', diff_f)

    int_f = sp.integrate(f)
    int_f = int_f - int_f.subs(t, -10)
    print('Integral of', f, 'is ', int_f)

    p1 = sp.plot(f, diff_f, int_f, show=False)
    p1[0].line_color='r'
    p1[1].line_color='b'
    p1[2].line_color='g'
    p1[0].label='Function'
    p1[1].label='Derivative'
    p1[2].label='Integral'
    p1.legend=True
    p1.title = 'Function = ' + name + '\n'
    p1.show()

  else:

    p1 = sp.plot(f, show=False)
    p1[0].line_color='r'
    p1[0].label='Function'
    p1.legend=True
    p1.title = 'Function = ' + name + '\n'
    p1.show()

def plot_alpha_func(t, f, df_dt):

  plt.figure()
  plt.subplot(2,1,1)
  plt.plot(t, f, 'r', label='Alpha function')
  plt.xlabel('Time (au)')
  plt.ylabel('Voltage')
  plt.title('Alpha function (f(t))')
  #plt.legend()

  plt.subplot(2,1,2)
  plt.plot(t, df_dt, 'b', label='Derivative')
  plt.title('Derivative of alpha function')
  plt.xlabel('Time (au)')
  plt.ylabel('df/dt')
  #plt.legend()

def plot_rate_and_gain(I, rate, gain):

    plt.figure()
    plt.subplot(1,2,1)
    plt.plot(I,rate)
    plt.xlabel('Injected current (au)')
    plt.ylabel('Output firing rate (normalized)')
    plt.title('Transfer function')

    plt.subplot(1,2,2)
    # Uncomment to plot
    plt.plot(I[0:-1], gain)
    plt.xlabel('Injected current (au)')
    plt.ylabel('Gain')
    plt.title('Gain')

def plot_charge_transfer(t, PSP, numerical_integral):

  fig, axes = plt.subplots(1, 2)

  axes[0].plot(t, PSP)
  axes[0].set(xlabel = 't', ylabel = 'PSP')

  axes[1].plot(t, numerical_integral)
  axes[1].set(xlabel = 't', ylabel = 'Charge Transferred')
```

---
# Section 1: What is differentiation and integration?

```python
#@title Video 2: What is differentiation and integration?
# Insert the ID of the corresponding youtube video
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="eOyGG3m-7gA", width=854, height=480, fs=1)
print("Video available at https://youtu.be/" + video.id)
video
```

Calculus is a part of mathematics concerned with **continous change**. There are two branches of calculus: differential calculus and integral calculus. Both of these concepts are useful not only in science, but also in daily life. We encounter differentiation and integration everywhere.

Differentiation of a function $f(t)$ gives you the derivative of that function $\frac{d(f(t))}{dt}$. A derivative captures how sensitive a function is to slight changes in the input for different ranges of inputs. Geometrically, the derivative of a function at a certain input is the slope of the function at that input. For example, as you drive, the distance traveled changes continuously with time. If you take the derivative of the distance traveled with respect to time, you get the velocity of the vehicle at each point in time. The velocity tells you the rate of change of the distance traveled at different points in time. If you have slow velocity (a small derivative), the distance traveled doesn't change much for small changes in time. A high velocity (big derivative) means that the distance traveled changes a lot for small changes in time.

The sign of the derivative of a function (or signal) tells whether the signal is increasing or decreasing. For a signal going through changes as a function of time, the derivative will become zero when the signal changes its direction of change (e.g. from increasing to decreasing). That is, at local minimum or maximum values, the slope of the signal will be zero. This property is used in optimizing problems. But we can also use it to find peaks in a signal.

Integration can be thought of as the reverse of differentation. If we integrate the velocity with respect to time, we can calculate the distance traveled. By integrating a function, we are basically trying to find functions that would have the original one as their derivative. When we integrate a function, our integral will have an added unknown scalar constant, $C$.
For example, if $$ g(t) = 1.5t^2 + 4t - 1$$,
our integral function $f(t)$ will be:
$$ f(t) = \int g(t) dt = 0.5t^3 + 2t^2 - t + C$$.

This constant exists because the derivative of a constant is 0 so we cannot know what the constant should be. This is an indefinite integral. If we compute a definite integral, that is the integral between two limits of the input, we will not have this unknown constant and the integral of a function will capture the area under the curve of that function between those two limits.

Some functions, when differentiated or integrated, equal a scalar times the same function. This is a similar idea to eigenvectors of a matrix being those that, when multipled by the matrix, equal a scalar times themselves, as you saw yesterday!

When

\begin{align*}
\frac{d(f(t)}{dt} = a\cdot f(t),
\end{align*}

we say that $f(t)$ is an **eigenfunction** for derivative operator, where $a$ is a scaling factor. Similarly, when

\begin{align*}
\int f(t)dt = a\cdot f(t),
\end{align*}

we say that $f(t)$ is an **eigenfunction** for integral operator.

As you can imagine, working with eigenfunctions can make mathematical analysis easy.

### Interactive Demo 1: Geometrical understanding

In the interactive demo below, you can pick different functions to examine in the drop down menu. You can then choose to show the derivative function and/or the integral function.

For the integral, we have chosen the unknown constant $C$ so that the integral function at the left x-axis limit is 0 (f(t = -10) = 0). So the integral will reflect the area under the curve starting from that position.

For each function:

*  Examine just the function first. Discuss and predict what the derivative and integral will look like. Remember that derivative = slope of function, integral = area under curve from t = -10 to that t.
*  Check the derivative - does it match your expectations?
*  Check the integral - does it match your expectations?
*  Identify whether the function is an eigenfunction for the derivative operator, an eigenfunction for the integral operator, or neither.

```python
# @markdown Execute this cell to enable the widget
function_options = widgets.Dropdown(
    options=['Linear', 'Parabolic', 'Exponential', 'Sine', 'Sigmoid'],
    description='Function',
    disabled=False,
)

derivative = widgets.Checkbox(
    value=False,
    description='Show derivative',
    disabled=False,
    indent=False
)

integral = widgets.Checkbox(
    value=False,
    description='Show integral',
    disabled=False,
    indent=False
)

def on_value_change(change):
    derivative.value = False
    integral.value = False

function_options.observe(on_value_change, names='value')

interact(plot_functions, function = function_options, show_derivative = derivative, show_integral = integral);
```

```python
# to_remove explanation

# The exponential function is an eigenfunction of both the derivative and integral operators
```

---
# Section 2: Analytical & Numerical Differentiation

```python
#@title Video 3: Analytical & Numerical Differentiation
# Insert the ID of the corresponding youtube video
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="C7U8zgI5rdk", width=854, height=480, fs=1)
print("Video available at https://youtu.be/" + video.id)
video
```

In this section, we will delve into how we actually find the derivative of a function, both analytically and numerically.

## Section 2.1: Analytical Differentiation

When we find the derivative analytically, we are finding the exact formula for the derivative function.

To do this, instead of having to do some fancy math every time, we can often consult [an online resource](https://en.wikipedia.org/wiki/Differentiation_rules) for a list of common derivatives, in this case our trusty friend Wikipedia.

If I told you to find the derivative of $f(t) = x^3$, you could consult that site and find in Section 2.1, that if $f(t) = x^r$, then $\frac{d(f(t))}{dt} = rx^{r-1}$. So you would be able to tell me that the derivative of $f(t) = x^3$ is $\frac{d(f(t))}{dt} = 3x^{2}$.

This list of common derivatives often contains only very simple functions. Luckily, as we'll see in the next two sections, we can often break the derivative of a complex function down into the derivatives of more simple components.

### Section 2.1.1: Product Rule
Sometimes we encounter functions which are the product of two functions that both depend on the variable.
How do we take the derivative of such functions? For this we use the [Product Rule](https://en.wikipedia.org/wiki/Product_rule).

\begin{align}
f(t) = u(t)\cdot v(t)\\
\frac{d(f(t))}{dt} = v\cdot \frac{du}{dt} + u\cdot \frac{dv}{dt}\\
\end{align}

#### Coding Exercise 2.1.1: Derivative of the postsynaptic potential alpha function

Let's use the product rule to get the derivative of the post-synaptic potential alpha function. As we saw in Video 3, the shape of the postsynaptic potential is given by the so called alpha function:

\begin{align*}
f(t) = t \cdot exp(-\frac{t}{\tau})
\end{align*}

Here $f(t)$ is a product of $t$ and $exp(-\frac{t}{\tau})$. The variable $\tau$ is the time constant of the synapse.

We have defined $u(t)$ and $v(t)$ in the code below, in terms of the variable $t$ which is an array of time steps from 0 to 10. Define $\frac{du}{dt}$ and $\frac{dv}{dt}$, the compute the full derivative of the alpha function using the product rule. You can always consult wikipedia to figure out $\frac{du}{dt}$ and $\frac{dv}{dt}$!

```python
# Define time, time constant
t = np.arange(0, 10, .1)
tau = 0.5

# Compute alpha function
f = t * np.exp(-t/tau)

# Define u(t), v(t)
u_t = t
v_t = np.exp(-t/tau)

# Define du/dt, dv/dt
du_dt = ...
dv_dt = ...

# Define full derivative
df_dt = ...

# Uncomment below to visualize
#plot_alpha_func(t, f, df_dt)
```

```python
# to_remove solution

# Define time, time constant
t = np.arange(0, 10, .1)
tau = 0.5

# Compute alpha function
f = t * np.exp(-t/tau)

# Define u(t), v(t)
u_t = t
v_t = np.exp(-t/tau)

# Define du/dt, dv/dt
du_dt = 1
dv_dt = -1/tau * np.exp(-t/tau)

# Define full derivative
df_dt = u_t * dv_dt + v_t * du_dt

# Uncomment below to visualize
with plt.xkcd():
  plot_alpha_func(t, f, df_dt)
```

### Section 2.1.2: Chain Rule
Many times we encounter situations in which the variable $a$ is changing with time ($t$) and affecting another variable $r$. How can we estimate the derivative of $r$ with respect to $a$ i.e. $\frac{dr}{da} = ?$

To calculate $\frac{dr}{da}$ we use the [Chain Rule](https://en.wikipedia.org/wiki/Chain_rule).

\begin{align}
\frac{dr}{da} = \frac{dr}{dt}\cdot\frac{dt}{da}
\end{align}

That is, we calculate the derivative of both variables with respect to time and divide the time derivative of $r$ by that of time derivative of $a$.

We will step back from applications for a second: we can use this to simplify taking derivatives of complex functions, as you will see in the next exercise.

#### Math Exercise 2.1.2: Chain Rule

Let's say that:
$$ r(a) = e^{a^4 + 1} $$

What is $\frac{dr}{da}$? This is a more complex function so we can't simply consult a table of common derivatives. Can you use the chain rule to help?

Hint: we didn't define t but you could set t equal to the function in the exponent

```python
# to_remove explanation

"""

We can set t equal to the exponent:
t = a^4 + 1
r(t) = e^t

Then:
 dt/da = 4a^3
 dr/dt = e^t

Now we can use the chain rule:
dr/da = dr/dt * dt/da
      = e^t(4a^3)
      = 4a^3e^{a^4 + 1}
"""
```

### Section 2.2.3: Derivatives in Python using Sympy

There is a useful Python library for getting the analytical derivatives of functions: Sympy. We actually used in Interactive Demo 1, under the hood.

See the following cell for an example of setting up a sympy function and finding the derivative.

```python
# For sympy we first define our symbolic variables
f, t = sp.symbols('f, t')

# Function definition (sigmoid)
f = 1/(1 + sp.exp(-(t-5)))

# Get the derivative
diff_f = sp.diff(f)

# Print the resulting function
print('Derivative of', f, 'is ', diff_f)
```

## Section 2.2: Numerical Differentiation

Formally, the derivative of a function $\mathcal{f}(x)$ at any value $a$ is given by the finite difference formula (FD):

\begin{align*}
FD = \frac{f(a+h) - f(a)}{h}
\end{align*}

As $h\rightarrow 0$, the FD approaches the actual value of the derivative. Let's check this.

*Note that the numerical estimate of the derivative will result
in a time series whose length is one short of the original time series.*

### Interactive Demo 2.2: Numerical Differentiation of the Sine Function

Below, we find the numerical derivative of the sine function for different values of $h$, and and compare the result the analytical solution.

- What values of h result in more accurate numerical derivatives?

```python
# @markdown *Execute this cell to enable the widget.*
def numerical_derivative_demo(h = 0.2):
  # Now lets create a sequence of numbers which change according to the sine function
  dt = 0.01
  tx = np.arange(-10, 10, dt)
  sine_fun = np.sin(tx)

  # symbolic diffrentiation tells us that the derivative of sin(t) is cos(t)
  cos_fun = np.cos(tx)

  # Numerical derivative using difference formula
  n_tx = np.arange(-10,10,h) # create new time axis
  n_sine_fun = np.sin(n_tx) # calculate the sine function on the new time axis
  sine_diff = (n_sine_fun[1:] - n_sine_fun[0:-1]) / h

  fig = plt.figure()
  ax = plt.subplot(111)
  plt.plot(tx, sine_fun, label='sine function')
  plt.plot(tx, cos_fun, label='analytical derivative of sine')

  with plt.xkcd():
    # notice that numerical derivative will have one element less
    plt.plot(n_tx[0:-1], sine_diff, label='numerical derivative of sine')
    plt.xlim([-10, 10])
    plt.xlabel('Time (au)')
    plt.ylabel('f(x) or df(x)/dt')
    ax.legend(loc='upper center', bbox_to_anchor=(0.5, 1.05),
              ncol=3, fancybox=True)
    plt.show()

_ = widgets.interact(numerical_derivative_demo, h = (0.01, 0.5, .02))
```

```python
# to_remove explanation

"""
The smaller the value of $h$, the better the estimate of the derivative of the
function at $x=a$.
"""
```

## Section 2.3: Transfer Function and Gain of a Neuron
When we inject a constant current (DC) in a neuron, its firing rate changes as a function of strength of the injected current. This is called the **input-output transfer function** or just the *transfer function* or *I/O Curve* of the neuron. For most neurons this can be approximated by a sigmoid function e.g.

\begin{align}
rate(I) = \frac{1}{1+\text{e}^{-a*(I-\theta)}} - \frac{1}{exp(a*\theta)} + \eta
\end{align}

where $I$ is injected current, $rate$ is the neuron firing rate and $\eta$ is noise (Gaussian noise with zero mean and $\sigma$ standard deviation).

*You will visit this equation in a different context in Week 3*

### Coding Exercise 2.1: Calculating the Transfer Function and Gain of a Neuron
The slope of a neurons input-output transfer function ($\frac{d(r(I)}{dI}$) is called the **gain** of the neuron, as it tells how the neuron output will change if the input is changed.

Estimate the gain of the following neuron transfer function using numerical differentiaton. We will use our timestep as h.

```python
# @markdown *Execute this cell to enable the numerical differentiation function: `numerical_derivative`*

def numerical_derivative(x, h):
  '''Numerical derivative calculation
    Args:
      x: array of number
      h: time step for differentiation

    Returns:
      Numerical derivative of f for a time step of h
  '''

  dxdt = np.zeros(len(x)-1)
  dxdt = (x[1:] - x[0:-1])/h
  return dxdt
```

```python
def compute_rate_and_gain(I, a, theta, current_timestep):
  """ Compute rate and gain of neuron based on parameters

  Args:
    I (ndarray): different possible values of the current
    a (scalar): parameter of the transfer function
    theta (scalar): parameter of the transfer function
    current_timestep (scalar): the time we're using to take steps

  Returns:
    (ndarray, ndarray): rate and gain for each possible value of I
  """

  ########################################################################
  ## TODO for students: calculate the gain of the neural firing rate ##
  ## Complete line of code and remove
  raise NotImplementedError("Calculate the gain")
  ########################################################################

  # Compute rate
  rate = (1+np.exp(-a*(I-theta)))**-1 - (1+np.exp(a*theta))**-1

  # Compute gain
  gain = ...

  return rate, gain

current_timestep = 0.1
I = np.arange(0, 8, current_timestep)

# Neuron transfer function
a = 1.2     # You can change this value
theta = 5 # You can change this value

# Compute rate and gain
rate, gain = compute_rate_and_gain(I, a, theta, current_timestep)

# Visualize rate and gain
plot_rate_and_gain(I, rate, gain)
```

```python
# to_remove solution
def compute_rate_and_gain(I, a, theta, current_timestep):
  """ Compute rate and gain of neuron based on parameters

  Args:
    I (ndarray): different possible values of the current
    a (scalar): parameter of the transfer function
    theta (scalar): parameter of the transfer function
    current_timestep (scalar): the time we're using to take steps

  Returns:
    (ndarray, ndarray): rate and gain for each possible value of I
  """

  # Compute rate
  rate = (1+np.exp(-a*(I-theta)))**-1 - (1+np.exp(a*theta))**-1

  # Compute gain
  gain = numerical_derivative(rate, current_timestep)

  return rate, gain

current_timestep = 0.1
I = np.arange(0, 8, current_timestep)

# Neuron transfer function
a = 1.2     # You can change this value
theta = 5 # You can change this value

# Compute rate and gain
rate, gain = compute_rate_and_gain(I, a, theta, current_timestep)

# Visualize rate and gain

with plt.xkcd():
  plot_rate_and_gain(I, rate, gain)
```

The slope of the transfer function tells us in which range of inputs the neuron is most sensitive to changes in its input. Change the parameters of the neuron transfer function (i.e. $a$ and $\theta$) and see if you can predict the value of $I$ for which the neuron has maximal slope and which parameter determines the peak value of the gain.

# Section 3: Functions of Multiple Variables

```python
#@title Video 4: Functions of Multiple Variables
# Insert the ID of the corresponding youtube video
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="rLsLOWsNOGw", width=854, height=480, fs=1)
print("Video available at https://youtu.be/" + video.id)
video
```

In the previous section, you looked at function of single variable $t$ or $x$. In most cases, we encounter functions of multiple variables. For example, in the brain, the firing rate of a neuron is a function of both excitatory and inhibitory input rates. In the following, we will look into how to calculate derivatives of such functions.

First, let's create a function of two variables. We take the example of a neuron driven by excitatory and inhibitory inputs. Because this is for illustrative purposes, we will not go in the details of the numerical range of the input and output variables.

In the function below, we assume that the firing rate of a neuron increases motonotically with an increase in excitation and decreases monotonically with an increase in inhibition. The inhibition is modelled as a subtraction. Like for the 1-dimensional transfer function, here we assume that we can approximate the transfer function as a sigmoid function.

```python
# @markdown Execute this cell to visualize the neuron firing rate surface
def sigmoid_function(x,a,theta):
    '''
    Population activation function.

    Expects:
    x     : the population input
    a     : the gain of the function
    theta : the threshold of the function

    Returns:
    the population activation response F(x) for input x
    '''
    # add the expression of f = F(x)
    f = (1+np.exp(-a*(x-theta)))**-1 - (1+np.exp(a*theta))**-1

    return f

# Neuron Transfer function
step_size = 0.1
exc_input = np.arange(2,9,step_size)
inh_input = np.arange(0,7,step_size)
exc_a = 1.2
exc_theta = 2.4
inh_a = 1.
inh_theta = 4.

rate = np.zeros((len(exc_input),len(inh_input)))

for ii in range(len(exc_input)):
  for jj in range(len(inh_input)):
    rate[ii,jj] = sigmoid_function(exc_input[ii],exc_a,exc_theta) - sigmoid_function(inh_input[jj],inh_a,inh_theta)*0.5

with plt.xkcd():
  X, Y = np.meshgrid(exc_input, inh_input)
  fig = plt.figure(figsize=(15,15))
  ax1 = fig.add_subplot(2,2,1)
  lg_txt = 'Inhibition = ' + str(inh_input[0])
  ax1.plot(exc_input,rate[:,0],label=lg_txt)
  lg_txt = 'Inhibition = ' + str(inh_input[20])
  ax1.plot(exc_input,rate[:,20],label=lg_txt)
  lg_txt = 'Inhibition = ' + str(inh_input[40])
  ax1.plot(exc_input,rate[:,40],label=lg_txt)
  ax1.legend()
  ax1.set_xlabel('Excitatory input (au)')
  ax1.set_ylabel('Neuron output rate (au)');

  ax2 = fig.add_subplot(2,2,2)
  lg_txt = 'Excitation = ' + str(exc_input[0])
  ax2.plot(inh_input,rate[0,:],label=lg_txt)
  lg_txt = 'Excitation = ' + str(exc_input[20])
  ax2.plot(inh_input,rate[20,:],label=lg_txt)
  lg_txt = 'Excitation = ' + str(exc_input[40])
  ax2.plot(inh_input,rate[40,:],label=lg_txt)
  ax2.legend()
  ax2.set_xlabel('Inhibitory input (au)')
  ax2.set_ylabel('Neuron output rate (au)');

  ax3 = fig.add_subplot(2, 1, 2, projection='3d')
  surf= ax3.plot_surface(Y.T, X.T, rate, rstride=1, cstride=1,
                  cmap='viridis', edgecolor='none')
  ax3.set_xlabel('Inhibitory input (au)')
  ax3.set_ylabel('Excitatory input (au)')
  ax3.set_zlabel('Neuron output rate (au)');
  fig.colorbar(surf)
```

In the **Top-Left** plot, we see how the neuron output rate increases as a function of excitatory input (e.g. the blue trace). However, as we increase inhibition, expectedly the neuron output decreases and the curve is shifted downwards. This constant shift in the curve suggests that the effect of inhibition is subtractive, and the amount of subtraction does not depend on the neuron output.

We can alternatively see how the neuron output changes with respect to inhibition and study how excitation affects that. This is visualized in the **Top-Right** plot.

This type of plotting is very intuitive, but it becomes very tedious to visualize when there are larger numbers of lines to be plotted. A nice solution to this visualization problem is to render the data as color, as surfaces, or both.

This is what we have done in the plot on the bottom. The colormap on the right shows the output of the neuron as a function of inhibitory input and excitatory input. The output rate is shown both as height along the z-axis and as the color. Blue means low firing rate and yellow means high firing rate (see the color bar).

In the above plot, the output rate of the neuron goes below zero. This is of course not physiological. In models, we either choose the operating point such that the output does not go below zero, or else we clamp the neuron output to zero if it goes below zero. You will learn about it more in Week 3.

## Section 3.1: Partial derivatives
The above function is like a surface and when we are thinking of the derivative of the surface we can make a physical analogy.

Consider putting a ball on this surface. In which direction the ball will move?

The movement along one of the directions (along the x-axis) will be determined by inhibitory input and in the other direction (along y-axis) it will be determined by excitatory inputs. The effective movement direction will be the vector sum of the two (perhaps you recall vector sum from yesterday).

That is, we can calculate the derivative of the surface for the inhibitory input and then for the excitatory inputs.

When we take the derrivative of a multivariable function with respect to one of the variables it is called the **partial derivative**. For example if we have a function:

\begin{align}
f(x,y) = x^2 + 2xy + y^2
\end{align}

The we can define the partial derivatives as

\begin{align}
\frac{\partial(f(x,y))}{\partial x} = 2x + 2y + 0 \\\\
\frac{\partial(f(x,y))}{\partial y} = 0 + 2x + 2y
\end{align}

In the above, the derivative of the last term ($y^2$) with respect to $x$ is zero because it does not change with respect to $x$. Similarly, the derivative of $x^2$ with respect to $y$ is also zero.

### Interactive Demo 3.1: Visualize partial derivatives

In the demo below, you can input any function of x and y and then visualize both the function and partial derivatives.

We visualized the 2-dimensional function as a surface plot in which the values of the function are rendered as color. Yellow represents a high value and blue represents a low value. The height of the surface also shows the numerical value of the function. The first plot is that of our function. And the two bottom plots are the derivative surfaces with respect to $x$ and $y$ variables.

1.   Ensure you understand how the plots relate to each other - if not, review the above material
2.   Can you come up with a function where the partial derivative with respect to x will be a linear plane and the derivative with respect to y will be more curvy?
3.   What happens to the partial derivatives if there are no terms involving multiplying x and y together?

```python
# @markdown Execute this widget to enable the demo

# Let's use sympy to calculate Partial derivatives of a function of 2-variables
@interact(f2d_string = 'x**2 + 2*x*y + y**2')
def plot_partial_derivs(f2d_string):
  f, x, y = sp.symbols('f, x, y')

  f2d = eval(f2d_string)
  f2d_dx = sp.diff(f2d,x)
  f2d_dy = sp.diff(f2d,y)

  print('Partial derivative of ', f2d, 'with respect to x is', f2d_dx)
  print('Partial derivative of ', f2d, 'with respect to y is', f2d_dy)

  p1 = sp.plotting.plot3d(f2d, (x, -5, 5), (y, -5, 5),show=True,xlabel='x', ylabel='y', zlabel='f(x,y)',title='Our function')

  p2 = sp.plotting.plot3d(f2d_dx, (x, -5, 5), (y, -5, 5),show=True,xlabel='x', ylabel='y', zlabel='df(x,y)/dx',title='Derivative w.r.t. x')

  p3 = sp.plotting.plot3d(f2d_dy, (x, -5, 5), (y, -5, 5),show=True,xlabel='x', ylabel='y', zlabel='df(x,y)/dy',title='Derivative w.r.t. y')
```

```python
# to_remove explanation

"""
2) Change a term involving y to involve more than y or y^2. For example, x**2 +  2*x*y + y**3

3) If there are no terms involving both x and y, the partial derivative with respect to x will
   just depend on x and the partial derivative with respect to y will just depend on y.
"""
```

### Section 3.2: Numerical calculation of partial derivatives
Now that you have an intuition about multivariable functions and partial derivatives we can go back to the neuron transfer function we evaluated earlier.
To evaluate the partial derivatives we can use the same numerical differentiation as before but now we apply it to each row and column separately.

```python
# @markdown Execute this cell to visualize the transfer function
# Neuron Transfer Function
step_size = 0.1
exc_input = np.arange(1,10,step_size)
inh_input = np.arange(0,7,step_size)
exc_a = 1.2
exc_theta = 2.4
inh_a = 1.
inh_theta = 4.

rate = np.zeros((len(exc_input),len(inh_input)))
for ii in range(len(exc_input)):
  for jj in range(len(inh_input)):
    rate[ii,jj] = sigmoid_function(exc_input[ii],exc_a,exc_theta) - sigmoid_function(inh_input[jj],inh_a,inh_theta)*0.5

# Derivative with respect to excitatory input rate
rate_de = np.zeros((len(exc_input)-1,len(inh_input)))# this will have one row less than the rate matrix
for ii in range(len(inh_input)):
  rate_de[:,ii] = (rate[1:,ii] - rate[0:-1,ii])/step_size

# Derivative with respect to inhibitory input rate
rate_di = np.zeros((len(exc_input),len(inh_input)-1))# this will have one column less than the rate matrix
for ii in range(len(exc_input)):
  rate_di[ii,:] = (rate[ii,1:] - rate[ii,0:-1])/step_size

with plt.xkcd():
  X, Y = np.meshgrid(exc_input, inh_input)
  fig = plt.figure(figsize=(20,8))
  ax1 = fig.add_subplot(1, 3, 1, projection='3d')
  surf1 = ax1.plot_surface(Y.T, X.T, rate, rstride=1, cstride=1, cmap='viridis', edgecolor='none')
  ax1.set_xlabel('Inhibitory input (au)')
  ax1.set_ylabel('Excitatory input (au)')
  ax1.set_zlabel('Neuron output rate (au)')
  ax1.set_title('Rate as a function of Exc. and Inh');
  ax1.view_init(45, 10)
  fig.colorbar(surf1)

  Xde, Yde = np.meshgrid(exc_input[0:-1], inh_input)
  ax2 = fig.add_subplot(1, 3, 2, projection='3d')
  surf2 = ax2.plot_surface(Yde.T, Xde.T, rate_de, rstride=1, cstride=1, cmap='viridis', edgecolor='none')
  ax2.set_xlabel('Inhibitory input (au)')
  ax2.set_ylabel('Excitatory input (au)')
  ax2.set_zlabel('Neuron output rate (au)');
  ax2.set_title('Derivative wrt Excitation');
  ax2.view_init(45, 10)
  fig.colorbar(surf2)

  Xdi, Ydi = np.meshgrid(exc_input, inh_input[:-1])
  ax3 = fig.add_subplot(1, 3, 3, projection='3d')
  surf3 = ax3.plot_surface(Ydi.T, Xdi.T, rate_di, rstride=1, cstride=1, cmap='viridis', edgecolor='none')
  ax3.set_xlabel('Inhibitory input (au)')
  ax3.set_ylabel('Excitatory input (au)')
  ax3.set_zlabel('Neuron output rate (au)');
  ax3.set_title('Derivative wrt Inhibition');
  ax3.view_init(15, -115)
  fig.colorbar(surf3)
```

Is this what you expeced? Vary the inputs and see if your intuitions are correct. Change the time varying variable and test your intuitions.

---
# Section 4: Numerical Integration

```python
#@title Video 5: Numerical Integration
# Insert the ID of the corresponding youtube video
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="sj_83_811j0", width=854, height=480, fs=1)
print("Video available at https://youtu.be/" + video.id)
video
```

Geometrically, integration is the area under the curve. This interpretation gives two formal ways to calculate the integral of a function numerically.

**[Riemann sum](https://en.wikipedia.org/wiki/Riemann_sum)**:
If we wish to integrate a function $f(t)$ with respect to $t$, then first we divide the function into $n$ intervals of size $dt = a-b$, where $a$ is the starting of the interval. Thus, each interval gives a rectangle with height $f(a)$ and width $dt$. By summing the area of all the rectangles, we can approximate the area under the curve. As the size $dt$ approaches to zero, our estimate of the integral approcahes the analytical calculation. Essentially, the Riemann sum is cutting the region under the curve in vertical stripes, calculating area of the each stripe and summing them up.

**[Lebesgue integral](https://en.wikipedia.org/wiki/Lebesgue_integral)**: In the Lebesgue integral, we divide the area under the curve into horizontal stripes. That is, instead of the independent variable, the range of the function $f(t)$ is divided into small intervals.

## Section 4.1: Demonstration of the Riemann Sum

### Interactive Demo 4.1: Riemann Sum vs. Analytical Integral with changing step size

Below, we will compare numerical integration using the Riemann Sum with the analytical solution. You can change the interval size $dt$ using the slider.

1.   What values of dt result in the best numerical integration?
2.   What is the downside of choosing that value of dt?
3.   With large dt, why are we underestimating the integral (as opposed to overestimating?

```python
# @markdown Run this cell to enable the widget!
def riemann_sum_demo(dt = 0.5):
  step_size = 0.1
  min_val = 0.
  max_val = 10.
  tx = np.arange(min_val, max_val, step_size)

  # Our function
  ftn = tx**2 - tx + 1
  # And the integral analytical formula calculates using sympy
  int_ftn = tx**3/3 - tx**2/2 + tx

  # Numerical integration of f(t) using Riemann Sum
  n = int((max_val-min_val)/dt)
  r_tx = np.zeros(n)
  fun_value = np.zeros(n)
  for ii in range(n):
    a = min_val+ii*dt
    fun_value[ii] = a**2 - a + 1
    r_tx[ii] = a;

  # Riemann sum is just cumulative sum of the fun_value multiplied by the
  r_sum = np.cumsum(fun_value)*dt
  with plt.xkcd():
    plt.figure(figsize=(20,5))
    ax = plt.subplot(1,2,1)
    plt.plot(tx,ftn,label='Function')

    for ii in range(n):
      plt.plot([r_tx[ii], r_tx[ii], r_tx[ii]+dt, r_tx[ii]+dt], [0, fun_value[ii], fun_value[ii], 0] ,color='r')

    plt.xlabel('Time (au)')
    plt.ylabel('f(t)')
    plt.title('f(t)')
    plt.grid()

    plt.subplot(1,2,2)
    plt.plot(tx,int_ftn,label='Analytical')
    plt.plot(r_tx+dt,r_sum,color = 'r',label='Riemann Sum')
    plt.xlabel('Time (au)')
    plt.ylabel('int(f(t))')
    plt.title('Integral of f(t)')
    plt.grid()
    plt.legend()
    plt.show()

_ = widgets.interact(riemann_sum_demo, dt = (0.1, 1., .02))

```

```python
# to_remove explanation

"""
1. The smallest value of dt results in the best numerical integration (most similar
   to the analytical solution)

2. We have to compute the area of more rectangles.

3. Here we used the forward Riemann sum, and it results in an underestimate of the
   integral for positive values of $T$ and underestimate for negativeve values of
   $T$. We could also use the backward Riemann sum, and that will give an overestimate
   of the integral for positive values of $T$.

"""
```

There are more advanced methods of numerical integration, such as Rungga-Kutta. In any case, the Riemann sum is the basis of Euler's method of integration for solving ordinary differential equations - something you will do in a later tutorial today.

See Bonus Section 1 to work through some examples of neural applications of numerical integration.

## Section 4.2: Neural Applications of Numerical Integration

### Coding Exercise 4.2: Calculating Charge Transfer with Excitatory Input
An incoming spike elicits a change in the post-synaptic membrane potential which can be captured by the following function

\begin{align}
PSP(t) = J\times t\times exp\big(-\frac{t-t_{sp}}{\tau_{s}}\big)
\end{align}

where $J$ is the synaptic amplitude, $t_{sp}$ is the spike time and $\tau_s$ is the synaptic time constant.

Estimate the total charge transfered to the postsynaptic neuron during an PSP with amplitude $J=1.0$, $\tau_s = 1.0$ and $t_{sp} = 1.$ (that is the spike occured at 1ms). The total charge will be the integral of the PSP function.

```python
# Set up parameters
J = 1
tau_s = 1
t_sp = 1
dt = .1
t = np.arange(0, 10, dt)

# Code PSP formula
PSP = ...

# Compute numerical integral
# We already have PSP at every time step (height of rectangles). We need to
#.  multiply by width of rectangles (dt) to get areas
rectangle_areas = ...

# Cumulatively sum rectangles (hint: use np.cumsum)
numerical_integral = ...

# Visualize
# plot_charge_transfer(t, PSP, numerical_integral)
```

```python
# to_remove solution

# Set up parameters
J = 1
tau_s = 1
t_sp = 1
dt = .1
t = np.arange(0, 10, dt)

# Code PSP formula
PSP = J * t * np.exp(- (t-t_sp)/tau_s)

# Compute numerical integral
# We already have PSP at every time step (height of rectangles). We need to
#.  multiply by width of rectangles (dt) to get areas
rectangle_areas = PSP *dt

# Cumulatively sum rectangles (hint: use np.cumsum)
numerical_integral = np.cumsum(rectangle_areas)

# Visualize
with plt.xkcd():
  plot_charge_transfer(t, PSP, numerical_integral)
```

You can see from the figure that the total charge transferred is a little over 2.5.

---
# Section 5: Integration and Differentiation as Filtering Operations

```python
#@title Video 6: Filtering Operations
# Insert the ID of the corresponding youtube video
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="TQ0t-S3__OA", width=854, height=480, fs=1)
print("Video available at https://youtu.be/" + video.id)
video
```

In the above, we used the notions that geometrically integration is the area under the curve and differentiation is the slope of the curve. There is another interpretation of these two operations.

As we calculate the derivative of a function, we take the difference of adjacent values of the function. This results in the removal of common part between the two values. As a consequence, we end up removing the unchanging part of the signal. If we now think in terms of frequencies, differentiation removes low frequencies, or slow changes. That is, differentiation acts as a high pass filter.

Integration does the opposite because in the estimation of an integral we keep adding adjacent values of the signal. So, again thinking in terms of frequencies, integration is akin to the removal of high frequencies or fast changes (low-pass filter). The shock absorbers in your bike are an example of integrators.

We can see this behavior the demo below. Here we will not work with functions, but with signals. As such, functions and signals are the same. Just that in most cases our signals are measurements with respect to time.

```python
# @markdown Execute this cell to see visualization
h = 0.01
tx = np.arange(0,2,h)
noise_signal = np.random.uniform(0,1,(len(tx)))*0.5
x1 = np.sin(0.5*np.pi*tx) + noise_signal # This will generate a 1 Hz sin wave
# In the signal x1 we have added random noise which contributs the high frequencies

# Take the derivative equivalent of the signal i.e. subtract the adjacent values
x1_diff = (x1[1:] - x1[:-1])

# Take the integration equivalent of the signal i.e. sum the adjacent values. Ans divide by 2 (take average essentially)
x1_integrate = (x1[1:] + x1[:-1])/2

plt.figure(figsize=(15,10))
plt.subplot(3,1,1)
plt.plot(tx,x1,label='Original Signal')
#plt.xlabel('Time (sec)')
plt.ylabel('Signal Value(au)')
plt.legend()

plt.subplot(3,1,2)
plt.plot(tx[0:-1],x1_diff,label='Differentiated Signal')
# plt.xlabel('Time (sec)')
plt.ylabel('Differentiated Value(au)')
plt.legend()

plt.subplot(3,1,3)
plt.plot(tx,x1,label='Original Signal')
plt.plot(tx[0:-1],x1_integrate,label='Integrate Signal')
plt.xlabel('Time (sec)')
plt.ylabel('Integrate Value(au)')
plt.legend()

```

Notice how the differentiation operation amplifies the fast changes which were contributed by noise. By contrast, the integration operation supresses the fast changing noise. Such sums and subtractions form the basis of digital filters.
Vary the signal characteristic to see how fast you can use these operations to enhance or supress noise.

Also if you want to be adventurous, you may try to use these operations in series and see what happens.

---
# Summary
* Geometrically, integration is the area under the curve and differentiation is the slope of the function
* The concepts of slope and area can be easily extended to higher dimensions. We saw this when we took the derivative of a 2-dimensional transfer function of a neuron
* Numerical estimates of both derivatives and integrals require us to choose a time step $h$. The smaller the $h$, the better the estimate, but for small values of $h$, more computations are needed. So there is always some tradeoff.
* Partial derivatives are just the estimate of the slope along one of the many dimensions of the function. We can combine the slopes in different directions using vector sum to find the direction of the slope.
* Because the derivative of a function is zero at the local peak or trough, derivatives are used to solve optimization problems.
* When thinking of signal, integration operation is equivalent to smoothening the signals (i.e. remove fast changes)
* Differentiation operations remove slow changes and enhance high frequency content of a signal


# Matrices Solutions

```
import sys
import os
sys.path.insert(1, os.path.join(os.path.pardir, "ipython_doctester"))
from sympy import *
from ipython_doctester import test
# Work around a bug in IPython. This will disable the ability to paste things with >>>
def notransform(line): return line
from IPython.core import inputsplitter
inputsplitter.transform_classic_prompt = notransform
```

Use `row_del` and `row_insert` to go from one Matrix to the other.

```
@test
def matrix1(M):
    """
    >>> M = Matrix([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
    >>> M
    [1, 2, 3]
    [4, 5, 6]
    [7, 8, 9]
    >>> matrix1(M)
    [4, 5, 6]
    [0, 0, 0]
    [7, 8, 9]
    """
    M.row_del(0)
    M = M.row_insert(1, Matrix([[0, 0, 0]]))
    return M
```

Success!

## Matrix Constructors

Use the matrix constructors to construct the following matrices.  There may be more than one correct answer.

$$\left[\begin{array}{ccc}4 & 0 & 0\\\\
0 & 4 & 0\\\\
0 & 0 & 4\end{array}\right]$$

```
@test
def matrix2():
    """
    >>> matrix2()
    [4, 0, 0]
    [0, 4, 0]
    [0, 0, 4]
    """
    return eye(3)*4
    # OR return diag(4, 4, 4)
```

Success!

$$\left[\begin{array}{}1 & 1 & 1 & 0\\\\1 & 1 & 1 & 0\\\\0 & 0 & 0 & 1\end{array}\right]$$

```
@test
def matrix3():
    """
    >>> matrix3()
    [1, 1, 1, 0]
    [1, 1, 1, 0]
    [0, 0, 0, 1]
    """
    return diag(ones(2, 3), 1)
    # OR diag(ones(2, 3), ones(1, 1))
```

Success!

$$\left[\begin{array}{}-1 & -1 & -1 & 0 & 0 & 0\\\\-1 & -1 & -1 & 0 & 0 & 0\\\\-1 & -1 & -1 & 0 & 0 & 0\\\\0 & 0 & 0 & 0 & 0 & 0\\\\0 & 0 & 0 & 0 & 0 & 0\\\\0 & 0 & 0 & 0 & 0 & 0\end{array}\right]$$

```
@test
def matrix4():
    """
    >>> matrix4()
    [-1, -1, -1, 0, 0, 0]
    [-1, -1, -1, 0, 0, 0]
    [-1, -1, -1, 0, 0, 0]
    [ 0,  0,  0, 0, 0, 0]
    [ 0,  0,  0, 0, 0, 0]
    [ 0,  0,  0, 0, 0, 0]
    """
    return diag(-ones(3, 3), zeros(3, 3))
    # OR diag(-ones(3, 3), 0, 0, 0)
```

Success!

## Advanced Methods

Recall that if $f$ is an analytic function, then we can define $f(M)$ for any square matrix $M$ by "plugging" $M$ into the power series formula for $f(x)$. In other words, if $$f(x) = \sum_{n=0}^\infty a_n x^n,$$ then we define $f(M)$ by $$f(M) = \sum_{n=0}^\infty a_n M^n,$$ where $M^0$ is $I$, the identity matrix.

Furthermore, if $M$ is a diagonalizable matrix, that is, $M=PDP^{-1}$, where $D$ is diagonal, then $M^n = PD^nP^{-1}$ (because $M^n = \left(PDP^{-1}\right)\left(PDP^{-1}\right)\cdots\left(PDP^{-1}\right)=PD\left(P^{-1}P\right)D\left(P^{-1}P\right)\cdots DP^{-1} = PD^nP^{-1}$).

But if

$$ D = \begin{bmatrix}
         d_1 & 0 & \cdots & 0 \\\\
         0 & d_2 & \cdots & 0 \\\\
         \vdots & \vdots & \ddots & \vdots \\\\
         0 & 0 & \cdots & d_n
      \end{bmatrix}
$$

is a diagonal matrix, then

$$ D^n = \begin{bmatrix}
         d_1^n & 0 & \cdots & 0 \\\\
         0 & d_2^n & \cdots & 0 \\\\
         \vdots & \vdots & \ddots & \vdots \\\\
         0 & 0 & \cdots & d_n^n
      \end{bmatrix}
$$

so that

$$
\sum_{n=0}^\infty a_n M^n = \sum_{n=0}^\infty a_n PD^nP^{-1} = P\cdot\begin{bmatrix}
    \sum_{n=0}^\infty a_n d_1^n & 0 & \cdots & 0 \\\\
         0 & \sum_{n=0}^\infty a_n d_2^n & \cdots & 0 \\\\
         \vdots & \vdots & \ddots & \vdots \\\\
         0 & 0 & \cdots & \sum_{n=0}^\infty a_n d_n^n
\end{bmatrix}\cdot P^{-1} = P\cdot\begin{bmatrix}
    f(d_1) & 0 & \cdots & 0 \\\\
         0 & f(d_2) & \cdots & 0 \\\\
         \vdots & \vdots & \ddots & \vdots \\\\
         0 & 0 & \cdots & f(d_n)
\end{bmatrix}\cdot P^{-1}
$$

Let's create some square matrices, which we will use throughout the exercises.

```
x = symbols('x')
A = Matrix([[1, 1], [1, 0]])
M = Matrix([[3, 10, -30], [0, 3, 0], [0, 2, -3]])
N = Matrix([[-1, -2, 0, 2], [-1, -1, 2, 1], [0, 0, 2, 0], [-1, -2, 2, 2]])
```

First, verify that these matrices are indeed diagonalizable.

```
print A.is_diagonalizable()
print M.is_diagonalizable()
print N.is_diagonalizable()
```

    True
    True
    True

Now, we want to write a function that computes $f(M)$, for diagonalizable matrix $M$ and analytic function $f$.

However, there is one complication. We can use `diagonalize` to get `P` and `D`, but we need to apply the function to the diagonal of `D`. We might think that we could use `eigenvals` to get the eigenvalues of the matrix, since the diagonal values of `D` are just the eigenvalues of `M`, but the issue is that they could be in any order in `D`.

Instead, we can use matrix slicing to get the diagonal values (or indeed, any value) of a matrix. There is not enough time in this tutorial (or room in this document) to discuss the full details of matrix slicing. For now, we just note that `M[i, j]` returns the element at position `i, j` (which is the `i + 1, j + 1`th element of the matrix, due to Python's 0-indexing). For example

```
M
```

    [3, 10, -30]
    [0,  3,   0]
    [0,  2,  -3]

```
M[0, 1]
```

    10

That should be enough information to write the following function.

```
@test
def matrix_func(M, func):
    """
    Computes M at func. Assumes that M is square diagonalizable.

    >>> matrix_func(M, exp)
    [exp(3), -5*exp(-3)/3 + 5*exp(3)/3, -5*exp(3) + 5*exp(-3)]
    [     0,                    exp(3),                     0]
    [     0,     -exp(-3)/3 + exp(3)/3,               exp(-3)]

    Note that for the function exp, we can also just use M.exp()

    >>> matrix_func(M, exp) == M.exp()
    True

    But for other functions, we have to do it this way.

    >>> M.sin()
    Traceback (most recent call last):
    ...
    AttributeError: Matrix has no attribute sin.
    >>> matrix_func(N, sin)
    [-sin(1), -2*sin(1),      0, 2*sin(1)]
    [-sin(1),   -sin(1), sin(2),   sin(1)]
    [      0,         0, sin(2),        0]
    [-sin(1), -2*sin(1), sin(2), 2*sin(1)]

    Note that we could also use this to compute the series expansion of a matrix,
    if we know the closed form of that expansion. For example, suppose we wanted to compute

    I + M + M**2 + M**3 + …

    The series

    1 + x + x**2 + x**3 + …

    is equal to the function 1/(1 - x).

    >>> matrix_func(M, Lambda(x, 1/(1 - x))) # Note, Lambda works just like lambda, but is symbolic
    [-1/2, -5/4, 15/4]
    [   0, -1/2,    0]
    [   0, -1/4,  1/4]
    """
    P, D = M.diagonalize()
    diags = [func(D[i, i]) for i in range(M.shape[0])]
    return P*diag(*diags)*P**-1
```

Success!

Now lets investigate how this works in relation to the series expansion definition. Write a function that uses `matrix_func` and `series` to compute the approximation of a matrix evaluated at a function up to $O(M^n)$.

```
@test
def matrix_func_series(M, func, n):
    """
    Computes the approximation of the func(M) using the series definition up to O(M**n).

    >>> matrix_func_series(M, exp, 10)
    [22471/1120,  14953/448, -44859/448]
    [         0, 22471/1120,          0]
    [         0, 14953/2240,    83/2240]
    >>> matrix_func_series(M, exp, 10).evalf()
    [20.0633928571429, 33.3772321428571,  -100.131696428571]
    [               0, 20.0633928571429,                  0]
    [               0, 6.67544642857143, 0.0370535714285714]
    >>> matrix_func(M, exp).evalf()
    [20.0855369231877, 33.3929164246997,  -100.178749274099]
    [               0, 20.0855369231877,                  0]
    [               0, 6.67858328493993, 0.0497870683678639]

    It's pretty close. Basically what we might expect for those values up to O(x**10).

    >>> matrix_func_series(N, sin, 3)
    [-1, -2, 0, 2]
    [-1, -1, 2, 1]
    [ 0,  0, 2, 0]
    [-1, -2, 2, 2]
    >>> matrix_func(N, sin).evalf()
    [-0.841470984807897,  -1.68294196961579,                 0,  1.68294196961579]
    [-0.841470984807897, -0.841470984807897, 0.909297426825682, 0.841470984807897]
    [                 0,                  0, 0.909297426825682,                 0]
    [-0.841470984807897,  -1.68294196961579, 0.909297426825682,  1.68294196961579]

    It's not as close, because we used O(x**3), but clearly still the same thing.

    >>> matrix_func_series(M, Lambda(x, 1/(1 - x)), 10)
    [29524, 73810, -221430]
    [    0, 29524,       0]
    [    0, 14762,  -14762]
    >>> matrix_func(M, Lambda(x, 1/(1 - x)))
    [-1/2, -5/4, 15/4]
    [   0, -1/2,    0]
    [   0, -1/4,  1/4]

    Woah! That one's not close at all. What is happening here?  Let's try more terms

    >>> matrix_func_series(M, Lambda(x, 1/(1 - x)), 100)
    [257688760366005665518230564882810636351053761000, 644221900915014163795576412207026590877634402500, -1932665702745042491386729236621079772632903207500]
    [                                               0, 257688760366005665518230564882810636351053761000,                                                  0]
    [                                               0, 128844380183002832759115282441405318175526880500,  -128844380183002832759115282441405318175526880500]

    It just keeps getting bigger. In fact, the series diverges. Recall that
    1/(1 - x) = 1 + x + x**2 + x**3 + … *only if* |x| < 1.  But the eigenvalues
    of M are bigger than 1 in absolute value.

    >>> M.eigenvals()
    {3: 2, -3: 1}

    In fact, 1/(1 - M) is mathematically defined via the analytic continuation
    of the series expansion 1 + x + x**2 + …, which is just 1/(1 - x).  This is
    well-defined as long as none of the eigenvalues of M are equal to 1.  Let's
    try it on N.

    >>> matrix_func(N, Lambda(x, 1/(1 - x)))
    [nan, -oo, nan,  oo]
    [nan, nan, nan, nan]
    [nan, nan, nan, nan]
    [nan, -oo, nan,  oo]

    That didn't work. What are the eigenvalues of N?

    >>> N.eigenvals()
    {1: 1, 2: 1, -1: 1, 0: 1}

    Ah, the first one is 1, so we cannot define 1/(1 - N).
    """
    x = Dummy('x') # This works even if func already contains Symbol('x')
    series_func = Lambda(x, func(x).series(x, 0, n).removeO())
    return matrix_func(M, series_func)
```

Success!

```

```


```python
from IPython.display import Image
Image('../../Python_probability_statistics_machine_learning_2E.png',width=200)
```

The Bernoulli distribution we studied earlier answers the question of which of
two outcomes ($Y \in \lbrace 0,1 \rbrace$) would be selected with probability, $p$.

$$
\mathbb{P}(Y) = p^Y (1-p)^{ 1-Y }
$$

 We also know how to solve the corresponding likelihood function for
the maximum likelihood estimate of $p$ given observations of the output,
$\lbrace Y_i \rbrace_{i=1}^n$. However, now we want to include other factors in
our estimate of $p$. For example, suppose we observe not just the outcomes, but
a corresponding continuous variable, $x$. That is, the observed data is now
$\lbrace (x_i,Y_i) \rbrace_{i=1}^n$  How can we incorporate $x$ into our
estimation of $p$?

The most straightforward idea is to model $p= a x + b$ where $a,b$ are
parameters of a fitted line. However, because $p$ is a probability with value
bounded between zero and one, we need to wrap this estimate in another function
that can map the entire real line into the $[0,1]$ interval. The logistic
(a.k.a. sigmoid) function has this property,

$$
\theta(s) = \frac{e^s}{1+e^s}
$$

 Thus, the new parameterized estimate for $p$ is the following,

$$
\begin{equation}
\hat{p} = \theta(a x+b)= \frac{e^{a x + b}}{1+e^{a x + b}}
\label{eq:prob} \tag{1}
\end{equation}
$$

 The *logit* function is defined as the following,

$$
\texttt{logit}(t)= \log \frac{t}{1-t}
$$

 It has the important property of extracting the regression components
from the probability estimator,

$$
\texttt{logit}(p) = b + a x
$$

 More continuous variables can be accommodated easily as

$$
\texttt{logit}(p) = b + \sum_k a_k x_k
$$

 This can be further extended beyond the binary case to multiple
target labels. The maximum likelihood estimate of this uses
numerical optimization methods that are implemented in Scikit-learn.

Let's construct some data to see how this works. In the following, we assign
class labels to a set of randomly scattered points in the two-dimensional
plane,

```python
%matplotlib inline

import numpy as np
from matplotlib.pylab import subplots
v = 0.9
@np.vectorize
def gen_y(x):
    if x<5: return np.random.choice([0,1],p=[v,1-v])
    else:   return np.random.choice([0,1],p=[1-v,v])

xi = np.sort(np.random.rand(500)*10)
yi = gen_y(xi)
```

**Programming Tip.**

The `np.vectorize` decorator used in the code above makes it easy to avoid
looping in code that uses Numpy arrays by embedding the looping semantics
inside of the so-decorated function. Note, however, that this does not
necessarily accelerate the wrapped function. It's mainly for convenience.

[Figure](#fig:logreg_001) shows a scatter plot of the data we constructed in
the above code, $\lbrace (x_i,Y_i) \rbrace$. As constructed, it is more
likely that large values of $x$ correspond to $Y=1$. On the other hand, values
of $x \in [4,6]$ of either category are heavily overlapped. This means that $x$
is not a particularly strong indicator of $Y$ in this region.  [Figure](#fig:logreg_002) shows the fitted logistic regression curve against the same
data. The points along the curve are the probabilities that each point lies in
either of the two categories. For large values of $x$ the curve is near one,
meaning that the probability that the associated $Y$ value is equal to one. On
the other extreme, small values of $x$ mean that this probability is close to
zero.  Because there are only two possible categories, this means that the
probability of $Y=0$ is thereby higher. The region in the middle corresponding
to the middle probabilities reflect the ambiguity between the two catagories
because of the overlap in the data for this region. Thus, logistic regression
cannot make a strong case for one category here.
The following code fits the logistic regression model,

```python
fig,ax=subplots()
_=ax.plot(xi,yi,'o',color='gray',alpha=.3)
_=ax.axis(ymax=1.1,ymin=-0.1)
_=ax.set_xlabel(r'$X$',fontsize=22)
_=ax.set_ylabel(r'$Y$',fontsize=22)
fig.savefig('fig-machine_learning/logreg_001.png')
```

```python
from sklearn.linear_model import LogisticRegression
lr = LogisticRegression()
lr.fit(np.c_[xi],yi)
```

    /home/unpingco/.conda/envs/pypsml2E/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.
      FutureWarning)

    LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,
                       intercept_scaling=1, l1_ratio=None, max_iter=100,
                       multi_class='warn', n_jobs=None, penalty='l2',
                       random_state=None, solver='warn', tol=0.0001, verbose=0,
                       warm_start=False)

```python
fig,ax=subplots()
xii=np.linspace(0,10,20)
_=ax.plot(xii,lr.predict_proba(np.c_[xii])[:,1],'k-',lw=3)
_=ax.plot(xi,yi,'o',color='gray',alpha=.3)
_=ax.axis(ymax=1.1,ymin=-0.1)
_=ax.set_xlabel(r'$x$',fontsize=20)
_=ax.set_ylabel(r'$\mathbb{P}(Y)$',fontsize=20)
fig.savefig('fig-machine_learning/logreg_002.png')
```

This scatterplot shows the binary $Y$ variables and the corresponding $x$ data for each category.

This shows the fitted logistic regression on the data shown in [Figure](#fig:logreg_001). The points along the curve are the probabilities that each point lies in either of the two categories.

For a deeper understanding of logistic regression, we need to alter our
notation slightly and once again use our projection methods. More generally we
can rewrite Equation ([1](#eq:prob)) as the following,

$$
\begin{equation}
p(\mathbf{x}) = \frac{1}{1+\exp(-\boldsymbol{\beta}^T \mathbf{x})}
\label{eq:probbeta} \tag{2}
\end{equation}
$$

 where $\boldsymbol{\beta}, \mathbf{x}\in \mathbb{R}^n$.  From our
prior work on projection we know that the signed perpendicular distance between
$\mathbf{x}$ and the linear boundary described by $\boldsymbol{\beta}$ is
$\boldsymbol{\beta}^T \mathbf{x}/\Vert\boldsymbol{\beta}\Vert$.  This means
that the probability that is assigned to any point in $\mathbb{R}^n$ is a
function of how close that point is to the linear boundary described by the
following equation,

$$
\boldsymbol{\beta}^T \mathbf{x} = 0
$$

 But there is something subtle hiding here. Note that
for any $\alpha\in\mathbb{R}$,

$$
\alpha\boldsymbol{\beta}^T \mathbf{x} = 0
$$

 describes the *same* hyperplane. This means that we can multiply
$\boldsymbol{\beta}$ by an arbitrary scalar and still get the same geometry.
However, because of $\exp(-\alpha\boldsymbol{\beta}^T \mathbf{x})$ in Equation
([2](#eq:probbeta)), this scaling determines the intensity of the probability
attributed to $\mathbf{x}$. This is illustrated in [Figure](#fig:logreg_003).
The panel on the left shows two categories (squares/circles) split by the
dotted line that is determined by $\boldsymbol{\beta}^T\mathbf{x}=0$. The
background colors show the probabilities assigned to points in the plane.  The
right panel shows that by scaling with $\alpha$, we can increase the
probabilities of class membership for the given points, given the exact same
geometry. The points near the boundary have lower probabilities because they
could easily be on the opposite side.  However, by scaling by $\alpha$, we can
raise those probabilities to any desired level at the cost of driving the
points further from the boundary closer to one. Why is this a problem? By
driving the probabilities arbitrarily using $\alpha$, we can overemphasize the
training set at the cost of out-of-sample data. That is, we may wind up
insisting on emphatic class membership of yet unseen points that are close to
the boundary that otherwise would have more equivocal probabilities (say, near
$1/2$).  Once again, this is another manifestation of bias/variance trade-off.

Scaling can arbitrarily increase the probabilities of points near the decision boundary.

Regularization is a method that controls this effect by penalizing the size of
$\beta$ as part of its solution. Algorithmically, logistic regression works by
iteratively solving a sequence of weighted least squares problems. Regression
adds a $\Vert\boldsymbol{\beta}\Vert/C$ term to the least squares error. To see
this in action, let's create some data from a logistic regression and see if we
can recover it using Scikit-learn. Let's start with a scatter of points in the
two-dimensional  plane,

```python
x0,x1=np.random.rand(2,20)*6-3
X = np.c_[x0,x1,x1*0+1] # stack as columns
```

 Note that `X` has a third column of all ones. This is a
trick to allow the corresponding line to be offset from the origin
in the two-dimensional plane. Next, we create a linear boundary
and assign the class probabilities according to proximity to the
boundary.

```python
beta = np.array([1,-1,1]) # last coordinate for affine offset
prd = X.dot(beta)
probs = 1/(1+np.exp(-prd/np.linalg.norm(beta)))
c = (prd>0) # boolean array class labels
```

  This establishes the training data.  The next block
creates the logistic regression object and fits the data.

```python
lr = LogisticRegression()
_=lr.fit(X[:,:-1],c)
```

    /home/unpingco/.conda/envs/pypsml2E/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.
      FutureWarning)

 Note that we have to omit the third dimension because of
how Scikit-learn internally breaks down the components of the
boundary. The resulting code extracts the corresponding
$\boldsymbol{\beta}$ from the `LogisticRegression` object.

```python
betah = np.r_[lr.coef_.flat,lr.intercept_]
```

**Programming Tip.**

The Numpy `np.r_`  object provides a quick way to stack Numpy
arrays horizontally instead of using `np.hstack`.

 The resulting boundary is shown in the left panel in [Figure](#fig:logreg_004). The crosses and triangles represent the two classes we
created above, along with the separating gray line.  The logistic regression
fit produces the dotted black line. The dark circle is the point that logistic
regression categorizes incorrectly. The regularization parameter is $C=1$ by
default. Next, we can change the strength of the regularization parameter as in
the following,

```python
lr = LogisticRegression(C=1000)
```

 and the re-fit the data to produce the right panel in
[Figure](#fig:logreg_004). By increasing the regularization
parameter, we essentially nudged the fitting algorithm to
*believe* the data more than the general model. That is, by doing
this we accepted more variance in exchange for better bias.

The left panel shows the resulting boundary (dashed line) with $C=1$ as the regularization parameter. The right panel is for $C=1000$. The gray line is the boundary used to assign the class membership for the synthetic data. The dark circle is the point that logistic regression categorizes incorrectly.

### Maximum Likelihood Estimation for Logistic Regression

Let us again consider the binary classification problem.  We define $y_k =
\mathbb{P}(C_1\vert \mathbf{x}_k)$, the conditional probability of the data as
a member of given class. Our construction of this problem provides

$$
y_k = \theta([\mathbf{w},w_0] \cdot [\mathbf{x}_k,1])
$$

 where $\theta$ is the logistic function.  Recall that there are only
two classes for this problem. The data set looks like the following,

$$
\lbrace(\mathbf{x}_0,r_0),\ldots,(\mathbf{x}_k,r_k),\ldots,(\mathbf{x}_{n-1},r_{n-1})\rbrace
$$

 where $r_k\in \lbrace 0,1 \rbrace$. For example, we could have the
following sequence of observed classes,

$$
\lbrace C_0,C_1,C_1,C_0,C_1 \rbrace
$$

 For this case the likelihood is then the following,

$$
\ell= \mathbb{P}(C_0\vert\mathbf{x}_0)\mathbb{P}(C_1\vert\mathbf{x}_1)\mathbb{P}(C_1\vert\mathbf{x}_1) \mathbb{P}(C_0\vert\mathbf{x}_0)\mathbb{P}(C_1\vert\mathbf{x}_1)
$$

 which we can rewrite as the following,

$$
\ell(\mathbf{w},w_0)= (1-y_0) y_1 y_2 (1-y_3) y_4
$$

 Recall that there are two mutually exhaustive classes. More
generally, this can be written as the following,

$$
\ell(\mathbf{w}\vert\mathcal{X})=\prod_k^n y_k^{r_k} (1-y_k)^{1-r_k}
$$

 Naturally, we want to compute the logarithm of this as the cross-entropy,

$$
E = -\sum_k r_k \log(y_k) + (1-r_k)\log(1-y_k)
$$

 and then minimize this to find $\mathbf{w}$ and $w_0$. This is
difficult to do with calculus because the derivatives have non-linear terms in
them that are hard to solve for.

### Multi-Class Logistic Regression Using Softmax

The logistic regression problem provides a solution for the probability
between exactly two alternative classes. To extend to the multi-class
problem, we need the *softmax* function. Consider the likelihood
ratio between the $i^{th}$ class and the reference class, $\mathcal{C}_k$,

$$
\log\frac{p(\mathbf{x}\vert \mathcal{C}_i)}{p(\mathbf{x}\vert\mathcal{C}_k)} = \mathbf{w}_i^T \mathbf{x}
$$

 Taking the exponential of this and normalizing across all
the classes gives the softmax function,

$$
y_i=p(\mathcal{C}_i \vert\mathbf{x})=\frac{\exp\left(\mathbf{w}_i^T\mathbf{x}\right)}{\sum_k \exp\left( \mathbf{w}_k^T \mathbf{x}\right)}
$$

 Note that $\sum_i y_i=1$. If the $\mathbf{w}_i^T \mathbf{x}$ term is
larger than the others, after the exponentiation and normalization, it
automatically suppresses the other $y_j \forall j \neq i$, which acts like the
maximum function, except this function is differentiable, hence *soft*, as in
*softmax*.  While that is all straightforward, the trick is deriving the
$\mathbf{w}_i$ vectors from the training data $\lbrace\mathbf{x}_i,y_i\rbrace$.

Once again, the launching point is the likelihood function. As with the
two-class logistic regression problem, we have the likelihood as the following,

$$
\ell = \prod_k \prod_i (y_i^k)^{r_i^k}
$$

 The log-likelihood of this is the same as the cross-entropy,

$$
E = - \sum_k \sum_i r_i^k \log y_i^k
$$

 This is the error function we want to minimize. The computation works
as before with logistic regression, except there are more derivatives to keep
track of in this case.

### Understanding Logistic Regression

To generalize this technique beyond logistic regression, we need to re-think
the problem more abstractly as the data set $\lbrace x_i, y_i \rbrace$.  We
have the $y_i \in \lbrace 0,1 \rbrace$ data modeled as Bernoulli random
variables. We also have the $x_i$ data associated with each $y_i$, but it is
not clear how to exploit this association.  What we would like is to construct
$\mathbb{E}(Y|X)$ which we already know (see [ch:prob](#ch:prob)) is the
best MSE estimator. For this problem, we have

$$
\mathbb{E}(Y|X) =\mathbb{P}(Y|X)
$$

 because only $Y=1$ is nonzero in the summation. Regardless, we
don't have the conditional probabilities anyway. One way to look at
logistic regression is as a way to build in the functional relationship
between $y_i$ and $x_i$. The simplest thing we could do is
approximate,

$$
\mathbb{E}(Y|X) \approx \beta_0 + \beta_1 x := \eta(x)
$$

 If this is the model, then the target would be the $y_i$ data. We can
force the output of this linear regression into the interval $[0,1]$ by composing
it with a sigmoidal function,

$$
\theta(x) = \frac{1}{1+\exp(-x)}
$$

 Then we have a new function $\theta(\eta(x))$ to match against $y_i$ using

$$
J(\beta_0,\beta_1) = \sum_i (\theta(\eta(x_i))-y_i)^2
$$

 This is a nice setup for an optimization problem. We could certainly
solve this numerically using `scipy.optimize`. Unfortunately, this would take
us into the black box of the optimization algorithm where we would lose all of
our intuitions and experience with linear regression.  We can take the opposite
approach. Instead of trying to squash the output of the linear estimator into
the desired domain, we can map the $y_i$ data into the unbounded space of the
linear estimator. Thus, we define the inverse of the above $\theta$ function as
the *link* function.

$$
g(y) = \log \left( \frac{y}{1-y} \right)
$$

 This means that our approximation to the
unknown conditional expectation is the following,

$$
g(\mathbb{E}(Y|X)) \approx \beta_0 + \beta_1 x := \eta(x)
$$

 We cannot apply this directly to the $y_i$, so we compute the Taylor
series expansion centered on $\mathbb{E}(Y|X)$, up to the linear term, to obtain the
following,

$$
\begin{align*}
g(Y) & \approx & g(\mathbb{E}(Y|X)) + (Y-\mathbb{E}(Y|X)) g'(\mathbb{E}(Y|X))  \\
     & \approx & \eta(x) + (Y-\theta(\eta(x))) g'(\theta(\eta(x))) := z
\end{align*}
$$

 Because we do not know the conditional expectation, we replaced these
terms with our earlier $\theta(\eta(x))$ function.  This new approximation
defines our transformed data that we will use to feed the linear model.  Note
that the $\beta$ parameters are embedded in this transformation.  The
$(Y-\theta(\eta(x)))$ term acts as the usual additive noise term. Also,

$$
g'(x) = \frac{1}{x(1-x)}
$$

 The following code applies this transformation to the `xi,yi` data

```python
import numpy as np
v = 0.9
@np.vectorize
def gen_y(x):
    if x<5: return np.random.choice([0,1],p=[v,1-v])
    else:   return np.random.choice([0,1],p=[1-v,v])

xi = np.sort(np.random.rand(500)*10)
yi = gen_y(xi)
```

```python
b0, b1 = -2,0.5
g = lambda x: np.log(x/(1-x))
theta = lambda x: 1/(1+np.exp(-x))
eta = lambda x: b0 + b1*x
theta_ = theta(eta(xi))
z=eta(xi)+(yi-theta_)/(theta_*(1-theta_))
```

```python
from matplotlib.pylab import subplots
fig,ax=subplots()
_=ax.plot(xi,z,'k.',alpha=.3)
_=ax2 = ax.twinx()
_=ax2.plot(xi,yi,'.r',alpha=.3)
_=ax2.set_ylabel('Y',fontsize=16,rotation=0,color='r')
_=ax2.spines['right'].set_color('red')
_=ax.set_xlabel('X',fontsize=16)
_=ax.set_ylabel('Z',fontsize=16,rotation=0)
_=ax2.tick_params(axis='y', colors='red')
fig.savefig('fig-machine_learning/glm_001.png')
```

The transformation underlying logistic regression.

Note the two vertical scales shown in [Figure](#fig:glm_001). The red scale on the
right is the $\lbrace 0,1 \rbrace$ domain of the $y_i$ data (red dots) and the
left scale is transformed $z_i$ data (black dots). Note that the transformed
data is more linear where the original data is less equivocal at the extremes.
Also, this transformation used a specific pair of  $\beta_i$ parameters. The
idea is to iterate over this transformation and derive new $\beta_i$
parameters.  With this approach,  we have

$$
\mathbb{V}(Z|X) = (g')^2 \mathbb{V}(Y|X)
$$

 Recall that, for this binary variable, we have

$$
\mathbb{P}(Y|X) = \theta(\eta(x)))
$$

 Thus,

$$
\mathbb{V}(Y|X) = \theta(\eta(x)) (1-\theta(\eta(x)))
$$

 from which we obtain

$$
\mathbb{V}(Z|X) = \left[ \theta(\eta(x))(1-\theta(\eta(x))) \right]^{-1}
$$

 The important fact here is the variance is a function of the $X$
(i.e., heteroskedastic). As we discussed with Gauss-Markov, the appropriate
linear regression is  weighted least-squares where the weights at each data
point are inversely proportional to the variance. This ensures that the
regression process accounts for this heteroskedasticity. Numpy has a weighted
least squares implemented in the `polyfit` function,

```python
w=(theta_*(1-theta_))
p=np.polyfit(xi,z,1,w=np.sqrt(w))
```

 The output of this fit is shown in [Figure](#fig:glm_002), along
with the raw data and $\mathbb{V}(Z|X)$ for this particular fitted $\beta_i$.
Iterating a few more times refines the estimated line but it does not take many
such iterations to converge. As indicated by the variance line, the fitted line
favors the data at either extreme.

```python
fig,ax=subplots()
ax2 = ax.twinx()
theta_ = theta(eta(xi))
_=ax.plot(xi,
        z,
        '.',alpha=.3,
        label='z'
       );
_=ax.axis(ymax=30,ymin=-30);
_=ax2.plot(xi,theta_*(1-theta_),'.r',alpha=.3,label='$\mathbb{V}(Z|X)$');
_=ax2.axis(ymax=0.27,ymin=0);
p=np.polyfit(xi,z,1,w=np.sqrt(w));
_=ax.plot(xi,np.polyval(p,xi),'k',label='fitted');
_=ax2.legend(fontsize=16);
_=ax.legend(loc=2);
_=ax.set_xlabel('X');
_=ax2.tick_params(axis='y', colors='red');
fig.savefig('fig-machine_learning/glm_002.png');
```

The output of the weighted least squares fit is shown, along with the raw data and $\mathbb{V}(Z|X)$.


# Panel Data

Panel data is an important
area in applied econometrics, simply because much of the available
data has this structure.

Panel data combines cross sectional and time series data: we have
a time series for each of the agents observed in a cross section.
- The addition of temporal information to a cross sectional model can
in principle allow us to investigate issues such as persistence, habit
formation, and dynamics.
- Starting from the perspective of a single time series, the addition
of cross-sectional information allows investigation of heterogeneity.
- In both cases, if parameters are common across units or over time,
the additional data allows for more precise estimation.

The basic idea is to allow variables to have two indices, $i=1,2,...,n$
and $t=1,2,...,T$. The simple linear model becomes
$$
y_{it}=\alpha+x_{it}\beta+\epsilon_{it}
$$
We could think of allowing the parameters to change over time and
over cross sectional units. This would give
$$
y_{it}=\alpha_{it}+x_{it}\beta_{it}+\epsilon_{it}
$$
The problem here is that there are more parameters than observations,
so the model is not identified. We need some restraint! The proper
restrictions to use of course depend on the problem at hand, and a
single model is unlikely to be appropriate for all situations. For
example, one could have time and cross-sectional dummies, and slopes
that are constant over time and across agents:
$$
y_{it}=\alpha_{i}+\gamma_{t}+x_{it}\beta+\epsilon_{it}
$$
There is a lot of room for playing around here. We also need to consider
whether or not $n$ and $T$ are fixed or growing. We'll need at least
one of them to be growing in order to do asymptotics.

To provide some focus, we'll consider common slope parameters, but
agent-specific intercepts, which:
\begin{equation}
y_{it}=\alpha_{i}+x_{it}\beta+\epsilon_{it}\label{eq:simple linear panel model}
\end{equation}

- I will refer to this as the ''simple linear panel model''. We assume
that the regressors $x_{it}$ are exogenous, with no correlation with
the error term.
- This is the model most often encountered in the applied literature.
It is like the original cross-sectional model, in that the $\beta's$
are constant over time for all $i.$ However we're now allowing for
the constant to vary across $i$ (some individual heterogeneity).
- We can consider what happens as $n\rightarrow\infty$ but $T$ is
fixed. This would be relevant for microeconometric panels, (e.g.,
the PSID data) where a survey of a large number of individuals may
be done for a limited number of time periods.
- Macroeconometric applications might look at longer time series for
a small number of cross-sectional units (e.g., 40 years of quarterly
data for 15 European countries). For that case, we could keep $n$
fixed (seems appropriate when dealing with the EU countries), and
do asymptotics as $T$ increases, as is normal for time series.
- The asymptotic results depend on how we do this, of course.

**Why bother using panel data, what are the benefits?**

- The model
$$
y_{it}=\alpha_{i}+x_{it}\beta+\epsilon_{it}
$$
 is a restricted version of
$$
y_{it}=\alpha_{i}+x_{it}\beta_{i}+\epsilon_{it}
$$
which could be estimated for each $i$ in turn, using time series
data. Why use the panel approach?
Because the restrictions that $\beta_{i}=\beta_{j}=...=\beta,$ if
true, lead to more efficient estimation. Estimation for each $i$
in turn will be very uninformative if $T$ is small.
-Panel data allows us to estimate parameters
that are not identified by cross sectional (time series) data. For
example, if the model is
$$
y_{it}=\alpha_{i}+\gamma_{t}+x_{it}\beta+\epsilon_{it}
$$
and we have only cross sectional data, we cannot estimate the $\alpha_{i}$.
If we have only time series data on a single cross sectional unit
$i=1$, we cannot estimate the $\gamma_{t}$. Cross-sectional variation
allows us to estimate parameters indexed by time, and time series
variation allows us to estimate parameters indexed by cross-sectional
unit. Parameters indexed by both $i$ and $t$ will require other
forms of restrictions in order to be estimable.
- $\alpha_{i}$ can absorb
any missing variables in the regression that don't change over time,
and $\gamma_{t}$ can absorb missing variables that don't change across
$i$.

For example, suppose we have the model
\begin{equation}
y_{it}=\alpha+x_{it}\beta+z_{i}\gamma+\epsilon_{it}\label{eq:simple panel model}
\end{equation}
where the variables in $z_{i}$ are unobserved, but are
constant over time. Assume that, as is usually the case, there is
some correlation between the variables in $x_{it}$ and $z_{i}$.
That is to say, there is some ordinary collinearity of the regressors.
- If we have only one time period, then we have to estimate the model
$$
y_{i}=\alpha+x_{i}\beta+z_{i}\gamma+\epsilon_{i}
$$
using the observations $i=1,2,...,n$. Because $z_{i}$ is unobserved, we have to let it be absorbed in the
error term. For convenience, and to keep the notation simple, assume
that the mean of $z_{i}\gamma$ is zero (this does not affect the
argument in any important way), so the model we can actually estimate
is
\begin{align*}
y_{i} & =\alpha+x_{i}\beta+v_{i}
\end{align*}
where $v_{i}=z_{i}\gamma+\epsilon_{i}$. This model has correlation between the regressors and the error, so
the OLS estimates would be inconsistent. Furthermore, we don't have
any natural instruments to estimate the model by IV.

However, suppose we have at least two time periods of data,
and $n$ cross-sectional observations. Then, we can let $z_{i}\gamma$
move into the constant, and we get the model
\begin{align*}
y_{it} & =\alpha+x_{it}\beta+z_{i}\gamma+\epsilon_{it}\\
y_{it} & =\alpha_{i}+x_{it}\beta+\epsilon_{it}
\end{align*}
where $\alpha_{i}=\alpha+z_{i}\gamma$. This is the simple linear
panel data model.
    - Notice that the problematic $z_{i}$ have now disappeared!
    - It turns out that OLS estimation of this model will give consistent estimates of the $\beta$ parameters, as the cross sectional size of the sample, $n$ increases, as long as the regressors are exogenous. If it's not clear how this can be estimated by OLS, then consider estimating it using first differences: that model is pretty obviously consistently estimable using OLS.

To begin with, assume that:
- the $x_{it}$ are weakly exogenous variables (uncorrelated with $\epsilon_{it})$
- the model is static: $x_{it}$ does not contain lags of $y_{it}$.
- then the basic problem we have in the panel data model $y_{it}=\alpha_{i}+x_{it}\beta+\epsilon_{it}$
is the presence of the $\alpha_{i}$. These are individual-specific
parameters. Or, possibly more accurately, they can be thought of as
individual-specific variables that are not observed (latent variables).
The reason for thinking of them as variables is because the agent
may choose their values following some process, or may choose other
variable taking these ones as given.

Define $\alpha=E(\alpha_{i})$, so $E(\alpha_{i}-\alpha)=0,$ where
the expectation is with respect to the density that describes the
distribution of the $\alpha_{i}$ in the population. Our model $y_{it}=\alpha_{i}+x_{it}\beta+\epsilon_{it}$
may be written
\begin{align*}
y_{it} & =\alpha_{i}+x_{it}\beta+\epsilon_{it}\\
 & =\alpha+x_{it}\beta+(\alpha_{i}-\alpha+\epsilon_{it})\\
 & =\alpha+x_{it}\beta+\eta_{it}
\end{align*}
Note that $E(\eta_{it})=0.$ A way of thinking about the data generating
process is this:
- First, $\alpha_{i}$ is drawn, from the population density
- then $T$ values of $x_{it}$ are drawn from $f_{X}(z|\alpha_{i}).$
- the important point is that the distribution of $x$ may vary
depending on the realization of $\alpha_{i}$.

For example, if $y$ is the quantity demanded of a luxury good, then
a high value of $\alpha_{i}$ means that agent $i$ will buy a large
quantity, on average. This may be possible only when the agent's income
is also high. Thus, it may be possible to draw high values of $\alpha_{i}$
only when income is also high, otherwise, the budget constraint would
be violated. If income is one of the variables in $x_{it},$ then
$\alpha_{i}$ and $x_{it}$ are not independent.

Another example: consider returns to education, modeling wage as a
function of education. $\alpha_{i}$ could be an individual specific
measure of ability. Ability could affect wages, but it could also
affect the number of years of education. When education is a regressor
and ability is a component of the error, we may expect an endogeneity
problem.

Thus, there may be correlation between $\alpha_{i}$ and $x_{it}$,
in which case $E(x_{it}\eta_{it})\ne$0 in the above equation.
- This means that OLS estimation of the model would lead to biased and
inconsistent estimates.
- However, it is possible (but unlikely for economic data) that $x_{it}$
and $\eta_{it}$ are independent or at least uncorrelated, if the
distribution of $x_{it}$ is constant with respect to the realization
of $\alpha_{i}$. In this case OLS estimation would be consistent.

**Fixed effects**: when $E(x_{it}$$\eta_{it})\ne$0,
the model is called the "fixed effects model"

**Random effects**: when $E(x_{it}\eta_{it})=0,$ the model is
called the "random effects model"

I find this to be pretty poor nomenclature, because the issue is not
whether ''effects'' are fixed or random (they are always random,
unconditional on $i$). The issue is whether or not the ''effects''
are correlated with the other regressors. In economics, it seems likely
that the unobserved variable $\alpha$ is probably correlated with
the observed regressors, $x$ (this is simply the presence of collinearity
between observed and unobserved variables, and collinearity is usually
the rule rather than the exception). So, we expect that the ''fixed
effects'' model is probably the relevant one unless special circumstances
imply that the $\alpha_{i}$ are uncorrelated with the $x_{it}$.

### Example: Agricultural Production

Suppose
$$y_{it} = \beta x_{it} + \eta_{it} + v_{it}$$
represents the Cobb-Douglas production function of an agricultural product. The index $i$ denotes farms and $t$ time periods.

- $y_{it}$ = log output
- $x_{it}$ = log input (labor)
- $\eta_{i}$ = input that remains constant over time (soil quality)
- $v_{it}$ = stochastic input which is outside of the farmer's control (rainfall)

Suppose $\eta_i$ is known by the farmer but not by the econometrician.
- If farmers maximize expected
profits there will be cross-sectional correlation between labour and soil quality.
- Therefore, the population coefficient in a simple regression of $y_{i1}$ on $x_{i1}$ will differ from $\beta$.

If $\eta$ were observed by the
econometrician,
- the coefficient on $x$ in a multiple cross-sectional regression of $y_{i1}$ on $x_{i1}$ and $\eta_i$ will
coincide with $\beta$.

Now suppose that data on $y_{i2}$ and $x_{i2}$ for a second period become available. Moreover,
suppose that rainfall in the second period is unpredictable from rainfall in the first period (permanent
differences in rainfall are part of $\eta_i$), so that rainfall is independent of a farm’s labour demand in
the two periods. Thus, even in the absence of data on ηi the availability of panel data affords the
identification of the technological parameter $\beta$.

## Fixed Effects: the within estimator

The within estimator  involves subtracting the time series average
from each cross sectional unit.
\begin{align}
\overline{x}_{i} & =\frac{1}{T}\sum_{t=1}^{T}x_{it}\nonumber \\
\overline{\epsilon_{i}} & =\frac{1}{T}\sum_{t=1}^{T}\epsilon_{it}\nonumber \\
\overline{y}_{i} & =\frac{1}{T}\sum_{t=1}^{T}y_{it}=\alpha_{i}+\frac{1}{T}\sum_{t=1}^{T}x_{it}\beta+\frac{1}{T}\sum_{t=1}^{T}\epsilon_{it}\nonumber \\
\overline{y}_{i} & =\alpha_{i}+\overline{x}_{i}\beta+\overline{\epsilon_{i}}\label{eq:time averages}
\end{align}
The transformed model is
\begin{align}
y_{it}-\overline{y}_{i} & =\alpha_{i}+x_{it}\beta+\epsilon_{it}-\alpha_{i}-\overline{x}_{i}\beta-\overline{\epsilon_{i}}\label{eq:within estimator}\\
y_{it}^{*} & =x_{it}^{*}\beta+\epsilon_{it}^{*}\nonumber
\end{align}
where $x_{it}^{*}=x_{it}-\overline{x}_{i}$ and $\epsilon_{it}^{*}=\epsilon_{it}-\overline{\epsilon}_{i}$.
In this model, it is clear that $x_{it}^{*}$ and $\epsilon_{it}^{*}$
are uncorrelated, as long as the original regressors $x_{it}$ are
exogenous with respect to the original error $\epsilon_{it}$
($E(x_{it}\epsilon_{is})=0,\,\forall t,s$). In this case, OLS will
give consistent estimates of the parameters of this model, $\beta.$

What about the $\alpha_{i}?$

Can they be consistently estimated?

An estimator is
$$
\hat{\alpha}_{i}=\frac{1}{T}\sum_{t=1}^{T}\left(y_{it}-x_{it}\hat{\beta}\right)
$$
It's fairly obvious that this is a consistent estimator if $T\rightarrow\infty$.
For a short panel with fixed $T,$ this estimator is not consistent.
Nevertheless, the variation in the $\hat{\alpha}_{i}$ can be fairly
informative about the heterogeneity.

An equivalent approach is to estimate the model
$$
y_{it}=\sum_{j=1}^{n}d_{j,it}\alpha_{j}+x_{it}\beta+\epsilon_{it}
$$
 by OLS. The $d_{j}$, $j=1,2,...,n$ are $n$ dummy variables that
take on the value $1$ if $j=i,$ zero otherwise. They are indicators
of the cross sectional unit of the observation. For example, with
3 cross sectional units and 3 time periods, and a single $x$ regressor,
the model in matrix form would look like
$$
\left[\begin{array}{c}
y_{11}\\
y_{12}\\
y_{13}\\
y_{21}\\
y_{22}\\
y_{23}\\
y_{31}\\
y_{32}\\
y_{33}
\end{array}\right]=\left[\begin{array}{cccc}
1 & 0 & 0 & x_{11}\\
1 & 0 & 0 & x_{12}\\
1 & 0 & 0 & x_{13}\\
0 & 1 & 0 & x_{21}\\
0 & 1 & 0 & x_{22}\\
0 & 1 & 0 & x_{23}\\
0 & 0 & 1 & x_{31}\\
0 & 0 & 1 & x_{32}\\
0 & 0 & 1 & x_{33}
\end{array}\right]\left[\begin{array}{c}
\alpha_{1}\\
\alpha_{2}\\
\alpha_{3}\\
\beta
\end{array}\right]+\left[\begin{array}{c}
\epsilon_{11}\\
\epsilon_{12}\\
\epsilon_{13}\\
\epsilon_{21}\\
\epsilon_{22}\\
\epsilon_{23}\\
\epsilon_{31}\\
\epsilon_{32}\\
\epsilon_{33}
\end{array}\right]
$$
Estimating this model directly by OLS gives numerically exactly the
same results as the OLS version of the ''within'' estimator, and
you get the $\hat{\alpha}_{i}$ automatically. See Cameron and Trivedi,
section 21.6.4 for details. An interesting and important result known
as the Frisch-Waugh-Lovell Theorem can be used to show that the two
means of estimation give identical results.

### Example in which Panel Data Does Not Work: Returns to Education

“Structural”
returns to education are important in the assessment of educational policies. It has been widely
believed in the literature that cross-sectional regression estimates of the returns could not be trusted
because of omitted “ability” potentially correlated with education attainment. In the earlier notation:
- $y_{it}$ = Log wage (or earnings).
- $x_{it}$ = Years of full-time education.
- $\eta_i$ = Unobserved ability.
- $\beta$ = Returns to education.
The problem in this example is that $x_{it}$ typically lacks time variation. So a regression in firstdifferences will not be able to identify β in this case. In this context data on siblings and cross-sectional
instrumental variables have proved more useful for identifying returns to schooling free of ability bias
than panel data.

This example illustrates a more general problem. Information about β in the regression in firstdifferences will depend on the ratio of the variances of ∆v and ∆x. In the earnings−education equation,
we are in the extreme situation where Var(∆x)=0, but if Var(∆x) is small regressions in changes
may contain very little information about parameters of interest even if the cross-sectional sample size
is very large.

## Random Effects

$\alpha_i$ is assumed to be random.

The original model is
$$
y_{it}=\alpha_{i}+x_{it}\beta+\epsilon_{it}
$$
This can be written as
\begin{eqnarray}
y_{it} & = & \alpha+x_{it}\beta+\left(\alpha_{i}-\alpha+\epsilon_{it}\right)\nonumber \\
y_{it} & = & \alpha+x_{it}\beta+\eta_{it}\label{eq:simple linear panel random effects}
\end{eqnarray}
Under random effects, the $\alpha_{i}$ are assumed not to be correlated
with the $x_{it},$ so $E(\eta_{it})=0,$ and $E(x_{it}\eta_{it})=0$.
As such, the OLS estimator of this model is consistent. We can recover
estimates of the $\alpha_{i}$ as discussed above.

It is to be noted
that the error $\eta_{it}$ is almost certainly heteroscedastic and
autocorrelated, so OLS will not be efficient, and inferences based
on OLS need to be done taking this into account. One could attempt
to use GLS, or panel-robust covariance matrix estimation, or both,
as above.

There are other estimators when we have random effects, a well-known
example being the ''between'' estimator, which operates on the time
averages of the cross sectional units. There is no advantage to doing
this, as the overall estimator is already consistent, and averaging
looses information (efficiency loss). One would still need to deal
with cross sectional heteroscedasticity when using the between estimator,
so there is no gain in simplicity, either.

It is to be emphasized that ''random effects'' is not a plausible
assumption with most economic data, so use of this estimator is discouraged,
even if your statistical package offers it as an option. Think carefully
about whether the assumption is warranted before trusting the results
of this estimator.

## Hausman Test

Suppose you're doubting about whether fixed or random effects are
present.
- If we have correlation between $x_{it}$ and $\alpha_{i}$ (fixed
effects), then the ''within'' estimator will be consistent, but
the random effects estimator of the previous section will not.
- A Hausman test statistic can be computed, using the difference between
the two estimators.
    - The null hypothesis is that the effects are uncorrelated with the
    regressors in $x_{it}$ (''random effects'') so that both estimators
    are consistent under the null.
    - When the test rejects, we conclude that fixed effects are present,
    so the ''within'' estimator should be used.
    - Now, what happens if the test does not reject? One could optimistically
    turn to the random effects model, but it's probably more realistic
    to conclude that the test may have low power. Failure to reject does
    not mean that the null hypothesis is true. After all, estimation of
    the covariance matrices needed to compute the Hausman test is a non-trivial
    issue, and is a source of considerable noise in the test statistic
    (noise=low power).