```python
import numpy as np

#extra imports
import sympy as sm
import scipy as sp
from scipy import linalg
from scipy import optimize
from scipy import interpolate
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
%matplotlib inline
```

# Linear regression

Consider the following **linear equation:**

$$y_i = \beta_0 + \beta_1 x_{1,i} + \beta_2 x_{2,i} + \epsilon_i$$

Assume you have access to data of the **independent variables** ($x_{1,i}$, $x_{2,i}$) and the **dependent variable** ($y_i$) for $N$ individuals, where $i$ indexes individuals. The variable $\epsilon_i$ is a mean-zero **stochastic shock**.

Assume the **data generating process** is given by:

```python
def DGP(N):

    # a. independent variables
    x1 = np.random.normal(0,1,size=N)
    x2 = np.random.normal(0,1,size=N)

    # b. errors
    eps = np.random.normal(0,1,size=N)

    extreme = np.random.uniform(0,1,size=N)
    eps[extreme < 0.05] += np.random.normal(-5,1,size=N)[extreme < 0.05]
    eps[extreme > 0.95] += np.random.normal(5,1,size=N)[extreme > 0.95]

    # c. dependent variable
    y = 0.1 + 0.3*x1 + 0.5*x2 + eps

    return x1, x2, y
```

**The data you have access to is:**

```python
np.random.seed(2020)
x1,x2,y = DGP(10000)
```

**Question 1:** Estimate the vector of coefficients $\mathbf{\beta} = (\beta_0,\beta_1,\beta_2)$ using **ordinary least squares (OLS)** implemented with **matrix algebra** by

$$ \hat{\mathbf{\beta}} = (\mathbf{X}^{\prime}\mathbf{X})^{-1}\mathbf{X}^{\prime}\mathbf{y} $$

where $\mathbf{X}^{\prime}$ is the transpose of $\mathbf{X}$ and

$$\mathbf{y} =
\pmatrix{ y_1 \cr y_2 \cr  \vdots \cr y_N
}
, \quad \mathbf{X} = \pmatrix{
1 & x_{1,1} & x_{2,1} \cr
1 & x_{1,2} & x_{2,2} \cr
\vdots & \vdots \cr
1 & x_{1,N} & x_{2,N}
}$$

```python
# construct beta0 vector of ones
ones = np.ones(10000)

# construct X matrix
X_trans = np.array([ones,x1, x2])
X = X_trans.T

# set up equation and print answer
beta_hat = linalg.inv(X_trans@X)@X_trans@y
print('the estimate of beta is beta_hat =' + f' ({beta_hat[0]:.2f}, {beta_hat[1]:.2f}, {beta_hat[2]:.2f})')
```

    the estimate of beta is beta_hat = (0.10, 0.29, 0.50)

**Question 2:** Construct a 3D plot, where the data is plotted as scattered points, and the prediction of the model is given by the plane

$$\hat{y}_i = \hat{\beta}_0 + \hat{\beta}_1 x_{1,i} + \hat{\beta}_2 x_{2,i}$$

```python
# predict y_hat
mesh1,mesh2 = np.meshgrid(x1,x2,indexing='ij')
y_plane = beta_hat[0]+beta_hat[1]*mesh1+beta_hat[2]*mesh2

# plot the predicted plane and the data points (be patient...)
plt.style.use('seaborn-whitegrid')

fig_reg = plt.figure(figsize=(12,8))
ax = fig_reg.add_subplot(1,1,1,projection='3d')
ax.scatter(x1,x2,y,s=5)
ax.plot_surface(mesh1,mesh2,y_plane,cmap=cm.jet)

ax.set_title('actual and predicted values')
ax.set_xlabel('$x_1$')
ax.set_ylabel('$x_2$')
ax.set_zlabel('$y$');
```

**Question 3:** Estimate the vector of coefficients $\mathbf{\beta} = (\beta_0,\beta_1,\beta_2)$ using a **numerical solver** to solve the ordinary least square problem, shown below, directly. Compare your results with the matrix algebra results.

$$ \min_{\mathbf{\beta}} \sum^N_{i=1} (y_i - (\beta_0 + \beta_1 x_{1,i} + \beta_2 x_{2,i}) )^2 $$

```python
# define sum of squares function
def sum_of_squares(beta):
    """
    Arguments:
        beta: vector of beta coefficients

    Returns:
        the sum of squares of the data generating process

    """
    return np.sum(np.square(y-(beta[0]+beta[1]*x1+beta[2]*x2)))

```

```python
# initial guess
beta_guess = [1,1,1]

#minimize sum of squares
ols = optimize.minimize(sum_of_squares,x0=beta_guess,method='SLSQP')

#unpack betas
beta_0_ols = ols.x[0]
beta_1_ols = ols.x[1]
beta_2_ols = ols.x[2]

print(f' beta0 = {beta_0_ols:.2f}, beta1 = {beta_1_ols:.2}, beta3 = {beta_2_ols:.2f}')
```

     beta0 = 0.10, beta1 = 0.29, beta3 = 0.50

```python
# compare with matrix algebra
beta_ols = np.array([beta_0_ols,beta_1_ols,beta_2_ols])
np.allclose(beta_hat,beta_ols)
```

    True

From the comparison, we see that the OLS method makes a (numerically) exact replica of the matrix algebra results.

**Question 4:** Estimate the vector of coefficients $\mathbf{\beta} = (\beta_0,\beta_1,\beta_2)$ using **least absolute deviations (LAD)** using a numerical solver to solve the following problem directly:

$$  \min_{\beta} \sum^N_{i=1} |y_i - (\beta_0 + \beta_1 x_{1,i} + \beta_2 x_{2,i}) | $$

where $|z|$ is the absolute value of $z$.

```python
# define new loss function
def least_abs_dev(beta):
    """
    Arguments:
        beta: vector of beta coefficients

    Returns:
        the least absolute deviations of the data generating process

    """
    return np.sum(np.abs(y-(beta[0]+beta[1]*x1+beta[2]*x2)))

#minimize least absolute deviations
lad = optimize.minimize(least_abs_dev,x0=beta_guess,method='SLSQP')

#unpack betas
beta_0_lad = lad.x[0]
beta_1_lad = lad.x[1]
beta_2_lad = lad.x[2]

print(f' beta0 = {beta_0_lad:.2f}, beta1 = {beta_1_lad:.2}, beta3 = {beta_2_lad:.2f}')
```

     beta0 = 0.09, beta1 = 0.31, beta3 = 0.51

```python
# compare with matrix algebra
beta_lad = np.array([beta_0_lad,beta_1_lad,beta_2_lad])
np.allclose(beta_hat,beta_lad)
```

    False

From the comparison, we see that the LAD method fails to make a (numerically) exact replica of the matrix algebra results.

**Question 5:** Set $N = 50$. Repeat the estimation using the **OLS** and **LAD** methods $K=5000$ times, drawing a new random sample from the data generating process each time. Compare the estimates from each method using histograms. Which method do you prefer? Explain your choice.

```python
# intialize
N = 50
K = 5000

beta0_ols_vec = np.empty(K)
beta1_ols_vec = np.empty(K)
beta2_ols_vec = np.empty(K)

beta0_lad_vec = np.empty(K)
beta1_lad_vec = np.empty(K)
beta2_lad_vec = np.empty(K)

# run monte carlo
for k in range(K):
    # run DGP
    x1,x2,y = DGP(N)

    # find betas
    ols = optimize.minimize(sum_of_squares,x0=beta_guess,method='SLSQP')
    lad = optimize.minimize(least_abs_dev,x0=beta_guess,method='SLSQP')

    # store betas
    beta0_ols_vec[k] = ols.x[0]
    beta1_ols_vec[k] = ols.x[1]
    beta2_ols_vec[k] = ols.x[2]

    beta0_lad_vec[k] = lad.x[0]
    beta1_lad_vec[k] = lad.x[1]
    beta2_lad_vec[k] = lad.x[2]
```

```python
# plot the simulation results
fig_mc, ax_mc = plt.subplots(ncols=3,figsize=(20,5))

ax_mc[0].hist(beta0_ols_vec,bins=50,alpha=0.5,label='OLS',density=True)
ax_mc[0].hist(beta0_lad_vec,bins=50,alpha=0.5, label='LAD',density=True)
ax_mc[0].axvline(0.1,linestyle='dashed',linewidth=2)
ax_mc[0].set_ylabel('Share of beta estimates in bin')
ax_mc[0].legend(loc='upper left')
ax_mc[0].set_xlabel('Realizations of beta0');

ax_mc[1].hist(beta1_ols_vec,bins=50,alpha=0.5,label='OLS',density=True)
ax_mc[1].hist(beta1_lad_vec,bins=50,alpha=0.5, label='LAD',density=True)
ax_mc[1].axvline(0.3,linestyle='dashed',linewidth=2)
ax_mc[1].legend(loc='upper left')
ax_mc[1].set_xlabel('Realizations of beta1');

ax_mc[2].hist(beta2_ols_vec,bins=50,alpha=0.5,label='OLS',density=True)
ax_mc[2].hist(beta2_lad_vec,bins=50,alpha=0.5, label='LAD',density=True)
ax_mc[2].axvline(0.5,linestyle='dashed',linewidth=2)
ax_mc[2].legend(loc='upper left')
ax_mc[2].set_xlabel('Realizations of beta2');

plt.show()
```

The above figures show the realizations of the MC simulation of the DGP. The dashed line indicates the true value of $\beta_i$. Both methods look unbiased, but it seems the consistency of LAD is better than for OLS. Therefore, we would choose LAD as our preferred method in this case.

```python
len(beta0_ols_vec)
```

    5000

# Durable purchases

Consider a **household** living in two periods.

In the **second period** it gets utility from **non-durable consumption**, $c$, and **durable consumption**, $d+\chi x$:

$$
\begin{aligned}
v_{2}(m_{2},d)&= \max_{c}\frac{(c^{\alpha}(d+\chi x)^{1-\alpha})^{1-\rho}}{1-\rho}\\
\text{s.t.} \\
x &= m_{2}-c \\
c &\in [0,m_{2}]
\end{aligned}
$$

where

* $m_2$ is cash-on-hand in the beginning of period 2
* $c$ is non-durable consumption
* $d$ is pre-commited durable consumption
* $x = m_2 - c$ is extra durable consumption
* $\rho > 1$ is the risk aversion coefficient
* $\alpha \in (0,1)$ is the utility weight on non-durable consumption
* $\chi \in (0,1)$ implies that extra durable consumption is *less* valuable than pre-comitted durable consumption
* the second constraint ensures the household *cannot* die in debt

The **value function** $v_2(m_2,d)$ measures the household's value of having $m_2$ at the beginning of period 2 with precomitted durable consumption of $d$. The optimal choice of non-durable consumption is denoted $c^{\ast}(m_2,d)$. The optimal extra durable consumption function is $x^{\ast}(m_2,d) = m_2-c^{\ast}(m_2,d)$.

Define the so-called **end-of-period 1 value function** as:

$$
\begin{aligned}
w(a,d)&\equiv\beta\mathbb{E}_{1}\left[v_2(m_2,d)\right]
\end{aligned}
$$

where

$$
\begin{aligned}
m_2&= (1+r)a+y \\
y &= \begin{cases}
1-\Delta & \text{with prob. }\frac{1}{3}\\
1 & \text{with prob. }\frac{1}{3}\\
1+\Delta & \text{with prob. }\frac{1}{3}
\end{cases}\\
\end{aligned}
$$

and

* $a$ is assets at the end of period 1
* $\beta > 0$ is the discount factor
* $\mathbb{E}_1$ is the expectation operator conditional on information in period 1
* $y$ is income in period 2
* $\Delta \in (0,1)$ is the level of income risk (mean-preserving)
* $r$ is the return on savings

In the **first period**, the household chooses it's pre-comitted level of durable consumption for the next-period,

$$
\begin{aligned}
v_{1}(m_{1})&=\max_{d} w(a,d)\\&\text{s.t.}&\\
a&= m_{1}-d \\
d&\in [0,m_{1}]\\
\end{aligned}
$$

where $m_1$ is cash-on-hand in period 1. The second constraint ensures the household *cannot* borrow. The **value function** $v_1(m_1)$ measures the household's value of having $m_1$ at the beginning of period 1. The optimal choice of pre-committed durable consumption is denoted $d^{\ast}(m_1)$.

The **parameters** and **grids** for $m_1$, $m_2$ and $d$ should be:

```python
# a. parameters
rho = 2
alpha = 0.8
beta = 0.96
r = 0.04
Delta = 0.25
chi = 0.9 # should be 0.9

# b. grids
m1_vec = np.linspace(1e-8,10,100)
m2_vec = np.linspace(1e-8,10,100)
d_vec = np.linspace(1e-8,5,100)
```

**Question 1:** Find and plot the functions $v_{2}(m_{2},d)$, $c^{\ast}(m_2,d)$, and $x^{\ast}(m_2,d)$. Comment.

First, we define the basic functions:

```python
# basic functions
def x(m2,c):
    x=m2-c
    return x

def v2(c,d,m2,chi,alpha,rho):
    return ((c**(alpha)*(d+chi*x(m2,c))**(1-alpha))**(1-rho))/(1-rho)
```

Then, we define the **solution function for period 2:**

```python
# period 2 solution function
def solve_period_2(rho,alpha,chi):

    # a. grids
    v2_grid = np.empty((100,100))
    c_grid = np.empty((100,100))
    x_grid = np.empty((100,100))

    # b. solve for each m2 in grid
    for i,m2 in enumerate(m2_vec):
        for j,d in enumerate(d_vec):

            # i. objective
            obj = lambda c: -v2(c,d,m2,chi,alpha,rho)

            # ii. initial value (consume third)
            x0 = m2/3

            # iii. optimizer
            result = optimize.minimize_scalar(obj,x0,method='bounded',bounds=[1e-8,m2])

            # iv. save results
            v2_grid[i,j] = -result.fun
            c_grid[i,j] = result.x
            x_grid[i,j] = m2-result.x

    return x_grid, v2_grid,c_grid
```

Now we can unpack the solution and plot the functions:

```python
# a. unpack solution and construct grids
x_grid,v2_grid,c_grid = solve_period_2(rho,alpha,chi)
m2_grid,d_grid = np.meshgrid(m2_vec,d_vec,indexing='ij')

# b. initialize figure
fig = plt.figure(figsize=(30,5))

#subplots
# subplot value in period 2
ax = fig.add_subplot(1,3,1,projection='3d')
cs = ax.plot_surface(m2_grid,d_grid,v2_grid,cmap=cm.jet)

# add labels
ax.set_title('utility in period 2')
ax.set_xlabel('$m_2$')
ax.set_ylabel('$d$')
ax.set_zlabel('$v$')

# invert xaxis
ax.invert_xaxis()

# add colorbar
fig.colorbar(cs);

# subplot consumption
ax = fig.add_subplot(1,3,2,projection='3d')
cs = ax.plot_surface(m2_grid,d_grid,c_grid,cmap=cm.jet)

# add labels
ax.set_title('non durable consumption in period 2')
ax.set_xlabel('$m_2$')
ax.set_ylabel('$d$')
ax.set_zlabel('$c$')

# invert xaxis
ax.invert_xaxis()

# add colorbar
fig.colorbar(cs);

# subplot extra durables
ax = fig.add_subplot(1,3,3,projection='3d')
cs = ax.plot_surface(m2_grid,d_grid,x_grid,cmap=cm.jet)

# add labels and title
ax.set_title('extra durable consumption in period 2')
ax.set_xlabel('$m_2$')
ax.set_ylabel('$d$')
ax.set_zlabel('$x$')

# invert xaxis
ax.invert_xaxis()

# add colorbar
fig.colorbar(cs);
```

**Comment:** We see from the plof of $x$ that for extra durable good consumption to be positive, the consumer must have a relatively high amount of cash on hand and/or a relatively low amount of initial durable good consumption. This makes good sense, as the marginal utility from consumption in period 2 ($\alpha=0.8)$ is considerably higher than the marginal utility of durable good consumption ($1-\alpha=0.2$). The kink in the x function is, in a sense, mirrored in the consumption function, as the consumer for a relatively high amount of $d$ spends its full cash endowment on consumption.

The plot of $v_2$ is not that easy to interpret. This is due to a property of the utility function (more precisely the "Cobb-Douglas part"), which tends towards negative infinity for very low values of $d$ and $m_2$. Hence, it is difficult to see that there actually is a change in the dark red area as it appears flat. One workaround is to adjust the starting point of the `np.linspace()` vectors m2_vec and d_vec from 1-e8 to e.g. 0.3, such that values very close to zero are excluded and the readability of the plot improves.

```python
# check if all of m2 is used for consumption for d approximately 2.5
np.allclose(m2_vec[99],c_grid[99,50])
```

    True

**Question 2:** Find and plot the functions $v_{1}(m_{1})$ and $d^{\ast}(m_1)$. Comment.

**Hint:** For interpolation of $v_2(m_2,d)$ consider using `interpolate.RegularGridInterpolator([GRID-VECTOR1,GRID-VECTOR2],VALUE-MATRIX,bounds_error=False,fill_value=None)`.

```python
# basic function
def v1(m1,d,Delta,r,beta,v2_interp):
    a = m1-d

    # a. v2 value, if low income
    m2_low = (1+r)*a+(1-Delta)
    v2_low = v2_interp([m2_low,d])[0]

    # b. v2 value, if mid income
    m2_mid = (1+r)*a+1
    v2_mid = v2_interp([m2_mid,d])[0]

    # b. v2 value, if high income
    m2_high = (1+r)*a+(1+Delta)
    v2_high = v2_interp([m2_high,d])[0]

    # c. expected v2 value
    v2_exp = 1/3*v2_low+ 1/3*v2_mid+ 1/3*v2_high

    v1 = beta*v2_exp

    # d. total value
    return v1
```

```python
# a. define solve function
def solve_period_1(alpha,rho,beta,r,Delta,v1,v2_interp):

    # grids
    v1_vec = np.empty(100)
    d1_vec = np.empty(100)

    # solve for each m1 in grid
    for i,m1 in enumerate(m1_vec):

        # i. objective
        obj = lambda d: -v1(m1,d,Delta,r,beta,v2_interp)

        # ii. initial guess (durable consume half)
        x0 = [m1/2]

        # iii. optimize
        result = optimize.minimize_scalar(obj,x0,method='bounded', bounds=(1e-8,m1))

        # iv. save
        v1_vec[i] = -result.fun
        d1_vec[i] = result.x

    return v1_vec,d1_vec

# b. construct interpolator
v2_interp = interpolate.RegularGridInterpolator((m2_vec,d_vec),v2_grid,
                                                bounds_error=False,fill_value=None)

# c. solve period 1
v1_vec,d1_vec = solve_period_1(alpha,rho,beta,r,Delta,v1,v2_interp)

# d. plot
fig = plt.figure(figsize=(15,5))

ax = fig.add_subplot(1,2,1)
ax.plot(m1_vec,v1_vec)
ax.set_title('value function period 1')
ax.set_xlabel('$m_1$')
ax.set_ylabel('$v_1$');

ax = fig.add_subplot(1,2,2)
ax.plot(m1_vec,d1_vec)
ax.set_title('durable consumption')
ax.set_xlabel('$m_1$')
ax.set_ylabel('$d$');
```

**Comment:** We see from the plots that both $v_1$ and $d$ are increasing functions of $m_1$. This makes good sense; the more money you have, the more you can consume (both durably and non-durably), which increases the utility of the agent.

Next, consider an **extension** of the model, where there is also a **period 0**. In this period, the household makes a choice whether to stick with the level of durables it has, $z = 0$, or adjust its stock of durables, $z = 1$. If adjusting, the household loses a part of the value of its durable stock; more specificaly it incurs a proportional loss of $\Lambda \in (0,1)$.

Mathematically, the **household problem in period 0** is:

$$
\begin{aligned}
v_{0}(m_{0},d_{0}) &= \max_{z\in\{0,1\}} \begin{cases}
w(m_{0},d_{0}) & \text{if } z = 0\\
v_1(m_0+(1-\Lambda) d_{0}) & \text{if } z = 1\\
\end{cases}\\
\end{aligned}
$$

The **parameters** and **grids** for $m_0$ and $d_0$ should be:

```python
Lambda = 0.2
m0_vec = np.linspace(1e-8,6,100)
d0_vec = np.linspace(1e-8,3,100)
```

**Question 3:** For which values of $m_0$ and  $d_0$ is the optimal choice not to adjust, i.e. $z = 0$? Show this in a plot. Give an interpretion of your results.

```python
# case 1: z = 0
def solve_period_0_nochange(alpha,rho,beta,r,Delta,v1,v2_interp):

    # a. grids
    v0_vec = np.empty(100)
    d0_vec = np.empty(100)

    # b. solve for each m1 in grid
    for i,m0 in enumerate(m0_vec):

        # i. objective
        obj = lambda d: -v1(m1,d,Delta,r,beta,v2_interp)

        # ii. initial guess (durable consume half)
        x0 = [m1/2]

        # iii. optimize
        result = optimize.minimize_scalar(obj,x0,method='bounded', bounds=(1e-8,m1))

        # iv. save
        v1_vec[i] = -result.fun
        d1_vec[i] = result.x

    return v1_vec,d1_vec

# b. construct interpolator
v2_interp = interpolate.RegularGridInterpolator((m2_vec,d_vec),v2_grid,
                                                bounds_error=False,fill_value=None)

# c. solve period 1
v1_vec,d1_vec = solve_period_1(alpha,rho,beta,r,Delta,v1,v2_interp)
```

```python
def w_solve_period_0_nochange(m0,d0,Delta,r,beta,v2_interp):

    a = m0-d

    # a. v2 value, if low income
    m2_low = (1+r)*a+(1-Delta)
    v2_low = v2_interp([m2_low,d])[0]

    # b. v2 value, if mid income
    m2_mid = (1+r)*a+1
    v2_mid = v2_interp([m2_mid,d])[0]

    # b. v2 value, if high income
    m2_high = (1+r)*a+(1+Delta)
    v2_high = v2_interp([m2_high,d])[0]

    # c. expected v2 value
    v2_exp = 1/3*v2_low+ 1/3*v2_mid+ 1/3*v2_high

    v1 = beta*v2_exp

    # d. total value
    return v1

#def solve_period_0
```

# Gradient descent

Let $\boldsymbol{x} = \left[\begin{array}{c}
x_1 \\
x_2\\
\end{array}\right]$ be a two-dimensional vector. Consider the following algorithm:

**Algorithm:** `gradient_descent()`

**Goal:** Minimize the function $f(\boldsymbol{x})$.

1. Choose a tolerance $\epsilon>0$, a scale factor $ \Theta > 0$, and a small number $\Delta > 0$
2. Guess on $\boldsymbol{x}_0$ and set $n=1$
3. Compute a numerical approximation of the jacobian for $f$ by

    $$
    \nabla f(\boldsymbol{x}_{n-1}) \approx \frac{1}{\Delta}\left[\begin{array}{c}
    f\left(\boldsymbol{x}_{n-1}+\left[\begin{array}{c}
    \Delta\\
    0
    \end{array}\right]\right)-f(\boldsymbol{x}_{n-1})\\
    f\left(\boldsymbol{x}_{n-1}+\left[\begin{array}{c}
    0\\
    \Delta
    \end{array}\right]\right)-f(\boldsymbol{x}_{n-1})
    \end{array}\right]
    $$

4. Stop if the maximum element in $|\nabla f(\boldsymbol{x}_{n-1})|$ is less than $\epsilon$
5. Set $\theta = \Theta$
6. Compute $f^{\theta}_{n} = f(\boldsymbol{x}_{n-1} - \theta \nabla f(\boldsymbol{x}_{n-1}))$
7. If $f^{\theta}_{n} < f(\boldsymbol{x}_{n-1})$ continue to step 9
8. Set $\theta = \frac{\theta}{2}$ and return to step 6
9. Set $x_{n} = x_{n-1} - \theta \nabla f(\boldsymbol{x}_{n-1})$
10. Set $n = n + 1$ and return to step 3

**Question:** Implement the algorithm above such that the code below can run.

**Optimizer function:**

```python
def gradient_descent(f,x0,epsilon=1e-6,Theta=0.1,Delta=1e-8,max_iter=10000): #step 1
    """
    Algorithm to minimize a two dimensional vector function.

    Arguments:
    f (func): function to be minimized
    x0 (vector): 2D initial guess vector
    epsilon (float): tolerance
    Theta (float): scale factor
    Delta (float): small number to be used in the numerical jacobian
    max_iter (float): maximum number of iterations

    Returns:
    x (vector): vector of coordinates that minimize the function within the tolerance
    """
    # step 2
    x = x0 # guess
    n = 1 # iterations

    # step 3-10
    while n < max_iter:

        # save current x as xprev
        xprev = x

        # step 3: compute numerical jacobian
        jac_array = np.array([f([x[0]+Delta,x[1]])-f(x),f([x[0],x[1]+Delta])-f(x)])
        num_jacx = 1/Delta*jac_array

        # step 4: check convergence
        if abs(np.max(num_jacx)) < epsilon:
            break

        else:
            # step 5:
            theta = Theta

            while True:

                # step 6: compute f_theta
                f_theta = f(x-theta*num_jacx)

                # step 7: check ineq
                if f_theta < f(x):
                    break

                else:
                    # step 8: update theta
                    theta /= 2

        # step 9: update guess
        x = xprev-theta*num_jacx

        # step 10:
        n += 1

    return x,n
```

**Test case:**

```python
def rosen(x):
    return (1.0-x[0])**2+2*(x[1]-x[0]**2)**2

x0 = np.array([1.1,1.1])
try:
    x,it = gradient_descent(rosen,x0)
    print(f'minimum found at ({x[0]:.4f},{x[1]:.4f}) after {it} iterations')
    assert np.allclose(x,[1,1])
except:
    print('not implemented yet')
```

    minimum found at (1.0000,1.0000) after 306 iterations


```python
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
```

# Using finite differences to approximate derivatives

The material in this notebook is intended to give a brief introduction, and is not comprehensive.

Suppose that you're given a table of values of some "secret" function $f(x)$, but you are not given the function itself.  For example:

```python
x_values = (0, 1, 2, 3, 4)
y_values = (0, 1, 8, 27, 64)
```

In order to understand the function that generated these values, we could plot them:

```python
plt.plot(x_values, y_values, 'o')
```

Each of the points can be connected by a line, and you might use the slope of this line if you wanted to estimate the derivative of the function.  For instance, to estimate the slope at $x=2$, you could use the slope of the line between $x=2$ and $x=3$:

```python
plt.plot(x_values, y_values, 'o')
plt.hold(True)
plt.plot(x_values[2:4], y_values[2:4], '-r', linewidth=5)
```

$$\left. f'(x) \right|_{x=2} \approx \frac{f(3)-f(2)}{3-2} = 19.$$

Alternatively, you might use the slope of the line between $x=1$ and $x=2$:

```python
plt.plot(x_values,y_values,'-o')
plt.hold(True)
plt.plot(x_values[1:3],y_values[1:3],'-r',linewidth=5)
```

$$\left. f'(x) \right|_{x=2} \approx \frac{f(2)-f(1)}{2-1} = 7.$$

Remember that the derivative is defined as the limit of a formula rather like our first approximation above (referred to as a *forward difference*):

$$ f'(x) = \lim_{h\to 0} \frac{f(x+h)-f(x)}{(x+h)-x} = \lim_{h\to 0} \frac{f(x+h)-f(x)}{h}.$$

Notice that the fraction in the definition is just the slope of the line connecting the values of $f$ at $x$ and $x+h$.  We could equally well define the derivative in terms of a *backward difference*, similar to our second formula above:

$$ f'(x) = \lim_{h\to 0} \frac{f(x)-f(x-h)}{x-(x-h)} = \lim_{h\to 0} \frac{f(x)-f(x-h)}{h}.$$

Yet another way of defining the derivative would be to use the slope of the line connecting points $x+h$ and $x-h$, leading to

$$ f'(x) = \lim_{h \to 0} \frac{f(x+h) - f(x-h)}{2h}.$$

Here the factor $2h$ in the denominator is just the length of the interval $(x-h,x+h)$.  Using this formula with our values gives

```python
plt.plot(x_values,y_values,'-ok')
plt.hold(True)
plt.plot((x_values[1],x_values[3]),(y_values[1],y_values[3]),'-r',linewidth=5)
```

$$\left. f'(x) \right|_{x=2} \approx \frac{f(3)-f(1)}{3-1} = 13.$$

When we only know the values of a function at a finite set of points, we can't compute these limits, so it makes sense to use a finite value of $h$ in order to approximate the derivative.  This approach is known as the *finite difference method*.  Soon we will use it to solve differential equations, but first let's examine its effectiveness in approximating derivatives.

You may have guessed that the "secret" function that generated these values is $f(x)=x^3$.  Let's compare our three estimates of the derivative with the true derivative: $f'(2) = 3\cdot 2^2 = 12$.  Clearly the third formula (known as a *centered difference*) is the best approximation.  We can also see this by plotting the approximations and the tangent line (whose slope is that of the true derivative):

```python
def f(x):
    return x**3

x = np.linspace(0,4)
tangent = 8 + 12*(x-2)
plt.plot(x,f(x),'k',linewidth=2)
plt.plot(x_values[2:4],y_values[2:4],'-g',linewidth=3)
plt.plot(x_values[1:3],y_values[1:3],'-b',linewidth=3)
plt.plot((x_values[1],x_values[3]),(y_values[1],y_values[3]),'-r',linewidth=3)
plt.plot(x,tangent,'--k',linewidth=3)
plt.legend(['f(x)','Forward','Backward','Centered','Tangent'],loc='best')
plt.axis((0.5,3.5,0,40))
```

It's clear that the centered difference approximation (slope of the red line) is closest to the derivative (slope of the dotted line).  Would this be true if we picked another function $f(x)$, or if we used function values at different points?  Let's answer the second question with an experiment.

```python
x = 2.
df = 12.

for h in (0.1, 0.05, 0.025):
    forward = (f(x+h)-f(x))/h
    forward_error = forward - df
    print(forward_error)
```

    0.6100000000000101
    0.30249999999994515
    0.15062499999994827

It seems clear from the results that reducing $h$ by a factor of two also reduces the error in the forward difference approximation by a factor of two.

**Now add backward and centered difference approximations to the code above and try to figure out what happens to their errors as $h$ decreases.  Make a table like Table 1.1 on page 5 of the textbook.**  Is the centered difference always the most accurate?  Do you think it will still be the most accurate if we continue reducing $h$?

**Now redefine the function $f(x)$ to be $x^2$ and generate the same table.  Do you see anything surprising?  Can you explain it?**  If not, make a guess.

## Estimating truncation errors

The error made by a finite difference approximation is called *truncation error*.  Why?  Well, we can estimate the error by expanding each function value in a *Taylor series*:

$$f(x+h) = f(x) + h f'(x) + \frac{1}{2}h^2 f''(x) + \frac{1}{6} h^3 f'''(x) + {\mathcal O}(h^4)$$

Here ${\mathcal O}(h^4)$ indicates that the rest of the terms in the series vanish at least as quickly as $h^4$ when $h\to 0$ (see Appendix A of the text).
Substituting this series in our forward difference formula gives
\begin{align}
\frac{f(x+h) - f(x)}{h} & = \frac{f(x) + h f'(x) + \frac{1}{2}h^2 f''(x) + \frac{1}{6} h^3 f'''(x) + {\mathcal O}(h^4) - f(x)}{h} \\
& = f'(x) + \frac{1}{2}h f''(x) + \frac{1}{6} h^2 f'''(x) + {\mathcal O}(h^3) \\
& = f'(x) + \frac{1}{2}h f''(x) + {\mathcal O}(h^2).
\end{align}

This analysis confirms our intuition that the forward difference approximates $f'(x)$, but it tells us much more.  Most importantly, it shows that the largest term in the error in this approximation is proportional to $h$.  That's why we saw that decreasing $h$ by a factor of two caused the error to decrease by the same amount.

Notice that if we truncated the Taylor series after the first term, we would get the forward difference formula exactly.  That is why the error is referred to as *truncation error* -- it's the error we get from truncating an infinite series.  The term $\frac{1}{2} h f''(x)$ is referred to as the *leading truncation error* because when $h$ is very small we expect that term to be much bigger than all the ones that come after it.

Since in our simple example we know the function $f(x)$, we could evaluate error terms in the series above to get a better approximation of the error.  But typically we won't know what the function is (if we did, why would we need finite differences?), so we'll be most interested in knowing what power of $h$ multiplies the leading error term.

**Write down the Taylor series for $f(x-h)$ about $x$.  Use that (and the series above) to work out the leading truncation error terms for the backward and centered difference formulas.**  Does your result for the centered formula allow you to explain the results you found before when applying it to $f(x)=x^2$?

## Polynomial interpolation and finite difference formulas

In sections 1.2-1.5 of the text, a method for finding finite difference formulas is given, based on Taylor series.  Here we will approach the same topic from a different route, using polynomial interpolation.  The idea is this: to approximate the derivative of a function whose point values are known, we first find a polynomial that interpolates those values, then evaluate the derivative of that polynomial.

To begin, suppose we are given three function values: $f(x_0-h), f(x_0), f(x_0+h)$, which we'll denote by $(f_1,f_2,f_3)$.  We wish to find a polynomial that passes through these three points.  As you may know, a set of $n$ values uniquely defines a polynomial of degree $n-1$, so we will look for a quadratic polynomial.  To make the computation simpler, we'll write it this way: $p(x) = a + b (x-x_0) + c (x-x_0)^2$.  We know that $p$ and $f$ must agree at the three given points, which means
\begin{align}
a + b(-h) + c (-h)^2  & = f_1 \\
a + b(0) + c(0)^2 & = f_2 \\
a + b (h) + c (h)^2 & = f_3 \\
\end{align}

or simply

\begin{align}
a - hb + h^2c  & = f_1 \\
a & = f_2 \\
a + hb + h^2c & = f_3 \\
\end{align}

We can rewrite this system of equations in matrix form:

\begin{align}
\begin{pmatrix}
1 & -h & h^2 \\
1 & 0   & 0 \\
1 & h & h^2 \\
\end{pmatrix}
\begin{pmatrix} a \\ b \\ c \end{pmatrix}
& = \begin{pmatrix} f_1 \\ f_2 \\ f_3 \end{pmatrix}
\end{align}

This is a linear system that we can solve for the coefficients $a,b,c$ in terms of $h$, and the values of $f$.  The result is

\begin{align}
a & = f_2, & b & = \frac{f_3-f_1}{2h}, & c & = \frac{f_1 -2 f_2 + f_3}{2h^2}.
\end{align}

Thus

\begin{align}
p(x) = f_2 + \frac{x-x_0}{2h}(f_3-f_1) + \frac{(x-x_0)^2}{2h^2}(f_1-2f_2+f_3).
\end{align}

Now suppose we wish to approximate the derivative of $f$.  We have an easy approach: use the derivative of $p$:

\begin{align}
p'(x) = \frac{f_3-f_1}{2h} + \frac{x-x_0}{h^2}(f_1 -2 f_2 + f_3).
\end{align}

In particular, if want to approximate the derivative at $x=x_0$, we get simply

$$f'(x) \approx p'(x_0) = \frac{f_3-f_1}{2h} = \frac{f(x_0+h) - f(x_0-h)}{2h}.$$

**Do you recognize this approximation?  Now use $p(x)$ to get an approximation for the second derivative of $f(x)$.  Write down the resulting formula.**

<font color='red'>
Derive a formula for $f''(x)$ based on the values $f(x), f(x-h), f(x-3h)$ by determining the interpolating polynomial and differentiating it.  How accurate is your formula?
</font>

**Can you think of a similar approach that would allow you to derive rules for numerical quadrature (integration)?**

```python

```


# Supervised Learning: Neural Networks

```python
%matplotlib inline
import warnings
warnings.filterwarnings("ignore")

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
sns.set(style='ticks')

import tensorflow as tf
from scipy import optimize
from ipywidgets import interact
from IPython.display import SVG
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import scale
```

## McCulloch and Pitts Neuron

In 1943, McCulloch and Pitts introduced a mathematical model of a neuron. It consisted of three components:

1. A set of **weights** $w_i$ corresponding to synapses (inputs)
2. An **adder** for summing input signals; analogous to cell membrane that collects charge
3. An **activation function** for determining when the neuron fires, based on accumulated input

The neuron model is shown schematically below. On the left are input nodes $\{x_i\}$, usually expressed as a vector. The strength with which the inputs are able to deliver the signal along the synapse is determined by their corresponding weights $\{w_i\}$. The adder then sums the inputs from all the synapses:

$$h = \sum_i w_i x_i$$

The parameter $\theta$ determines whether or not the neuron fires given a weighted input of $h$. If it fires, it returns a value $y=1$, otherwise $y=0$. For example, a simple **activation function** is using $\theta$ as a simple fixed threshold:

$$y = g(h) = \left\{ \begin{array}{l}
1, \text{if } h \gt \theta \\
0, \text{if } h \le \theta
\end{array} \right.$$

this activation function may take any of several forms, such as a logistic function.

A single neuron is not interesting, nor useful, from a learning perspective. It cannot learn; it simply receives inputs and either fires or not. Only when neurons are joined as a **network** can they perform useful work.

Learning takes place by changing the weights of the connections in a neural network, and by changing the parameters of the activation functions of neurons.

## Perceptron

A collection of McCullough and Pitts neurons, along with a set of input nodes connected to the inputs via weighted edges, is a perceptron, the simplest neural network.

Each neuron is independent of the others in the perceptron, in the sense that its behavior and performance depends only on its own weights and threshold values, and not of those for the other neurons. Though they share inputs, they operate independently.

The number of inputs and outputs are determined by the data. Weights are stored as a `N x K` matrix, with N observations and K neurons, with $w_{ij}$ specifying the weight on the *i*th observation on the *j*th neuron.

In order to use the perceptron for statistical learning, we compare the outputs $y_j$ from each neuron to the obervation target $t_j$, and adjust the input weights when they do not correspond (*e.g.* if a neuron fires when it should not have).

$$t_j - y_j$$

We use this difference to update the weight $w_{ij}$, based on the input and a desired **learning rate**. This results in an update rule:

$$w_{ij} \leftarrow w_{ij} + \eta (t_j - y_j) x_i$$

After an incremental improvement, the perceptron is shown the training data again, resulting in another update. This is repeated until the performance no longer improves. Having a learning rate less than one results in a more stable learning rate, though this stability is traded off against having to expose the network to the data multiple times. Typical learning rates are in the 0.1-0.4 range.

An additional input node is typically added to the perceptron model, which is a constant value (usually -1, 0, or 1) that acts analogously to an intercept in a regression model. This establishes a baseline input for the case when all inputs are zero.

## Learning with Perceptrons

1. Initialize weights $w_{ij}$ to small, random numbers.
2. For each t in T iterations
    * compute activation for each neuron *j* connected to each input vector *i*
    $$y_j = g\left( h=\sum_i w_{ij} x_i \right) = \left\{ \begin{array}{l}
1, \text{if } h \gt 0 \\
0, \text{if } h \le 0
\end{array} \right.$$
    * update weights
    $$w_{ij} \leftarrow w_{ij} + \eta (t_j - y_j) x_i$$

This algorithm is $\mathcal{O}(Tmn)$

### Example: Logical functions

Let's see how the perceptron learns by training it on a couple of of logical functions, AND and OR. For two variables `x1` and `x2`, the AND function returns 1 if both are true, or zero otherwise; the OR function returns 1 if either variable is true, or both. These functions can be expressed as simple lookup tables.

```python
AND = pd.DataFrame({'x1': (0,0,1,1), 'x2': (0,1,0,1), 'y': (0,0,0,1)})
AND
```

      x1
      x2
      y

      0
      0
      0
      0

      1
      0
      1
      0

      2
      1
      0
      0

      3
      1
      1
      1

First, we need to initialize weights to small, random values (can be positive and negative).

```python
w = np.random.randn(3)*1e-4
```

Then, a simple activation function for calculating $g(h)$:

```python
g = lambda inputs, weights: np.where(np.dot(inputs, weights)>0, 1, 0)
```

Finally, a training function that iterates the learning algorithm, returning the adapted weights.

```python
def train(inputs, targets, weights, eta, n_iterations):

    # Add the inputs that match the bias node
    inputs = np.c_[inputs, -np.ones((len(inputs), 1))]

    for n in range(n_iterations):

        activations = g(inputs, weights);
        weights -= eta*np.dot(np.transpose(inputs), activations - targets)

    return(weights)
```

Let's test it first on the AND function.

```python
inputs = AND[['x1','x2']]
target = AND['y']

w = train(inputs, target, w, 0.25, 10)
```

Checking the performance:

```python
g(np.c_[inputs, -np.ones((len(inputs), 1))], w)
```

    array([0, 0, 0, 1])

Thus, it has learned the function perfectly. Now for OR:

```python
OR = pd.DataFrame({'x1': (0,0,1,1), 'x2': (0,1,0,1), 'y': (0,1,1,1)})
OR
```

      x1
      x2
      y

      0
      0
      0
      0

      1
      0
      1
      1

      2
      1
      0
      1

      3
      1
      1
      1

```python
w = np.random.randn(3)*1e-4
```

```python
inputs = OR[['x1','x2']]
target = OR['y']

w = train(inputs, target, w, 0.25, 20)
```

```python
g(np.c_[inputs, -np.ones((len(inputs), 1))], w)
```

    array([0, 1, 1, 1])

Also 100% correct.

### Exercise: XOR

Now try running the model on the XOR function, where a one is returned for either `x1` or `x2` being true, but *not* both. What happens here?

```python
XOR = pd.DataFrame({'x1': (0,0,1,1), 'x2': (0, 1, 0, 1), 'y':(0, 1, 1,0)})
XOR
```

      x1
      x2
      y

      0
      0
      0
      0

      1
      0
      1
      1

      2
      1
      0
      1

      3
      1
      1
      0

#### Initializing weights

```python
w = np.random.randn(3) * 1e-4
```

```python
inputs = XOR[['x1','x2']]
target = XOR['y']

w = train(inputs, target, w, 0.25, 20)
```

```python
g(np.c_[inputs, -np.ones((len(inputs), 1))], w)
```

    array([1, 1, 1, 1])

Let's explore the problem graphically:

```python
AND.plot(kind='scatter', x='x1', y='x2', c='y', s=50, colormap='winter')
plt.plot(np.linspace(0,1.4), 1.5 - 1*np.linspace(0,1.4), 'k--');
```

```python
OR.plot(kind='scatter', x='x1', y='x2', c='y', s=50, colormap='winter')
plt.plot(np.linspace(-.4,1), .5 - 1*np.linspace(-.4,1), 'k--');
```

```python
XOR = pd.DataFrame({'x1': (0,0,1,1), 'x2': (0,1,0,1), 'y': (0,1,1,0)})

XOR.plot(kind='scatter', x='x1', y='x2', c='y', s=50, colormap='winter');
```

The perceptron tries to find a separating hyperplane for the two response classes. Namely, a set of weights that satisfies:

$$\mathbf{x_1}\mathbf{w}^T=0$$

and:

$$\mathbf{x_2}\mathbf{w}^T=0$$

Hence,

$$\begin{aligned}
\mathbf{x}_1\mathbf{w}^T &= \mathbf{x}_2\mathbf{w}^T \\
\Rightarrow (\mathbf{x}_1 - \mathbf{x}_2) \mathbf{w}^T &= 0
\end{aligned}$$

This means that either the norms of $\mathbf{x}_1 - \mathbf{x}_2$ or $\mathbf{w}$ are zero, or the cosine of the angle between them is equal to zero, due to the identity:

$$\mathbf{a}\mathbf{b} = \|a\| \|b\| \cos \theta$$

Since there is no reason for the norms to be zero in general, we need the two vectors to be at right angles to one another. So, we need a weight vector that is perpendicular to the decision boundary.

Clearly, for the XOR function, the output classes are not linearly separable. So, the algorithm does not converge on an answer, but simply cycles through two incorrect solutions.

## Multi-layer Perceptron

The solution to fitting more complex (*i.e.* non-linear) models with neural networks is to use a more complex network that consists of more than just a single perceptron. The take-home message from the perceptron is that all of the learning happens by adapting the synapse weights until prediction is satisfactory. Hence, a reasonable guess at how to make a perceptron more complex is to simply **add more weights**.

There are two ways to add complexity:

1. Add backward connections, so that output neurons feed back to input nodes, resulting in a **recurrent network**
2. Add neurons between the input nodes and the outputs, creating an additional ("hidden") layer to the network, resulting in a **multi-layer perceptron**

The latter approach is more common in applications of neural networks.

How to train a multilayer network is not intuitive. Propagating the inputs forward over two layers is straightforward, since the outputs from the hidden layer can be used as inputs for the output layer. However, the process for updating the weights based on the prediction error is less clear, since it is difficult to know whether to change the weights on the input layer or on the hidden layer in order to improve the prediction.

Updating a multi-layer perceptron (MLP) is a matter of:

1. moving forward through the network, calculating outputs given inputs and current weight estimates
2. moving backward updating weights according to the resulting error from forward propagation.

In this sense, it is similar to a single-layer perceptron, except it has to be done twice, once for each layer (in principle, we can add additional hidden layers, but without sacrificing generality, I will keep it simple).

### Error back-propagation

We update the weights in a MLP using **back-propagation** of the prediction errors, which is essentially a form of gradient descent, as we have used previously for optimization.

First, for the multi-layer perceptron we need to modify the error function, which in the single-layer case was a simple difference between the predicted and observed outputs. Because we will be summing errors, we have to avoid having errors in different directions cancelling each other out, so a sum of squares error is more appropriate:

$$E(t,y) = \frac{1}{2} \sum_i (t_i - y_i)^2$$

It is on this function that we will perform gradient descent, since the goal is to minimize the error. Specificially, we will differentiate with respect to the weights, since it is the weights that we are manipulating in order to get better predictions.

Recall that the error is a function of the threshold function

$$E(\mathbf{w}) = \frac{1}{2} \sum_i (t_i - y_i)^2 = \frac{1}{2} \sum_i \left(t_i - g\left[ \sum_j w_{ij} a_j \right]\right)^2$$

So, we will also need to differentiate that. However, the threshold function we used in the single-layer perceptron was discontinuous, making it non-differentiable. Thus, we need to modify it as well. An alternative is to employ some type of sigmoid function, such as the logistic, which can be parameterized to resemble a threshold function, but varies smoothly across its range.

$$g(h) = \frac{1}{1 + \exp(-\beta h)}$$

```python
logistic = lambda h, beta: 1./(1 + np.exp(-beta * h))

@interact(beta=(-1, 25))
def logistic_plot(beta=5):
    hvals = np.linspace(-2, 2)
    plt.plot(hvals, logistic(hvals, beta))
```

This has the advantage of having a simple derivative:

$$\frac{dg}{dh} = \beta g(h)(1 - g(h))$$

Alternatively, the hyperbolic tangent function is also sigmoid:

$$g(h) = \tanh(h) = \frac{\exp(h) - \exp(-h)}{\exp(h) + \exp(-h)}$$

```python
hyperbolic_tangent = lambda h: (np.exp(h) - np.exp(-h)) / (np.exp(h) + np.exp(-h))

@interact(theta=(-1, 25))
def tanh_plot(theta=5):
    hvals = np.linspace(-2, 2)
    h = hvals*theta
    plt.plot(hvals, hyperbolic_tangent(h))
```

Notice that the hyperbolic tangent function asymptotes at -1 and 1, rather than 0 and 1, which is sometimes beneficial, and its derivative is simple:

$$\frac{d \tanh(x)}{dx} = 1 - \tanh^2(x)$$

Performing gradient descent will allow us to change the weights in the direction that optimially reduces the error. The next trick will be to employ the **chain rule** to decompose how the error changes as a function of the input weights into the change in error as a function of changes in the inputs to the weights, mutliplied by the changes in input values as a function of changes in the weights.

$$\frac{\partial E}{\partial w} = \frac{\partial E}{\partial h}\frac{\partial h}{\partial w}$$

This will allow us to write a function describing the activations of the output weights as a function of the activations of the hidden layer nodes and the output weights, which will allow us to propagate error backwards through the network.

The second term in the chain rule simplifies to:

$$\begin{align}
\frac{\partial h_k}{\partial w_{jk}} &= \frac{\partial \sum_l w_{lk} a_l}{\partial w_{jk}}  \\
&= \sum_l \frac{\partial w_{lk} a_l}{\partial w_{jk}} \\
& = a_j
\end{align}$$

where $a_j$ is the activation of the jth hidden layer neuron.

For the first term in the chain rule above, we decompose it as well:

$$\frac{\partial E}{\partial h_k} = \frac{\partial E}{\partial y_k}\frac{\partial y_k}{\partial h_k} = \frac{\partial E}{\partial g(h_k)}\frac{\partial g(h_k)}{\partial h_k}$$

The second term of this chain rule is just the derivative of the activation function, which we have chosen to have a conveneint form, while the first term simplifies to:

$$\frac{\partial E}{\partial g(h_k)} = \frac{\partial}{\partial g(h_k)}\left[\frac{1}{2} \sum_k (t_k - y_k)^2 \right] = t_k - y_k$$

Combining these, and assuming (for illustration) a logistic activiation function, we have the gradient:

$$\frac{\partial E}{\partial w} = (t_k - y_k) y_k (1-y_k) a_j$$

Which ends up getting plugged into the weight update formula that we saw in the single-layer perceptron:

$$w_{jk} \leftarrow w_{jk} - \eta (t_k - y_k) y_k (1-y_k) a_j$$

Note that here we are *subtracting* the second term, rather than adding, since we are doing gradient descent.

We can now outline the MLP learning algorithm:

1. Initialize all $w_{jk}$ to small random values
2. For each input vector, conduct forward propagation:
    * compute activation of each neuron $j$ in hidden layer (here, sigmoid):
    $$h_j = \sum_i x_i v_{ij}$$
    $$a_j = g(h_j) = \frac{1}{1 + \exp(-\beta h_j)}$$
    * when the output layer is reached, calculate outputs similarly:
    $$h_k = \sum_k a_j w_{jk}$$
    $$y_k = g(h_k) = \frac{1}{1 + \exp(-\beta h_k)}$$
3. Calculate loss for resulting predictions:
    * compute error at output:
    $$\delta_k = (t_k - y_k) y_k (1-y_k)$$
4. Conduct backpropagation to get partial derivatives of cost with respect to weights, and use these to update weights:
    * compute error of the hidden layers:
    $$\delta_{hj} = \left[\sum_k w_{jk} \delta_k \right] a_j(1-a_j)$$
    * update output layer weights:
    $$w_{jk} \leftarrow w_{jk} - \eta \delta_k a_j$$
    * update hidden layer weights:
    $$v_{ij} \leftarrow v_{ij} - \eta \delta_{hj} x_i$$

Return to (2) and iterate until learning completes. Best practice is to shuffle input vectors to avoid training in the same order.

Its important to be aware that because gradient descent is a hill-climbing (or descending) algorithm, it is liable to be caught in local minima with respect to starting values. Therefore, it is worthwhile training several networks using a range of starting values for the weights, so that you have a better chance of discovering a globally-competitive solution.

One useful performance enhancement for the MLP learning algorithm is the addition of **momentum** to the weight updates. This is just a coefficient on the previous weight update that increases the correlation between the current weight and the weight after the next update. This is particularly useful for complex models, where falling into local mimima is an issue; adding momentum will give some weight to the previous direction, making the resulting weights essentially a weighted average of the two directions. Adding momentum, along with a smaller learning rate, usually results in a more stable algorithm with quicker convergence. When we use momentum, we lose this guarantee, but this is generally seen as a small price to pay for the improvement momentum usually gives.

A weight update with momentum looks like this:

$$w_{jk} \leftarrow w_{jk} - \eta \delta_k a_j + \alpha \Delta w_{jk}^{t-1}$$

where $\alpha$ is the momentum (regularization) parameter and $\Delta w_{jk}^{t-1}$ the update from the previous iteration.

The multi-layer pereptron is implemented below in the `MLP` class. The implementation uses the scikit-learn interface, so it is uses in the same way as other supervised learning algorithms in that package.

```python
softmax = lambda a: a / np.sum(a, axis=1, keepdims=True)

class MLP:

    def __init__(self, alpha=0.01, eta=0.01, n_hidden_dim=25):

        self.alpha = alpha
        self.eta = eta
        self.n_hidden_dim = n_hidden_dim

    # Helper function to evaluate the total loss on the dataset
    def calculate_loss(self, X, y):

        # Forward propagation to calculate our predictions
        z1 = X.dot(self.w1) + self.b1
        a1 = np.tanh(z1)
        z2 = a1.dot(self.w2) + self.b2
        exp_scores = np.exp(z2)
        probs = softmax(exp_scores)

        # Calculating the loss
        data_loss = -np.log(probs[range(num_examples), y]).sum()

        # Add regulatization term to loss (optional)
        data_loss += self.alpha/2 * np.square(self.w1).sum() + np.square(self.w2).sum()

        return 1./num_examples * data_loss

    # Helper function to predict an output (0 or 1)
    def predict(self, x):

        # Forward propagation
        z1 = x.dot(self.w1) + self.b1
        a1 = np.tanh(z1)
        z2 = a1.dot(self.w2) + self.b2
        exp_scores = np.exp(z2)
        probs = softmax(exp_scores)

        return np.argmax(probs, axis=1)

    def fit(self, X, y, num_passes=20000, print_loss=False, seed=42):

        num_examples, nn_input_dim = X.shape
        nn_output_dim = len(set(y))

        # Initialize the parameters to random values. We need to learn these.
        np.random.seed(seed)
        self.w1 = np.random.randn(nn_input_dim, self.n_hidden_dim) / np.sqrt(nn_input_dim)
        self.b1 = np.zeros((1, self.n_hidden_dim))
        self.w2 = np.random.randn(self.n_hidden_dim, nn_output_dim) / np.sqrt(self.n_hidden_dim)
        self.b2 = np.zeros((1, nn_output_dim))

        # Gradient descent. For each batch...
        for i in range(num_passes):

            # Forward propagation
            z1 = X.dot(self.w1) + self.b1
            a1 = np.tanh(z1)
            z2 = a1.dot(self.w2) + self.b2
            exp_scores = np.exp(z2)

            # Backpropagation
            delta3 = softmax(exp_scores)
            delta3[range(num_examples), y] -= 1
            dw2 = (a1.T).dot(delta3)
            db2 = np.sum(delta3, axis=0, keepdims=True)
            delta2 = delta3.dot(self.w2.T) * (1 - np.power(a1, 2))
            dw1 = np.dot(X.T, delta2)
            db1 = np.sum(delta2, axis=0)

            # Add regularization terms (b1 and b2 don't have regularization terms)
            dw2 += self.alpha * self.w2
            dw1 += self.alpha * self.w1

            # Gradient descent parameter update
            self.w1 += -self.eta * dw1
            self.b1 += -self.eta * db1
            self.w2 += -self.eta * dw2
            self.b2 += -self.eta * db2

            # Optionally print the loss.
            # This is expensive because it uses the whole dataset, so we don't want to do it too often.
            if print_loss and i % 1000 == 0:
              print("Loss after iteration %i: %f" %(i, self.calculate_loss(X, y)))

```

Let's initialize a MLP classifier, specifying the conjugate gradient minimization method.

```python
mlp = MLP()
```

Now we can confirm that it solves a non-linear classification, using the simple XOR example

```python
X = XOR[['x1','x2']].values
y = XOR['y'].values
```

```python
mlp.fit(X, y, num_passes=100)
```

```python
mlp.predict(X)
```

    array([0, 1, 1, 0])

For a somewhat more sophisiticated example, we can use scikit-learn to simulate some data with a non-linear boundary.

```python
# Generate a dataset and plot it
np.random.seed(0)
X, y = datasets.make_moons(200, noise=0.20)
X = X.astype(np.float32)
y = y.astype(np.int32)
plt.scatter(X[:,0], X[:,1], s=40, c=y, cmap=plt.cm.Set1)
```

```python
clf = MLP(n_hidden_dim=3)
clf.fit(X, y)
```

```python
def plot_decision_boundary(pred_func, X=X, y=y):
    # Set min and max values and give it some padding
    x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
    y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
    h = 0.01
    # Generate a grid of points with distance h between them
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
    # Predict the function value for the whole gid
    Z = pred_func(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)
    # Plot the contour and training examples
    plt.contourf(xx, yy, Z, cmap=plt.cm.summer_r)
    plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Greens)
```

```python
plot_decision_boundary(lambda x: clf.predict(x))
plt.title("Decision Boundary for hidden layer size 3")
```

Since we used the scikit-learn interface, its easy to take advantage of the `metrics` module to evaluate the MLP's performance.

```python
X, y = datasets.make_moons(50, noise=0.20)
X = X.astype(np.float32)
y = y.astype(np.int32)
```

```python
from sklearn.metrics import accuracy_score, confusion_matrix

accuracy_score(y, clf.predict(X))
```

    0.94

```python
confusion_matrix(y, clf.predict(X))
```

    array([[23,  2],
           [ 1, 24]])

# Varying the hidden layer size

In the example above we picked a hidden layer size of 3. Let's now get a sense of how varying the hidden layer size affects the result.

```python
plt.figure(figsize=(16, 32))
n_hidden_dim = [1, 2, 3, 4, 5, 20, 50]
for i, h in enumerate(n_hidden_dim):
    plt.subplot(5, 2, i+1)
    plt.title('Hidden Layer size %d' % h)
    model = MLP(n_hidden_dim=h)
    model.fit(X, y)
    plot_decision_boundary(lambda x: model.predict(x))
```

### Neural network specification

The MLP implemented above uses a single hidden layer, though it allows a user-specified number of hidden layer nodes (defaults to 25). It is worth considering whether it is useful having **multiple hidden layers**, and whether more hidden nodes is desirable.

Unfortunately, there is no theory to guide the choice of hidden node number. As a result, we are left to experiment with this parameter, perhaps in some systematic fashion such as cross-validation.

Adding additional layers presents only additional "bookkeeping" overhead to the user, with the weight updating becoming more complicated as layers are added. So, we don't want to add more hidden layers if it does not pay off in performance. It turns out that two or three layers (including the output layer) can be shown to approximate almost any smooth function. Combining 3 sigmoid functions allows local responses to be approximated with arbitrary accuracy. This is sufficient for determining any decision boundary.

### Neural network validation

Just as with other supervised learning algorithms, neural networks can be under- or over-fit to a training dataset. The degree to which a network is trained to a particular dataset depends on how long we train it on that dataset. Every time we run the MLP learning algorithm over a dataset (an **epoch**), it reduces the prediction error for that dataset. Thus, the number of epochs should be tuned as a hyperparameter, stopping when the testing-training error gap begins to widen.

Note that though we can also use cross-validation to tune the number of hidden layers in the network, there is no risk of overfitting by having too many layers.

### Exercise: Epoch tuning for MLPs

The dataset `pima-indians-diabetes.data.txt` in your data folder contains eight measurements taken from a group of Pima Native Americans in Arizona, along with an indicator of the onset of diabetes. Use the MLP class to fit a neural network classifier to this dataset, and use cross-validation to examine the optimal number of epochs to use in training.

1. Number of times pregnant
2. Plasma glucose concentration a 2 hours in an oral glucose tolerance test
3. Diastolic blood pressure (mm Hg)
4. Triceps skin fold thickness (mm)
5. 2-Hour serum insulin (mu U/ml)
6. Body mass index (weight in kg/(height in m)^2)
7. Diabetes pedigree function
8. Age (years)
9. Class variable (0 or 1)

```python
pima = pd.read_csv('../data/pima-indians-diabetes.data.txt', header=None)
pima.head()
```

      0
      1
      2
      3
      4
      5
      6
      7
      8

      0
      6
      148
      72
      35
      0
      33.6
      0.627
      50
      1

      1
      1
      85
      66
      29
      0
      26.6
      0.351
      31
      0

      2
      8
      183
      64
      0
      0
      23.3
      0.672
      32
      1

      3
      1
      89
      66
      23
      94
      28.1
      0.167
      21
      0

      4
      0
      137
      40
      35
      168
      43.1
      2.288
      33
      1

```python
# Write your answer here
```

## Example: Mulitilayer perceptron in TensorFlow

TensorFlow is designed to evaluate expressions efficiently, and one of its key features is that it automatically differentiates expressions. This saves us from having to code a gradient function by hand.

For this, we will use the MNIST dataset, which we will introduce more formally later.

```python
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
```

    WARNING:tensorflow:From <ipython-input-60-c3d55fec490c>:2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.
    Instructions for updating:
    Please use alternatives such as official/mnist/dataset.py from tensorflow/models.
    WARNING:tensorflow:From /Users/victor2/anaconda3/envs/cqsml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.
    Instructions for updating:
    Please write your own downloading logic.
    WARNING:tensorflow:From /Users/victor2/anaconda3/envs/cqsml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:252: _internal_retry.<locals>.wrap.<locals>.wrapped_fn (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.
    Instructions for updating:
    Please use urllib or similar directly.
    Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.
    WARNING:tensorflow:From /Users/victor2/anaconda3/envs/cqsml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.
    Instructions for updating:
    Please use tf.data to implement this functionality.
    Extracting /tmp/data/train-images-idx3-ubyte.gz
    Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.
    WARNING:tensorflow:From /Users/victor2/anaconda3/envs/cqsml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.
    Instructions for updating:
    Please use tf.data to implement this functionality.
    Extracting /tmp/data/train-labels-idx1-ubyte.gz
    WARNING:tensorflow:From /Users/victor2/anaconda3/envs/cqsml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.
    Instructions for updating:
    Please use tf.one_hot on tensors.
    Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.
    Extracting /tmp/data/t10k-images-idx3-ubyte.gz
    Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.
    Extracting /tmp/data/t10k-labels-idx1-ubyte.gz
    WARNING:tensorflow:From /Users/victor2/anaconda3/envs/cqsml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.
    Instructions for updating:
    Please use alternatives such as official/mnist/dataset.py from tensorflow/models.

```python
# Parameters
learning_rate = 0.001
training_epochs = 15
batch_size = 100
display_step = 1

# Network Parameters
nn_hdim = 256 # hidden layer number of neurons
nn_input_dim = 784 # MNIST data input (img shape: 28*28)
nn_output_dim = 10 # MNIST total classes (0-9 digits)
```

## Defining the Computation Graph in TensorFlow

The first thing we need to is define our computations using TensorFlow. We start by defining our input data matrix `X` and our training labels `y`:

```python
# Our data vectors
X = tf.placeholder('float', [None, nn_input_dim], name='X')
Y = tf.placeholder('float', [None, nn_output_dim], name='y')
```

Remember, we have not assigned any values to `X` or `y`. All we have done is defined mathematical expressions for them. We can use these expressions in subsequent calculations. If we want to evaluate an expression we can call its `eval` method. For example, to evaluate the expression `X * 2` for a given value of `X` we could do the following:

```python
tf.InteractiveSession()
```

    <tensorflow.python.client.session.InteractiveSession at 0x1270c7f60>

This creates a TensorFlow `Session` that can be used interactively. More on this later.

```python
(Y * 2).eval(feed_dict={Y : np.random.randn(10)[None, :]})
```

    array([[ 2.201744  , -0.04348226, -1.0957438 ,  2.008915  ,  0.77484846,
            -1.9866828 ,  1.8271554 ,  0.10985135, -0.8148851 ,  4.9540443 ]],
          dtype=float32)

Recall also that in addition to tensor objects, Theano also has shared variables which have values associated with them. Their value that is kept in memory and can be shared by all functions that use them. Shared variables can also be updated, and Theano includes low-level optimizations that makes updating them very efficient, especially on GPUs. Our network parameters $W_1, b_1, W_2, b_2$ are constantly updated using gradient descent, so they can be represented by shared variables:

```python
# Store layers weight & bias
weights = {
    'W1': tf.Variable(tf.random_normal([nn_input_dim, nn_hdim])),
    'W2': tf.Variable(tf.random_normal([nn_hdim, nn_hdim])),
    'out': tf.Variable(tf.random_normal([nn_hdim, nn_output_dim]))
}
biases = {
    'b1': tf.Variable(tf.random_normal([nn_hdim])),
    'b2': tf.Variable(tf.random_normal([nn_hdim])),
    'out': tf.Variable(tf.random_normal([nn_output_dim]))
}
```

We can now build our forward propagation step for the perceptron, in TensorFlow.

Note that we don't need to explicitly do a forward propagation here. TensorFlow knows that our gradients depend on our predictions from the forward propagation and it will handle all the necessary calculations for us. It does everything it needs to update the values.

```python
def multilayer_perceptron(x):
    # Hidden fully connected layer with 256 neurons
    layer_1 = tf.add(tf.matmul(x, weights['W1']), biases['b1'])
    # Hidden fully connected layer with 256 neurons
    layer_2 = tf.add(tf.matmul(layer_1, weights['W2']), biases['b2'])
    # Output fully connected layer with a neuron for each class
    out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
    return out_layer
```

Rather than calling the `eval` method to evaluate our TF expressions, we can instead define operators for expressions we want to evaluate.

For example, to calculate the loss, we need to know the values for $X$ and $y$, and pass them to our loss function.

```python
# Construct model
logits = multilayer_perceptron(X)

# Define loss and optimizer
loss_op = tf.reduce_mean(tf.losses.softmax_cross_entropy(
    logits=logits, onehot_labels=Y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
train_op = optimizer.minimize(loss_op)
# Initializing the variables
init = tf.global_variables_initializer()
```

```python
with tf.Session() as sess:
    sess.run(init)

    # Training cycle
    for epoch in range(training_epochs):
        avg_cost = 0.
        total_batch = int(mnist.train.num_examples/batch_size)
        # Loop over all batches
        for i in range(total_batch):
            batch_x, batch_y = mnist.train.next_batch(batch_size)
            # Run optimization op (backprop) and cost op (to get loss value)
            _, c = sess.run([train_op, loss_op], feed_dict={X: batch_x,
                                                            Y: batch_y})
            # Compute average loss
            avg_cost += c / total_batch
        # Display logs per epoch step
        if epoch % display_step == 0:
            print("Epoch:", '%04d' % (epoch+1), "cost={:.9f}".format(avg_cost))
    print("Optimization Finished!")

    # Test model
    pred = tf.nn.softmax(logits)  # Apply softmax to logits
    correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(Y, 1))
    # Calculate accuracy
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
    print("Accuracy:", accuracy.eval({X: mnist.test.images, Y: mnist.test.labels}))
```

    Epoch: 0001 cost=314.691833565
    Epoch: 0002 cost=96.875646191
    Epoch: 0003 cost=71.626063349
    Epoch: 0004 cost=58.613202802
    Epoch: 0005 cost=50.011053205
    Epoch: 0006 cost=43.523445813
    Epoch: 0007 cost=38.593564915
    Epoch: 0008 cost=34.231494266
    Epoch: 0009 cost=32.110081593
    Epoch: 0010 cost=28.955673105
    Epoch: 0011 cost=27.387018789
    Epoch: 0012 cost=25.070321978
    Epoch: 0013 cost=24.535828133
    Epoch: 0014 cost=22.481265009
    Epoch: 0015 cost=21.020586926
    Optimization Finished!
    Accuracy: 0.896

---
## References

[The TensorFlow Playground](https://playground.tensorflow.org/)

T. Hastie, R. Tibshirani and J. Friedman. (2009) [Elements of Statistical Learning: Data Mining, Inference, and Prediction](http://statweb.stanford.edu/~tibs/ElemStatLearn/), second edition. Springer.

X. Glorot, A. Bordes and Y. Bengio (2011). [Deep sparse rectifier neural networks (PDF)](http://proceedings.mlr.press/v15/glorot11a/glorot11a.pdf). AISTATS.

S. Marsland. (2009) [Machine Learning: An Algorithmic Perspective](Machine Learning: An Algorithmic Perspectivehttp://seat.massey.ac.nz/personal/s.r.marsland/MLBook.html). CRC Press.

D. Rodriguez. (2013) [Basic [1 hidden layer] neural network on Python](http://danielfrg.com/blog/2013/07/03/basic-neural-network-python/).

D. Britz. (2015) [Implementing a Neural Network from Scratch](http://www.wildml.com/2015/09/implementing-a-neural-network-from-scratch/)


```python
from math import *
from sympy import Symbol, symbols, Matrix, sin, cos
import matplotlib.pyplot as plt
import numpy as np
import scipy.stats as stats
```

```python
#set variables

mu, sigma = symbols('\mu, \sigma')
```

# The Total Probability Rule

the total probability rule (tpr) is

$P(A) = P(A | B) P(B) + P(A | B^C) P(B^C)$

# Probability Density Function

The normal probability density function (pdf) is

$f(x | \mu, \sigma) = \frac{1}{\sigma\sqrt{2\pi}} e^{- \frac{(\mu - x)^2}{2\sigma^2}}$
---

```python
 def gaussian(mu, sigma, x):
          # calculates the probability of x for 1-dim Gaussian with mean mu and var. sigma
          return exp(- ((mu - x) ** 2) / (sigma ** 2) / 2.0) / sqrt(2.0 * pi * (sigma ** 2))
```

# New Mean and Variance

$\mu^{'} = \frac{\sigma^{2}_{2} \dot \mu_{1} + \sigma^{2}_{1} \dot \mu_{2}}{\sigma^{2}_{2} + \sigma^{2}_{1}}$

$\sigma^{'} = \frac{1}{\frac{1}{\sigma^{2}_{2}} + \frac{1}{\sigma^{2}_{1}}}$

```python
def SumGaussian(m1, m2, s1, s2):
  m3 = (s2**2*m1 + s1**2*m2)/(s2**2+ s1**2)
  s3 = 1/((1/s2**2)+ (1/s1**2))
  return m3, s3
```

```python
mu1 = 10
sigma1 = 0.5

x1 = np.linspace(mu1 - 3*sigma1, mu1 + 3*sigma1, 100)
plt.plot(x1, stats.norm.pdf(x1, mu1, sigma1), color="blue", label="P(z|x)")

mu2 = 8
sigma2 = 0.3

x2 = np.linspace(mu2 - 3*sigma2, mu2 + 3*sigma2, 100)
plt.plot(x2, stats.norm.pdf(x2, mu2, sigma2), color="red", label="P(x)")

mu3, sigma3 = SumGaussian(mu1, mu2, sigma1, sigma2)

x3 = np.linspace(mu3 - 3*sigma3, mu3 + 3*sigma3, 100)
plt.plot(x3, stats.norm.pdf(x3, mu3, sigma3), color="green", label="P(x|z)"
)

plt.legend()

plt.show()

```


We will introduce ideas for programming using a real life example. We will use Newton's second law of motion for the motion of a falling body in the air. From the free body diagram, a balance of forces tells us that the mass times the acceleration equals to the sum of forces acting on the body.
\begin{equation}
m a = \sum F
\end{equation}
where $m$ is the mass in kg, $a$ is the acceleration in m/s$^2$, and $F$ is a force acting on the body.

# Mathematical Model

The first step in any engineering formulation of a physics problem is to infer the mathematical description of the physical system.

So, what forces can we consider for a body falling in the air?

* weight ($ m g$)
* drag - due to air resistance

A popular form for drag is $F = c * u$ where $u$ is the speed of the falling body. Plugging everything in, we have
\begin{equation}
m a = mg - c u
\end{equation}

But that doesn't help us a lot: on the left handside we have the acceleration and on the right hand side we have the speed. Fortunately, they are related to each other via:
\begin{equation}
a = \frac{\text{d} u}{\text{d} t}
\end{equation}

So now, our mathematical physics model looks like
\begin{equation}
m \frac{\text{d} u}{\text{d} t} = m g - c u
\end{equation}
or, rearranging terms, we have
\begin{equation}
\frac{\text{d} u}{\text{d} t} = g - \tfrac{c}{m} u
\end{equation}

# Analytical Solution?

In many cases, an anlytical solution is available. In this case, the solution for the speed (or velocity) is
\begin{equation}
u(t) = (u_0 - \frac{m g}{c}) e^{-\frac{c}{m} t} + \frac{mg}{c}
\end{equation}
Here, $u_0$ is the initial speed with which the parachutist leaves the plane, and is usually 0.

Now let's do some predictions!

```python
u0 = 0.0 # m/s
m = 60 # kg
c = 10 # drag coef (can you figure out the units? # hint: drag = force = c * u = Newtons)
g = 9.81 # m/s/s
t = 100000
u = (u0 - m*g/c)*math.exp(-c/m*t) + m*g/c
print(u)
```

It turns out that we need to tell Python where to get the exponential from. There is a module in python called math that you have to import

```python
from math import exp
u0 = 0.0 # m/s
m = 60.0 # kg
c = 13.0 # drag coef (can you figure out the units? # hint: drag = force = c * u = Newtons)
g = 9.81 # m/s/s
t = 10
u = (u0 - m*g/c)*exp(-c/m*t) + m*g/c
print(u, 'm/s')
```

    40.09005110968492 m/s

# Let's Create a Function

But this is really cumbersome: everytime we need to change a parameter, we have to type everything again.

Let's create a function instead that we can call everytime. Functions in Python can be define via def.

```python
def fall_speed():
    u = (u0 - m*g/c)*exp(- c/m * t) + m*g/c
    return u
```

But this doesn't make sense! Can you tell me why?

It doesn't make sense because we haven't supplied any of the parameters to the function. The time t, the mass of the object, the gravitational acceleration, etc...

```python
def fall_speed(m,c,t):
    g = 9.81
    u = (u0 - m*g/c)*exp(- c/m * t) + m*g/c
    return u
```

```python
fall_speed(60,13,10)
```

    40.09005110968492

# Seeing the Forest

We are still lost in the trees at the moment. We can't see the trends yet. Let's use the function that we defined to make a a plot!

To create a plot, we first should think what we are plotting vs what variable. In this case, the most natural thing is to plot the velocity $u$ as a function of time $t$.

For this, we will need a list of points that represent time.

```python
t = [1,2,3,4]
exp(t)
```

It turns out that the math module works for single values - not lists or arrays. To deal with arrays, we need numpy.

```python
import numpy as np
np.exp(t)
```

    array([ 2.71828183,  7.3890561 , 20.08553692, 54.59815003])

```python
def fall_speed(m,c,t):
    g = 9.81
    u0 = 0
    u = (u0 - m*g/c)*np.exp(- c/m * t) + m*g/c
    return u
```

```python
import matplotlib.pyplot as plt
```

```python
plt.plot(t,fall_speed(60,12,t))
```

It turns out we can't multiply a list by a number c/m * t. We need something better than just a list. We need a numpy array.

```python
t = np.array([1,2,3,4,5,6,7])
plt.plot(t,fall_speed(60,12,t))
```

```python
t = np.linspace(0,100)
plt.plot(t,fall_speed(60,12,t))
```

```python
plt.plot(t,fall_speed(60,12,t))
plt.plot(t,fall_speed(70,12,t))
plt.plot(t,fall_speed(80,12,t))
```

```python
plt.plot(t,fall_speed(60,12,t))
plt.plot(t,fall_speed(60,24,t))
plt.plot(t,fall_speed(60,36,t))
```

# Solving this Numerically

We were very lucky this time to have an analytical solution. in the vast majority of cases, we do not have one. Say the drag model was more complex for example - then you would certainly need a numerical solution.

To solve this numerically, without even starting numerical methods, we are going to approximate the derivative term

\begin{equation}
\frac{\text{d} u}{\text{d} t} \approx \frac{\Delta u}{\Delta t} = \frac{u(t + \Delta t) - u(t)}{\Delta t}
\end{equation}

This gives us the formula
\begin{equation}
\frac{u(t + \Delta t) - u(t)}{\Delta t} = g - \tfrac{c}{m} u(t)
\end{equation}
or
\begin{equation}
u^{n+1} = u^n + \Delta t (g - \tfrac{c}{m} u^n)
\end{equation}

Now we have a sequence of number, starting with $u^0 = u(t=0)$, we have:

$u^0$ given

$u^1 = u^0 + \Delta t (g-\tfrac{c}{m}u^0)$

$u^2 = u^1 + \Delta t (g-\tfrac{c}{m}u^1)$

etc...

If you think we will write down every single one of those, then you're not thinking as a computer :)

The greatest thing about a sequence like this is that it can be repeated, as many times as you like! So let's create a loop for this.

```python
m = 900.0 # kg
c = 13.0 # drag coef, kg/s
g = 9.81 # m/s/s, gravitational acceleration
u0 = 0.0 # initial condition, m/s

dt = 0.1 # timestep size, s
t = 0.0 # initial time
tend = 100 # end time - run the sequence until tend
uold = u0
while t < tend:
    unew = uold + dt * (g - c/m * uold)
    t = t + dt
    uold = unew
uexact = fall_speed(m,c,tend)
print('Numerical value:', unew, ' \n Exact value:', uexact)
print('absolute error =', abs(unew - uexact)*100,'%' )
print('relative error =', (unew - uexact)/uexact*100,'%' )
```

    Numerical value: 519.355364651743
     Exact value: 518.957018024558
    absolute error = 39.8346627184992 %
    relative error = 0.07675907895056958 %

Great! but wouldn't it be nice to plot the numerical solution?

Indeed. We can do this several ways, but for this example, we will simply save each value of the sequence into a list.

```python
m = 60.0 # kg
c = 13.0 # drag coef, kg/s
g = 9.81 # m/s/s, gravitational acceleration
u0 = 0.0 # initial condition, m/s

dt = 5 # timestep size, s
t = 0.0 # initial time
tend = 30 # end time - run the sequence until tend
uold = u0

unumerical = []
tvals = []
while t < tend:
    tvals.append(t)
    unumerical.append(uold)
    unew = uold + dt * (g - c/m * uold)
    t = t + dt
    uold = unew
plt.plot(tvals, unumerical,'*-')
plt.plot(tvals, fall_speed(m,c,np.array(tvals)))
```

```python

```


## Volume primerjava

Tukaj primerjam volumne.

Best case:

$$
\begin{equation}
\label{volume3}
    \frac{V_{hyperball}}{V_{hypercube}} = \frac{ \frac{2}{d}\frac{\pi^{d/2}}{\Gamma \left( d/2 \right)} r^d }{\left(2r\right)^d } = \frac{2}{d}\frac{\pi^{d/2}}{\Gamma \left( d/2 \right) \cdot 2^d }
\end{equation}
$$

Worst case:

$$
\begin{equation}
\label{volume4}
    \frac{V_{hyperball}}{V_{hypercube}} = \frac{ \frac{2}{d}\frac{\pi^{d/2}}{\Gamma \left( d/2 \right)} \left( r \sqrt{d} \right) ^d }{\left(2 r \right)^d } = \frac{2}{d}\frac{\pi^{d/2}}{\Gamma \left( d/2 \right) \cdot 2^d } \cdot d^{\frac{d}{2}}
\end{equation}
$$

```python
# %matplotlib qt
%matplotlib inline

import numpy as np
from scipy.special import gamma
import matplotlib.pyplot as plt
from math import pi
```

```python
d_vals = np.arange(2, 18 +1)
cube = np.ones( d_vals.shape )

frac_best = np.array([])
frac_worst = np.array([])

for d in d_vals:
    el = 2/d * (pi**(d/2))/(gamma(d/2)* 2**d)
    frac_best = np.append( frac_best,   el )
    frac_worst = np.append( frac_worst,  el* d**(d/2) )
```

```python
plt.figure(figsize=(8,4))

plt.plot( d_vals, cube, label="enotska hiperkocka" )
plt.plot( d_vals, frac_best, label="najboljši hiperelipsoid" )
plt.plot( d_vals, frac_worst, label="najslabši hiperelipsoid" )

plt.yscale("log")

plt.legend(loc=3, prop={'size': 12})
plt.xlabel("Število dimenzij "+r'$d$', fontsize=14)
plt.ylabel("Volumen", fontsize=14 )

plt.show()
```

```python
plt.figure(figsize=(8,4))

plt.plot( d_vals, cube, label="enotska hiperkocka" )
plt.plot( d_vals, frac_best, label="najboljši hiperelipsoid" )
plt.plot( d_vals, frac_worst, label="najslabši hiperelipsoid" )

plt.yscale("log")

plt.legend(loc=3, prop={'size': 12})
plt.xlabel("Število dimenzij "+r'$d$', fontsize=14)
plt.ylabel("Volumen", fontsize=14 )

plt.scatter([12,12,12,12], [6.894,1.353, 4.204, 4.986], c="red", s=30, label="raztezani hiperelipsoid")
plt.scatter([12,12,12,12], [0.1007,0.07501, 0.08465, 0.04670], c="purple", s=30, label="elipsoidna metoda")

plt.legend(loc=3, prop={'size': 12})

plt.show()
```

```python
print(frac_best[10])
print(frac_worst[10])
print( frac_worst[10] / frac_best[10])
```

    0.00032599188692738996
    973.4065584949956
    2985984.0

```python

```


```python
import numpy as np
from matplotlib import pyplot
```

```python
from sympy import sin, cos, Matrix, sqrt, simplify, solve
from sympy.abc import rho, phi, x, y, z, X, Y, Z, R
```

```python
def normalized(x,y,z):
    v = np.array((x,y,z))
    return v / np.linalg.norm(v)
```

```python
# default Stereographics projection
R = sqrt(X*X + Y*Y)
x_ = 2*X/(1 + R*R)
y_ = 2*Y/(1 + R*R)
z_ = (-1+R*R)/(1 + R*R)
```

```python
Finv = Matrix([x_, y_, z_])
Finv
```

```python
# compute jacobian determinant for integral transformation / transformation of probability density
# https://en.wikipedia.org/wiki/Stereographic_projection
J = Finv.jacobian(Matrix([X,Y]))
J
```

```python
fu = sqrt(simplify(simplify((J.T*J).det())))
fu
```

```python
def distortion(r):
    return simplify(fu.subs({X : 0, Y : r})).evalf()
```

```python
distortion(1.)
```

```python

```


# Part 1 - Scalars and Vectors

For the questions below it is not sufficient to simply provide answer to the questions, but you must solve the problems and show your work using python (the NumPy library will help a lot!) Translate the vectors and matrices into their appropriate python  representations and use numpy or functions that you write yourself to demonstrate the result or property.

## 1.1 Create a two-dimensional vector and plot it on a graph

```
import math
import matplotlib
import matplotlib.pyplot as plt
matplotlib.style.use('ggplot')
import numpy as np

# Vector Examples

blue = [.2, 5]
red = [.1, .3]

#plt.arrow(0, 0, yellow[0], yellow[1], head_width=.02, head_length=0.01, color = 'y')
plt.arrow(0, 0, .2, .5, head_width=.02, head_length=0.01, color = 'b')
plt.arrow(0, 0, .1, .3,  head_width=.02, head_length=0.01, color = 'r')
plt.show()
```

## 1.2 Create a three-dimensional vecor and plot it on a graph

```
from mpl_toolkits.mplot3d import Axes3D
import numpy as np
import matplotlib.pyplot as plt

yellow = [.4, .5, .5]
red = [.2, .4, .0]
blue = [.9, .3, .3]

vectors = np.array([[0, 0, 0, .5, .5, .5],
                    [0, 0, 0, .2, .1, .0],
                    [0, 0, 0, .1, .3, .3]])

X, Y, Z, U, V, W = zip(*vectors)
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.quiver(X, Y, Z, U, V, W, length=1)
ax.set_xlim([0, 1])
ax.set_ylim([0, 1])
ax.set_zlim([0, 1])
ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')
plt.show()
```

## 1.3 Scale the vectors you created in 1.1 by $5$, $\pi$, and $-e$ and plot all four vectors (original + 3 scaled vectors) on a graph. What do you notice about these vectors?

```
from math import e, pi
print(e)
print(pi)
```

    2.718281828459045
    3.141592653589793

```
import math
import matplotlib
import matplotlib.pyplot as plt
matplotlib.style.use('ggplot')
import numpy as np

blue   = [.1, .3]#
blue_pi = [i*pi for i in blue]
blue_e  = [i*e for i in blue]
blue_5  = [i*5 for i in blue]

red   = [.2, .5]#
red_pi = [i*pi for i in red]
red_e  = [i*e for i in red]
red_5  = [i*5 for i in red]

plt.arrow(0, 0, blue[0], blue[1], head_width=.02, head_length=0.01, color = 'b')
plt.arrow(0, 0, blue_pi[0], blue_pi[1],  head_width=.02, head_length=0.01, color = 'b')
plt.arrow(0, 0, blue_e[0], blue_e[1],  head_width=.02, head_length=0.01, color = 'b')
plt.arrow(0, 0, blue_5[0], blue_5[1],  head_width=.02, head_length=0.01, color = 'b')

plt.arrow(0, 0, red[0], red[1], head_width=.02, head_length=0.01, color = 'r')
plt.arrow(0, 0, red_pi[0], red_pi[1],  head_width=.02, head_length=0.01, color = 'r')
plt.arrow(0, 0, red_e[0], red_e[1],  head_width=.02, head_length=0.01, color = 'r')
plt.arrow(0, 0, red_5[0], red_5[1],  head_width=.02, head_length=0.01, color = 'r')
plt.title('Scale vectors in 1.1 by 5,π,and −e')
plt.show()
```

```

```

## 1.4 Graph vectors $\vec{a}$ and $\vec{b}$ and plot them on a graph

\begin{align}
\vec{a} = \begin{bmatrix} 5 \\ 7 \end{bmatrix}
\qquad
\vec{b} = \begin{bmatrix} 3 \\4 \end{bmatrix}
\end{align}

```
import math
import matplotlib
import matplotlib.pyplot as plt
matplotlib.style.use('ggplot')
import numpy as np

red = [5, 7]
blue = [3, 4]

plt.arrow(0, 0, 5, 7, head_width=.02, head_length=0.01, color = 'r')
plt.arrow(0, 0, 3, 4,  head_width=.02, head_length=0.01, color = 'b')
plt.title('Vector Examples')
plt.show()
```

## 1.5 find $\vec{a} - \vec{b}$ and plot the result on the same graph as $\vec{a}$ and $\vec{b}$. Is there a relationship between vectors $\vec{a} \thinspace, \vec{b} \thinspace \text{and} \thinspace \vec{a-b}$

```
import math
import matplotlib
import matplotlib.pyplot as plt
matplotlib.style.use('ggplot')
import numpy as np

red = [5, 7]
blue = [3, 4]
sub  =[2, 3]

plt.arrow(0, 0, 5, 7, head_width=.02, head_length=0.01, color = 'r')
plt.arrow(0, 0, 3, 4,  head_width=.02, head_length=0.01, color = 'b')
plt.arrow(0, 0, 2, 3, head_width=.02, head_length=0.01, color = 'g')
plt.title('Vector Examples')
plt.show()
```

## 1.6 Find $c \cdot d$ $Dot Product$

\begin{align}
\vec{c} = \begin{bmatrix}7 & 22 & 4 & 16\end{bmatrix}
\qquad
\vec{d} = \begin{bmatrix}12 & 6 & 2 & 9\end{bmatrix}
\end{align}

```
# a.b = (7)(12) + (22)(6) + (4)(2) + (16)(9)
84 + 132 + 8 + 144
```

    368

##  1.7 Find $e \times f$

\begin{align}
\vec{e} = \begin{bmatrix} 5 \\ 7 \\ 2 \end{bmatrix}
\qquad
\vec{f} = \begin{bmatrix} 3 \\4 \\ 6 \end{bmatrix}
\end{align}

```
# cross product
#[5,7,2],[3,4,6]
[7*6-2*4]-[5*6-2*3]+[5*4-7*3] # crossproduct
[34, -24,  -1]
```

#Norm of a Vector (Magnitude or length)
## 1.8 Find $||g||$ and then find $||h||$. Which is longer?

\begin{align}
\vec{g} = \begin{bmatrix} 1 \\ 1 \\ 1 \\ 8 \end{bmatrix}
\qquad
\vec{h} = \begin{bmatrix} 3 \\3 \\ 3 \\ 3 \end{bmatrix}
\end{align}

```
g=math.sqrt((1)^2 + (1)^2 + (1)^2 + (8)^2)
h=math.sqrt((3)^2 + (3)^2 + (3)^2 + (3)^2)
print(g,h)
```

    3.0 2.0

###g is longer

# Part 2 - Matrices

```
mn * np -> mp
CA, CD
```

## 2.1 What are the dimensions of the following matrices? Which of the following can be multiplied together? See if you can find all of the different legal combinations.
\begin{align}
A = \begin{bmatrix}
1 & 2 \\
3 & 4 \\
5 & 6
\end{bmatrix}
\qquad
B = \begin{bmatrix}
2 & 4 & 6 \\
\end{bmatrix}
\qquad
C = \begin{bmatrix}
9 & 6 & 3 \\
4 & 7 & 11
\end{bmatrix}
\qquad
D = \begin{bmatrix}
1 & 0 & 0 \\
0 & 1 & 0 \\
0 & 0 & 1
\end{bmatrix}
\qquad
E = \begin{bmatrix}
1 & 3 \\
5 & 7
\end{bmatrix}
\end{align}

## 2.2 Find the following products: CD, AE, and BA. What are the dimensions of the resulting matrices? How does that relate to the dimensions of their factor matrices?

```
# CD     [[9 6 3  4 7 11]] * [[1 0 0  0 1 0  0 0 1 ]]

# [(9*1)+(6*0)+(3*0)   (9*0)+(6*1)+(3*0)   (9*0)+(6*0)+(3*1) ]
# [(4*1)+(7*0)+(11*0)  (4*0)+(7*1)+(11*0)  (4*0)+(7*0)+(11*1)]
#   (9) + (0) + (0)     (0)  +(6) + (0)     (0) + (0) +  (3)
#   (4) + (0) + (0)     (0)  +(7) + (0)     (0) + (0) + (11)

#   [13 + 13 + 14]
```

```
# AE   [1 5   3 7] [ 1 2  3 4  5 6]
#      1*1 3*2 5*1 7*2
#      1*3 3*4 5*4 7*4
#      2*5 3*6 5*6 7*6
#      1 +  6 +  6 +  9
#      4 + 12 + 20 + 28
#     10 + 18 + 30 + 42

#    [[15  36
#      56  79]]
```

    186

## 2.3  Find $F^{T}$. How are the numbers along the main diagonal (top left to bottom right) of the original matrix and its transpose related? What are the dimensions of $F$? What are the dimensions of $F^{T}$?

\begin{align}
F =
\begin{bmatrix}
20 & 19 & 18 & 17 \\
16 & 15 & 14 & 13 \\
12 & 11 & 10 & 9 \\
8 & 7 & 6 & 5 \\
4 & 3 & 2 & 1
\end{bmatrix}
\end{align}

```
20 16 12 8 4
19 15 11 7 3
18 14 10 6 2
17 13  9 5 1
```

# Part 3 - Square Matrices

## 3.1 Find $IG$ (be sure to show your work) 😃

\begin{align}
G=
\begin{bmatrix}
12 & 11 \\
7 & 10
\end{bmatrix}
\end{align}

```

```

## 3.2 Find $|H|$ and then find $|J|$.

\begin{align}
H=
\begin{bmatrix}
12 & 11 \\
7 & 10
\end{bmatrix}
\qquad
J=
\begin{bmatrix}
0 & 1 & 2 \\
7 & 10 & 4 \\
3 & 2 & 0
\end{bmatrix}
\end{align}

```

```

## 3.3 Find $H^{-1}$ and then find $J^{-1}$

```

```

## 3.4 Find $HH^{-1}$ and then find $J^{-1}J$. Is $HH^{-1} == J^{-1}J$? Why or Why not?

```
inverse
```

# Stretch Goals:

A reminder that these challenges are optional. If you finish your work quickly we welcome you to work on them. If there are other activities that you feel like will help your understanding of the above topics more, feel free to work on that. Topics from the Stretch Goals sections will never end up on Sprint Challenges. You don't have to do these in order, you don't have to do all of them.

- Write a function that can calculate the dot product of any two vectors of equal length that are passed to it.
- Write a function that can calculate the norm of any vector
- Prove to yourself again that the vectors in 1.9 are orthogonal by graphing them.
- Research how to plot a 3d graph with animations so that you can make the graph rotate (this will be easier in a local notebook than in google colab)
- Create and plot a matrix on a 2d graph.
- Create and plot a matrix on a 3d graph.
- Plot two vectors that are not collinear on a 2d graph. Calculate the determinant of the 2x2 matrix that these vectors form. How does this determinant relate to the graphical interpretation of the vectors?


Traçar um esboço do gráfico e obter uma equação da parábola que satisfaça as condições dadas.

22. Foco: $F(4,-5)$; diretriz $d: y = 1$

Fazendo um esboço é possível perceber que a parábola é paralela ao eixo $y$, logo sua equação é dada por $(x-h)^2 = 2p(y-k)$

Sabendo que a distância da diretriz até o foco é $p$, podemos calcular sua distância para achar $\frac{p}{2}$ usando o ponto $P(4,1)$ da diretriz

$p = \sqrt{(4-4)^2 + (-5-1)^2}$

$p = \sqrt{0 + (-6)^2}$

$p = \pm \sqrt{36}$

$p = -6$

$\frac{p}{2} = -3$

Somando $\frac{p}{2}$ no eixo $y$ da diretriz, obtemos as coordenadas do vértice

Vértice: $V(4, -2)$

Substituindo agora os pontos dos vértice e o valor de $p$ na fórmula, temos que

$(x-4)^2 = 2 \cdot -6 \cdot (y+2)$

$(x-4)^2 = -12(y+2)$

$x^2 - 8x + 16 = -12y - 24$

$x^2 - 8x + 12y + 16 + 24$

$x^2 - 8x + 12y + 40$

Gráfico da parábola

```python
from sympy import *
from sympy.plotting import plot_implicit
x, y = symbols("x y")
plot_implicit(Eq((x-4)**2, -12*(y+2)), (x,-20,20), (y,-20,20),
title=u'Gráfico da parábola', xlabel='x', ylabel='y');
```

Problemas Propostos. Pag. 172 - 175/22.ipynb


# Geometric Series for Elementary Economics

```python
import matplotlib.pyplot as plt
%matplotlib inline
import numpy as np
import sympy as sym
from sympy import init_printing
from matplotlib import cm
from mpl_toolkits.mplot3d import Axes3D
```

```python
# True present value of a finite lease
def finite_lease_pv_true(T, g, r, x_0):
    G = (1 + g)
    R = (1 + r)
    return (x_0 * (1 - G**(T + 1) * R**(-T - 1))) / (1 - G * R**(-1))
# First approximation for our finite lease

def finite_lease_pv_approx_1(T, g, r, x_0):
    p = x_0 * (T + 1) + x_0 * r * g * (T + 1) / (r - g)
    return p

# Second approximation for our finite lease
def finite_lease_pv_approx_2(T, g, r, x_0):
    return (x_0 * (T + 1))

# Infinite lease
def infinite_lease(g, r, x_0):
    G = (1 + g)
    R = (1 + r)
    return x_0 / (1 - G * R**(-1))
```

```python
def plot_function(axes, x_vals, func, args):
    axes.plot(x_vals, func(*args), label=func.__name__)

T_max = 50

T = np.arange(0, T_max+1)
g = 0.02
r = 0.03
x_0 = 1

our_args = (T, g, r, x_0)
funcs = [finite_lease_pv_true,
        finite_lease_pv_approx_1,
        finite_lease_pv_approx_2]
        ## the three functions we want to compare

fig, ax = plt.subplots()
ax.set_title('Finite Lease Present Value $T$ Periods Ahead')
for f in funcs:
    plot_function(ax, T, f, our_args)
ax.legend()
ax.set_xlabel('$T$ Periods Ahead')
ax.set_ylabel('Present Value, $p_0$')
plt.show()
```

```python
# Convergence of infinite and finite
T_max = 1000
T = np.arange(0, T_max+1)
fig, ax = plt.subplots()
ax.set_title('Infinite and Finite Lease Present Value $T$ Periods Ahead')
f_1 = finite_lease_pv_true(T, g, r, x_0)
f_2 = np.ones(T_max+1)*infinite_lease(g, r, x_0)
ax.plot(T, f_1, label='T-period lease PV')
ax.plot(T, f_2, '--', label='Infinite lease PV')
ax.set_xlabel('$T$ Periods Ahead')
ax.set_ylabel('Present Value, $p_0$')
ax.legend()
plt.show()
```

```python
# First view
# Changing r and g
fig, ax = plt.subplots()
ax.set_title('Value of lease of length $T$')
ax.set_ylabel('Present Value, $p_0$')
ax.set_xlabel('$T$ periods ahead')
T_max = 10
T=np.arange(0, T_max+1)

rs, gs = (0.9, 0.5, 0.4001, 0.4), (0.4, 0.4, 0.4, 0.5),
comparisons = ('$\gg$', '$>$', r'$\approx$', '$<$')
for r, g, comp in zip(rs, gs, comparisons):
    ax.plot(finite_lease_pv_true(T, g, r, x_0), label=f'r(={r}) {comp} g(={g})')

ax.legend()
plt.show()
```

```python
# Second view
fig = plt.figure()
T = 3
ax = fig.gca(projection='3d')
r = np.arange(0.01, 0.99, 0.005)
g = np.arange(0.011, 0.991, 0.005)

rr, gg = np.meshgrid(r, g)
z = finite_lease_pv_true(T, gg, rr, x_0)

# Removes points where undefined
same = (rr == gg)
z[same] = np.nan
surf = ax.plot_surface(rr, gg, z, cmap=cm.coolwarm,
    antialiased=True, clim=(0, 15))
fig.colorbar(surf, shrink=0.5, aspect=5)
ax.set_xlabel('$r$')
ax.set_ylabel('$g$')
ax.set_zlabel('Present Value, $p_0$')
ax.view_init(20, 10)
ax.set_title('Three Period Lease PV with Varying $g$ and $r$')
plt.show()
```

```python
# Creates algebraic symbols that can be used in an algebraic expression
g, r, x0 = sym.symbols('g, r, x0')
G = (1 + g)
R = (1 + r)
p0 = x0 / (1 - G * R**(-1))
init_printing()
print('Our formula is:')
p0
```

```python
print('dp0 / dg is:')
dp_dg = sym.diff(p0, g)
dp_dg
```

```python
print('dp0 / dr is:')
dp_dr = sym.diff(p0, r)
dp_dr
```

```python
# Function that calculates a path of y
def calculate_y(i, b, g, T, y_init):
    y = np.zeros(T+1)
    y[0] = i + b * y_init + g
    for t in range(1, T+1):
        y[t] = b * y[t-1] + i + g
    return y

# Initial values
i_0 = 0.3
g_0 = 0.3
# 2/3 of income goes towards consumption
b = 2/3
y_init = 0
T = 100

fig, ax = plt.subplots()
ax.set_title('Path of Aggregate Output Over Time')
ax.set_xlabel('$t$')
ax.set_ylabel('$y_t$')
ax.plot(np.arange(0, T+1), calculate_y(i_0, b, g_0, T, y_init))
# Output predicted by geometric series
ax.hlines(i_0 / (1 - b) + g_0 / (1 - b), xmin=-1, xmax=101, linestyles='--')
plt.show()
```

```python
bs = (1/3, 2/3, 5/6, 0.9)

fig,ax = plt.subplots()
ax.set_title('Changing Consumption as a Fraction of Income')
ax.set_ylabel('$y_t$')
ax.set_xlabel('$t$')
x = np.arange(0, T+1)
for b in bs:
    y = calculate_y(i_0, b, g_0, T, y_init)
    ax.plot(x, y, label=r'$b=$'+f"{b:.2f}")
ax.legend()
plt.show()
```

```python
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(6, 10))
fig.subplots_adjust(hspace=0.3)

x = np.arange(0, T+1)
values = [0.3, 0.4]

for i in values:
    y = calculate_y(i, b, g_0, T, y_init)
    ax1.plot(x, y, label=f"i={i}")
for g in values:
    y = calculate_y(i_0, b, g, T, y_init)
    ax2.plot(x, y, label=f"g={g}")

axes = ax1, ax2
param_labels = "Investment", "Government Spending"
for ax, param in zip(axes, param_labels):
    ax.set_title(f'An Increase in {param} on Output')
    ax.legend(loc ="lower right")
    ax.set_ylabel('$y_t$')
    ax.set_xlabel('$t$')
plt.show()
```

```python

```

Geometric Series for Elementary Economics.ipynb


$(x_1,y_1),(x_2,y_2),...,(x_n,y_n)$
$$\hat y=\hat A+\hat Bx$$
$$y=A+Bx+\epsilon$$
$$S(x)=\sum_{i=0}^{n-1}x_i$$
$$S(y)=\sum_{i=0}^{n-1}y_i$$
$$SS(x)=\sum_{i=0}^{n-1}x_i^2$$
$$SXY(X,Y)=\sum_{i=0}^{n-1}x_iy_i$$
$$\Delta(x)=n\sum_{i=0}^{n-1}x_i^2-\left(\sum_{i=0}^{n-1}x_i\right)^2$$
Punto de corte:
\begin{equation}
A=\frac{SS(X)*S(Y)-S(X)*SXY(X,Y)}{\Delta(X)}
\end{equation}
Pendiente:
\begin{equation}
B=\frac{n*SXY(X,Y)-S(X)*S(Y)}{\Delta(X)}
\end{equation}
Incertidumbre en Y:
\begin{equation}
\sigma_y=\sqrt{\frac{\sum (y_{i}-A-Bx)^2}{n-2}}
\end{equation}
Incertidumbre en A:
\begin{equation}
\Delta A=\sigma_{y}\frac{\sqrt{\sum x_i^2}}{\Delta(X)}
\end{equation}

```python

```

```python

```