**Import packages:**

```python
import numpy as np
import scipy as sp
from scipy import linalg
from scipy import optimize
from scipy import interpolate
import sympy as sm
import pandas

%matplotlib inline
import matplotlib.pyplot as plt
from matplotlib import cm
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.ticker import PercentFormatter

# To print nicely:
sm.init_printing(use_unicode=True)
```

# 1. Human capital accumulation

**We define the variables used in the analysis:**

w: wage rate

b: unemployment benefit

$c_t$: consumption in period t

$h_t$: human capital in period t

$l_t$: labor supply in period t

$\rho$: risk aversion coefficient

$\gamma$: degree of disutility from working

$\Delta$: stochastic experience gain

$\beta$: discount factor

**Defining the necessary equations and the values of l.**

```python
l = [0, 1]

def consumption(w,h,b,l):
    """
    If the person doesn't work (l = 0), she will recieve an unemployment benefit (b) instead of a wage income.
    If the person works (l = 1), she will receive a wage income based on the wage (w) and her human capital (h).
    """
    if l == 0:
        c = b
    elif l == 1:
        c = w*h
    return c

def utility(rho,h,w,b,l):
    return consumption(w,h,b,l)**(1-rho)/(1-rho)

def disutility(l,gamma):
    return gamma*l

def v2(l2,h2,w,b,rho,gamma):
    return utility(rho,h2,w,b,l2) - disutility(l2,gamma)
```

**Setting parameter values:**

```python
rho = 2
beta = 0.96
gamma = 0.1
w = 2
b = 1
Delta = 0.1
```

**Question 1:** Solve the model in period 2 and illustrate the solution (including labor supply as a function of human capital).

```python
def solve_period_2(w,b,rho,gamma):
    """
    The function solves the utility maximizing problem in period 2 for different values of human capital in period 2.
    This is done, by comparing the utility when working to the utility when not working,
    for a given level of human capital and thereby identifying the utility maximizing behavior.

    It returns the utility, the labor supply and the consumption for the different values of human capital.
    """
    # a. grids
    h2_vec = np.linspace(0.1,1.5,100)
    l2_vec = np.empty(100)
    c2_vec = np.empty(100)
    v2_vec = np.empty(100)

    # b. solve for each h2 in grid
    for i,h2 in enumerate(h2_vec):

        # i. Calculating v when working, l[0] and when not working, l[1].
        v2_0 = v2(l[0],h2,w,b,rho,gamma)
        v2_1 = v2(l[1],h2,w,b,rho,gamma)

        # ii. Finding the maximum value of v and the value of l and c associated with it, and saving these values.
        v2_vec[i] = max(v2_0,v2_1)
        l2_vec[i] = v2_1 > v2_0
        c2_vec[i] = consumption(w,h2,b,l2_vec[i])

    return h2_vec,v2_vec,l2_vec,c2_vec
```

```python
# solving for period 2 and assigning values to different vectors.
h2_vec,v2_vec,l2_vec,c2_vec = solve_period_2(w,b,rho,gamma);
```

```python
# Presents consumption, labor supply and utility as a function of human capital,
# based on the utility maximizing problem in period 2.
fig = plt.figure(figsize=(20,4))
ax = fig.add_subplot(1,3,1)
ax.plot(h2_vec,c2_vec)
ax.grid()
ax.set_xlabel('$h_2$')
ax.set_ylabel('$c_2$')
ax.set_title('1a: consumption function in period 2')

ax = fig.add_subplot(1,3,2)
ax.plot(h2_vec,l2_vec)
ax.grid()
ax.set_xlabel('$h_2$')
ax.set_ylabel('$l_2$')
ax.set_title('1b: labor function in period 2')
ax.set_ylim([-0.1,1.1])

ax = fig.add_subplot(1,3,3)
ax.plot(h2_vec,v2_vec)
ax.grid()
ax.set_xlabel('$h_2$')
ax.set_ylabel('$v_2$')
ax.set_title('1c: utility function in period 2');
```

**Identifying the level of human capital necessary for the individual to work:**

```python
index = np.where(l2_vec == 1)[0][0];
print('She chooses to work when h ≥ ' + str(round(h2_vec[index],3)))
```

    She chooses to work when h ≥ 0.567

**Question 2:** Solve the model in period 1 and illustrate the solution (including labor supply as a function of human capital).

**Construct interpolator:**

```python
# Defining an interpolating function based on data from figure 1c.
v2_interp = interpolate.RegularGridInterpolator([h2_vec], v2_vec, bounds_error=False,fill_value=None)
```

***Defining $v_1$:***

We use the interpolated function to define the expected utility in period 2, accounting for two possible scenarios,
 one scenario with a stochastic experience gain to the human capital in period 2 and one without.

```python
def v1(l1,h1,w,b,rho,gamma,beta,Delta,v2_interp):

    # a. v2 value without a stochastic experience gain
    h2_low = h1 + l1
    v2_low = v2_interp([h2_low])[0]

    # b. v2 value with a stochastic experience gain
    h2_high = h1 + l1 + Delta
    v2_high = v2_interp([h2_high])[0]

    # c. expected value of v2
    v2 = 0.5 * v2_low + 0.5 * v2_high

    # d. total value
    return utility(rho,h1,w,b,l1) - disutility(l1,gamma) + beta*v2
```

```python
def solve_period_1(w,b,rho,gamma,Delta,beta,v2_interp):
    """
    The function solves the intertemporal utility maximizing problem in period 1,
    for different values of human capital in period 1 considering both periods.
    This is done, by comparing the utility when working to the utility when not working,
    for a given level of human capital and thereby identifying the utility maximizing behavior.

    It returns the utility, the labor supply and the consumption for the different values of human capital.
    """
    # a. grids
    h1_vec = np.linspace(0.1,1.5,100)
    l1_vec = np.empty(100)
    c1_vec = np.empty(100)
    v1_vec = np.empty(100)

    # b. solve for each h2 in grid
    for i,h1 in enumerate(h1_vec):

        # i. Calculating v when working, l[0] and when not working, l[1].
        v1_0 = v1(l[0],h1,w,b,rho,gamma,beta,Delta,v2_interp)
        v1_1 = v1(l[1],h1,w,b,rho,gamma,beta,Delta,v2_interp)

        # ii. Finding the maximum value of v and the value of l and c associated with it, and saving these values.
        v1_vec[i] = max(v1_0,v1_1)
        l1_vec[i] = v1_1 > v1_0
        c1_vec[i] = consumption(w,h1,b,l1_vec[i])

    return h1_vec,v1_vec,l1_vec,c1_vec
```

```python
# solving for period 1 and assigning values to different vectors.
h1_vec,v1_vec,l1_vec,c1_vec = solve_period_1(w,b,rho,gamma,Delta,beta,v2_interp)
```

```python
# Presents consumption, labor supply and utility as a function of human capital,
# based on the intertemporal utility maximizing problem in period 1.
fig = plt.figure(figsize=(20,4))
ax = fig.add_subplot(1,3,1)
ax.plot(h1_vec,c1_vec)
ax.grid()
ax.set_xlabel('$h_1$')
ax.set_ylabel('$c_1$')
ax.set_title('2a: consumption function in period 1')

ax = fig.add_subplot(1,3,2)
ax.plot(h1_vec,l1_vec)
ax.grid()
ax.set_xlabel('$h_1$')
ax.set_ylabel('$l_1$')
ax.set_title('2b: labor function in period 1')
ax.set_ylim([-0.1,1.1])

ax = fig.add_subplot(1,3,3)
ax.plot(h1_vec,v1_vec)
ax.grid()
ax.set_xlabel('$h_1$')
ax.set_ylabel('$v_1$')
ax.set_title('2c: utility function in period 1');
```

**Identifying the level of human capital necessary for the individual to work:**

```python
index = np.where(l1_vec == 1)[0][0];
print('She chooses to work when h ≥ ' + str(round(h1_vec[index],3)))
```

    She chooses to work when h ≥ 0.355

**Question 3:** Will the worker never work if her potential wage income is lower than the unemployment benefits she can get? Explain and illustrate why or why not.

As can be seen in figure 2a the worker will choose to work even though her potential wage income is lower than the unemployment benefit, i.e. her consumption is lower when she chooses to work.
The reason for this, is that she improves her human capital in period 2 by working in period 1. This will then improve her consumption more in period 2 than it reduces her consumption in period 1.

# 2. AS-AD model

**Defining symbols:**

```python
pi = sm.symbols("pi_t")
y = sm.symbols("y_t")
pi_e = sm.symbols("pi^e_t")
s = sm.symbols("s_t")
v = sm.symbols("v_t")
pi1 = sm.symbols("pi_t-1")
y1 = sm.symbols("y_t-1")
s1 = sm.symbols("s_t-1")
pistar = sm.symbols("pi^*")
ystar = sm.symbols("y^*")
alpha = sm.symbols("alpha")
gamma = sm.symbols("gamma")
phi = sm.symbols("phi")
h = sm.symbols("h")
b = sm.symbols("b")
```

**Question 1:** Use the ``sympy`` module to solve for the equilibrium values of output, $y_t$, and inflation, $\pi_t$, (where AD = SRAS) given the parameters ($\alpha$, $h$, $b$, $\alpha$, $\gamma$) and $y_{t-1}$ , $\pi_{t-1}$, $v_t$, $s_t$, and $s_{t-1}$.

**Defining our AD-curve:**

```python
AD = sm.Eq(pi,(v - (1 + b*alpha)*y)/(h*alpha))
AD
```

**Defining our SRAS-curve:**

```python
SRAS = sm.Eq(pi,pi1 + gamma*y - phi*gamma*y1 + s - phi*s1)
SRAS
```

**Finding equilibrium values for output gap, $y_t$, and inflation gap, $\pi_t$:**

```python
# Solving
Eq = sm.solve([AD, SRAS], [y, pi])
pi_eq = Eq[pi]
y_eq = Eq[y]

# Creating functions of pi and y for later use
pi_func = sm.lambdify((pi1, y1, v, s, s1, alpha, h, b, phi, gamma), pi_eq)
y_func = sm.lambdify((pi1, y1, v, s, s1, alpha, h, b, phi, gamma), y_eq)
```

**Output gap:**

```python
sm.Eq(y,y_eq)
```

**Inflation gap:**

```python
sm.Eq(pi,pi_eq)
```

**Question 2:** Find and illustrate the equilibrium when $y_{t-1} = \pi_{t-1} = v_t = s_t = s_{t-1} = 0$. Illustrate how the equilibrium changes when instead $v_t = 0.1$.

**Setting the parameters of the model:**

```python
par = {}

par['alpha'] = 5.76
par['h'] = 0.5
par['b'] = 0.5
par['phi'] = 0
par['gamma'] = 0.075
```

**Assigning values to variables:**

```python
y1 = 0
pi1 = 0
s = 0
s1 = 0
v_vec = [0,0.1]
y_vec = np.linspace(-2,2,40)
```

**Calculating $\pi_t$ and $y_t$ for $v_t$ = 0 and $v_t$ = 0.1:**

```python
# creating empty arrays for pi, y and our supply and demand curves
SRAS_vec = []
AD1_vec = []
AD2_vec = []
pi_result = []
y_result = []

# calculating pi, y and our supply and demand curves
for y in y_vec:
    SRAS_vec.append(par['gamma']*y)
    AD1_vec.append((v_vec[0] - (1 + par['b']*par['alpha'])*y)/(par['h']*par['alpha']))
    AD2_vec.append((v_vec[1] - (1 + par['b']*par['alpha'])*y)/(par['h']*par['alpha']))

for val in v_vec:
    pi_result.append(pi_func(0,0,val,0,0,par['alpha'],par['h'],par['b'],par['phi'],par['gamma']))
    y_result.append(y_func(0,0,val,0,0,par['alpha'],par['h'],par['b'],par['phi'],par['gamma']))
```

**Plotting the results and marking equilibrium points:**

```python
fig = plt.figure()
ax = fig.add_subplot(1,1,1)
ax.plot(y_vec, SRAS_vec, label='SRAS')
ax.plot(y_vec, AD1_vec, label='AD, $v_t$ = 0')
ax.plot(y_vec, AD2_vec, label='AD, $v_t$ = 0.1')
ax.plot(y_result[0], pi_result[0], marker='o')
ax.plot(y_result[1], pi_result[1], marker='o')
ax.grid()
ax.legend()
ax.set_xlabel('$y_t$')
ax.set_ylabel('$\pi_t$')
ax.set_xlim([-0.1,0.1])
ax.set_ylim([-0.2,0.2])
ax.set_title('3: Equilibrium values for different demand shocks');
```

```python
# creating a table with the equilibrium values of y and pi for the different values of v
headers = ["v = 0", "v = 0.1"]
rows = ["$\pi_t$", "$y_t$"]
data = np.array([[pi_result[0],pi_result[1]],[y_result[0],y_result[1]]])
pandas.DataFrame(data,rows,headers)
```

      v = 0
      v = 0.1

      $\pi_t$
      0.0
      0.001831

      $y_t$
      0.0
      0.024414

We can see that a positive demand shock increases both inflation gap and output gap but the output gap increases the most.

**Question 3:** Starting from $y_{-1} = \pi_{-1} = s_{-1} = 0$, how does the economy evolve for $x_0 = 0.1$, $x_t = 0, \forall t > 0$ and $c_t = 0, \forall t \geq 0$?

**Defining functions for demand and supply shocks:**

```python
v_func = lambda v1,x: delta*v1 + x
s_func = lambda s1,c: omega*s1 + c
```

**Setting new parameter values:**

```python
delta = par['delta'] = 0.80
omega = par['omega'] = 0.15
```

**Creating empty vectors for initial values:**

```python
pi_vec_q3 = [0]
y_vec_q3 = [0]
v_vec = [0]
s_vec = [0]
```

**Choosing the period length and defining x and c for each period:**

```python
T = 7
c = np.zeros(T)
x = np.zeros(T)

# setting x0 value
x[1] = 0.1
```

**Running the simulation and appending values to vectors:**

```python
# creating a dataframe to present the results
output = pandas.DataFrame(index=np.arange(0,T-1),columns=('t','$\pi_t$', '$y_t$'))

# looping through the periods
for t in range(1,T):

    # i. update v and s
    v_vec.append(v_func(v_vec[t-1], x[t]))
    s_vec.append(s_func(s_vec[t-1], c[t]))

    # ii. compute y og pi
    y_vec_q3.append(y_func(pi_vec_q3[t-1], y_vec_q3[t-1], v_vec[t], s_vec[t], s_vec[t-1], par['alpha'],
                           par['h'], par['b'], par['phi'], par['gamma']))
    pi_vec_q3.append(pi_func(pi_vec_q3[t-1], y_vec_q3[t-1], v_vec[t], s_vec[t], s_vec[t-1], par['alpha'],
                           par['h'], par['b'], par['phi'], par['gamma']))

    # appending y and pi values to our dataframe
    output.iloc[t-1] = [t-2, round(pi_vec_q3[t-1],4), round(y_vec_q3[t-1],4)]

# removing the index column and printing the output table
blankIndex=[''] * len(output)
output.index=blankIndex
output
```

      t
      $\pi_t$
      $y_t$

      -1
      0
      0

      0
      0.0018
      0.0244

      1
      0.0032
      0.0182

      2
      0.0042
      0.0134

      3
      0.0049
      0.0095

      4
      0.0054
      0.0065

**Question 4:** Simulate the AS-AD model for 1,000 periods. Calculate the following five statistics:

1. Variance of $y_t$, $var(y_t)$
2. Variance of $\pi_t$, $var(\pi_t)$
3. Correlation between $y_t$ and $\pi_t$, $corr(y_t,\pi_t)$
4. Auto-correlation between $y_t$ and $y_{t-1}$, $corr(y_t,y_{t-1})$
5. Auto-correlation between $\pi_t$ and $\pi_{t-1}$, $corr(\pi_t,\pi_{t-1})$

**The standard deviation of the shocks are:**

```python
sigma_x = par['sigma_x'] = 3.492
sigma_c = par['sigma_c'] = 0.2
```

**Defining stochastic process:**

```python
# number of periods
T = 1000

# seed number
np.random.seed(1234)

# creating x and c vectors
x = np.random.normal(0,sigma_x,T)
c = np.random.normal(0,sigma_c,T)

def simulation(T,phi_custom):

    # creating pi, y, v and s vectors with initial values
    pi_vec_q4 = [0]
    y_vec_q4 = [0]
    v_vec = [0]
    s_vec = [0]

    for t in range(1,T):

        # i. update v and s
        v_vec.append(v_func(v_vec[t-1], x[t]))
        s_vec.append(s_func(s_vec[t-1], c[t]))

        # ii. compute y and pi
        y_vec_q4.append(y_func(pi_vec_q4[t-1], y_vec_q4[t-1], v_vec[t], s_vec[t], s_vec[t-1], par['alpha'],
                           par['h'], par['b'], phi_custom, par['gamma']))
        pi_vec_q4.append(pi_func(pi_vec_q4[t-1], y_vec_q4[t-1], v_vec[t], s_vec[t], s_vec[t-1], par['alpha'],
                           par['h'], par['b'], phi_custom, par['gamma']))

    # convert lists to numpy arrays
    pi_vec_q4 = np.array(pi_vec_q4)
    y_vec_q4 = np.array(y_vec_q4)

    # saving the correlation coefficient
    stats_q4 = []
    stats_q4.append(y_vec_q4.var())
    stats_q4.append(pi_vec_q4.var())
    stats_q4.append(np.corrcoef(y_vec_q4, pi_vec_q4)[1,0])
    stats_q4.append(np.corrcoef(y_vec_q4[1:], y_vec_q4[:-1])[1,0])
    stats_q4.append(np.corrcoef(pi_vec_q4[1:], pi_vec_q4[:-1])[1,0])

    return y_vec_q4, pi_vec_q4, stats_q4

y_vec_q4, pi_vec_q4, stats_q4 = simulation(T, par['phi'])
```

```python
# creating a dataframe with the statistics and printing the output table
row_q4 = ['Var($y_t$)', 'Var($\pi_t$)', 'corr($y_t,\pi_t$)', 'corr($y_t,y_t-1$)','corr($\pi_t,\pi_t-1$)']
header_q4 = ['']
stats = pandas.DataFrame(stats_q4,row_q4,header_q4)
stats
```

      Var($y_t$)
      1.584397

      Var($\pi_t$)
      0.976962

      corr($y_t,\pi_t$)
      -0.102136

      corr($y_t,y_t-1$)
      0.739915

      corr($\pi_t,\pi_t-1$)
      0.977730

**Question 5:** Plot how the correlation between $y_t$ and $\pi_t$ changes with $\phi$. Use a numerical optimizer or root finder to choose $\phi\in(0,1)$ such that the simulated correlation between $y_t$ and $\pi_t$ comes close to 0.31.

**Creating a vector containing different values of $\phi$:**

```python
phi_q5 = np.linspace(0,1,50)
```

**Running the simulation for different values of $\phi$:**

```python
# creating an empty list
corr = []

# looping through values of phi and appending correlation coefficients to empty list, corr
for p in phi_q5:
    y_vec_q4, pi_vec_q4, stats_q4 = simulation(T, p)
    corr.append(stats_q4[2])

# plotting the correlation coefficients as a function of phi
fig = plt.figure()
ax = fig.add_subplot(1,1,1)
ax.plot(phi_q5,corr)
ax.grid()
ax.set_xlabel('$\phi$')
ax.set_ylabel('$corr(y_t,\pi_t)$')
ax.set_title('4: Correlation between $y_t$ and $\pi_t$ for different values of $\phi$');
```

**Defining the objective function that we want to minimize:**

```python
# The function defines the difference between the simulated correlation coefficient and the desired coefficient
obj = lambda phi: simulation(T,phi)[2][2] - 0.31
```

**Using a root finder to identify the value of $\phi$ that minimizes the objective function:**

```python
# run the root finder and save the optimal phi value
phi_opt = optimize.root(obj, x0 = 0.9)
phi_opt = phi_opt.x

# print results
print('The value of phi that generates the desired correlation coefficient is: ' + str(round(phi_opt[0],2)))
print('Check for correct correlation coefficient. Corr: ' + str(round(simulation(T,phi_opt)[2][2],2)))
```

    The value of phi that generates the desired correlation coefficient is: 1.01
    Check for correct correlation coefficient. Corr: 0.31

**Quesiton 6:** Use a numerical optimizer to choose $\sigma_x>0$, $\sigma_c>0$ and $\phi\in(0,1)$ to make the simulated statistics as close as possible to US business cycle data where:

1. $var(y_t) = 1.64$
2. $var(\pi_t) = 0.21$
3. $corr(y_t,\pi_t) = 0.31$
4. $corr(y_t,y_{t-1}) = 0.84$
5. $corr(\pi_t,\pi_{t-1}) = 0.48$

**Creating array with the above values:**

```python
stats_comparison = np.array([1.64, 0.21, 0.31, 0.84, 0.48])
```

**Defining new simulation function that can also takes sigma values as input:**

```python
def simulation_q6(params):

    # parameter inputs unpacked
    phi_custom, sigma_x_custom, sigma_c_custom = params

    # time periods
    T = 1000

    # setting seed number for creating x and c vectors
    np.random.seed(1234)

    # creating pi, y, v and s vectors with initial values
    pi_vec_q6 = [0]
    y_vec_q6 = [0]
    v_vec = [0]
    s_vec = [0]

    # creating x and c vectors
    x = np.random.normal(0,sigma_x_custom,T)
    c = np.random.normal(0,sigma_c_custom,T)

    for t in range(1,T):

        # i. update v and s
        v_vec.append(v_func(v_vec[t-1], x[t]))
        s_vec.append(s_func(s_vec[t-1], c[t]))

        # ii. compute y and pi
        y_vec_q6.append(y_func(pi_vec_q6[t-1], y_vec_q6[t-1], v_vec[t], s_vec[t], s_vec[t-1], par['alpha'],
                           par['h'], par['b'], phi_custom, par['gamma']))
        pi_vec_q6.append(pi_func(pi_vec_q6[t-1], y_vec_q6[t-1], v_vec[t], s_vec[t], s_vec[t-1], par['alpha'],
                           par['h'], par['b'], phi_custom, par['gamma']))

    # convert lists to numpy arrays
    pi_vec_q6 = np.array(pi_vec_q6)
    y_vec_q6 = np.array(y_vec_q6)

    # saving the correlation coefficient
    stats_q6 = []
    stats_q6.append(y_vec_q6.var())
    stats_q6.append(pi_vec_q6.var())
    stats_q6.append(np.corrcoef(y_vec_q6, pi_vec_q6)[1,0])
    stats_q6.append(np.corrcoef(y_vec_q6[1:], y_vec_q6[:-1])[1,0])
    stats_q6.append(np.corrcoef(pi_vec_q6[1:], pi_vec_q6[:-1])[1,0])

    return stats_q6
```

**Creating lists used in the numerical optimizer:**

```python
# initial guess is equal to the parameter values from question 5
x0 = np.array([phi_opt, sigma_x, sigma_c])

# defining parameter list used in our objective function below
parameters = [phi, sigma_x, sigma_c]

# setting parameter bounds
bnds = [[0,1],[0,50],[0,50]]
```

**Defining our objective function:**

```python
# minimizing the squared difference between the simulated statistics and the desired statistics
obj_q6 = lambda parameters: np.sum((simulation_q6(parameters) - stats_comparison)**2)
```

**Using a numerical optimizer to identify the value of $\phi$, $\sigma_x$ and $\sigma_c$ that minimizes the objective function:**

```python
# running the optimizer
params_opt = optimize.minimize(obj_q6, x0, bounds = bnds)

# printing the function value
print('The minimized sum of squared difference is: ' + str(round(params_opt.fun,4)))

# saving the phi and sigma values
params_opt = params_opt.x

# calculating the statistics based on our optimal values of phi and sigma_x and sigma_c
sim_val = simulation_q6(params_opt)
print('Variance of y: ' + str(round(sim_val[0],2)))
print('Variance of pi: ' + str(round(sim_val[1],2)))
print('Correlation between y and pi: ' + str(round(sim_val[2],2)))
print('Autocorrelation of y: ' + str(round(sim_val[3],2)))
print('Autocorrelation of pi: ' + str(round(sim_val[4],2)))
```

    The minimized sum of squared difference is: 0.0298
    Variance of y: 1.65
    Variance of pi: 0.05
    Correlation between y and pi: 0.28
    Autocorrelation of y: 0.77
    Autocorrelation of pi: 0.49

From the values above we see that we cant find values of $\phi$, $\sigma_x$ and $\sigma_c$ such that we get the same statistics but based on our approach this is as close as we get

# 3. Exchange economy

The **parameters** and **random preferences and endowments** are given by:

```python
# a. parameters
N = 50000
mu = np.array([3,2,1])
Sigma = np.array([[0.25, 0, 0], [0, 0.25, 0], [0, 0, 0.25]])
gamma = 0.8
zeta = 1

# b. random draws
seed = 1986
np.random.seed(seed)

# preferences
alphas = np.exp(np.random.multivariate_normal(mu, Sigma, size=N))
betas = alphas/np.reshape(np.sum(alphas,axis=1),(N,1))

# endowments
e1 = np.random.exponential(zeta,size=N)
e2 = np.random.exponential(zeta,size=N)
e3 = np.random.exponential(zeta,size=N)
```

**Question 1:** Plot the histograms of the budget shares for each good across agents.

From the Cobb-Douglas utility function we can see that the budget shares are the beta values.
Therefore we plot the beta values in histograms below

```python
fig = plt.figure(figsize=(20,8))
ax = fig.add_subplot(1,3,1)
ax.hist(betas[:,0],bins=100,histtype='bar')
ax.set_xlabel('$share$')
ax.set_ylabel('consumers', fontsize=15)
ax.set_title('5a: Budget share of good 1', fontsize=15)

ax = fig.add_subplot(1,3,2)
ax.hist(betas[:,1],bins=100,density=True,histtype='bar')
ax.set_xlabel('$share$')
ax.set_title('5b: Budget share of good 2', fontsize=15)

ax = fig.add_subplot(1,3,3)
ax.hist(betas[:,2],bins=100,density=True,histtype='bar')
ax.set_xlabel('$share$')
ax.set_title('5c: Budget share of good 3', fontsize=15);
```

Consider the **excess demand functions:**

$$ z_i(p_1,p_2) = \sum_{j=1}^N x_{i}^{\star j}(p_{1},p_{2},\boldsymbol{e}^{j}) - e_i^j$$

**Question 2:** Plot the excess demand functions.

**Defining the demand functions and excess demand function for good 1, 2 and 3:**

```python
# demand functions. We exclude p3 as it is normalized to 1
def demand_good_1(betas,p1,p2,e1,e2,e3):

    # i. income
    I = p1*e1 + p2*e2 + e3

    # ii. demand
    d1 = betas[:,0] * I/p1

    return d1

def demand_good_2(betas,p1,p2,e1,e2,e3):

    # i. income
    I = p1*e1 + p2*e2 + e3

    # ii. demand
    d2 = betas[:,1] * I/p2

    return d2

def demand_good_3(betas,p1,p2,e1,e2,e3):

    # i. income
    I = p1*e1 + p2*e2 + e3

    # ii. demand
    d3 = betas[:,2] * I

    return d3

# excess demand functions
def excess_demand_good_1(betas,p1,p2,e1,e2,e3):

    # total demand for good 1
    total_d1 = np.sum(demand_good_1(betas,p1,p2,e1,e2,e3))

    # total supply of good 1
    total_e1 = np.sum(e1)

    # excess demand of good 1
    excess_1 = total_d1 - total_e1

    return excess_1

def excess_demand_good_2(betas,p1,p2,e1,e2,e3):

    # total demand for good 2
    total_d2 = np.sum(demand_good_2(betas,p1,p2,e1,e2,e3))

    # total supply of good 2
    total_e2 = np.sum(e2)

    # excess demand of good 2
    excess_2 = total_d2 - total_e2

    return excess_2

# creating price arrays
p1_a = np.linspace(0.1,10,100)
p2_a = np.linspace(0.1,10,100)

# creating empty grids for excess demands
excess_grid_1 = np.empty((100,100))
excess_grid_2 = np.empty((100,100))

# creating grids from our price vectors
p1_grid, p2_grid = np.meshgrid(p1_a, p2_a, indexing='ij')

# calculating excess demand for different prices
for i,p1 in enumerate(p1_a):
    for j,p2 in enumerate(p2_a):
        excess_grid_1[i,j] = excess_demand_good_1(betas,p1,p2,e1,e2,e3)
        excess_grid_2[i,j] = excess_demand_good_2(betas,p1,p2,e1,e2,e3)
```

**Plotting the excess demand curves for good 1 and good 2:**

```python
fig = plt.figure(figsize=(12,5))
ax = fig.add_subplot(1,2,1, projection='3d')
ax.plot_surface(p1_grid, p2_grid, excess_grid_1)
ax.invert_xaxis()
ax.set_title('Excess demand for good 1')
ax.set_xlabel('$p_1$')
ax.set_ylabel('$p_2$')

ax = fig.add_subplot(1,2,2, projection='3d')
ax.plot_surface(p1_grid, p2_grid, excess_grid_2)
ax.invert_xaxis()
ax.set_title('Excess demand for good 2')
ax.set_xlabel('$p_1$')
ax.set_ylabel('$p_2$');
```

We see from our graphs above that the excess demand of a good, either good 1 or 2, increases in the price of the other good. This makes sense as you would expect the demand of good 1 to increase as the price for good 2 increases, because good 1 then becomes cheaper relative to good 2.

**Quesiton 3:** Find the Walras-equilibrium prices, $(p_1,p_2)$, where both excess demands are (approximately) zero, e.g. by using the following tâtonnement process:

1. Guess on $p_1 > 0$, $p_2 > 0$ and choose tolerance $\epsilon > 0$ and adjustment aggressivity parameter, $\kappa > 0$.
2. Calculate $z_1(p_1,p_2)$ and $z_2(p_1,p_2)$.
3. If $|z_1| < \epsilon$ and $|z_2| < \epsilon$ then stop.
4. Else set $p_1 = p_1 + \kappa \frac{z_1}{N}$ and $p_2 = p_2 + \kappa \frac{z_2}{N}$ and return to step 2.

**Guessing initial values:**

```python
p1_0 = 1
p2_0 = 1
```

**Defining equilibrium function:**

```python
def equilibrium(betas,p1,p2,e1,e2,e3,kappa=0.5,eps=1e-6,maxiter=10000):

    t=0
    while True:

        # i. step 2: calculate excess demand
        z1 = excess_demand_good_1(betas,p1,p2,e1,e2,e3)
        z2 = excess_demand_good_2(betas,p1,p2,e1,e2,e3)

        # ii. step 3: stop?
        if (np.abs(z1) < eps and np.abs(z2) < eps) or t >= maxiter:
            print(f'Iteration nr. {t}: (p1, p2) = ({p1:.2f}, {p2:.2f}) => excess demand (z1, z2) = ({z1:.2f}, {z1:.2f})')
            break

        # iii. step 4: update prices and return to step 2
        p1 = p1 + kappa*z1/N
        p2 = p2 + kappa*z2/N

        # next iteration
        t += 1

    return p1, p2
```

```python
# running the equilibrium function and saving equilibrium prices
p1, p2 = equilibrium(betas,p1_0,p2_0,e1,e2,e3)
```

    Iteration nr. 2342: (p1, p2) = (6.49, 2.62) => excess demand ($z_1$, z2) = (0.00, 0.00)

**Question 4:** Plot the distribution of utility in the Walras-equilibrium and calculate its mean and variance.

**Defining utility function:**

```python
def utility_func(betas,p1,p2,e1,e2,e3,gamma):

    # i. creating demand functions
    d1 = demand_good_1(betas,p1,p2,e1,e2,e3)
    d2 = demand_good_2(betas,p1,p2,e1,e2,e3)
    d3 = demand_good_3(betas,p1,p2,e1,e2,e3)

    # ii. creating an empty array for utility values
    u = np.zeros(N-1)

    # iii. calculating utility for each individual
    for i in range(0,N-1):
        u[i] = (d1[i]**betas[i,0]*d2[i]**betas[i,1]*d3[i]**betas[i,2])**gamma

    # Calculating mean and variance of utility
    u_mean = np.mean(u)
    u_var = np.var(u)

    # printing mean and variance
    print(f'Mean(u) = {u_mean:.3f}')
    print(f'Var(u) = {u_var:.3f}')

    return u
```

```python
# running the utility function
u = utility_func(betas,p1,p2,e1,e2,e3,gamma)
```

    Mean(u) = 1.010
    Var(u) = 0.317

**Plotting the utility distribution:**

```python
plt.hist(u,bins=100)
plt.title('distribution of utility')
plt.xlabel('utility')
plt.ylabel('consumers');
```

**Question 5:** Find the Walras-equilibrium prices if instead all endowments were distributed equally. Discuss the implied changes in the distribution of utility. Does the value of $\gamma$ play a role for your conclusions?

**Defining new and equally distributed endowments based on the original total endowments:**

```python
e1_new = np.ones(N) * np.sum(e1)/N
e2_new = np.ones(N) * np.sum(e2)/N
e3_new = np.ones(N) * np.sum(e3)/N
```

**Calculating equilibrium prices and utility values for new endowments and different $\gamma$ values:**

```python
p1_new, p2_new = equilibrium(betas,p1_0,p2_0,e1_new,e2_new,e3_new)
u_new = utility_func(betas,p1_new,p2_new,e1_new,e2_new,e3_new,gamma)
u_new_newgamma = utility_func(betas,p1_new,p2_new,e1_new,e2_new,e3_new,2)
```

    Iteration nr. 2339: (p1, p2) = (6.49, 2.62) => excess demand ($z_1$, z2) = (0.00, 0.00)
    Mean(u) = 1.046
    Var(u) = 0.003
    Mean(u) = 1.126
    Var(u) = 0.027

**Plotting the different utility distributions:**

```python
fig = plt.figure(figsize=(20,8))
plt.hist(u, bins=50, label='initial unequal endowments, $\gamma$ = 0.8')
plt.hist(u_new, bins=50, color='red', label='equal endowments, $\gamma$ = 0.8')
plt.hist(u_new_newgamma, bins=50, alpha=0.6, color='green', label='equal endowments, $\gamma$ = 2.0')
plt.legend()
plt.title('distribution of utility')
plt.xlabel('utility')
plt.ylabel('consumers');
```

The distribution of endowments has no effect on prices. However, the mean utility is higher when initial endowments
are equally distributed, and the variance is lower.

When we increase gamma both the mean utility and the variance increases, so higher average utility but greater inequality.


# Constrained optimization

## Bound constraints
Often the parameters of an optimization problems are subject to (often abbreviated as s. t.) constraints. In practice, constraints make solving an optimization problem much more difficult. In the simplest case, these can be simple box constraints for the parameters.

Consider the Rosenbrock function with box constraints:

$$
\begin{align}
\underset{x, y}{argmin} (1-x)^2+100(y-x^2)^2 \\
\text{s. t. } x\ \in [-1, 0.5],\ y\in [0.5 , 1.5]
\end{align}
$$

Bounds have to be provided as a list of (lower_bound, upper_bound) for each coordinate:

```python
import simplenlopt
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Patch

def rosenbrock(pos):

    x, y = pos
    return (1-x)**2 + 100 * (y - x**2)**2

x0 = np.array([-0.5, 0.6])
bounds=[(-1, 0.5), (0.5, 1.5)]
res = simplenlopt.minimize(rosenbrock, x0, bounds = bounds)
print(res.x)

X = np.linspace(-2, 2, 500)
Y = np.linspace(-2, 2, 500)

XX, YY = np.meshgrid(X, Y)
Z = (1-XX)*(1-XX)+100*(YY-XX*XX)**2

fig, ax = plt.subplots()
plt.pcolormesh(XX, YY, Z, vmin = 0, vmax = 10)
box = plt.Rectangle((-1, 0.5),1.5, 1, facecolor='r', edgecolor='r', lw = 2, alpha = 0.3)
plt.colorbar(extend='max', label="$f(x, y)$")
ax.add_patch(box)
ax.scatter([res.x[0]], [res.x[1]], c='r', s = 15, label='Optimum')
handles, labels = ax.get_legend_handles_labels()
legend_patch = Patch(facecolor='red', edgecolor='r', alpha = 0.3,
                         label='Feasible region')
handles.append(legend_patch)
ax.legend(handles = handles)
ax.set_xlabel("x")
ax.set_ylabel("y")
plt.show()
```

## Inequality and equality constraints

General inequality and equality constraints reduce the parameter space by linear or nonlinear functions. Let's say we are interested in the minimum of the rosenbrock function which fulfills that $y\geq 0.5-x$ and $y\leq 1-x^2$. In typical optimization notation, constraints are formulated by defining a function of the optimization parameters $f(\mathbf{x})\leq 0$ for inequality constraints and $f(\mathbf{x})=0$ for equality constraints. In this notation, our constrained Rosenbrock problem would be written as

$$
\begin{align}
\underset{x, y}{argmin} (1-x)^2 & +100(y-x^2)^2 \\
s.\ t.\ 0.5 -x -y &\leq 0\\
y+x^2-1 &\leq 0
\end{align}
$$

In SciPy/simplenlopt style, a constraint has to be provided as a dictionary with at least two keys: ``const={type:'ineq'/'eq', 'fun'}``. ``'fun'`` must be of the form ``fun(x, *args)`` just like the objective function. All constraints then have to be passed as a list to minimize.

```python
def rosenbrock(pos):

    x, y = pos
    return (1-x)**2 + 100 * (y - x**2)**2

def constraint_1(pos):
    x, y = pos
    return 0.5-x-y

def constraint_2(pos):
    x, y = pos
    return y+x*x -1

ineq_constraint_1 = {'type':'ineq', 'fun':constraint_1}
ineq_constraint_2 = {'type':'ineq', 'fun':constraint_2}

x0 = np.array([0.5, 0.5])
res = simplenlopt.minimize(rosenbrock, x0, constraints=[ineq_constraint_1, ineq_constraint_2])
print("Found optimum position: ", res.x)
print("Number of function evaluations: ", res.nfev)

#Plot the feasiable solution space and the found minimum
fig, ax = plt.subplots()
plt.pcolormesh(XX, YY, Z, vmin = 0, vmax = 10, alpha = 0.5)

y_constraint_1 = 0.5-X
y_constraint_2 = 1 - X*X

constraint_1_region_lower_bound = np.maximum(y_constraint_1, y_constraint_2)
constraint_2_region_upper_bound = np.minimum(y_constraint_1, y_constraint_2)

ax.fill_between(X, constraint_1_region_lower_bound, np.full_like(X, 2), color='green', edgecolor = 'green', alpha=0.1)
ax.fill_between(X, constraint_2_region_upper_bound, np.full_like(X, -2), color='blue', edgecolor = 'blue', alpha=0.45)

ax.fill_between(X, y_constraint_1, y_constraint_2, where=(y_constraint_2 > y_constraint_1), color='r', alpha=0.2)#where=(y1 > y2)
plt.colorbar(extend='max', label="$f(x, y)$")
ax.scatter([res.x[0]], [res.x[1]], c='r', s = 15, zorder=2, label='Optimum')
handles, labels = ax.get_legend_handles_labels()
legend_patch_feasible = Patch(facecolor='red', edgecolor='r', alpha = 0.3,
                         label='Feasible region')
legend_patch_constraint1 = Patch(facecolor='green', edgecolor='g', alpha = 0.3,
                         label='$y\geq 0.5-x$')
legend_patch_constraint2 = Patch(facecolor='blue', edgecolor='blue', alpha = 0.3,
                         label='$y\leq 1-x^2$')
handles.extend((legend_patch_feasible, legend_patch_constraint1, legend_patch_constraint2))
ax.set_xlim(-2, 2)
ax.set_ylim(-2, 2)
ax.legend(handles = handles, loc='lower center')
ax.set_xlabel("x")
ax.set_ylabel("y")
plt.show()
```

## Augmented Lagrangian

Many optimization algorithms were originally not developed to deal with constraints. NLopt provides a powerful way around this: the augmented Lagrangian. Similarly to regularization in machine learning, the augmented lagrangian adds increasing penalty terms to penalize violation of the constraints until they are met. This makes it possible to also use algorithms which were originally not designed to handle constraints. In simplenlopt, the ``minimize`` function automatically calls the augmented lagrangian if an optimizer cannot deal with the supplied constraints. Here, we will use the BOBYQA algorithm for the same constrained Rosenbrock problem.

```python
res = simplenlopt.minimize(rosenbrock, x0, method = 'bobyqa', constraints=[ineq_constraint_1, ineq_constraint_2])
print("Found optimum position: ", res.x)
print("Number of function evaluations: ", res.nfev)
```

    Found optimum position:  [0.70747058 0.49948457]
    Number of function evaluations:  753

    C:\Users\danie\.conda\envs\simplenlopt_env\lib\site-packages\simplenlopt\_Core.py:479: RuntimeWarning: Method bobyqa does not support constraints. Constraints will be handled by augmented lagrangian. In case of problems consider method='COBYLA'.
      warn("Method {} does not support constraints. "

The augmented lagrangian can also be directly called with by the function 'auglag' which uses the same arguments as minimize. As auglag does not apply the same checks as minimize, calling it directly reduces some small overhead.

```python
res = simplenlopt.auglag(rosenbrock, x0, method = 'bobyqa', constraints=[ineq_constraint_1, ineq_constraint_2])
print(res.x)
```

    [0.70747058 0.49948457]

Note that the augmented lagrangian is only implemented for the local optimizers but not for the global optimizers!


# Statistics

## 1.1 Sales for the past week was the following amounts: [3505, 2400, 3027, 2798, 3700, 3250, 2689]. Without using library functions, what is the mean, variance, and standard deviation of of sales from last week? (for extra bonus points, write your own function that can calculate these two values for any sized list)

```
# Fuction to find the mean of a list of values:
def findMean(num_list):
    '''
    Return the mean out of any list of numbers.

        - Params:
        num_list: a list of numeric values
    '''
    return round(sum(num_list)/len(num_list),2)

# Function to find th variance from a list of numbers:
def findVariance(num_list):
    '''
    Return the variance from a list of numbers.

        - Params:
        num_list: a list of numeric values
    '''
    l = []
    a = sum(num_list)/len(num_list)
    for i in num_list:
        b = i-a
        c = b**2
        l.append(c)
    d = len(l)
    e = sum(l)/d
    return round(e,2)

# Function to find the standard deviation from a list of numbers:
def findStd(num_list):
    '''
    Return the standard deviation from a list of numbers.

        - Params:
        num_list: a list of numeric values
    '''
    l = []
    a = sum(num_list)/len(num_list)
    for i in num_list:
        b = i-a
        c = b**2
        l.append(c)
    d = len(l)
    e = sum(l)/d
    return round(e**(.5),2)
```

```
# Sale numbers of the past week:
sales_amounts = [3505,2400,3027,2798,3700,3250,2689]

print('Mean: ',findMean(sales_amounts))
print('Variance: ',findVariance(sales_amounts))
print('Standard Deviation: ',findStd(sales_amounts))
```

    Mean:  3052.71
    Variance:  183761.06
    Standard Deviation:  428.67

## 1.2 Find the covariance between last week's sales numbers and the number of customers that entered the store last week: [127, 80, 105, 92, 120, 115, 93] (you may use librray functions for calculating the covariance since we didn't specifically talk about its formula)

```
import numpy as np
# Sale numbers of the past week:
sales_amounts = [3505,2400,3027,2798,3700,3250,2689]
# Number of customers from the past week:
num_of_cust = [127,80,105,92,120,115,93]

# Covariance between sales and customers:
cov = round(np.cov(sales_amounts,num_of_cust)[0][1],2)
print('Covariance: ',cov)
```

    Covariance:  7604.36

## 1.3 Find the standard deviation of customers who entered the store last week. Then, use the standard deviations of both sales and customers to standardize the covariance to find the correlation coefficient that summarizes the relationship between sales and customers. (You may use library functions to check your work.)

```
# Standard deviation of the number of customers that entered the store in the past week:
cust_std = findStd(num_of_cust)
# Standard deviation of the sales from the past week:
sales_std = findStd(sales_amounts)
# Find the covariance:
cov = np.cov(np.array([sales_amounts],[num_of_cust])
# Denominator for covariance standardization:
denominator = sales_std*cust_std
# Standardize the covariance to find the correlation coefficient:
corr = cov/denominator
print('By hand Correlation Coeficient: \n',corr)

print('Correlation Coef. useing Numpy: ',np.corrcoef(sales_amounts,num_of_cust)[0][1])

# Find the Correlation Coefficient using pandas:
import pandas as pd
# Create a dataframe:
df = pd.DataFrame({'sales':sales_amounts,'customers':num_of_cust})
# Find the correlation coefficient:
df['sales'].corr(df['customers'])
print('Correlation Coef. using Pandas: ',np.array(df.corr())[0][1])

np.linalg.det(corr)
```

    By hand Correlation Coeficient:
     [[31.67342724  1.12345915]
     [ 1.12345915  0.04298498]]
    Correlation Coef. useing Numpy:  0.9628339778148909
    Correlation Coef. using Pandas:  0.9628339778148908

    0.09932107295422878

## 1.4 Use pandas to import a cleaned version of the titanic dataset from the following link: [Titanic Dataset](https://raw.githubusercontent.com/Geoyi/Cleaning-Titanic-Data/master/titanic_clean.csv)

## Calculate the variance-covariance matrix and correlation matrix for the titanic dataset's numeric columns. (you can encode some of the categorical variables and include them as a stretch goal if you finish early)

```
import pandas as pd
# Read in the cleaned up titanic dataset:
titanic = pd.read_csv('https://raw.githubusercontent.com/Geoyi/Cleaning-Titanic-Data/master/titanic_clean.csv', index_col='Unnamed: 0')
# Get the variance-covariance matrix:
cov = titanic.cov()
# Get the correlation matrix:
corr = titanic.corr()

print("Variance-CoVariance Matrix:")
display(cov)
print('\n\n',"Correlation Matrix:")
display(corr)
print('\n\n')

import matplotlib.pyplot as plt
import seaborn as sns
fig, ax = plt.subplots(figsize=(7,6))
sns.heatmap(corr)
ax.set_title('Titanic Data Correlation Matrix',fontsize=20)
plt.show();

```

# Orthogonality

## 2.1 Plot two vectors that are orthogonal to each other. What is a synonym for orthogonal?

**Answer:**
> A synonym for orthogonal is perpendicular.

```
# Creat orthogonal vectors:
vec_a = [.2,.5]
vec_b = [.5,-.2]

# Plot the orthogonal vectors:
fig, ax = plt.subplots()
ax.grid()
plt.xlim(-.4,1.0)
plt.ylim(-.4,1.0)
plt.arrow(0,0,vec_a[0],vec_a[1],
          width=.03,head_width=.1,head_length=.1,length_includes_head=True)
plt.arrow(0,0,vec_b[0],vec_b[1],
          width=.03,head_width=.1,head_length=.1,length_includes_head=True,
          color='green',ec='k');

np.dot(vec_a,vec_b)
```

## 2.2 Are the following vectors orthogonal? Why or why not?

\begin{align}
a = \begin{bmatrix} -5 \\ 3 \\ 7 \end{bmatrix}
\qquad
b = \begin{bmatrix} 6 \\ -8 \\ 2 \end{bmatrix}
\end{align}

**Answer:**
> These vectors are not orthogonal, because the are not perpendicular and the dot product is not equal to zero.

```
# Create vectors:
vec_c,vec_d = [-.5,.3,.7],[.6,-.8,.2]

# Check orthoganality by finding the dot product of the 2 vectors:
print(np.dot(vec_c,vec_d))

# Check orthoganality by plotting these vectors:
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.set_xlim(-.10,1)
ax.set_ylim(-.10,1)
ax.set_zlim(-.10,1)
ax.quiver(0,0,0,vec_c[0],vec_c[1],vec_c[2])
ax.quiver(0,0,0,vec_d[0],vec_d[1],vec_d[2])
```

## 2.3 Compute the following values: What do these quantities have in common?

## What is $||c||^2$?

## What is $c \cdot c$?

## What is $c^{T}c$?

\begin{align}
c = \begin{bmatrix} 2 & -15 & 6 & 20 \end{bmatrix}
\end{align}

```
# Create vector:
c = [2,-15,6,20]

# Find the length of the vector:
norm_c = np.linalg.norm(c)**2
# Find the dot product of the vector:
dot_c = np.dot(c,c)
# Calculate the vector.Transpose by itself:
transpose_c = np.array(c).T*np.array(c)

print(norm_c)
print(dot_c)
print(sum(transpose_c))

# All of these calculations compute the same thing.
```

    665.0
    665
    665

# Unit Vectors

## 3.1 Using Latex, write the following vectors as a linear combination of scalars and unit vectors:

\begin{align}
d = \begin{bmatrix} 7 \\ 12 \end{bmatrix}
\qquad
e = \begin{bmatrix} 2 \\ 11 \\ -8  \end{bmatrix}
\end{align}

**Answer:**
>$d = 7\hat{i}+12\hat{j}$

>$e = 2\hat{i}+11\hat{j}+-8\hat{k}$

## 3.2 Turn vector $f$ into a unit vector:

\begin{align}
f = \begin{bmatrix} 4 & 12 & 11 & 9 & 2 \end{bmatrix}
\end{align}

```
# Create Vector:
f = [4,12,11,9,2]
# Convert f into a numpy array:
f = np.array(f)
# Turn vector into a unit vector
f = f/np.linalg.norm(f)
f
```

    array([0.20908335, 0.62725005, 0.57497921, 0.47043754, 0.10454167])

# Linear Independence / Dependence

## 4.1 Plot two vectors that are linearly dependent and two vectors that are linearly independent (bonus points if done in $\mathbb{R}^3$).

```
import numpy as np
import pandas as pd
# Create matrix of linearly dependent vectors:
dep_vec_matrix = pd.DataFrame({'vec_a':[.2,.2,.2],'vec_b':[.3,.3,.3]})
print('Rank ',np.linalg.matrix_rank(dep_vec_matrix))
# Create matrix of linearly independent vectors:
ind_vec_matrix = pd.DataFrame({'vec_a':[.3,.6,.2],'vec_b':[.1,.4,.5]})
print('Rank ',np.linalg.matrix_rank(ind_vec_matrix))
# Plot vectors:
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.set_xlim(0,1)
ax.set_ylim(0,1)
ax.set_zlim(0,1)
ax.quiver(0,0,0,dep_vec_matrix['vec_a'][0],dep_vec_matrix['vec_a'][1],dep_vec_matrix['vec_a'][2],color='red')
ax.quiver(0,0,0,dep_vec_matrix['vec_b'][0],dep_vec_matrix['vec_b'][1],dep_vec_matrix['vec_b'][2],color='black')
ax.quiver(0,0,0,ind_vec_matrix['vec_a'][0],ind_vec_matrix['vec_a'][1],ind_vec_matrix['vec_a'][2],color='green')
ax.quiver(0,0,0,ind_vec_matrix['vec_b'][0],ind_vec_matrix['vec_b'][1],ind_vec_matrix['vec_b'][2],color='green');
```

# Span

## 5.1 What is the span of the following vectors?

\begin{align}
g = \begin{bmatrix} 1 & 2 \end{bmatrix}
\qquad
h = \begin{bmatrix} 4 & 8 \end{bmatrix}
\end{align}

```
# Create vectors:
g = np.array([.1,.2])
h = np.array([.4,.8])

# Find the span:
print('Span: Rank ',np.linalg.matrix_rank([g,h]),'\n\n')

# Plot the span of the vectors to show its 1-d:
fig, ax = plt.subplots()
ax.grid()
plt.xlim(-.4,1)
plt.ylim(-.4,1)
plt.arrow(0,0,h[0],h[1],color='red',head_width=.06,width=.04)
plt.arrow(0,0,g[0],g[1],head_width=.05,width=.02);
```

## 5.2 What is the span of $\{l, m, n\}$?

\begin{align}
l = \begin{bmatrix} 1 & 2 & 3 \end{bmatrix}
\qquad
m = \begin{bmatrix} -1 & 0 & 7 \end{bmatrix}
\qquad
n = \begin{bmatrix} 4 & 8  & 2\end{bmatrix}
\end{align}

```
# Create Vectors:
l = np.array([1,2,3])
m = np.array([-1,0,7])
n = np.array([4,8,2])
# Find span of vectors:
print('Span: Rank',np.linalg.matrix_rank([l,m,n]))
```

    Span: Rank 3

# Basis

## 6.1 Graph two vectors that form a basis for $\mathbb{R}^2$

```
# Creat vectors:
vec_one,vec_two = [-.2,.3],[.2,.5]

# Plot vectors that form a basis for rank2:
fig, ax = plt.subplots()
ax.grid()
plt.xlim(-.3,.6)
plt.ylim(-.3,.6)
plt.arrow(0,0,vec_one[0],vec_one[1],
          head_width=.03, width=.01, color='green')
plt.arrow(0,0,vec_two[0],vec_two[1],
          head_width=.03, width=.01, color='red')
```

## 6.2 What does it mean to form a basis?

# Rank

## 7.1 What is the Rank of P?

\begin{align}
P = \begin{bmatrix}
1 & 2 & 3 \\
 -1 & 0 & 7 \\
4 & 8  & 2
\end{bmatrix}
\end{align}

```
# Create matrix:
p = np.array([[1,-1,4],
             [2,0,8],
             [3,7,2]])
# Find Rank:
np.linalg.matrix_rank(p.T)
```

    3

## 7.2 What does the rank of a matrix tell us?

**Answer:**
>It shows us the maximum number of linearly independent column vectors in the matrix or the maximum number of linearly independent row vectors in the matrix.

# Linear Projections

## 8.1 Line $L$ is formed by all of the vectors that can be created by scaling vector $v$
\begin{align}
v = \begin{bmatrix} 1 & 3 \end{bmatrix}
\end{align}

\begin{align}
w = \begin{bmatrix} -1 & 2 \end{bmatrix}
\end{align}

## find $proj_{L}(w)$

## graph your projected vector to check your work (make sure your axis are square/even)

```
v = np.array([1,3])
w = np.array([-1,2])

v_w = np.dot(v,w)
w_w = np.dot(w,w)

frac = v_w/w_w

projection = np.multiply(frac,w)

fig, ax = plt.subplots()
ax.grid()
plt.xlim(-2,10)
plt.ylim(-2,10)
ax.arrow(0,0,projection[0],projection[1])
```

# Stretch Goal

## For vectors that begin at the origin, the coordinates of where the vector ends can be interpreted as regular data points. (See 3Blue1Brown videos about Spans, Basis, etc.)

## Write a function that can calculate the linear projection of each point (x,y) (vector) onto the line y=x. run the function and plot the original points in blue and the new projected points on the line y=x in red.

## For extra points plot the orthogonal vectors as a dashed line from the original blue points to the projected red points.

```
import pandas as pd
import matplotlib.pyplot as plt

# Creating a dataframe for you to work with -Feel free to not use the dataframe if you don't want to.
x_values = [1, 4, 7, 3, 9, 4, 5 ]
y_values = [4, 2, 5, 0, 8, 2, 8]

data = {"x": x_values, "y": y_values}

df = pd.DataFrame(data)

df.head()

plt.scatter(df.x, df.y)
plt.show()
```

```

```


## Description
This notebook aims at creating visualization for the [note of objective functions](https://www.notion.so/Objective-Functions-21cdb3d560ac4b2fb7755c37f441209b). Please refer to the note to view the practical usage of objective functions in Tensorflow.

## Import packages, Environment Setting

```python
import os
import numpy as np
import tensorflow as tf

from bokeh.io import output_notebook, export_png, reset_output
from bokeh.plotting import figure, show
from bokeh.models import Range1d

reset_output()
output_notebook()

physical_devices = tf.config.list_physical_devices('GPU')
tf.config.experimental.set_memory_growth(physical_devices[0], True)
```

    Loading BokehJS ...

```python
def visualize(name, X, y, ymin=None, ymax=None, fig=None, export=True, x_axis_label='y - ŷ',
    classification=False, **kwargs):
    X = X.numpy()
    y = y.numpy()

    xmin, xmax = np.min(X), np.max(X)
    if classification:
        ymin, ymax = 0, np.max(y)
    else:
        ymin, ymax = 0, np.max(np.power(X, 2))
    if not fig:
        fig = figure(title=name, tools=[], x_axis_label=x_axis_label, y_axis_label='Loss')
        fig.x_range=Range1d(xmin, xmax)
        fig.y_range=Range1d(ymin, ymax)

    fig.line(X, y, line_width=5, **kwargs)

    if export:
        file = os.path.join('assets', f'objective_functions_{name}.png'.lower().replace(' ', '_'))
        export_png(fig, filename=file)

    return fig
```

```python
X = tf.linspace(-10., 10., 100)
```

## Regression Objective Functions
Suppose $\mathbf{y}$ is the actual values we want to predict, and $\mathbf{\hat{y}} \in \mathbb{R}^m$ is the prediction made by our model. In other words, $\mathbf{\hat{y}}=f(\mathbf{X}, \mathbf{w})$.  we can use several objective functions to evaluate our model. In this section, we will discuss the most commonly used ones: `MeanAbsoluteError`,  `MeanSquaredError` and `HuberError`.

### Mean Absolute Error (L1 Loss)
Equation

$L(\mathbf{y}, \mathbf{\hat{y}})=\frac{1}{m}\sum\limits_{i=1}^m|y_i - \hat{y}_i|$

Derivative

$\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\mathbf{\hat{y}}}=[d_i]=\begin{cases}\frac{1}{m}\;&\text{if } \hat{y}_i > y_i\\-\frac{1}{m}&\text{otherwise}\end{cases}$

Properties
* Less sensitive to samples with large residual between prediction and actual value.

```python
y = tf.abs(X)
_ = visualize('Mean Absolute Error', X, y, **{'color': 'deepskyblue'})
```

### Mean Squared Error (L2 Loss)
Equation

$L(\mathbf{y}, \mathbf{\hat{y}})=\frac{1}{m}\sum\limits_{i=1}^m(y_i - \hat{y}_i)^2$

Derivative

$\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\mathbf{\hat{y}}}=-\frac{2}{m}(\mathbf{y}-\mathbf{\hat{y}})$

Properties
* More sensitive to samples with large residual between prediction and actual value.

```python
y = tf.pow(X, 2)
_ = visualize('Mean Squared Error', X, y, **{'color': 'lightcoral'})
```

### Huber Loss
Equation

$\begin{align}
L(\mathbf{y}, \mathbf{\hat{y}})=\sum\limits_{i=1}^m\begin{cases}\frac{1}{2}(y_i-\hat{y}_i)^2\;\;\;&\text{if}\;|y_i-\hat{y}_i|\leq\delta\\\delta(|y_i-\hat{y}_i|-\frac{1}{2}\delta)&\text{otherwise}\end{cases}
\end{align}$

Derivative

$\begin{align}
\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial \mathbf{\hat{y}}}=[d_i]=\begin{cases}\hat{y}_i-y_i\;\;\;&\text{if}\;|y_i-\hat{y}_i|\leq\delta\\-\delta&\text{if}\;|y_i-\hat{y}_i|>\delta\text{ and }|y_i-\hat{y}_i|>0\\\delta&\text{otherwise}\end{cases}
\end{align}$

Properties
* Hyperparameter $\delta$ can be used to control how many penalties should be given to samples with large residual.

```python
delta = 1
y = tf.where(tf.abs(X) <= delta, 0.5 * tf.pow(X, 2), delta * (tf.abs(X) - 0.5 * delta))
fig = visualize('Huber Loss', X, y, export=False, **{'color': 'thistle', 'legend_label': f'delta:{1:>3}'})
delta = 5
y = tf.where(tf.abs(X) <= delta, 0.5 * tf.pow(X, 2), delta * (tf.abs(X) - 0.5 * delta))
fig = visualize('Huber Loss', X, y, fig=fig, export=False, **{'color': 'plum', 'legend_label': f'delta:{5:>3}'})
delta = 10
y = tf.where(tf.abs(X) <= delta, 0.5 * tf.pow(X, 2), delta * (tf.abs(X) - 0.5 * delta))
fig = visualize('Huber Loss', X, y, fig=fig, **{'color': 'mediumorchid', 'legend_label': f'delta:{10:>3}'})
```

## Classification Loss
### Cross Entropy
Consider $y_{ij}=1$ denotes the sample $i$ when it belongs to class $j$, $y_i=0$ denotes the sample when it dows not belong to class $j$, $\hat{y}_{ij}$ denotes the prediction of the **probability** to assign sample $i$ to class $j$

Equation:

$H(\mathbf{y}, \mathbf{\hat{y}})=-\frac{1}{m}\sum\limits_{i=1}^m\sum\limits_{j=1}^Ky_{ij}\ln\hat{y}_{ij}$

Derivative:

$\frac{\partial H(\mathbf{y}, \mathbf{\hat{y}})}{\partial\mathbf{\hat{y}}}=[d_{ij}]=-\frac{y_{ij}}{\hat{y}_{ij}}$

Properties:

- The inferred probability $\mathbf{\hat{y}}$ is more accurate compare to hinge loss.
- **Sigmoid** or **softmax** activation functions are preferred to used in the output layer.

Notes:
- [CategoricalCrossentropy](https://www.tensorflow.org/api_docs/python/tf/keras/losses/CategoricalCrossentropy): Used when `y_true` is an one-hot encoded vector.
- [SparseCategoricalCrossentropy](https://www.tensorflow.org/api_docs/python/tf/keras/losses/SparseCategoricalCrossentropy): Used when `y_true` is a label encoded vector.

```python
y_pred = tf.linspace(0.01, 0.99, 100)
loss = -1 * tf.math.log(y_pred)
fig = visualize(
    'Cross Entropy',
    y_pred,
    loss,
    classification=True,
    export=False,
    x_axis_label='ŷ (Class A)',
    **{'color': 'limegreen', 'legend_label': 'Truth (Class 0)'}
)
loss = -1 * tf.math.log(1 - y_pred)
fig = visualize(
    'Cross Entropy',
    y_pred,
    loss,
    fig=fig,
    classification=True,
    **{'color': 'palegreen', 'legend_label': 'Truth (Class 1)'}
)
```

### Hinge Loss (Crammer and Singer)

Consider $y_{ij}=1$ denotes the sample $i$ when it belongs to class $j$, $y_i=0$ denotes the sample when it dows not belong to class $j$, $\hat{y}_{ij}$ denotes the prediction of the **distance** to assign sample $i$ to class $j$, the positive value suggests higher confidence while the negative value suggests lower confidence. The prediction value can be an **unbounded continuous value**.

Equation:

$\text{neg}_i =\text{max}_{j}((1-y_{ij})\hat{y}_{ij})$

$\text{pos}=\sum\limits_{j=1}^my_{ij}\hat{y}_{ij}$

$L(\mathbf{y}, \mathbf{\hat{y}})=\frac{1}{m}\sum\limits_{i=1}^m\text{max}(0, 1 + \text{neg}_i - \text{pos})$

Derivative:

$\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial \mathbf{\hat{y}}}=[d_{ij}]=\begin{cases}1-2y_{ij}\;&\text{if }{(1-y_{ij})\hat{y}_{ij}}=\text{neg}_i\text{ and neg}_i-\text{pos}>-1\\-y_{ij}\;&\text{if }{(1-y_{ij})\hat{y}_{ij}}\ne\text{neg}_i\text{ and neg}_i-\text{pos}>-1\\0&\text{otherwise}\end{cases}$

Properties:

- Data point far away from the decision boundary do not contribute to the loss function.
- **Linear** or **hyperbolic tangent** activation functions are preferred to used in the output layer.
- Need additional methods (e.g. **Platt scaling**) to estimate probability for each class.

```python
y_pred = tf.linspace(-3., 3., 100)
loss = tf.where(1. + y_pred > 0., 1. + y_pred, tf.zeros_like(y_pred))
fig = visualize(
    'Hinge Loss',
    y_pred,
    loss,
    x_axis_label='ŷ',
    classification=True,
    export=False,
    **{'color': 'blueviolet', 'legend_label': 'Truth (Class 0)'}
)
loss = tf.where(1. - y_pred > 0., 1. - y_pred, tf.zeros_like(y_pred))
fig = visualize(
    'Hinge Loss',
    y_pred,
    loss,
    fig=fig,
    classification=True,
    **{'color': 'plum', 'legend_label': 'Truth (Class 1)'}
)
```

## Why are MSE and MAE not suitable for classification
### Binary Classification with Probability

Let's first decompose the binary classification problem with **sigmoid** output:

$\mathbf{\hat{y}}=\sigma(\mathbf{\hat{X}}),\;\;\;\mathbf{\hat{x}}=\mathbf{X}\mathbf{w}+b$

Now, consider the gradient of mean square error with respect to $\mathbf{\hat{y}}$:

$\nabla_{\mathbf{\hat{y}}}L(\mathbf{y}, \mathbf{\hat{y}})=\begin{bmatrix}\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{y}_1} \\\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{y}_2} \\\vdots \\\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{y}_m}\end{bmatrix}=\begin{bmatrix}-\frac{2}{m}(y_1 - \hat{y}_1) \\-\frac{2}{m}(y_2 - \hat{y}_2) \\\vdots \\-\frac{2}{m}(y_1 - \hat{y}_m)\end{bmatrix}=\frac{2}{m}(\mathbf{y}-\mathbf{\hat{y}})$

And the Jacobian matrix of sigmoid function with respect to $\mathbf{\hat{x}}$ (check **Activation Functions** in [Neural Network Basics](https://www.notion.so/Neural-Network-Basics-6c71b218abc14bb89e2fa21f35066c54) ):

$\frac{\partial\mathbf{\hat{y}}}{\partial\mathbf{\mathbf{\hat{x}}}}=\begin{bmatrix}    \frac{\partial\hat{y}_1}{\partial\hat{x}_1} & \frac{\partial\hat{y}_1}{\partial\hat{x}_2} & \cdots & \frac{\partial\hat{y}_1}{\partial\hat{x}_m} \\    \frac{\partial\hat{y}_2}{\partial\hat{x}_1} & \frac{\partial\hat{y}_2}{\partial\hat{x}_2} & \cdots & \frac{\partial\hat{y}_2}{\partial\hat{x}_m} \\    \vdots & \vdots & \ddots & \vdots \\     \frac{\partial\hat{y}_m}{\partial\hat{x}_1} & \frac{\partial\hat{y}_m}{\partial\hat{x}_2} & \cdots & \frac{\partial\hat{y}_m}{\partial\hat{x}_m}\end{bmatrix}=\begin{bmatrix}\sigma(\hat{x}_1)(1-\sigma(\hat{x}_1)) & 0 & \cdots & 0 \\0 & \sigma(\hat{x}_2)(1-\sigma(\hat{x}_2))  & \cdots & 0 \\\vdots & \vdots & \ddots & \vdots \\ 0 & 0 & \cdots & \sigma(\hat{x}_m)(1-\sigma(\hat{x}_m)) \\\end{bmatrix}=\mathbf{1}\sigma(\mathbf{\hat{x}})^T\otimes(\mathbf{I}-\sigma(\mathbf{\hat{x}})\mathbf{1}^T)$

And the Jacobian matrix of $\mathbf{\hat{x}}$ with respect to $\mathbf{w}$:

$\mathbf{\frac{\partial \hat{x}}{\partial w}}=\begin{bmatrix}    \frac{\partial\hat{x}_1}{\partial w_1} & \frac{\partial\hat{x}_1}{\partial w_2} & \cdots & \frac{\partial\hat{x}_1}{\partial w_n} \\    \frac{\partial\hat{x}_2}{\partial w_1} & \frac{\partial\hat{x}_2}{\partial w_2} & \cdots & \frac{\partial\hat{x}_2}{\partial w_n} \\    \vdots & \vdots & \ddots & \vdots \\     \frac{\partial\hat{x}_m}{\partial w_1} & \frac{\partial\hat{x}_m}{\partial w_2} & \cdots & \frac{\partial\hat{x}_m}{\partial w_n}\end{bmatrix}=\mathbf{X}$

We can then compute the gradient of $L(\mathbf{y}, \mathbf{\hat{y}})$ with respect to ${\mathbf{\hat{x}}}$:

$\nabla_{\mathbf{\hat{x}}}L(\mathbf{y}, \mathbf{\hat{y}})=\begin{bmatrix}\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{x}_1} \\\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{x}_2} \\\vdots \\\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{x}_m} \\\end{bmatrix}=\begin{bmatrix}\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{y}_1}\frac{\partial\hat{y}_1}{\partial\hat{x}_1} + \frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{y}_2}\frac{\partial\hat{y}_2}{\partial\hat{x}_1}+\cdots+\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{y}_m}\frac{\partial\hat{y}_m}{\partial\hat{x}_1}\\\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{y}_1}\frac{\partial\hat{y}_1}{\partial\hat{x}_2} + \frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{y}_2}\frac{\partial\hat{y}_2}{\partial\hat{x}_2}+\cdots+\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{y}_m}\frac{\partial\hat{y}_m}{\partial\hat{x}_2}\\\vdots \\\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{y}_1}\frac{\partial\hat{y}_1}{\partial\hat{x}_m} + \frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{y}_2}\frac{\partial\hat{y}_2}{\partial\hat{x}_m}+\cdots + \frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{y}_m}\frac{\partial\hat{y}_m}{\partial\hat{x}_m}\\\end{bmatrix}=(\frac{\partial\mathbf{\hat{y}}}{\partial\mathbf{\mathbf{\hat{x}}}})^T\nabla_{\mathbf{\hat{y}}}L(\mathbf{y}, \mathbf{\hat{y}})$

And finally, the gradient of $L(\mathbf{y}, \mathbf{\hat{y}})$ with respect to ${\mathbf{w}}$:

$\nabla_{\mathbf{w}}L(\mathbf{y}, \mathbf{\hat{y}})=\begin{bmatrix}\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial w_1} \\\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial w_2} \\\vdots \\\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial w_n} \\\end{bmatrix}=\begin{bmatrix}\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{x}_1}\frac{\partial\hat{x}_1}{\partial w_1} + \frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{x}_2}\frac{\partial\hat{x}_2}{\partial w_1}+\cdots+\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{x}_m}\frac{\partial\hat{x}_m}{\partial w_1}\\\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{x}_1}\frac{\partial\hat{x}_1}{\partial w_2} + \frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{x}_2}\frac{\partial\hat{x}_2}{\partial w_2}+\cdots+\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{x}_m}\frac{\partial\hat{x}_m}{\partial w_2}\\\vdots \\\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{x}_1}\frac{\partial\hat{x}_1}{\partial w_n} + \frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{x}_2}\frac{\partial\hat{x}_2}{\partial w_n}+\cdots + \frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial\hat{x}_m}\frac{\partial\hat{x}_m}{\partial w_n}\\\end{bmatrix}=(\mathbf{\frac{\partial \hat{x}}{\partial w}})^T(\frac{\partial\mathbf{\hat{y}}}{\partial\mathbf{\mathbf{\hat{x}}}})^T\nabla_{\mathbf{\hat{y}}}L(\mathbf{y}, \mathbf{\hat{y}})=\mathbf{X}^T [\mathbf{1}\sigma(\mathbf{\hat{x}})^T\otimes(\mathbf{I}-\sigma(\mathbf{\hat{x}})\mathbf{1}^T)] ^ T \frac{2}{m}(\mathbf{y}-\mathbf{\hat{y}})$

We can see that $\nabla_{\mathbf{w}}L(\mathbf{y}, \mathbf{\hat{y}})\to0$ when $\sigma(\mathbf{\hat{x}})\to1$ or $\sigma(\mathbf{\hat{x}})\to0$ (whether or not the prediction is correct, the weight will not be updated if the model is confident about the prediction). Softmax and mean absolute error derive similar results. On the other hand, one can easily see that cross-entropy does not have this problem (substitute the first term in the final gradient with the derivative of cross-entropy).

### Binary Classification with Distance

If the activation function of the output layer is **linear** or **hyperbolic tangent** (in this case, $y_i=1$ for the sample being labelled as class $0$, $y_i=-1$ as class $1$, $f(x_i)>0$ suggests network predict the sample being classified as class A). The predictor makes correct prediction if $y_i\hat{y}_i>0$. However, the MSE and MAE increase when $y_i\hat{y}_i>1$. Therefore, MSE and MAE are not suitable to be used in this setting neither.

```python
y_pred = tf.linspace(-3., 3., 500)
y_true = tf.ones(500)

loss = tf.abs(y_true - y_pred)
fig = visualize(
    'y·ŷ and loss',
    y_true * y_pred,
    loss,
    export=False,
    x_axis_label='y·ŷ',
    **{'color': 'deepskyblue', 'legend_label': 'Mean Absolute Error'}
)
loss = tf.pow(y_true - y_pred, 2)
fig = visualize(
    'Mean Squared Error',
    y_true * y_pred,
    loss,
    fig=fig,
    export=False,
    **{'color': 'lightcoral', 'legend_label': 'Mean Squared Error'}
)
loss = tf.math.log(1 + np.exp(-1 * y_pred * y_true)) / np.log(2)
loss = -1 * y_true * tf.math.log(1 / (1 + tf.math.exp(-1 * y_pred))) / tf.math.log(2.)
fig = visualize(
    'Sigmoid with Cross Entropy',
    y_true * y_pred, loss,
    fig=fig,
    export=False,
    **{'color': 'limegreen', 'legend_label': 'Sigmoid with Cross Entropy', 'line_dash': 'dotted'}
)
loss = tf.maximum(tf.zeros_like(y_pred * y_true), tf.ones_like(y_pred * y_true) - y_pred * y_true) + 0.025
fig = visualize(
    'Hinge Loss',
    y_true * y_pred,
    loss,
    fig=fig,
    export=False,
    **{'color': 'blueviolet', 'legend_label': 'Hinge Loss', 'line_dash': 'dashed'}
)
loss = tf.where(y_true * y_pred < 0, tf.ones_like(y_true * y_pred), tf.zeros_like(y_true * y_pred))
fig = visualize(
    'Ideal Loss',
    y_true * y_pred,
    loss,
    fig=fig,
    **{'color': 'black', 'legend_label': 'Ideal Loss'}
)
```

```python

```


# Weyl Tensor calculations using Symbolic module

```python
import sympy
from sympy import cos, sin, sinh
from einsteinpy.symbolic import MetricTensor, WeylTensor

sympy.init_printing()
```

### Defining the Anti-de Sitter spacetime Metric

```python
syms = sympy.symbols("t chi theta phi")
t, ch, th, ph = syms
m = sympy.diag(-1, cos(t) ** 2, cos(t) ** 2 * sinh(ch) ** 2, cos(t) ** 2 * sinh(ch) ** 2 * sin(th) ** 2).tolist()
metric = MetricTensor(m, syms)
```

### Calculating the Weyl Tensor (with all indices covariant)

```python
weyl = WeylTensor.from_metric(metric)
weyl.tensor()
```

```python
weyl.config
```

    'llll'

docs/source/examples/Weyl Tensor symbolic calculation.ipynb


# 01: Introduction to Monte Carlo Methods

A great reference for this material is W. Krauth's excellent book ''Statistical Mechanics: Algorithms and Computations''

```python
import numpy as np
import matplotlib.pyplot as plt
%config InlineBackend.figure_format = 'retina'
%matplotlib inline
try: plt.style.use('./mc_notebook.mplstyle')
except: pass
```

## Estimating π

We will attempt to estimate $\pi$ by considering the unit circle embedded in a square of side 1

```python
plt.figure(figsize=(4,4))
cx = np.linspace(-1,1,1000)
cy = np.sqrt(1-cx**2)

plt.fill_between(cx,-cy,cy, color='gray')
plt.xlabel('x')
plt.ylabel('y')
plt.xticks([]);
plt.yticks([]);
```

We know the area of the circle is $A_\circ = \pi$ while the area of the square is $A_\square = 4$ thus we can compute $\pi$ as the ratio:

\begin{equation}
\pi = 4 \frac{A_\circ}{A_\square}
\end{equation}

This ratio can be estimated using a simple ''children's game'' of pebble toss.

### Direct Sampling

1. Randomly toss a pebble into the square.
2. Record the position where it lands $\vec{r} = (x,y)$
3. Determine whether the stone fell inside the circle: $r < 1$
4. Repeat $N$ times and determine the total number of hits $N_{\rm hits}$

We can then determine:

\begin{equation}
\frac{N_{\rm hits}}{N} = \frac{A_\circ}{A_\square} = \frac{\pi}{4}\, .
\end{equation}

```python
N = 2**14

# sample inside the square
x = np.random.uniform(-1,1,N)
y = np.random.uniform(-1,1,N)

# count the number of hits
N_hits = np.sum(x**2+y**2<1)
print('π ≃ %6.4f' % (4*N_hits/N))
```

    π ≃ 3.1648

```python
plt.figure(figsize=(4,4))
cx = np.linspace(-1,1,1000)
cy = np.sqrt(1-cx**2)

plt.plot(x,y, 'k', marker='o', linewidth=0, markersize=2, markeredgewidth=0)

plt.fill_between(cx,-cy,cy, color='gray')
plt.xlabel('x')
plt.ylabel('y')
plt.xticks([]);
plt.yticks([]);
```

### Markov Chain Sampling

Suppose that we are not able reach the entire area of the square by throwing pebbles.  An alternative sampling approach proceeds by:

1. Start in the corner of the squre $\vec{r} = (1,1)$
2. Throw a pebble
    * if the new pebble is inside the square, move to its position
    * if not, drop the pebble at your current location
3. Record the position (new or current) $\vec{r} = (x,y)$
4. Determine if $r < 1$
5. Repeat $N$ times and determine the relative number of hits

The hallmark of this type of sampling is that the $(n+1)^{\rm th}$ configuration only depends on the $n^{\rm th}$ one.

```python
N = 2**12
x = np.ones(N)
y = np.ones(N)
δ = 0.3

for n in range(1,N):
    # throw in a box of side δ
    Δx = np.random.uniform(-δ,δ)
    Δy = np.random.uniform(-δ,δ)

    # update only if the new position is inside the square
    x[n] = x[n-1] + Δx*(np.abs(x[n-1]+Δx) < 1)
    y[n] = y[n-1] + Δy*(np.abs(y[n-1]+Δy) < 1)

# count the number of hits
N_hits = np.sum(x**2+y**2<1)
print('π ≃ %6.4f' % (4*N_hits/N))
```

    π ≃ 3.0156

```python
plt.figure(figsize=(4,4))
cx = np.linspace(-1,1,1000)
cy = np.sqrt(1-cx**2)

plt.plot(x,y, 'k', marker='o', linewidth=0.2, markersize=2, markeredgewidth=0)

plt.fill_between(cx,-cy,cy, color='gray')
plt.xlabel('x')
plt.ylabel('y')
plt.xticks([]);
plt.yticks([]);
```

## What are we doing?

We are using different sampling schemes to compute the ratio of two-dimensional integrals:

\begin{align*}
\frac{A_\circ}{A_\square} = \frac{\pi}{4} &=
\frac{\int_{-1}^1 dx \int_{-\sqrt{1-x^2}}^{\sqrt{1-x^2}} dy}{\int_{-1}^1 dx \int_{-1}^1 dy}\\
&= \frac{\int_{-1}^1 dx \int_{-1}^1 dy\, \mathcal{O}(x,y)}{\int_{-1}^1 dx \int_{-1}^1 dy}
\end{align*}

where we have defined a function:

\begin{equation}
\mathcal{O}(x,y) =
\begin{cases}
1 &;& x^2 + y^2 < 1 \\
0 &;& \text{otherwise}
\end{cases}.
\end{equation}

This integral can be re-written as an ''average'' of the function $\mathcal{O}(x,y)$ over the square with side $2$ by defining the uniform probability distribution:

\begin{equation}
\pi(x,y) = \frac{1}{{\int_{-1}^1 dx \int_{-1}^1 dy}}
\end{equation}

which yields:

\begin{align*}
\langle \mathcal{O} \rangle &= \int_{-1}^1 dx \int_{-1}^1 dy\, \pi (x,y)\, \mathcal{O}(x,y) \newline
&\simeq \frac{1}{N} \sum_{n=0}^{N-1} \mathcal{O}(x_n,y_n) = \frac{N_{\rm hits}}{N} \simeq \frac{\pi}{4}
\end{align*}

where $x_n,y_n \in \pi(x,y)$; i.e. in the last line the probablity distribution doesn't explicity apear, **it is sampled**!

Using this idea we can re-write **any** d-dimensional integral as a Monte Carlo sampling problem:

\begin{align*}
I = \frac{\int_{\Omega} d^d x\, f(x_1,\ldots,x_d)}{\int_\Omega d^d x\,} & = \int_\Omega d^d x\, \pi(x_1,\ldots,x_n) \, f(x_1,\ldots,x_n)
= \langle f \rangle \newline
&\simeq \frac{1}{N} \sum_n f(\mathbf{x}_n)
\end{align*}

where $\mathbf{x}_n$ is a vector sampled accoridng to the $d$-dimensional uniform distribution $\pi$.

```python

```


# Домашнее задание "Производная функции нескольких аргументов".

```python
import numpy as np
from sympy import *
from scipy.optimize import approx_fprime
```

### Уровень 0:

Посчитайте частные производные функций:

1) $f(x,y)=2x^2y^3 + 1/x + y^2x + 7$, а для этой функции также градиент в точке $(1,2)$

2) $f(x,y)=x^2y - sin(xy) + cos(x^2) + 6y$

```python
x = symbols('x')
y = symbols('y')
```

```python
# first function
f = 2 * x**2 * y**3 + 1/x + y**2 * x + 7
```

```python
f1x = diff(f, x)
print(format(f1x))
```

    4*x*y**3 + y**2 - 1/x**2

```python
f1y = diff(f, y)
print(format(f1y))
```

    6*x**2*y**2 + 2*x*y

```python
# second function
f = x**2 * y - sin(x * y) + cos(x**2) + 6 * y
```

```python
f1x = diff(f, x)
print(format(f1x))
```

    2*x*y - 2*x*sin(x**2) - y*cos(x*y)

```python
f1y = diff(f, y)
print(format(f1y))
```

    x**2 - x*cos(x*y) + 6

```python
# gradient of the first function
def func(t):
    return 2 * t[0] ** 2 * t[1] ** 3 + 1 / t[0] + t[1] ** 2 * t[0] + 7

eps = np.sqrt(np.finfo(float).eps)
grad = approx_fprime([1, 2], func, [eps, eps])
print(grad)
```

    [35.00000024 28.00000024]

### Уровень 1:

Градиентный спуск своими руками:

```python
def f(t):
    return (t[0] ** 2) + (t[1] ** 2)
```

```python
x0 = np.array([100, 200])
learning_rate = 0.1
```

```python
eps = np.sqrt(np.finfo(float).eps)
x = x0
for i in range(100):
    # Calculate gradient
    grad = approx_fprime(x, f, [eps, eps])

    # Update x with gradient
    x = np.array([
        x[0] - learning_rate * grad[0],
        x[1] - learning_rate * grad[1]
    ])

print("Minimum is in: ", x)
print("Minimum value is: ", f(x))
```

    Minimum is in:  [1.29197811e-08 3.32901401e-08]
    Minimum value is:  1.2751541739575636e-15

Проверка с помощью встроенной функции numpy:

```python
from scipy.optimize import minimize

res = minimize(f, x0, method='nelder-mead', options={'xtol': 1e-8, 'disp': True})
print(res)
```

    Optimization terminated successfully.
             Current function value: 0.000000
             Iterations: 85
             Function evaluations: 164
     final_simplex: (array([[ 3.31391559e-09, -1.82888492e-09],
           [-3.91522747e-09, -3.79451217e-09],
           [-4.97134432e-09,  7.91645290e-09]]), array([1.43268566e-17, 2.97273287e-17, 8.73844908e-17]))
               fun: 1.4326856592347756e-17
           message: 'Optimization terminated successfully.'
              nfev: 164
               nit: 85
            status: 0
           success: True
                 x: array([ 3.31391559e-09, -1.82888492e-09])

```python

```


# Matrix addition

```python
import numpy as np
# Initializing an array
x = np.array([[1, 1], [2, 2]])
y = np.array([[10, 10], [20, 20]])
```

```python
x+y
```

    array([[11, 11],
           [22, 22]])

```python
np.add(x,y)
```

    array([[11, 11],
           [22, 22]])

# Matrix multiplication

```python
x = np.array([[1, 1], [2, 2]])
```

```python
2*x
```

    array([[2, 2],
           [4, 4]])

# Matrix arithmetic

```python
# subtraction
x-y
```

    array([[ -9,  -9],
           [-18, -18]])

```python
x*y
```

    array([[10, 10],
           [40, 40]])

```python
# division
x/y
```

    array([[ 0.1,  0.1],
           [ 0.1,  0.1]])

```python
# exponential
x**y
```

    array([[      1,       1],
           [1048576, 1048576]], dtype=int32)

# Matrix matrix multiplication

```python
x=np.array([[1, 1], [2, 2]])
y=np.array([[10, 10], [20, 20]])
```

```python
np.dot(x,y)
```

    array([[30, 30],
           [60, 60]])

# Matrix transpose

```python
# Initialize a matrix
A = np.array([[1,2],[3,4]])
A.transpose()
```

    array([[1, 3],
           [2, 4]])

# Matrix inversion

```python
from scipy import linalg
```

```python
A = np.array([[1,2],[3,4]])
```

```python
linalg.inv(A)
```

    array([[-2. ,  1. ],
           [ 1.5, -0.5]])

# Solving linear systems using matrices

```python
from scipy import linalg
```

```python
A=[[1,3,5],[2,5,1],[2,3,8]]
b=[[10],[8],[3]]
```

```python
np.dot(linalg.inv(A),b)
```

    array([[-9.28],
           [ 5.16],
           [ 0.76]])

```python
linalg.solve(A, b)
```

    array([[-9.28],
           [ 5.16],
           [ 0.76]])

# Determinant of a matrix

```python
linalg.det(A)
```

    -25.000000000000004

# Null space of a matrix

```python
import sympy
M = sympy.Matrix([[1, 2, 3, 0, 0], [4, 10, 0, 0, 1]])
```

```python
M.nullspace()
```

    [Matrix([
     [-15],
     [  6],
     [  1],
     [  0],
     [  0]]), Matrix([
     [0],
     [0],
     [0],
     [1],
     [0]]), Matrix([
     [   1],
     [-1/2],
     [   0],
     [   0],
     [   1]])]

```python
x=[
[-15],
[  6],
[  1],
[  0],
[  0]]

```

```python
np.dot(np.array(M),x)
```

    array([[0],
           [0]], dtype=object)

# Range of a matrix

```python
x=np.array([[1,100],[2,200]])
```

```python
np.ptp(x)
```

    199

```python
np.ptp(x,axis=1)
```

    array([ 99, 198])

```python
np.ptp(x,axis=0)
```

    array([  1, 100])

# Rank and nullity of a matrix

```python
a=np.array([[3,2,-1],[2,-3,-5],[-1,-4,-3]])
```

```python
np.linalg.matrix_rank(a)
```

    2

```python
a.shape[1]
```

    3

```python
a.shape[1] - np.linalg.matrix_rank(a)
```

    1

# LU decomposition of a matrix

```python
A = np.array([[2., 1., 1.], [1., 3., 2.], [1., 0., 0.]])
```

```python
import scipy.linalg as linalg
```

```python
P, L, U = linalg.lu(A)
```

```python
print(L)
```

    [[ 1.   0.   0. ]
     [ 0.5  1.   0. ]
     [ 0.5 -0.2  1. ]]

```python
print(U)
```

    [[ 2.   1.   1. ]
     [ 0.   2.5  1.5]
     [ 0.   0.  -0.2]]

```python
np.dot(L,U)
```

    array([[  2.00000000e+00,   1.00000000e+00,   1.00000000e+00],
           [  1.00000000e+00,   3.00000000e+00,   2.00000000e+00],
           [  1.00000000e+00,   0.00000000e+00,  -2.77555756e-17]])

# QR decomposition of a matrix

```python
import scipy.linalg as linalg
import numpy as np

```

```python
A = np.array([ [2., 1., 1.], [1., 3., 2.], [1., 0., 0]])
```

```python
Q, R = linalg.qr(A)
```

```python
print(Q)
```

    [[-0.81649658  0.27602622 -0.50709255]
     [-0.40824829 -0.89708523  0.16903085]
     [-0.40824829  0.34503278  0.84515425]]

```python
print(R)
```

    [[-2.44948974 -2.04124145 -1.63299316]
     [ 0.         -2.41522946 -1.51814423]
     [ 0.          0.         -0.16903085]]

```python
print(np.dot(Q,Q.T))
```

    [[  1.00000000e+00  -1.05102508e-17  -3.90671821e-17]
     [ -1.05102508e-17   1.00000000e+00   2.51394744e-17]
     [ -3.90671821e-17   2.51394744e-17   1.00000000e+00]]

```python
print(np.dot(Q,R))
```

    [[  2.00000000e+00   1.00000000e+00   1.00000000e+00]
     [  1.00000000e+00   3.00000000e+00   2.00000000e+00]
     [  1.00000000e+00   1.11022302e-16  -7.63527929e-17]]

# Eigenvalue and eigenvector of a matrix

```python
a = np.array([[1, 2], [3, 4]])
```

```python
la, v = linalg.eig(a)
```

```python
print(la)
```

    [-0.37228132+0.j  5.37228132+0.j]

```python
print(v)
```

    [[-0.82456484 -0.41597356]
     [ 0.56576746 -0.90937671]]

```python
np.dot(a,v)
```

    array([[ 0.30697009, -2.23472698],
           [-0.21062466, -4.88542751]])

```python
np.dot(la,v)
```

    array([ 3.34643208+0.j, -4.73056832+0.j])

# Diagonalizing a matrix

```python
import numpy as np
```

```python
a= np.array([1,2,3])
```

```python
A=np.diag(a)
print(A)
```

    [[1 0 0]
     [0 2 0]
     [0 0 3]]

```python
la, v = linalg.eig(A)
print(la)
print(v)

```

    [ 1.+0.j  2.+0.j  3.+0.j]
    [[ 1.  0.  0.]
     [ 0.  1.  0.]
     [ 0.  0.  1.]]

```python
a = np.array([[1, 2,0], [2,1,0],[0,0,-3]])
la, v = linalg.eig(a)
print(la)
print(v)
```

    [ 3.+0.j -1.+0.j -3.+0.j]
    [[ 0.70710678 -0.70710678  0.        ]
     [ 0.70710678  0.70710678  0.        ]
     [ 0.          0.          1.        ]]

```python
np.dot(np.dot(np.linalg.inv(v),a),v)
```

    array([[  3.00000000e+00,   4.44089210e-16,   0.00000000e+00],
           [  6.66133815e-16,  -1.00000000e+00,   0.00000000e+00],
           [  0.00000000e+00,   0.00000000e+00,  -3.00000000e+00]])

```python
np.diag(la)
```

    array([[ 3.+0.j,  0.+0.j,  0.+0.j],
           [ 0.+0.j, -1.+0.j,  0.+0.j],
           [ 0.+0.j,  0.+0.j, -3.+0.j]])

```python
np.dot(np.dot(np.linalg.inv(v),a),v)-np.diag(la)
```

    array([[ -4.44089210e-16+0.j,   4.44089210e-16+0.j,   0.00000000e+00+0.j],
           [  6.66133815e-16+0.j,  -4.44089210e-16+0.j,   0.00000000e+00+0.j],
           [  0.00000000e+00+0.j,   0.00000000e+00+0.j,   0.00000000e+00+0.j]])

# Jordan form of a matrix

```python
import numpy as np
from sympy import Matrix

```

```python
a = np.array([[5, 4, 2, 1], [0, 1, -1, -1], [-1, -1, 3, 0], [1, 1, -1, 2]])
```

```python
m = Matrix(a)
```

```python
P, J = m.jordan_form()
```

```python
J
```

    Matrix([
    [1, 0, 0, 0],
    [0, 2, 0, 0],
    [0, 0, 4, 1],
    [0, 0, 0, 4]])

# SVD of a matrix

```python
a = np.array([[1, 2], [3, 4]])
```

```python
from scipy import linalg
```

```python
linalg.svd(a)
```

    (array([[-0.40455358, -0.9145143 ],
            [-0.9145143 ,  0.40455358]]),
     array([ 5.4649857 ,  0.36596619]),
     array([[-0.57604844, -0.81741556],
            [ 0.81741556, -0.57604844]]))

```python
U = np.array([[-0.40455358, -0.9145143 ],
        [-0.9145143 ,  0.40455358]])
sigma = np.array([[ 5.4649857 ,  0],
                  [0,0.36596619]])
V=np.array([[-0.57604844, -0.81741556],
        [ 0.81741556, -0.57604844]])

```

```python
np.dot(np.dot(U,sigma),V)
```

    array([[ 0.99999999,  1.99999998],
           [ 3.00000003,  4.00000001]])

# Sparse matrix

```python
import numpy as np
from scipy import sparse

```

```python
A = np.array([[1,2,0],[0,0,3],[1,0,4]])
```

```python
sA = sparse.csr_matrix(A)
```

```python
print(sA)
```

      (0, 0)	1
      (0, 1)	2
      (1, 2)	3
      (2, 0)	1
      (2, 2)	4

```python
from scipy.sparse import lil_matrix
from scipy.sparse.linalg import spsolve
from numpy.linalg import solve, norm
from numpy.random import rand

```

```python
A = lil_matrix((10000, 10000))
```

```python
A[0, :100] = rand(100)
A[1, 100:200] = A[0, :100]

```

```python
A.setdiag(rand(10000))
```

```python
b = rand(10000)
```

```python
A = A.tocsr()
```

```python
x = spsolve(A, b)
```

```python
import time
start_time=time.time()
x = spsolve(A.tocsr(), b)
end_time=time.time()
print(end_time-start_time)
```

    0.00500035285949707

```python
import time
start_time=time.time()
x = solve(A.toarray(), b)
end_time=time.time()
print(end_time-start_time)
```

    6.414999961853027

```python

```


Showing that the pseudoinverse solution minimizes the sums of squares of the estimated parameters $B$ for $Y = X B + \epsilon$, in the case of a particular rank deficient matrix, from a tutorial by Cyril Pernet.

```
import numpy as np
import numpy.linalg as npl
```

```
# Setting up the design
a_block = np.array([-0.1, 0, 0.1])
baseline = 10
activation = 11
n_on_off = 3
# The data
on_off = np.hstack((a_block + baseline, a_block + activation))
Y = np.tile(on_off, (n_on_off,))
Y = Y[:, None] # Make Y into a column vector
# Regressors in the redundant design
x_on = np.hstack((np.zeros(len(a_block)), np.ones(len(a_block))))
x_off = 1 - x_on
X_over_part = np.column_stack((x_on, x_off, np.ones_like(x_on)))
X_over = np.tile(X_over_part, (n_on_off, 1))
Y, X_over
```

    (array([[  9.9],
            [ 10. ],
            [ 10.1],
            [ 10.9],
            [ 11. ],
            [ 11.1],
            [  9.9],
            [ 10. ],
            [ 10.1],
            [ 10.9],
            [ 11. ],
            [ 11.1],
            [  9.9],
            [ 10. ],
            [ 10.1],
            [ 10.9],
            [ 11. ],
            [ 11.1]]), array([[ 0.,  1.,  1.],
            [ 0.,  1.,  1.],
            [ 0.,  1.,  1.],
            [ 1.,  0.,  1.],
            [ 1.,  0.,  1.],
            [ 1.,  0.,  1.],
            [ 0.,  1.,  1.],
            [ 0.,  1.,  1.],
            [ 0.,  1.,  1.],
            [ 1.,  0.,  1.],
            [ 1.,  0.,  1.],
            [ 1.,  0.,  1.],
            [ 0.,  1.,  1.],
            [ 0.,  1.,  1.],
            [ 0.,  1.,  1.],
            [ 1.,  0.,  1.],
            [ 1.,  0.,  1.],
            [ 1.,  0.,  1.]]))

The design matrix `X_over` is not full rank:

```
npl.matrix_rank(X_over)
```

    2

That means it has a *null space* - a set of vectors $k$ for which $X k = 0$

```
import sympy
sympy.init_printing(use_latex=True)
```

```
X = sympy.Matrix(X_over)
null_space = X.nullspace()
null_space
```

The null space is a list of all the basis vectors spanning the null space:

```
type(null_space)
```

    list

In our case, the matrix is only rank deficient by one column, and there is only one basis vector for the null space:

```
len(null_space)
```

```
null_vector = null_space[0]
null_vector
```

The null space is all vectors $k$ formed by multiplying a scalar $a$ times the null space basis vector `null_vector`.

This in turn means that if we find some parameter vector $B$ as the solution for $Y = X B$, then $Y = X B + X k = X (B + k)$ also.

For example, the parameters that went into forming $Y$ in our case are $1, 0, 10$ (activation, baseline, mean signal).  Not surprisingly, this vector minimizes the sum of squares for $\epsilon = Y - X B$:

```
beta_in = np.array([[1, 0, 10]]).T
Y - X_over.dot(beta_in)
```

    array([[-0.1],
           [ 0. ],
           [ 0.1],
           [-0.1],
           [ 0. ],
           [ 0.1],
           [-0.1],
           [ 0. ],
           [ 0.1],
           [-0.1],
           [ 0. ],
           [ 0.1],
           [-0.1],
           [ 0. ],
           [ 0.1],
           [-0.1],
           [ 0. ],
           [ 0.1]])

But we get exactly the same fit by adding any multiple of `null_vector` to $B$:

```
null_numeric = np.array(null_vector).astype(float)
Y - X_over.dot(beta_in + null_numeric * 42)
```

    array([[-0.1],
           [ 0. ],
           [ 0.1],
           [-0.1],
           [ 0. ],
           [ 0.1],
           [-0.1],
           [ 0. ],
           [ 0.1],
           [-0.1],
           [ 0. ],
           [ 0.1],
           [-0.1],
           [ 0. ],
           [ 0.1],
           [-0.1],
           [ 0. ],
           [ 0.1]])

This means that the residuals $\epsilon$ cannot distinguish between any of the possible parameter vectors $B + k$, where $k$ is a vector in the null space of $X$.

The pseudoinverse operator chooses one of the infinite possible solutions from $B + k$:

```
beta_hat = npl.pinv(X_over).dot(Y)
beta_hat
```

    array([[ 4.],
           [ 3.],
           [ 7.]])

In fact, this is the solution that minimizes the sum of squares of the chosen parameters in `beta_hat`:

```
# The fitted parameters can be beta_in (the input paramters) + a * null_vector
a = sympy.symbols('a')
fitted_params = beta_in + a * null_vector
fitted_params
```

```
# pinv minimized the sums of squares of the fitted parameters
ss = sympy.simplify(sum(e ** 2 for e in fitted_params))
ss
```

```
# What multiple `a` of the null vector minimizes the sums of squares?
a_min_ss = sympy.solve(sympy.diff(ss), a)[0]
a_min_ss
```

```
# That's what we get from the pseudoinverse model fit
sympy.Matrix(beta_in + a_min_ss * null_vector)
```


# Chebychev polynomial and spline approximantion of various functions

**Randall Romero Aguilar, PhD**

This demo is based on the original Matlab demo accompanying the  Computational Economics and Finance 2001 textbook by Mario Miranda and Paul Fackler.

Original (Matlab) CompEcon file: **demapp05.m**

Running this file requires the Python version of CompEcon. This can be installed with pip by running

    !pip install compecon --upgrade

Last updated: 2021-Oct-01

## About

Demonstrates Chebychev polynomial, cubic spline, and linear spline approximation for the following functions
\begin{align}
    y &= 1 + x + 2x^2 - 3x^3 \\
    y &= \exp(-x) \\
    y &= \frac{1}{1+25x^2} \\
    y &= \sqrt{|x|}
\end{align}

## Initial tasks

```python
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from compecon import BasisChebyshev, BasisSpline, nodeunif
```

### Functions to be approximated

```python
funcs = [lambda x: 1 + x + 2 * x ** 2 - 3 * x ** 3,
         lambda x: np.exp(-x),
         lambda x: 1 / ( 1 + 25 * x ** 2),
         lambda x: np.sqrt(np.abs(x))]

fst = ['$y = 1 + x + 2x^2 - 3x^3$', '$y = \exp(-x)$',
       '$y = 1/(1+25x^2)$', '$y = \sqrt{|x|}$']
```

Set degree of approximation and endpoints of approximation interval

```python
n = 7   # degree of approximation
a = -1  # left endpoint
b = 1   # right endpoint
```

Construct uniform grid for error ploting

```python
x = np.linspace(a, b, 2001)
```

```python
def subfig(f,  title):

    # Construct interpolants
    C = BasisChebyshev(n, a, b, f=f)
    S = BasisSpline(n, a, b, f=f)
    L = BasisSpline(n, a, b, k=1, f=f)

    data = pd.DataFrame({
        'actual': f(x),
        'Chebyshev': C(x),
        'Cubic Spline': S(x),
        'Linear Spline': L(x)},
        index = x)

    fig1, axs = plt.subplots(2,2, figsize=[12,6], sharex=True, sharey=True)
    fig1.suptitle(title)
    data.plot(ax=axs, subplots=True)

    errors = data[['Chebyshev', 'Cubic Spline']].subtract(data['actual'], axis=0)

    fig2, ax = plt.subplots(figsize=[12,3])
    fig2.suptitle("Approximation Error")
    errors.plot(ax=ax)

```

## Polynomial
$y = 1 + x + 2x^2 - 3x^3$

```python
subfig(lambda x: 1 + x + 2*x**2 - 3*x**3, '$y = 1 + x + 2x^2 - 3x^3$')
```

## Exponential
$y = \exp(-x)$

```python
subfig(lambda x: np.exp(-x),'$y = \exp(-x)$')
```

## Rational
$y = 1/(1+25x^2)$

```python
subfig(lambda x: 1 / ( 1 + 25 * x ** 2),'$y = 1/(1+25x^2)$')
```

## Kinky
$y = \sqrt{|x|}$

```python
subfig(lambda x: np.sqrt(np.abs(x)), '$y = \sqrt{|x|}$')
```

_build/html/_sources/notebooks/app/05 Chebychev polynomial and spline approximantion of various functions.ipynb


Basic Math

```python
import math
```

```python
1 + 1 # Addition
```

    2

```python
2 - 1 # Subtraction
```

    1

```python
8 - 9
```

    -1

```python
2 * 8 # Multiplication
```

    16

```python
8 / 4 # Division
```

    2.0

```python
20 // 6 # Division, removed devimal values
```

    3

```python
20 % 6 # Shows remainder for division
```

    2

```python
3  ** 2 # ** means raised to the power
```

    9

```python
16 ** (1/2) # Squared root
```

    4.0

```python
8 ** (1/3) # Squared cube
```

    2.0

```python
abs(-8) # Absolute Value
```

    8

```python
7%3 # 7 mod 3 - modulo
```

    1

```python
# round decimal point to 2 places
round(3.1468, 2)
```

    3.15

```python
# Logarithmic of x to base b
# log(x, b)
math.log(64, 2)
```

    6.0

PEMDAS - Parentheses, Exponents, Multiply, Divide, Add, Subtract

```python
3 + 3 * 3
```

    12

```python
(8 - 5) * 2
```

    6

```python
12 - 8 + 4 * 2 - (6 / 3)
```

    10.0

Fractions

```python
from fractions import Fraction
# Fraction(numerator, denominator)

d = (0.25, 0.5, 0.75, 1.25)
for the_decimal in d:
    print(Fraction(the_decimal))
```

    1/4
    1/2
    3/4
    5/4

```python
from decimal import Decimal

f = (1/4, 1/2, 3/4, 7/2)
for the_fraction in f:
    print(Decimal(the_fraction))
```

    0.25
    0.5
    0.75
    3.5

```python
Fraction(1,7) + Fraction(2,7)
```

    Fraction(3, 7)

```python
Fraction(1/2) + Fraction(1/4)
```

    Fraction(3, 4)

```python
Fraction(1/2)  + 6 * 3
```

    Fraction(37, 2)

```python
Fraction(1/2) * Fraction(1/4)
```

    Fraction(1, 8)

Complex Numbers

```python
# For complex number use "j" instead of "i"
# Example complex number is "6 + 3i";however, in Python as "6 + 3j"
a = 4 + 2i # Does not work
type(a)
```

```python
a = 4 + 2j
type(a)
```

    complex

```python
a = complex(4, 8)
print("Complex Number:", a)
```

    Complex Number: (4+8j)

```python
b = 2 + 2j
c = 3 + 3j
```

```python
a + b
```

    (6+10j)

```python
a - b
```

    (2+6j)

```python
a * b
```

    (-8+24j)

```python
a / b
```

    (3+1j)

```python
# Show which number is real and imaginary numbers
print("Real number:", a.real)
print("Imaginary number:", a.imag)
```

    Real number: 4.0
    Imaginary number: 8.0

```python
from sympy import *

x = Symbol('x')
y = Symbol('y')
z = Symbol('z')
```

```python
a = x*y + x*y
a
```

    2*x*y

```python
c = (x + 6)*(x + 4)
c
```

    (x + 4)*(x + 6)

```python
# Expand the factor
expand(c)
```

    x**2 + 10*x + 24

```python
d = (x+y)**2
expand(d)
```

    x**2 + 2*x*y + y**2

```python
e = (x-y)**2
expand(e)
```

    x**2 - 2*x*y + y**2

```python
g = (x+y)**3
expand(g)
```

    x**3 + 3*x**2*y + 3*x*y**2 + y**3

```python
from sympy import pprint # Shows math expression
pprint(g)
```

           3
    (x + y)

```python
u = 3*x**3 + 6*x*y - 3*y**3
pprint(u)
```

       3              3
    3⋅x  + 6⋅x⋅y - 3⋅y

```python
from math import sqrt
import cmath # has complex number
# Quandratic Formula
# https://en.wikipedia.org/wiki/Quadratic_formula
# General Equation ax^2 + bx + c = 0
# x = -b + sqrt(b**2 - 4ac) / 2a
# x = -b - sqrt(b**2 - 4ac) / 2a
a = 2
b = 4
c = 8

# Check if the discriminant is positive, negative, or zero
d = b**2-4*a*c # discriminant

if d < 0:
    print("No Solutions!") # It has imaginary number
elif d == 0:
    x = -b / (2*a)
    print("One solution:", x) # Same solutions
else:
    x1 = (-b+sqrt((b**2)-(4*(a*c))))/(2*a)
    x2 = (-b-sqrt((b**2)-(4*(a*c))))/(2*a)
    print("Two solutions: ", x1, " and", x2)
```

    No Solutions!

```python
a = 9
b = 12
c = 4
d = b**2-4*a*c # discriminant

if d < 0:
    print("No Solutions!")
elif d == 0:
    x = -b / (2*a)
    print("One solution:", x)
else:
    x1 = (-b+sqrt((b**2)-(4*(a*c))))/(2*a)
    x2 = (-b-sqrt((b**2)-(4*(a*c))))/(2*a)
    print("Two solutions: ", x1, " and", x2)
```

    One solution: -0.6666666666666666

```python
a = 1
b = 4
c = 3
d = b**2-4*a*c # discriminant

if d < 0:
    print("No Solutions!")
elif d == 0:
    x = -b / (2*a)
    print("One solution:", x)
else:
    x1 = (-b+sqrt((b**2)-(4*(a*c))))/(2*a)
    x2 = (-b-sqrt((b**2)-(4*(a*c))))/(2*a)
    print("Two solutions: ", x1, " and", x2)
```

    Two solutions:  -1.0  and -3.0

```python
# This shows real and imaginary numbers

a = 3
b = 4
c = 2
d = b**2-4*a*c # discriminant

if d < 0:
    x1=(-b+cmath.sqrt(d))/(2*a)
    x2=(-b-cmath.sqrt(d))/(2*a)
    print("Complex Number")
    print("x1=",x1," and x2=",x2)
elif d > 0:
    x1=(-b+sqrt(d))/(2*a)
    x2=(-b-sqrt(d))/(2*a)
    print("Real Number")
    print("x1=",x1," and x2=",x2)
elif d==0:
    x=-b/(2*a)
    print("Equal to Zero" )
    print("x=",x)
else:
    print("This is NOT Quadratic Equation!")
```

    Complex Number
    x1= (-0.6666666666666666+0.47140452079103173j)  and x2= (-0.6666666666666666-0.47140452079103173j)

```python

a = 3
b = 4
c = 2
d = b**2-4*a*c # discriminant

if d < 0:
    x1=(-b+cmath.sqrt(d))/(2*a)
    x2=(-b-cmath.sqrt(d))/(2*a)
    print("Complex Number")
    print("x1=",x1.real, x1.imag,"and x2=",x2.real, x2.imag)
elif d > 0:
    x1=(-b+sqrt(d))/(2*a)
    x2=(-b-sqrt(d))/(2*a)
    print("Real Number")
    print("x1=",x1," and x2=",x2)
elif d==0:
    x=-b/(2*a)
    print("Equal to Zero" )
    print("x=",x)
else:
    print("This is NOT Quadratic Equation!")
```

    Complex Number
    x1= -0.6666666666666666 0.47140452079103173 and x2= -0.6666666666666666 -0.47140452079103173

Graphing

```python
import matplotlib.pyplot as plt
```

```python
y = [1,2,3,4,5,6]   # List of numbers
plt.plot(y)         # Plot graph
plt.show()          # Show the plot
```

```python
# Parabola
a = []
b = []
for x in range(-20,20,1):
    y=x**2
    a.append(x)
    b.append(y)

plt.plot(a,b)
plt.show()
```

```python
# Trignometry
import math

def cos(x):
    return cos(radians(x))
def sin(x):
    return sin(radians(x))
def tan(x):
    return tan(radians(x))
def arc_tan(x):
    return math.degrees(atan(x))
def arc_sine(x):
    return math.degrees(asin(x))
def arc_cos(x):
    return math.degrees(acos(x))

print("Radians")
print(math.cos(180))
print(math.sin(90))
print(math.tan(45))
print("Degrees")
print(arc_tan(1))
print(arc_sine(1))
print(arc_cos(1))
```

    Radians
    -0.5984600690578581
    0.8939966636005579
    1.6197751905438615
    Degrees
    45.0
    90.0
    0.0