```python
import os

import numpy as np
import pandas as pd

import pint
import sympy

import sissotools
from sissotools import io
from sissotools import dimensional_analysis
from sissotools import regression
from sissotools import plotting
```

```python
%matplotlib inline
```

# I/O Options

```python
input_filename = "./data_AD.txt"
working_directory = "./demo"
```

```python
if not os.path.isdir(working_directory):
    os.mkdir(working_directory)
```

# Read Data

```python
# parse tab-separated-value file
df_ad = io.read_tsv(input_filename)
```

```python
df_ad
```

      Tc
      wlog
      w1
      w2
      wph
      eta
      lambd
      mus

      material

      Pb
      7.20
      56.0
      60.0
      65.0
      110.0
      2.4
      1.550
      0.105

      In
      3.40
      68.0
      79.0
      89.0
      179.0
      1.3
      0.805
      0.097

      Sn
      3.75
      99.0
      110.0
      121.0
      209.0
      2.2
      0.720
      0.092

      Hg
      4.19
      29.0
      38.0
      49.0
      162.0
      1.4
      1.600
      0.098

      Tl
      2.36
      52.0
      58.0
      64.0
      127.0
      1.2
      0.795
      0.111

      Ta
      4.48
      132.0
      140.0
      148.0
      228.0
      4.9
      0.690
      0.093

      alpha-Ga
      8.56
      55.0
      77.0
      101.0
      291.0
      2.1
      1.620
      0.095

      beta-Ga
      5.90
      87.0
      108.0
      129.0
      285.0
      2.0
      0.970
      0.092

      Tl0.9Bi0.1
      2.30
      48.0
      55.0
      62.0
      120.0
      1.1
      0.780
      0.099

      Pb0.4Tl0.6
      4.60
      48.0
      56.0
      62.0
      121.0
      1.6
      1.150
      0.094

      Pb0.6Tl0.4
      5.90
      50.0
      57.0
      62.0
      119.0
      2.0
      1.380
      0.103

      Pb0.8Tl0.2
      6.80
      50.0
      56.0
      61.0
      116.0
      2.1
      1.530
      0.101

      Pb0.6Tl0.2Bi0.2
      7.26
      48.0
      53.0
      58.0
      112.0
      2.3
      1.810
      0.111

      Pb0.9Bi0.1
      7.65
      50.0
      56.0
      60.0
      108.0
      2.2
      1.660
      0.081

      Pb0.8Bi0.2
      7.95
      46.0
      52.0
      57.0
      109.0
      2.3
      1.880
      0.093

      Pb0.7Bi0.3
      8.45
      47.0
      52.0
      57.0
      110.0
      2.4
      2.010
      0.092

      Pb0.65Bi0.35
      8.95
      45.0
      50.0
      55.0
      110.0
      2.4
      2.130
      0.093

      In0.9Tl0.1
      3.28
      63.0
      75.0
      86.0
      176.0
      1.4
      0.850
      0.103

      In0.73Tl0.27
      3.36
      55.0
      67.0
      77.0
      166.0
      1.4
      0.930
      0.110

      In0.67Tl0.33
      3.26
      57.0
      68.0
      79.0
      167.0
      1.4
      0.900
      0.110

      In0.57Tl0.43
      2.60
      53.0
      64.0
      74.0
      165.0
      1.3
      0.850
      0.117

      In0.5Tl0.5
      2.52
      53.0
      64.0
      73.0
      163.0
      1.3
      0.830
      0.110

      In0.27Tl0.73
      3.64
      42.0
      53.0
      63.0
      151.0
      1.4
      1.090
      0.094

      In0.17Tl0.83
      3.19
      45.0
      55.0
      63.0
      144.0
      1.3
      0.980
      0.101

      In0.07Tl0.93
      2.77
      49.0
      56.0
      63.0
      131.0
      1.3
      0.890
      0.107

      In2Bi
      5.60
      46.0
      57.0
      67.0
      174.0
      1.6
      1.400
      0.096

      Sb2Tl7
      5.20
      37.0
      48.0
      58.0
      134.0
      1.6
      1.430
      0.102

      Bi2Tl
      6.40
      47.0
      53.0
      59.0
      120.0
      2.1
      1.630
      0.101

      alpha-Pb0.45Bi0.55
      7.00
      29.0
      38.0
      47.0
      128.0
      2.1
      2.590
      0.116

# Data Inspection

```python
# consider the target and a subset of features
column_names = ["Tc", "lambd", "wlog", "w2", "mus"]
```

```python
# substitutions for visualization
latex_subs = {"mus": r"\mu^{\ast}",
              "wlog": r"\omega_{\mathrm{log}}",
              "w2": r"\omega_2",
              'lambd': r"\lambda",
              'Tc': r"T_c"}
label_substitutions = {k: r"${}$".format(v) for k, v in latex_subs.items()}
```

```python
# visualize target and feature distributions
fig, ax = plotting.plot_distributions(df_ad,
                                      column_names,
                                      label_substitutions=label_substitutions)
fig.tight_layout()
```

# Unit checking

```python
# Consider known equations (McMillan 1968, Allen and Dynes 1975)
expr_mm = r'wlog / 1.2 * exp((-1.04 * (1 + lambd)) / (lambd - mus * (1 + 0.62 * lambd)))'

expr_f1 = r'((1 + ((lambd/(2.46*(1 + 3.8 * mus)))**1.5))**(1/3))'
expr_f2 = r'(1 + (((w2/wlog) - 1) * lambd**2)/(lambd**2 + (1.82*(1 + 6.3*mus)*(w2/wlog))**2))'
expr_ad = r'({}) * ({}) * ({})'.format(expr_mm, expr_f1, expr_f2)
```

```python
# Dummy equation
expr_dummy = r"wlog * w2 * lambd * mus"

# Invalid equations
expr_dims = r"wlog - lambd"
expr_zero = r"w2 / (wlog - wlog)"
```

```python
# Define units per feature
unit_ref = {'wlog': 'K',
            'w2': 'K',
            'lambd': 'dimensionless',
            'mus': 'dimensionless'}
```

```python
# Check units
expr_list = [expr_mm, expr_f1, expr_f2, expr_ad,
             expr_dummy, expr_dims, expr_zero]
name_list = ["McMillan Equation",
             "Allen-Dynes f1",
             "Allen-Dynes f2",
             "Allen-Dynes Equation",
             "\nDummy equation",
             "Invalid equation",
             "Invalid equation"]

print("{0:<20} {1:>20} [{2}]".format("Name", "Units", "Degree"))
print("="*50)
for expr, name in zip(expr_list, name_list):
    dims, degree = dimensional_analysis.check_dimensionality(expr,
                                                             unit_ref)
    dims = dims.replace('[', '').replace(']', '')
    print("{0:<20} {1:>20} [{2}]".format(name, dims, degree))
```

    Name                                Units [Degree]
    ==================================================
    McMillan Equation             temperature [1]
    Allen-Dynes f1              dimensionless [0]
    Allen-Dynes f2              dimensionless [0]
    Allen-Dynes Equation          temperature [1]

    Dummy equation               temperature [2]
    Invalid equation       Dimension Mismatch [0]
    Invalid equation            Zero Division [0]

# Evaluating models

```python
# Predict Tc using equations
slope = 1 # no slope
intercept = 0  # no intercept
coefficients = (slope, intercept)

features = column_names[1:]  # subset of features

values_mm, _ = regression.evaluate_model(expr_mm,
                                         df_ad,
                                         features,
                                         coefficients)
values_ad, _ = regression.evaluate_model(expr_ad,
                                         df_ad,
                                         features,
                                         coefficients)

# Validate against experimental reference measurements
fig, ax = plotting.pretty_scatter(df_ad['Tc'], values_mm, units="K")
ax.set_title("McMillan Equation")
fig.tight_layout()

fig, ax = plotting.pretty_scatter(df_ad['Tc'], values_ad, units="K")
ax.set_title("Allen-Dynes Equation")
fig.tight_layout()
```

# Writing SISSO inputs

```python
# train.dat
column_names = ["Tc", "wlog", "lambd", "mus"]  # feature subset
df_train = df_ad[column_names]

filename = os.path.join(working_directory, "train.dat")
io.create_tsv(df_train, filename)
```

```python
# the first dimension group includes the first feature
dimension_groups = "(1:1)"
# ungrouped features (i.e. 2 and 3) are dimensionless

n_iterations = 3                     # number of subpsace expansion iterations (max 3)
n_train = len(df_train)              # number of training samples
n_features = len(column_names) - 1   # number of features
n_sis = 1000                         # SIS subspace size
n_models = 1000                      # number of models to save

problem_type = 1  # regression
operation_set = "'(+)(-)(*)(/)(exp)(exp-)(^-1)(^2)(^3)(sqrt)(log)'"
fit_intercept = False

parameters = dict(ptype=problem_type,
                  dimclass=dimension_groups,
                  rung=n_iterations,
                  opset=operation_set,
                  nsample=n_train,
                  nsf=n_features,
                  subs_sis=n_sis,
                  nm_output=n_models,
                  fit_intercept=fit_intercept)

filename = os.path.join(working_directory, "SISSO.in")
io.create_sisso_input(parameters, filename=filename)
```

```python

```


```python
import numpy as np

A = np.array(
    [
    [-1, 2, -3]
    ], dtype=float
)

(m,n) = np.shape(A)

print('order of \n', A, '= ', f'{m} x {n}')
```

    order of
     [[-1.  2. -3.]] =  1 x 3

```python
columns = 5
rows = 15

m = rows
n = columns

print('order = ', f'{m} x {n}')
```

    order =  15 x 5

```python

```

```python
import numpy as np

A = np.array(
    [
        [1, -1, -1],
        [0, -2, 3],
        [2, 1, 5],
    ], dtype=float
)

B = np.array(
    [
        [1, 1, 1],
        [1, 1, 1],
    ], dtype=float
)

try:
    print('A*B=\n', A * B)
except ValueError:
    print('A*B=\nError multiplying A*B')

print()

try:
    print('B*A=\n', B @ A)
except ValueError:
    print('B*A=\nError multiplying B*A')

print()

try:
    print('A*B^T=\n', A @ np.transpose(B))
except ValueError:
    print('A*B^T=\nError multiplying A*B^T')

print()

try:
    print('B*A^T=\n', B @ np.transpose(A))
except ValueError:
    print('B*A^T=\nError multiplying B*A^T=')

```

    A*B=
    Error multiplying A*B

    B*A=
     [[ 3. -2.  7.]
     [ 3. -2.  7.]]

    A*B^T=
     [[-1. -1.]
     [ 1.  1.]
     [ 8.  8.]]

    B*A^T=
     [[-1.  1.  8.]
     [-1.  1.  8.]]

```python
import numpy as np

A = np.array(
    [
        [1, -1, -1, 0],
        [0, -2, 3, 2],
        [2, 1, 5, 1],
        [-2, 1, -2, 9],
    ], dtype=float
)

np.linalg.det(A)
```

    -227.99999999999986

```python
import numpy as np

A = np.array(
    [
        [1, -1, -1, 0],
        [0, -2, 3, 2],
        [2, 1, 5, 1],
        [-2, 1, -2, 9],
    ]
)

b = np.array(
    [
        [4],
        [4],
        [4],
        [4],
    ]
)

x = np.linalg.solve(A,b)

print('x = \n', x)

```

    x =
     [[ 2.57894737]
     [-1.21052632]
     [-0.21052632]
     [ 1.10526316]]

```python
from sympy import *

x,y,z=symbols('x y z')

linsolve(
    Matrix(
        (
            [3, -1, -1, 4],
            [4, -2, 3, 2],
            [2, 1, -5, -1]
        )
    ),
    (x, y, z)
)

```

$\displaystyle \left\{\left( - \frac{21}{13}, \  -7, \  - \frac{24}{13}\right)\right\}$

```python
import numpy as np
from fractions import Fraction

M = np.array(
    [
        [3, -1, -1, 4],
        [4, -2, 3, 2],
        [2, 1, -5, -1],
    ],dtype=float
)

print('M =\n', M)
(m,n) = M.shape
print('mxn =\n', m, 'x', n)
print('|A| = \n', M[:3,:3], '\n =', np.linalg.det(M[:3,:3]))

k = 0
p = 0

for p in range(0,m-1):
    for i in range(p+1,m):
        if M[p,p] == 0:
            # change row
            for j in range(i,m):
                if M[j,p] != 0:
                    k += 1
                    Mp = np.copy(M[p])
                    M[p] = M[j]
                    M[j] = Mp

                    print('Mx =\n', M)
                    break

        M[i] = M[i,:] - M[p,:]*M[i,p]/M[p,p]
        print('M =\n', M)

detA = 1
for p in range(m):
    detA *= M[p,p]

print('|A| =', detA)

```

    M =
     [[ 3. -1. -1.  4.]
     [ 4. -2.  3.  2.]
     [ 2.  1. -5. -1.]]
    mxn =
     3 x 4
    |A| =
     [[ 3. -1. -1.]
     [ 4. -2.  3.]
     [ 2.  1. -5.]]
     = -13.0
    M =
     [[ 3.         -1.         -1.          4.        ]
     [ 0.         -0.66666667  4.33333333 -3.33333333]
     [ 2.          1.         -5.         -1.        ]]
    M =
     [[ 3.         -1.         -1.          4.        ]
     [ 0.         -0.66666667  4.33333333 -3.33333333]
     [ 0.          1.66666667 -4.33333333 -3.66666667]]
    M =
     [[  3.          -1.          -1.           4.        ]
     [  0.          -0.66666667   4.33333333  -3.33333333]
     [  0.           0.           6.5        -12.        ]]
    |A| = -12.999999999999995

```python

```

$$
\left[
    \begin{matrix}
        (1-\alpha_{11k}) & -\alpha_{12k} & -\alpha_{13k} & -\alpha_{14k} & -\alpha_{15k} & -\alpha_{16k} & -\alpha_{17k} & -\alpha_{18k} & -\alpha_{19k} & -\alpha_{110k} \\

    \end{matrix}
\right]
$$

$$
\left[
    \begin{matrix}
        (1-\alpha_{11k}) & -\alpha_{12k} & -\alpha_{13k} & -\alpha_{14k} & -\alpha_{15k} & -\alpha_{16k} & -\alpha_{17k} & -\alpha_{18k} & -\alpha_{19k} & -\alpha_{110k} \\
        -\alpha_{21k} & (1-\alpha_{21k}) & -\alpha_{23k} & -\alpha_{24k} & -\alpha_{25k} & -\alpha_{26k} & -\alpha_{27k} & -\alpha_{28k} & -\alpha_{29k} & -\alpha_{210k} \\
        -\alpha_{31k} & -\alpha_{31k} & (1-\alpha_{33k}) & -\alpha_{34k} & -\alpha_{35k} & -\alpha_{36k} & -\alpha_{37k} & -\alpha_{38k} & -\alpha_{39k} & -\alpha_{310k} \\
        -\alpha_{41k} & (1-\alpha_{41k}) & -\alpha_{43k} & (1-\alpha_{44k}) & -\alpha_{45k} & -\alpha_{46k} & -\alpha_{47k} & -\alpha_{48k} & -\alpha_{49k} & -\alpha_{410k} \\

    \end{matrix}
\right]
$$

```python
from sympy import *

a, b, c = symbols('a b c')

linsolve(Matrix((
        [3, -1, -1, 6],
        [3, -1, 4, 2],
        [3, -4, 1, 0],
        )), (a,b,c))
```

$\displaystyle \left\{\left( \frac{20}{9}, \  \frac{22}{15}, \  - \frac{4}{5}\right)\right\}$

```python

```


# Regression on simulated data with mariner ship
Model for mariner ship taken from: (Wang et al., 2021) with hydrodynamic derivatives originally from (Chislett & Strom-Tejsen, 1965).

## Purpose
* The regression of simulated data works extremly well with linear data as seen in: [12.01_regression_simulated_data.ipynb](12.01_regression_simulated_data.ipynb)
* Also works for linear equation of motion : [12.02_regression_simulated_data_nonlinear_EOM.ipynb](12.02_regression_simulated_data_nonlinear_EOM.ipynb)
* Now it is time to look at the full nonlinear model using the mariner ship as one example.

## Methodology
* Simulate mariner ship with coefficients from (Chislett & Strom-Tejsen, 1965).
* Regress the simulated data
* What is the accuray?
* Are the coefficients the same?

## Results
Describe and comment the most important results.

## Setup

```python
# %load imports.py
## Local packages:

%matplotlib inline
%load_ext autoreload
%autoreload 2
%config Completer.use_jedi = False  ## (To fix autocomplete)

## External packages:
import pandas as pd
#pd.options.plotting.backend = "plotly"
pd.options.display.max_rows = 999
pd.options.display.max_columns = 999
pd.set_option("display.max_columns", None)

import numpy as np
import os
import matplotlib.pyplot as plt
#if os.name == 'nt':
#    plt.style.use('presentation.mplstyle')  # Windows

import plotly.express as px
import plotly.graph_objects as go

import seaborn as sns
import sympy as sp
from sympy.physics.mechanics import (dynamicsymbols, ReferenceFrame,
                                      Particle, Point)
from sympy.physics.vector.printing import vpprint, vlatex
from IPython.display import display, Math, Latex
from src.substitute_dynamic_symbols import run, lambdify

import pyro

import sklearn
import pykalman
from statsmodels.sandbox.regression.predstd import wls_prediction_std
import statsmodels.api as sm

from scipy.integrate import solve_ivp

## Local packages:
from src.data import mdl

from src.symbols import *
from src.parameters import *
import src.symbols as symbols
from src import prime_system
from src.models import regression
from src.visualization.regression import show_pred
from src.visualization.plot import track_plot
from src.equation import Equation

## Load models:
# (Uncomment these for faster loading):
import src.models.vmm_mariner  as vmm
```

```python
from src.models.vessels import mariner as ship
```

```python
ship.df_parameters.head()
```

```python
meta_data = pd.Series({
    'lpp' : 160.93,
    'rho' : 1000,
    'beam': 20,  # Guessing...
})

```

## Ship parameters

```python
#T_ = (meta_data.TA + meta_data.TF)/2
L_ = meta_data.lpp
#m_ = meta_data.mass
rho_ = meta_data.rho
B_ = meta_data.beam
#I_z_ = 900
m_ = ship.m_prime_*1/2*rho_*L_**3
xG_ = ship.xG_prime_*L_
Iz_ = ship.Iz_prime_*1/2*rho_*L_**5

ship_parameters = {
        'L' : L_,
        'B' : B_,
        'rho' : rho_,
        'x_G' : xG_,
        'm' : m_,
        'I_z': Iz_,
    }

ps = prime_system.PrimeSystem(**ship_parameters)  # ship
```

```python
ship_parameters
```

```python
ship_parameters_prime = ps.prime(ship_parameters)
```

```python
ship_parameters_prime
```

## Simulate data

```python
type(vmm.simulator)
```

```python
parameters=ship.df_parameters['prime'].copy()

# Removing the interia from the coefficients:
parameters.Xrr+=(-ship.m_prime_*ship.xG_prime_)
parameters.Xvr+=(-ship.m_prime_)
parameters.Yur+=ship.m_prime_
parameters.Nur+=ship.m_prime_*ship.xG_prime_

t_ = np.linspace(0,700,1000)
df = pd.DataFrame(index=t_)

df['u'] = 7.7175
df['v'] = 0
df['r'] = 0
df['x0'] = 0
df['y0'] = 0
df['psi'] = 0
df['U'] = np.sqrt(df['u']**2 + df['v']**2)
df['beta'] = -np.arctan2(df['v'],df['u'])

df['delta'] = 0
df.loc[100:,'delta'] = -np.deg2rad(35)

result = vmm.simulator.simulate(df_=df, parameters=parameters, ship_parameters=ship_parameters,
                                  control_keys=['delta'], primed_parameters=True,prime_system=ps)
```

```python
result.track_plot();
```

```python
result.plot()
```

```python
result.X_qs.plot()
```

```python
df_result = result.result.copy()
df_result = pd.concat([df_result,result.accelerations], axis=1)
```

```python
df_result.plot(y='v1d')
```

## Regression

```python
df = ps.prime(df_result, U=df_result['U'])  # OBS primed!
df['u']-=df.iloc[0]['u']  # Only preturbed!!!
```

```python
df.plot(y='u')
```

## N

```python
vmm.N_eq
```

```python
N_ = sp.symbols('N_')

diff_eq_N = regression.DiffEqToMatrix(ode=vmm.N_qs_eq.subs(N_qs,N_),
                                      label=N_, base_features=[delta,u,v,r])
```

```python
X = diff_eq_N.calculate_features(data=df)
y = diff_eq_N.calculate_label(y=df['r1d'])

model_N = sm.OLS(y,X)
results_N = model_N.fit()

show_pred(X=X,y=y,results=results_N, label=r'$\dot{r}$')
```

```python
X = run(function=diff_eq_N.X_lambda, inputs=df)
X = X.reshape(X.shape[0],X.shape[-1]).T
X.shape
```

```python
diff_eq_N.eq_X.rhs
```

```python
type(diff_eq_N.eq_X.rhs)
```

```python
1 in diff_eq_N.eq_X.rhs
```

```python
lamda = lambdify(sp.matrices.immutable.ImmutableDenseMatrix(diff_eq_N.eq_X.rhs[1:]))
```

```python
diff_eq_N.eq_X.rhs[1:]
```

```python
df.shape
```

```python
diff_eq_N.X_lambda
```

```python
X = run(function=lamda, inputs=df)
X.shape
```

```python
diff_eq_N.X_lambda(delta=df['delta'].values, r=df['r'].values, u=df['u'].values, v=df['v'].values).shape
```

```python

```

```python
Math(vlatex(diff_eq_N.eq_X))
```

```python
diff_eq_N.feature_names_subs()
```

```python
diff_eq_N.eq_beta
```

```python
diff_eq_N.acceleration_equation_x
```

```python
A_,b_ = sp.linear_eq_to_matrix([diff_eq_N.acceleration_equation_x.rhs],diff_eq_N.xs)
A_
```

```python

A_
```

```python
sp.linear_eq_to_matrix([diff_eq_N.acceleration_equation_x.rhs],diff_eq_N.xs)[1]
```

```python
diff_eq_N.eq_beta
```

```python
diff_eq_N.xs
```

```python
diff_eq_N.coefficients
```

```python
diff_eq_N.parts
```

```python
sp.linear_eq_to_matrix([diff_eq_N.acceleration_equation_x.rhs],diff_eq_N.xs)[1]
```

## Y

```python
Y_ = sp.symbols('Y_')
diff_eq_Y = regression.DiffEqToMatrix(ode=vmm.Y_qs_eq.subs(Y_qs,Y_),
                                      label=Y_, base_features=[delta,u,v,r])
```

```python
Math(vlatex(diff_eq_Y.acceleration_equation))
```

```python
X = diff_eq_Y.calculate_features(data=df)
y = diff_eq_Y.calculate_label(y=df['v1d'])

model_Y = sm.OLS(y,X)
results_Y = model_Y.fit()

show_pred(X=X,y=y,results=results_Y, label=r'$\dot{v}$')
```

## X

```python
X_ = sp.symbols('X_')
diff_eq_X = regression.DiffEqToMatrix(ode=vmm.X_qs_eq.subs(X_qs,X_),
                                      label=X_, base_features=[delta,u,v,r,thrust])
```

```python
diff_eq_X.acceleration_equation
```

```python
X = diff_eq_X.calculate_features(data=df)
y = diff_eq_X.calculate_label(y=df['u1d'])

model_X = sm.OLS(y,X)
results_X = model_X.fit()

show_pred(X=X,y=y,results=results_X, label=r'$\dot{u}}$')
```

```python
results_summary_X = regression.results_summary_to_dataframe(results_X)
results_summary_Y = regression.results_summary_to_dataframe(results_Y)
results_summary_N = regression.results_summary_to_dataframe(results_N)
```

### Decoupling

```python
u1d,v1d,r1d = sp.symbols('u1d, v1d, r1d')
u_,v_,r_ = sp.symbols('u, v, r')

X_qs_, Y_qs_, N_qs_ = sp.symbols('X_qs, Y_qs, N_qs')

X_eq = vmm.X_eom.subs([
    (X_force,sp.solve(vmm.fx_eq,X_force)[0]),
])

Y_eq = vmm.Y_eom.subs([
    (Y_force,sp.solve(vmm.fy_eq,Y_force)[0]),
])

N_eq = vmm.N_eom.subs([
    (N_force,sp.solve(vmm.mz_eq,N_force)[0]),
])

subs = [
    #(X_qs,X_qs_),
    #(Y_qs,Y_qs_),
    #(N_qs,N_qs_),

    (u1d,u1d),
    (v1d,v1d),
    (r1d,r1d),

    (u,u_),
    (v,v_),
    (r,r_),

]
eq_X_ = X_eq.subs(subs)
eq_Y_ = Y_eq.subs(subs)
eq_N_ = N_eq.subs(subs)

A,b = sp.linear_eq_to_matrix([eq_X_,eq_Y_,eq_N_],[u1d,v1d,r1d])
```

```python
subs = {value:key for key,value in p.items()}
A_ = A*sp.matrices.MutableDenseMatrix([A_coeff,B_coeff,C_coeff])
A_lambda=lambdify(A_.subs(subs))
```

```python
A_coeff_ = results_summary_X['coeff']
B_coeff_ = results_summary_Y['coeff']
C_coeff_ = results_summary_N['coeff']

coeffs = run(A_lambda,A_coeff=A_coeff_.values, B_coeff=B_coeff_.values, C_coeff=C_coeff_.values,
    **ship.df_parameters['prime'], **ship_parameters_prime)
```

The way that the regression is formulated, inertial forces, such as centrifugal force will be included into the derivatives (I think) which means that centrifugal force : $-m \cdot r \cdot u$ will be included into $Y_{ur}$ coefficient. This coefficient is therefore not pure hydrodynamic, and can potentially be counted twice..?
The coefficients are recalculated below to avooid this:

```python
results_summary_X['decoupled'] = coeffs[0][0]
results_summary_Y['decoupled'] = coeffs[1][0]
results_summary_N['decoupled'] = coeffs[2][0]

x_G_ = ship_parameters_prime['x_G']
m_ = ship_parameters_prime['m']

results_summary_X.loc['Xrr','decoupled']+=(-m_*x_G_)
results_summary_X.loc['Xvr','decoupled']+=(-m_)
results_summary_Y.loc['Yur','decoupled']+=m_
results_summary_N.loc['Nur','decoupled']+=m_*x_G_

#results_summary_X.loc['Xr','decoupled']+=(-m_*x_G_)
#results_summary_Y.loc['Yr','decoupled']+=m_
#results_summary_N.loc['Nr','decoupled']+=m_*x_G_

```

```python
results_summary_N
```

## Add the regressed parameters
Hydrodynamic derivatives that depend on acceleration cannot be obtained from the VCT regression. They are however essential if a time simulation should be conducted. These values have then been taken from Brix semi empirical formulas for the simulations below.

```python
df_parameters_all = ship.df_parameters.copy()
for other in [results_summary_X, results_summary_Y, results_summary_N]:
    df_parameters_all = df_parameters_all.combine_first(other)

df_parameters_all.rename(columns={'decoupled':'regressed'}, inplace=True)
df_parameters_all.drop(columns=['brix_lambda'], inplace=True)

df_parameters_all['regressed'] = df_parameters_all['regressed'].combine_first(df_parameters_all['prime'])  # prefer regressed
#df_parameters_all['regressed'].fillna(0,inplace=True)
```

# Simulate forces

```python
df_result['X_qs_pred'] = run(function=vmm.simulator.X_qs_lambda, inputs=df_result, **df_parameters_all['regressed'])
df_result['Y_qs_pred'] = run(function=vmm.simulator.Y_qs_lambda, inputs=df_result, **df_parameters_all['regressed'])
df_result['N_qs_pred'] = run(function=vmm.simulator.N_qs_lambda, inputs=df_result, **df_parameters_all['regressed'])

```

```python
X = diff_eq_X.calculate_features(data=df)
X_forces = X*results_summary_X['decoupled']
X_forces.index = df.index

X = diff_eq_Y.calculate_features(data=df)
Y_forces = X*results_summary_Y['decoupled']
Y_forces.index = df.index

X = diff_eq_N.calculate_features(data=df)
N_forces = X*results_summary_N['decoupled']
N_forces.index = df.index

```

```python
display(px.line(X_forces, y=X_forces.columns, width=1000, height=400))
display(px.line(Y_forces, y=Y_forces.columns, width=1000, height=400))
display(px.line(N_forces, y=N_forces.columns, width=1000, height=400))
```

# Simulation

```python
parameters=df_parameters_all['regressed'].copy()

result_regression = vmm.simulator.simulate(df_=df_result, parameters=parameters, ship_parameters=ship_parameters,
                                  control_keys=['delta'], primed_parameters=True,prime_system=ps)
```

```python
fig,ax=plt.subplots()
key = 'psi'
result.result.plot(y=key, ax=ax)
result_regression.result.plot(y=key, style='--', ax=ax)
```

```python
result_regression.plot_compare()
```

```python
df_parameters_all.head()
```

```python
df_plot_parameters = df_parameters_all[['regressed','prime']].copy()
mask = df_plot_parameters['regressed'].abs() > 0

fig,ax=plt.subplots()
fig.set_size_inches(17,7)
df_plot_parameters.loc[mask].plot(kind='bar', ax=ax);
```

```python

```


 Social network Graph Link Prediction - Facebook Challenge

```
#Importing Libraries
# please do go through this python notebook:
import warnings
warnings.filterwarnings("ignore")

import csv
import pandas as pd#pandas to create small dataframes
import datetime #Convert to unix time
import time #Convert to unix time
# if numpy is not installed already : pip3 install numpy
import numpy as np#Do aritmetic operations on arrays
# matplotlib: used to plot graphs
import matplotlib
import matplotlib.pylab as plt
import seaborn as sns#Plots
from matplotlib import rcParams#Size of plots
from sklearn.cluster import MiniBatchKMeans, KMeans#Clustering
import math
import pickle
import os
# to install xgboost: pip3 install xgboost
import xgboost as xgb

import warnings
import networkx as nx
import pdb
import pickle
from pandas import HDFStore,DataFrame
from pandas import read_hdf
from scipy.sparse.linalg import svds, eigs
import gc
from tqdm import tqdm
```

```
from google.colab import drive
drive.mount('/content/drive')
```

    Mounted at /content/drive

# 1. Reading Data

```
if os.path.isfile('data/after_eda/train_pos_after_eda.csv'):
    train_graph=nx.read_edgelist('data/after_eda/train_pos_after_eda.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)
    print(nx.info(train_graph))
else:
    print("please run the FB_EDA.ipynb or download the files from drive")
```

    Name:
    Type: DiGraph
    Number of nodes: 1780722
    Number of edges: 7550015
    Average in degree:   4.2399
    Average out degree:   4.2399

# 2. Similarity measures

## 2.1 Jaccard Distance:

\begin{equation}
j = \frac{|X\cap Y|}{|X \cup Y|}
\end{equation}

```
#for followees
def jaccard_for_followees(a,b):
    try:
        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:
            return 0
        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/\
                                    (len(set(train_graph.successors(a)).union(set(train_graph.successors(b)))))
    except:
        return 0
    return sim
```

```
#for followers
def jaccard_for_followers(a,b):
    try:
        if len(set(train_graph.predecessors(a))) == 0  | len(set(g.predecessors(b))) == 0:
            return 0
        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/\
                                 (len(set(train_graph.predecessors(a)).union(set(train_graph.predecessors(b)))))
        return sim
    except:
        return 0
```

## 2.2 Cosine distance

\begin{equation}
CosineDistance = \frac{|X\cap Y|}{|X|\cdot|Y|}
\end{equation}

```
#for followees
def cosine_for_followees(a,b):
    try:
        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:
            return 0
        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/\
                                    (math.sqrt(len(set(train_graph.successors(a)))*len((set(train_graph.successors(b))))))
        return sim
    except:
        return 0
```

```
def cosine_for_followers(a,b):
    try:

        if len(set(train_graph.predecessors(a))) == 0  | len(set(train_graph.predecessors(b))) == 0:
            return 0
        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/\
                                     (math.sqrt(len(set(train_graph.predecessors(a))))*(len(set(train_graph.predecessors(b)))))
        return sim
    except:
        return 0
```

## 3. Ranking Measures

## 3.1 Page Ranking

```
if not os.path.isfile('data/fea_sample/page_rank.p'):
    pr = nx.pagerank(train_graph, alpha=0.85)
    pickle.dump(pr,open('data/fea_sample/page_rank.p','wb'))
else:
    pr = pickle.load(open('data/fea_sample/page_rank.p','rb'))
```

```
print('min',pr[min(pr, key=pr.get)])
print('max',pr[max(pr, key=pr.get)])
print('mean',float(sum(pr.values())) / len(pr))
```

    min 1.6556497245737814e-07
    max 2.7098251341935827e-05
    mean 5.615699699389075e-07

```
#for imputing to nodes which are not there in Train data
mean_pr = float(sum(pr.values())) / len(pr)
print(mean_pr)
```

    5.615699699389075e-07

# 4. Other Graph Features

## 4.1 Shortest path:

Getting Shortest path between twoo nodes, if nodes have direct path i.e directly connected then we are removing that edge and calculating path.

```
#if has direct edge then deleting that edge and calculating shortest path
def compute_shortest_path_length(a,b):
    p=-1
    try:
        if train_graph.has_edge(a,b):
            train_graph.remove_edge(a,b)
            p= nx.shortest_path_length(train_graph,source=a,target=b)
            train_graph.add_edge(a,b)
        else:
            p= nx.shortest_path_length(train_graph,source=a,target=b)
        return p
    except:
        return -1
```

## 4.2 Checking for same community

```
#getting weekly connected edges from graph
wcc=list(nx.weakly_connected_components(train_graph))
def belongs_to_same_wcc(a,b):
    index = []
    if train_graph.has_edge(b,a):
        return 1
    if train_graph.has_edge(a,b):
            for i in wcc:
                if a in i:
                    index= i
                    break
            if (b in index):
                train_graph.remove_edge(a,b)
                if compute_shortest_path_length(a,b)==-1:
                    train_graph.add_edge(a,b)
                    return 0
                else:
                    train_graph.add_edge(a,b)
                    return 1
            else:
                return 0
    else:
            for i in wcc:
                if a in i:
                    index= i
                    break
            if(b in index):
                return 1
            else:
                return 0
```

```
belongs_to_same_wcc(861, 1659750)
```

    0

```
belongs_to_same_wcc(669354,1635354)
```

    0

## 4.3 Adamic/Adar Index:

```
#adar index
def calc_adar_in(a,b):
    sum=0
    try:
        n=list(set(train_graph.successors(a)).intersection(set(train_graph.successors(b))))
        if len(n)!=0:
            for i in n:
                sum=sum+(1/np.log10(len(list(train_graph.predecessors(i)))))
            return sum
        else:
            return 0
    except:
        return 0
```

## 4.4 Is persion was following back:

```
def follows_back(a,b):
    if train_graph.has_edge(b,a):
        return 1
    else:
        return 0
```

## 4.6 Hits Score
The HITS algorithm computes two numbers for a node. Authorities estimates the node value based on the incoming links. Hubs estimates the node value based on outgoing links.

```
if not os.path.isfile('data/fea_sample/hits.p'):
    hits = nx.hits(train_graph, max_iter=100, tol=1e-08, nstart=None, normalized=True)
    pickle.dump(hits,open('data/fea_sample/hits.p','wb'))
else:
    hits = pickle.load(open('data/fea_sample/hits.p','rb'))
```

```
print('min',hits[0][min(hits[0], key=hits[0].get)])
print('max',hits[0][max(hits[0], key=hits[0].get)])
print('mean',float(sum(hits[0].values())) / len(hits[0]))
```

    min 0.0
    max 0.004868653378780953
    mean 5.615699699344123e-07

# 5. Featurization

## 5. 1 Reading a sample of Data from both train and test

```
import random
if os.path.isfile('data/after_eda/train_after_eda.csv'):
    filename = "data/after_eda/train_after_eda.csv"
    # here we have hardcoded the number of lines as 15100030
    # n_train = sum(1 for line in open(filename)) #number of records in file (excludes header)
    n_train =  15100028
    s = 100000 #desired sample size
    skip_train = sorted(random.sample(range(1,n_train+1),n_train-s))
```

```
if os.path.isfile('data/after_eda/train_after_eda.csv'):
    filename = "data/after_eda/test_after_eda.csv"
    # here we have hardcoded the number of lines as 3775008
    # n_test = sum(1 for line in open(filename)) #number of records in file (excludes header)
    n_test = 3775006
    s = 50000 #desired sample size
    skip_test = sorted(random.sample(range(1,n_test+1),n_test-s))
```

```
print("Number of rows in the train data file:", n_train)
print("Number of rows we are going to elimiate in train data are",len(skip_train))
print("Number of rows in the test data file:", n_test)
print("Number of rows we are going to elimiate in test data are",len(skip_test))
```

    Number of rows in the train data file: 15100028
    Number of rows we are going to elimiate in train data are 15000028
    Number of rows in the test data file: 3775006
    Number of rows we are going to elimiate in test data are 3725006

```
df_final_train = pd.read_csv('data/after_eda/train_after_eda.csv', skiprows=skip_train, names=['source_node', 'destination_node'])
df_final_train['indicator_link'] = pd.read_csv('data/train_y.csv', skiprows=skip_train, names=['indicator_link'])
print("Our train matrix size ",df_final_train.shape)
df_final_train.head(2)
```

    Our train matrix size  (100002, 3)

      source_node
      destination_node
      indicator_link

      0
      273084
      1505602
      1

      1
      832016
      1543415
      1

```
df_final_test = pd.read_csv('data/after_eda/test_after_eda.csv', skiprows=skip_test, names=['source_node', 'destination_node'])
df_final_test['indicator_link'] = pd.read_csv('data/test_y.csv', skiprows=skip_test, names=['indicator_link'])
print("Our test matrix size ",df_final_test.shape)
df_final_test.head(2)
```

    Our test matrix size  (50002, 3)

      source_node
      destination_node
      indicator_link

      0
      848424
      784690
      1

      1
      483294
      1255532
      1

## 5.2 Adding a set of features

__we will create these each of these features for both train and test data points__

jaccard_followers

jaccard_followees

cosine_followers

cosine_followees

num_followers_s

num_followees_s

num_followers_d

num_followees_d

inter_followers

inter_followees

```
if not os.path.isfile('data/fea_sample/storage_sample_stage1.h5'):
    #mapping jaccrd followers to train and test data
    df_final_train['jaccard_followers'] = df_final_train.apply(lambda row:
                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)
    df_final_test['jaccard_followers'] = df_final_test.apply(lambda row:
                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)

    #mapping jaccrd followees to train and test data
    df_final_train['jaccard_followees'] = df_final_train.apply(lambda row:
                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)
    df_final_test['jaccard_followees'] = df_final_test.apply(lambda row:
                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)

        #mapping jaccrd followers to train and test data
    df_final_train['cosine_followers'] = df_final_train.apply(lambda row:
                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)
    df_final_test['cosine_followers'] = df_final_test.apply(lambda row:
                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)

    #mapping jaccrd followees to train and test data
    df_final_train['cosine_followees'] = df_final_train.apply(lambda row:
                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)
    df_final_test['cosine_followees'] = df_final_test.apply(lambda row:
                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)
```

```
def compute_features_stage1(df_final):
    #calculating no of followers followees for source and destination
    #calculating intersection of followers and followees for source and destination
    num_followers_s=[]
    num_followees_s=[]
    num_followers_d=[]
    num_followees_d=[]
    inter_followers=[]
    inter_followees=[]
    for i,row in df_final.iterrows():
        try:
            s1=set(train_graph.predecessors(row['source_node']))
            s2=set(train_graph.successors(row['source_node']))
        except:
            s1 = set()
            s2 = set()
        try:
            d1=set(train_graph.predecessors(row['destination_node']))
            d2=set(train_graph.successors(row['destination_node']))
        except:
            d1 = set()
            d2 = set()
        num_followers_s.append(len(s1))
        num_followees_s.append(len(s2))

        num_followers_d.append(len(d1))
        num_followees_d.append(len(d2))

        inter_followers.append(len(s1.intersection(d1)))
        inter_followees.append(len(s2.intersection(d2)))

    return num_followers_s, num_followers_d, num_followees_s, num_followees_d, inter_followers, inter_followees
```

```
if not os.path.isfile('data/fea_sample/storage_sample_stage1.h5'):
    df_final_train['num_followers_s'], df_final_train['num_followers_d'], \
    df_final_train['num_followees_s'], df_final_train['num_followees_d'], \
    df_final_train['inter_followers'], df_final_train['inter_followees']= compute_features_stage1(df_final_train)

    df_final_test['num_followers_s'], df_final_test['num_followers_d'], \
    df_final_test['num_followees_s'], df_final_test['num_followees_d'], \
    df_final_test['inter_followers'], df_final_test['inter_followees']= compute_features_stage1(df_final_test)

    hdf = HDFStore('data/fea_sample/storage_sample_stage1.h5')
    hdf.put('train_df',df_final_train, format='table', data_columns=True)
    hdf.put('test_df',df_final_test, format='table', data_columns=True)
    hdf.close()
else:
    df_final_train = read_hdf('data/fea_sample/storage_sample_stage1.h5', 'train_df',mode='r')
    df_final_test = read_hdf('data/fea_sample/storage_sample_stage1.h5', 'test_df',mode='r')
```

## 5.3 Adding new set of features

__we will create these each of these features for both train and test data points__

adar index

is following back

belongs to same weakly connect components

shortest path between source and destination

```
if not os.path.isfile('data/fea_sample/storage_sample_stage2.h5'):
    #mapping adar index on train
    df_final_train['adar_index'] = df_final_train.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)
    #mapping adar index on test
    df_final_test['adar_index'] = df_final_test.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)

    #--------------------------------------------------------------------------------------------------------
    #mapping followback or not on train
    df_final_train['follows_back'] = df_final_train.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)

    #mapping followback or not on test
    df_final_test['follows_back'] = df_final_test.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)

    #--------------------------------------------------------------------------------------------------------
    #mapping same component of wcc or not on train
    df_final_train['same_comp'] = df_final_train.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)

    ##mapping same component of wcc or not on train
    df_final_test['same_comp'] = df_final_test.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)

    #--------------------------------------------------------------------------------------------------------
    #mapping shortest path on train
    df_final_train['shortest_path'] = df_final_train.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)
    #mapping shortest path on test
    df_final_test['shortest_path'] = df_final_test.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)

    hdf = HDFStore('data/fea_sample/storage_sample_stage2.h5')
    hdf.put('train_df',df_final_train, format='table', data_columns=True)
    hdf.put('test_df',df_final_test, format='table', data_columns=True)
    hdf.close()
else:
    df_final_train = read_hdf('data/fea_sample/storage_sample_stage2.h5', 'train_df',mode='r')
    df_final_test = read_hdf('data/fea_sample/storage_sample_stage2.h5', 'test_df',mode='r')
```

## 5.4 Adding new set of features

__we will create these each of these features for both train and test data points__

Weight Features

weight of incoming edges

weight of outgoing edges

weight of incoming edges + weight of outgoing edges

weight of incoming edges * weight of outgoing edges

2*weight of incoming edges + weight of outgoing edges

weight of incoming edges + 2*weight of outgoing edges

Page Ranking of source

Page Ranking of dest

katz of source

katz of dest

hubs of source

hubs of dest

authorities_s of source

authorities_s of dest

#### Weight Features

In order to determine the similarity of nodes, an edge weight value was calculated between nodes. Edge weight decreases as the neighbor count goes up. Intuitively, consider one million people following a celebrity on a social network then chances are most of them never met each other or the celebrity. On the other hand, if a user has 30 contacts in his/her social network, the chances are higher that many of them know each other.

\begin{equation}
W = \frac{1}{\sqrt{1+|X|}}
\end{equation}

it is directed graph so calculated Weighted in and Weighted out differently

```
#weight for source and destination of each link
Weight_in = {}
Weight_out = {}
for i in  tqdm(train_graph.nodes()):
    s1=set(train_graph.predecessors(i))
    w_in = 1.0/(np.sqrt(1+len(s1)))
    Weight_in[i]=w_in

    s2=set(train_graph.successors(i))
    w_out = 1.0/(np.sqrt(1+len(s2)))
    Weight_out[i]=w_out

#for imputing with mean
mean_weight_in = np.mean(list(Weight_in.values()))
mean_weight_out = np.mean(list(Weight_out.values()))
```

    100%|████████████████████████████████████████████████████████████████████| 1780722/1780722 [00:11<00:00, 152682.24it/s]

```
if not os.path.isfile('data/fea_sample/storage_sample_stage3.h5'):
    #mapping to pandas train
    df_final_train['weight_in'] = df_final_train.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))
    df_final_train['weight_out'] = df_final_train.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))

    #mapping to pandas test
    df_final_test['weight_in'] = df_final_test.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))
    df_final_test['weight_out'] = df_final_test.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))

    #some features engineerings on the in and out weights
    df_final_train['weight_f1'] = df_final_train.weight_in + df_final_train.weight_out
    df_final_train['weight_f2'] = df_final_train.weight_in * df_final_train.weight_out
    df_final_train['weight_f3'] = (2*df_final_train.weight_in + 1*df_final_train.weight_out)
    df_final_train['weight_f4'] = (1*df_final_train.weight_in + 2*df_final_train.weight_out)

    #some features engineerings on the in and out weights
    df_final_test['weight_f1'] = df_final_test.weight_in + df_final_test.weight_out
    df_final_test['weight_f2'] = df_final_test.weight_in * df_final_test.weight_out
    df_final_test['weight_f3'] = (2*df_final_test.weight_in + 1*df_final_test.weight_out)
    df_final_test['weight_f4'] = (1*df_final_test.weight_in + 2*df_final_test.weight_out)
```

```
if not os.path.isfile('data/fea_sample/storage_sample_stage3.h5'):

    #page rank for source and destination in Train and Test
    #if anything not there in train graph then adding mean page rank
    df_final_train['page_rank_s'] = df_final_train.source_node.apply(lambda x:pr.get(x,mean_pr))
    df_final_train['page_rank_d'] = df_final_train.destination_node.apply(lambda x:pr.get(x,mean_pr))

    df_final_test['page_rank_s'] = df_final_test.source_node.apply(lambda x:pr.get(x,mean_pr))
    df_final_test['page_rank_d'] = df_final_test.destination_node.apply(lambda x:pr.get(x,mean_pr))
    #================================================================================

    #Katz centrality score for source and destination in Train and test
    #if anything not there in train graph then adding mean katz score
    df_final_train['katz_s'] = df_final_train.source_node.apply(lambda x: katz.get(x,mean_katz))
    df_final_train['katz_d'] = df_final_train.destination_node.apply(lambda x: katz.get(x,mean_katz))

    df_final_test['katz_s'] = df_final_test.source_node.apply(lambda x: katz.get(x,mean_katz))
    df_final_test['katz_d'] = df_final_test.destination_node.apply(lambda x: katz.get(x,mean_katz))
    #================================================================================

    #Hits algorithm score for source and destination in Train and test
    #if anything not there in train graph then adding 0
    df_final_train['hubs_s'] = df_final_train.source_node.apply(lambda x: hits[0].get(x,0))
    df_final_train['hubs_d'] = df_final_train.destination_node.apply(lambda x: hits[0].get(x,0))

    df_final_test['hubs_s'] = df_final_test.source_node.apply(lambda x: hits[0].get(x,0))
    df_final_test['hubs_d'] = df_final_test.destination_node.apply(lambda x: hits[0].get(x,0))
    #================================================================================

    #Hits algorithm score for source and destination in Train and Test
    #if anything not there in train graph then adding 0
    df_final_train['authorities_s'] = df_final_train.source_node.apply(lambda x: hits[1].get(x,0))
    df_final_train['authorities_d'] = df_final_train.destination_node.apply(lambda x: hits[1].get(x,0))

    df_final_test['authorities_s'] = df_final_test.source_node.apply(lambda x: hits[1].get(x,0))
    df_final_test['authorities_d'] = df_final_test.destination_node.apply(lambda x: hits[1].get(x,0))
    #================================================================================

    hdf = HDFStore('data/fea_sample/storage_sample_stage3.h5')
    hdf.put('train_df',df_final_train, format='table', data_columns=True)
    hdf.put('test_df',df_final_test, format='table', data_columns=True)
    hdf.close()
else:
    df_final_train = read_hdf('data/fea_sample/storage_sample_stage3.h5', 'train_df',mode='r')
    df_final_test = read_hdf('data/fea_sample/storage_sample_stage3.h5', 'test_df',mode='r')
```

## 5.5 Adding new set of features

__we will create these each of these features for both train and test data points__

SVD features for both source and destination

```
def svd(x, S):
    try:
        z = sadj_dict[x]
        return S[z]
    except:
        return [0,0,0,0,0,0]
```

```
#for svd features to get feature vector creating a dict node val and inedx in svd vector
sadj_col = sorted(train_graph.nodes())
sadj_dict = { val:idx for idx,val in enumerate(sadj_col)}
```

```
Adj = nx.adjacency_matrix(train_graph,nodelist=sorted(train_graph.nodes())).asfptype()
```

```
U, s, V = svds(Adj, k = 6)
print('Adjacency matrix Shape',Adj.shape)
print('U Shape',U.shape)
print('V Shape',V.shape)
print('s Shape',s.shape)
```

    Adjacency matrix Shape (1780722, 1780722)
    U Shape (1780722, 6)
    V Shape (6, 1780722)
    s Shape (6,)

```
if not os.path.isfile('data/fea_sample/storage_sample_stage4.h5'):
    #===================================================================================================

    df_final_train[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \
    df_final_train.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)

    df_final_train[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \
    df_final_train.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)
    #===================================================================================================

    df_final_train[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \
    df_final_train.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)

    df_final_train[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \
    df_final_train.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)
    #===================================================================================================

    df_final_test[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \
    df_final_test.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)

    df_final_test[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \
    df_final_test.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)

    #===================================================================================================

    df_final_test[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \
    df_final_test.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)

    df_final_test[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \
    df_final_test.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)
    #===================================================================================================

    hdf = HDFStore('data/fea_sample/storage_sample_stage4.h5')
    hdf.put('train_df',df_final_train, format='table', data_columns=True)
    hdf.put('test_df',df_final_test, format='table', data_columns=True)
    hdf.close()
```


# Optimización media-varianza

## [Evaluación de medio término](http://cursos.iteso.mx/course/view.php?id=1480)

La **teoría de portafolios** es uno de los avances más importantes en las finanzas modernas e inversiones.
- Apareció por primera vez en un [artículo corto](https://www.math.ust.hk/~maykwok/courses/ma362/07F/markowitz_JF.pdf) llamado "Portfolio Selection" en la edición de Marzo de 1952 de "the Journal of Finance".
- Escrito por un desconocido estudiante de la Universidad de Chicago, llamado Harry Markowitz.
- Escrito corto (sólo 14 páginas), poco texto, fácil de entender, muchas gráficas y unas cuantas referencias.
- No se le prestó mucha atención hasta los 60s.

Finalmente, este trabajo se convirtió en una de las más grandes ideas en finanzas, y le dió a Markowitz el Premio Nobel casi 40 años después.
- Markowitz estaba incidentalmente interesado en los mercados de acciones e inversiones.
- Estaba más bien interesado en entender cómo las personas tomaban sus mejores decisiones cuando se enfrentaban con "trade-offs".
- Principio de conservación de la miseria. O, dirían los instructores de gimnasio: "no pain, no gain".
- Si queremos más de algo, tenemos que perder en algún otro lado.
- El estudio de este fenómeno era el que le atraía a Markowitz.

De manera que nadie se hace rico poniendo todo su dinero en la cuenta de ahorros. La única manera de esperar altos rendimientos es si se toma bastante riesgo. Sin embargo, riesgo significa también la posibilidad de perder, tanto como ganar.

Pero, ¿qué tanto riesgo es necesario?, y ¿hay alguna manera de minimizar el riesgo mientras se maximizan las ganancias?
- Markowitz básicamente cambió la manera en que los inversionistas pensamos acerca de esas preguntas.
- Alteró completamente la práctica de la administración de inversiones.
- Incluso el título de su artículo era innovador. Portafolio: una colección de activos en lugar de tener activos individuales.
- En ese tiempo, un portafolio se refería a una carpeta de piel.
- En el resto de este módulo, nos ocuparemos de la parte analítica de la teoría de portafolios, la cual puede ser resumida en dos frases:
 - No pain, no gain.
 - No ponga todo el blanquillo en una sola bolsa.

**Objetivos:**
- ¿Qué es la línea de asignación de capital?
- ¿Qué es el radio de Sharpe?
- ¿Cómo deberíamos asignar nuestro capital entre un activo riesgoso y un activo libre de riesgo?

*Referencia:*
- Notas del curso "Portfolio Selection and Risk Management", Rice University, disponible en Coursera.
___

## 1. Línea de asignación de capital

### 1.1. Motivación

El proceso de construcción de un portafolio tiene entonces los siguientes dos pasos:
1. Escoger un portafolio de activos riesgosos.
2. Decidir qué tanto de tu riqueza invertirás en el portafolio y qué tanto invertirás en activos libres de riesgo.

Al paso 2 lo llamamos **decisión de asignación de activos**.

Preguntas importantes:
1. ¿Qué es el portafolio óptimo de activos riesgosos?
 - ¿Cuál es el mejor portafolio de activos riesgosos?
 - Es un portafolio eficiente en media-varianza.
2. ¿Qué es la distribución óptima de activos?
 - ¿Cómo deberíamos distribuir nuestra riqueza entre el portafolo riesgoso óptimo y el activo libre de riesgo?
 - Concepto de **línea de asignación de capital**.
 - Concepto de **radio de Sharpe**.

Dos suposiciones importantes:
- Funciones de utilidad media-varianza.
- Inversionista averso al riesgo.

La idea sorprendente que saldrá de este análisis, es que cualquiera que sea la actitud del inversionista de cara al riesgo, el mejor portafolio de activos riesgosos es idéntico para todos los inversionistas.

Lo que nos importará a cada uno de nosotros en particular, es simplemente la desición óptima de asignación de activos.
___

### 1.2. Línea de asignación de capital

Sean:
- $r_s$ el rendimiento del activo riesgoso,
- $r_f$ el rendimiento libre de riesgo, y
- $w$ la fracción invertida en el activo riesgoso.

 Realizar deducción de la línea de asignación de capital en el tablero.

**Tres doritos después...**

#### Línea de asignación de capital (LAC):
$E[r_p]$ se relaciona con $\sigma_p$ de manera afín. Es decir, mediante la ecuación de una recta:

$$E[r_p]=r_f+\frac{E[r_s-r_f]}{\sigma_s}\sigma_p.$$

- La pendiente de la LAC es el radio de Sharpe $\frac{E[r_s-r_f]}{\sigma_s}=\frac{E[r_s]-r_f}{\sigma_s}$,
- el cual nos dice qué tanto rendimiento obtenemos por unidad de riesgo asumido en la tenencia del activo (portafolio) riesgoso.

Ahora, la pregunta es, ¿dónde sobre esta línea queremos estar?
___

### 1.3. Resolviendo para la asignación óptima de capital

Recapitulando de la clase pasada, tenemos las curvas de indiferencia: **queremos estar en la curva de indiferencia más alta posible, que sea tangente a la LAC**.

 Ver en el tablero.

Analíticamente, el problema es

$$\max_{w} \quad E[U(r_p)]\equiv\max_{w} \quad E[r_p]-\frac{1}{2}\gamma\sigma_p^2,$$

donde los puntos $(\sigma_p,E[r_p])$ se restringen a estar en la LAC, esto es $E[r_p]=r_f+\frac{E[r_s-r_f]}{\sigma_s}\sigma_p$ y $\sigma_p=w\sigma_s$. Entonces el problema anterior se puede escribir de la siguiente manera:

$$\max_{w} \quad r_f+wE[r_s-r_f]-\frac{1}{2}\gamma w^2\sigma_s^2.$$

 Encontrar la $w$ que maximiza la anterior expresión en el tablero.

**Tres doritos después...**

La solución es entonces:

$$w^\ast=\frac{E[r_s]-r_f}{\gamma\sigma_s^2}.$$

De manera intuitiva:
- $w^\ast\propto E[r_s-r_f]$: a más exceso de rendimiento que se obtenga del activo riesgoso, más querremos invertir en él.
- $w^\ast\propto \frac{1}{\gamma}$: mientras más averso al riesgo seas, menos querrás invertir en el activo riesgoso.
- $w^\ast\propto \frac{1}{\sigma_s^2}$: mientras más riesgoso sea el activo, menos querrás invertir en él.
___

## 2. Ejemplo de asignación óptima de capital: acciones y billetes de EU

Pongamos algunos números con algunos datos, para ilustrar la derivación que acabamos de hacer.

En este caso, consideraremos:
- **Portafolio riesgoso**: mercado de acciones de EU (representados en algún índice de mercado como el S&P500).
- **Activo libre de riesgo**: billetes del departamento de tesorería de EU (T-bills).

Tenemos los siguientes datos:

$$E[r_{US}]=11.9\%,\quad \sigma_{US}=19.15\%, \quad r_f=1\%.$$

Recordamos que podemos escribir la expresión de la LAC como:

\begin{align}
E[r_p]&=r_f+\left[\frac{E[r_{US}-r_f]}{\sigma_{US}}\right]\sigma_p\\
      &=0.01+\text{S.R.}\sigma_p,
\end{align}

donde $\text{S.R}=\frac{0.119-0.01}{0.1915}\approx0.569$ es el radio de Sharpe (¿qué es lo que es esto?).

Grafiquemos la LAC con estos datos reales:

```python
# Importamos librerías que vamos a utilizar
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
```

```python
# Datos
Ers = 0.119
ss = 0.1915
rf = 0.01
# Radio de Sharpe para este activo
RS = (Ers - rf) / ss
# Vector de volatilidades del portafolio (sugerido: 0% a 50%)
sp = np.linspace(0, 0.5)
# LAC
Erp = RS *sp + rf
```

```python
# Gráfica
plt.figure(figsize=(6,4))
plt.axhline(y = 0, color = 'k')
plt.axvline(x = 0, color = 'k')
plt.plot(sp, Erp, 'r', lw = 2, label = "LAC")
plt.plot(0, rf, 'og', ms = 5, label = 'Activo libre de riesgo')
plt.plot(ss, Ers, 'ob', ms = 5, label = 'Activo riesgoso')
plt.grid()
plt.xlabel('Volatilidad $\sigma$')
plt.ylabel('Rendimiento esperado $E[r]$')
plt.legend(loc='best')
```

Bueno, y ¿en qué punto de esta línea querríamos estar?
- Pues ya vimos que depende de tus preferencias.
- En particular, de tu actitud de cara al riesgo, medido por tu coeficiente de aversión al riesgo.

Solución al problema de asignación óptima de capital:

$$\max_{w} \quad E[U(r_p)]$$

$$w^\ast=\frac{E[r_s-r_f]}{\gamma\sigma_s^2}$$

Dado que ya tenemos datos, podemos intentar para varios coeficientes de aversión al riesgo:

```python
# importar pandas
import pandas as pd
```

```python
# Crear un DataFrame con los pesos, rendimiento
# esperado y volatilidad del portafolio óptimo
# entre los activos riesgoso y libre de riesgo
# cuyo índice sean los coeficientes de aversión
# al riesgo del 1 al 10 (enteros)
gamma = np.arange(1, 11, 1)
df = pd.DataFrame({'$\gamma$': gamma, '$W^\ast$': (Ers - rf) / (gamma * ss**2)})
```

```python
df
```

      $\gamma$
      $W^st$

      0
      1
      2.972275

      1
      2
      1.486137

      2
      3
      0.990758

      3
      4
      0.743069

      4
      5
      0.594455

      5
      6
      0.495379

      6
      7
      0.424611

      7
      8
      0.371534

      8
      9
      0.330253

      9
      10
      0.297227

¿Cómo se interpreta $w^\ast>1$?
- Cuando $0<w^\ast<1$, entonces $0<1-w^\ast<1$. Lo cual implica posiciones largas en el mercado de activos y en el activo libre de riesgo.
- Por el contrario, cuando $w^\ast>1$, tenemos $1-w^\ast<0$. Lo anterior implica una posición corta en el activo libre de riesgo (suponiendo que se puede) y una posición larga (de más del 100%) en el mercado de activos: apalancamiento.

# Anuncios parroquiales.

## 1. Segunda entrega Tarea 5 para el viernes.

## 2. Quiz la siguiente clase.

## 3. Pueden consultar sus calificaciones en el siguiente [enlace](https://docs.google.com/spreadsheets/d/10Ziyn-G6OjS4OlSc1lEbC0s-fBwkrA-rvN0MRggTDt4/edit?usp=sharing).

Created with Jupyter by Esteban Jiménez Rodríguez.


```python
#!pip install seaborn
import sympy as sym
from sympy import apart

import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
sns.set()
sns.set_style("whitegrid", {'grid.linestyle': '--'})
sym.init_printing()
# %matplotlib inline
%matplotlib notebook
```

# Respuesta dinámica

Ya se ha establecido que el primer paso para analizar un sistema es obtener un modelo.

En el área de la **ingeniería de control**, se configura una base de comparación del comportamiento de sistemas controlados contra señales de prueba particulares.

## Señales de prueba

Las señales de prueba más comunes corresponden a funciones impulso, escalón, rampa, parábola, etc.

Como ya se ha estudiado, la respuesta ante la señal impulso permite caracterizar completamente al sistema, sin embargo, no es posible obtener esta señal en laboratorio.

La señal de prueba más apropiada para un sistema debe determinarse por la forma más frecuente de las excitaciones en su operación normal.

### Impulso
Entendido como la derivada del escalón.
\begin{equation}
    \delta(t) = \frac{d\epsilon}{dt}
\end{equation}

Aplicando la transformada de Laplace se obtiene:
\begin{equation}
    \mathcal{L} \{ \delta(t) \} = 1
\end{equation}

Determina completamente el comportamiento del sistema.

### Escalón
\begin{equation}
\epsilon(t) = \begin{cases} 0 & t<0 \\  1 & t > 0 \end{cases}
\end{equation}

Aplicando la transformada de Laplace se obtiene:
\begin{equation}
    \mathcal{L} \{ \epsilon(t) \} = \frac{1}{s}
\end{equation}

Apropiada para visualizar el comportamiento de sistemas que buscan mantener variables fijas.

### Rampa
Entendida como la integral del escalón.

\begin{equation}
    rampa(t) = \int_{-\infty}^{t} \epsilon(\tau) d\tau = t \cdot \epsilon(t)
\end{equation}

Aplicando la transformada de Laplace se obtiene:
\begin{equation}
    \mathcal{L} \{ rampa(t) \} = \frac{1}{s^2}
\end{equation}

Apropiada para visualizar el comportamiento de sistemas que buscan variables que cambian con velocidad constante.

### Parábola
Entendida como la integral de la rampa.

\begin{equation}
    parabola(t) = \int_{-\infty}^{t} rampa(\tau) d\tau   = \frac{t^2}{2} \cdot \epsilon(t)
\end{equation}

Aplicando la transformada de Laplace se obtiene:
\begin{equation}
    \mathcal{L} \{ parabola(t) \} = \frac{1}{s^3}
\end{equation}

Apropiada para visualizar el comportamiento de sistemas que buscan variables que cambian con aceleración constante.

## Respuesta transitoria y respuesta estacionaria

Cuando un [sistema estable](https://en.wikipedia.org/wiki/BIBO_stability) es excitado en el instante $t=0$, este responde de manera continua (respuesta transitoria) desde un estado inicial relacionado con las condiciones iniciales hasta que logra equilibrio (respuesta estacionaria).

## Sistemas de primer orden

Considere la siguiente función de transferencia:

\begin{equation}
G(s) = \frac{C(s)}{R(s)} = \frac{a}{b s + c}
\end{equation}

Este sistema tiene $3$ parámetros ($a$, $b$ y $c$), sin embargo, para facilitar el análisis de estos sistemas puede transformarse la función a su forma canónica al dividir numerador y denominador por $c$.

\begin{equation}
G(s) = \frac{C(s)}{R(s)}  = \frac{\frac{a}{c}}{\frac{b}{c} s + \frac{c}{c}}= \frac{k}{\tau s + 1}
\end{equation}

En la forma canónica, se definen los parámetro como:
- $k$: ganancia en estado estacionario.
- $\tau$: constante de tiempo.

### Respuesta al escalón

Si el sistema es excitado con un escalón unitario, la señal de salida en el dominio de la frecuencia es:

\begin{equation}
C(s) = G(s) \cdot R(s) = \frac{k}{\tau s + 1} \cdot \frac{1}{s}
\end{equation}

Observe que esta expresión puede reescrbirse como:

\begin{equation}
C(s)= k \cdot \left ( \frac{1}{s} - \frac{\tau}{\tau s + 1} \right )
\end{equation}

La transformada inversa de Laplace permite obtener una expresión para la señal de salida en el dominio del tiempo $c(t)$.

\begin{equation}
c(t)= k \left ( 1 - e^{-t/\tau} \right ) \epsilon(t)
\end{equation}

Observe que:

- $c(t<0)=0$ (condción inicial).
- $\lim_{t\rightarrow \infty} c(t)=k$ (respuesta estacionaria).
- La respuesta transitoria es dominada por $e^{-t/\tau}$.
- $c(\tau) = 0.632k$, el sistema alcanza el $63.2\%$ del valor final después de un tiempo $\tau$.
- $c(2\tau) = 0.865k$, el sistema alcanza el $86.5\%$ del valor final después de un tiempo $2\tau$.
- $c(3\tau) = 0.95k$, el sistema alcanza el $95\%$ del valor final después de un tiempo $3\tau$.
- $\frac{dc}{dt}\big\vert_{t = 0^+} = \frac{d}{dt} \left ( k \left ( 1 - e^{-t/\tau} \right ) \right ) \big\vert_{t = 0^+} = \frac{k}{\tau}  e^{-t/\tau} \big\vert_{t = 0^+} =\frac{k}{\tau}$
- El sistema tiene un polo ubicado en $s=\frac{-1}{\tau}$

```python
k, tau = sym.symbols('k, tau',real=True, positive=True)
t = sym.symbols('t',real=True, positive=True)

s = sym.Symbol('s')

def L(f):
    return sym.laplace_transform(f, t, s, noconds=True)
def invL(F):
    return sym.inverse_laplace_transform(F, s, t)
```

```python
# la entrada r(t) es escalon unitario
r = sym.Heaviside(t)
r
```

```python
# la entrada R(s) es la transformada de Laplace de r(t)
R = L(r)
R
```

```python
# Se define la función de transferencia del sistema G(s)
G = k/(tau*s + 1)
g =invL(G)
G
```

```python
# la salida se calcula como C(s) = G(s)R(s)
C = G*R
C
```

```python
# la salida c(t) es la transformada inversa de Laplace de C(s)
c = invL(C)
c
```

Hasta este punto se ha encontrado la respuesta general de un sistema de primer orden ante una entrada escalón. Se recomienda variar los parámetros del sistema para identificar cómo varía su comportamiento.

```python
# Para valores particulares de k y tau
K = 3
T = 30
kt = {k: K, tau: T}
ckt = c.subs(kt)
ckt
```

```python
plt.rcParams['figure.figsize'] = 10, 5
g0 = sym.plot(K,(t,0,500),show = False, line_color='green');
g1 = sym.plot(ckt, (t,0,500),show = False, line_color='blue')
g2 = sym.plot(t*K/T,(t,0,T),show = False, line_color='red');
g0.extend(g1)
g0.extend(g2)
g0.show()
kt
```

### Respesta ante la rampa

Si el sistema es excitado con una rampa unitaria, la señal de salida en el dominio de la frecuencia es:

\begin{equation}
C(s) = G(s) \cdot R(s) = \frac{k}{\tau s + 1} \cdot \frac{1}{s^2}
\end{equation}

Observe que esta expresión puede reescrbirse como:

\begin{equation}
C(s)= k \cdot \left ( \frac{1}{s^2} - \frac{\tau}{s} + \frac{\tau^2}{\tau s + 1} \right )
\end{equation}

La transformada inversa de Laplace permite obtener una expresión para la señal de salida en el dominio del tiempo $c(t)$.

\begin{equation}
c(t)= k \left ( t - \tau + \tau e^{-t/\tau} \right ) \epsilon(t)
\end{equation}

Observe que:

- $c(t)$ se compone de una rampa ($t$), una constante ($\tau$) y una exponencial ($\tau e^{-t/\tau}$).
- $c(t<0)=0$ (condción inicial).
- A medida que $t\rightarrow \infty$, $\tau e^{-t/\tau} \rightarrow 0$, por lo que $\lim_{t\rightarrow \infty} c(t)=k \left ( t - \tau \right )$, es decir hace un seguimiento de la rampa con una desviación que depende de $k$ y $\tau$.

- $\lim_{t\rightarrow \infty} \frac{dc}{dt} = \lim_{t\rightarrow \infty} k \left (1 - e^{-t/\tau} \right ) = k$

```python
# la salida se calcula como C(s) = G(s)R(s)
C_ramp = G*(1/s**2)
C_ramp
```

```python
# la salida c_ramp(t) es la transformada inversa de Laplace de C(s)
c_ramp = invL(C_ramp)
c_ramp
```

```python
# Para valores particulares de k y tau
K = 0.9
T = 1
kt = {k: K, tau: T}
c_rampkt = c_ramp.subs(kt)
c_rampkt
```

```python
plt.rcParams['figure.figsize'] = 9, 3
g0 = sym.plot(t,(t,0,10),show = False, line_color='green');
g1 = sym.plot(c_rampkt, (t,0,10),show = False, line_color='blue')
g2 = sym.plot(t*K,(t,0,10),show = False, line_color='red');
g0.extend(g1)
g0.extend(g2)
g0.show()
kt
```

```python
from ipywidgets import interact

evalfimpulse = sym.lambdify((k, tau, t), g , 'numpy')
evalfstep = sym.lambdify((k, tau, t), c, 'numpy')
evalframp = sym.lambdify((k, tau, t), c_ramp, 'numpy')

ts = np.linspace(0, 10)
def orden1(tau_in, K_in):
    plt.figure(figsize=(7, 5))
    ax_impulse = plt.subplot2grid((3, 2), (0, 0))
    ax_step = plt.subplot2grid((3, 2), (1, 0))
    ax_ramp = plt.subplot2grid((3, 2), (2, 0))
    ax_complex = plt.subplot2grid((2, 2), (0, 1), rowspan=2)

    ax_impulse.plot(ts, evalfimpulse(K_in, tau_in, ts))
    ax_impulse.set_title('Resp. al impulso')
    ax_impulse.set_ylim(0, 10)

    tau_height = 1 - np.exp(-1)
    ax_step.set_title('Resp. al escalón')
    ax_step.plot(ts, evalfstep(K_in, tau_in, ts),ts,K_in*ts**0)
    ax_step.set_ylim(0, 10)

    ax_ramp.set_title('Resp. a la rampa')
    ax_ramp.plot(ts, evalframp(K_in, tau_in, ts),ts,K_in*ts)
    ax_ramp.set_ylim(0, 100)

    ax_complex.set_title('Polos y ceros')
    ax_complex.scatter(-1/tau_in, 0, marker='x', s=30)
    ax_complex.axhline(0, color='black')
    ax_complex.axvline(0, color='black')
    ax_complex.axis([-10, 10, -10, 10])

def orden11(tau_in, K_in):
    plt.figure(figsize=(6, 3))
    ax_step = plt.subplot2grid((1, 2), (0, 0))
    ax_complex = plt.subplot2grid((1, 2), (0, 1), rowspan=2)

    tau_height = 1 - np.exp(-1)
    ax_step.set_title('Resp. al escalón')
    ax_step.plot(ts, evalfstep(K_in, tau_in, ts),ts,K_in*ts**0)
    ax_step.set_ylim(-6, 6)

    ax_complex.set_title('Polos y ceros')
    ax_complex.scatter(-1/tau_in, 0, marker='x', s=30)
    ax_complex.axhline(0, color='black')
    ax_complex.axvline(0, color='black')
    ax_complex.axis([-5, 5, -5, 5])

interact(orden11, tau_in=(-2.0, 3.0), K_in=(-5.0, 5.0));
```

    interactive(children=(FloatSlider(value=0.5, description='tau_in', max=3.0, min=-2.0), FloatSlider(value=0.0, …

Una vez analizadas las simulaciones anteriores, comparta sus ideas para obtener modelos de primer orden de forma experimental.

## Sistemas de segundo orden

Considere la siguiente función de transferencia:

\begin{equation}
G(s) = \frac{C(s)}{R(s)} = \frac{a}{b s^2 + c s + d}
\end{equation}

Este sistema tiene $4$ parámetros ($a$, $b$, $c$ y $d$), sin embargo, para facilitar el análisis de estos sistemas, puede transformarse la función a su forma canónica al dividir numerador y denominador por $b$.

\begin{equation}
G(s) = \frac{C(s)}{R(s)} = \frac{\frac{a}{b}}{\frac{b}{b} s^2 + \frac{c}{b} s + \frac{d}{b}} = \frac{k\omega_0^2}{s^2 + 2 \zeta\omega_0s + \omega_0^2}
\end{equation}

En la forma canónica, se definen los parámetro como:
- $k$: ganancia en estado estacionario.
- $\omega_0$: frecuencia natural.
- $\zeta$: coeficiente de amortiguación.

Considerando que el sistema es estable, puede observarse que:
- Este sistema no tiene **ceros**, pues no existe un valor de $s$ que haga que el numerador $k\omega_0^2$ sea igual a cero.
- El sistema tiene dos **polos** ubicados en

$$s_1,s_2 =-\zeta \omega_0 \pm \omega_0\sqrt{\zeta^2 - 1}$$

Así, deben considerarse 3 casos:
1. $\zeta < 1$, por lo cual $\sqrt{\zeta^2 - 1}$ arroja un parte imaginaria y el sistema tiene dos polos complejos conjugados. En este caso, el sistema es **subamortiguado**.

2. $\zeta = 1$, por lo cual $\sqrt{\zeta^2 - 1} = 0$ y el sistema tendría dos polos reales repetidos. En este caso, el sistema es **críticamente amortiguado**.

3. $\zeta > 1$, por lo cual $\sqrt{\zeta^2 - 1}$ arroja un parte real y el sistema tiene dos polos reales diferentes. En este caso, el sistema es **sobreamortiguado**.

### Caso subamortiguado

La función de transferencia del sistema es:

\begin{equation}
G(s) = \frac{C(s)}{R(s)} = \frac{k\omega_0^2}{s^2 + 2 \zeta\omega_0s + \omega_0^2}
\end{equation}

En este caso se considera que $\zeta < 1$, por lo cual $\sqrt{\zeta^2 - 1}$ arroja un parte imaginaria y el sistema tiene dos polos complejos conjugados.

#### Respuesta al escalón

Si el sistema es excitado con un escalón unitario, la señal de salida en el dominio de la frecuencia es:

\begin{equation}
C(s) = G(s) \cdot R(s) = \frac{k\omega_0^2}{s^2 + 2 \zeta\omega_0s + \omega_0^2} \cdot \frac{1}{s}
\end{equation}

Observe que esta expresión puede reescrbirse como:

\begin{equation}
C(s)= k \cdot \left ( \frac{1}{s} - \frac{s + 2\zeta\omega_0}{s^2 + 2\zeta\omega_0s + \omega_0^2} \right )
\end{equation}

Suponiendo que la parte imaginaria de los polos es $\omega_d = \omega_0\sqrt{1-\zeta^2}$, la respuesta se puede reescribir como:

\begin{equation}
C(s)= k \cdot \left ( \frac{1}{s} - \frac{s + \zeta\omega_0}{(s+\zeta\omega_0)^2 + \omega_d^2} - \frac{\zeta\omega_0}{(s+\zeta\omega_0)^2 + \omega_d^2} \right )
\end{equation}

La transformada inversa de Laplace permite obtener una expresión para la señal de salida en el dominio del tiempo $c(t)$.

\begin{align}
c(t) &= k \left ( 1 - e^{-\zeta\omega_0 t} \cos{\omega_d t} - \frac{\zeta\omega_0}{\omega_d}e^{-\zeta\omega_0 t} \sin{\omega_d t}\right ) \epsilon(t) \\
&= k \left ( 1 - e^{-\zeta\omega_0 t} \cos{\omega_d t} - \frac{\zeta}{\sqrt{1-\zeta^2}}e^{-\zeta\omega_0 t} \sin{\omega_d t}\right ) \epsilon(t) \\
&= k \left ( 1 - \frac{e^{-\zeta\omega_0 t}}{\sqrt{1-\zeta^2}} \sin \left (\omega_d t + \tan^{-1} \frac{\sqrt{1-\zeta^2}}{\zeta} \right )\right ) \epsilon(t) \\
&= k \left ( 1 - \frac{e^{-\zeta\omega_0 t}}{\sqrt{1-\zeta^2}} \sin \left (\omega_0 \sqrt{1-\zeta^2} t + \tan^{-1} \frac{\sqrt{1-\zeta^2}}{\zeta} \right )\right ) \epsilon(t)
\end{align}

Observe que:

- $c(t<0)=0$ (condción inicial).
- $\lim_{t\rightarrow \infty} c(t)=k$ (respuesta estacionaria).
- La respuesta transitoria es dominada por $e^{-\zeta\omega_0 t}$ siendo modulada por una oscilación cuya frecuencia es $\omega_d = \omega_0 \sqrt{1-\zeta^2}$.
- Si $\zeta = 0 $, el sistema oscilaría eternamente.
- El sistema tiene dos polos ubicado en $s_1,s_2 =-\zeta \omega_0 \pm j \omega_d$

```python
k, omega0, zeta, omegad = sym.symbols('k, omega_0, zeta, omega_d',real=True, positive=True)
t = sym.symbols('t',real=True, positive=True)
wd = {omegad : omega0*sym.sqrt(1-zeta**2)}

s = sym.Symbol('s')

G2_sub = k*omega0**2/((s + zeta*omega0 + omegad*1j)*(s + zeta*omega0 - omegad*1j))
G2_sub # La función de transferencia
```

```python
G2_sub_step = G2_sub*(1/s) # La respuesta ante el escalón
G2_sub_step = G2_sub_step.apart(s)   # Descompone larespuesta al escalón en fracciones parciales
G2_sub_step
```

```python
g2_sub_step = invL(G2_sub_step).subs(wd).factor().simplify()
g2_sub_step
```

```python
parametros = {k: 1,omega0: 8, zeta: 0.25}
sym.plot(g2_sub_step.subs(parametros),(t,0,10))

```

    <IPython.core.display.Javascript object>

    <sympy.plotting.plot.Plot at 0x2a8e75aeba8>

### Caso críticamente amortiguado

En este caso se considera que $\zeta = 1$, por lo cual $\sqrt{\zeta^2 - 1}$ arroja un parte imaginaria igual a cero polos reales repetidos. La función de transferencia del sistema queda:

\begin{equation}
G(s) = \frac{C(s)}{R(s)} = \frac{k\omega_0^2}{s^2 + 2 \omega_0s + \omega_0^2} =  \frac{k\omega_0^2}{\left (s + \omega_0 \right )^2}
\end{equation}

#### Respuesta al escalón

Si el sistema es excitado con un escalón unitario, la señal de salida en el dominio de la frecuencia es:

\begin{equation}
C(s) = G(s) \cdot R(s) = \frac{k\omega_0^2}{\left (s + \omega_0 \right )^2} \cdot \frac{1}{s}
\end{equation}

Observe que esta expresión puede reescrbirse como:

\begin{equation}
C(s)= k \cdot \left ( \frac{1}{s} - \frac{\omega_0}{\left ( s + \omega_0 \right )^2} - \frac{k}{s + \omega_0} \right )
\end{equation}

Aplicando la transformada inversa se obtiene:

\begin{align}
c(t) &= k \left ( 1 - \omega_0 t e^{-\omega_0 t} - e^{-\omega_0 t} \right ) \epsilon(t) \\
&= k \left ( 1 - (1 + \omega_0 t  )e^{-\omega_0 t} \right ) \epsilon(t)
\end{align}

- $c(t<0)=0$ (condción inicial).
- $\lim_{t\rightarrow \infty} c(t)=k$ (respuesta estacionaria).
- La respuesta transitoria es dominada por $(1 + \omega_0 t  )e^{-\omega_0 t}$.
- El sistema tiene dos polos ubicados en $s_1,s_2 =-\omega_0$

```python
k, omega0 = sym.symbols('k, omega_0',real=True, positive=True)
t = sym.symbols('t',real=True, positive=True)

s = sym.Symbol('s')

G2_cri = k*omega0**2/(s + omega0)**2
G2_cri # La función de transferencia
```

```python
G2_cri_step = G2_cri*(1/s) # La respuesta ante el escalón
G2_cri_step = G2_cri_step.apart(s)   # Descompone larespuesta al escalón en fracciones parciales
G2_cri_step
```

```python
g2_cri_step = invL(G2_cri_step).factor().simplify()
g2_cri_step
```

```python
parametros = {k: 5,omega0: 6}
sym.plot(g2_cri_step.subs(parametros),(t,0,2))
```

    <IPython.core.display.Javascript object>

    <sympy.plotting.plot.Plot at 0x2a8eb3b8b00>

### Caso sobreamortiguado

En este caso se considera que $\zeta > 1$, por lo cual $\sqrt{\zeta^2 - 1}$ arroja un parte imaginaria igual a cero y polos reales diferentes en $s_1,s_2 = \omega_0 (-\zeta  \pm \sqrt{\zeta^2 - 1})$. La función de transferencia del sistema queda:

\begin{equation}
G(s) = \frac{C(s)}{R(s)} = \frac{k\omega_0^2}{s^2 + 2 \zeta \omega_0s + \omega_0^2} =  \frac{k s_1 s_2}{\left (-s + s_1 \right ) \left (-s + s_2 \right )}
\end{equation}

#### Respuesta al escalón

Si el sistema es excitado con un escalón unitario, la señal de salida en el dominio de la frecuencia es:

\begin{equation}
C(s) = G(s) \cdot R(s) = \frac{k s_1 s_2}{\left (-s + s_1 \right ) \left (-s + s_2 \right )} \cdot \frac{1}{s}
\end{equation}

Observe que esta expresión puede reescrbirse como:

\begin{equation}
C(s)= k \cdot \left ( \frac{1}{s} - \frac{s_1}{s_1 - s_2}\frac{1}{s+s_2} + \frac{s_2}{s_1 - s_2}\frac{1}{s+s_1} \right )
\end{equation}

\begin{equation}
C(s)= k \cdot \left ( \frac{1}{s} +  \frac{1}{s_1 - s_2} \left (  \frac{s_1}{s+s_2} + \frac{s_2}{s+s_1} \right )  \right )
\end{equation}

Aplicando la transformada inversa se obtiene:

\begin{equation}
c(t)= k \cdot \left ( 1 +  \frac{1}{s_1 - s_2} \left (s_1 e^{-s_2 t} +  s_2 e^{-s_1 t} \right )  \right )
\end{equation}

- $c(t<0)=0$ (condción inicial).
- $\lim_{t\rightarrow \infty} c(t)=k$ (respuesta estacionaria).
- La respuesta transitoria es dominada por dos exponenciales reales.
- El sistema tiene dos polos reales ubicados en $s_1,s_2 = \omega_0 (-\zeta  \pm \sqrt{\zeta^2 - 1})$

```python
k, s1,s2,zeta,omega0 = sym.symbols('k, s_1, s_2,\zeta,\omega_0',real=True, positive=True)
t = sym.symbols('t',real=True, positive=True)

s = sym.Symbol('s')

G2_sob = k*s1*s2/((-s + s1)*(-s+s2))
G2_sob # La función de transferencia
```

```python
G2_sob_step = G2_sob*(1/s) # La respuesta ante el escalón
G2_sob_step = G2_sob_step.apart(s)   # Descompone larespuesta al escalón en fracciones parciales
G2_sob_step
```

```python
g2_sob_step = invL(G2_sob_step).factor().simplify()
g2_sob_step
```

```python
s1s2 = {s1: omega0*(-zeta + sym.sqrt(zeta**2-1)) ,s2:  omega0*(-zeta - sym.sqrt(zeta**2-1))}
# \omega_0 (-\zeta  \pm \sqrt{\zeta^2 - 1})
g2_sob_step = g2_sob_step.subs(s1s2).simplify()
g2_sob_step
```

```python
parametros1 = {k: 1.0,omega0: 2.0, zeta: 1.1}
parametros2 = {k: 1.0,omega0: 2.0, zeta: 2.5}
parametros3 = {k: 1.0,omega0: 2.0, zeta: 5}

g1 = sym.plot(g2_sob_step.subs(parametros1),
              (t,0,10),show=False,line_color='green')
g2 = sym.plot(g2_sob_step.subs(parametros2),
              (t,0,10),show=False, line_color='blue')
g3 = sym.plot(g2_sob_step.subs(parametros3),
              (t,0,10),show=False, line_color='red')
g1.extend(g2)
g1.extend(g3)
g1.show()
```

**Pregunta**
¿Qué esperaría que ocurra si el coeficiente de amoriguamiento de un sistema de segundo orden es "muy grande"?


# Basic New Keynesian (NK) model

This introductory notebook demonstrates the basics of the New-Keynesian (NK) model used in economics

The materials and equations are from Chapter 3 of following book:
- Costa, C. (2018). Understanding DSGE models: Theory and Applications. Vernon Press.
    - [Amazon link for book](https://www.amazon.com/gp/product/B01LXPPOA6?pf_rd_r=ZHR3A69BS93PKMTAXK0P&pf_rd_p=edaba0ee-c2fe-4124-9f5d-b31d6b1bfbee)

This book is brilliant and I highly recommend to anyone interested in studying the DSGE modeling regime used in macroeconomics.

# Table of contents

- [1. Theoretical review](#1.-Theoretical-review)
- [2. The NK model](#2.-The-NK-Model)
- [3. Equilibrium conditions](#3.-Equilibrium-conditions)
- [4. Solve steady state](#4.-Solve-steady-state)
- [5. Log-linearization](#5.-Log-linearization)

```python
#Import libraries
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import warnings
warnings.filterwarnings("ignore")
plt.style.use('seaborn-darkgrid')
```

# 1. Theoretical review

#### NK model structure
- Household behavior: similar to RBC models

- Firm behavior: vastly different than RBC because we no longer assume imperfect competition
    - As a result we now include two sectors of firms:
        1. Wholesalers: producers of intermediate goods
            - Imperfect competition only assumed in the wholesaler market!
        2. Retailers: producers of final goods
            - Perfect competition assumed

## 1.1 Differentiated products and the consumption aggregator

Consumption aggregator function
$$C=C(c_1,c_2,\cdots, C_N)=\Big[(c_1)^{\frac{\psi-1}{\psi}}+(c_2)^{\frac{\psi-1}{\psi}}(c_N)^{\frac{\psi-1}{\psi}}\Big]^{{\frac{\psi}{\psi-1}}}$$
Where:
- $c_1$ = Type 1 of consumer good
- $c_2$ = Type 2 of consumer good
- $\psi$= Elasticity of substitution between different goods
    - $\psi\rightarrow\infty$: goods are perfect substitutes
    - However, NK models usually assume $\frac{\psi-1}{\psi}$

Must satisfy:
1. $\frac{\partial c (.)}{\partial c_j}>0$
2. $\frac{\partial^2 c (.)}{\partial c_j^2}<0$
    - Total consumption is an increasing consumption function of a j type good

### Elasticity of substitution between two goods

$$\phi = \frac{\partial(c_1/c_2)}{\partial MRS}\cdot\frac{MRS}{c_1/c_2}=\frac{\partial Ln(c_1/c_2)}{\partial LnMRS}$$

## 1.2 Firms in monopolistic competition

- Firms in monopolistic competition have some degree of markets power
    - As a result the demand curve is negatievely sloped
    - Max profit: $P_j>MC_j$
    - NK models use monopolisitic competition to model the goods market

### 1.2.1 Python example: Monopolistic competition

```python
def imperfect_comp(Q, P, Total_cost):
    df = pd.DataFrame(data=np.vstack([Q,P,Total_cost]).T, columns= ['Q', 'P', 'TC'])
    df["TR"] = Q*P
    df["Profit"] = df.TR-df.TC
    df["MC"] = df.TC.diff()/df.Q.diff()
    df["ATC"] = df.TC/df.Q
    df["MR"] = df.TR.diff()/df.Q.diff() #Same as demand curve
    production = list(df.loc[df.MR==df.MC, ['Q','P']].values)
    plt.plot(Q,df.MC, label='Marginal cost')
    plt.plot(Q,df.MR, label='Marginal rev')
    plt.plot(Q,df.ATC, label='ATC')
    plt.title('Firms produce @ Q:{} P:{}'.format(production[0][0],production[0][1]))
    plt.legend();
    return round(df,2)

Quantities = np.linspace(10,80,8)
Price = [23,20,18,16,14,12,10,8]
Total_cost = [340,400,480,580,700,840,1020,1280]
imperfect_comp(Quantities, Price, Total_cost)
```

## 1.3 Price stickiness
- NK models also focus on the stickiness of:
    1. Nominal wages
    2. Product prices

- Stickness can apply to both real and nominal prices

- Stickiness occurs when price of a good or wages do not change in response shifts in demand and supply

# 2. The NK Model

## 2.1 Model assumptions from RBC
- Assumption 2.2.1. The economy is closed, with no government or financial sector.
- Assumption 2.2.2. This economy does not have a currency (Barter economy)
- Assumption 2.2.3. Adjustment costs are not present

## 2.2 Households

### 2.2.1 Households assumptions
- Assumption 2.2.4. The model is made up of a unitary set of households indexed by $j \in[0,1]$ whose goal is to maximize intertemporal welfare function.
    - A utility function is used, additively separable into consumption (C) and labor (L).
- Assumption 2.2.5. Consumption is intertemporally additively separable (no habit formation).
- Assumption 2.2.6. Population growth is ignored
- Assumption 2.2.7. The labor market is of perfect competition (no wage rigidity).

### 2.2.2 Equations from the [RBC model](https://github.com/corybaird/DSGE_models/blob/master/RBC_Basic_Costa2018.ipynb)
- Format: (RBC eq #/NK eq #)

#### Capital accumulation over time:
$$K_{j,t+1} = (1-\delta)K_{j,t}+I_{j,t}\tag{3.1}$$

#### Household labor supply equation
Consumption-leisure MRS = Consumption-leisure relative price

$$
C_{j,t}^\sigma L^\varphi_{j,t} = \frac{W_t}{P_t}\tag{3.2}
$$

#### Euler equation
$$\Big(\frac{E_t C_{j,t+1}}{C_{j,t}}\Big)^\sigma = \beta\Big[(1-\delta)+ E_t\frac{R_{t+1}}{P_{t+1}}\Big]
\tag{3.3}$$

## 2.3 Firms
### 2.3.1 Firm assumptions
- Assumption 2.2.9. Productivity growth is ignored.

But we have a new assumption for the NK model:
- Assumption 3.2.1. It is assumed that the market structure is one of monopolistic competition.
    - Assumption 2.2.8 (Perfect competition) no longer holds

Producing sector divided into two parts:
1. Wholesale firms: Intermediate goods
    - Many companies each producing differentiable goods
        - They are able to decide quantity and prices

2. Retail firms: Final goods

### 2.3.2 Retail firms (Final goods)
Assumption 3.2.2. Aggregate good sold by one retail firm within a structure of perfect competition. Assume all retailers are identical

- Similar to the firm assumptions made in RBC model

- There are many wholesale goods each indexed within the interval [0,1].
- Each good is produced by a single wholesaler and is imperfectly substitutable.
    - This gives wholesalers a degree of monopoly power.

To represent the problem retail firms face one widely functional form is the Dixit-Stiglitz aggregator (Dixit and Stiglitz, 1977):

$$Y_t=\Big(\int_0^1 Y_{j,t}^{\frac{\psi-1}{\psi}} dj\Big)^{\frac{\psi}{\psi-1}}\tag{3.4}$$

Where:
- $Y_t$= Product of retailers in period t
- $Y_{j,t} \text{for} \ \  j \in[0,1]=$ wholesale good j
- $\psi>1$= substitution between wholesale goods

#### Retail firm maximizing its profit function:

$P_t$= nominal price of retail product $P_{j,t}$= nominal price of wholesale good j

$$\max_{Y_{j,t}}P_tY_t=\int_0^1 P_{j,t}Y_{j,t} dj\tag{3.5}$$

$3.4\rightarrow3.5$

$$\max_{Y_{j,t}}P_t \Big(\int_0^1 Y_{j,t}^{\frac{\psi-1}{\psi}} dj\Big)^{\frac{\psi}{\psi-1}}- P_{j,t}\int_0^1 Y_{j,t} dj\tag{3.6}$$

First order condition of 3.6

Rearrange

Re-write 3.4

Eliminate RHS of f.o.c

Raise expression to power of $\psi$
$$Y_{j,t}=Y_t\big(\frac{P_t}{P_{j,t}}\big)^\psi\tag{3.7}$$

$3.7\rightarrow3.4$

#### Pricing rule for final (retail goods)
$$P_t = \Big[\int_0^1P_{j,t}^{1-\psi}dj\Big]^{\frac{1}{1-\psi}}\tag{3.8}$$

### 2.3.3 Whole-sale firms (intermediate goods)

#### 2.3.3.1 Assumptions
- Assumption 3.2.3. Monopolistic competition
    - Wholesale firms with differentiated prodcuts have a degree of market power and can thus set prices.

- Assumption 3.2.4. Fixed production costs do not exist.
    - VC=ATC.

- Assumption 3.2.5. Per unit production cost of a wholesale product is always the same regardless of the scale of production.
    - CRS

Assumptions 3.2.4 & 3.2.5 mean that TC = Q*MC

#### 2.3.3.2 Minimization & constraint
Whole sale firms solve maximization problem in two stages:
1. The firms takes prices on the factors of production
2. The firms determines the amount of capital and labor it will use to minimize production costs

$$\min_{L_{j,t},K_{j,t}} W_tL_{j,t}+R_tK_{j,t}\tag{3.9}$$

s.j.t technology
$$Y_{j,t}=A_tK_{j,t}^\alpha L_{j,t}^{1-\alpha}\tag{3.10}$$

With the law of motion of productivity
$$logA_t =(1-\rho_a)logA_{ss}+\rho_A logA_{t-1}+\varepsilon_t \tag{3.11}$$

Where:
- $A_{ss}$= value of productivity at steady state
- $\rho_{A}$= autoregressive paramater of productiity

#### 2.3.3.3 Optimization: Lagrangian
$$L= W_tL_{j,t}+R_tK_{j,t}+\mu_{j,t}\Big(Y_{j,t}-A_tK_{j,t}^\alpha L_{j,t}^{1-\alpha}\Big)\tag{3.12}$$

F.O.C.

$$
\frac{\partial L}{\partial L_{j,t}}= W_t -(1-\alpha)\mu_{j,t}A_tK_{j,t}^\alpha L^\alpha_{j,t}\tag{3.13}
$$
$$
\frac{\partial L}{\partial K_{j,t}}= R_t -\alpha\mu_{j,t}A_tK_{j,t}^{\alpha-1} L^{1-\alpha}_{j,t}\tag{3.14}
$$

With $\mu_{j,t}=MC_{j,t}$ 3.13. and 3.14 become

$$L_{j,t}= (1-\alpha)MC_{j,t}\frac{Y_{j,t}}{W_t}\tag{3.15}$$
$$K_{j,t}= \alpha MC_{j,t}\frac{Y_{j,t}}{R_t}\tag{3.16}$$

#### Production technology is the same as RBC
$$
MC_{j,t} = \frac{1}{A_t}\Big(\frac{W_t}{1-\alpha}\Big)^{1-\alpha}\Big(\frac{R_t}{\alpha}\Big)^\alpha\tag{3.17}
$$

#### 2.3.3.4 Calvo pricing
- Whole sale firms decide the price of its good
- The firms deides how much to produce in each period according to the Calvo rule

The following stickiness rules exist:
1. Maintain the previous periods price
$$P_{j,t}=P_{j,t-1}$$

2. Update price using steady state gross inflation
$$P_{j,t}=\pi_{ss}P_{j,t-1}$$

3. Update the price using previous period's gross inflation rate
$$P_{j,t}=\pi_{t-1}P_{j,t-1}$$

- Assumption 3.2.6.  Rule to determine price stickiness: $P_{j,t} = P_{j,t−1}$

- Wholesale firm has a $\theta$ probability of price of its good fixed in next period

Whole sale firm capable of adjusting price of its goods:

$$\max_{P^*_{j,t}}E_t \sum_{i=0}^{\infty} (\beta \theta)^i (P^*_{j,t}Y_{j,t+1}-TC_{j,t+i})\tag{3.18}$$

$3.7\rightarrow3.18$

Take problem's F.O.C

Rearrange

#### Optimal price level
$$P^*_{j,t}=\Big(\frac{\psi}{\psi-1}\Big) E_t \sum_{i=0}^\infty (\beta \theta)^i MC_{j,t+i}\tag{3.20}$$

In periods $P^*_{j,t}$ is the same price for all 1-$\theta$ firms$ which set their prices

Combine 3.8 pricing rule with reality that firms within certain groups of firms define their prices w.r.t stickiness of prices

#### General (aggregate) price level:

$$P_t = \Big[\theta P_{t-1}^{1-\psi}+(1-\theta)P_t^{*1-\psi}\Big]^{\frac{1}{1-\psi}}\tag{3.21}$$

# 3. Equilibrium conditions

### 3.4.1 Structure of the model

$$
\begin{align}
C_t^\sigma L_t^\varphi & = \frac{W_t}{P_t} & \text{Labor supply, 2.8} \\
\Big(\frac{E_t C_{j,t+1}}{C_{j,t}}\Big)^\sigma &= \beta\Big[(1-\delta)+ E_t\frac{R_{t+1}}{P_{t+1}}\Big]& \text{Euler, 2.9} \\
K_{t+1} & = (1-\delta)K_t + I_t& \text{Law of motion of capital, 2.3}\\
y_t & = A_t K_t^\alpha L_t^{1 - \alpha} & \text{Production function, 2.10} \\
K_t & =\alpha MC_t \frac{Y_t}{R_t} & \text{Demand for capital, 3.16} \\
L_t & =(1-\alpha)\frac{Y_t}{\frac{W_t}{P_t}} & \text{Demand for labor, 3.15} \\
P^*_{j,t} & =\Big(\frac{\psi}{\psi-1}\Big) E_t \sum_{i=0}^\infty (\beta \theta)^i MC_{j,t+i} & \text{Optimal price level, 3.20 }\\
P_t & = \Big[\theta P_{t-1}^{1-\psi}+(1-\theta)P_t^{*1-\psi}\Big]^{\frac{1}{1-\psi}}& \text{General price level, 3.21 }\\
\pi_t & =\frac{P_t}{P_{t-1}}& \text{Gross inflation rate}\\
Y_t & =C_t+I_t & \text{Equilibrium condition}\\
logA_t & =(1-\rho_a)logA_{ss}+\rho_A logA_{t-1}+\varepsilon_t& \text{Productivity shock, 2.16}
\end{align}
$$

## 3.4.2 Steady state

$$
A_{ss} = 1 \\
P_{ss} = 1 \\
R_{ss} = P_{ss}\Big[\frac{1}{\beta}-(1-\delta)\Big]\\
MC_{ss}= \Big(\frac{\psi}{\psi-1}\Big) (1-\beta\theta)P_{ss} \ \ \leftarrow 3.21 \\
W_{ss} = (1-\delta)MC_{ss}^{\frac{1}{1-\alpha}}\Big(\frac{\alpha}{R_{ss}}\Big)^{\frac{\alpha}{1-\alpha}}\\
Y_{ss} = \Big(\frac{R_{ss}}{R_{ss}-\delta\alpha MC_{ss}}\Big)^{\frac{\sigma}{\sigma+\varphi}}+\Bigg[\frac{W_{ss}}{P_{ss}}\Big(\frac{W_{ss}}{(1-\alpha) MC_{ss}}\Big)^{\varphi}\Bigg]^{\frac{1}{\sigma+\varphi}}\\
I_{ss}=\Big(\frac{\delta\alpha MC_{ss}}{R_{ss}}\Big)Y_{ss}\\
C_{ss}= \frac{1}{Y_{ss}^{\frac{\varphi}{\sigma}}}\Big[\frac{W_{ss}}{P_{ss}}\Big(\frac{W_{ss}}{(1-\alpha)MC_{ss}}\Big)^{\varphi}\Big]^{1/\sigma}\\
L_{ss}=(1-\alpha)MC_{ss}\frac{Y_{ss}}{W_{ss}}\\
K_{ss}=\alpha MC_{ss}\frac{Y_{ss}}{R_{ss}}\\
$$

# 4. Solve steady state

## 4.1 Set parameters names and values

```python
parameter_names = [
    'relative risk',
    'marginal disutility of labor',
    'elasticity of production w.r.t K',
    'discount factor',
    'depreciation rate',
    'price',
    'price stickiness',
    'elasticity of substitution between intermediate goods'
]
# Setup fixed parameters
parameters = pd.DataFrame({
    'name': parameter_names,
    'value': [2, 1.5, 0.35, 0.985, 0.025,1, .75,8]
})
parameters.T
```

      0
      1
      2
      3
      4
      5
      6
      7

      name
      relative risk
      marginal disutility of labor
      elasticity of production w.r.t K
      discount factor
      depreciation rate
      price
      price stickiness
      elasticity of substitution between intermediat...

      value
      2
      1.5
      0.35
      0.985
      0.025
      1
      0.75
      8

## 4.2 Solve model

### 4.2.1 Create model function

```python
class New_Keynes(object):
    def __init__(self, params=None):
        # Model dimensions
        self.k_params = 8

        # Initialize parameters
        if params is not None:
            self.update(params)

    def update(self, params):
        self.risk = params[0]
        self.disutility_labor = params[1]
        self.elasticity_capital = params[2]
        self.discount_factor = params[3]
        self.depreciation = params[4]
        self.price = params[5]
        self.sticky = params[6]
        self.intermediate_sub = params[7]

    def steady_state(self):
        #Rates: Idential to RBC
        rates = self.price*((1/self.discount_factor)-(1-self.depreciation))

        # Marginal cost: New to NK model
        marginal = ((self.intermediate_sub-1)/(self.intermediate_sub))*(1-self.discount_factor*self.sticky)*self.price

        # Wages: New to NK model
        wages = (1-self.elasticity_capital)*marginal**(1/(1-self.elasticity_capital))*(self.elasticity_capital/rates)**(self.elasticity_capital/(1-self.elasticity_capital))

        #Output: New to NK model
        output_1 = ((rates)/(rates-(self.depreciation*self.elasticity_capital*marginal)))**((self.risk)/(self.risk+self.disutility_labor))
        output_2 = ((wages/self.price)*((wages)/(((1-self.elasticity_capital)*marginal)))**(self.disutility_labor))**((1/(self.risk+self.disutility_labor)))
        output = output_1*output_2

        #Investment: New to NK model
        investment = ((self.depreciation*self.elasticity_capital*marginal)/rates)*output #self.depreciation*self.elasticity_capital*MARGINAL

        #Consumption: New to NK model
        cons_1 = (1/(output**(self.disutility_labor/self.risk)))
        cons_2 = ((wages/self.price)*((wages)/((1-self.elasticity_capital)*marginal))**(self.disutility_labor))**(1/self.risk)
        consumption = cons_1*cons_2

        #Capital: New to NK model
        labor = (1-self.elasticity_capital)*marginal*(output/wages)

        #Labor: New to NK model
        capital = (self.elasticity_capital*marginal)*((output)/(rates))

        return (rates, marginal, wages, output, investment, consumption, labor,capital)
```

### 4.2.2 Steady state solution

```python
NK_solution = New_Keynes(parameters['value'])
Steady_state_solution = NK_solution.steady_state()
```

### 4.2.3 Print results
- Solution matches table 2.3 Costa pg. 47

```python
variable_names = [
    'Interest rates','Marginal cost', 'Wages', 'Output',
    'Investment', 'Consumption', 'Labor','Capital'
]
steady_state = pd.DataFrame({
    'Solution': Steady_state_solution,
}, index=variable_names)

round(steady_state.T,3)
```

      Interest rates
      Marginal cost
      Wages
      Output
      Investment
      Consumption
      Labor
      Capital

      Solution
      0.04
      0.229
      0.215
      0.778
      0.039
      0.739
      0.537
      1.547

# 5. Log-linearization
$$\sigma \tilde{C_t}+\varphi\tilde{L_t} = \tilde{W_t}-\tilde{P_t}\tag{2.39}$$
$$\frac{\sigma}{\beta}(E_t \tilde{C_{t+1}}-\tilde{C_t}) = \frac{R_{ss}}{P_{ss}} E_t(\tilde{R_{t+1}}-\tilde{P_{t+1}})\tag{2.40}$$
$$\tilde{R_t}-\tilde{P_t}= \tilde{Y_t}-\tilde{K_t}\tag{2.41}$$
$$\tilde{W_t}-\tilde{P_t}= \tilde{Y_t}-\tilde{L_t}\tag{2.42}$$
$$\tilde{Y_t}= \tilde{A_t}+\alpha\tilde{K_t}+(1-\alpha)\tilde{L_t}\tag{2.43}$$
$$\tilde{K_{t+1}}=(1-\delta)\tilde{K_t}+\delta\tilde{I_t}\tag{2.44}$$
$$Y_{tt}\tilde{Y_{t}}=C_{ss}\tilde{C_t}+I_{ss}\tilde{I_t}\tag{2.45}$$
$$Y_{tt}\tilde{Y=A_{t}}= \rho_A \tilde{A_{t-1}}+\varepsilon_t\tag{2.46}$$
$$\tilde{MC_t} = (1-\alpha) \tilde{W_t}+\alpha \tilde{R_t}-\tilde{A_t}\tag{3.46}$$

## 5.1 Determining the NK Phillips curve

$$\tilde{\pi_t} = \beta E_t\tilde{\pi_{t+1}}+\Big[\frac{(1-\theta)(1-\beta\theta)}{\theta}\Big](\tilde{MC_t}-\tilde{P_t})\tag{3.50}$$