```python
from sympy import symbols, cos, sin, pi, simplify, pprint, tan, expand_trig, sqrt, trigsimp, atan2
from sympy.matrices import Matrix
```

```python
q1, q2, q3, q4, q5, q6= symbols('q1:7')
alpha, beta, gamma = symbols('alpha beta gamma', real = True)
px, py, pz = symbols('px py pz', real = True)
```

```python
R03 = Matrix([
[sin(q2 + q3)*cos(q1), cos(q1)*cos(q2 + q3), -sin(q1)],
[sin(q1)*sin(q2 + q3), sin(q1)*cos(q2 + q3),  cos(q1)],
[        cos(q2 + q3),        -sin(q2 + q3),        0]])

R03T = Matrix([
[sin(q2 + q3)*cos(q1), sin(q1)*sin(q2 + q3),  cos(q2 + q3)],
[cos(q1)*cos(q2 + q3), sin(q1)*cos(q2 + q3), -sin(q2 + q3)],
[            -sin(q1),              cos(q1),             0]])
```

```python
R36 = Matrix([
[-sin(q4)*sin(q6) + cos(q4)*cos(q5)*cos(q6), -sin(q4)*cos(q6) - sin(q6)*cos(q4)*cos(q5), -sin(q5)*cos(q4)],
[                           sin(q5)*cos(q6),                           -sin(q5)*sin(q6),          cos(q5)],
[-sin(q4)*cos(q5)*cos(q6) - sin(q6)*cos(q4),  sin(q4)*sin(q6)*cos(q5) - cos(q4)*cos(q6),  sin(q4)*sin(q5)]])
```

```python
R0u = Matrix([
[1.0*cos(alpha)*cos(beta), -1.0*sin(alpha)*cos(gamma) + sin(beta)*sin(gamma)*cos(alpha), 1.0*sin(alpha)*sin(gamma) + sin(beta)*cos(alpha)*cos(gamma)],
[1.0*sin(alpha)*cos(beta),  sin(alpha)*sin(beta)*sin(gamma) + 1.0*cos(alpha)*cos(gamma), sin(alpha)*sin(beta)*cos(gamma) - 1.0*sin(gamma)*cos(alpha)],
[          -1.0*sin(beta),                                     1.0*sin(gamma)*cos(beta),                                    1.0*cos(beta)*cos(gamma)]])
```

```python
Rgu_eval = Matrix([[0, 0, 1], [0, -1.00000000000000, 0], [1.00000000000000, 0, 0]])
RguT_eval = Matrix([[0, 0, 1], [0, -1.00000000000000, 0], [1.00000000000000, 0, 0]])
```

```python
# Total transform wrt gripper given
# yaw (alpha), pitch (beta), roll (beta)
# position px, py, pz

T0g_b = Matrix([
[1.0*sin(alpha)*sin(gamma) + sin(beta)*cos(alpha)*cos(gamma),  1.0*sin(alpha)*cos(gamma) - 1.0*sin(beta)*sin(gamma)*cos(alpha), 1.0*cos(alpha)*cos(beta), px],
[sin(alpha)*sin(beta)*cos(gamma) - 1.0*sin(gamma)*cos(alpha), -1.0*sin(alpha)*sin(beta)*sin(gamma) - 1.0*cos(alpha)*cos(gamma), 1.0*sin(alpha)*cos(beta), py],
[                                   1.0*cos(beta)*cos(gamma),                                        -1.0*sin(gamma)*cos(beta),           -1.0*sin(beta), pz],
[                                                          0,                                                                0,                        0,  1]])
```

```python
# Total transform wrt gripper given
# angles q1, q2, q3, q4, q5, q6

T0g_a = Matrix([
[((sin(q1)*sin(q4) + sin(q2 + q3)*cos(q1)*cos(q4))*cos(q5) + sin(q5)*cos(q1)*cos(q2 + q3))*cos(q6) - (-sin(q1)*cos(q4) + sin(q4)*sin(q2 + q3)*cos(q1))*sin(q6), -((sin(q1)*sin(q4) + sin(q2 + q3)*cos(q1)*cos(q4))*cos(q5) + sin(q5)*cos(q1)*cos(q2 + q3))*sin(q6) + (sin(q1)*cos(q4) - sin(q4)*sin(q2 + q3)*cos(q1))*cos(q6), -(sin(q1)*sin(q4) + sin(q2 + q3)*cos(q1)*cos(q4))*sin(q5) + cos(q1)*cos(q5)*cos(q2 + q3), -0.303*sin(q1)*sin(q4)*sin(q5) + 1.25*sin(q2)*cos(q1) - 0.303*sin(q5)*sin(q2 + q3)*cos(q1)*cos(q4) - 0.054*sin(q2 + q3)*cos(q1) + 0.303*cos(q1)*cos(q5)*cos(q2 + q3) + 1.5*cos(q1)*cos(q2 + q3) + 0.35*cos(q1)],
[ ((sin(q1)*sin(q2 + q3)*cos(q4) - sin(q4)*cos(q1))*cos(q5) + sin(q1)*sin(q5)*cos(q2 + q3))*cos(q6) - (sin(q1)*sin(q4)*sin(q2 + q3) + cos(q1)*cos(q4))*sin(q6), -((sin(q1)*sin(q2 + q3)*cos(q4) - sin(q4)*cos(q1))*cos(q5) + sin(q1)*sin(q5)*cos(q2 + q3))*sin(q6) - (sin(q1)*sin(q4)*sin(q2 + q3) + cos(q1)*cos(q4))*cos(q6), -(sin(q1)*sin(q2 + q3)*cos(q4) - sin(q4)*cos(q1))*sin(q5) + sin(q1)*cos(q5)*cos(q2 + q3),  1.25*sin(q1)*sin(q2) - 0.303*sin(q1)*sin(q5)*sin(q2 + q3)*cos(q4) - 0.054*sin(q1)*sin(q2 + q3) + 0.303*sin(q1)*cos(q5)*cos(q2 + q3) + 1.5*sin(q1)*cos(q2 + q3) + 0.35*sin(q1) + 0.303*sin(q4)*sin(q5)*cos(q1)],
[                                                                -(sin(q5)*sin(q2 + q3) - cos(q4)*cos(q5)*cos(q2 + q3))*cos(q6) - sin(q4)*sin(q6)*cos(q2 + q3),                                                                  (sin(q5)*sin(q2 + q3) - cos(q4)*cos(q5)*cos(q2 + q3))*sin(q6) - sin(q4)*cos(q6)*cos(q2 + q3),                                     -sin(q5)*cos(q4)*cos(q2 + q3) - sin(q2 + q3)*cos(q5),                                                                                -0.303*sin(q5)*cos(q4)*cos(q2 + q3) - 0.303*sin(q2 + q3)*cos(q5) - 1.5*sin(q2 + q3) + 1.25*cos(q2) - 0.054*cos(q2 + q3) + 0.75],
[                                                                                0,                                                                                0,                                                                                0,                                                                                1]])
```

```python
def get_hypotenuse(a, b):
  return sqrt(a*a + b*b)

def get_cosine_law_angle(a, b, c):
  cos_gamma = (a*a + b*b - c*c) / (2*a*b)
  sin_gamma = sqrt(1 - cos_gamma * cos_gamma)
  gamma = atan2(sin_gamma, cos_gamma)

  return gamma

def get_wrist_center(gripper_point, R0g, dg = 0.303):

  xu, yu, zu = gripper_point

  nx, ny, nz = R0g[0, 2], R0g[1, 2], R0g[2, 2]
  xw = xu - dg * nx
  yw = yu - dg * ny
  zw = zu - dg * nz

  return xw, yw, zw
def get_first_three_angles(wrist_center):

  x, y, z  = wrist_center

  a1, a2, a3 = 0.35, 1.25, -0.054
  d1, d4 = 0.75, 1.5
  l = 1.50097168527591
  phi = 1.53481186671284

  x_prime = get_hypotenuse(x, y)
  mx = x_prime -  a1
  my = z - d1
  m = get_hypotenuse(mx, my)
  alpha = atan2(my, mx)

  gamma = get_cosine_law_angle(l, a2, m)
  beta = get_cosine_law_angle(m, a2, l)

  q1 = atan2(y, x)
  q2 = pi/2 - beta - alpha
  q3 = -(gamma - phi)

  return q1, q2, q3

def get_last_three_angles(R):

  sin_q4 = R[2, 2]
  cos_q4 =  -R[0, 2]

  sin_q5 = sqrt(R[0, 2]**2 + R[2, 2]**2)
  cos_q5 = R[1, 2]

  sin_q6 = -R[1, 1]
  cos_q6 = R[1, 0]

  q4 = atan2(sin_q4, cos_q4)
  q5 = atan2(sin_q5, cos_q5)
  q6 = atan2(sin_q6, cos_q6)

  return q4, q5, q6
```

```python
def ik(x, y, z, roll, pitch, yaw, debug = False):

  # input: given position and orientation of the gripper wrt to URDFrame
  # output: angles q1, q2, q3, q4, q5, q6

  gripper_point = x, y, z

  R0u_eval = R0u.evalf(subs = {alpha: yaw, beta: pitch, gamma: roll})
  R0g_eval = R0u_eval * RguT_eval

  wrist_center = get_wrist_center(gripper_point, R0g_eval, dg = 0.303)

  j1, j2, j3 = get_first_three_angles(wrist_center)

  R03T_eval = R03T.evalf(subs = {q1: j1.evalf(), q2: j2.evalf(), q3: j3.evalf()})
  R36_eval = R03T_eval * R0g_eval

  j4, j5, j6 = get_last_three_angles(R36_eval)

  j1 = j1.evalf()
  j2 = j2.evalf()
  j3 = j3.evalf()
  j4 = j4.evalf()
  j5 = j5.evalf()
  j6 = j6.evalf()

  if debug:

    print()
    print("\n x:", x, "\n y:", y, "\n z:", z)
    print("\n roll:", roll, "\n pitch:", pitch, "\n yaw:", yaw)

    print()
    print(" j1:", j1, "\n j2:", j2, "\n j3:", j3)
    print(" j4:", j4, "\n j5:", j5, "\n j6:", j6)

    print()
    print("wrist_center", wrist_center)

    print()
    print("evaluated R0g:")
    pprint(R0g_eval)

    print()
    print("Total transform wrt gripper: given yaw (alpha), pitch (beta), roll (beta), px, py, pz")
    pprint(T0g_b.evalf(subs = {
      gamma: roll, beta: pitch, alpha: yaw, px: x, py: y, pz: z
    }))

    print()
    print("Total transform wrt gripper: given angles q1, q2, q3, q4, q5, q6")
    pprint(T0g_a.evalf(subs = {
      q1: j1, q2: j2, q3: j3, q4: j4, q5: j5, q6: j6
    }))

  return j1, j2, j3, j4, j5, j6
```

```python
qs = ik(x = 0.49792, y = 1.3673, z = 2.4988,
        roll = 0.366, pitch = -0.078, yaw = 2.561, debug = True)
```

     x: 0.49792
     y: 1.3673
     z: 2.4988

     roll: 0.366
     pitch: -0.078
     yaw: 2.561

     j1: 1.01249809363771
     j2: -0.275800363737724
     j3: -0.115686651053751
     j4: 1.63446527240323
     j5: 1.52050002599430
     j6: -0.815781306199679

    wrist_center (0.750499428337951, 1.20160389781975, 2.47518995758694)

    evaluated R0g:
    ⎡0.257143295038827   0.48887208255965   -0.833595473062543⎤
    ⎢                                                         ⎥
    ⎢0.259329420712765  0.796053601157403    0.54685182237706 ⎥
    ⎢                                                         ⎥
    ⎣0.93092726749696   -0.356795110642117  0.0779209320563015⎦

    Total transform wrt gripper: given yaw (alpha), pitch (beta), roll (beta), px, py, pz
    ⎡0.257143295038827   0.48887208255965   -0.833595473062543  0.49792⎤
    ⎢                                                                  ⎥
    ⎢0.259329420712765  0.796053601157403    0.54685182237706   1.3673 ⎥
    ⎢                                                                  ⎥
    ⎢0.93092726749696   -0.356795110642117  0.0779209320563015  2.4988 ⎥
    ⎢                                                                  ⎥
    ⎣        0                  0                   0             1.0  ⎦

    Total transform wrt gripper: given angles q1, q2, q3, q4, q5, q6
    ⎡0.257143295038827   0.48887208255965   -0.833595473062543  0.497919999999998⎤
    ⎢                                                                            ⎥
    ⎢0.259329420712765  0.796053601157403    0.54685182237706        1.3673      ⎥
    ⎢                                                                            ⎥
    ⎢0.93092726749696   -0.356795110642117  0.0779209320563016  2.49880000000001 ⎥
    ⎢                                                                            ⎥
    ⎣        0                  0                   0                  1.0       ⎦

```python
qs = ik(x = 2.3537, y = -0.1255546, z = 2.841452,
        roll = 0.131008, pitch = -0.10541, yaw = 0.0491503, debug = True)

# -0.07, 0.41, -1.07, 0.32, 0.46, 0
```

     x: 2.3537
     y: -0.1255546
     z: 2.841452

     roll: 0.131008
     pitch: -0.10541
     yaw: 0.0491503

     j1: -0.0682697289101386
     j2: 0.434273483083027
     j3: -1.13476160607020
     j4: 0.206486955261342
     j5: 0.604353673052791
     j6: -0.0272724984420472

    wrist_center (2.05274568076731, -0.140358517861560, 2.80957188470638)

    evaluated R0g:
    ⎡-0.0977692193362551  0.0624374999845176  0.993248578325719 ⎤
    ⎢                                                           ⎥
    ⎢-0.135600778861087   -0.989558155295888  0.0488578147246194⎥
    ⎢                                                           ⎥
    ⎣ 0.985927790724374   -0.129908490419534  0.105214901959143 ⎦

    Total transform wrt gripper: given yaw (alpha), pitch (beta), roll (beta), px, py, pz
    ⎡-0.0977692193362551  0.0624374999845176  0.993248578325719     2.3537  ⎤
    ⎢                                                                       ⎥
    ⎢-0.135600778861087   -0.989558155295888  0.0488578147246194  -0.1255546⎥
    ⎢                                                                       ⎥
    ⎢ 0.985927790724374   -0.129908490419534  0.105214901959143    2.841452 ⎥
    ⎢                                                                       ⎥
    ⎣         0                   0                   0              1.0    ⎦

    Total transform wrt gripper: given angles q1, q2, q3, q4, q5, q6
    ⎡-0.0977692193362551  0.0624374999845176  0.993248578325719   2.35369999999999
    ⎢
    ⎢-0.135600778861087   -0.989558155295888  0.0488578147246194     -0.1255546
    ⎢
    ⎢ 0.985927790724374   -0.129908490419534  0.105214901959143       2.841452
    ⎢
    ⎣         0                   0                   0                 1.0

    ⎤
    ⎥
    ⎥
    ⎥
    ⎥
    ⎥
    ⎦

```python
qs = ik(x = 2.3537, y = -0.1255546, z = 2.841452,
        yaw = 0.131008, pitch = -0.10541, roll = 0.0491503, debug = True)

# -0.07, 0.41, -1.07, 0.32, 0.46, 0
```

     x: 2.3537
     y: -0.1255546
     z: 2.841452

     roll: 0.0491503
     pitch: -0.10541
     yaw: 0.131008

     j1: -0.0800813023075095
     j2: 0.439897727669121
     j3: -1.14290690404898
     j4: 0.363106480125535
     j5: 0.626901765463759
     j6: -0.226818468786107

    wrist_center (2.05496387941051, -0.164916872597119, 2.80957188470638)

    evaluated R0g:
    ⎡-0.0977692193362551   0.135600778861087   0.985927790724374⎤
    ⎢                                                           ⎥
    ⎢-0.0624374999845176  -0.989558155295888   0.129908490419534⎥
    ⎢                                                           ⎥
    ⎣ 0.993248578325719   -0.0488578147246194  0.105214901959143⎦

    Total transform wrt gripper: given yaw (alpha), pitch (beta), roll (beta), px, py, pz
    ⎡-0.0977692193362551   0.135600778861087   0.985927790724374    2.3537  ⎤
    ⎢                                                                       ⎥
    ⎢-0.0624374999845176  -0.989558155295888   0.129908490419534  -0.1255546⎥
    ⎢                                                                       ⎥
    ⎢ 0.993248578325719   -0.0488578147246194  0.105214901959143   2.841452 ⎥
    ⎢                                                                       ⎥
    ⎣         0                    0                   0             1.0    ⎦

    Total transform wrt gripper: given angles q1, q2, q3, q4, q5, q6
    ⎡-0.0977692193362552   0.135600778861087   0.985927790724374   2.3536999999999
    ⎢
    ⎢-0.0624374999845176  -0.989558155295888   0.129908490419534  -0.1255545999999
    ⎢
    ⎢ 0.993248578325719   -0.0488578147246194  0.105214901959143   2.8414520000000
    ⎢
    ⎣         0                    0                   0                 1.0

    9 ⎤
      ⎥
    99⎥
      ⎥
    1 ⎥
      ⎥
      ⎦

```python

```


```
import pandas as pd
import pydotplus
import numpy  as np
import matplotlib.pyplot as plt
import seaborn as sns
import statsmodels.api as sm

from io import StringIO
from IPython.display import Image, SVG
from graphviz import Source
from matplotlib import cm
from matplotlib.colors import ListedColormap, LinearSegmentedColormap
from matplotlib.patches import Patch
from scipy.stats import chi2_contingency, norm
from sklearn.tree import export_graphviz
from sklearn import tree
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_curve, auc, accuracy_score
from sklearn.model_selection import StratifiedKFold, train_test_split
from statsmodels.formula.api import ols
from sympy import var, plot_implicit, Eq
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import roc_auc_score

plt.style.use('ggplot')
# ignore log(0) and divide by 0 warning
np.seterr(divide='ignore');
SEED=1
```

## Decision trees and random forests

##Decision trees##

*   In contrast with linear models, like logistic regression, decision trees can capture non-linear relationship in data.
*   Futhermore, decision trees don't require the features to be on the same scale
* The decision region: region in the feauture space where all instances are assigned to one class label. Produces rectangular decision regions in the feature-space

```
df = pd.read_csv('/content/drive/MyDrive/data_davivienda/Lending_club.csv',
                 dtype = {'loan_status':'category', 'annual_inc':'float',
                          'verification_status':'category', 'emp_length':'category',
                          'home_ownership':'category', 'int_rate':'object',
                          'loan_amnt':'float', 'purpose':'category',
                          'term':'category', 'grade':'category'})

#Change loan_status, verification_status, emp_length, term and grade to category type
df.loan_status = df.loan_status.astype(pd.api.types.CategoricalDtype(categories=['Charged Off', 'Fully Paid']))
df.verification_status = df.verification_status.astype(pd.api.types.CategoricalDtype(categories=['Not Verified', 'Source Verified', 'Verified']))
df.emp_length = df.emp_length.astype(pd.api.types.CategoricalDtype(categories=['< 1 year', '1 year', '2 years', '3 years', '4 years', \
                                                             '5 years', '6 years', '7 years', '8 years', '9 years', \
                                                             '10+ years']))
df.home_ownership = df.home_ownership.astype(pd.api.types.CategoricalDtype(categories=['RENT','MORTGAGE','OWN','OTHER']))
df.term = df.term.astype(pd.api.types.CategoricalDtype(categories=[' 36 months', ' 60 months']))
df.grade = df.grade.astype(pd.api.types.CategoricalDtype(categories=['A','B','C','D','E','F','G']))
df.purpose = df.purpose.astype(pd.api.types.CategoricalDtype(categories=['debt_consolidation','credit_card','other','home_improvement','major_purchase','small_business','car', 'wedding','medical','moving','house','vacation','educational','renewable_energy']))

#In addition, the original data in int_rate contains strings of the form 'x.xx%',
#we remove the % and change the vaules to float:
df.int_rate = df.int_rate.str.rstrip('%').astype('float')

```

```
df_t=df.copy()
for var in ['loan_status', 'term','verification_status','grade','home_ownership']:
    df_t[var]=df_t[var].cat.codes

clf = DecisionTreeClassifier(max_depth = 3)

# Train Decision Tree Classifer
covariates=['loan_amnt']

clf = clf.fit(df_t[covariates], df_t['loan_status'])

dot_data = StringIO()
export_graphviz(clf, out_file=dot_data, feature_names = covariates,
                class_names=['Charged Off', 'Fully Paid'],
                filled=True, rounded=True,
                special_characters=True,
                impurity='gini', proportion=True)

graph = pydotplus.graph_from_dot_data(dot_data.getvalue())
Image(graph.create_png())
```

We need to determine which variable we are going to put at the beginning of the tree. The algorithm does that with the concept of gain information:

> $\text{IG(f, sp)}= \text{IG(parent)}- ( \frac{N_l}{N}\text{IG(left)}+\frac{N_r}{N}\text{IG(right)})$

##training##

```
df_pre = pd.concat([(df.loan_amnt - df.loan_amnt.mean())/df.loan_amnt.std(),\
                     (df.int_rate - df.int_rate.mean())/df.int_rate.std(),\
                     (df.annual_inc - df.annual_inc.mean())/df.annual_inc.std(),\
                     pd.get_dummies(df.home_ownership, prefix='home', drop_first=True), \
                     pd.get_dummies(df.purpose, prefix='purpose', drop_first=True), \
                     pd.get_dummies(df.grade, prefix='grade',drop_first=True)], axis=1)
df_pre['verification_status'] = df.verification_status.cat.codes
df_pre['emp_length'] = df.emp_length.cat.codes
df_pre['term'] = df.term.cat.codes
```

```
y=df.loan_status.cat.codes
X=df_pre
X_train, X_test, y_train, y_test= train_test_split(X, y,test_size=0.2, stratify=y,random_state=SEED)
```

```
dt = DecisionTreeClassifier(random_state=SEED)
```

```
#params
params_dt = {'max_depth':[2,3,4,6], 'min_samples_leaf':[0.12, 0.14, 0.16, 0.18,0.25]}
```

```
# Instantiate grid_dt
grid_dt = GridSearchCV(estimator=dt,
                       param_grid=params_dt,
                       scoring='roc_auc',
                       cv=5,
                       n_jobs=-1)
#training
grid_dt.fit(X_train, y_train)
```

    GridSearchCV(cv=5, error_score=nan,
                 estimator=DecisionTreeClassifier(ccp_alpha=0.0, class_weight=None,
                                                  criterion='gini', max_depth=None,
                                                  max_features=None,
                                                  max_leaf_nodes=None,
                                                  min_impurity_decrease=0.0,
                                                  min_impurity_split=None,
                                                  min_samples_leaf=1,
                                                  min_samples_split=2,
                                                  min_weight_fraction_leaf=0.0,
                                                  presort='deprecated',
                                                  random_state=1, splitter='best'),
                 iid='deprecated', n_jobs=-1,
                 param_grid={'max_depth': [2, 3, 4, 6],
                             'min_samples_leaf': [0.12, 0.14, 0.16, 0.18, 0.25]},
                 pre_dispatch='2*n_jobs', refit=True, return_train_score=False,
                 scoring='roc_auc', verbose=0)

```
# Extract the best estimator
best_model = grid_dt.best_estimator_
best_hyperparams = grid_dt.best_params_
print('Best hyerparameters:\n', best_hyperparams)

# Predict the test set probabilities of the positive class
y_pred_proba = best_model.predict_proba(X_test)[:,1]

# Compute test_roc_auc
test_roc_auc = roc_auc_score(y_test, y_pred_proba)

# Print test_roc_auc
print('Test set ROC AUC score: {:.3f}'.format(test_roc_auc))

```

    Best hyerparameters:
     {'max_depth': 4, 'min_samples_leaf': 0.12}
    Test set ROC AUC score: 0.661

### Decision boundary of a decision tree

The decision boundary of decision tree can be highly non-linear. It usually looks like patches due to the fact that we are dichotomizing the dataset at multiple levels. The following is an example for a tree built with `annual_inc` and `loan_amnt`:

```
n_classes = 2
plot_colors = 'br'

labels = ['Charged off', 'Fully paid']

clf = DecisionTreeClassifier(random_state=SEED).fit(df[['annual_inc', 'loan_amnt']], df['loan_status'].cat.codes)

# Plot the decision boundary

plot_step = 100
viridis = cm.get_cmap('viridis', 2)
newcolors = viridis(np.linspace(0, 1, 2))
blue = np.array([0, 0, 1, 1])
red = np.array([1, 0, 0, 1])
newcolors[0, :] = blue
newcolors[1, :] = red
newcmp = ListedColormap(newcolors)

np.random.seed(0)
idx_plot = np.random.choice(df.shape[0], 100, replace = False )
df_plot = df.iloc[idx_plot]

x_min, x_max = df_plot['annual_inc'].min() - 1000, df_plot['annual_inc'].max() + 1000
y_min, y_max = df_plot['loan_amnt'].min() - 1000, df_plot['loan_amnt'].max() + 1000
xx, yy = np.meshgrid(np.arange(x_min, x_max, plot_step),
                     np.arange(y_min, y_max, plot_step))

fig, ax = plt.subplots()
Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
cs = plt.contourf(xx, yy, Z, cmap=newcmp, alpha = 0.7)

plt.xlabel('Annual income')
plt.ylabel('Loan amount')
plt.axis('tight');

for i, color in zip(range(n_classes), plot_colors):
    idx = np.where(df_plot['loan_status'].cat.codes == i)
    plt.scatter(df_plot.iloc[idx]['annual_inc'], df_plot.iloc[idx]['loan_amnt'], c=color,
                label=labels[i], edgecolors='black')

legend_elements = [Patch(facecolor='blue', edgecolor='blue', label='Charged off'),
                   Patch(facecolor='red', edgecolor='red', label='Fully paid')]

plt.axis('tight')
plt.title('Decision surface of a decision tree')
leg1 = ax.legend(loc='upper left', title='Observed outcome')
ax.legend(handles=legend_elements, loc='lower right', title='Predicted outcome');
ax.add_artist(leg1);
```

*   The area of a single rectangule depends of hyperparameters of DT
*   The max_depth parameter defines whether the model raises a bias or variance problem

##Random Forest

The idea behind random forests is to construct a lot of decision trees, then use a voting mechanism to produce a final prediction that is agreed upon by most of the decision trees. When training these individual trees, we also subsample on the dataset as well as the covariates that are available in order to get as much built-in cross-validation as possible. As a result, the problem of overfitting and stability in a single decision tree model is resolved to a large degree.

```
rf = RandomForestClassifier(random_state=SEED)
```

```
#params
params_rf = {'max_features':['log2','sqrt'],'n_estimators':[300,400,500,800],'max_depth':[2,3,4,6], 'min_samples_leaf':[0.12, 0.14, 0.16, 0.18,0.25]}
```

```
# Instantiate grid_dt
grid_rf = GridSearchCV(estimator=rf,
                       param_grid=params_rf,
                       scoring='neg_mean_squared_error',
                       cv=5,
                       n_jobs=-1)
#training
grid_rf.fit(X_train, y_train)
```

    GridSearchCV(cv=5, error_score=nan,
                 estimator=RandomForestClassifier(bootstrap=True, ccp_alpha=0.0,
                                                  class_weight=None,
                                                  criterion='gini', max_depth=None,
                                                  max_features='auto',
                                                  max_leaf_nodes=None,
                                                  max_samples=None,
                                                  min_impurity_decrease=0.0,
                                                  min_impurity_split=None,
                                                  min_samples_leaf=1,
                                                  min_samples_split=2,
                                                  min_weight_fraction_leaf=0.0,
                                                  n_estimators=100, n_jobs=None,
                                                  oob_score=False, random_state=1,
                                                  verbose=0, warm_start=False),
                 iid='deprecated', n_jobs=-1,
                 param_grid={'max_depth': [2, 3, 4, 6],
                             'max_features': ['log2', 'sqrt'],
                             'min_samples_leaf': [0.12, 0.14, 0.16, 0.18, 0.25],
                             'n_estimators': [300, 400, 500, 800]},
                 pre_dispatch='2*n_jobs', refit=True, return_train_score=False,
                 scoring='neg_mean_squared_error', verbose=0)

```
# Extract the best estimator
best_model = grid_rf.best_estimator_
best_hyperparams = grid_rf.best_params_
print('Best hyerparameters:\n', best_hyperparams)

# Predict the test set probabilities of the positive class
y_pred_proba = best_model.predict_proba(X_test)[:,1]

# Compute test_roc_auc
test_roc_auc = roc_auc_score(y_test, y_pred_proba)

# Print test_roc_auc
print('Test set ROC AUC score: {:.3f}'.format(test_roc_auc))
```

    Best hyerparameters:
     {'max_depth': 2, 'max_features': 'log2', 'min_samples_leaf': 0.12, 'n_estimators': 300}
    Test set ROC AUC score: 0.678

## Feature Importance##

```
plt.rcParams["figure.figsize"] = (10,10)
importance_rf=pd.Series(best_model.feature_importances_, index=X.columns)

#Sort importance_rf

sorted_importance_rf= importance_rf.sort_values()
sorted_importance_rf.plot(kind='barh');
```

Contrast results RF- DC

```
def cv_roc_tree(X, y, n_split, algo='Tree'):

    cv = StratifiedKFold(n_split)

    tprs = []
    aucs = []
    mean_fpr = np.linspace(0, 1, 100)
    for train, test in cv.split(X, y):
        if algo == 'Tree':
            clf = DecisionTreeClassifier()
        else:
            clf = RandomForestClassifier(n_estimators=100, max_depth=3, random_state=0)

        clf = clf.fit(X.iloc[train], y.iloc[train])
        probas_ = clf.predict_proba(X.iloc[test])

        # Compute ROC curve and area the curve
        fpr, tpr, thresholds = roc_curve(y.iloc[test], probas_[:,1])
        tprs.append(np.interp(mean_fpr, fpr, tpr))
        tprs[-1][0] = 0.0
        roc_auc = auc(fpr, tpr)
        aucs.append(roc_auc)

    return tprs, aucs, mean_fpr
```

```
tprs_t, aucs_t, mean_fpr_t = cv_roc_tree(df_pre, df['loan_status'].cat.codes, 5)
tprs_f, aucs_f, mean_fpr_f = cv_roc_tree(df_pre, df['loan_status'].cat.codes,5, algo='forest')

plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='blue', label='Random guess', alpha=.8)

mean_tpr_t = np.mean(tprs_t, axis=0)
mean_tpr_f = np.mean(tprs_f, axis=0)
mean_tpr_t[-1] = 1.0
mean_tpr_f[-1] = 1.0

mean_auc_t = auc(mean_fpr_t, mean_tpr_t)
mean_auc_f = auc(mean_fpr_f, mean_tpr_f)
std_auc_t = np.std(aucs_t)
std_auc_f = np.std(aucs_f)

plt.plot(mean_fpr_t, mean_tpr_t, color='darkorange', lw=2, alpha=.8,
         label=f"Decision Tree: Cross-validated Mean ROC (AUC = {mean_auc_t:0.2f} $\pm$ {std_auc_t:0.2f})")
plt.plot(mean_fpr_f, mean_tpr_f, color='red', lw=2, alpha=.8,
         label=f"Random Forest: Cross-validated Mean ROC (AUC = {mean_auc_f:0.2f} $\pm$ {std_auc_f:0.2f})")

plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.legend(loc='lower right')
plt.title('Tree-based Classification of Defaulters: 5-fold Cross-validation')
plt.show()
```

```

```


# Gaussian Process Distribution of Relaxation Times

## In this tutorial we will reproduce Figure 7 of the article https://doi.org/10.1016/j.electacta.2019.135316

GP-DRT is our newly developed approach that can be used to obtain both the mean and covariance of the DRT from the EIS data by assuming that the DRT is a Gaussian process (GP). The GP-DRP can  predict the DRT and the imaginary part of the impedance at frequencies that were not previously measured.

To obtain the DRT from the impedance we take that $\gamma(\xi)$ is a GP where $f$ is the frequency and $\xi=\log f$. Under the DRT model and considering that GPs are closed linear transformations, it follows that $Z^{\rm DRT}_{\rm im}\left(\xi\right)$ is also a GP.

More precisely we can write

$$\begin{pmatrix}
\gamma(\xi) \\
Z^{\rm DRT}_{\rm im}\left(\xi\right)
\end{pmatrix}\sim \mathcal{GP}\left(\mathbf 0, \begin{pmatrix}
k(\xi, \xi^\prime) & \mathcal L^{\rm im}_{\xi^\prime} \left(k(\xi, \xi^\prime)\right)\\
\mathcal L^{\rm im}_{\xi} k(\xi, \xi^\prime) & \mathcal L^{\rm im}_{\xi^\prime}\left(\mathcal L^{\rm im}_{\xi} \left(k(\xi, \xi^\prime)\right)\right)
\end{pmatrix}\right)$$

where

$$\mathcal L^{\rm im}_\xi \left(\cdot\right) = -\displaystyle \int_{-\infty}^\infty \frac{2\pi \displaystyle e^{\xi-\hat \xi}}{1+\left(2\pi \displaystyle e^{\xi-\hat \xi}\right)^2}  \left(\cdot\right) d \hat \xi$$

is a linear functional. The latter functional, transforms the DRT to the imaginary part of the impedance.

Assuming we have $N$ observations, we can set $\left(\mathbf Z^{\rm exp}_{\rm im}\right)_n = Z^{\rm exp}_{\rm im}(\xi_n)$ with $\xi_n =\log f_n$ and $n =1, 2, \ldots N $. The corresponding multivariate Gaussian random variable can be written as

$$\begin{pmatrix}
\boldsymbol{\gamma} \\
\mathbf Z^{\rm exp}_{\rm im}
\end{pmatrix}\sim \mathcal{N}\left(\mathbf 0, \begin{pmatrix}
\mathbf K & \mathcal L_{\rm im} \mathbf K\\
\mathcal L_{\rm im}^\sharp \mathbf K & \mathcal L^2_{\rm im} \mathbf K + \sigma_n^2 \mathbf I
\end{pmatrix}\right)$$

where

$$\begin{align}
(\mathbf K)_{nm} &= k(\xi_n, \xi_m)\\
(\mathcal L_{\rm im} \mathbf K)_{nm} &= \left. \mathcal L^{\rm im}_{\xi^\prime} \left(k(\xi, \xi^\prime)\right) \right |_{\xi_n, \xi_m}\\
(\mathcal L_{\rm im}^\sharp \mathbf K)_{nm} &= \left.\mathcal L^{\rm im}_{\xi} \left(k(\xi, \xi^\prime)\right) \right|_{\xi_n, \xi_m}\\
(\mathcal L^2_{\rm im} \mathbf K)_{nm} &= \left.\mathcal L^{\rm im}_{\xi^\prime}\left(\mathcal L^{\rm im}_{\xi} \left(k(\xi, \xi^\prime)\right)\right) \right|_{\xi_n, \xi_m}
\end{align}$$

and $\mathcal L_{\rm im} \mathbf K^\top = \mathcal L_{\rm im}^\sharp \mathbf K$.

To obtain the DRT from impedance, the distribution of $\mathbf{\gamma}$ conditioned on $\mathbf Z^{\rm exp}_{\rm im}$ can be written as

$$\boldsymbol{\gamma}|\mathbf Z^{\rm exp}_{\rm im}\sim \mathcal N\left( \mathbf \mu_{\gamma|Z^{\rm exp}_{\rm im}}, \mathbf\Sigma_{\gamma| Z^{\rm exp}_{\rm im}}\right)$$

with

$$\begin{align}
\mathbf \mu_{\gamma|Z^{\rm exp}_{\rm im}} &= \mathcal L_{\rm im} \mathbf K \left(\mathcal L^2_{\rm im} \mathbf K + \sigma_n^2 \mathbf I \right)^{-1} \mathbf Z^{\rm exp}_{\rm im} \\
\mathbf \Sigma_{\gamma| Z^{\rm exp}_{\rm im}} &= \mathbf K- \mathcal L_{\rm im} \mathbf K \left(\mathcal L^2_{\rm im} \mathbf K + \sigma_n^2 \mathbf I \right)^{-1}\mathcal L_{\rm im} \mathbf K^\top
\end{align}$$

The above formulas depend on 1) the kernel, $k(\xi, \xi^\prime)$; 2) the noise level, $\sigma_n$; and 3) the experimental data, $\mathbf Z^{\rm exp}_{\rm im}$ (at the log-frequencies $\mathbf \xi$).

```python
from math import cos, pi, sin

import GP_DRT
import matplotlib.pyplot as plt
import numpy as np
from scipy.optimize import minimize

%matplotlib inline
```

## 1) Define parameters of the ZARC circuit which will be used for the synthetic experiment generation

The impedance of a ZARC can be written as
$$
Z^{\rm exact}(f) = R_\infty + \displaystyle \frac{1}{\displaystyle \frac{1}{R_{\rm ct}}+C \left(i 2\pi f\right)^\phi}
$$

where $\displaystyle C = \frac{\tau_0^\phi}{R_{\rm ct}}$.

The analytical DRT can be computed analytically as

$$
\gamma(\log \tau) =  \displaystyle \frac{\displaystyle R_{\rm ct}}{\displaystyle 2\pi} \displaystyle \frac{\displaystyle \sin\left((1-\phi)\pi\right)}{\displaystyle \cosh(\phi \log(\tau/\tau_0))-\cos(\pi(1-\phi))}
$$

```python
# define the frequency range
N_freqs = 81
freq_vec = np.logspace(-4.0, 4.0, num=N_freqs, endpoint=True)
xi_vec = np.log(freq_vec)
tau = 1 / freq_vec

# define the frequency range used for prediction
# note: we could have used other values
freq_vec_star = np.logspace(-4.0, 4.0, num=81, endpoint=True)
xi_vec_star = np.log(freq_vec_star)

# parameters for ZARC model, the impedance and analytical DRT are calculated as the above equations
R_inf = 10
R_ct = 50
phi = 0.8
tau_0 = 1.0

C = tau_0 ** phi / R_ct
Z_exact = R_inf + 1.0 / (1.0 / R_ct + C * (1j * 2.0 * pi * freq_vec) ** phi)
gamma_fct = (
    (R_ct)
    / (2.0 * pi)
    * sin((1.0 - phi) * pi)
    / (np.cosh(phi * np.log(tau / tau_0)) - cos((1.0 - phi) * pi))
)

# we will use a finer mesh for plotting the results
freq_vec_plot = np.logspace(-4.0, 4.0, num=10 * (N_freqs - 1), endpoint=True)
tau_plot = 1 / freq_vec_plot
gamma_fct_plot = (
    (R_ct)
    / (2.0 * pi)
    * sin((1.0 - phi) * pi)
    / (np.cosh(phi * np.log(tau_plot / tau_0)) - cos((1.0 - phi) * pi))
)  # for plotting only

# we will add noise to the impedance computed analytically
rng = np.random.seed(214975)
sigma_n_exp = 1.0
Z_exp = Z_exact + sigma_n_exp * (
    np.random.normal(0, 1, N_freqs) + 1j * np.random.normal(0, 1, N_freqs)
)
```

## 2) Show the synthetic impedance in the Nyquist plot - this is similar to Figure 7 (a)

```python
# Nyquist plot of the impedance
plt.plot(np.real(Z_exact), -np.imag(Z_exact), linewidth=4, color="black", label="exact")
plt.plot(
    np.real(Z_exp), -np.imag(Z_exp), "o", markersize=10, color="red", label="synth exp"
)
plt.plot(
    np.real(Z_exp[20:60:10]),
    -np.imag(Z_exp[20:60:10]),
    "s",
    markersize=10,
    color="black",
)

plt.rc("text", usetex=True)
plt.rc("font", family="serif", size=15)
plt.rc("xtick", labelsize=15)
plt.rc("ytick", labelsize=15)
plt.legend(frameon=False, fontsize=15)
plt.axis("scaled")

plt.xticks(range(10, 70, 10))
plt.yticks(range(0, 60, 10))
plt.gca().set_aspect("equal", adjustable="box")
plt.xlabel(r"$Z_{\rm re}/\Omega$", fontsize=20)
plt.ylabel(r"$-Z_{\rm im}/\Omega$", fontsize=20)
# label the frequency points
plt.annotate(
    r"$10^{-2}$",
    xy=(np.real(Z_exp[20]), -np.imag(Z_exp[20])),
    xytext=(np.real(Z_exp[20]) - 2, 10 - np.imag(Z_exp[20])),
    arrowprops=dict(arrowstyle="-", connectionstyle="arc"),
)
plt.annotate(
    r"$10^{-1}$",
    xy=(np.real(Z_exp[30]), -np.imag(Z_exp[30])),
    xytext=(np.real(Z_exp[30]) - 2, 6 - np.imag(Z_exp[30])),
    arrowprops=dict(arrowstyle="-", connectionstyle="arc"),
)
plt.annotate(
    r"$1$",
    xy=(np.real(Z_exp[40]), -np.imag(Z_exp[40])),
    xytext=(np.real(Z_exp[40]), 10 - np.imag(Z_exp[40])),
    arrowprops=dict(arrowstyle="-", connectionstyle="arc"),
)
plt.annotate(
    r"$10$",
    xy=(np.real(Z_exp[50]), -np.imag(Z_exp[50])),
    xytext=(np.real(Z_exp[50]) - 1, 10 - np.imag(Z_exp[50])),
    arrowprops=dict(arrowstyle="-", connectionstyle="arc"),
)
plt.show()
```

## 3) Obtain the optimal hyperparameters of the GP-DRT model by minimizing the negative marginal log likelihood (NMLL)

We constrain the kernel to be a squared exponential, _i.e._

$$
k(\xi, \xi^\prime) = \sigma_f^2 \exp\left(-\frac{1}{2 \ell^2}\left(\xi-\xi^\prime\right)^2 \right)
$$

and modify its two parameters, $\sigma_f$ and $\ell$ as well as the noise level $\sigma_n$. Therefore, the vector of hyperparameters of the GP-DRT is assumed to be $\boldsymbol \theta = \begin{pmatrix} \sigma_n, \sigma_f, \ell  \end{pmatrix}^\top$.

Following the same derivation from the article we can write that

$$
\log p(\mathbf Z^{\rm exp}_{\rm im}|\boldsymbol \xi, \boldsymbol \theta)= - \frac{1}{2} {\mathbf Z^{\rm exp}_{\rm im}}^\top \left(\mathcal L^2_{\rm im} \mathbf K +\sigma_n^2\mathbf I \right)^{-1} \mathbf Z^{\rm exp}_{\rm im} -\frac{1}{2} \log \left| \mathcal L^2_{\rm im} \mathbf K+\sigma_n^2\mathbf I  \right| - \frac{N}{2} \log 2\pi
$$

We will call $L(\boldsymbol \theta)$ the negative (and shifted) MLL (NMLL):
$$
L(\boldsymbol \theta) = - \log p(\mathbf Z^{\rm exp}_{\rm im}|\boldsymbol \xi, \boldsymbol \theta) - \frac{N}{2} \log 2\pi
$$

the experimental evidence is maximized for

$$
\boldsymbol \theta = \arg \min_{\boldsymbol \theta^\prime}L(\boldsymbol \theta^\prime)
$$

The above minimization problem is solved using the `optimize` method provided by `scipy` package

```python
# initialize the parameter for global 3D optimization to maximize the marginal log-likelihood as shown in eq (31)
sigma_n = sigma_n_exp
sigma_f = 5.0
ell = 1.0

theta_0 = np.array([sigma_n, sigma_f, ell])
seq_theta = np.copy(theta_0)

def print_results(theta):
    global seq_theta
    seq_theta = np.vstack((seq_theta, theta))
    print("{0:.7f}  {1:.7f}  {2:.7f}".format(theta[0], theta[1], theta[2]))

GP_DRT.NMLL_fct(theta_0, Z_exp, xi_vec)
GP_DRT.grad_NMLL_fct(theta_0, Z_exp, xi_vec)
print("sigma_n,   sigma_f,   ell")

# minimize the NMLL L(\theta) w.r.t sigma_n, sigma_f, ell using the Newton-CG method as implemented in scipy
res = minimize(
    GP_DRT.NMLL_fct,
    theta_0,
    args=(Z_exp, xi_vec),
    method="Newton-CG",
    jac=GP_DRT.grad_NMLL_fct,
    callback=print_results,
    options={"disp": True},
)

# collect the optimized parameters
sigma_n, sigma_f, ell = res.x
```

## 4) Core of the GP-DRT

### 4a) Compute matrices
Once we have identified the optimized parameters we can compute $\mathbf K$, $\mathcal L_{\rm im} \mathbf K$, and $\mathcal L^2_{\rm im} \mathbf K$, which are given in equation `(18)` in the article

```python
K = GP_DRT.matrix_K(xi_vec, xi_vec, sigma_f, ell)
L_im_K = GP_DRT.matrix_L_im_K(xi_vec, xi_vec, sigma_f, ell)
L2_im_K = GP_DRT.matrix_L2_im_K(xi_vec, xi_vec, sigma_f, ell)
Sigma = (sigma_n ** 2) * np.eye(N_freqs)
```

### 4b) Factorize the matrices and solve the linear equations
We are computing
$$
\boldsymbol{\gamma}|\mathbf Z^{\rm exp}_{\rm im}\sim \mathcal N\left( \boldsymbol \mu_{\gamma|Z^{\rm exp}_{\rm im}}, \boldsymbol \Sigma_{\gamma| Z^{\rm exp}_{\rm im}}\right)
$$

using
$$
\begin{align}
\boldsymbol \mu_{\gamma|Z^{\rm exp}_{\rm im}} &= \mathcal L_{\rm im} \mathbf K\left(\mathcal L^2_{\rm im} \mathbf K+\sigma_n^2\mathbf I\right)^{-1}\mathbf Z^{\rm exp}_{\rm im} \\
\boldsymbol \Sigma_{\gamma| Z^{\rm exp}_{\rm im}} &= \mathbf K-\mathcal L_{\rm im} \mathbf K\left(\mathcal L^2_{\rm im} \mathbf K+\sigma_n^2\mathbf I\right)^{-1}\mathcal L_{\rm im} \mathbf K^\top
\end{align}
$$

The key ingredient is to do Cholesky factorization of $\mathcal L^2_{\rm im} \mathbf K+\sigma_n^2\mathbf I$, _i.e._, `K_im_full`

```python
# the matrix $\mathcal L^2_{\rm im} \mathbf K + \sigma_n^2 \mathbf I$ whose inverse is needed
K_im_full = L2_im_K + Sigma

# Cholesky factorization, L is a lower-triangular matrix
L = np.linalg.cholesky(K_im_full)

# solve for alpha
alpha = np.linalg.solve(L, Z_exp.imag)
alpha = np.linalg.solve(L.T, alpha)

# estimate the gamma of eq (21a), the minus sign, which is not included in L_im_K, refers to eq (65)
gamma_fct_est = -np.dot(L_im_K.T, alpha)

# covariance matrix
inv_L = np.linalg.inv(L)
inv_K_im_full = np.dot(inv_L.T, inv_L)

# estimate the sigma of gamma for eq (21b)
cov_gamma_fct_est = K - np.dot(L_im_K.T, np.dot(inv_K_im_full, L_im_K))
sigma_gamma_fct_est = np.sqrt(np.diag(cov_gamma_fct_est))
```

### 4c) Plot the obtained DRT against the analytical DRT

```python
# plot the DRT and its confidence region
plt.semilogx(freq_vec_plot, gamma_fct_plot, linewidth=4, color="black", label="exact")
plt.semilogx(freq_vec, gamma_fct_est, linewidth=4, color="red", label="GP-DRT")
plt.fill_between(
    freq_vec,
    gamma_fct_est - 3 * sigma_gamma_fct_est,
    gamma_fct_est + 3 * sigma_gamma_fct_est,
    color="0.4",
    alpha=0.3,
)
plt.rc("text", usetex=True)
plt.rc("font", family="serif", size=15)
plt.rc("xtick", labelsize=15)
plt.rc("ytick", labelsize=15)
plt.axis([1e-4, 1e4, -5, 25])
plt.legend(frameon=False, fontsize=15)
plt.xlabel(r"$f/{\rm Hz}$", fontsize=20)
plt.ylabel(r"$\gamma/\Omega$", fontsize=20)
plt.show()
```

### 4d) Predict the $\gamma$ and the imaginary part of the GP-DRT impedance

This part is explained in Section `2.3.3` of the main article

```python
# initialize the imaginary part of impedance vector
Z_im_vec_star = np.empty_like(xi_vec_star)
Sigma_Z_im_vec_star = np.empty_like(xi_vec_star)

gamma_vec_star = np.empty_like(xi_vec_star)
Sigma_gamma_vec_star = np.empty_like(xi_vec_star)

# calculate the imaginary part of impedance at each $\xi$ point for the plot
for index, val in enumerate(xi_vec_star):
    xi_star = np.array([val])

    # compute matrices shown in eq (18), k_star corresponds to a new point
    k_star = GP_DRT.matrix_K(xi_vec, xi_star, sigma_f, ell)
    L_im_k_star = GP_DRT.matrix_L_im_K(xi_vec, xi_star, sigma_f, ell)
    L2_im_k_star = GP_DRT.matrix_L2_im_K(xi_vec, xi_star, sigma_f, ell)
    k_star_star = GP_DRT.matrix_K(xi_star, xi_star, sigma_f, ell)
    L_im_k_star_star = GP_DRT.matrix_L_im_K(xi_star, xi_star, sigma_f, ell)
    L2_im_k_star_star = GP_DRT.matrix_L2_im_K(xi_star, xi_star, sigma_f, ell)

    # compute Z_im_star mean and standard deviation using eq (26)
    Z_im_vec_star[index] = np.dot(L2_im_k_star.T, np.dot(inv_K_im_full, Z_exp.imag))
    Sigma_Z_im_vec_star[index] = L2_im_k_star_star - np.dot(
        L2_im_k_star.T, np.dot(inv_K_im_full, L2_im_k_star)
    )

    # compute Z_im_star mean and standard deviation
    gamma_vec_star[index] = -np.dot(L_im_k_star.T, np.dot(inv_K_im_full, Z_exp.imag))
    Sigma_gamma_vec_star[index] = k_star_star - np.dot(
        L_im_k_star.T, np.dot(inv_K_im_full, L_im_k_star)
    )
```

### 4e) Plot the imaginary part of the GP-DRT impedance together with the exact one and the synthetic experiment

```python
plt.semilogx(
    freq_vec_star, -np.imag(Z_exact), ":", linewidth=4, color="blue", label="exact"
)
plt.semilogx(
    freq_vec, -Z_exp.imag, "o", markersize=10, color="black", label="synth exp"
)
plt.semilogx(freq_vec_star, -Z_im_vec_star, linewidth=4, color="red", label="GP-DRT")
plt.fill_between(
    freq_vec_star,
    -Z_im_vec_star - 3 * np.sqrt(abs(Sigma_Z_im_vec_star)),
    -Z_im_vec_star + 3 * np.sqrt(abs(Sigma_Z_im_vec_star)),
    alpha=0.3,
)
plt.rc("text", usetex=True)
plt.rc("font", family="serif", size=15)
plt.rc("xtick", labelsize=15)
plt.rc("ytick", labelsize=15)
plt.axis([1e-4, 1e4, -5, 25])
plt.legend(frameon=False, fontsize=15)
plt.xlabel(r"$f/{\rm Hz}$", fontsize=20)
plt.ylabel(r"$-Z_{\rm im}/\Omega$", fontsize=20)
plt.show()
```


# Simple Linear Dynamical System Demo

Linear Dynamical Systems (LDS) are used to model data that evolve in principled ways. They are widely used in control systems, signal processing, neuroscience, and other fields. Our focus in on learning and inference for LDS, and a good reference for this is Chapter 13.3 of [Pattern Recognition and Machine Learning](http://users.isr.ist.utl.pt/~wurmd/Livros/school/Bishop%20-%20Pattern%20Recognition%20And%20Machine%20Learning%20-%20Springer%20%202006.pdf) by Christopher Bishop.

For LDS with Poisson observations, especially as they are used in neuroscience, [Estimating State and parameters in state space models of spike trains](https://www.cambridge.org/core/services/aop-cambridge-core/content/view/FAB8634C2790F3461E3E86BB632EAE6F/9781139941433c6_p137-159_CBO.pdf/estimating_state_and_parameters_in_state_space_models_of_spike_trains.pdf), by J.H Macke _et al_ is a good reference.

## 1. Setup
The line `import ssm` imports the package for use. Here, we have also imported a few other packages for plotting.

**Note**: We recently added Seaborn to the list of dependencies. If you have an older installation of SSM, you may need to manually install Seaborn using `pip install seaborn` or `conda install seaborn`.

```python
import autograd.numpy as np
import autograd.numpy.random as npr
npr.seed(0)

import matplotlib.pyplot as plt
%matplotlib inline

from matplotlib.gridspec import GridSpec

import seaborn as sns
sns.set_style("white")
sns.set_context("talk")

color_names = ["windows blue",
               "red",
               "amber",
               "faded green",
               "dusty purple",
               "orange",
               "clay",
               "pink",
               "greyish",
               "mint",
               "light cyan",
               "steel blue",
               "forest green",
               "pastel purple",
               "salmon",
               "dark brown"]

colors = sns.xkcd_palette(color_names)

import ssm
from ssm.util import random_rotation
from ssm.plots import plot_dynamics_2d

# Specify whether or not to save figures
save_figures = False
```

## 2. Create an LDS Object

A Linear Dynamical System consists of a continuous state variable, $x \in \mathbb{R}^D$, which evolves according to linear dynamics.
At time-step $t$, we observe an emission (sometimes called an observation) $y_t \in \mathbb{R}^N$. Similarly, we use $x_t$ to denote the state at time $t$.

The state dynamics obey the following relation (Bishop 2006):

$$
\begin{align}
x_t &= A x_{t-1} + V u_{t} + b + w_t \\
\end{align}
$$

The vector $b$ is an offset vector, which can drive the dynamics in a particular direction.
The terms $w$ and $v$ are noise terms, which perturb the states and observations.
Most commonly these are modeled as zero-mean multivariate Gaussians,
but one nice feature of SSM is that it supports many distributions for these noise terms.

We support the following dynamics models, which can be specified by passing an argument to the constructor the LDS class, e.g `dynamics="gaussian"`:

1. **`none`**: Model has no dynamics. At each time-step, the state is drawn independently from a Gaussian. This is the equivalent of filling the dynamics matrix $A$ with zeros.
1. **`gaussian`**: Model has linear-gaussian dynamics. $w \sim \mathcal{N}(0, \Sigma_w )$.
1. **`diagonal_gaussian`**: This is a special case of the Gaussian dynamics model where the noise term has diagonal covariance.
1. **`studentst`**: $w \sim t(0, \Sigma_w, \nu_w)$.
1. **`diagonal_studentst`**: Constrains covariance matrix to be diagonal.

To specify an LDS, we also need to specify how the emissions $y_t$ are generated (the emissions are what we observe).
The most common emission model is the simple linear-Gaussian emission model:
$$
y_t = Cx_t + d + F u_t + v_t
$$
In this case, $C$ is called the measurement matrix, $d$ is an offset or bias term, and $F$ is called the feedthrough matrix or passthrough matrix (it passes the input directly to the emission).
$v_t \sim \mathcal{N}(0, \Sigma_v)$ is a noise term, which in the simplest case is Gaussian, but can also be modeled to come from other distributions.
We support the following emission models.

1. **`gaussian`**: The most common emission model. $y_t \sim \mathcal{N}(C x_t + d + F u_t, \Sigma_v)$
1. **`studentst`**: Same as above, but uses the Students-T distribution instead of the normal distribution.
1. **`poisson`**: Useful for modeling count or event data. Under this model, the observations are nonnegative integers drawn from a poisson distribution (Macke 2011).
The latent state controls the rate parameter, in a non-linear fashion as follows. Let $y_{i,t}$ be the $i$-th dimension of our observation at time $t$.
We'll also make use of an excitation vector $g_t = C x_t + d + F u_t$, and additionally a mean function $\eta(\bullet)$.
The observations are drawn from a poisson distribution at rates specified by the excitation vector:
$$P( y_{i,t} \mid g_{i,t}) = \frac{1}{y_{i,t}!} \eta(g_{i,t})^{y_{i,t}}\exp{(-\eta(g_{i,t}))}.$$ By default, the mean function is set to $\exp(\bullet)$.

1. **`bernoulli`**:
As in the Poisson case, we have an excitation vector $g_t = C x_t + d + F u_t$, and a mean function $\eta(\bullet)$. In this case, though, the mean function is always the logistic function, becuase its outputs need to be between zero and one. The observations are drawn from a bernoulli distribution where $y_{i,t}$ is 1 with probability $\eta(g_{i,t})$.

For all of the above we support orthogonal variants, in which the measurement matrix $C$ is constrained to be orthogonal.

_Note: SSM supports many other emission models. We are in the process of creating full-standalone documentation to describe them. For now, the best way to learn about SSM's other functionality is look at the source code. The emission models are described in emissions.py._

The below line creates an instance of the SSM's `LDS` class:
`true_lds = ssm.LDS(obs_dim, state_dim, dynamics="gaussian", emissions="gaussian")`

Following that, we modify the dynamics matrix (the matrix $A$ in the above equations) so that our system will have some interesting behavior.

The lines most relevant for a user of SSM are these:
`true_lds.dynamics.A = A
true_lds.dynamics.b = b`

The first line sets the dynamics matrix $A$ to be the one which we create. The second line sets the offset vector $b$.

```python
# Set the parameters of the LDS
time_bins = 200   # number of time bins
state_dim = 2     # number of latent dimensions
obs_dim = 10      # number of observed dimensions
```

```python
# Make an LDS with somewhat interesting dynamics parameters
true_lds = ssm.LDS(obs_dim, state_dim, emissions="gaussian_orthog")

A0 = .99 * random_rotation(state_dim, theta=np.pi/20)
S = np.arange(1, state_dim+1)
R = np.linalg.svd(npr.randn(state_dim, state_dim))[0] * S
A = R.dot(A0).dot(np.linalg.inv(R))
b = npr.randn(state_dim)

# Set the dynamics matrix (A) and the
true_lds.dynamics.A = A
true_lds.dynamics.b = b
```

## 3. Sample from the LDS and plot
Since our state variable $x$ is two-dimensional, we can plot its path over time in the 2D plane. Below, we plot the trajectory of the state, along with a vector field showing the dynamics of the system. The small blue arrows show the direction that the dynamics of the system will "push" the state at each point.

Though it's a bit beyond the scope of this notebook, a lot can be said about how the matrix $A$ determins the dynamics of the system. For example, if the spectral radius of $A$ is larger than 1, the system will be unstable, and the state will blow up as time goes to infinity (the spectral radius is the largest singular value).
We've been hand-wavy in defining stability and instability.
In our case, we have explicitly constructed a stable system, and so we see that the dynamics tend towards a fixed point.

We see that the trajectory (shown in black) follows the blue arrows quite closely, spiraling towards the center. Why doesn't the trajectory follow the arrows exactly? This is because we have noise on the dynamics equation (the $w_t$s in the equations above).

```python
# Sample from the LDS object.
states, emissions = true_lds.sample(time_bins)

# Plot the dynamics vector field
q = plot_dynamics_2d(A,
                     bias_vector=b,
                     mins=states.min(axis=0),
                     maxs=states.max(axis=0),
                     color=colors[0])

plt.plot(states[:,0], states[:,1], '-k', lw=3)
plt.xlabel("$x_1$")
plt.ylabel("$x_2$")
plt.title("Simulated Latent States")
plt.tight_layout()

if save_figures:
    plt.savefig("lds_1.pdf")
```

### Exercise 3.1: Modify trajectory
See if you can modify the LDS object created in Section 2 so that the system will be unstable. In other words, you want to set the matrix $A$ so that the largest singular value is greater than 1.

One way of doing this is by starting with the matrix $A$ we used previously, and taking the SVD of that matrix. After taking the SVD, you can modify the largest singular value, and then re-form the dynamics matrix.
After sampling from the system and plotting, do you notice anythign different?

### Exercise 3.2: Advancing $t$ timesteps.
Say the LDS starts at state $x_0$ at time zero. Can you derive an expression for the mode of $p(x_t \mid x_0)$, where $t$ is some number of timesteps in the future?
In other words, what is $ \mathbb{E} \left[x_t \mid x_0 \right]$ ?

## 4. Visualize States and Observations
In the above section, we were able to plot a trajectory for the latent state over time. Since our emissions are 10-dimensional, we can't show a trajectory for them in a 10-d plane (matplotlib doesn't support that yet). Instead, we'll plot each emission coordinate $y_i$ as a function over time. We do the same for the simulated latent states.

```python
plt.figure(figsize=(8, 6))
gs = GridSpec(2, 1, height_ratios=(1, obs_dim/state_dim))

# Plot the continuous latent states
lim = abs(states).max()
plt.subplot(gs[0])
for d in range(state_dim):
    plt.plot(states[:, d] + lim * d, '-k')
plt.yticks(np.arange(state_dim) * lim, ["$x_{}$".format(d+1) for d in range(state_dim)])
plt.xticks([])
plt.xlim(0, time_bins)
plt.title("Simulated Latent States")

lim = abs(emissions).max()
plt.subplot(gs[1])
for n in range(obs_dim):
    plt.plot(emissions[:, n] - lim * n, '-k')
plt.yticks(-np.arange(obs_dim) * lim, ["$y_{{ {} }}$".format(n+1) for n in range(obs_dim)])
plt.xlabel("time")
plt.xlim(0, time_bins)

plt.title("Simulated emissions")
plt.tight_layout()

if save_figures:
    plt.savefig("lds_2.pdf")
```

## 5. Fit an LDS to Data
Simulating a Linear-Gaussian LDS is all fine, but you're probably interested in fitting an LDS to your own datasets. This is useful, for instance, if you believe that your data are well described by linear dynamical system, and you're interested in examining the dynamics of that system.

In the below cells, we'll treat the variable `emissions` which we generated above as data which comes from an unknown LDS.
The line `data = emissions` is just to emphasize that our new LDS will be fit from data.

1. First, we make a new LDS object:
`lds = ssm.LDS(obs_dim, state_dim, emissions="gaussian")`

2. Then we fit the LDS:
`
elbos, q = lds.fit(data, method="laplace_em", num_iters=10)
`

3. Next, we get the posterior of the latent states with:
`state_means = q.mean_continuous_states[0]
`

4. Finally smooth the observations using our dynamics model:
`smoothed_obs = lds.smooth(state_means, data)`

**`LDS.fit`** returns a tuple of `posterior, elbos`. `posterior` is an object representing the posterior of the data given the observations, $p(x \mid y)$.
By default, SSM parameterizes the posterior as a Gaussian with a block-tridiagional precision matrix.
Of most use will probably be `posterior.mean_continuous_states[0]` which returns $\mathbb{E}q(x \mid y)$.
This is our best estimate of the state trajectory given our observations.
By default, calling `LDS.fit` will use the Laplace-EM algorithm, which is a variational inference method. In the case of Gaussian dynamics and Gaussian dynamics, Laplace-EM reduces to the normal EM update rules.
Other fitting methods, like `"svi"` (stochastic variational inference) are also supported, but don't tend to work as well in practice.
We'd recommend sticking to the default fitting methods, at least for now.

**`elbos`** stands for "evidence lower bound" and it is a lower bound on the log-likelihood of the data. It is used to monitor convergence of the EM algorithm

**Note 1**: You'll notice when you call `fit` that you actually see two progress bars, one labeled "LP" and one labeled "ELBO". This is because internally, the fitting is done in two steps. In order to get a good initialization of the paramters, we first initialize using an autoregressive HMM.
Then, we obtain the final parameters using the Laplace-EM algorithm. The first progress bar is showing progress for the initialization (fitting an ar-HMM).

**Note 2**: We realize some of the APIs are quite weird. For example, `mean_continuous_states[0]` is a bit strange. This is because the APIs are currently shared between vanilla LDS and switching LDS. We plan on fixing this soon, and will update the notebook.

```python
data = emissions

print("Fitting LDS with Laplace-EM using structured variational posterior")
lds = ssm.LDS(obs_dim, state_dim, emissions="gaussian_orthog")
elbos, q = lds.fit(data, method="laplace_em", num_iters=10)

# Get the posterior mean of the continuous states
state_means = q.mean_continuous_states[0]

# Smooth the data under the variational posterior
smoothed_obs = lds.smooth(state_means, data)
```

    Fitting LDS with Laplace-EM using structured variational posterior
    Initializing with an ARHMM using 25 steps of EM.

    HBox(children=(IntProgress(value=0, max=25), HTML(value='')))

    HBox(children=(IntProgress(value=0, max=10), HTML(value='')))

Below, we plot the ELBOs which we got from the `fit` function to check that our fitting procedure converged.

```python
# Plot the ELBOs
plt.plot(elbos, label="Laplace-EM")
plt.xlabel("Iteration")
plt.ylabel("ELBO")
plt.legend()
```

## 6. Visualize Model Fit
Below, we show the true and estimated states over time. Although there is an offset between the true state and our estimate, we see that the dynamics of the system seem to match pretty well: our estimate follows a similar trajectory as the true state.

We also use our model to "smooth" the emissions by removing noise. Once we have learned the dynamics of the system, we see that we can remove some of the jitter from the emissions.

```python
plt.figure(figsize=(8,4))
for d in range(state_dim):
    plt.plot(states[:,d] + 4 * d, '-k', label="True States" if d==0 else None)
    plt.plot(state_means[:,d] + 4 * d, '--', color=colors[0], label="Estimated States" if d==0 else None)
plt.ylabel("$x$")
plt.xlabel("time")
plt.legend(loc='upper right')
plt.title("True and Estimated States")
plt.show()
```

```python
# Plot the smoothed emissions
plt.figure(figsize=(8,4))
for n in range(obs_dim):
    plt.plot(emissions[:, n] + 4 * n, '-k', label="Original" if n == 0 else None)
    plt.plot(smoothed_obs[:, n] + 4 * n, '--', color=colors[0], label="Smoothed" if n == 0 else None)
plt.legend()
plt.xlabel("time")
plt.title("Original and Smoothed Observations")
plt.show()
```

Below, we visualize the true dynamics, as well as the dynamics of the system we learned. We would hope that the overall features would be the same, though the overall dynamics might be rotated.

```python
# Extract dynamics matrix from the true model
A_true = true_lds.dynamics.A
b_true = true_lds.dynamics.b

A_est = lds.dynamics.A
b_est = lds.dynamics.b

f, ax = plt.subplots(1, 2, figsize=(10, 4))
plot_dynamics_2d(A_true, b_true, npts=10, axis=ax[0], color=colors[0])
ax[0].set_xlabel("$x_1$")
ax[0].set_ylabel("$x_2$")
ax[0].set_title("True Dynamics")

plot_dynamics_2d(A_est, b_est, npts=10, axis=ax[1], color=colors[1])
ax[1].set_xlabel("$x_1$")
ax[1].set_ylabel("$x_2$")
ax[1].set_title("Inferred Dynamics")

plt.tight_layout()
```

### Exercise 6.1: Mis-specified Dynamics Model
What you expect to happen if you fit an HMM to your data when the true dataset was generated by an LDS? Assume that in both cases, the observation model is Gaussian, so that only the dynamics model is misspecified.

How would the performance of your model change as you increase the number of discrete states in the HMM?

## 7. Simulate Poisson data from a Poisson LDS with the same dynamics

In the below cells, we create an LDS with poisson observations that shares the dynamics of the LDS above.
The line:

`plds = ssm.LDS(obs_dim, state_dim, emissions="poisson_orthog", emission_kwargs=dict(link="softplus"))
`

creates an LDS object with Poisson observations.
Specifying `link="softplus"` specifes that the link function will be the softplus function.
The term `link` refers to the link a generalized linear model, which controls how the linear latent variable, in this case the state $x_t$ controls the emission distribution.
When creating a Poisson or Bernoulli LDS, the user specifies the link function, which allows in-turn sets the mean function of the emissions.

For Poisson LDS, SSM supports two link functions:
1. `link="softplus"`: The mean function $\eta(x)$ is set to the inverse softplus function.
2. `link="log"`: The mean function  $\eta(x)$ is set to the exponential function.

Below, we copy the parameters from our prior LDS to the new object:
`plds.dynamics.params = copy.deepcopy(true_lds.dynamics.params)
`

We use `copy.deepcopy` to make sure we don't accidentally change the parameters in the other LDS object.
The Poisson emissions are composed of a linear mapping from the continuous states:
$$z_t = C x_t + d + F u$$
In the below line, we set the bias vector $d$ in the above equation to zeros:
`
plds.emissions.ds = 0 * np.ones(obs_dim)
`

After sampling from our Poisson LDS, we sample and show the trajectory, as before. We then also visualize our Poisson observations. Since Poisson observations are nonnegative integers, we plot them as a color density.

```python
import copy

plds = ssm.LDS(obs_dim, state_dim, emissions="poisson_orthog", emission_kwargs=dict(link="softplus"))
plds.dynamics.params = copy.deepcopy(true_lds.dynamics.params)
plds.emissions.ds = 0 * np.ones(obs_dim)
states_plds, obs_plds = plds.sample(time_bins)
```

```python
# Plot the dynamics vector field
plot_dynamics_2d(plds.dynamics.A,
                 plds.dynamics.b,
                 mins=states_plds.min(axis=0),
                 maxs=states_plds.max(axis=0),
                 color=colors[0])
plt.plot(states_plds[:,0], states_plds[:,1], '-k', lw=3)
plt.xlabel("$x_1$")
plt.ylabel("$x_2$")
plt.title("Simulated Latent States")
plt.tight_layout()

if save_figures:
    plt.savefig("lds_6.pdf")
```

```python
# Plot the dynamics vector field
plt.figure(figsize=(8, 6))
gs = GridSpec(2, 1, height_ratios=(1, obs_dim/state_dim))

# Plot the continuous latent states
lim = abs(states_plds).max()
plt.subplot(gs[0])
for d in range(state_dim):
    plt.plot(states_plds[:, d] + lim * d, '-k')
plt.yticks(np.arange(state_dim) * lim, ["$x_{}$".format(d+1) for d in range(state_dim)])
plt.xticks([])
plt.xlim(0, time_bins)
plt.title("Simulated Latent States")

lim = abs(obs_plds).max()
plt.subplot(gs[1])
plt.imshow(obs_plds.T, cmap="Greys", aspect="auto")
plt.yticks(np.arange(obs_dim), ["$y_{{ {} }}$".format(n+1) for n in range(obs_dim)])
plt.xlabel("time")
plt.xlim(0, time_bins)
plt.title("Simulated Poisson Observations")
plt.colorbar()

plt.tight_layout()

if save_figures:
    plt.savefig("lds_7.pdf")
```

### Exercise 7.1: Mis-specified Observation Model
What you expect to happen if you fit an LDS with Gaussian observations to a system where the true observation model is Poisson?

You can try this by following the steps in Section 5 using the data we generated in Section 7.

notebooks/1b Simple Linear Dynamical System.ipynb


# Classification using NAG Second-order Conic Programming via CVXPY

## Correct Rendering of this notebook

This notebook makes use of the `latex_envs` Jupyter extension for equations and references.  If the LaTeX is not rendering properly in your local installation of Jupyter , it may be because you have not installed this extension.  Details at https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/nbextensions/latex_envs/README.html

## Installing the NAG library and running this notebook

This notebook depends on the NAG library for Python to run. Please read the instructions in the [Readme.md](https://github.com/numericalalgorithmsgroup/NAGPythonExamples/blob/master/local_optimization/Readme.md#install) file to download, install and obtain a licence for the library.

Instruction on how to run the notebook can be found [here](https://github.com/numericalalgorithmsgroup/NAGPythonExamples/blob/master/local_optimization/Readme.md#jupyter).

## Introduction

In this notebook, we demonstrate how NAG second-order conic programming (SOCP) solver can be used to find a quartic polynomial
\vspace{0.1cm}
\begin{equation}\label{poly}
p(x,y) = \sum_{i,j} a_{ij} x^i y^j ~\mbox{with}~ i+j\leq4
\end{equation}
\vspace{0.1cm}
that separates two sets of points in a plane robustly. The optimization model for this problem can be cast as
\vspace{0.1cm}
\begin{equation}\label{prob}
\begin{array}{ll}
\underset{a\in\Re^{15}}{\mbox{minimize}} & \|a\|\\[0.6ex]
\mbox{subject to} & p(x,y)\leq -1, \mbox{for x, y in set 1},\\[0.6ex]
     & p(x,y)\geq 1, \mbox{for x, y in set 2}.
\end{array}
\end{equation}
\vspace{0.1cm}

```python
# Import necessary packages
import numpy as np
import cvxpy as cvx
import matplotlib.pyplot as plt
```

## Data preparation
In this section, we generate two sets of random points used as training data for our model.

```python
# Fix random seed
np.random.seed(3)
# Generate two primary sets of points
# Number of points in each set
n = 80
set_1_base = np.random.uniform(-1.0, 1.0, (n,2))
set_2_base = np.random.uniform(-1.0, 1.0, (n,2))
# Scale the primary points to make set 1 surrounded by set 2
for i in range(n):
    set_1_base[i,0:] = 0.9 * set_1_base[i,0:] * np.random.rand() / \
                       np.linalg.norm(set_1_base[i,0:])
    set_2_base[i,0:] = set_2_base[i,0:] * (1.1 + np.random.rand() / \
                                np.linalg.norm(set_2_base[i,0:]))
# Further process the data to make abnormal shape of sets of points
maxnorm_set_1 = max(np.linalg.norm(set_1_base,axis=1))
set_2_pick = set_2_base[np.linalg.norm(set_2_base,axis=1)>maxnorm_set_1,0:]
set_1_pick = np.concatenate((set_1_base,
                             set_2_base[np.linalg.norm(set_2_base,axis=1)<=
                                        maxnorm_set_1,0:]))
# The shape of set 1 is primarily round, I punch set 1 from the left to make it abnormal
# Feel free to modify punch power to see the resulting graph
punch_power = 1.0
set_1 = set_1_pick[np.linalg.norm(set_1_pick-[-1.0,-0.5],axis=1)>punch_power,0:]
set_2 = np.concatenate((set_2_pick,
                        set_1_pick[np.linalg.norm(set_1_pick-[-1.0,-0.5],axis=1)<=
                                   punch_power,0:]))
```

Now the training data is ready to use. Visualize it.

```python
data = (set_1, set_2)
colors = ("red", "blue")
groups = ("set1", "set2")

fig = plt.figure()
ax = fig.add_subplot(1, 1, 1, alpha=1.0)
for data, color, group in zip(data, colors, groups):
    x = data[0:,0]
    y = data[0:,1]
    ax.scatter(x, y, alpha=0.8, c=color, edgecolors='none', s=10, label=group)
plt.xlim(-2.0, 2.0)
plt.ylim(-2.0, 2.0)
plt.title('Training Data')
plt.legend(loc=2)
plt.show()
```

The number of coefficient for a quartic polynomial is $15$, which is therefore our number of variables.

```python
# Number of variables
nvar = 15
```

Both constraints in (\ref{prob}) are linear with respect to variable $a$. Use set_1 and set_2 to set linear coefficients for those constraints.

```python
A_1 = np.zeros((set_1.shape[0], nvar))
A_2 = np.zeros((set_2.shape[0], nvar))

for i in range(set_1.shape[0]):
    counter = 0
    for j in range(5):
        for k in range(5-j):
            A_1[i,counter] = set_1[i,0]**j * set_1[i,1]**k
            counter = counter + 1
for i in range(set_2.shape[0]):
    counter = 0
    for j in range(5):
        for k in range(5-j):
            A_2[i,counter] = set_2[i,0]**j * set_2[i,1]**k
            counter = counter + 1
```

## Model our problem via CVXPY and solve

```python
# Define decision variables
x = cvx.Variable(nvar)
# Define objective function
objective = cvx.Minimize(cvx.norm(x))
# Define constraints
constraint = [A_1@x <= -1.0, A_2@x >= 1.0]
# Define the entire problem
problem = cvx.Problem(objective, constraint)
# Solve, Bing!
problem.solve(solver='NAG', verbose=True)
# Save result
coef = x.value
```

     ------------------------------------------------
      E04PT, Interior point method for SOCP problems
     ------------------------------------------------

     Begin of Options
         Print File                    =                  -1     * U
         Print Level                   =                   2     * d
         Print Options                 =                 Yes     * d
         Print Solution                =                  No     * d
         Monitoring File               =                   6     * U
         Monitoring Level              =                   2     * U
         Socp Monitor Frequency        =                   0     * d

         Infinite Bound Size           =         1.00000E+20     * d
         Task                          =            Minimize     * d
         Stats Time                    =                  No     * d
         Time Limit                    =         1.00000E+06     * d

         Socp Iteration Limit          =                 100     * d
         Socp Max Iterative Refinement =                   9     * d
         Socp Presolve                 =                 Yes     * d
         Socp Scaling                  =                None     * d
         Socp Stop Tolerance           =         1.05367E-08     * d
         Socp Stop Tolerance 2         =         1.05367E-08     * d
         Socp System Formulation       =                Auto     * d
     End of Options

     Problem Statistics
       No of variables                 32
         bounds               not defined
       No of lin. constraints         176
         nonzeroes                   2432
       No of quad.constraints           0
       No of cones                      1
         biggest cone size             16
       Objective function          Linear

     Presolved Problem Measures
       No of variables                191
       No of lin. constraints         175
         nonzeroes                   2590
       No of cones                      1

     ------------------------------------------------------------------------
      it|    pobj    |    dobj    |  p.inf  |  d.inf  |  d.gap  |   tau  | I
     ------------------------------------------------------------------------
       0  1.00000E+00  0.00000E+00  2.70E-02  5.35E-03  2.00E+00  1.0E+00
       1  5.13493E+01  8.32818E+01  1.29E-02  2.55E-03  9.55E-01  5.0E-02
       2  2.80351E+01  5.34418E+01  9.73E-03  1.93E-03  7.22E-01  3.6E-01
       3  1.11360E+02  2.17980E+02  4.46E-03  8.86E-04  3.31E-01  1.2E-01
       4  2.07571E+02  4.06965E+02  2.08E-03  4.13E-04  1.55E-01  5.9E-02
       5  3.02366E+02  5.91847E+02  1.38E-03  2.74E-04  1.03E-01  3.5E-02
       6  2.42309E+02  4.51372E+02  2.93E-04  5.82E-05  2.18E-02  2.0E-02
       7  1.29884E+02  1.56324E+02  8.30E-05  1.65E-05  6.16E-03  2.2E-02
       8  7.71822E+01  8.68099E+01  3.34E-05  6.63E-06  2.48E-03  3.4E-02
       9  6.67725E+01  7.13985E+01  1.75E-05  3.48E-06  1.30E-03  3.6E-02
      10  6.49722E+01  6.63180E+01  7.68E-06  1.52E-06  5.70E-04  3.2E-02
      11  6.27050E+01  6.28895E+01  2.58E-06  5.11E-07  1.91E-04  3.2E-02
      12  6.17692E+01  6.18968E+01  1.62E-06  3.21E-07  1.20E-04  2.9E-02
      13  6.14587E+01  6.14705E+01  4.56E-07  9.04E-08  3.38E-05  2.9E-02
      14  6.11965E+01  6.11981E+01  5.12E-08  1.02E-08  3.80E-06  2.8E-02
      15  6.11753E+01  6.11754E+01  2.23E-09  4.43E-10  1.66E-07  2.8E-02
      16  6.11747E+01  6.11747E+01  8.75E-11  1.72E-11  6.42E-09  2.8E-02
     ------------------------------------------------------------------------------
     Status: converged, an optimal solution found
     ------------------------------------------------------------------------------
     Final primal objective value         6.117468E+01
     Final dual objective value           6.117469E+01
     Absolute primal infeasibility        6.493819E-09
     Relative primal infeasibility        8.753236E-11
     Absolute dual infeasibility          3.212062E-09
     Relative dual infeasibility          1.718111E-11
     Absolute complementarity gap         6.424124E-09
     Relative complementarity gap         6.424124E-09
     Iterations                                     16

## Visualize the classifier

```python
# Generate a mesh
x = np.arange(-5.0,5.0,0.008)
y = np.arange(-5.0,5.0,0.008)
xx, yy = np.meshgrid(x,y)

# Ploynomial value on the mesh
polyval = np.zeros(xx.shape)
counter = 0
for i in range(5):
    for j in range(5-i):
        polyval = polyval + coef[counter]*np.power(xx,i)*np.power(yy,j)
        counter = counter + 1

# Plot the trained polynomial
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1, alpha=1.0)

ax.scatter(xx[polyval<=-1], yy[polyval<=-1], alpha=0.002)

data = (set_1, set_2)
colors = ("red", "blue")
groups = ("set1", "set2")
for data, color, group in zip(data, colors, groups):
    x = data[0:,0]
    y = data[0:,1]
    ax.scatter(x, y, alpha=0.8, c=color, edgecolors='none', s=10, label=group)
plt.xlim(-2.0, 2.0)
plt.ylim(-2.0, 2.0)
plt.title('Polynomial')
plt.legend(loc=2)
plt.show()
```


# Introduction to the Harmonic Oscillator

*Note:* Much of this is adapted/copied from https://flothesof.github.io/harmonic-oscillator-three-methods-solution.html

This week week we are going to begin studying molecular dynamics, which uses classical mechanics to study molecular systems. Our "hydrogen atom" in this section will be the 1D harmomic oscillator.

The harmonic oscillator is a system that, when displaced from its equilibrium position, experiences a restoring force F proportional to the displacement x:

$$F=-kx$$

The potential energy of this system is

$$V = {1 \over 2}k{x^2}$$

These are sometime rewritten as

$$ F=- \omega_0^2 m x, \text{ }  V(x) = {1 \over 2}  \omega_0^2 m {x^2}$$

Where $\omega_0  = \sqrt {{k \over m}} $

In classical mechanics, our goal is to determine the equations of motion, $x(t),y(t)$, that describe our system.

In this notebook we will use sympy to solve an second order, ordinary differential equation.

## 1. Solving differential equations with sympy

Soliving differential equations can be tough, and there is not always a set plan on how to proceed. Luckily for us, the harmonic osscillator is the classic second order diffferential eqations.

Consider the following second order differential equation

$$ay(t)''+by(t)'=c$$

where $y(t)'' = {{{d^2}y} \over {dt^2}}$, and $y(t)' = {{{d}y} \over {dt}}$

We can rewrite this as a homogeneous linear differential equations

$$ay(t)''+by(t)'-c=0$$

The goal here is to find $y(t)$, similar to our classical mechanics problems. Lets use sympy to solve this equation

### Second order ordinary differential equation

First we import the sympy library

```python
import sympy as sym
```

Next we initialize pretty printing

```python
sym.init_printing()
```

Next we will set our symbols

```python
t,a,b,c=sym.symbols("t,a,b,c")
```

Now for somehting new. We can define functions using `sym.Function("f")`

```python
y=sym.Function("y")
y(t)
```

Now, If I want to define a first or second derivative, I can use `sym.diff`

```python
sym.diff(y(t),(t,1)),sym.diff(y(t),(t,2))
```

My differential equation can be written as follows

```python
dfeq=a*sym.diff(y(t),(t,2))+b*sym.diff(y(t),(t,1))-c
dfeq
```

```python
sol = sym.dsolve(dfeq)
sol
```

The two constants $C_1$ and $C_2$ can be determined by setting boundry conditions.
First, we can set the condition $y(t=0)=y_0$

The next intial condition we will set is $y'(t=0)=v_0$

To setup the equality we want to solve, we are using `sym.Eq`. This function sets up an equaility between a lhs aand rhs of an equation

```python
# sym.Eq example
alpha,beta=sym.symbols("alpha,beta")
sym.Eq(alpha+2,beta)
```

Back to the actual problem

```python
x0,v0=sym.symbols("x_0,v_0")
ics=[sym.Eq(sol.args[1].subs(t, 0), x0),
     sym.Eq(sol.args[1].diff(t).subs(t, 0), v0)]
ics
```

We can use this result to first solve for $C_2$ and then solve for $C_1$.
Or we can use sympy to solve this for us.

```python
solved_ics=sym.solve(ics)
solved_ics
```

Substitute the result back into $y(t)$

```python
full_sol = sol.subs(solved_ics[0])
full_sol
```

We can plot this result too. Assume that $a,b,c=1$ and that the starting conditions are $y_0=0,v_0=0$

We will use two sample problems:

* case 1 : initial position is nonzero and initial velocity is zero
* case 2 : initial position is zero and initialvelocity is nonzero

```python
# Print plots
%matplotlib inline
```

#### Initial velocity set to zero

```python
case1 = sym.simplify(full_sol.subs({y0:0, v0:0, a:1, b:1, c:1}))
case1
```

```python
sym.plot(case1.rhs)
sym.plot(case1.rhs,(t,-2,2))
```

#### Initial velocity set to one

```python
case2 = sym.simplify(full_sol.subs({y0:0, v0:1, a:1, b:1, c:1}))
case2
```

```python
sym.plot(case2.lhs,(t,-2,2))
```

## Calculate the phase space

As we will see in lecture, the state of our classical systems are defined as points in phase space, a hyperspace defined by ${{\bf{r}}^N},{{\bf{p}}^N}$. We will convert our sympy expression into a numerical function so that we can plot the path of $y(t)$ in phase space $y,y'$.

```python
case1
```

```python
# Import numpy library
import numpy as np

# Make numerical functions out of symbolic expressions
yfunc=sym.lambdify(t,case1.rhs,'numpy')
vfunc=sym.lambdify(t,case1.rhs.diff(t),'numpy')

# Make list of numbers
tlst=np.linspace(-2,2,100)

# Import pyplot
import matplotlib
import matplotlib.pyplot as plt
# Make plot
plt.plot(yfunc(tlst),vfunc(tlst))
plt.xlabel('$y$')
plt.ylabel("$y'$")
plt.show()
```

### Exercise 1.1

Change the initial starting conditions and see how that changes the plots. Make three different plots with different starting conditions

```python
# Change starting velocity to 5
case3 = sym.simplify(full_sol.subs({y0:0, v0:5, a:1, b:1, c:1}))
tlst=np.linspace(-2,2,100)
sym.plot(case3.rhs,(t,-2,2))

# Change starting position to 3
case4 = sym.simplify(full_sol.subs({y0:3, v0:0, a:1, b:1, c:1}))
tlst=np.linspace(-2,2,100)
sym.plot(case4.rhs,(t,-2,2))

# Change starting velocity to 0.5
case5 = sym.simplify(full_sol.subs({y0:0, v0:0.5, a:1, b:1, c:1}))
tlst=np.linspace(-2,2,100)
sym.plot(case5.rhs,(t,-2,2))
```

```python
#
```

## 2. Harmonic oscillator

Applying the harmonic oscillator force to Newton's second law leads to the following second order differential equation

$$ F = m a $$

$$ F= - \omega_0^2 m x $$

$$ a = - \omega_0^2 x $$

$$ x(t)'' = - \omega_0^2 x $$

The final expression can be rearranged into a second order homogenous differential equation, and can be solved using the methods we used above

Your goal is determine and plot the equations of motion of a 1D harmomnic oscillator

### Exercise 2.1

1. Use the methodology above to determine the equations of motion $x(t), v(t)$ for a harmonic ocillator
1. Solve for any constants by using the following initial conditions: $x(0)=x_0, v(0)=v_0$
1. Show expressions for and plot the equations of motion for the following cases:
    1. $x(0)=0, v(0)=0$
    1. $x(0)=0, v(0)>0$
    1. $x(0)>0, v(0)=0$
    1. $x(0)<0, v(0)=0$
1. Plot the phasespace diagram for the harmonic oscillator

```python
# Equations of motion x(t). a= dv/dt
t,a,b,c,x,k,z, w0=sym.symbols("t,a,b,c,x,k,z,w0")
x=sym.Function("x")
x(t)
sym.diff(x(t),(t,1)),sym.diff(x(t),(t,2))
dfeq=a*sym.diff(x(t),(t,2))+b*sym.diff(x(t),(t,1))-c
dfeq

```

```python
sol = sym.dsolve(dfeq)
sol
```

```python
# sym.Eq example. limitation on C1 and C2
alpha,beta=sym.symbols("alpha,beta")
sym.Eq(alpha+2,beta)
```

```python
#x= x0, z'= v
x0,v0=sym.symbols("x_0,v_0")
ics=[sym.Eq(sol.args[1].subs(t, 0), x0),
     sym.Eq(sol.args[1].diff(t).subs(t, 0), v0)]
ics
full_sol = sol.subs(solved_ics[0])
full_sol
```

```python

```

```python
dfeq=sym.diff(x(t),(t,2))+w0**2*x(t)
dfeq

```

```python
sol = sym.dsolve(dfeq)
sol

alpha,beta=sym.symbols("alpha,beta")
sym.Eq(alpha+2,beta)

x0,v0=sym.symbols("x_0,v_0")
ics=[sym.Eq(sol.args[1].subs(t, 0), x0),
     sym.Eq(sol.args[1].diff(t).subs(t, 0), v0)]
ics

solved_ics=sym.solve(ics)
solved_ics
```

```python
full_sol= sol.subs(solved_ics[0])
full_sol
```

```python
#1. 𝑥(0)=0,𝑣(0)=0
casea = sym.simplify(full_sol.subs({x0:0, v0:0, w0:1}))
tlst=np.linspace(-2,2,100)
sym.plot(casea.rhs,(t,-2,2))

```

```python
#2. 𝑥(0)=0,𝑣(0)>0
caseb = sym.simplify(full_sol.subs({x0:0, v0:2, w0:1}))
tlst=np.linspace(-2,2,100)
sym.plot(caseb.rhs,(t,-2,2))
```

```python
#3. 𝑥(0)>0,𝑣(0)=0
casec = sym.simplify(full_sol.subs({x0:2, v0:0, wo:1}))
tlst=np.linspace(-2,2,100)
sym.plot(caseb.rhs,(t,-2,2))

```

```python
#4. 𝑥(0)<0,𝑣(0)=0
cased = sym.simplify(full_sol.subs({x0:-2, v0:0, w0:1))
tlst=np.linspace(-2,2,100)
sym.plot(caseb.rhs,(t,-2,2))
```

```python
# Phase Plot
import numpy as np

# Make numerical functions out of symbolic expressions
xfunc=sym.lambdify(t,caseb.rhs,'numpy')
vfunc=sym.lambdify(t,caseb.rhs.diff(t),'numpy')

# Make list of numbers
tlst=np.linspace(-10,10,100)

# Import pyplot
import matplotlib
import matplotlib.pyplot as plt
# Make plot
plt.plot(xfunc(tlst),vfunc(tlst))
plt.xlabel('$x$')
plt.ylabel("$x'$")
plt.show()
```

```python

```


```python
from sympy.physics.mechanics import *
import sympy as sp
mechanics_printing(pretty_print=True)
```

```python
m1,m2,m3,l01,l02,l03,k1,k2,k3 = sp.symbols(r'm_1,m_2,m_3,l_{0\,1},l_{0\,2},l_{0\,3},k_1,k_2,k_3')
t,g = sp.symbols('t,g')
x1, x2, x3, y1, y2, y3 = dynamicsymbols('x_1 x_2 x_3 y_1 y_2 y_3')
dx1, dx2, dx3, dy1, dy2, dy3 = dynamicsymbols('x_1 x_2 x_3 y_1 y_2 y_3', 1)
```

```python
V = m1*g*y1+m2*g*y2+m3*g*y3+sp.Rational(1,2)*k1*(sp.sqrt(x1**2+y1**2)-l01)**2+sp.Rational(1,2)*k2*(sp.sqrt((x2-x1)**2+(y2-y1)**2)-l02)**2+sp.Rational(1,2)*k3*(sp.sqrt((x3-x2)**2+(y3-y2)**2)-l03)**2
T = sp.Rational(1,2)*m1*(dx1**2+dy1**2)+sp.Rational(1,2)*m2*(dx2**2+dy2**2)+sp.Rational(1,2)*m3*(dx3**2+dy3**2)
L = sp.simplify(T - V)
```

```python
LM = LagrangesMethod(L, [x1, y1, x2, y2, x3, y3])
```

```python
soln = LM.form_lagranges_equations()
```

```python
solvedsoln = sp.solve((sp.Eq(soln[0]),sp.Eq(soln[1]),sp.Eq(soln[2]),sp.Eq(soln[3]),sp.Eq(soln[4]),sp.Eq(soln[5])),(x1.diff(t,t),y1.diff(t,t),x2.diff(t,t),y2.diff(t,t),x3.diff(t,t),y3.diff(t,t)))
```

```python
solvedsoln
```

\\[
\ddot{x}_1 =
\frac{k_2}{m_1}(x_2-x_1)\left(1-\frac{l_{0,2}}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)
+\frac{k_1}{m_1}x_1\left(\frac{l_{0,1}}{-1+\sqrt{x^2_1+y^2_1}}\right)
\\]

\\[
\ddot{x}_2 =
\frac{k_2}{m_2}(x_2-x_1)\left(-1+\frac{l_{0,2}}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)
+\frac{k_3}{m_2}(x_3-x_2)\left(1-\frac{l_{0,3}}{\sqrt{(x_3-x_2)^2+(y_3-y_2)^2}}\right)
\\]

\\[
\ddot{x}_3 = \frac{k_3}{m_3}(x_3-x_2)\left(-1+\frac{l_{0,3}}{\sqrt{(x_3-x_2)^2+(y_3-y_2)^2}}\right)
\\]

\\[
\ddot{y}_1 = -g+\frac{k_1}{m_1}y_1\left(\frac{l_{0,1}}{\sqrt{x^2_1+y^2_1}}-1\right)
+\frac{k_2}{m_1}(y_2-y_1)\left(1-\frac{l_{0,2}}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)
\\]

\\[
\ddot{y}_2 = -g+\frac{k_2}{m_2}(y_2-y_1)\left(-1+\frac{l_{0,2}}{\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}}\right)
+\frac{k_3}{m_2}(y_3-y_2)\left(1-\frac{l_{0,3}}{\sqrt{(x_3-x_2)^2+(y_3-y_2)^2}}\right)
\\]

\\[
\ddot{y}_3 = -g+\frac{k_3}{m_3}(y_3-y_2)\left(-1+\frac{l_{0,3}}{\sqrt{(x_3-x_2)^2+(y_3-y_2)^2}}\right)
\\]

```python

```