# 2. logistic_regression

```python
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
plt.style.use('fivethirtyeight')
import matplotlib.pyplot as plt
# import tensorflow as tf
from sklearn.metrics import classification_report
```

```python
data = pd.read_csv('ex2data1.txt', names=['exam1', 'exam2', 'admitted'])
data.head()
```

      exam1
      exam2
      admitted

      0
      34.623660
      78.024693
      0

      1
      30.286711
      43.894998
      0

      2
      35.847409
      72.902198
      0

      3
      60.182599
      86.308552
      1

      4
      79.032736
      75.344376
      1

```python
data.describe()
```

      exam1
      exam2
      admitted

      count
      100.000000
      100.000000
      100.000000

      mean
      65.644274
      66.221998
      0.600000

      std
      19.458222
      18.582783
      0.492366

      min
      30.058822
      30.603263
      0.000000

      25%
      50.919511
      48.179205
      0.000000

      50%
      67.032988
      67.682381
      1.000000

      75%
      80.212529
      79.360605
      1.000000

      max
      99.827858
      98.869436
      1.000000

```python
sns.set(context="notebook", style="darkgrid", palette=sns.color_palette("RdBu", 2))

sns.lmplot('exam1', 'exam2', hue='admitted', data=data,
           size=6,
           fit_reg=False,
           scatter_kws={"s": 50}
          )
plt.show()
```

```python
def get_X(df):
#     """
#     use concat to add intersect feature to avoid side effect
#     not efficient for big dataset though
#     """
    ones = pd.DataFrame({'ones': np.ones(len(df))})
    data = pd.concat([ones, df], axis=1)
    return data.iloc[:, :-1].as_matrix()

def get_y(df):
#     '''assume the last column is the target'''
    return np.array(df.iloc[:, -1])#df.iloc[:, -1]

def normalize_feature(df):
#     """Applies function along input axis(default 0) of DataFrame."""
    return df.apply(lambda column: (column - column.mean()) / column.std())
```

```python
X = get_X(data)
print(X.shape)

y = get_y(data)
print(y.shape)
```

    (100, 3)
    (100,)

```python
def sigmoid(z):
    return 1 / (1 + np.exp(-z))
```

```python
fig, ax = plt.subplots(figsize=(8, 6))
ax.plot(np.arange(-10, 10, step=0.01),
        sigmoid(np.arange(-10, 10, step=0.01)))
ax.set_ylim((-0.1,1.1))
ax.set_xlabel('z', fontsize=18)
ax.set_ylabel('g(z)', fontsize=18)
ax.set_title('sigmoid function', fontsize=18)
plt.show()
```

# cost function
> * $max(\ell(\theta)) = min(-\ell(\theta))$
> * choose $-\ell(\theta)$ as the cost function

$$\begin{align}
  & J\left( \theta  \right)=-\frac{1}{m}\sum\limits_{i=1}^{m}{[{{y}^{(i)}}\log \left( {{h}_{\theta }}\left( {{x}^{(i)}} \right) \right)+\left( 1-{{y}^{(i)}} \right)\log \left( 1-{{h}_{\theta }}\left( {{x}^{(i)}} \right) \right)]} \\
 & =\frac{1}{m}\sum\limits_{i=1}^{m}{[-{{y}^{(i)}}\log \left( {{h}_{\theta }}\left( {{x}^{(i)}} \right) \right)-\left( 1-{{y}^{(i)}} \right)\log \left( 1-{{h}_{\theta }}\left( {{x}^{(i)}} \right) \right)]} \\
\end{align}$$

```python
theta = theta=np.zeros(3) # X(m*n) so theta is n*1
theta
```

    array([ 0.,  0.,  0.])

```python

```

```python
def cost(theta, X, y):
    ''' cost fn is -l(theta) for you to minimize'''
    return np.mean(-y * np.log(sigmoid(X @ theta)) - (1 - y) * np.log(1 - sigmoid(X @ theta)))

# X @ thetaX.dot(theta)
```

```python
cost(theta, X, y)
```

    0.69314718055994529

# gradient descent
（batch gradient descent）
： $\frac{1}{m} X^T( Sigmoid(X\theta) - y )$
$$\frac{\partial J\left( \theta  \right)}{\partial {{\theta }_{j}}}=\frac{1}{m}\sum\limits_{i=1}^{m}{({{h}_{\theta }}\left( {{x}^{(i)}} \right)-{{y}^{(i)}})x_{_{j}}^{(i)}}$$

```python
def gradient(theta, X, y):
#     '''just 1 batch gradient'''
    return (1 / len(X)) * X.T @ (sigmoid(X @ theta) - y)
```

```python
gradient(theta, X, y)
```

    array([ -0.1       , -12.00921659, -11.26284221])

```python
import scipy.optimize as opt
```

```python
res = opt.minimize(fun=cost, x0=theta, args=(X, y), method='Newton-CG', jac=gradient)
```

```python
print(res)
```

         fun: 0.20349770172083584
         jac: array([  1.98942032e-06,   1.34698328e-04,   1.47259166e-04])
     message: 'Optimization terminated successfully.'
        nfev: 73
        nhev: 0
         nit: 30
        njev: 270
      status: 0
     success: True
           x: array([-25.16227358,   0.20623923,   0.20147921])

```python
def predict(x, theta):
    prob = sigmoid(x @ theta)
    return (prob >= 0.5).astype(int)
```

```python
final_theta = res.x
y_pred = predict(X, final_theta)

print(classification_report(y, y_pred))
```

                 precision    recall  f1-score   support

              0       0.87      0.85      0.86        40
              1       0.90      0.92      0.91        60

    avg / total       0.89      0.89      0.89       100

http://stats.stackexchange.com/questions/93569/why-is-logistic-regression-a-linear-classifier
> $X \times \theta = 0$  (this is the line)

```python
print(res.x) # this is final theta
```

    [-25.16227358   0.20623923   0.20147921]

```python
coef = -(res.x / res.x[2])  # find the equation
print(coef)

x = np.arange(130, step=0.1)
y = coef[0] + coef[1]*x
```

    [ 124.88769463   -1.0236254    -1.        ]

```python
data.describe()  # find the range of x and y
```

      exam1
      exam2
      admitted

      count
      100.000000
      100.000000
      100.000000

      mean
      65.644274
      66.221998
      0.600000

      std
      19.458222
      18.582783
      0.492366

      min
      30.058822
      30.603263
      0.000000

      25%
      50.919511
      48.179205
      0.000000

      50%
      67.032988
      67.682381
      1.000000

      75%
      80.212529
      79.360605
      1.000000

      max
      99.827858
      98.869436
      1.000000

> you know the intercept would be around 125 for both x and y

```python
sns.set(context="notebook", style="ticks", font_scale=1.5)

sns.lmplot('exam1', 'exam2', hue='admitted', data=data,
           size=6,
           fit_reg=False,
           scatter_kws={"s": 25}
          )

plt.plot(x, y, 'grey')
plt.xlim(0, 130)
plt.ylim(0, 130)
plt.title('Decision Boundary')
plt.show()
```

# 3- 正则化逻辑回归

```python
df = pd.read_csv('ex2data2.txt', names=['test1', 'test2', 'accepted'])
df.head()
```

      test1
      test2
      accepted

      0
      0.051267
      0.69956
      1

      1
      -0.092742
      0.68494
      1

      2
      -0.213710
      0.69225
      1

      3
      -0.375000
      0.50219
      1

      4
      -0.513250
      0.46564
      1

```python
sns.set(context="notebook", style="ticks", font_scale=1.5)

sns.lmplot('test1', 'test2', hue='accepted', data=df,
           size=6,
           fit_reg=False,
           scatter_kws={"s": 50}
          )

plt.title('Regularized Logistic Regression')
plt.show()
```

# feature mapping（特征映射）

polynomial expansion

```
for i in 0..i
  for p in 0..i:
    output x^(i-p) * y^p
```

```python
def feature_mapping(x, y, power, as_ndarray=False):
#     """return mapped features as ndarray or dataframe"""
    # data = {}
    # # inclusive
    # for i in np.arange(power + 1):
    #     for p in np.arange(i + 1):
    #         data["f{}{}".format(i - p, p)] = np.power(x, i - p) * np.power(y, p)

    data = {"f{}{}".format(i - p, p): np.power(x, i - p) * np.power(y, p)
                for i in np.arange(power + 1)
                for p in np.arange(i + 1)
            }

    if as_ndarray:
        return pd.DataFrame(data).as_matrix()
    else:
        return pd.DataFrame(data)

```

```python
x1 = np.array(df.test1)
x2 = np.array(df.test2)
```

```python
data = feature_mapping(x1, x2, power=6)
print(data.shape)
data.head()
```

    (118, 28)

      f00
      f01
      f02
      f03
      f04
      f05
      f06
      f10
      f11
      f12
      ...
      f30
      f31
      f32
      f33
      f40
      f41
      f42
      f50
      f51
      f60

      0
      1.0
      0.69956
      0.489384
      0.342354
      0.239497
      0.167542
      0.117206
      0.051267
      0.035864
      0.025089
      ...
      0.000135
      0.000094
      0.000066
      0.000046
      0.000007
      0.000005
      0.000003
      3.541519e-07
      2.477505e-07
      1.815630e-08

      1
      1.0
      0.68494
      0.469143
      0.321335
      0.220095
      0.150752
      0.103256
      -0.092742
      -0.063523
      -0.043509
      ...
      -0.000798
      -0.000546
      -0.000374
      -0.000256
      0.000074
      0.000051
      0.000035
      -6.860919e-06
      -4.699318e-06
      6.362953e-07

      2
      1.0
      0.69225
      0.479210
      0.331733
      0.229642
      0.158970
      0.110047
      -0.213710
      -0.147941
      -0.102412
      ...
      -0.009761
      -0.006757
      -0.004677
      -0.003238
      0.002086
      0.001444
      0.001000
      -4.457837e-04
      -3.085938e-04
      9.526844e-05

      3
      1.0
      0.50219
      0.252195
      0.126650
      0.063602
      0.031940
      0.016040
      -0.375000
      -0.188321
      -0.094573
      ...
      -0.052734
      -0.026483
      -0.013299
      -0.006679
      0.019775
      0.009931
      0.004987
      -7.415771e-03
      -3.724126e-03
      2.780914e-03

      4
      1.0
      0.46564
      0.216821
      0.100960
      0.047011
      0.021890
      0.010193
      -0.513250
      -0.238990
      -0.111283
      ...
      -0.135203
      -0.062956
      -0.029315
      -0.013650
      0.069393
      0.032312
      0.015046
      -3.561597e-02
      -1.658422e-02
      1.827990e-02

5 rows × 28 columns

```python
data.describe()
```

      f00
      f01
      f02
      f03
      f04
      f05
      f06
      f10
      f11
      f12
      ...
      f30
      f31
      f32
      f33
      f40
      f41
      f42
      f50
      f51
      f60

      count
      118.0
      118.000000
      118.000000
      118.000000
      1.180000e+02
      118.000000
      1.180000e+02
      118.000000
      118.000000
      118.000000
      ...
      1.180000e+02
      118.000000
      1.180000e+02
      118.000000
      1.180000e+02
      118.000000
      1.180000e+02
      1.180000e+02
      118.000000
      1.180000e+02

      mean
      1.0
      0.183102
      0.301370
      0.142350
      1.710985e-01
      0.115710
      1.257256e-01
      0.054779
      -0.025472
      0.015483
      ...
      5.983333e-02
      -0.005251
      9.432094e-03
      -0.001705
      1.225384e-01
      0.011812
      1.893340e-02
      5.196507e-02
      -0.000703
      7.837118e-02

      std
      0.0
      0.519743
      0.284536
      0.326134
      2.815658e-01
      0.299092
      2.964416e-01
      0.496654
      0.224075
      0.150143
      ...
      2.746459e-01
      0.096738
      5.455787e-02
      0.037443
      2.092709e-01
      0.072274
      3.430092e-02
      2.148098e-01
      0.058271
      1.938621e-01

      min
      1.0
      -0.769740
      0.000026
      -0.456071
      6.855856e-10
      -0.270222
      1.795116e-14
      -0.830070
      -0.484096
      -0.483743
      ...
      -5.719317e-01
      -0.296854
      -1.592528e-01
      -0.113448
      1.612020e-09
      -0.246068
      2.577297e-10
      -3.940702e-01
      -0.203971
      6.472253e-14

      25%
      1.0
      -0.254385
      0.061086
      -0.016492
      3.741593e-03
      -0.001072
      2.298277e-04
      -0.372120
      -0.178209
      -0.042980
      ...
      -5.155632e-02
      -0.029360
      -3.659760e-03
      -0.005749
      1.869975e-03
      -0.001926
      1.258285e-04
      -7.147973e-03
      -0.006381
      8.086369e-05

      50%
      1.0
      0.213455
      0.252195
      0.009734
      6.360222e-02
      0.000444
      1.604015e-02
      -0.006336
      -0.016521
      -0.000039
      ...
      -2.544062e-07
      -0.000512
      -1.473547e-07
      -0.000005
      2.736163e-02
      0.000205
      3.387050e-03
      -1.021440e-11
      -0.000004
      4.527344e-03

      75%
      1.0
      0.646562
      0.464189
      0.270310
      2.155453e-01
      0.113020
      1.001215e-01
      0.478970
      0.100795
      0.079510
      ...
      1.099616e-01
      0.015050
      1.370560e-02
      0.001024
      1.520801e-01
      0.019183
      2.090875e-02
      2.526861e-02
      0.002104
      5.932959e-02

      max
      1.0
      1.108900
      1.229659
      1.363569
      1.512062e+00
      1.676725
      1.859321e+00
      1.070900
      0.568307
      0.505577
      ...
      1.228137e+00
      0.369805
      2.451845e-01
      0.183548
      1.315212e+00
      0.304409
      2.018260e-01
      1.408460e+00
      0.250577
      1.508320e+00

8 rows × 28 columns

# regularized cost

$$J\left( \theta  \right)=\frac{1}{m}\sum\limits_{i=1}^{m}{[-{{y}^{(i)}}\log \left( {{h}_{\theta }}\left( {{x}^{(i)}} \right) \right)-\left( 1-{{y}^{(i)}} \right)\log \left( 1-{{h}_{\theta }}\left( {{x}^{(i)}} \right) \right)]}+\frac{\lambda }{2m}\sum\limits_{j=1}^{n}{\theta _{j}^{2}}$$

```python
theta = np.zeros(data.shape[1])
X = feature_mapping(x1, x2, power=6, as_ndarray=True)
print(X.shape)

y = get_y(df)
print(y.shape)
```

    (118, 28)
    (118,)

```python
def regularized_cost(theta, X, y, l=1):
#     '''you don't penalize theta_0'''
    theta_j1_to_n = theta[1:]
    regularized_term = (l / (2 * len(X))) * np.power(theta_j1_to_n, 2).sum()

    return cost(theta, X, y) + regularized_term
#正则化代价函数
```

```python
regularized_cost(theta, X, y, l=1)
```

    0.6931471805599454

this is the same as the not regularized cost because we init theta as zeros...

# regularized gradient
$$\frac{\partial J\left( \theta  \right)}{\partial {{\theta }_{j}}}=\left( \frac{1}{m}\sum\limits_{i=1}^{m}{\left( {{h}_{\theta }}\left( {{x}^{\left( i \right)}} \right)-{{y}^{\left( i \right)}} \right)} \right)+\frac{\lambda }{m}{{\theta }_{j}}\text{ }\text{             for  j}\ge \text{1}$$

```python
def regularized_gradient(theta, X, y, l=1):
#     '''still, leave theta_0 alone'''
    theta_j1_to_n = theta[1:]
    regularized_theta = (l / len(X)) * theta_j1_to_n

    # by doing this, no offset is on theta_0
    regularized_term = np.concatenate([np.array([0]), regularized_theta])

    return gradient(theta, X, y) + regularized_term
```

```python
regularized_gradient(theta, X, y)
```

    array([  8.47457627e-03,   7.77711864e-05,   3.76648474e-02,
             2.34764889e-02,   3.93028171e-02,   3.10079849e-02,
             3.87936363e-02,   1.87880932e-02,   1.15013308e-02,
             8.19244468e-03,   3.09593720e-03,   4.47629067e-03,
             1.37646175e-03,   5.03446395e-02,   7.32393391e-03,
             1.28600503e-02,   5.83822078e-03,   7.26504316e-03,
             1.83559872e-02,   2.23923907e-03,   3.38643902e-03,
             4.08503006e-04,   3.93486234e-02,   4.32983232e-03,
             6.31570797e-03,   1.99707467e-02,   1.09740238e-03,
             3.10312442e-02])

```python
import scipy.optimize as opt
```

```python
print('init cost = {}'.format(regularized_cost(theta, X, y)))

res = opt.minimize(fun=regularized_cost, x0=theta, args=(X, y), method='Newton-CG', jac=regularized_gradient)
res
```

    init cost = 0.6931471805599454

         fun: 0.5290027297127737
         jac: array([  1.05541650e-07,   6.13738958e-08,  -7.04772465e-08,
            -1.36777274e-08,  -2.84652117e-08,  -3.41659162e-08,
            -5.15630062e-08,  -1.74645557e-08,   7.50726574e-09,
             1.18564041e-08,   1.78853446e-08,   1.12436692e-08,
             8.73092759e-09,   5.56139873e-08,   4.12686771e-09,
            -2.49410770e-08,  -6.34978298e-09,  -1.34271390e-08,
            -2.13487848e-09,   4.54710087e-10,  -4.37439525e-09,
            -1.26745782e-09,  -3.40985521e-09,   7.34158338e-09,
            -7.74561697e-09,  -9.84723852e-11,   9.65250750e-10,
            -1.18501368e-08])
     message: 'Optimization terminated successfully.'
        nfev: 7
        nhev: 0
         nit: 6
        njev: 66
      status: 0
     success: True
           x: array([ 1.27273981,  1.18108974, -1.43166669, -0.17513036, -1.19281478,
           -0.45635758, -0.9246528 ,  0.62527237, -0.9174247 , -0.35723884,
           -0.27470605, -0.29537769, -0.14388711, -2.0199599 , -0.36553508,
           -0.61555685, -0.27778507, -0.32738029,  0.12400668, -0.05098942,
           -0.04473108,  0.01556645, -1.45815829, -0.20600596, -0.29243192,
           -0.24218804,  0.02777165, -1.04320421])

```python
final_theta = res.x
y_pred = predict(X, final_theta)

print(classification_report(y, y_pred))
```

                 precision    recall  f1-score   support

              0       0.90      0.75      0.82        60
              1       0.78      0.91      0.84        58

    avg / total       0.84      0.83      0.83       118

#  $\lambda$
#
*  $X\times \theta = 0$
* instead of solving polynomial equation, just create a coridate x,y grid that is dense enough, and find all those $X\times \theta$ that is close enough to 0, then plot them

```python
def draw_boundary(power, l):
#     """
#     power: polynomial power for mapped feature
#     l: lambda constant
#     """
    density = 1000
    threshhold = 2 * 10**-3

    final_theta = feature_mapped_logistic_regression(power, l)
    x, y = find_decision_boundary(density, power, final_theta, threshhold)

    df = pd.read_csv('ex2data2.txt', names=['test1', 'test2', 'accepted'])
    sns.lmplot('test1', 'test2', hue='accepted', data=df, size=6, fit_reg=False, scatter_kws={"s": 100})

    plt.scatter(x, y, c='R', s=10)
    plt.title('Decision boundary')
    plt.show()
```

```python
def feature_mapped_logistic_regression(power, l):
#     """for drawing purpose only.. not a well generealize logistic regression
#     power: int
#         raise x1, x2 to polynomial power
#     l: int
#         lambda constant for regularization term
#     """
    df = pd.read_csv('ex2data2.txt', names=['test1', 'test2', 'accepted'])
    x1 = np.array(df.test1)
    x2 = np.array(df.test2)
    y = get_y(df)

    X = feature_mapping(x1, x2, power, as_ndarray=True)
    theta = np.zeros(X.shape[1])

    res = opt.minimize(fun=regularized_cost,
                       x0=theta,
                       args=(X, y, l),
                       method='TNC',
                       jac=regularized_gradient)
    final_theta = res.x

    return final_theta
```

```python
def find_decision_boundary(density, power, theta, threshhold):
    t1 = np.linspace(-1, 1.5, density)
    t2 = np.linspace(-1, 1.5, density)

    cordinates = [(x, y) for x in t1 for y in t2]
    x_cord, y_cord = zip(*cordinates)
    mapped_cord = feature_mapping(x_cord, y_cord, power)  # this is a dataframe

    inner_product = mapped_cord.as_matrix() @ theta

    decision = mapped_cord[np.abs(inner_product) < threshhold]

    return decision.f10, decision.f01
```

```python
draw_boundary(power=6, l=1)#lambda=1
```

```python
draw_boundary(power=6, l=0)  # no regularization, over fitting，#lambda=0
```

```python
draw_boundary(power=6, l=100)  # underfitting，#lambda=100
```

```python

```


This is my attempt to recreate the model from [chredlin_model_truth.ipynb](chredlin_model_truth.ipynb).

```python
import numpy as np
import pandas as pd

np.set_printoptions(suppress=True)
```

# Ordinary Least Squares

Suppose our model is that $Y \sim \mathcal{N}\left(X^\intercal \beta, \sigma^2\right)$.

## Parameter Estimates

We want to find $\beta$ such that $L\left(\beta\right) = \left\lVert X\beta - y \right\rVert_2^2$ is minimized. That is, we want to find the projection of $y$ onto to the hyperplane spanned by the columns of $X$. Thus, we must have that $X^\intercal\left(y - X\hat{\beta}\right) = 0$ since the residuals will orthogonal to the columns of $X$ if $X\hat{\beta}$ is the projection that minimizes the squared error. Solving for $\hat{\beta}$, we have that

\begin{equation}
\hat{\beta} = \left(X^\intercal X\right)^{-1} X^\intercal y.
\end{equation}

## Residual Standard Error

Let us derive an unbiased estimator for residual standard error. Consider the residual random vector.

\begin{equation}
R = y - X\hat{\beta}
\end{equation}

As stated earlier, the residuals are orthogonal to hyperplane spanned by the columns of $X$, so they must lie in some orthonormal hyperplane of $N - p$ vectors, where $p = \dim(\beta)$. Thus, the residuals are $y$ projected down to this space.

Let $w_1,\ldots,w_{n-p}$ be an orthonormal basis of this space. Let $W$ be matrix with these basis vectors as the columns.

We have that

\begin{align}
R &= y - X\hat{\beta} \\
&= W\left(W^\intercal y\right) \\
&= W\left(W^\intercal\left(X\beta + \sigma\epsilon\right)\right) \\
&= W\left(W^\intercal X\right)\beta + \sigma W\left(W^\intercal\epsilon\right) \\
&= \sigma W\left(W^\intercal\epsilon\right).
\end{align}

Now, $W^\intercal\epsilon \sim \mathcal{N}\left(0, I_{n-p}\right)$. To see this, note that the $i$th entry is $\sum_{j=1}^n w_{ij}\epsilon_j \sim \mathcal{N}\left(0, 1\right)$, and for $i \neq i^\prime$,

\begin{align}
\operatorname{Cov}\left(\left(W^\intercal\epsilon\right)_i, \left(W^\intercal\epsilon\right)_{i^\prime}\right) &=
\mathbb{E}\left[
\left(\sum_{j=1}w_{ij}\epsilon_j\right)\left(\sum_{k=1}w_{i^\prime k}\epsilon_k\right)
\right] \\
&= \sum_{j=1}^n\mathbb{E}\left[w_{ij}w_{i^\prime j} \epsilon_j^2\right] +
2\sum_{j=1}^{n-1}\sum_{k=j+1}^n \mathbb{E}\left[w_{ij}w_{i^\prime k} \epsilon_j\epsilon_k\right] \\
&= w_i^\intercal w_{i^\prime} + 2\sum_{j=1}^{n-1}\sum_{k=j+1}^n w_{ij}w_{i^\prime k} \mathbb{E}\left[\epsilon_j\epsilon_k\right] \\
&= 0,
\end{align}
where the first term disappears by since the two vectors are orthonormal, and the second term disappears because of independence of the errors.

Thus, we have that

\begin{equation}
R^\intercal R = \sigma^2 \left(W^\intercal\epsilon\right)^\intercal W^\intercal W \left(W^\intercal\epsilon\right) =\sigma^2 \left(W^\intercal\epsilon\right)^\intercal\left(W^\intercal\epsilon\right)
\sim \sigma^2 \chi^2_{n-p}.
\end{equation}

Finally, we have that

\begin{equation}
\mathbb{E}\left[R^\intercal R\right] = \sigma^2\left(n - p\right)
\Rightarrow
\mathbb{E}\left[\frac{\sum_{i=1}^n \left(y - X\hat{\beta}\right)^2}{n-p}\right] = \sigma^2.
\end{equation}

Our consistent estimator is

\begin{equation}
\boxed{\hat{\sigma}^2 = \frac{\sum_{i=1}^n \left(y - X\hat{\beta}\right)^2}{n-p}.}
\end{equation}

## Hypothesis Testing

We can rewrite $y$ as $y = X\beta + \sigma \epsilon$, where each element of $\epsilon$ is drawn from $\mathcal{N}\left(0, 1\right)$. Substituting, we have that

\begin{align}
\hat{\beta} &= \left(X^\intercal X\right)^{-1}X^\intercal\left(X\beta + \sigma\epsilon\right) \\
&= \beta + \sigma\left(X^\intercal X\right)^{-1}X^\intercal \epsilon.
\end{align}

Thus, $\hat{\beta}_j \sim \mathcal{N}\left(\beta_j, \sigma^2\left(X^\intercal X\right)^{-1}_{jj}\right)$.

This gives us that

\begin{equation}
\frac{\hat{\beta}_j - \beta_j}{\sqrt{\sigma^2\left(X^\intercal X\right)^{-1}_{jj}}} \sim
\mathcal{N}\left(0, 1\right).
\end{equation}

From the previous part,

\begin{equation}
(n - p)\frac{\hat{\sigma}^2}{\sigma^2} \sim \chi^2_{n-p}.
\end{equation}

$\hat{\beta}$ and $\hat{\sigma}^2$ are independent by Basu's theorem: $\hat{\sigma}^2$ is an ancillary statistic that does not depend on the model parameters, $\beta$. Thus, we have that

\begin{equation}
\left.
\frac{\hat{\beta}_j - \beta_j}{\sqrt{\sigma^2\left(X^\intercal X\right)^{-1}_{jj}}}
\middle/
\sqrt{\frac{(n - p)\frac{\hat{\sigma}^2}{\sigma^2}}{n-p}}
\right.
= \frac{\hat{\beta}_j - \beta_j}{\sqrt{\hat{\sigma}^2\left(X^\intercal X\right)^{-1}_{jj}}}
\sim t_{n-p}.
\end{equation}

That is, we have $t$ distribution with $n - p$ degrees of freedom.

## Estimating the Model Parameters

The code for `LinearRegression` can be found in [linear_regression.py](https://github.com/ppham27/stat570/tree/master/stat570/linear_model/linear_regression.py).

```python
from chredlin import load_data
from stat570.linear_model.linear_regression import LinearRegression

chredlin = load_data()

COVARIATES = ['race', 'fire', 'theft', 'age', 'log_income']
RESPONSE = 'involact'

linear_model = LinearRegression.from_data_frame(chredlin, COVARIATES, RESPONSE)

with open('p1_residual_standard_error.txt', 'w') as f:
    f.write(str(np.sqrt(linear_model.residual_variance_)))

with open('p1_model_parameters.tex', 'w') as f:
    f.write(linear_model.coefficients_.to_latex())

print('Residual standard error is {}.'.format(np.sqrt(linear_model.residual_variance_)))
linear_model.coefficients_
```

    Residual standard error is 0.3345267301243203.

      estimate
      std_error
      t-statistic
      p-value

      (intercept)
      -1.185540
      1.100255
      -1.077514
      0.287550

      race
      0.009502
      0.002490
      3.816831
      0.000449

      fire
      0.039856
      0.008766
      4.546588
      0.000048

      theft
      -0.010295
      0.002818
      -3.653264
      0.000728

      age
      0.008336
      0.002744
      3.037749
      0.004134

      log_income
      0.345762
      0.400123
      0.864137
      0.392540

## Response Confidence Interval

Let us $0$-index the columns of $X$ so that the $0$th column is all $1$s. Let us $0$-index $\beta$ so the $0$th entry is the intercept. Since $\hat{\beta}$ satisfies $\left(X^\intercal X\right)\hat{\beta} = X^\intercal y$, we have that the intercept estimate is

\begin{equation}
\hat{\beta}_0 = \bar{y} - \sum_{j=1}^p \hat{\beta}_j \bar{X}_{:,j}.
\end{equation}

Consider trying to predict $\hat{y} = x^\intercal\hat{\beta}$ for some $x$. We have that

\begin{equation}
\hat{y} = \bar{y} + \sum_{j=1}^p \left(x_i - \bar{X}_{:,j}\right)\hat{\beta}_j,
\end{equation}

so the variance of the prediction increases with values far from data.

Let $\bar{X}$ be the vector of column-wise means of $X$. Since $\bar{\epsilon}$ is an ancillary statistic, this can also be written as

\begin{equation}
\hat{y} \mid x \sim \mathcal{N}\left(
x^\intercal \beta, \sigma^2 \left(\frac{1}{n} + \left(x - \bar{X}\right)^\intercal\left(X^\intercal X\right)^{-1}\left(x - \bar{X}\right)\right)
\right).
\end{equation}

Following the same method as before, if we replace $\beta$ with $\hat{\beta}$ and $\sigma^2$ with $\hat{\sigma}^2$, we have

\begin{equation}
\frac{\hat{y} - x^\intercal\hat{\beta}}{
\sqrt{\hat{\sigma}^2\left(\frac{1}{n} + \left(x - \bar{X}\right)^\intercal\left(X^\intercal X\right)^{-1}\left(x - \bar{X}\right)\right)}
}
\sim t_{n-p}.
\end{equation}

## Modified Model

`log_income` doesn't add much value to the model. From [chredlin_explore.ipynb](chredlin_explore.ipynb), we see this is because income correlates so strongly with `race` and `fire`. Let's remove it.

```python
linear_model_custom = LinearRegression.from_data_frame(
    chredlin,
    [covariate for covariate in COVARIATES if covariate != 'log_income'],
    RESPONSE)

with open('p1_residual_standard_error_custom.txt', 'w') as f:
    f.write(str(np.sqrt(linear_model_custom.residual_variance_)))

with open('p1_model_parameters_custom.tex', 'w') as f:
    f.write(linear_model_custom.coefficients_.to_latex())

print('Residual standard error is {}.'.format(np.sqrt(linear_model_custom.residual_variance_)))
linear_model_custom.coefficients_
```

    Residual standard error is 0.3335165792871865.

      estimate
      std_error
      t-statistic
      p-value

      (intercept)
      -0.243118
      0.145054
      -1.676054
      0.101158

      race
      0.008104
      0.001886
      4.296913
      0.000100

      fire
      0.036646
      0.007916
      4.629173
      0.000035

      theft
      -0.009592
      0.002690
      -3.565847
      0.000921

      age
      0.007210
      0.002408
      2.994369
      0.004595


```python
import sympy as sy
import scipy as sp
import numpy as np
import random as rd
import plotly as py
from ipywidgets import interact
from matplotlib import pyplot
from stat_dist import *
```

##### Gaussian distribution is:

$$ f(x; \mu, \sigma) = \frac{1}{\sqrt{2 \pi \sigma^2}}  e^{-\frac{(x-\mu)^2}{2\sigma}}$$

where $\mu$ is mean value ($1^{st}$ moment) and $\sigma$ is standard deviation (square rot of $2^{nd}$ moment).

```python
N = int(input())
l, r = map(float, input().split())
```

```python
def g(mu, sigma):
    x = gauss(mu-3*sigma, mu+3*sigma, mu, sigma, N)
    stat(x, mu-3*sigma, mu+3*sigma, 0.001, N)

interact(g, mu = (-10, 10, 0.01), sigma = (0.01, 10, 0.01))
```

#### Uniform distribution on set $\Omega$ is:

$$p(x; \Omega) = \frac{1}{\mu(\Omega)}$$

where $\mu(\Omega) = \int_{\Omega} d\mu$ is Lebesgue measure of a set $\Omega$.

For example:
1. $\Omega = [a,b]$: $$p(x) = \frac{1}{b-a}$$,
2. $\Omega = \{a_1,a_2,...,a_N\}$: $$p(a_i) = \frac{1}{N}$$

and so on...

```python
N = int(input())
l, r = map(float, input().split())
x = uniform(l, r, N)
stat(x, l, r, 0.001, N)
```

#### Gamma distribution is:

$$f(x; \alpha, \beta) = \frac{\beta^{\alpha} x^{\alpha-1}e^{-\beta x}}{\Gamma(\alpha)}$$

```python
N = int(input())
h = float(input())
```

```python
def gam(alfa, beta):
    x = gamma(h, alfa, beta, N)
    stat(x, 0, h, 0.001, N)

interact(gam, alfa = (0.01, 10, 0.01), beta = (0.01, 10, 0.01))
```

stat_dist.py contains some more probability distribution functions, whose graphs can be constructed in the same maneer.

```python

```


## Run with your first code

```python
from sympy import init_printing; init_printing()

from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all" # display all expression in one cell instead of the last one
```

#### Wave Equation
The tensor coefficient matrix in electromagnetic wave equation

$$\mathit{k}\times(\mathit{k}\times \tilde{\mathit{E}}) + k_0^2 \mathbf{\kappa}\cdot \tilde{\mathit{E}}=\mathit{0}$$

$$
\left(\begin{bmatrix}
-k_z^2 &  & k_x k_z \\
 & -k_x^2-k_z^2 & \\
 k_x k_z &  & -k_x^2
\end{bmatrix} + k_0^2
\begin{bmatrix}
\kappa_\perp & -j \kappa_\times & \\
j\kappa_\times & \kappa_\perp  & \\
 &  & \kappa_\parallel
\end{bmatrix}\right)\cdot
\begin{bmatrix}
\tilde{E}_x\\
\tilde{E}_y\\
\tilde{E}_z
\end{bmatrix} =
\begin{bmatrix}
0\\
0\\
0
\end{bmatrix}$$

$$
\mathbf{T}\cdot
\begin{bmatrix}
\tilde{E}_x\\
\tilde{E}_y\\
\tilde{E}_z
\end{bmatrix} =
\begin{bmatrix}
0\\
0\\
0
\end{bmatrix}
$$

You can acquire the `sympy.Array` representation of the above (tensor) matrix coefficients by the `WaveEq` class' coeff_matrix() method. For the relevant variable, you can get them by attribute *e.g.*, `wave_eq.wave.k[1]` means the wave's $k_y$ component of $\vec{k}$ vector.

```python
from sinupy import mediums, waves
from sinupy.waves import EM
plasma = mediums.ColdMagnetizedPlasma()     # to specify a kind of metarial
wave_eq = waves.EM.WaveEq(plasma)           # wave eqution
wave = wave_eq.wave
wave_eq.coeff_matrix()                      # get the k parameter of cross part
wave_eq
```

$\displaystyle \left[\begin{matrix}- k_{y }^{2} - k_{z }^{2} + \frac{\kappa_{\perp} \omega_{}^{2}}{c^{2}} & - \frac{i \kappa_{\times} \omega_{}^{2}}{c^{2}} + k_{x } k_{y } & k_{x } k_{z }\\\frac{i \kappa_{\times} \omega_{}^{2}}{c^{2}} + k_{x } k_{y } & - k_{x }^{2} - k_{z }^{2} + \frac{\kappa_{\perp} \omega_{}^{2}}{c^{2}} & k_{y } k_{z }\\k_{x } k_{z } & k_{y } k_{z } & - k_{x }^{2} - k_{y }^{2} + \frac{\kappa_{\parallel} \omega_{}^{2}}{c^{2}}\end{matrix}\right]$

$\displaystyle \left[\begin{matrix}k_{y } \left(- E_{x } k_{y } + E_{y } k_{x }\right) - k_{z } \left(E_{x } k_{z } - E_{z } k_{x }\right) & - k_{x } \left(- E_{x } k_{y } + E_{y } k_{x }\right) + k_{z } \left(- E_{y } k_{z } + E_{z } k_{y }\right) & k_{x } \left(E_{x } k_{z } - E_{z } k_{x }\right) - k_{y } \left(- E_{y } k_{z } + E_{z } k_{y }\right)\end{matrix}\right] = \left[\begin{matrix}- \frac{\omega_{}^{2} \left(E_{x } \kappa_{\perp} - i E_{y } \kappa_{\times}\right)}{c^{2}} & - \frac{\omega_{}^{2} \left(i E_{x } \kappa_{\times} + E_{y } \kappa_{\perp}\right)}{c^{2}} & - \frac{E_{z } \kappa_{\parallel} \omega_{}^{2}}{c^{2}}\end{matrix}\right]$

The function `sinupy.waves.EM.solve_N2(WaveEq)` returns the solutions of $N^2(\omega, \theta)$, which is usually tedious for hand-deduction.

```python
EM.solve_N2(wave_eq)        # solve the eqution
```

docs/source/Run with your first code.ipynb

docs/build/html/_sources/Run with your first code.ipynb.txt


# Shells 2D

## Init symbols for *sympy*

```python
from sympy import *
from sympy.vector import CoordSys3D
N = CoordSys3D('N')
x1, x2 = symbols("x_1 x_2")
alpha1, alpha2 = symbols("alpha_1 alpha_2")
R, L, ga, gv = symbols("R L g_a g_v")
init_printing()
```

## Cylindrical coordinates

```python
a1 = pi / 2 + (L / 2 - alpha1)/R

x = R * cos(a1)
y = R * sin(a1)

r = x*N.i + y*N.j
```

#### Curve in 2D coordinates system will be defined with the following vector $\vec{r}=\vec{r(\alpha_1)}$

```python
r
```

#### Tangent to curve

```python
v_temp = r.diff(alpha1)
dr_len = v_temp.magnitude()
v = v_temp / dr_len
v = trigsimp(v)
v_temp
```

```python
trigsimp(dr_len)
```

#### Normal to curve

```python
n_temp = v.diff(alpha1)
k=trigsimp(n_temp.magnitude())
n = n_temp/k
q=1/(R*sqrt(1/R**2))
n = trigsimp(n).subs(q, 1)
n
```

```python
v.dot(n)
```

```python
n.dot(v)
```

#### Curvature

```python
sympify(k)
```

#### Derivative of base vectors
Let's find
$\frac { d\vec{n} } { d\alpha_1}$
$\frac { d\vec{v} } { d\alpha_1}$
$\frac { d\vec{n} } { d\alpha_2}$
$\frac { d\vec{v} } { d\alpha_2}$

```python
n.diff(alpha1)
```

$ \frac { d\vec{n} } { d\alpha_1} = -\frac {1}{R} \vec{v} = -k \vec{v} $

```python
v.diff(alpha1)
```

$ \frac { d\vec{v} } { d\alpha_1} = \frac {1}{R} \vec{n} = k \vec{n} $

#### Derivative of vectors

$ \vec{u} = u_v \vec{v} + u_n\vec{n} $

$ \frac { d\vec{u} } { d\alpha_1} =  \frac { d(u_v\vec{v}) } { d\alpha_1} + \frac { d(u_n\vec{n}) } { d\alpha_1} =
 \frac { du_n } { d\alpha_1} \vec{n} + u_n \frac { d\vec{n} } { d\alpha_1} + \frac { du_v } { d\alpha_1} \vec{v} + u_v \frac { d\vec{v} } { d\alpha_1} = \frac { du_n } { d\alpha_1} \vec{n} - u_n k \vec{v} + \frac { du_v } { d\alpha_1} \vec{v} + u_v k \vec{n}$

Then
$ \frac { d\vec{u} } { d\alpha_1} = \left( \frac { du_v } { d\alpha_1} - u_n k \right) \vec{v} + \left( \frac { du_n } { d\alpha_1} + u_v k \right) \vec{n}$

$ \frac { d\vec{u} } { d\alpha_2} = \frac { d(u_n\vec{n}) } { d\alpha_2} + \frac { d(u_v\vec{v}) } { d\alpha_2} =
 \frac { du_n } { d\alpha_2} \vec{n} + u_n \frac { d\vec{n} } { d\alpha_2} + \frac { du_v } { d\alpha_2} \vec{v} + u_v \frac { d\vec{v} } { d\alpha_2} = \frac { du_n } { d\alpha_2} \vec{n} + \frac { du_v } { d\alpha_2} \vec{v} $

#### Base Vectors $\vec{R}_1, \vec{R}_2$

```python
R_alpha=r+alpha2*n
R_alpha
```

```python
R1=R_alpha.diff(alpha1)
R2=R_alpha.diff(alpha2)
```

```python
trigsimp(R1)
```

```python
R2
```

#### Let's find Jacobi matrix:

$ A = \left(
\begin{array}{cc}
\frac{\partial x_1}{\partial \alpha_1} & \frac{\partial x_1}{\partial \alpha_2} \\
\frac{\partial x_2}{\partial \alpha_1} & \frac{\partial x_2}{\partial \alpha_2}
\end{array}
\right)$

$ \left[
\begin{array}{cc}
\vec{R}_1 & \vec{R}_2
\end{array}
\right] = \left[
\begin{array}{cc}
\vec{e}_1 & \vec{e}_2
\end{array}
\right] \cdot  \left(
\begin{array}{cc}
\frac{\partial x_1}{\partial \alpha_1} & \frac{\partial x_1}{\partial \alpha_2} \\
\frac{\partial x_2}{\partial \alpha_1} & \frac{\partial x_2}{\partial \alpha_2}
\end{array}
\right) = \left[
\begin{array}{cc}
\vec{e}_1 & \vec{e}_2
\end{array}
\right] \cdot A$

$ \left[
\begin{array}{cc}
\vec{e}_1 & \vec{e}_2
\end{array}
\right] = \left[
\begin{array}{cc}
\vec{R}_1 & \vec{R}_2
\end{array}
\right] \cdot A^{-1}$

```python
m11=R1.dot(N.i)
m12=R2.dot(N.i)
m21=R1.dot(N.j)
m22=R2.dot(N.j)
A=Matrix([[m11, m12], [m21, m22]])
A
```

```python
A_inv = trigsimp(A**-1)
sympify(trigsimp(Matrix([R1, R2]).T*A_inv))
```

```python
trigsimp(A.det())
```

#### Metric tensor

```python
g11=R1.dot(R1)
g12=R1.dot(R2)
g21=R2.dot(R1)
g22=R2.dot(R2)

G=Matrix([[g11, g12],[g21, g22]])
G=trigsimp(G)
G
```

```python
G_inv = G**-1
```

#### Derivative of base vectors

```python
dR1dalpha1 = trigsimp(R1.diff(alpha1))
dR1dalpha1
```

$ \frac { d\vec{R_1} } { d\alpha_1} = \frac {1}{R} \left( 1-\frac{\alpha_2}{R} \right) \vec{R_2} $

```python
dR1dalpha2 = trigsimp(R1.diff(alpha2))
dR1dalpha2
```

$ \frac { d\vec{R_1} } { d\alpha_2} = -\frac {1}{R} \frac {1}{1-\frac{\alpha_2}{R}}  \vec{R_1} $

```python
dR2dalpha1 = trigsimp(R2.diff(alpha1))
dR2dalpha1
```

$ \frac { d\vec{R_2} } { d\alpha_1} = -\frac {1}{R} \frac {1}{1-\frac{\alpha_2}{R}}  \vec{R_1} $

```python
dR2dalpha2 = trigsimp(R2.diff(alpha2))
dR2dalpha2
```

$ \frac { d\vec{R_2} } { d\alpha_2} = \vec{0} $

#### Derivative of vectors

$ \vec{u} = u^1 \vec{R_1} + u^2\vec{R_2} $

$ \frac { d\vec{u} } { d\alpha_1} =  \frac { d(u^1\vec{R_1}) } { d\alpha_1} + \frac { d(u^2\vec{R_2}) } { d\alpha_1} =
 \frac { du^1 } { d\alpha_1} \vec{R_1} + u^1 \frac { d\vec{R_1} } { d\alpha_1} + \frac { du^2 } { d\alpha_1} \vec{R_2} + u^2 \frac { d\vec{R_2} } { d\alpha_1} = \frac { du^1 } { d\alpha_1} \vec{R_1} + u^1 \frac {1}{R} \left( 1-\frac{\alpha_2}{R} \right) \vec{R_2} + \frac { du^2 } { d\alpha_1} \vec{R_2} - u^2 \frac {1}{R} \frac {1}{1-\frac{\alpha_2}{R}}  \vec{R_1}$

Then
$ \frac { d\vec{u} } { d\alpha_1} = \left( \frac { du^1 } { d\alpha_1} - u^2 \frac {1}{R} \frac {1}{1-\frac{\alpha_2}{R}}  \right) \vec{R_1} + \left( \frac { du^2 } { d\alpha_1} + u^1 \frac {1}{R} \left( 1-\frac{\alpha_2}{R} \right) \right) \vec{R_2}$

$ \frac { d\vec{u} } { d\alpha_2} = \frac { d(u^1\vec{R_1}) } { d\alpha_2} + \frac { d(u^2\vec{R_2}) } { d\alpha_2} =
 \frac { du^1 } { d\alpha_2} \vec{R_1} + u^1 \frac { d\vec{R_1} } { d\alpha_2} + \frac { du^2 } { d\alpha_2} \vec{R_2} + u^2 \frac { d\vec{R_2} } { d\alpha_2} = \frac { du^1 } { d\alpha_2} \vec{R_1} - u^1 \frac {1}{R} \frac {1}{1-\frac{\alpha_2}{R}}  \vec{R_1} + \frac { du^2 } { d\alpha_2} \vec{R_2} $

 Then
$ \frac { d\vec{u} } { d\alpha_2} = \left( \frac { du^1 } { d\alpha_2}  - u^1 \frac {1}{R} \frac {1}{1-\frac{\alpha_2}{R}} \right) \vec{R_1} + \frac { du^2 } { d\alpha_2} \vec{R_2}$

$\nabla_1 u^1 = \frac { \partial u^1 } { \partial \alpha_1} - u^2 \frac {1}{R} \frac {1}{1-\frac{\alpha_2}{R}}$

$\nabla_1 u^2 = \frac { \partial u^2 } { \partial \alpha_1} + u^1 \frac {1}{R} \left( 1-\frac{\alpha_2}{R} \right) $

$\nabla_2 u^1 = \frac { \partial u^1 } { \partial \alpha_2}  - u^1 \frac {1}{R} \frac {1}{1-\frac{\alpha_2}{R}}$

$\nabla_2 u^2 = \frac { \partial u^2 } { \partial \alpha_2}$

$ \nabla \vec{u} = \left(
\begin{array}{cc}
\nabla_1 u^1 & \nabla_1 u^2 \\
\nabla_2 u^1 & \nabla_2 u^2
\end{array}
\right)$

```python
u1=Function('u^1')
u2=Function('u^2')

u1_nabla1 = u1(alpha1, alpha2).diff(alpha1) - u2(alpha1, alpha2) / R * (S(1)/(1-alpha2/R))
u2_nabla1 = u2(alpha1, alpha2).diff(alpha1) + u1(alpha1, alpha2) / R * ( 1-alpha2/R)
u1_nabla2 = u1(alpha1, alpha2).diff(alpha2) - u1(alpha1, alpha2) / R * (S(1)/(1-alpha2/R))
u2_nabla2 = u2(alpha1, alpha2).diff(alpha2)
# $\nabla_2 u^2 = \frac { \partial u^2 } { \partial \alpha_2}$

grad_u = Matrix([[u1_nabla1, u2_nabla1],[u1_nabla2, u2_nabla2]])
grad_u
```

```python
q=Symbol('q')
grad_u_down=grad_u.subs(1-alpha2/R, q)*G.subs((R-alpha2)/R,q)
#grad_u_down=grad_u*G
expand(simplify(grad_u_down))#.subs((R-alpha2)/R, q)
```

$
\left(
\begin{array}{c}
\nabla_1 u_1 \\ \nabla_2 u_1 \\
\nabla_1 u_2 \\ \nabla_2 u_2
\end{array}
\right)
=
\left(
\begin{array}{c}
\left( 1-\frac{\alpha_2}{R} \right)^2  \frac { \partial u^1 } { \partial \alpha_1} - u^2 \frac {\left( 1-\frac{\alpha_2}{R} \right)}{R} \\
\left( 1-\frac{\alpha_2}{R} \right)^2  \frac { \partial u^1 } { \partial \alpha_2} - u^1 \frac {\left( 1-\frac{\alpha_2}{R} \right)}{R} \\
\frac { \partial u^2 } { \partial \alpha_1} + u^1 \frac {\left( 1-\frac{\alpha_2}{R} \right)}{R} \\
\frac { \partial u^2 } { \partial \alpha_2}
\end{array}
\right)
$

$
\left(
\begin{array}{c}
\nabla_1 u_1 \\ \nabla_2 u_1 \\
\nabla_1 u_2 \\ \nabla_2 u_2
\end{array}
\right)
=
\left(
\begin{array}{cccccc}
0 & \left( 1-\frac{\alpha_2}{R} \right)^2 & 0 & -\frac {\left( 1-\frac{\alpha_2}{R} \right)}{R} & 0 & 0 \\
-\frac {\left( 1-\frac{\alpha_2}{R} \right)}{R} & 0 & \left( 1-\frac{\alpha_2}{R} \right)^2 & 0  & 0 & 0 \\
\frac {\left( 1-\frac{\alpha_2}{R} \right)}{R} & 0 & 0 & 0 & 1 & 0  \\
0 & 0 & 0 & 0 & 0 & 1  \\
\end{array}
\right)
\left(
\begin{array}{c}
u^1 \\
\frac { \partial u^1 } { \partial \alpha_1} \\
\frac { \partial u^1 } { \partial \alpha_2} \\
u^2 \\
\frac { \partial u^2 } { \partial \alpha_1} \\
\frac { \partial u^2 } { \partial \alpha_2} \\
\end{array}
\right)
$

### Elasticity tensor(stiffness tensor)

```python
from sympy import MutableDenseNDimArray
C_x = MutableDenseNDimArray.zeros(3, 3, 3, 3)

for i in range(3):
    for j in range(3):
        for k in range(3):
            for l in range(3):
                elem_index = 'C^{{{}{}{}{}}}'.format(i+1, j+1, k+1, l+1)
                el = Symbol(elem_index)
                C_x[i,j,k,l] = el

C_x

```

```python
C_x_1 = MutableDenseNDimArray.zeros(3, 3, 3, 3)

def getCIndecies(index):
    if (index == 0):
        return 0, 0
    elif (index == 1):
        return 1, 1
    elif (index == 2):
        return 2, 2
    elif (index == 3):
        return 0, 1
    elif (index == 4):
        return 0, 2
    elif (index == 5):
        return 1, 2

for i in range(3):
    for j in range(i, 3):
        for k in range(3):
            for l in range(k, 3):
                elem_index = 'C^{{{}{}{}{}}}'.format(i+1, j+1, k+1, l+1)
                el = Symbol(elem_index)
                C_x_1[i,j,k,l] = el
                C_x_1[i,j,l,k] = el
                C_x_1[j,i,k,l] = el
                C_x_1[j,i,l,k] = el
                if (i >= k or j >= l):
                    C_x_1[k,l,i,j] = el
                    C_x_1[k,l,j,i] = el
                    C_x_1[l,k,i,j] = el
                    C_x_1[l,k,j,i] = el

C_x_1
```

```python
# for i in range(3):
#     for j in range(i, 3):
#         for k in range(3):
#             for l in range(k, 3):
#                 elem_index = 'C^{{{}{}{}{}}}'.format(i+1, j+1, k+1, l+1)
#                 el = Symbol(elem_index)
#                 C_x[i,j,k,l] = el
#                 C_x[i,j,l,k] = el
#                 C_x[j,i,k,l] = el
#                 C_x[j,i,l,k] = el

# C_x

# for i in range(3):
#     for j in range(3):
#         for k in range(3):
#             for l in range(3):
#                 elem_index = 'C^{{{}{}{}{}}}'.format(i+1, j+1, k+1, l+1)
#                 el = Symbol(elem_index)
#                 C_x[i,j,k,l] = el

#                 C_x[k,l,i,j] = el
#         if (s < 3 and t < 3):

#         elif (s==t):
#             i,j = getCIndecies(s)
#             k,l = getCIndecies(t)
#             elem_index = 'C^{{{}{}{}{}}}'.format(i+1, j+1, k+1, l+1)
#             el = Symbol(elem_index)
#             C_x[i,j,k,l] = el

# def getCIndecies(index):
#     if (index == 0):
#         return 0, 0
#     elif (index == 1):
#         return 1, 1
#     elif (index == 2):
#         return 2, 2
#     elif (index == 3):
#         return 0, 1
#     elif (index == 4):
#         return 0, 2
#     elif (index == 5):
#         return 1, 2

# def getCalpha(C, A, q, p, s, t):
#     res = S(0)
#     for i in range(3):
#         for j in range(3):
#             for k in range(3):
#                 for l in range(3):
#                     res += C[i,j,k,l]*A[q,i]
#     return simplify(trigsimp(res))

# C_alpha = MutableDenseNDimArray.zeros(3, 3, 3, 3)
# C_alpha_empty = MutableDenseNDimArray.zeros(3, 3, 3, 3)

# m11=R1.dot(N.i)
# m12=R2.dot(N.i)
# m21=R1.dot(N.j)
# m22=R2.dot(N.j)
# A=Matrix([[m11, m12, 0], [m21, m22, 0], [0,0,1]])
# A_inv=A**-1

# for s in range(6):
#     for t in range(s, 6):
#         if (s < 3 and t < 3):
#             i,j = getCIndecies(s)
#             k,l = getCIndecies(t)
#             elem_index = 'C^{}{}{}{}'.format(i+1, j+1, k+1, l+1)
#             el = Symbol(elem_index)
#             C_x[i,j,k,l] = el
#             C_x[k,l,i,j] = el
#         elif (s==t):
#             i,j = getCIndecies(s)
#             k,l = getCIndecies(t)
#             elem_index = 'C^{}{}{}{}'.format(i+1, j+1, k+1, l+1)
#             el = Symbol(elem_index)
#             C_x[i,j,k,l] = el

# for i in range(3):
#     for j in range(3):
#         for k in range(3):
#             for l in range(3):
#                 c = getCalpha(C_x, A_inv, i, j, k, l)
#                 C_alpha[i,j,k,l] = c
#                 if (c != 0):
#                     elem_index = 'C_{{\alpha}}^{}{}{}{}'.format(i+1, j+1, k+1, l+1)
#                     el = Symbol(elem_index)
#                     C_alpha_empty[i,j,k,l] = el

# # trigsimp(C_alpha[0,2,0,2])

# C_alpha_empty
```

```python
C_x
```

#### Virtual work

```python
def contraction3DSameRank(A,B):
    res = S(0)
    for i in range(3):
        for j in range(3):
            res += A[i,j]*B[j,i]
    return res

def contraction3D(C,e):
    res = MutableDenseNDimArray.zeros(3, 3)
    for i in range(3):
        for j in range(3):
            res[i,j] = S(0)
            for k in range(3):
                for l in range(3):
                    res[i,j] += C[i,j,k,l]*e[k,l]
    return res

```

```python
e11 = Symbol("e_{11}")
e12 = Symbol("e_{12}")
e22 = Symbol("e_{22}")
e13 = Symbol("e_{13}")
e23 = Symbol("e_{23}")
e33 = Symbol("e_{33}")
# s11 = Symbol("s_{11}")
# s12 = Symbol("s_{12}")
# s22 = Symbol("s_{22}")

e=Matrix([[e11, e12, e13], [e12, e22, e23], [e13, e23, e33]])

s=contraction3D(C_alpha, e)

E=contraction3DSameRank(s, e)

```

```python
# e_alpha=G*A_inv*e*A_inv.T*G
# #e_alpha=A*e*A.T

# s_alpha=A_inv*s*A_inv.T
# E_alpha=contraction2D(s_alpha, e_alpha)

```


# Linear Algebra and Optimisation

###### COMP4670/8600 - Introduction to Statistical Machine Learning - Tutorial 1B

$\newcommand{\trace}[1]{\operatorname{tr}\left\{#1\right\}}$
$\newcommand{\Norm}[1]{\lVert#1\rVert}$
$\newcommand{\RR}{\mathbb{R}}$
$\newcommand{\inner}[2]{\langle #1, #2 \rangle}$
$\newcommand{\DD}{\mathscr{D}}$
$\newcommand{\grad}[1]{\operatorname{grad}#1}$
$\DeclareMathOperator*{\argmin}{arg\,min}$

Setting up python environment ([do not use pylab](http://carreau.github.io/posts/10-No-PyLab-Thanks.ipynb.html))

```
import matplotlib.pyplot as plt
import numpy as np
import scipy as sp
import scipy.optimize as opt
import time

%matplotlib inline
```

Consider the following cost function $ f(X) $ defined
over the space of real $ n \times p $ matrices
\begin{equation}
  f(X) = \frac{1}{2} \trace{X^T C X N} + \mu \frac{1}{4} \Norm{N - X^T X}^2_F
\end{equation}
where $ X \in \RR^{n \times p} $, $ n \ge p $, and the matrix $ C $ is symmetric,
such that $ C = C^T $. The scalar $ \mu $ is assumed to be larger than the $p$th smallest
eigenvalue of $ C $. The matrix $ N $ is diagonal with distinct positive entries
on the diagonal.
The trace of a square matrix $ A $ is denoted as $ \trace{A} $, and
the Frobenius norm of an arbitrary matrix $ A $ is defined as $ \Norm{A}_F = \sqrt{\trace{A^T A}} $.

## Frobenious Norm

Implement a Python function ```frobenius_norm``` which accepts an arbitrary matrix $ A $ and returns
$ \Norm{A}_F $ using the formula given. (Use ```numpy.trace``` and ```numpy.sqrt```.)
1. Given a matrix $ A \in \RR^{n \times p} $, what is the complexity of your implementation of ```frobenius_norm```
using the formula above?
2. Can you come up with a faster implementation, if you were additionally told that $ p \ge n $ ?
3. Can you find an even faster implementation than in 1. and 2.?

### Solution

Given $A \in \RR^{n \times p} $, $ p \le n $, the straightforward implementation of $ \trace{A^T A} = \trace{A A^T}$ is first multiplying the matrices
and then taking the trace. This is of
complexity $ O(p^2 n) $ (or even $ O(p n^2) $ if you are not careful).
But this trace can be reformulated as
\begin{equation}
  \trace{A^T A} = \sum_{i=1}^p (A^T A)_{i,i}
                = \sum_{i=1}^p \sum_{j=1}^n \underbrace{(A^T)_{i,j}}_{=A_{j, i}} A_{j,i}
                = \sum_{i=1}^p \sum_{j=1}^n A_{j,i}^2
\end{equation}
So we can implement it with complexity $ O(np)$.

```
# Solution
def frobenius_norm(A):
    """Calculate the Frobenius norm of an array or matrix.
    A -- array or matrix
    """
    return np.sqrt((np.asarray(A)**2).sum(axis=None))

M = np.random.rand(5,3)
print(M)
print(frobenius_norm(M))
```

## Cost Function $f(X)$ with matrix argument

Implement the cost function defined as $f(X)$ above as a function ```cost_function_for_matrix```
in Python.

Hint: As good programmers, we do not use global variables in subroutines.

```
# Solution
def cost_function_for_matrix(X, C, N, mu):
    """
    Calculate the cost function at point X given as a matrix.
    X -- current point in matrix form
    C -- symmetric matrix
    N -- diagonal matrix
    mu -- scalar
    returns the value of the cost function as a scalar.
    """
    if not isinstance(X, np.matrix):
        raise TypeError("X is not a matrix")

    if not isinstance(C, np.matrix):
        raise TypeError("C is not a matrix")

    if not isinstance(N, np.matrix):
        raise TypeError("N is not a matrix")

    r1 = 0.5  * np.trace(X.T * C * X * N)
    r2 = 0.25 * mu * frobenius_norm(N - X.T * X)**2
    return r1 + r2

X = np.matrix(np.random.rand(5,3))
C = np.random.rand(5,5)
C = np.matrix(C+C.T)
N = np.matrix(np.diag(np.random.rand(3)))
print(cost_function_for_matrix(X,C,N,np.random.rand()))
```

## Cost Function $f(X)$ with vector argument

Many standard optimisation routines work only with vectors. Fortunately, as vector spaces,
the space of matrices $ \RR^{n \times p} $
and the space of vectors $ \RR^{n p} $ are isomorphic. What does this mean?

Implement the cost function $ f(X) $ given $ X $ as a vector and call it ```cost_function_for_vector```.
Which extra arguments do you need for the cost function?

```
# Solution
def cost_function_for_vector(X, C, N, mu, n, p):
    """Calculate the cost function at point X given as 1-D array
    X  -- current point as 1-D array
    C  -- symmetric matrix
    N  -- diagonal matrix
    mu -- scalar
    n  -- row dimension of X
    p  -- column dimension of X
    returns the value of the cost function as a scalar
    """
    if not isinstance(X, np.ndarray):
        raise TypeError("X is not a matrix")

    if X.ndim != 1:
        raise ValueError("X is not a 1-D vector")

    Xmatrix = np.matrix(X.reshape((n, p)))
    return cost_function_for_matrix(Xmatrix, C, N, mu)

```

## Construction of a random matrix $C$ with given eigenvalues

A diagonal matrix has the nice property that the eigenvalues can be directly read off
the diagonal. Given a diagonal matrix $ C \in \RR^{n \times n} $ with distinct eigenvalues,
how many different diagonal matrices have the same set of eigenvalues?

Given a diagonal matrix $ C \in \RR^{n \times n} $ with distinct eigenvalues,
how many different matrices have the same set of eigenvalues?

Given a set of $ n $ distinct real eigenvalues $ \mathcal{E} = \{e_1, \dots, e_n \} $,
write a Python function \lstinline$random_matrix_from_eigenvalues$ which takes a list of
eigenvalues $ E $ and returns a random symmetric matrix $ C $ having the same eigenvalues.

### Solution

There are $ n! $ permutations of diagonal elements, but infinitely many matrices
with the same set of eigenvalues.

In order to construct a random matrix with given eigenvalues $\lambda_i$, $i=1,\dots,n$
we first create a diagonal matrix $ \Lambda $ with those eigenvalues on the
diagonal. Then we can get another matrix $ A $ with the same eigenvalues as $ \Lambda $
if we apply an arbitrary nonsingular matrix $ B $ to get $ A = B \Lambda B^{-1} $.
(Can you prove that $ A $ and $ \Lambda $ have the same set of eigenvalues?)

If $ B $ is an orthogonal matrix $ Q $, then $ Q^{-1} = Q^T $ and therefore the above
formula results in $ A = Q \Lambda Q^T $ which is much faster to calculate then
using the inverse of a matrix.

How to get a random orthogonal matrix? We use the QR-decomposition of a matrix which
decomposes every arbitrary matrix $ B $ into an orthogonal matrix $ Q $ and an
upper-triangular matrix $ R $, $ B = Q R $.

The algorithm therefore is
1. Choose a random matrix $ B $ by randomly choosing the elements of $ B $.
2. Calculate the QR-decomposition $ B = Q R $. (Check that $ B $ is nonsingular
      by checking the diagonal of $ R $ has nonzero elements.)
3. Calculate $ A =  Q \Lambda Q^T $, the wanted arbitrary matrix with the
      same eigenvalues as $ \Lambda $.

Note: $ A $ and $ \Lambda $ share the same \emph{set} of eigenvalues. The order can
be arbitrary.

```
# Solution
def random_matrix_from_eigenvalues(E):
    """Create a square random matrix with a given set of eigenvalues
    E -- list of eigenvalues
    return the random matrix
    """
    n    = len(E)
    # Create a random orthogonal matrix Q via QR decomposition
    # of a random matrix A
    A    = np.matrix(np.random.rand(n,n))
    Q, R = np.linalg.qr(A)
    #  similarity transformation with orthogonal
    #  matrix leaves eigenvalues intact
    return Q * np.diag(E) * Q.T

```

## Minimising the cost function $f(X)$

Use the minimisation functions ```fmin``` or ```fmin_powell``` provided in the
Python package ```scipy.optimize``` to minimise the cost function ```cost_function_for_vector```.

Hint: Use the argument ```args``` in the minimisation functions  ```fmin``` or ```fmin_powell```
to provide the extra parameters to
your cost function ```cost_function_for_vector```.

```
# Solution
def minimise_f_using_fmin(initialise_proc):
    """Run minimisation with simplex algorithm."""
    C, N, mu, n, p, X0 = initialise_proc()

    X_at_min = opt.fmin(cost_function_for_vector,
                                 X0,
                                 args=(C, N, mu, n, p),
                                 xtol=0.0001,
                                 ftol=0.0001,
                                 maxiter=None,
                                 maxfun=None,
                                 full_output = 0,
                                 disp=1,
                                 retall=0,
                                 callback=None)
    X_at_min = np.matrix(X_at_min.reshape((n, p)))
    show_results(X_at_min, C)

def minimise_f_using_fmin_powell(initialise_proc):
    """Run minimisation with Powell algorithm"""
    C, N, mu, n, p, X0 = initialise_proc()

    X_at_min = opt.fmin_powell(cost_function_for_vector,
                                 X0,
                                 args=(C, N, mu, n, p),
                                 xtol=0.0001,
                                 ftol=0.0001,
                                 maxiter=None,
                                 maxfun=None,
                                 full_output = 0,
                                 disp=1,
                                 retall=0,
                                 callback=None,
                                 direc=None)
    X_at_min = np.matrix(X_at_min.reshape((n, p)))
    show_results(X_at_min, C)

```

## Gradient of $f(X)$

Calculate the gradient for the cost function $f(X)$ given the
inner product on the space of real matrices $ n \times p $ is defined as
\begin{equation}
  \inner{A}{B} = \trace{A^T B}
\end{equation}

Implement a function ```gradient_for_vector``` which takes $ X $ as a vector, and
returns the gradient of $ f(X) $ as a vector.

### Solution

The definition of the directional derivative is in the slides.
A straightforward calculation using the definition gives
the directional derivative of the cost function as

\begin{align*}
  \DD f(X) (\xi) = & \phantom{+} \frac{1}{2} \trace{\xi^T C X N} \\
                   & + \frac{1}{2} \trace{X^T C \xi N} \\
                   & + \frac{1}{4} \mu \trace{(- \xi^T X)(N - X^T X)} \\
                   & + \frac{1}{4} \mu \trace{(- X^T \xi)(N - X^T X)} \\
                   & + \frac{1}{4} \mu \trace{(N - X^T X)(- \xi^T X)} \\
                   & + \frac{1}{4} \mu \trace{(N - X^T X)(- X^T \xi)} .
\end{align*}

Note, the shortcut was to replace each occurrence of the variable $ X $
in the function $ f(X) $ once with $ \xi $ and then add together
all those expressions. Reason: The directional derivative gives the
linear approximation of the infinitesimal change of the function $ f(X) $
at $ X $ in direction $ \xi $. Therefore it must be linear in $ \xi $.

The above expression can be simplified by using that for any matrices
$ A, B $,

\begin{align*}
 \trace{A^T} & = \trace{A}, \\
 \trace{A B} & = \trace{B A}
\end{align*}

From this we get therefore the simplified form
\begin{align*}
  \DD f(X) (\xi) & = \trace{\xi^T C X N} - \mu \trace{\xi^T X(N - X^T X)} \\
                 & = \trace{\xi^T \left(C X N - \mu X(N - X^T X) \right)}
\end{align*}

For the given inner product $ \inner{A}{B} = \trace{A^T B} $, the gradient
will therefore be

\begin{equation}
   \grad f(X) = C X N - \mu X(N - X^T X)
\end{equation}

```
# Solution
def gradient_for_matrix(X, C, N, mu):
    """Calculate the gradient for the cost function at a point X
    X  -- current point in matrix form
    C  -- symmetric matrix
    N  -- diagonal matrix
    mu -- scalar
    returns the gradient of the cost function as matrix
    """
    gradient = C * X * N - mu * X * (N - X.T * X)
    return gradient

def gradient_for_vector(X, C, N, mu, n, p):
    """Calculate the gradient for the cost function at a point X
    X  -- current point as 1-D array
    C  -- symmetric matrix
    N  -- diagonal matrix
    mu -- scalar
    n  -- row dimension of X
    p  -- column dimension of X
    returns the gradient of the cost function as 1-D array
    """
    Xmatrix = np.matrix(X.reshape((n, p)))
    gradient =  gradient_for_matrix(Xmatrix, C, N, mu)
    return np.asarray(gradient).reshape((n*p,))

```

## Minimising the cost function $f(X)$ using the gradient

Use the minimisation functions ```fmin_cg``` or ```fmin_bfgs``` provided in the
Python package ```scipy.optimize``` to minimise the cost function ```cost_function_for_vector``` utilising the gradient ```gradient_for_vector```.

Compare the speed of convergence to the minimisation with ```fmin``` or ```fmin_powell```.

```
# Solution
def normalize_columns(A):
    """Normalise the columns of a 2-D array or matrix to length one
    A - array or matrix (which will be modified)
    """
    if A.ndim != 2:
        raise ValueError("A is not a 2-D array")

    number_of_columns = A.shape[1]
    for i in range(number_of_columns):
        A[:,i] /= np.linalg.norm(A[:,i], ord=2)

def show_results(X_at_min, C):
    """Display the found arg min and compare with eigenvalues of C
    X_at_min -- arguement at minimum found
    C        -- symmetric matrix
    """
    n,p = X_at_min.shape

    normalize_columns(X_at_min)

    # Get the eigenvectors belonging to the smallest eigenvalues
    Eigen_Values, Eigen_Vectors = np.linalg.eig(C)
    Permutation = Eigen_Values.argsort()
    Smallest_Eigenvectors = Eigen_Vectors[:, Permutation[:p]]

    if n < 10:
        print("X_at_min               :\n", X_at_min)
        print()
        print("Smallest_Eigenvectors  :\n", Smallest_Eigenvectors)
        print()
    else:
        Project_into_Eigenvectorspace = \
          Smallest_Eigenvectors * Smallest_Eigenvectors.T * X_at_min
        Normal_Component = X_at_min - Project_into_Eigenvectorspace

        print("norm(Normal_Component)/per entry :", \
            np.linalg.norm(Normal_Component, ord=2) / float(n*p))

def minimise_f_using_fmin_cg(initialise_proc):
    """Run minimisation with conjugate gradient algorithm"""
    C, N, mu, n, p, X0 = initialise_proc()

    X_at_min = opt.fmin_cg(cost_function_for_vector,
                                 X0,
                                 fprime=gradient_for_vector,
                                 args=(C, N, mu, n, p),
                                 gtol=1.0000000000000001e-05,
                                 norm=2,
                                 epsilon=1.49011611938e-08,
                                 maxiter=None,
                                 full_output=0,
                                 disp=1,
                                 retall=0,
                                 callback=None)
    X_at_min = np.matrix(X_at_min.reshape((n, p)))
    show_results(X_at_min, C)

def minimise_f_using_fmin_bfgs(initialise_proc):
    """Run minimisation with BFGS algorithm"""
    C, N, mu, n, p, X0 = initialise_proc()

    X_at_min = opt.fmin_bfgs(cost_function_for_vector,
                                 X0,
                                 fprime=gradient_for_vector,
                                 args=(C, N, mu, n, p),
                                 gtol=1.0000000000000001e-05,
                                 norm=2,
                                 epsilon=1.49011611938e-08,
                                 maxiter=None,
                                 full_output=0,
                                 disp=1,
                                 retall=0,
                                 callback=None)
    X_at_min = np.matrix(X_at_min.reshape((n, p)))
    show_results(X_at_min, C)

```

## Minima of $f(X)$

Compare the columns $x_1,\dots, x_p$ of the matrix $X^\star$ which minimises $ f(X) $
\begin{equation}
  X^\star = \argmin_{X \in \RR^{n \times p}} f(X)
\end{equation}

with the eigenvectors related to the smallest eigenvalues of $ C $.

### Solution

The minimum of the given cost function are matrices $ X $ which contain
the $ p $ eigenvectors of $ C $ which are associated with the $ p $ smallest
eigenvalues of $ C $. The order of the eigenvector in the minimum $ X $
is defined by the order of the diagonal elements in $ N $.

```
# Solution

def initialise_low_dimensional_data():
    """Initialise the data, low dimensions"""
    n = 3
    p = 2
    mu = 2.7

    N = np.matrix(np.diag([2.5, 1.5]))
    E = [1, 2, 3]
    C = random_matrix_from_eigenvalues(E)
    X0 = np.random.rand(n*p)

    return C, N, mu, n, p, X0

def initialise_higher_dimensional_data():
    """Initialise the data, higher dimensions"""
    n  = 20
    p  =  5
    mu = p + 0.5

    N = np.matrix(np.diag(np.arange(p, 0, -1)))
    E = np.arange(1, n+1)
    C = random_matrix_from_eigenvalues(E)
    X0 = np.random.rand(n*p)

    return C, N, mu, n, p, X0

def run_and_time_all_tests():
    """Run all test and time them using a list of function names"""
    List_of_Test_Names = ["minimise_f_using_fmin",
                 "minimise_f_using_fmin_powell",
                 "minimise_f_using_fmin_cg",
                 "minimise_f_using_fmin_bfgs"]

    List_of_Initialisations = ["initialise_low_dimensional_data",
                               "initialise_higher_dimensional_data"]

    for test_name in List_of_Test_Names:
        for init_routine in List_of_Initialisations:
            task_string  = test_name + "(" + init_routine + ")"
            line_length  = 76
            spaces       = 2
            left_padding = (line_length - len(task_string)) // 2
            right_padding = line_length - left_padding - len(task_string)
            print("=" * line_length)
            print("=" * (left_padding - spaces) + " " * spaces + task_string + \
                " " * spaces + "=" * (right_padding - spaces))
            print("=" * line_length)

            start = time.clock()
            exec(task_string)
            run_time = time.clock() - start
            print("run_time :", run_time)

run_and_time_all_tests()
```

```

```


### ¿Cómo funciona la suspensión de un auto?
> Una primer aproximación al modelo de la suspensión de un automovil es considerar el oscilador armónico amortiguado. El cual se representa a través de la siguiente ecuación diferencial.

\begin{equation}
m\ddot{x} + k x + B \dot{x} = 0
\end{equation}
donde $k$ es la constante del muelle, y $B$ la constante de amortiguación.

Referencia:
   - https://docs.scipy.org/doc/scipy/reference/generated/scipy.integrate.odeint.html
   - https://docs.scipy.org/doc/scipy/reference/index.html

___
<div>

 </div>

Esta es una ecuación diferencial ordinaria (EDO). En python existe una función llamada _odeint_ del paquete _integrate_ de la libreria _scipy_, que permite integrar sistemas del tipo
\begin{equation}
\frac{dy}{dt} = f(x,y)
\end{equation}
con condiciones iniciales $y(0) = y_{0}$. Ahora bien, si nos fijamos bien, la ecuación diferencial que tenemos es de segundo orden. No hay problema. La podemos simplificar como un sistema de ecuaciones de primer orden como sigue:

\begin{align}
\dot{x} & = y \\
\dot{y} & = -\frac{k}{m} x - \frac{B}{m} y
\end{align}

```python
import matplotlib.pyplot as plt
import numpy as np
from scipy.integrate import odeint
%matplotlib inline
```

```python
k = 3.0 # Constante del muelle
m = 1.0   # Masa
B = .5 # Constante de amortiguación

def armonico(variables, t):
    x, y = variables
    return [y, -k * x / m - B / m * y]

inicial = [0.6, 0.4] # Vector de posición inicial y velocidad inicial
# condiciones iniciales x(t=0)=0.6 [m] y(t=0)=0.4 [m/s]

#tiempo = np.linspace(0, 15)  # Dominio temporal de 0 a 15
tiempo = np.arange(0, 20, .01)

resultado = odeint(armonico, inicial, tiempo)
#El sistema se resuelve  con
#odeint(sistema, condiciones iniciales, rango donde graficaremos)
```

```python
xx, yy = resultado.T  # extraer posición y velocidad.
```

```python
import matplotlib as mpl
label_size = 14
mpl.rcParams['xtick.labelsize'] = label_size
mpl.rcParams['ytick.labelsize'] = label_size
```

```python
plt.plot(tiempo, xx, c = 'r', label="Posicion")
plt.plot(tiempo, yy, c = 'k', label="Velocidad")
plt.legend(loc = 'best', prop={'size': 14})
plt.xlabel('tiempo', fontsize = 14)
plt.show()
```

```python
omega0 = k/m
```

```python
plt.figure(figsize = (6,6))
plt.scatter(xx, yy/omega0, lw = 0, s = 3, cmap = 'viridis', c = xx)
plt.show()
```

```python
from ipywidgets import *
```

```python
def amortiguado(t = 0):
    fig = plt.figure()
    ax = fig.add_subplot(1, 1, 1)
    ax.plot(tiempo, xx,  'k-')
    ax.plot(tiempo[t], xx[t],  'ro')
    ax.text(4, .6, 'tiempo =    %s'%tiempo[t])
    ax.text(4, .5, 'posición =  %s'%xx[t])
    fig.canvas.draw()
t_f = len(tiempo)
interact_manual(amortiguado, t = (0, t_f, 1));
```

    interactive(children=(IntSlider(value=0, description='t', max=2000), Button(description='Run Interact', style=…

### Casos

Teníamos
\begin{equation}
m\ddot{x} + k x + B \dot{x} = 0
\end{equation}

si recordamos que $\omega_0 ^2 = k/m$ y definimos $B/m\equiv 2\Gamma$, tendremos

\begin{equation}
\ddot{x} + 2\Gamma \dot{x}+ \omega_0^2 x = 0
\end{equation}

### Amortiguado

Si $\omega_0^2 > \Gamma^2$ se tiene movimiento oscilatorio amortiguado.

```python
omega0 = k/m
Gamma = B/(2*m)
```

```python
omega0**2, Gamma**2
```

    (9.0, 0.0625)

```python
omega0**2 > Gamma**2
```

    True

Entonces, el primer caso que ya habíamos presentado corresponde a movimiento amortiguado.

```python
plt.plot(tiempo, xx, c = 'r', label="Posicion")
plt.plot(tiempo, yy, c = 'k', label="Velocidad")
plt.legend(loc = 'best', prop={'size': 14})
plt.xlabel('tiempo', fontsize = 14)
plt.show()
```

### Sobreamortiguado

Si $\omega_0^2 < \Gamma^2$ se tiene movimiento oscilatorio amortiguado.

```python
k = .1 # Constante del muelle
m = 1.0   # Masa
B = .5 # Constante de amortiguación
```

```python
omega0 = k/m
Gamma = B/(2*m)
```

```python
omega0**2, Gamma**2
```

    (0.010000000000000002, 0.0625)

```python
omega0**2 < Gamma**2
```

    True

```python
inicial = [0.6, 0.4]
tiempo = np.arange(0, 20, .01)
resultado = odeint(armonico, inicial, tiempo)
```

```python
xxA, yyA = resultado.T  # extraer posición y velocidad.
```

```python
plt.plot(tiempo, xxA, c = 'r', label="Posicion")
plt.plot(tiempo, yyA, c = 'k', label="Velocidad")
plt.legend(loc = 'best', prop={'size': 14})
plt.xlabel('tiempo', fontsize = 14)
plt.show()
```

### Amortiguamiento crítico

Si $\omega_0^2 = \Gamma^2$ se tiene movimiento aperiódico crítico (amortiguamiento crítico).

```python
k = np.sqrt(.0625) # Constante del muelle
m = 1.0   # Masa
B = .5 # Constante de amortiguación
```

```python
omega0 = k/m
Gamma = B/(2*m)
```

```python
omega0**2, Gamma**2
```

    (0.0625, 0.0625)

```python
omega0**2 == Gamma**2
```

    True

```python
inicial = [0.6, 0.4]
tiempo = np.arange(0, 20, .01)
resultado = odeint(armonico, inicial, tiempo)
```

```python
xxC, yyC = resultado.T  # extraer posición y velocidad.
```

```python
plt.plot(tiempo, xxC, c = 'r', label="Posicion")
plt.plot(tiempo, yyC, c = 'k', label="Velocidad")
plt.legend(loc = 'best',prop={'size': 14})
plt.xlabel('tiempo', fontsize = 14)
plt.show()
```

En resumen, se tiene entonces:

```python
fig, ((ax1, ax2, ax3), (ax4, ax5, ax6)) = plt.subplots(2, 3, sharex='col',
                                                       sharey='row',figsize =(10,6))
ax1.plot(tiempo, xx, c = 'k')
ax1.set_title('Amortiguado', fontsize = 14)
ax1.set_ylabel('Posición', fontsize = 14)
ax2.plot(tiempo, xxA, c = 'b')
ax2.set_title('Sobreamortiguado', fontsize = 14)
ax3.plot(tiempo, xxC, c = 'r')
ax3.set_title('Crítico', fontsize = 16)
ax4.plot(tiempo, yy, c = 'k')
ax4.set_ylabel('Velocidad', fontsize = 14)
ax4.set_xlabel('tiempo', fontsize = 14)
ax5.plot(tiempo, yyA, c = 'b')
ax5.set_xlabel('tiempo', fontsize = 14)
ax6.plot(tiempo, yyC, c = 'r')
ax6.set_xlabel('tiempo', fontsize = 14)
plt.show()
```

> **Actividad**. ¿Cómo se ve el espacio fase para los diferentes casos así como para diferentes condiciones iniciales?

```python

```


# Non Linear Regression Analysis

If the data shows a curvy trend, then linear regression will not produce very accurate results when compared to a non-linear regression because, as the name implies, linear regression presumes that the data is linear.
Let's learn about non linear regressions and apply an example on python. In this notebook, we fit a non-linear model to the datapoints corrensponding to China's GDP from 1960 to 2014.

### Importing required libraries

```
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
```

Though Linear regression is very good to solve many problems, it cannot be used for all datasets. First recall how linear regression, could model a dataset. It models a linear relation between a dependent variable y and independent variable x. It had a simple equation, of degree 1, for example y = 2*(x) + 3.

```
x = np.arange(-5.0, 5.0, 0.1)

##You can adjust the slope and intercept to verify the changes in the graph
y = 2*(x) + 3
y_noise = 2 * np.random.normal(size=x.size)
ydata = y + y_noise
#plt.figure(figsize=(8,6))
plt.plot(x, ydata,  'bo')
plt.plot(x,y, 'r')
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

Non-linear regressions are a relationship between independent variables $x$ and a dependent variable $y$ which result in a non-linear function modeled data. Essentially any relationship that is not linear can be termed as non-linear, and is usually represented by the polynomial of $k$ degrees (maximum power of $x$).

$$ \ y = a x^3 + b x^2 + c x + d \ $$

Non-linear functions can have elements like exponentials, logarithms, fractions, and others. For example: $$ y = \log(x)$$

Or even, more complicated such as :
$$ y = \log(a x^3 + b x^2 + c x + d)$$

Let's take a look at a cubic function's graph.

```
x = np.arange(-5.0, 5.0, 0.1)

##You can adjust the slope and intercept to verify the changes in the graph
y = 1*(x**3) + 1*(x**2) + 1*x + 3
y_noise = 20 * np.random.normal(size=x.size)
ydata = y + y_noise
plt.plot(x, ydata,  'bo')
plt.plot(x,y, 'r')
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

As you can see, this function has $x^3$ and $x^2$ as independent variables. Also, the graphic of this function is not a straight line over the 2D plane. So this is a non-linear function.

Some other types of non-linear functions are:

### Quadratic

$$ Y = X^2 $$

```
x = np.arange(-5.0, 5.0, 0.1)

##You can adjust the slope and intercept to verify the changes in the graph

y = np.power(x,2)
y_noise = 2 * np.random.normal(size=x.size)
ydata = y + y_noise
plt.plot(x, ydata,  'bo')
plt.plot(x,y, 'r')
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

### Exponential

An exponential function with base c is defined by $$ Y = a + b c^X$$ where b ≠0, c > 0 , c ≠1, and x is any real number. The base, c, is constant and the exponent, x, is a variable.

```
X = np.arange(-5.0, 5.0, 0.1)

##You can adjust the slope and intercept to verify the changes in the graph

Y= np.exp(X)

plt.plot(X,Y)
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

### Logarithmic

The response $y$ is a results of applying logarithmic map from input $x$'s to output variable $y$. It is one of the simplest form of __log()__: i.e. $$ y = \log(x)$$

Please consider that instead of $x$, we can use $X$, which can be polynomial representation of the $x$'s. In general form it would be written as
\begin{equation}
y = \log(X)
\end{equation}

```
X = np.arange(-5.0, 5.0, 0.1)

Y = np.log(X)

plt.plot(X,Y)
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

### Sigmoidal/Logistic

$$ Y = a + \frac{b}{1+ c^{(X-d)}}$$

```
X = np.arange(-5.0, 5.0, 0.1)

Y = 1-4/(1+np.power(3, X-2))

plt.plot(X,Y)
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

# Non-Linear Regression example

For an example, we're going to try and fit a non-linear model to the datapoints corrensponding to China's GDP from 1960 to 2014. We download a dataset with two columns, the first, a year between 1960 and 2014, the second, China's corresponding annual gross domestic income in US dollars for that year.

```
import numpy as np
import pandas as pd

#downloading dataset
!wget -nv -O china_gdp.csv https://s3-api.us-geo.objectstorage.softlayer.net/cf-courses-data/CognitiveClass/ML0101ENv3/labs/china_gdp.csv

df = pd.read_csv("china_gdp.csv")
print('Size data : ', len(df))
df.head(10)
```

    2019-07-22 03:42:56 URL:https://s3-api.us-geo.objectstorage.softlayer.net/cf-courses-data/CognitiveClass/ML0101ENv3/labs/china_gdp.csv [1218/1218] -> "china_gdp.csv" [1]
    Size data :  55

      Year
      Value

      0
      1960
      5.918412e+10

      1
      1961
      4.955705e+10

      2
      1962
      4.668518e+10

      3
      1963
      5.009730e+10

      4
      1964
      5.906225e+10

      5
      1965
      6.970915e+10

      6
      1966
      7.587943e+10

      7
      1967
      7.205703e+10

      8
      1968
      6.999350e+10

      9
      1969
      7.871882e+10

__Did you know?__ When it comes to Machine Learning, you will likely be working with large datasets. As a business, where can you host your data? IBM is offering a unique opportunity for businesses, with 10 Tb of IBM Cloud Object Storage: [Sign up now for free](http://cocl.us/ML0101EN-IBM-Offer-CC)

### Plotting the Dataset ###
This is what the datapoints look like. It kind of looks like an either logistic or exponential function. The growth starts off slow, then from 2005 on forward, the growth is very significant. And finally, it deaccelerates slightly in the 2010s.

```
plt.figure(figsize=(8,5))
x_data, y_data = (df["Year"].values, df["Value"].values)
plt.plot(x_data, y_data, 'ro')
plt.ylabel('GDP')
plt.xlabel('Year')
plt.show()
```

### Choosing a model ###

From an initial look at the plot, we determine that the logistic function could be a good approximation,
since it has the property of starting with a slow growth, increasing growth in the middle, and then decreasing again at the end; as illustrated below:

```
X = np.arange(-5.0, 5.0, 0.1)
Y = 1.0 / (1.0 + np.exp(-X))

plt.plot(X,Y)
plt.ylabel('Dependent Variable')
plt.xlabel('Indepdendent Variable')
plt.show()
```

The formula for the logistic function is the following:

$$ \hat{Y} = \frac1{1+e^{\beta_1(X-\beta_2)}}$$

$\beta_1$: Controls the curve's steepness,

$\beta_2$: Slides the curve on the x-axis.

### Building The Model ###
Now, let's build our regression model and initialize its parameters.

```
def sigmoid(x, Beta_1, Beta_2):
     y = 1 / (1 + np.exp(-Beta_1*(x-Beta_2)))
     return y
```

Lets look at a sample sigmoid line that might fit with the data:

```
beta_1 = 0.10
beta_2 = 1990.0

#logistic function
Y_pred = sigmoid(x_data, beta_1 , beta_2)

#plot initial prediction against datapoints
plt.plot(x_data, Y_pred*15000000000000.)
plt.plot(x_data, y_data, 'ro')
```

Our task here is to find the best parameters for our model. Lets first normalize our x and y:

```
# Lets normalize our data
xdata =x_data/max(x_data)
ydata =y_data/max(y_data)
```

#### How we find the best parameters for our fit line?
we can use __curve_fit__ which uses non-linear least squares to fit our sigmoid function, to data. Optimal values for the parameters so that the sum of the squared residuals of sigmoid(xdata, *popt) - ydata is minimized.

popt are our optimized parameters.

```
from scipy.optimize import curve_fit
popt, pcov = curve_fit(sigmoid, xdata, ydata)
#print the final parameters
print(" beta_1 = %f, beta_2 = %f" % (popt[0], popt[1]))
```

     beta_1 = 690.453017, beta_2 = 0.997207

Now we plot our resulting regresssion model.

```
x = np.linspace(1960, 2015, 55)
x = x/max(x)
plt.figure(figsize=(8,5))
y = sigmoid(x, *popt)
plt.plot(xdata, ydata, 'ro', label='data')
plt.plot(x,y, linewidth=3.0, label='fit')
plt.legend(loc='best')
plt.ylabel('GDP')
plt.xlabel('Year')
plt.show()
```

## Practice
Can you calculate what is the accuracy of our model?

```
# write your code here

# split data into train/test
msk = np.random.rand(len(df)) < 0.8
train_x = xdata[msk]
test_x = xdata[~msk]
train_y = ydata[msk]
test_y = ydata[~msk]

# build the model using train set
popt, pcov = curve_fit(sigmoid, train_x, train_y)

# predict using test set
y_hat = sigmoid(test_x, *popt)

# evaluation
print("Mean absolute error: %.2f" % np.mean(np.absolute(y_hat - test_y)))
print("Residual sum of squares (MSE): %.2f" % np.mean((y_hat - test_y) ** 2))
from sklearn.metrics import r2_score
print("R2-score: %.2f" % r2_score(y_hat , test_y) )
```

    Mean absolute error: 0.05
    Residual sum of squares (MSE): 0.00
    R2-score: 0.95

Double-click __here__ for the solution.

## Want to learn more?

IBM SPSS Modeler is a comprehensive analytics platform that has many machine learning algorithms. It has been designed to bring predictive intelligence to decisions made by individuals, by groups, by systems – by your enterprise as a whole. A free trial is available through this course, available here: [SPSS Modeler](http://cocl.us/ML0101EN-SPSSModeler).

Also, you can use Watson Studio to run these notebooks faster with bigger datasets. Watson Studio is IBM's leading cloud solution for data scientists, built by data scientists. With Jupyter notebooks, RStudio, Apache Spark and popular libraries pre-packaged in the cloud, Watson Studio enables data scientists to collaborate on their projects without having to install anything. Join the fast-growing community of Watson Studio users today with a free account at [Watson Studio](https://cocl.us/ML0101EN_DSX)

### Thanks for completing this lesson!

Notebook created by: Saeed Aghabozorgi

Copyright © 2018 [Cognitive Class](https://cocl.us/DX0108EN_CC). This notebook and its source code are released under the terms of the [MIT License](https://bigdatauniversity.com/mit-license/).


```python
import numpy as np
from sympy import symbols, expand, simplify, sqrt, Matrix, cos, sin
from sympy.abc import x,v,h,t,y
```

```python
# Answer
def position_verlet(dt, x, v, force_rule):
    """Does one iteration/timestep using the Position verlet scheme

    Parameters
    ----------
    dt : float
        Simulation timestep in seconds
    x : float/array-like
        Quantity of interest / position of COM
    v : float/array-like
        Quantity of interest / velocity of COM
    force_rule : ufunc
        A function, f, that takes one argument and
        returns the instantaneous forcing

    Returns
    -------
    x_n : float/array-like
        The quantity of interest at the Next time step
    v_n : float/array-like
        The quantity of interest at the Next time step
    """
    temp_x = x + 0.5*dt*v
    v_n = v + dt * force_rule(temp_x)
    x_n = temp_x + 0.5 * dt * v_n
    return x_n, v_n
```

```python
# Answer
def euler_fwd(dt, x, v, force_rule):
    """Does one iteration/timestep using the Euler forward scheme

    Parameters
    ----------
    dt : float
        Simulation timestep in seconds
    x : float/array-like
        Quantity of interest / position of COM
    v : float/array-like
        Quantity of interest / velocity of COM
    force_rule : ufunc
        A function, f, that takes one argument and
        returns the instantaneous forcing

    Returns
    -------
    x_n : float/array-like
        The quantity of interest at the Next time step
    v_n : float/array-like
        The quantity of interest at the Next time step
    """
    x_n = x + dt * v
    v_n = v + dt * force_rule(x)
    return x_n, v_n
```

```python
def runge_kutta4(dt, x, v, force_rule):
    """Does one iteration/timestep using the RK4 scheme

    Parameters
    ----------
    dt : float
        Simulation timestep in seconds
    x : float/array-like
        Quantity of interest / position of COM
    v : float/array-like
        Quantity of interest / velocity of COM
    force_rule : ufunc
        A function, f, that takes one argument and
        returns the instantaneous forcing

    Returns
    -------
    x_n : float/array-like
        The quantity of interest at the Next time step
    v_n : float/array-like
        The quantity of interest at the Next time step
    """

    def vector_func(y):
        return Matrix([y[1], force_rule(y[0])])

    # Base
    u = Matrix([x,v])

    # Stage 1
    k_1 = dt*vector_func(u)

    # Stage 2
    k_2 = dt * vector_func(u + 0.5*k_1)

    # Stage 3
    k_3 = dt * vector_func(u + 0.5*k_2)

    # Stage 4
    k_4 = dt * vector_func(u + k_3)

    u_n = u + (1./6.)*(k_1 + 2.*k_2 + 2.* k_3 + k_4)

    return u_n[0], u_n[1]
```

```python
def harmonic(t_x):
    return -t_x
```

```python
init_energy = x**2 + y**2
u, v = x, y
for i in range(2):
    u, v = position_verlet(h,u,v,harmonic)
fin_energy = u**2 + v**2
```

```python
init_energy
```

```python
simplify(fin_energy.subs([(x,cos(t)), (y,sin(t))]))
```

```python
temp = expand(fin_energy) - init_energy
```

```python
new_temp = temp.subs(y,sqrt(1-x**2))
```

```python
expand(new_temp)
```

```python
init_energy = x**2 + y**2
u, v = x, y
for i in range(5):
    u, v = euler_fwd(h,u,v,harmonic)
fin_energy = u**2 + v**2
```

```python
expand(simplify(fin_energy.subs([(x,cos(t)), (y,sin(t))])))
```

```python
init_energy = x**2 + y**2
u, v = x, y
for i in range(2):
    u, v = runge_kutta4(h,u,v,harmonic)
fin_energy = u**2 + v**2
```

```python
expand(simplify(fin_energy.subs([(x,cos(t)), (y,sin(t))])))
```

```python

```