```python
import opt_mo
import numpy as np
import sympy as sym
import itertools

sym.init_printing()

```

    //anaconda3/envs/opt-mo/lib/python3.6/site-packages/sklearn/externals/joblib/__init__.py:15: DeprecationWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.
      warnings.warn(msg, category=DeprecationWarning)

```python
p_1, p_2, p_3, p_4 = sym.symbols("p_1, p_2, p_3, p_4")
q_1, q_2, q_3, q_4 = sym.symbols("q_1, q_2, q_3, q_4")

```

```python
p = (p_1, p_2, p_3, p_4)
q = (q_1, q_2, q_3, q_4)

```

```python
pi_1, pi_2, pi_3, pi_4 = sym.symbols("pi_1, pi_2, pi_3, pi_4")
pi = (pi_1, pi_2, pi_3, pi_4)

```

```python
R, S, T, P = sym.symbols("R, S, T, P")
```

```python
def pd_utility(S_p, x, y):
    """
    Returns the utility as a function
    of p, q vectors. Based on Press and Dyson, 2012.
    """
    B = sym.Matrix(opt_mo.make_B(S_p, x, y))
    unit = sym.Matrix(opt_mo.make_B((1, 1, 1, 1), x, y))
    return B.det() / unit.det()
```

**Theorem 1 Proof**

As discrubed in Section 2 the utility of a memory one player against another is given by the steady states of M
multiplied by the payoffs.

```python
M = opt_mo.mem_one_match_markov_chain(player=p, opponent=q)

```

```python
ss = opt_mo.steady_states(M, pi)

```

```python
v = sym.Matrix([[ss[pi_1]], [ss[pi_2]], [ss[pi_3]], [ss[pi_4]]])

```

```python
first_ss = v[0].collect(p_2 * p_4).collect(p_2 * p_3).collect(p_3 * p_4)
```

```python
second_ss = v[1].collect(p_1 * p_3).collect(p_1 * p_4).collect(p_3 * p_4)
```

```python
(sym.fraction(second_ss)[-1] - sym.fraction(first_ss)[-1]).simplify()
```

```python
third_ss = v[2].factor().collect(p_1 * p_2).collect(p_1 * p_4).collect(p_2 * p_4)
```

```python
fourth_ss = v[3].factor()
```

```python
(sym.fraction(third_ss)[-1] - sym.fraction(fourth_ss)[-1]).expand()
```

```python
(sym.fraction(third_ss)[-1] - sym.fraction(second_ss)[-1]).simplify()
```

```python
utility = v.dot(np.array([R, S, T, P]))
```

```python
S_p = sym.Matrix([[R, S, T, P]])
```

```python
utility = pd_utility(S_p, p, q)
```

```python
numerator, denominator = sym.fraction(utility.simplify())
```

```python
numerator
```

```python
numerator_elements = [[numerator.coeff(f1 * f2).factor() * f1 * f2 for f2 in p] for f1 in p]
```

```python
flat_elements = list(itertools.chain.from_iterable(numerator_elements))
```

```python
cross_prod = sum(flat_elements) / 2
```

```python
cross_prod
```

```python
sym.latex(sum(flat_elements) / 2).replace("\\left", "").replace("\\right", "")
```

    '- p_{1} p_{2} (q_{1} - q_{3}) (P q_{2} - P - T q_{4}) + p_{1} p_{3} (q_{1} - q_{2}) (P q_{3} - S q_{4}) + p_{1} p_{4} (q_{1} - q_{4}) (S q_{2} - S - T q_{3}) + p_{2} p_{3} (q_{2} - q_{3}) (P q_{1} - P - R q_{4}) - p_{2} p_{4} (q_{3} - q_{4}) (R q_{2} - R - T q_{1} + T) + p_{3} p_{4} (q_{2} - q_{4}) (R q_{3} - S q_{1} + S)'

```python
linear_expr = numerator.subs({p_2: 0, p_3: 0, p_4: 0}).coeff(p_1).factor() * p_1
linear_expr += numerator.subs({p_1: 0, p_3: 0, p_4: 0}).coeff(p_2).factor() * p_2
linear_expr += numerator.subs({p_1: 0, p_2: 0, p_4: 0}).coeff(p_3).factor() * p_3
linear_expr += numerator.subs({p_1: 0, p_2: 0, p_3: 0}).coeff(p_4).factor() * p_4
```

```python
linear_expr
```

```python
sym.latex(linear_expr).replace("\\left", "").replace("\\right", "")
```

    'p_{1} q_{1} (P q_{2} - P - T q_{4}) - p_{2} (q_{3} - 1) (P q_{2} - P - T q_{4}) + p_{3} (- P q_{1} q_{2} + P q_{2} q_{3} + P q_{2} - P q_{3} + R q_{2} q_{4} - S q_{2} q_{4} + S q_{4}) + p_{4} (- R q_{2} q_{4} + R q_{4} + S q_{2} q_{4} - S q_{2} - S q_{4} + S + T q_{1} q_{4} - T q_{3} q_{4} + T q_{3} - T q_{4})'

```python
constant = numerator.subs({p_2: 0, p_3: 0, p_4: 0, p_1: 0})
```

```python
constant
```

```python
sym.latex(constant)
```

    '- P q_{2} + P + T q_{4}'

```python
((constant - linear_expr - cross_prod) - numerator).simplify()
```

```python
((cross_prod + linear_expr + constant) - numerator).factor()
```

**Denominator**

```python
denominator_elements = [[denominator.coeff(f1 * f2) * f1 * f2 for f2 in p] for f1 in p]
```

```python
flat_elements = list(itertools.chain.from_iterable(denominator_elements))
```

```python
cross_prod = sum(flat_elements) / 2
```

```python
cross_prod
```

```python
sym.latex(cross_prod).replace("\\left", "").replace("\\right", "")
```

    'p_{1} p_{2} (q_{1} q_{2} - q_{1} q_{4} - q_{1} - q_{2} q_{3} + q_{3} q_{4} + q_{3}) + p_{1} p_{3} (- q_{1} q_{3} + q_{1} q_{4} + q_{2} q_{3} - q_{2} q_{4}) + p_{1} p_{4} (- q_{1} q_{2} + q_{1} q_{3} + q_{1} + q_{2} q_{4} - q_{3} q_{4} - q_{4}) + p_{2} p_{3} (- q_{1} q_{2} + q_{1} q_{3} + q_{2} q_{4} + q_{2} - q_{3} q_{4} - q_{3}) + p_{2} p_{4} (- q_{1} q_{3} + q_{1} q_{4} + q_{2} q_{3} - q_{2} q_{4}) + p_{3} p_{4} (q_{1} q_{2} - q_{1} q_{4} - q_{2} q_{3} - q_{2} + q_{3} q_{4} + q_{4})'

```python
linear_expr = denominator.subs({p_2: 0, p_3: 0, p_4: 0}).coeff(p_1) * p_1
linear_expr += denominator.subs({p_1: 0, p_3: 0, p_4: 0}).coeff(p_2) * p_2
linear_expr += denominator.subs({p_1: 0, p_2: 0, p_4: 0}).coeff(p_3) * p_3
linear_expr += denominator.subs({p_1: 0, p_2: 0, p_3: 0}).coeff(p_4) * p_4
```

```python
sym.latex(linear_expr).replace("\\left", "").replace("\\right", "")
```

    'p_{1} (- q_{1} q_{2} + q_{1} q_{4} + q_{1}) + p_{2} (q_{2} q_{3} - q_{2} - q_{3} q_{4} - q_{3} + q_{4} + 1) + p_{3} (q_{1} q_{2} - q_{2} q_{3} - q_{2} + q_{3} - q_{4}) + p_{4} (- q_{1} q_{4} + q_{2} + q_{3} q_{4} - q_{3} + q_{4} - 1)'

```python
constant = denominator.subs({p_2: 0, p_3: 0, p_4: 0, p_1: 0})
```

```python
constant
```

```python
sym.latex(constant)
```

    'q_{2} - q_{4} - 1'


```python
import numpy as np
import pandas as pd
import emcee
import corner
import time
import scipy.optimize as op
from allsn_info import get_at2019dge
from helper.arnett import model_arnett_Ltph
from multiprocessing import Pool
from helper import phys
from helper.mcmcfit import mylinear_fit
from helper.models import model_piro15_bol_recast

import matplotlib
import matplotlib.pyplot as plt
fs = 14
matplotlib.rcParams['font.size']=fs
```

## Method1: Model Fitting

```python
filename = "./helper/piromodel/2.0/sampler.h5"
reader = emcee.backends.HDFBackend(filename)
samples = reader.get_chain(discard=1000, flat=True)
lgR_sigmas = np.percentile(samples[:,0], (0.13, 2.27, 15.87, 50, 84.13, 97.73, 99.87))
lgM_sigmas = np.percentile(samples[:,1], (0.13, 2.27, 15.87, 50, 84.13, 97.73, 99.87))
t0_sigmas = np.percentile(samples[:,2], (0.13, 2.27, 15.87, 50, 84.13, 97.73, 99.87))
E51_sigmas = np.percentile(samples[:,3], (0.13, 2.27, 15.87, 50, 84.13, 97.73, 99.87))
Eenvs_sigmas = np.percentile(samples[:,4], (0.13, 2.27, 15.87, 50, 84.13, 97.73, 99.87)) * 1e+49

Eenv = Eenvs_sigmas[3]
Renv = 10**lgR_sigmas[3]
Menv = 10**lgM_sigmas[3]
t0 =t0_sigmas[3]
E51 = E51_sigmas[3]
```

```python
data = pd.read_csv('../data/otherSN/Yao2020/bbdata.csv')
t_data = data['phase'].values - t0
L_data = data['Lbb'].values
L_unc_data = data['Lbb_unc'].values
lgL_data = data['lgLbb'].values
lgL_unc_data = data['lgLbb_unc'].values
```

```python
tgrid = np.linspace(0, 30, 100)
Lp15 = model_piro15_bol_recast(tgrid, Renv, Menv, E51, Eenv / 1e+49)
lgLp15 = np.log10(Lp15)
```

```python
result = get_at2019dge()
tb0 = result['tb']
```

```python
tb0 = tb0[tb0['filter'].values=='r']
tb0 = tb0[tb0.instrument!="P60+SEDM"]
tb0 = tb0[(tb0.tmax_of-t0) > max(t_data)]
t_quasi = tb0["tmax_of"].values -t0
Lquasi = tb0["Llambda"].values * tb0['wave'].values
Lquasi_unc = tb0["Llambda_unc"].values * tb0['wave'].values
lgLquasi = np.log10(Lquasi)
lgLquasi_unc = Lquasi_unc / Lquasi / np.log(10)
```

```python
Lp15_data = model_piro15_bol_recast(t_data, Renv, Menv, E51, Eenv / 1e+49)
```

```python
L_data_resi = L_data - Lp15_data
lgL_data_resi = np.log10(L_data_resi)
```

    /anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:2: RuntimeWarning: invalid value encountered in log10

```python
plt.figure()
plt.plot(t_data, Lp15_data)
plt.plot(t_data, L_data, 'ko')
```

    <IPython.core.display.Javascript object>

    [<matplotlib.lines.Line2D at 0x14a98b390>]

```python
%matplotlib notebook
```

```python
plt.figure(figsize=(6, 5))
ax = plt.subplot(111)
ax.errorbar(t_quasi, lgLquasi, lgLquasi_unc, fmt='--o', color = "grey", markerfacecolor='none', zorder = 3, markersize=7)
ax.errorbar(t_data, lgL_data_resi, lgL_unc_data, fmt='ok', zorder = 3, markersize=5)
```

    <IPython.core.display.Javascript object>

    <ErrorbarContainer object of 3 artists>

```python
x1 = t_data
y1 = lgL_data_resi
ey1 = lgL_unc_data
ix1 = (~np.isnan(y1))&(x1 > 2)
x1 = x1[ix1]
y1 = y1[ix1]
ey1 = ey1[ix1]
ix = x1 > 6
x1 = x1[ix]
y1 = y1[ix]
ey1 = ey1[ix]
```

```python
x3 = t_quasi
y3 = lgLquasi
ey3 = lgLquasi_unc
```

```python
x = np.hstack([x1, x3]) + t0
y = np.hstack([y1, y3])
ey = np.hstack([ey1, ey3])
ix = np.argsort(x)
x = x[ix]
y = y[ix]
ey = ey[ix]
```

```python
xyey = np.vstack([x, y, ey])
```

```python
np.savetxt("./helper/Lbb_p15subtracted.txt", xyey)
```

### Photospheric phase Arnett model -- modified

```python
from helper.arnett import main_arnettrun
```

```python
# main_arnettrun()
# This takes some time to run
```

```python
filename = "./helper/arnettmodel/sampler.h5"
reader = emcee.backends.HDFBackend(filename)
```

```python
samples = reader.get_chain(discard=200, flat=True)
lgprobs = reader.get_log_prob(discard=200, flat=True)
print (samples.shape)
print (lgprobs.shape)
```

    (600000, 4)
    (600000,)

```python
taum_sigmas = np.percentile(samples[:,0], (0.13, 2.27, 15.87, 50, 84.13, 97.73, 99.87))
lgMni_sigmas = np.percentile(samples[:,1], (0.13, 2.27, 15.87, 50, 84.13, 97.73, 99.87))
t0_sigmas = np.percentile(samples[:,2], (0.13, 2.27, 15.87, 50, 84.13, 97.73, 99.87))
tfl_sigmas = np.percentile(samples[:,3], (0.13, 2.27, 15.87, 50, 84.13, 97.73, 99.87))
texp = tfl_sigmas[3]
```

```python
paramsNames= [r"$\tau_{\rm m}$",
                 'lg' +r'$M_{\rm Ni}$',
                 r"$t_0$",
             r"$t_{\rm fl}$"]
quantiles=[0.1587, 0.5, 0.8413]
```

```python
samples_final = samples[:, :3]
```

```python
corner.corner(samples_final, labels = paramsNames[:3], quantiles = quantiles,
              range = [0.99, 0.99, 0.99],
              show_titles=True, plot_datapoints=False,
               title_kwargs = {"fontsize": fs})
plt.savefig("../paper/figures/corner_arnett_modified_all.pdf")
# plt.close()
```

    <IPython.core.display.Javascript object>

```python
taum_ = taum_sigmas[3]
taum_unc_right = taum_sigmas[4]-taum_sigmas[3]
taum_unc_left = taum_sigmas[3]-taum_sigmas[2]
print ("tau_m = %.2f + %.2f - %.2f day"%(taum_, taum_unc_right, taum_unc_left))
```

    tau_m = 6.35 + 0.18 - 0.18 day

```python
Mni = 10**lgMni_sigmas[3]
Mni_unc_left = 10**lgMni_sigmas[3] - 10**lgMni_sigmas[2]
Mni_unc_right = 10**lgMni_sigmas[4] - 10**lgMni_sigmas[3]
print ("%.2f (+%.2f) (-%.2f) 1e-2 Msun"%(Mni*100, Mni_unc_right*100, Mni_unc_left*100))
```

    1.61 (+0.04) (-0.03) 1e-2 Msun

```python
t0_ = t0_sigmas[3]
t0_unc_right = t0_sigmas[4]-t0_sigmas[3]
t0_unc_left = t0_sigmas[3]-t0_sigmas[2]
print ("t0 = %.2f + %.2f - %.2f day"%(t0_, t0_unc_right, t0_unc_left))
```

    t0 = 24.04 + 0.76 - 0.73 day

```python
from helper.arnett import model_arnett_modified
tgrid = np.linspace(0.1, 70, 200)
Lnidecay = model_arnett_modified(tgrid, taum_ = taum_, Mni_ = Mni, t0_ = t0_, texp = 0)
lgLnidecay = np.log10(Lnidecay)
```

```python
plt.figure()
plt.errorbar(x-texp, y, ey, fmt= ".k")
plt.plot(tgrid, lgLnidecay)
```

    <IPython.core.display.Javascript object>

    [<matplotlib.lines.Line2D at 0x154f5f048>]

Estimate ejecta mass

```python
kappa_opt = 0.07 # relevant for stripped envelope supernova
v_ej = 6000 * 1e+5
Mej_ = (taum_ * 24 * 3600)**2 * 13.8 * phys.c / 2 / kappa_opt * v_ej / phys.sm
Mej_unc_right = ((taum_+taum_unc_right) * 24 * 3600)**2 * 13.8 * phys.c / 2 / kappa_opt * v_ej / phys.sm - Mej_
Mej_unc_left = Mej_ - ((taum_-taum_unc_left) * 24 * 3600)**2 * 13.8 * phys.c / 2 / kappa_opt * v_ej / phys.sm
print ("Mej = %.2f (+%.2f) (-%.2f) Msun"%(Mej_, Mej_unc_right, Mej_unc_left))
```

    Mej = 0.27 (+0.02) (-0.01) Msun

```python
Ekin_ = 0.3 * Mej_ * phys.sm * v_ej**2
Ekin_unc_left = 0.3 * Mej_unc_left * phys.sm * v_ej**2
Ekin_unc_right = 0.3 * Mej_unc_right * phys.sm * v_ej**2
print ("Ekin = %.2f (+%.2f) (-%.2f) e+49 erg"%(Ekin_ / 1e+49, Ekin_unc_right / 1e+49, Ekin_unc_left / 1e+49))
```

    Ekin = 5.76 (+0.34) (-0.32) e+49 erg

```python
ind_max = np.argsort(Lnidecay)[-1]
```

```python
tpeak = (tgrid[ind_max]) * 86400
Lpeak = Lnidecay[ind_max]
```

```python
print ("tpeak = %.1f day"%(tpeak / 86400))
print ("Lpeak = %.1f e+41 erg/s"%(Lpeak / 1e+41))
```

    tpeak = 8.5 day
    Lpeak = 5.9 e+41 erg/s

## Method2: KK19 Equations
An improvement from the Arnett relations.

```python
ts = 8.8*86400
beta = 4/3
L0 = Lpeak * beta**2 * tpeak**2 / 2 / ts**2 / (1 - (1 + beta*tpeak/ts)*np.exp(-beta * tpeak / ts))
epsilon_Ni = 3.9e+10 # erg / g / s
M_Ni = L0 / (epsilon_Ni) / phys.sm
```

```python
print ("M_Ni = %.2f e-2 Msun"%(M_Ni/1e-2))
```

    M_Ni = 1.70 e-2 Msun

\begin{align}
x &= \frac{t_{\rm peak}}{t_{\rm d}}\\
x &= 0.11 {\rm ln} (1 + 9 \frac{8.8}{9}x) + 0.36
\end{align}

```python
x = np.linspace(0.1, 1, 100)
y = 0.11 * np.log(1 + 8.8*x) + 0.36
```

```python
plt.figure(figsize=(4,4))
plt.plot(x, x)
plt.plot(x, y)
```

    <IPython.core.display.Javascript object>

    [<matplotlib.lines.Line2D at 0x15619ecf8>]

```python
ix = np.argsort(abs(y-x))

x_solved = x[ix[0]]
```

```python
td = tpeak / x_solved
```

```python
td / 86400
```

    15.382239064173326

```python
kappa = 0.07 # DD19 default
vej = 6000 * 1e+5
Mej_kk = td**2 * vej * phys.c / kappa / phys.sm
print (Mej_kk)
```

    0.2282549926628625

```python
0.3 * Mej_kk * phys.sm * vej**2
```

    4.901897966453538e+49

```python

```


# Deep Learning - Assignment 1

## Outline (15 points)
#### About the laboratory exercises
The laboratory exercises in computer room are for providing an environment doing the assignments and project，except the first exercise that will also contain a pytorch tutorial, explaining how to install python, pytorch on your own computer, or how to use them in the computer room.

Extra information about access to pytorch environment can be found in the FAQ section in [https://moodle.oulu.fi/mod/page/view.php?id=73734](https://moodle.oulu.fi/mod/page/view.php?id=73734).

We will give at least 2 weeks for each assignment and more for the final project. So it happens that the new assignment is published while the deadline of the previous assignment is not reached. Good luck.

#### In this assignment, you will learn:
* How to generate random data using python.
* Building linear models for simple regression problem on the generated data.
* Training the linear models with gradient descent algorithm.
* How to alleviate over-fitting for your model.
* **Concepts** you will learn: ***Regularization***, ***Model selection***, ***Gradient descent***, ***Over-fitting***, ***Weight decay***, ***Training/Validation/Testing***.

#### Tasks
In this assignment, we are going to solve a basic linear regression problem by fitting a polynomial function, which we shall use as a running example to motivate a number of key concepts mentioned above.
* **Part 1.** Generate training and testing data using python.
* **Part 2.** Linear regression with polynomials without regularization.
* **Part 3.** Linear regression with polynomials with regularization.

#### Environment
Python 3.

Other libraries should be installed correctly such as numpy, matplotlib, *etc*., according to the dependencies of the assignment.

#### Database
* Randomly synthesized data.

## Part 1. Generate training and testing data using python
Suppose we observe a real-valued input variable $x$ and we wish to use this observation to predict the value of a real-valued target variable $y$. For the present purposes, it is instructive to consider an artificial example using synthetically generated data because we then know the precise process that generated the data for comparison against any learned model.

We therefore generate our training data which consists of 10 pairs of $\{x_i, y_i\}$ where $\{x_i\}$ are generated uniformly in range (0, 1), and the corresponding target values $\{y_i\}$ are obtained by first computing the corresponding values of the function $sin(2\pi x)$ and then adding random noise with a Gaussian distribution having standard deviation 0.3.
\begin{align*}
y_i = sin(2\pi x_i) + \varepsilon, \varepsilon\sim \mathcal{N}(0,\,0.3^{2}).
\end{align*}

The testing data is created in the same way as the creation of training data but the number of pairs is 100, thus to give a more complete evaluation of the model to be trained.

Please create the required training and testing data below and plot them. You could use numpy to create the data and matplotlib.pyplot to show them.

```python
# import libraries
import numpy as np
import matplotlib.pyplot as plt

x_train = np.random.rand(10, 1)
y_train = np.sin(2 * np.pi * x_train) + np.random.normal(0, 0.3, (10, 1))

# your code here
x_test = np.random.rand(100, 1)
y_test = np.sin(2 * np.pi * x_test) + np.random.normal(0, 0.3, (100, 1))

plt.plot(x_train, y_train, marker='o', linestyle='none',label='Train Data')
plt.plot(x_test, y_test, fillstyle='none', marker='o', color='g', linestyle='none',label='Test Data')
plt.legend(loc='upper right')
plt.show()
```

--------------------1 point--------------------

## Part 2. Linear regression with polynomials without regularization
Congratulations if you generate the data correctly! However, to make sure all of you use the same data, we have generated it in advance.

```python
import pickle
with open('data/rawdata.pickle', 'rb') as f:
    data = pickle.load(f)

x_train = data['x_train']
y_train = data['y_train']

x_test = data['x_test']
y_test = data['y_test']
```

The task in this part is to train a linear model to fit the curve on the training data. Now that we have $\{x, y\}$, we consider using a polynomial function of the form
\begin{align}
f(x, \pmb{w}) = \omega_0 + \omega_1x + \omega_2x^2 + ... + \omega_Mx^M = \sum_{j=0}^{M}\omega_jx^j,
\tag{1}
\end{align}
where $M$ is the *order* of the polynomial, and $x^j$ denotes $x$ raised to the power of j. The values of the coefficients will be determined by fitting the polynomial to the training data. To do so, we introduce the error function to be minimized:
\begin{equation}
E(\pmb{w}) = \frac{1}{2}\sum_{i=1}^{N}\{f(x_i, \pmb{w}) - y_i\}^2,
\tag{2}
\end{equation}
where, $N$ is 10 in our case.

First of all, to use matrix operation in this case, we re-write the polynomial function (1) in the form of matrix multiplication:
$$
\begin{bmatrix}
f(x_1, \pmb{w}) \\ f(x_2, \pmb{w}) \\ . \\ . \\ . \\ f(x_N, \pmb{w})
\end{bmatrix}
=
\begin{bmatrix}
1 & x_1 & x_1^2 & . & . & . & x_1^M \\ 1 & x_2 & x_2^2 & . & . & . & x_2^M \\ . & . & . & . & . & . & . \\ . & . & . & . & . & . & . \\ . & . & . & . & . & . & . \\ 1 & x_N & x_N^2 & . & . & . & x_N^M
\end{bmatrix}
\begin{bmatrix}
\omega_0 \\ \omega_1 \\ . \\ . \\ . \\ \omega_M
\end{bmatrix}
$$
We call the first matrix on the right hand side feature matrix. We next build the feature matrix and try to solve above minimization problem using [`Gradient Descent Algorithm`](https://towardsdatascience.com/an-introduction-to-gradient-descent-c9cca5739307) and the closed-form analytical solution. Because the result from the closed-form solution is theoretically global-optimal for the problem, by doing this, we could know how well our gradient descent algorithm performs by comparing its results with the global optimal.

Anyway, I'll give an guidance using $M=3$ as following.

### Part 2.1. Method based on closed-form analysis
First, let's quickly solve this using the closed-form analytical solution. In this case, we luckily have this solution to solve the polynomial regression problem. However, in practical problems using deep learning models, which are non-linear and often very complex, there doesn't exist a closed-form solution or extremely difficult to find the solution analytically, the most common way is using gradient descent algorithm.

In our case, we will first solve the regression problem using closed-form analysis to get the optimal solution, which can also be seen as the upper bound the gradient descent algorithm goes (because gradient descent algorithm will alwayse find an approximate solution which is close to the optimal).

First, we calculate the feature matrix based on the matrix formulation and apply [`sklearn.linear_model.LinearRegression`](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LinearRegression.html) to solve the regression problem.

```python
feature_matrix = np.ones_like(x_train)
M = 3
for i in range(1, M+1):
    feature_matrix = np.concatenate((feature_matrix, x_train ** i), axis=1)

from sklearn.linear_model import LinearRegression

model = LinearRegression(fit_intercept=False).fit(feature_matrix, y_train)
```

Then, show the original training data points and the draw the curve of the function obtained on a single figure.

```python
x_curve = np.linspace(0, 1, 10000).reshape(-1, 1)
features_curve = np.ones_like(x_curve)
for i in range(1, M+1):
    features_curve = np.concatenate((features_curve, x_curve ** i), axis=1)

y_curve = model.predict(features_curve)
plt.plot(x_train, y_train, marker='o', linestyle='None')
plt.plot(x_curve, y_curve, 'r')
plt.show()
```

Now, we have got all the coefficients (noted as $\pmb{w}^*$) of the polynomial function by minimizing the error function (2). Therefore, we can evaluate the residual value of $E(\pmb{w}^*)$ given by (2) for the training and testing data. However, it is sometimes more convenient to use the *root-mean-square* (RMS) error defined by
$$
E_{RMS} = \sqrt{2E(\pmb{w}^*)/N}
\tag{3}
$$
in which the division by $N$ allows us to compare different sizes of data sets on an equal footing, also with the same scale (and in the same units) as the target variable $y$.

In order to evaluate the polynomial function we obtained, we do the following:
* Calculate and print the RMS error on training data based on equation (3),
* Calculate and print the RMS error on testing data based on equation (3),
* Get and print the coefficients of the trained function $\omega_0$ ~ $\omega_M$ using [`LinearRegression.coef_`](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LinearRegression.html#sklearn.linear_model.LinearRegression).

```python
prediction_train = model.predict(feature_matrix)
rms_train = np.sqrt(np.sum((prediction_train - y_train) ** 2) / len(prediction_train))

feature_matrix_test = np.ones_like(x_test)
for i in range(1, M+1):
    feature_matrix_test = np.concatenate((feature_matrix_test, x_test ** i), axis=1)

prediction_test = model.predict(feature_matrix_test)
rms_test = np.sqrt(np.sum((prediction_test - y_test) ** 2) / len(prediction_test))

print('rms error on training data: ', rms_train)
print('rms error on testing data: ', rms_test)
print('coefficients of the trained function: ', model.coef_)
```

    rms error on training data:  0.16066897173745565
    rms error on testing data:  0.30943910311337264
    coefficients of the trained function:  [[ -0.16311694  10.9380811  -30.08577056  19.11555865]]

### Part 2.2. Method based on Gradient Descent Algorithm
Once we get the closed-form solution, which is also the optimal solution, the next thing we are supposed to do is to re-solve the problem by gradient descent algorithm, because gradient descent matters a lot in practical applications since we cannot have a closed-form solution for most of the practical problems. Generally, the ingredients make up a gradient descent algorithm are:
* defining the learning rate to update our parameters,
* defining a loss function or objective function which you should minimize (here we use the error function based on equation (2)),
* calculating the gradients of parameters w.r.t. the loss,
* update parameters according to the learning rate and gradients to a direction that makes the loss smaller.

So, let's do this in that way.

```python
# defining a learning rate, you could change it
learning_rate = 0.1

# initialize the parameters, here we simply assign each parameter to 1
# becuase M=3, so we totally have 4 parameters (w0, w1, w2, w3)
W = np.ones((M+1, 1))

# calculate the feature matrix as before, in order to compute the loss
feature_matrix = np.ones_like(x_train)
for i in range(1, 4):
    feature_matrix = np.concatenate((feature_matrix, x_train ** i), axis=1)

# iteratively do gradient descent, here we just iterate it 50 times
for i in range(50):
    # calculate the values of the polynomial function
    F = np.matmul(feature_matrix, W)

    # calculate the loss, print it every 10 iterations
    if i % 10 ==0:
        loss = 0.5 * np.sum((F - y_train) ** 2)
        print('loss: ', loss)

    # calcualte the gradients of the parameters
    # here we have 10 data samples, so we calculate the mean.
    # please understand this formulation by inferencing the gradients
    # on your own with what you learnt in undergraduate school.
    G = np.mean((F - y_train) * feature_matrix, axis=0).reshape(-1, 1)

    # update parameters to a direction that makes the loss smaller
    # so we use '-' here, it is also why we call gradient "descent"
    W = W - learning_rate * G
```

    loss:  34.380066478803926
    loss:  4.012740605942455
    loss:  2.436307391471592
    loss:  1.958278267953077
    loss:  1.6626874106866916

Again, we show the original training data points and the draw the curve of the function obtained on a single figure.

```python
# your code here

x_curve = np.linspace(0, 1, 10000).reshape(-1, 1)
features_curve = np.ones_like(x_curve)
for i in range(1, M+1):
    features_curve = np.concatenate((features_curve, x_curve ** i), axis=1)

y_curve = np.matmul(features_curve, W)
plt.plot(x_train, y_train, marker='o', linestyle='None')
plt.plot(x_curve, y_curve, 'r')
plt.show()
```

--------------------2 points--------------------

Also, in order to evaluate the polynomial function we obtained, we do the following:
* Calculate and print the RMS error on training data based on equation (3),
* Calculate and print the RMS error on testing data based on equation (3),
* Print the coefficients (parameters).

```python
# your code here
prediction_train = np.matmul(feature_matrix, W)
rms_train = np.sqrt(np.sum((prediction_train - y_train) ** 2) / len(prediction_train))

feature_matrix_test = np.ones_like(x_test)

for i in range(1, M+1):
    feature_matrix_test = np.concatenate((feature_matrix_test, x_test ** i), axis=1)

prediction_test = np.matmul(feature_matrix_test, W)
rms_test = np.sqrt(np.sum((prediction_test - y_test) ** 2) / len(prediction_test))

coeff = W.reshape(1,-1)

print('rms error on training data: ',rms_train )
print('rms error on testing data: ',rms_test )
print('coefficients of the trained function: ',coeff)
```

    rms error on training data:  0.5422276467732126
    rms error on testing data:  0.5918422558573816
    coefficients of the trained function:  [[ 0.23861297 -0.26124322 -0.24765614 -0.13137266]]

--------------------3 points--------------------

#### Question 1. What's the difference between closed-form method and gradient descent method?

#### Your answer:

**Closed-form method** is a procedure to solve the model parameters analytically, which works better in small dataset. It is accurate.

**Gradient descent method** is a procedure to estimate optimal parameters of function, which works better in big dataset.

--------------------3.5 points--------------------

### Part 2.3. Model selection with closed-form solution
There remains the problem of choosing the order $M$ of the polynomial, this is actually an example of the concept *model selection*. To see how changing $M$ affects the regression problem being solved, please change $M=$ from 1 to 9, and get the following objectives based on **closed-form analysis**:
* Draw all the 9 figures as done above (make sure the function curve and training data points are in the same figure for each case, using plt.subplots),
* Calculate all the error values using equation (3) with respect to trainig and testing data for all the 9 cases, and plot this values on a single figure (that means 18 points would be drawn in the figure, with x axis representing $M$ and y axis representing the error value, please use two colors to distinguish training and testing),
* Print the coefficients for the 9 polynomial functions.

```python
# your code here
fig, ax = plt.subplots(3, 3, figsize=(15, 15))

# Define Feature matrix, X-curve and test matrix
feature_matrix_Model = np.ones_like(x_train)
x_curve = np.linspace(0, 1, 10000).reshape(-1, 1)
features_curve = np.ones_like(x_curve)

feature_matrix_test = np.ones_like(x_test)

# Define list to store train rms and test rms
train_rms = []
test_rms = []

for M in range(1, 10):

    # Define matrix and model along with M value
    feature_matrix_Model = np.concatenate((feature_matrix_Model, x_train ** M), axis=1)
    model = LinearRegression(fit_intercept=False).fit(feature_matrix_Model, y_train)
    features_curve = np.concatenate((features_curve, x_curve ** M), axis=1)

    # Define feature test matrix
    feature_matrix_test = np.concatenate((feature_matrix_test, x_test ** i), axis=1)

    # Use model to calculate y_curve
    y_curve = model.predict(features_curve)

    # Plot all curves and dots
    ax[(M-1)//3][(M-1)%3].plot(x_curve, y_curve)
    ax[(M-1)//3][(M-1)%3].plot(x_train, y_train, marker='o', linestyle='None')

    # Calculate rms for train and test data
    prediction_train = model.predict(feature_matrix_Model)
    rms_train = np.sqrt(np.sum((prediction_train - y_train) ** 2) / len(prediction_train))

    prediction_test = model.predict(feature_matrix_test)
    rms_test = np.sqrt(np.sum((prediction_test - y_test) ** 2) / len(prediction_test))

    # Record rms value
    train_rms.append(rms_train)
    test_rms.append(rms_test)

    print('\n Results when M = %d' % M)
    print('rms error on training / testing data: %f / %f' % (rms_train, rms_test))
    print('coefficients of the trained function: ', model.coef_)

fig.tight_layout()
plt.show()

M_x = np.linspace(1, 9, 9)
# draw the 18 points here
plt.plot(M_x, train_rms, 'ro',label='Train RMS error')
plt.plot(M_x, test_rms, 'bo',label='Test RMS error')
plt.legend(loc='upper right')
```

--------------------5.5 points--------------------

In fact, when $M = 9$, the 10 training points would be exactly fitted, and the training loss becomes zero, since the degree of freedom for this regression problem is 10. However, the fitted curve oscillates wildly and gives a very poor representation of the function $sin(2\pi x)$. This behavior is known as **over-fitting**.
#### Question 2. Which M is the best one for this probelm and why? Why the over-fitting happens when $M$ is big?

#### Your answer:

**2** is the best value for $M$. When $M$ is 2, we can get smallest rms error for test sets than other value.

When $M$ is bigger, the polynomial function is more and more suitable for training sets which is unable to reflect the universality of data.

--------------------6.5 points--------------------

### Part 2.4. Gradient descent practice
Set $M=9$, based on **gradient descent algorithm**, do the following (During training, print the loss every $n$ iterations to show you are running the gradient descent algorithm to train the model, you could select a $n$ as you like):
* Change learning rate and see how fast the training converges,
* Change number of iteration to see whether the training loss would be smaller,
* Draw the figure as done above when you feel the training is good enough (make sure the function curve and training data points are in the same figure),
* Calculate and print the error values using equation (3) with respect to trainig and testing data,
* Print the coefficients (parameters) for the polynomial function.

```python
# your code here
# defining a learning rate
learning_rate = 0.80

# initialize the parameters
M = 9
W = np.ones((M+1, 1))

# calculate the feature matrix as before, in order to compute the loss
feature_matrix = np.ones_like(x_train)
for i in range(1, M+1):
    feature_matrix = np.concatenate((feature_matrix, x_train ** i), axis=1)

# iteratively do gradient descent
iterations = 100
for i in range(iterations):

    # calculate the values of the polynomial function
    F = np.matmul(feature_matrix, W)

    # calculate the loss and print it
    if i % 10 ==0:
        loss = 0.5 * np.sum((F - y_train) ** 2)
        print('loss: ', loss)

    # calcualte the gradients of the parameters
    G = np.mean((F - y_train) * feature_matrix, axis=0).reshape(-1, 1)

    # update parameters to a direction that makes the loss smaller
    W = W - learning_rate * G
```

    loss:  108.9576867114645
    loss:  1.3277842162748286
    loss:  0.9356622923098687
    loss:  0.8633900656938968
    loss:  0.8213955911783352
    loss:  0.7940135285020612
    loss:  0.7740305795276011
    loss:  0.7579693551248701
    loss:  0.7441417749268406
    loss:  0.7317178277101527

Draw the figure

```python
x_curve = np.linspace(0, 1, 10000).reshape(-1, 1)
features_curve = np.ones_like(x_curve)
for i in range(1, M+1):
    features_curve = np.concatenate((features_curve, x_curve ** i), axis=1)

y_curve = np.matmul(features_curve, W)
plt.plot(x_train, y_train, marker='o', linestyle='None')
plt.plot(x_curve, y_curve, 'r')
plt.show()
```

```python
prediction_train = np.matmul(feature_matrix, W)
rms_train = np.sqrt(np.sum((prediction_train - y_train) ** 2) / len(prediction_train))

feature_matrix_test = np.ones_like(x_test)

for i in range(1, M+1):
    feature_matrix_test = np.concatenate((feature_matrix_test, x_test ** i), axis=1)

prediction_test = np.matmul(feature_matrix_test, W)
rms_test = np.sqrt(np.sum((prediction_test - y_test) ** 2) / len(prediction_test))

coeff = W.reshape(1,-1)

print('rms error on training data: ',rms_train )
print('rms error on testing data: ',rms_test )
print('coefficients of the trained function: ',coeff)
```

    rms error on training data:  0.37954700678755726
    rms error on testing data:  0.4004883084519169
    coefficients of the trained function:  [[ 0.70587409 -0.26760124 -1.17617027 -0.94815637 -0.52156432 -0.12875747
       0.18808766  0.43479006  0.62635021  0.77666943]]

--------------------7.5 points--------------------

#### Question 3. Actually, when using gradient descent algorithm, even in the case that $M=9$, it is hard to observe the over-fitting phenomenon. It is because it's difficult to reach the theoretically optimal solution that makes the training loss zero. Regardless of over-fitting, please give your idea to make the training loss smaller when using gradient descent algorithm in this case.

#### Your answer:

A suitable learning rate and bigger iterations will help us to get a better value. Multiple iterations can always help us close to the local optimal value, but we still need to consider about time consuming.

--------------------8 points--------------------

## Part 3. Linear regression with polynomials with regularization
Before introducing regularization, let's think about a way to solve the over-fitting problem.

Definitely, through the experiments done above, you somewhat have found that the most direct way is reducing $M$. However, what if we use more training data?

### Part 3.1. Re-visit polynomial regression with extra training data and $M=9$
Finish this step as in Part 2.1 based on closed-form analysis. (*note*: please calculate the RMS error on the original training and testing data)

```python
with open('data/extradata.pickle', 'rb') as f:
    extra_data = pickle.load(f)

x_extra = extra_data['x_extra']
y_extra = extra_data['y_extra']

x_extra_train = np.concatenate((x_train, x_extra), axis=0)
y_extra_train = np.concatenate((y_train, y_extra), axis=0)

# your code here
feature_matrix = np.ones_like(x_extra_train)
M = 9
for i in range(1, M+1):
    feature_matrix = np.concatenate((feature_matrix, x_extra_train ** i), axis=1)

from sklearn.linear_model import LinearRegression

model = LinearRegression(fit_intercept=False).fit(feature_matrix, y_extra_train)

x_curve = np.linspace(0, 1, 10000).reshape(-1, 1)
features_curve = np.ones_like(x_curve)
for i in range(1, M+1):
    features_curve = np.concatenate((features_curve, x_curve ** i), axis=1)

y_curve = model.predict(features_curve)
plt.plot(x_train, y_train, marker='o', linestyle='None')
plt.plot(x_curve, y_curve, 'r')
plt.show()
```

```python
# In order to use original data, we redefine feature matrix
# your code here
feature_matrix = np.ones_like(x_train)
M = 9
for i in range(1, M+1):
    feature_matrix = np.concatenate((feature_matrix, x_train ** i), axis=1)

prediction_train = model.predict(feature_matrix)
rms_train = np.sqrt(np.sum((prediction_train - y_train) ** 2) / len(prediction_train))

feature_matrix_test = np.ones_like(x_test)
for i in range(1, M+1):
    feature_matrix_test = np.concatenate((feature_matrix_test, x_test ** i), axis=1)

prediction_test = model.predict(feature_matrix_test)
rms_test = np.sqrt(np.sum((prediction_test - y_test) ** 2) / len(prediction_test))

print('rms error on training data: ', rms_train)
print('rms error on testing data: ', rms_test)
print('coefficients of the trained function: ', model.coef_)
```

    rms error on training data:  0.12703545085564324
    rms error on testing data:  0.29462627148498843
    coefficients of the trained function:  [[-1.83865514e-02  2.07504406e+01 -3.11916156e+02  2.43682755e+03
      -9.77774964e+03  2.15345964e+04 -2.71503685e+04  1.93217142e+04
      -7.04728868e+03  9.73155968e+02]]

--------------------9 points--------------------

#### Question 4. Does introducing more training data in this case alleviate over-fitting? Why?

#### Your answer:

Yes. More training data can help parameters in model become more universal, which can find distribution of data set instead of the function of several data points.

--------------------10 points--------------------

### Part 3.2. Re-visit polynomial regression with regularization and $M=9$
We may alleviate the over-fitting problem through introducing more training data, but what if there is no extra training data? In fact, we donnot want to limit the number of parameters in a model according to the size of the available training set, because it also limits the capacity and flexibility of the model. One technique that is often used to control over the over-fitting phenomenon in such cases is that of **regularization**, which involves adding a penalty term to the error function (2) in order to discourage the coefficients from reaching large values. The simplest such penalty term takes the form of a sum of squares of all of the coefficients, leading to a modified error function:
$$
\widetilde{E}(\pmb{w})=\frac{1}{2}\sum_{i=1}^{N}\{f(x_i, \pmb{w}) - y_i\}^2 + \frac{\lambda}{2}||\pmb{w}||^2
\tag{4}
$$
where $||\pmb{w}||^2\equiv \pmb{w}^T\pmb{w} = \omega_0^2 + \omega_1^2 + ... + \omega_M^2$, and the coefficient $\lambda$ governs the relative importance of the regularization term compared with the sum-of-squares error term.

Again, this error function can be minimized exactly in closed form. The regression with such a quadratic regularizer is called *ridge regression*. In the context of neural networks, this approcah is known as **weight decay**.

Apart from training and testing data, we also need another data set to test how suitable the $\lambda$ is before we use the model on the testing data. We call this data set **validation** set. Usually, when we have large amount of training data, we create a validation set splitted from the original training data and use the rest of the training data as the new training data. In our case, we only have the training data consisting of 10 data points, so here we use the extra data on Part 3.1 as the validation data.

Please solve the ridge regression problem with the polynomial function using **training** data in the following 7 conditions:
* $M=9$, $ln\lambda=-35$ ($\lambda=e^{-35}$)
* $M=9$, $ln\lambda=-25$
* $M=9$, $ln\lambda=-20$
* $M=9$, $ln\lambda=-15$
* $M=9$, $ln\lambda=-10$
* $M=9$, $ln\lambda=-5$
* $M=9$, $ln\lambda=0$

with the following tasks:
1. Calculate the root-mean-square error based on (3) (not the modified one) on the **training** and **validation** set. Plot the curves of RMS error versus $\ln\lambda$ for training and validation set in a single figure (i.e., in this figure, two curves will be plotted, one for training data and one for validation data. x axis represents the value of $\ln\lambda$ and y axis represents RMS error).
2. Print the coefficients for the obtained 7 ploynomial functions.
3. Find the best value for $\lambda$ among these conditions (`best` means its corresponding RMS on the **validation** set is the smallest), calculate the corresponding RMS error on **testing** data and print it.
4. Draw the curve of the polynomial function in a figure w.r.t the best $\lambda$, in which the 10 original training data points and the 100 testing data points should also be plotted using different colors.

To sovle the ridge regression problem, we could use [`sklearn.linear_model.Ridge`](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html).

```python
# your code here (task 1 & 2)
import math
feature_matrix = np.ones_like(x_train)
for i in range(1, 10):
    feature_matrix = np.concatenate((feature_matrix, x_train ** i), axis=1)

feature_matrix_valid = np.ones_like(x_extra)
for i in range(1, 10):
    feature_matrix_valid = np.concatenate((feature_matrix_valid, x_extra ** i), axis=1)

from sklearn.linear_model import Ridge

rmss_train = []
rmss_valid = []
best_model = None
best_valid = np.finfo(float).max
M = 9

ln_lambdas = [-35, -25, -20, -15, -10, -5, 0]

for ln_lambda in ln_lambdas:

    model = Ridge(math.exp(ln_lambda)/2).fit(feature_matrix, y_train)

    prediction_train = model.predict(feature_matrix)
    rms_train = np.sqrt(np.sum((prediction_train - y_train) ** 2) / len(prediction_train))

    prediction_valid = model.predict(feature_matrix_valid)
    rms_valid = np.sqrt(np.sum((prediction_valid - y_extra) ** 2) / len(prediction_valid))

    rmss_train.append(rms_train)
    rmss_valid.append(rms_valid)

    print('\n Results when M = %d' % M)
    print('coefficients of the trained function: ', model.coef_)

    if rms_valid < best_valid:
        best_valid = rms_valid
        best_model = model

plt.plot(ln_lambdas, rmss_train, 'b', marker='o',label='Train RMS')
plt.plot(ln_lambdas, rmss_valid, 'r', marker='o',label='Valid RMS')
plt.legend(loc='upper right')
plt.xlabel('ln(lambda)')
plt.ylabel('RMS error')
plt.show()
print(rmss_valid)
```

--------------------12 points--------------------

```python
# (task 3) select the best model, print the rms error on testing data

feature_matrix_test = np.ones_like(x_test)
for i in range(1, 10):
    feature_matrix_test = np.concatenate((feature_matrix_test, x_test ** i), axis=1)

prediction_test = best_model.predict(feature_matrix_test)
rms_test = np.sqrt(np.sum((prediction_test - y_test) ** 2) / len(prediction_test))
print('rms error on testing data: ', rms_test)

# (task 4) drawing
x_curve = np.linspace(0, 1, 10000).reshape(-1, 1)
features_curve = np.ones_like(x_curve)
for i in range(1, 10):
    features_curve = np.concatenate((features_curve, x_curve ** i), axis=1)

y_curve = best_model.predict(features_curve)

plt.plot(x_train, y_train, marker='o', linestyle='None',label='Train Data')
plt.plot(x_test, y_test, fillstyle='none', marker='o', color='g', linestyle='None',label='Test Data')
plt.legend(loc='upper right')
plt.plot(x_curve, y_curve, 'r')
plt.show()
```

--------------------12.5 points--------------------

#### Question 5. Does regularization alleviate over-fitting? Why?

#### Your answer:

Yes. We can find that over-fitting problem will happen when $M$ is big. After regularization, the number of parameters, $M$, is also considered, which means a big $M$ will increase loss. So it alleviates over-fitting to certain extent.

--------------------13.5 points--------------------

### Part 3.3. Gradient descent practice
Choose a $\lambda$ as you like and set $M=9$, calculate and print its rms error on training, validation and testing data respectively, draw the function curve with all the training and testing data on it, based on **gradient descent algorithm**.

***Note***: the loss function is equation (4), when calculating the gradients, remember to also consider the regularizatin component.

```python
# your code here
# defining a learning rate
M = 9
learning_rate = 0.2
lmda = -25

# initialize the parameters
W = np.ones((M+1, 1))

# calculate the feature matrix as before, in order to compute the loss
feature_matrix = np.ones_like(x_train)
for i in range(1, M+1):
    feature_matrix = np.concatenate((feature_matrix, x_train ** i), axis=1)

# iteratively do gradient descent
for i in range(50):
    # calculate the values of the polynomial function
    F = np.matmul(feature_matrix, W)

    # calculate the loss, print it every 10 iterations
    if i % 10 ==0:
        loss = 0.5 * np.sum((F - y_train) ** 2)+ math.exp(ln_lambda)/2*sum(W*W)
        print('loss: ', loss)

    # calcualte the gradients of the parameters, considering regularization
    G = np.mean((F - y_train) * feature_matrix, axis=0).reshape(-1, 1)

    # update parameters to a direction that makes the loss smaller
    W = W - learning_rate * G

# draw the figure and print all the information

# Calculate train RMS Error
prediction_train =  np.matmul(feature_matrix, W)
rms_train = np.sqrt(np.sum((prediction_train - y_train) ** 2) / len(prediction_train))

# Calculate valid RMS Error
prediction_valid = np.matmul(feature_matrix_valid, W)
rms_valid = np.sqrt(np.sum((prediction_valid - y_extra) ** 2) / len(prediction_valid))

# Calculate test RMS Error
prediction_test = np.matmul(feature_matrix_test, W)
rms_test = np.sqrt(np.sum((prediction_test - y_test) ** 2) / len(prediction_test))

# Print RMS Error
print('Train RMS Error:', rms_train)
print('Valid RMS Error:', rms_valid)
print('Test RMS Error:', rms_test)

x_curve = np.linspace(0, 1, 10000).reshape(-1, 1)
features_curve = np.ones_like(x_curve)
for i in range(1, M+1):
    features_curve = np.concatenate((features_curve, x_curve ** i), axis=1)

y_curve = np.matmul(features_curve, W)
plt.plot(x_train, y_train, marker='o', linestyle='None',label='Train Data')
plt.plot(x_test, y_test, fillstyle='none', marker='o', color='g', linestyle='None',label='Test Data')
plt.legend(loc='upper right')
plt.plot(x_curve, y_curve, 'r')
plt.show()
```

--------------------15 points--------------------


# Cosmic Star Formation Rate

Figure 9.6 from Chapter 9 of *Interstellar and Intergalactic Medium* by Ryden & Pogge, 2021,
Cambridge University Press.

Plot of the cosmic star formation rate in units of solar masses per year per comoving cubic megaparsec.

Uses the parametric fit of [Madau & Dickinson 2014, ARA&A, 52, 415](https://ui.adsabs.harvard.edu/abs/2014ARA%26A..52..415M),  Equation 15.

```python
%matplotlib inline

import math
import numpy as np
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.ticker import MultipleLocator, LogLocator, NullFormatter

import warnings
warnings.filterwarnings('ignore',category=UserWarning, append=True)
warnings.filterwarnings('ignore',category=RuntimeWarning, append=True)
```

## Standard Plot Format

Setup the standard plotting format and make the plot. Fonts and resolution adopted follow CUP style.

```python
figName = 'Fig9_6'

# graphic aspect ratio = width/height

aspect = 4.0/3.0 # 4:3

# Text width in inches - don't change, this is defined by the print layout

textWidth = 6.0 # inches

# output format and resolution

figFmt = 'png'
dpi = 600

# Graphic dimensions

plotWidth = dpi*textWidth
plotHeight = plotWidth/aspect
axisFontSize = 10
labelFontSize = 6
lwidth = 0.5
axisPad = 5
wInches = textWidth
hInches = wInches/aspect

# Plot filename

plotFile = f'{figName}.{figFmt}'

# LaTeX is used throughout for markup of symbols, Times-Roman serif font

plt.rc('text', usetex=True)
plt.rc('font', **{'family':'serif','serif':['Times-Roman'],'weight':'bold','size':'16'})

# Font and line weight defaults for axes

matplotlib.rc('axes',linewidth=lwidth)
matplotlib.rcParams.update({'font.size':axisFontSize})

# axis and label padding

plt.rcParams['xtick.major.pad'] = f'{axisPad}'
plt.rcParams['ytick.major.pad'] = f'{axisPad}'
plt.rcParams['axes.labelpad'] = f'{axisPad}'
```

## Cosmic Star Formation Rate

Compute the cosmic SFR using the parametric fit of Madau & Dickinson 2014, equation 15:
\begin{equation}
   \psi(z) = 0.015\frac{(1+z)^{2.7}}{1+\left[(1+z)/2.9\right]^{5.6}} {\rm M_{\odot} yr^{-1} Mpc^{-3}}
\end{equation}

```python
z = np.linspace(0.0,9.0,501)
sfr = 0.015*((1+z)**2.7)/(1+((1+z)/2.9)**5.6)

# plotting limits

xMin = -0.5
xMax = 9.0

yMin = 0.005
yMax = 0.2
```

## Make the plot

```python
fig,ax = plt.subplots()

fig.set_dpi(dpi)
fig.set_size_inches(wInches,hInches,forward=True)

ax.tick_params('both',length=6,width=lwidth,which='major',direction='in',top='on',right='on')
ax.tick_params('both',length=3,width=lwidth,which='minor',direction='in',top='on',right='on')

# Limits

plt.xlim(xMin,xMax)
ax.xaxis.set_major_locator(MultipleLocator(2))
ax.xaxis.set_minor_locator(MultipleLocator(0.5))
plt.xlabel(r'z',fontsize=axisFontSize)

plt.ylim(yMin,yMax)
ax.set_yscale('log')
ax.set_yticks([0.01,0.1])
ax.set_yticklabels(['0.01','0.1'])
plt.ylabel(r'R$_{\rm sf}$/V$_0$ [M$_{\odot}$ yr$^{-1}$ Mpc$^{-3}$]',fontsize=axisFontSize)

plt.plot(z,sfr,'-',color='black',lw=1.0,zorder=10)

plt.plot()
plt.savefig(plotFile,bbox_inches='tight',facecolor='white')
```


This notebook is part of https://github.com/AudioSceneDescriptionFormat/splines, see also https://splines.readthedocs.io/.

[back to Euclidean splines](index.ipynb)

# Piecewise Monotone Interpolation

When interpolating
a sequence of one-dimensional data points,
it is sometimes desirable to limit the interpolant
between any two adjacent data points
to a monotone function.
This makes sure that there are no overshoots beyond the given data points.
In other words, if the data points are within certain bounds,
all interpolated data will also be within those same bounds.
It follows that if all data points are non-negative,
interpolated data will be non-negative as well.
Furthermore, this makes sure that monotone data leads to a monotone interpolant
(see also [Monotone Interpolation](#Monotone-Interpolation) below).

A Python implementation of
piecewise monotone one-dimensional cubic splines
is available in the
[splines.PiecewiseMonotoneCubic](../python-module/splines.rst#splines.PiecewiseMonotoneCubic)
class.

The SciPy package provides a similar tool with the
[pchip_interpolate()](https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.pchip_interpolate.html)
function and the
[PchipInterpolator](https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.PchipInterpolator.html)
class (see below for more details).

The 3D animation software [Blender](https://www.blender.org)
provides an
[Auto Clamped](https://docs.blender.org/manual/en/dev/editors/graph_editor/fcurves/properties.html#editors-graph-fcurves-settings-handles)
property for creating piecewise monotone animation cuves.

## Examples

```python
import matplotlib.pyplot as plt
import numpy as np
```

```python
import splines
```

We use a few helper functions from [helper.py](helper.py)
for plotting:

```python
from helper import plot_spline_1d, grid_lines
```

```python
values = 0, 3, 3, 7
times = 0, 3, 8, 10, 11
```

Let's compare a piecewise monotone spline with
a [Catmull--Rom spline](catmull-rom.ipynb)
and a [natural spline](natural.ipynb):

```python
plot_spline_1d(
    splines.PiecewiseMonotoneCubic(values, times, closed=True),
    label='piecewise monotone')
plot_spline_1d(
    splines.CatmullRom(values, times, endconditions='closed'),
    label='Catmull–Rom')
plot_spline_1d(
    splines.Natural(values, times, endconditions='closed'),
    label='natural spline')
plt.legend()
grid_lines(times)
```

```python
def plot_piecewise_monotone(*args, **kwargs):
    s = splines.PiecewiseMonotoneCubic(*args, **kwargs)
    plot_spline_1d(s)
    grid_lines(x=s.grid)
```

```python
plot_piecewise_monotone([0, 1, 3, 2, 1])
```

### Providing Slopes

By default, appropriate slopes are calculated automatically.
However, those slopes can be overridden if desired.
Specifying `None` falls back to the auto-generated default.

```python
plot_piecewise_monotone([0, 1, 3, 2, 1], slopes=[None, 0, None, -3, -1.5])
```

Slopes that would lead to non-monotone segments are prohibited:

```python
try:
    plot_piecewise_monotone([0, 1, 3, 2, 1], slopes=[None, 4, None, None, None])
except Exception as e:
    print(e)
    assert 'too steep' in str(e)
else:
    assert False
```

## Generating and Modifying the Slopes at Segment Boundaries

> In this paper we derive necessary and sufficient conditions for a cubic to be
monotone in an interval. These conditions are then used to develop an algorithm which
constructs a $\mathscr{C}^1$ monotone piecewise cubic interpolant to monotone data. The curve
produced contains no extraneous "bumps" or "wiggles", which makes it more readily
acceptable to scientists and engineers.
>
> ---(Fritsch and Carlson 1980), section 1: "Introduction"

(Fritsch and Carlson 1980)
derives necessary and sufficient conditions
for a cubic curve segment to be monotone,
based on the slopes of the secant lines (i.e. the piecewise linear interpolant)
and their endpoint derivatives.
Furthermore, they provide a two-step algorithm
to generate piecewise monotone cubics:

1. calculate initial tangents (with whatever method)
2. tweak the ones that don't fulfill the monotonicity conditions

> To implement Step 1 we have found the standard three-point difference formula to
be satisfactory for $d_2$, $d_3$, $\cdots$, $d_{n-1}$.
>
> ---(Fritsch and Carlson 1980), section 4: "Monotone piecewise cubic interpolation algorithm"

> This is what de Boor [1978, p. 53] calls cubic Bessel interpolation, in which
the interior derivatives are set using the standard three point difference formula.
>
> ---(Fritsch and Carlson 1980), section 5: "Numerical examples"

In the 2001 edition of
(de Boor 1978),
*piecewise cubic Bessel interpolation* is defined on page 42.
We have already seen the *standard three-point difference* in the
[tangent vectors of non-uniform Catmull--Rom splines](catmull-rom-non-uniform.ipynb#Tangent-Vectors).

For the following equations, we define the slope of the secant lines as

\begin{equation*}
S_i =
\frac{x_{i+1} - x_i}{t_{i+1} - t_i}.
\end{equation*}

We use $x_i$ to represent the given data points and
and $t_i$ to represent the corresponding parameter values.
The slope at those values is represented by $\dot{x}_i$.

Note

In the literature, the parameter values are often represented by $x_i$,
so try not to be confused!

Based on (Fritsch and Carlson 1980),
(Dougherty et al. 1989)
provides (in equation 4.2) an algorithm for modifying the initial slopes
to ensure monotonicity.
Adapted to our notation, it looks like this:

\begin{equation*}
\dot{x}_i \leftarrow
\begin{cases}
\min(\max(0, \dot{x}_i), 3 \min(|S_{i-1}|, |S_i|)), & \sigma_i > 0,\\
\max(\min(0, \dot{x}_i), -3 \min(|S_{i-1}|, |S_i|)), & \sigma_i < 0,\\
0, & \sigma_i=0,
\end{cases}
\end{equation*}

where $\sigma_i = \operatorname{sgn}(S_i)$ if $S_i S_{i-1} > 0$ and $\sigma_i=0$ otherwise.

This algoritm is implemented in the
[splines.PiecewiseMonotoneCubic](../python-module/splines.rst#splines.PiecewiseMonotoneCubic)
class.

## PCHIP/PCHIM

A different approach for obtaining slopes that ensure monotonicity
is described in
(Fritsch and Butland 1984),
equation (5):

\begin{equation*}
G(S_1, S_2, h_1, h_2) =
\begin{cases}
\frac{S_1 S_2}{\alpha S_2 + (1 - \alpha) S_1} \quad & \text{if } S_1 S_2 > 0,\\
0 & \text{otherwise,}
\end{cases}
\end{equation*}

where

\begin{equation*}
\alpha =
\frac{1}{3} \left(1 + \frac{h_2}{h_1 + h_2}\right) =
\frac{h_1 + 2 h_2}{3 (h_1 + h_2)}.
\end{equation*}

The function $G$ can be used to calculate the slopes at segment boundaries,
given the slopes $S_i$ of the neighboring secant lines
and the neighboring parameter intervals $h_i = t_{i+1} - t_i$.

Let's define this using [SymPy](https://www.sympy.org/) for later reference:

```python
import sympy as sp
```

```python
h1, h2 = sp.symbols('h1:3')
S1, S2 = sp.symbols('S1:3')
```

```python
alpha = (h1 + 2 * h2) / (3 * (h1 + h2))
G1 = (S1 * S2) / (alpha * S2 + (1 - alpha) * S1)
```

This has been implemented in a
[Fortran](https://en.wikipedia.org/wiki/Fortran) package described in
(Fritsch 1982),
which has coined the acronym PCHIP,
originally meaning *Piecewise Cubic Hermite Interpolation Package*.

> It features software to produce a monotone and "visually pleasing"
interpolant to monotone data.
>
> ---(Fritsch 1982)

The package contains many Fortran subroutines,
but the one that's relevant here is
`PCHIM`, which is short for
*Piecewise Cubic Hermite Interpolation to Monotone data*.

The source code (including some later modifications) is available at https://people.sc.fsu.edu/~jburkardt/f77_src/pchip/pchip.html.
This is the code snippet responsible for calculating the slopes:

```
C
C        USE BRODLIE MODIFICATION OF BUTLAND FORMULA.
C
   45    CONTINUE
         HSUMT3 = HSUM+HSUM+HSUM
         W1 = (HSUM + H1)/HSUMT3
         W2 = (HSUM + H2)/HSUMT3
         DMAX = MAX( ABS(DEL1), ABS(DEL2) )
         DMIN = MIN( ABS(DEL1), ABS(DEL2) )
         DRAT1 = DEL1/DMAX
         DRAT2 = DEL2/DMAX
         D(1,I) = DMIN/(W1*DRAT1 + W2*DRAT2)
```

This looks different from the function $G$ defined above,
but if we transform the Fortran code into math ...

```python
HSUM = h1 + h2
```

```python
W1 = (HSUM + h1) / (3 * HSUM)
W2 = (HSUM + h2) / (3 * HSUM)
```

... and use separate expressions
depending on which of the neighboring secant slopes is larger ...

```python
G2 = S1 / (W1 * S1 / S2 + W2 * S2 / S2)
G3 = S2 / (W1 * S1 / S1 + W2 * S2 / S1)
```

... we see that the two cases are mathematically equivalent ...

```python
assert sp.simplify(G2 - G3) == 0
```

... and they are in fact also equivalent
to the aforementioned equation from
(Fritsch and Butland 1984):

```python
assert sp.simplify(G1 - G2) == 0
```

Presumably, the Fortran code uses the larger one of the pair of secant slopes
in the denominator in order to reduce numerical errors
if one of the slopes is very close to zero.

Yet another variation of this theme is shown in
(Moler 2004), section 3.4,
which defines the slope $d_k$ as a weighted harmonic mean
of the two neighboring secant slopes:

\begin{equation*}
\frac{w_1 + w_2}{d_k} =
\frac{w_1}{\delta_{k-1}} +
\frac{w_2}{\delta_k},
\end{equation*}

with $w_1 = 2 h_k + h_{k-1}$ and $w_2 = h_k + 2 h_{k-1}$.
Using the notation from above, $d_k = \dot{x}_k$ and $\delta_k = S_k$.

Again, when defining this using SymPy ...

```python
w1 = 2 * h2 + h1
w2 = h2 + 2 * h1
```

```python
G4 = (w1 + w2) / (w1 / S1 + w2 / S2)
```

... we can see that it is actually equivalent to the previous equations:

```python
assert sp.simplify(G4 - G1) == 0
```

The `PCHIM` algorithm,
which is nowadays known by the less self-explanatory name PCHIP,
is available in the *SciPy* package in form of the
[pchip_interpolate()](https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.pchip_interpolate.html)
function and the
[PchipInterpolator](https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.PchipInterpolator.html)
class.

```python
from scipy.interpolate import PchipInterpolator
```

## More Examples

To illustrate the differences between the two approaches mentioned above,
let's plot a few examples.
Both methods are piecewise monotone,
but their exact shape is slightly different.
Decide for yourself which one is more "visually pleasing"!

```python
def compare_pchip(values, times):
    plot_times = np.linspace(times[0], times[-1], 100)
    plt.plot(
        plot_times,
        PchipInterpolator(times, values)(plot_times),
        label='PCHIP')
    plt.plot(
        plot_times,
        splines.PiecewiseMonotoneCubic(values, times).evaluate(plot_times),
        label='PiecewiseMonotoneCubic')
    plt.legend()
    grid_lines(x=times)
```

```python
compare_pchip([0, 0, 1.5, 4, 4], [-1, 0, 1, 8, 9])
```

```python
compare_pchip([0, 0, 1.5, 4, 4], [-1, 0, 6, 8, 9])
```

There is even a slight difference in the uniform case:

```python
compare_pchip([0, 0, 3.3, 4, 4], [-1, 0, 1, 2, 3])
```

```python
compare_pchip([0, 0, 0.7, 4, 4], [-1, 0, 1, 2, 3])
```

For differences at the beginning and the end of the curve,
see the [section about end conditions](#End-Conditions).

## Monotone Interpolation

When using the aforementioned piecewise monotone algorithms with monotone data,
the whole interpolant will be monotone.

The class
[splines.MonotoneCubic](../python-module/splines.rst#splines.MonotoneCubic)
works very much the same as
[splines.PiecewiseMonotoneCubic](../python-module/splines.rst#splines.PiecewiseMonotoneCubic),
except that it only allows monotone data values.

Since the resulting interpolation function is monotone,
it can be inverted.
Given a function value, the method
[.get_time()](../python-module/splines.rst#splines.MonotoneCubic.get_time)
can be used to find the associated parameter value.

```python
s = splines.MonotoneCubic([0, 2, 2, 6, 6], grid=[0, 2, 3, 6, 8])
```

```python
probes = 1, 3, 5
```

```python
fig, ax = plt.subplots()
plot_spline_1d(s)
ax.scatter(s.get_time(probes), probes)
grid_lines(x=s.grid)
```

If the solution is not unique (i.e. on plateaus),
the return value is `None`:

```python
assert s.get_time(2) is None
```

Closed curves are obviously not possible:

```python
try:
    splines.MonotoneCubic([0, 2, 2, 6, 6], closed=True)
except Exception as e:
    print(e)
    assert 'closed' in str(e)
else:
    assert False
```

## End Conditions

[The usual end conditions](end-conditions.ipynb)
don't necessarily lead to a monotone interpolant,
therefore we need to come up with custom end conditions
that preserve monotonicity.

> For the end derivatives, the noncentered three
point difference formula may be used, although it is sometimes necessary to modify $d_1$
and/or $d_n$ if the signs are not appropriate. In these cases we have obtained better results
setting $d_1$ or $d_n$ equal to zero, rather than equal to the slope of the secant line.
>
> ---(Fritsch and Carlson 1980), section 4: "Monotone piecewise cubic interpolation algorithm"

(Fritsch and Carlson 1980)
recommends using the *noncentered three point difference formula*,
however, it fails to mention what that actually is.
Luckily, we can have a look at the code at
https://people.sc.fsu.edu/~jburkardt/f77_src/pchip/pchip.html:

```
C
C  SET D(1) VIA NON-CENTERED THREE-POINT FORMULA, ADJUSTED TO BE
C     SHAPE-PRESERVING.
C
      HSUM = H1 + H2
      W1 = (H1 + HSUM)/HSUM
      W2 = -H1/HSUM
      D(1,1) = W1*DEL1 + W2*DEL2
      IF ( PCHST(D(1,1),DEL1) .LE. ZERO)  THEN
         D(1,1) = ZERO
      ELSE IF ( PCHST(DEL1,DEL2) .LT. ZERO)  THEN
C        NEED DO THIS CHECK ONLY IF MONOTONICITY SWITCHES.
         DMAX = THREE*DEL1
         IF (ABS(D(1,1)) .GT. ABS(DMAX))  D(1,1) = DMAX
      ENDIF
```

The function `PCHST` is a simple sign test:

```
      PCHST = SIGN(ONE,ARG1) * SIGN(ONE,ARG2)
      IF ((ARG1.EQ.ZERO) .OR. (ARG2.EQ.ZERO))  PCHST = ZERO
```

This implementation seems to be used by "modern" PCHIP/PCHIM implementations as well.

> This defines the `pchip` slopes at interior breakpoints,
but the slopes $d_1$ and $d_n$ at either end of the data interval
are determined by a slightly different,
one-sided analysis. The details are in `pchiptx.m`.
>
> ---(Moler 2004), section 3.4

Section 3.6 of (Moler 2004)
shows the implementation of `pchiptx.m`:

```octave
function d = pchipend(h1,h2,del1,del2)
%  Noncentered, shape-preserving, three-point formula.
    d = ((2*h1+h2)*del1 - h1*del2)/(h1+h2);
    if sign(d) ~= sign(del1)
        d = 0;
    elseif (sign(del1)~=sign(del2))&(abs(d)>abs(3*del1))
        d = 3*del1;
    end
```

Apparently, this is the same as the above Fortran implementation.

The class [scipy.interpolate.PchipInterpolator](https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.PchipInterpolator.html)
uses [the same implementation (ported to Python)](https://github.com/scipy/scipy/blob/v1.6.1/scipy/interpolate/_cubic.py#L237-L250).

This implementation ensures monotonicity,
but it might seem a bit strange that for calculating the first slope,
the second slope is not directly taken into account.

Another awkward property is that for calculating the inner slopes,
only the immediately neighboring secant slopes and time intervals are considered,
while for calculating the initial and final slopes,
both the neighboring segment and the one next to it are considered.
This makes the curve less locally controlled at the ends compared to the middle.

```python
def plot_pchip(values, grid, **kwargs):
    pchip = PchipInterpolator(grid, values)
    times = np.linspace(grid[0], grid[-1], 100)
    plt.plot(times, pchip(times), **kwargs)
    plt.scatter(grid, pchip(grid))
    grid_lines(x=grid)
```

```python
plot_pchip([0, 1, 0], [0, 1, 2])
plot_pchip([0, 1, 1], [0, 1, 2])
grid_lines([0, 1, 2])
```

```python
plot_pchip([0, 1, 0], [0, 1, 4])
plot_pchip([0, 1, 0], [0, 1, 1.5])
grid_lines([0, 1, 1.5, 4])
```

In both of the above examples,
the very left slope depends on properties of the very right segment.

The slope at $t = 1$ is clearly zero in both cases and apart from that fact,
the shape of the curve at $t > 1$ should, arguably,
not have any influence on the slope at $t = 0$.

To provide an alternative to this behavior, the class
[splines.PiecewiseMonotoneCubic](../python-module/splines.rst#splines.PiecewiseMonotoneCubic)
uses end conditions that depend on the slope at $t = 1$,
but not explicitly on the shape of the curve at $t > 1$:

```python
plot_piecewise_monotone([0, 1, 0], grid=[0, 1, 1.5])
plot_piecewise_monotone([0, 1, 0], grid=[0, 1, 4])
grid_lines(x=[0, 1, 1.5, 4])
```

The initial and final slopes of
[splines.PiecewiseMonotoneCubic](../python-module/splines.rst#splines.PiecewiseMonotoneCubic)
are implemented like this:

```python
def monotone_end_condition(inner_slope, secant_slope):
    if secant_slope < 0:
        return -monotone_end_condition(-inner_slope, -secant_slope)
    assert 0 <= inner_slope <= 3 * secant_slope
    if inner_slope <= secant_slope:
        return 3 * secant_slope - 2 * inner_slope
    else:
        return (3 * secant_slope - inner_slope) / 2
```

## Even More Examples

The following example plots show different slopes at the beginning and end
due to different end conditions.

```python
compare_pchip([1, 2, 1], [1, 3.5, 5])
```

```python
compare_pchip([1, 2, 3.5, 4, 3], [1, 1.5, 4, 5, 6])
```

```python
compare_pchip([1, 2, 1.9, 1], [1, 3, 4, 6])
```


```python
# ============================================================
# Notebook setup
# ============================================================

%load_ext autoreload
%autoreload 2

# Control figure size
interactive_figures = True
if interactive_figures:
    # Normal behavior
    %matplotlib widget
    figsize=(9, 3)
else:
    # PDF export behavior
    figsize=(14, 5)

from util import cmapss
import numpy as np
from matplotlib import pyplot as plt
import pandas as pd
from tensorflow import keras
from tensorflow.keras import layers, callbacks
from tensorflow_probability import distributions
from sklearn.metrics import r2_score

# Load data
data_folder = 'data'
data = cmapss.load_data(data_folder)

# Identify input columns
dt_in = list(data.columns[3:-1])

# Focus on a subset of the data
data_by_src = cmapss.split_by_field(data, field='src')
dt = data_by_src['train_FD004']

# Split training and test machines
tr_ratio = 0.75
np.random.seed(42)
machines = dt.machine.unique()
np.random.shuffle(machines)

sep = int(tr_ratio * len(machines))
tr_mcn = machines[:sep]
ts_mcn = machines[sep:]

tr, ts = cmapss.partition_by_machine(dt, tr_mcn)

# Standardization and normalization
trmean = tr[dt_in].mean()
trstd = tr[dt_in].std().replace(to_replace=0, value=1) # handle static fields
ts_s = ts.copy()
ts_s[dt_in] = (ts_s[dt_in] - trmean) / trstd
tr_s = tr.copy()
tr_s[dt_in] = (tr_s[dt_in] - trmean) / trstd

trmaxrul = tr['rul'].max()
ts_s['rul'] = ts['rul'] / trmaxrul
tr_s['rul'] = tr['rul'] / trmaxrul

# Define a cost model
failtimes = dt.groupby('machine')['cycle'].max()
safe_interval = failtimes.min()
maintenance_cost = failtimes.max()
cmodel = cmapss.RULCostModel(maintenance_cost=maintenance_cost, safe_interval=safe_interval)

# Sliding window input
wlen = 3
tr_sw, tr_sw_m, tr_sw_r = cmapss.sliding_window_by_machine(tr_s, wlen, dt_in)
ts_sw, ts_sw_m, ts_sw_r = cmapss.sliding_window_by_machine(ts_s, wlen, dt_in)
```

# RUL Prediction as Classification

## RUL Prediction as Classification

**RUL-based maintenance can also be tackled _using a classifier_**

* We build a classifier to determine whether a failure will occur in $\theta$ steps
* We stop as soon as the classifier outputs (say) a 0, i.e.

$$
f_\theta(x, \lambda) = 0
$$

* $f$ is the classifier, with parameter vector $\lambda$
* $\theta$ is the horizon for detecting a failure

**In a sense, we are trying to learn _directly_ a maintenance policy**

* The policy is the form "stop $\theta$ units before a failure"
* The classifier tries to learn it

## Classifier Architecture

**We can therefore immediately define our classifier architecture:**

```python
def build_classifier(hidden):
    input_shape = (len(dt_in), )
    model_in = keras.Input(shape=input_shape, dtype='float32')
    x = model_in
    for h in hidden:
        x = layers.Dense(h, activation='relu')(x)
    model_out = layers.Dense(1, activation='sigmoid')(x)
    model = keras.Model(model_in, model_out)
    return model
```

* Like in the regression case, we use a Multilayer Perceptron
* The only difference is the use of a sigmoid activation in the output layer
* For `hidden = []` we get Logistic Regression

## Training

**Before training, we need to define the classes**

In turn, this requires to define the detection horizon $\theta$:

```python
class_thr = 15
tr_lbl = (tr['rul'] >= class_thr)
ts_lbl = (ts['rul'] >= class_thr)
```

* The class is "1" if a failure is more than $\theta$ steps away
* The class if "0" otherwise

**Classification problems _tend to be easier_ than regression problems**

* On the other hand, learning the whole policy
* ...May be trickier than just estimating the RUL

## Training

**Let's start by training the simplest possible model**

```python
nn1 = build_classifier(hidden=[])
nn1.compile(optimizer='Adam', loss='binary_crossentropy')
cb = [callbacks.EarlyStopping(patience=10, restore_best_weights=True)]
history1 = nn1.fit(tr_s[dt_in], tr_lbl, validation_split=0.2,
                 callbacks=cb,
                 batch_size=32, epochs=20, verbose=1)
```

    2021-12-04 11:35:22.398388: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
    To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.

    Epoch 1/20
    1135/1135 [==============================] - 1s 785us/step - loss: 0.5272 - val_loss: 0.3904
    Epoch 2/20
    1135/1135 [==============================] - 1s 666us/step - loss: 0.3208 - val_loss: 0.2824
    Epoch 3/20
    1135/1135 [==============================] - 1s 662us/step - loss: 0.2486 - val_loss: 0.2430
    Epoch 4/20
    1135/1135 [==============================] - 1s 668us/step - loss: 0.2203 - val_loss: 0.2267
    Epoch 5/20
    1135/1135 [==============================] - 1s 665us/step - loss: 0.2071 - val_loss: 0.2180
    Epoch 6/20
    1135/1135 [==============================] - 1s 672us/step - loss: 0.1988 - val_loss: 0.2113
    Epoch 7/20
    1135/1135 [==============================] - 1s 662us/step - loss: 0.1925 - val_loss: 0.2052
    Epoch 8/20
    1135/1135 [==============================] - 1s 687us/step - loss: 0.1869 - val_loss: 0.2000
    Epoch 9/20
    1135/1135 [==============================] - 1s 691us/step - loss: 0.1820 - val_loss: 0.1952
    Epoch 10/20
    1135/1135 [==============================] - 1s 688us/step - loss: 0.1777 - val_loss: 0.1912
    Epoch 11/20
    1135/1135 [==============================] - 1s 650us/step - loss: 0.1739 - val_loss: 0.1879
    Epoch 12/20
    1135/1135 [==============================] - 1s 650us/step - loss: 0.1705 - val_loss: 0.1845
    Epoch 13/20
    1135/1135 [==============================] - 1s 641us/step - loss: 0.1673 - val_loss: 0.1819
    Epoch 14/20
    1135/1135 [==============================] - 1s 657us/step - loss: 0.1646 - val_loss: 0.1788
    Epoch 15/20
    1135/1135 [==============================] - 1s 644us/step - loss: 0.1621 - val_loss: 0.1761
    Epoch 16/20
    1135/1135 [==============================] - 1s 642us/step - loss: 0.1599 - val_loss: 0.1738
    Epoch 17/20
    1135/1135 [==============================] - 1s 647us/step - loss: 0.1576 - val_loss: 0.1720
    Epoch 18/20
    1135/1135 [==============================] - 1s 652us/step - loss: 0.1557 - val_loss: 0.1699
    Epoch 19/20
    1135/1135 [==============================] - 1s 688us/step - loss: 0.1540 - val_loss: 0.1681
    Epoch 20/20
    1135/1135 [==============================] - 1s 695us/step - loss: 0.1524 - val_loss: 0.1671

## Training

**Here's the loss evolution over time and its final value**

```python
cmapss.plot_training_history(history1, figsize=figsize)
tr1, vl1 = history1.history["loss"][-1], np.min(history1.history["val_loss"])
print(f'Final loss: {tr1:.4f} (training), {vl1:.4f} (validation)')
```

    Final loss: 0.1524 (training), 0.1671 (validation)

        Figure

## Training

**Let's try with a deeper model**

```python
nn2 = build_classifier(hidden=[32, 32])
nn2.compile(optimizer='Adam', loss='binary_crossentropy')
history2 = nn2.fit(tr_s[dt_in], tr_lbl, validation_split=0.2,
                 callbacks=cb,
                 batch_size=32, epochs=20, verbose=1)
```

    Epoch 1/20
    1135/1135 [==============================] - 1s 830us/step - loss: 0.2123 - val_loss: 0.1674
    Epoch 2/20
    1135/1135 [==============================] - 1s 760us/step - loss: 0.1322 - val_loss: 0.1287
    Epoch 3/20
    1135/1135 [==============================] - 1s 766us/step - loss: 0.1066 - val_loss: 0.1093
    Epoch 4/20
    1135/1135 [==============================] - 1s 753us/step - loss: 0.0913 - val_loss: 0.0820
    Epoch 5/20
    1135/1135 [==============================] - 1s 754us/step - loss: 0.0811 - val_loss: 0.0757
    Epoch 6/20
    1135/1135 [==============================] - 1s 751us/step - loss: 0.0761 - val_loss: 0.0731
    Epoch 7/20
    1135/1135 [==============================] - 1s 750us/step - loss: 0.0737 - val_loss: 0.0681
    Epoch 8/20
    1135/1135 [==============================] - 1s 801us/step - loss: 0.0716 - val_loss: 0.0629
    Epoch 9/20
    1135/1135 [==============================] - 1s 755us/step - loss: 0.0724 - val_loss: 0.0653
    Epoch 10/20
    1135/1135 [==============================] - 1s 751us/step - loss: 0.0704 - val_loss: 0.0690
    Epoch 11/20
    1135/1135 [==============================] - 1s 769us/step - loss: 0.0704 - val_loss: 0.0614
    Epoch 12/20
    1135/1135 [==============================] - 1s 759us/step - loss: 0.0696 - val_loss: 0.0737
    Epoch 13/20
    1135/1135 [==============================] - 1s 789us/step - loss: 0.0693 - val_loss: 0.0647
    Epoch 14/20
    1135/1135 [==============================] - 1s 749us/step - loss: 0.0688 - val_loss: 0.0712
    Epoch 15/20
    1135/1135 [==============================] - 1s 753us/step - loss: 0.0694 - val_loss: 0.0656
    Epoch 16/20
    1135/1135 [==============================] - 1s 751us/step - loss: 0.0700 - val_loss: 0.0646
    Epoch 17/20
    1135/1135 [==============================] - 1s 756us/step - loss: 0.0679 - val_loss: 0.0594
    Epoch 18/20
    1135/1135 [==============================] - 1s 759us/step - loss: 0.0677 - val_loss: 0.0575
    Epoch 19/20
    1135/1135 [==============================] - 1s 751us/step - loss: 0.0680 - val_loss: 0.0768
    Epoch 20/20
    1135/1135 [==============================] - 1s 750us/step - loss: 0.0670 - val_loss: 0.0583

## Training

**Here's the loss for the deeper model**

```python
cmapss.plot_training_history(history2, figsize=figsize)
tr2, vl2 = history2.history["loss"][-1], np.min(history2.history["val_loss"])
print(f'Final loss: {tr2:.4f} (training), {vl2:.4f} (validation)')
```

    Final loss: 0.0670 (training), 0.0575 (validation)

        Figure

* Depth pays off big in this case

## Predictions

**Our predictions can be interpreted as probabilities (of advancing)**

```python
tr_pred2_prob = nn2.predict(tr_s[dt_in]).ravel()
stop = 1095
cmapss.plot_rul(tr_pred2_prob[:stop], tr['rul'][:stop], same_scale=False, figsize=figsize)
```

        Figure

* The probability falls when closer to failures

## Predictions

**In practice, we need to convert the predictions to integers via rounding**

...Unless we want to deal with one more threshold (in addition to $\theta$)

```python
tr_pred2 = np.round(nn2.predict(tr_s[dt_in]).ravel())
cmapss.plot_rul(tr_pred2[:stop], tr['rul'][:stop], same_scale=False, figsize=figsize)
```

        Figure

* Still, the behavior seems to be reasonable

## Predictions

**Let's see the behavior on the test set**

```python
ts_pred2 = np.round(nn2.predict(ts_s[dt_in]).ravel())
cmapss.plot_rul(ts_pred2[:stop], ts['rul'][:stop], same_scale=False, figsize=figsize)
```

        Figure

* Apparently a decent degree of generalization

## Evaluation

**We can evaluate the classifier _directly_**

...Because it defines the whole policy, with no need for additional calibration!

* On one hand this makes this stage of the process _simpler_
* ...On the other, this is (apparently) a missed opportunity

```python
tr_c2, tr_f2, tr_s2 = cmodel.cost(tr['machine'].values, tr_pred2, 0.5, return_margin=True)
ts_c2, ts_f2, ts_s2 = cmodel.cost(ts['machine'].values, ts_pred2, 0.5, return_margin=True)
print(f'Cost: {tr_c2} (training), {ts_c2} (test)')
print(f'Avg. fails: {tr_f2/len(tr_mcn)} (training), {ts_f2/len(ts_mcn)} (test)')
print(f'Avg. slack: {tr_s2/len(tr_mcn):.2f} (training), {ts_s2/len(ts_mcn):.2f} (test)')
```

    Cost: -17497 (training), -6479 (test)
    Avg. fails: 0.0 (training), 0.0 (test)
    Avg. slack: 22.26 (training), 21.10 (test)

* Still pretty good results, but worse than the best regression approach
* We will investigate _two potential reasons_

# Balancing a Dataset

## Balancing a Dataset

**By construction, our dataset is likely to be strongly unbalanced**

```python
counts = tr_lbl.value_counts(normalize=True)
counts
```

    True     0.938526
    False    0.061474
    Name: rul, dtype: float64

* In these kind of situation, SGD optimization may have convergence issues
* The gradient will push strongly in the direction of the overrepresented class

**A common practice to address this issue is using _class weights_**

* Typically, we use weights inversely proportional to the counts
* ...So as to counter-balance the effect

```python
class_weight = {0: 1/counts[0], 1: 1/counts[1]}
```

## Class-Weights, Cross-Entropy, and Likelihood

**While are class frequencies typically used as weights?**

Let's say our classifier is trained via for _weighted cross-entropy_:
$$
\min_{y} \left\{ - w_1 \sum_{\hat{y}_i = 1} \log y_i -  w_0 \sum_{\hat{y}_i = 0} \log (1 - y_i) \right\}
$$
* Where $\hat{y}$ is the label vector and $y$ the vector of classifier outputs
* $w_0$ and $w_1$ are the weights for class 0 and 1

**If we switch optimization direction and we apply an exponential we get:**
$$
\max_{y} \left\{ e^{w_1 \sum_{\hat{y}_i = 1} \log y_i} e^{w_0 \sum_{\hat{y}_i = 0} \log (1 - y_i)} \right\}
$$
* The _optimal points_ are the same as before

## Class-Weights, Cross-Entropy, and Likelihood

**We with algebraic manipulations we finally obtain:**

$$
\max_{y} \left\{ \prod_{\hat{y}_i = 1} y_i^{w_1} \prod_{\hat{y}_i = 0} (1-y_i)^{w_0} \right\}
$$

**When training, we are _maximizing a likelihood expression_!**

* To see that, just interpret $y_i$ as the (estimated) probability of the class being 1

**The weights act as _exponents for the class probabilities_**

...Which is equivalent to adjusting their frequencies!

* E.g. from a mathematical point of view, setting $w_0$ to 2...
* ...Is equivalent to making a copy of each example having label 0

Hence, inverse class frequencies are often used as weights for dataset rebalancing

## Training with Non-uniform Class Weights

**Let's repeat the training process**

```python
nn3 = build_classifier(hidden=[32, 32])
nn3.compile(optimizer='Adam', loss='binary_crossentropy')
history3 = nn3.fit(tr_s[dt_in], tr_lbl, validation_split=0.2,
                 callbacks=cb, class_weight=class_weight,
                 batch_size=32, epochs=20, verbose=1)
```

    Epoch 1/20
    1135/1135 [==============================] - 1s 879us/step - loss: 0.8826 - val_loss: 0.2998
    Epoch 2/20
    1135/1135 [==============================] - 1s 806us/step - loss: 0.4422 - val_loss: 0.3385
    Epoch 3/20
    1135/1135 [==============================] - 1s 802us/step - loss: 0.3573 - val_loss: 0.1743
    Epoch 4/20
    1135/1135 [==============================] - 1s 802us/step - loss: 0.3320 - val_loss: 0.1880
    Epoch 5/20
    1135/1135 [==============================] - 1s 863us/step - loss: 0.3046 - val_loss: 0.1753
    Epoch 6/20
    1135/1135 [==============================] - 1s 802us/step - loss: 0.3046 - val_loss: 0.1272
    Epoch 7/20
    1135/1135 [==============================] - 1s 873us/step - loss: 0.2975 - val_loss: 0.1932
    Epoch 8/20
    1135/1135 [==============================] - 1s 804us/step - loss: 0.2879 - val_loss: 0.1430
    Epoch 9/20
    1135/1135 [==============================] - 1s 803us/step - loss: 0.2859 - val_loss: 0.1387
    Epoch 10/20
    1135/1135 [==============================] - 1s 804us/step - loss: 0.2781 - val_loss: 0.1651
    Epoch 11/20
    1135/1135 [==============================] - 1s 809us/step - loss: 0.2807 - val_loss: 0.1177
    Epoch 12/20
    1135/1135 [==============================] - 1s 847us/step - loss: 0.2847 - val_loss: 0.1384
    Epoch 13/20
    1135/1135 [==============================] - 1s 868us/step - loss: 0.2805 - val_loss: 0.1733
    Epoch 14/20
    1135/1135 [==============================] - 1s 805us/step - loss: 0.2667 - val_loss: 0.0998
    Epoch 15/20
    1135/1135 [==============================] - 1s 807us/step - loss: 0.2735 - val_loss: 0.1132
    Epoch 16/20
    1135/1135 [==============================] - 1s 801us/step - loss: 0.2731 - val_loss: 0.2934
    Epoch 17/20
    1135/1135 [==============================] - 1s 805us/step - loss: 0.2688 - val_loss: 0.0968
    Epoch 18/20
    1135/1135 [==============================] - 1s 803us/step - loss: 0.2587 - val_loss: 0.1426
    Epoch 19/20
    1135/1135 [==============================] - 1s 802us/step - loss: 0.2667 - val_loss: 0.1319
    Epoch 20/20
    1135/1135 [==============================] - 1s 855us/step - loss: 0.2567 - val_loss: 0.1322

## Predictions

**Let's check how the the raw predictions (probabilities) have changed**

```python
tr_pred3_prob = nn3.predict(tr_s[dt_in]).ravel()
cmapss.plot_rul(tr_pred3_prob[:stop], tr['rul'][:stop], same_scale=False, figsize=figsize)
```

        Figure

* They have actually become _worse_: there are mistakes in the initial section

## Evaluation

**Let's check the model performance in terms of cost**

```python
tr_pred3 = np.round(nn3.predict(tr_s[dt_in]).ravel())
ts_pred3 = np.round(nn3.predict(ts_s[dt_in]).ravel())
tr_c3, tr_f3, tr_s3 = cmodel.cost(tr['machine'].values, tr_pred3, 0.5, return_margin=True)
ts_c3, ts_f3, ts_s3 = cmodel.cost(ts['machine'].values, ts_pred3, 0.5, return_margin=True)
print(f'Cost: {tr_c3} (training), {ts_c3} (test)')
print(f'Avg. fails: {tr_f3/len(tr_mcn)} (training), {ts_f3/len(ts_mcn)} (test)')
print(f'Avg. slack: {tr_s3/len(tr_mcn):.2f} (training), {ts_s3/len(ts_mcn):.2f} (test)')
```

    Cost: -13607 (training), -5095 (test)
    Avg. fails: 0.0 (training), 0.0 (test)
    Avg. slack: 45.87 (training), 47.21 (test)

As expected, the results are worse

* The model now treats the two classes more fairly
* ...But this increases the chance of an (undesirable) early stop

**So, how shall we choose the balance?**

# Bayesian (Surrogate-Based) Optimization

## Taking a Step Back

**In the _regression_ case, we are formally solving:**

$$\begin{align}
&\min_{\theta} \sum_{k \in K} \mathit{cost}(f(x_k\, \lambda^*), \theta) \\
\text{ with: } \lambda^* &= \text{argmin}_\lambda L(f(x_k, \lambda), y_k)
\end{align}$$

* Where $\lambda^*$ is the optimal parameter vector (i.e. the network weights)
* ...And $L$ is the loss function (i.e. the MSE), and $\mathit{cost}$ is our cost model
* $\theta$ is chosen so as to minimize the cost

**This is a _bilevel optimization_ problem**

* However, since $\theta$ does appears neither in $L$ nor in $f$
* ...It can be _decomposed into two sequential subproblems_

## Taking a Step Back

**In the _classification_ case, we are formally solving:**

$$\begin{align}
&\text{min}_{\theta} \sum_{k \in K} \mathit{cost}(f(x_k\, \lambda^*), 0.5) \\
\text{ with: } \lambda^* &= \text{argmin}_\lambda L(f(x_k, \lambda), {\bf 1}_{y_k \geq \theta})
\end{align}$$

* In this case, we use a dummy threshold in the cost model (i.e. 0.5)
* ...And ${\bf 1}_{y_k \geq \theta}$ is the indicator function of $y_k \geq \theta$ (i.e. our class labels)

**Unlike the previous one, this problem _cannot be decomposed_**

...Because $\theta$ appears in the loss function!

* This means we need to _optimize $\theta$ and $\lambda$ at the same time_
* This is obviously complicated, but it's also an _opportunity_
* ...Since we can _adapt the training problem_ based on the threshold

## Taking a Step Back

**We can try to tackle the optimization process in a hierarchical fashion**

1. We search over the possible values of $\theta$
2. For the given $\theta$ value, we compute ${\bf 1}_{y_k \geq \theta}$ (i.e. the class labels)
3. For the given $\theta$ and ${\bf 1}_{y_k \geq \theta}$, we compute $\lambda^*$ (i.e. we train the model)
4. For the given $\theta$, ${\bf 1}_{y_k \geq \theta}$, and $\lambda^*$, we compute the cost

At the end of the process, we choose the configuration with the best cost

**The method is reasonable, but slow**

* We could reduce the runtime by warm-starting each training attempt
* ...And by searching over $\theta$ in a smart way

## Bayesian Optimization

**We will use an approach known as _Bayesian optimization_**

* It's actually a family of _surrogate-based optimization_ methods
* They are all designed to optimize _blackbox functions_
* I.e. functions with an unknown structure, that can only be evaluated

**Formally, they address problems in the form:**

$$
\min_{x \in B} f(x)
$$

* Where $B$ is a box, i.e. a specification of bounds for each component of $x$

In our case:

* The decision variable $x$ would be $\theta$
* The function to be optimized would be the cost

## Bayesian (or Surrogate-Based) Optimization

**Internally, methods in this family store:**

* A collection $\hat{x}$ of evaluated points
* A surrogate-model $\tilde{f}$ for $f$

**The basic loop of the algorithm is then as follow:**

Let $x^*$ be the current candidate optimum

* Based on the current $\tilde{f}$:
  - Determine whether there exists $x^\prime \in B$ that may improve over $x^*$
  - If there is no such point, return $x^*$
  - Otherwise
    + Evaluate $f(x^\prime)$ and possibly replace $x^*$
    + Add $x^\prime$ to $\hat{x}$, adjust $\tilde{f}$, and repeat

## Bayesian (or Surrogate-Based) Optimization

**A few considerations:**

In practice, the algorithm needs to balance _exploration_ and _exploitation_

* We need to explore regions where we cannot make confident predictions
* ...But we need also to focus on regions with promising cost values

**Different Bayesian optimization algorithms:**

* Make use of different surrogate models
* Rely on different criteria for choosing $x^\prime$
* Strike different trade-offs in terms of number of (expensive) evaluations of $f$
* ...And the quality of the obtained solutions

**For more information, see (e.g.) [this tutorial](https://machinelearningmastery.com/what-is-bayesian-optimization/)**

## Our Chosen Optimizer

**We will use the `bayesian-optimization` python module**

* The solver is designed for maximization
* Decent documentation can be found on [its github page](https://github.com/fmfn/BayesianOptimization)
* It's not as fast as (e.g.) [RBFOpt](https://github.com/coin-or/rbfopt), but easier to install and configure

**The solver relies on _Gaussian Processes_ as a surrogate model:**

GPs are guaranteed to (approximately) touch every point in the training set

* ...And behave reasonably well even with a simple RBF kernel
* ...Which is in fact the one used in this case

**GPs provide confidence intervals**

...Which the algorithm uses to balance exploration and exploitation

* Regions with wide C.I. may be worth exploring
* ...And regions with high (upper) C.I. may contain high-quality solutions

## Method Setup

**First, we need to define the function to be maximized**

This will be the (negative) cost, after retraining the classifier:

```python
nn4 = build_classifier(hidden=[32, 32])
nn4.compile(optimizer='Adam', loss='binary_crossentropy')
nn4.set_weights(nn2.get_weights())

bo_weights = {}
def classification_obj(class_thr, class0_weight):
    # Define new labels
    tr_lbl = (tr['rul'] >= class_thr)
    # Fit
    nn4.fit(tr_s[dt_in], tr_lbl, validation_split=0.2,
                     callbacks=cb, class_weight={1:1, 0:class0_weight},
                     batch_size=32, epochs=5, verbose=0)
    # Cost computation
    tr_pred = np.round(nn4.predict(tr_s[dt_in]).ravel())
    tr_cost = cmodel.cost(tr['machine'].values, tr_pred, 0.5)
    # Store the model weights
    bo_weights[class_thr, class0_weight] = nn4.get_weights()
    return -tr_cost
```

## Method Setup

**We rely on warm-starting to reduce the training time**

We start from (the weights of) our deep MLP

```python
nn4 = build_classifier(hidden=[32, 32])
nn4.compile(optimizer='Adam', loss='binary_crossentropy')
nn4.set_weights(nn2.get_weights())
```

...So that we can train for just a few epochs

```python
nn4.fit(tr_s[dt_in], tr_lbl, validation_split=0.2,
                 callbacks=cb, class_weight={1:1, 0:class0_weight},
                 batch_size=32, epochs=5, verbose=0)
```

* Since the training process is stochastic...
* The optimization result will be stochastic as well

## Method Setup

**We will optimize _both $\theta$ and the class weights_**

```python
def classification_obj(class_thr, class0_weight):
    ...
    nn4.fit(tr_s[dt_in], tr_lbl, validation_split=0.2,
                     callbacks=cb, class_weight={1:1, 0:class0_weight},
                     batch_size=32, epochs=5, verbose=0)
    ...
```

* We can afford it, since we have a much better optimizer than grid search

**We need to store the weights of each model**

...So that we can recover the optimal configuration:

```python
bo_weights[class_thr, class0_weight] = nn4.get_weights()
```

## Method Setup

**We now define the bounding box**

```python
box = {'class_thr': (1, 15), 'class0_weight': (1, 10)}
```

* We will consider stopping from 1 to 15 steps before a failure
* We will consider weights for class 0 from 1 to 10

**Then we can configure the optimizer:**

```python
from bayes_opt import BayesianOptimization
optimizer = BayesianOptimization(f=classification_obj,
                                 pbounds=box, random_state=42, verbose=2)
```

* We specify the function to be maximized, the box
* ...But also a random seed and the logging level

## Optimization

**Finally, we run the optimization process**

```python
optimizer.maximize(n_iter=10, init_points=3)
```

    |   iter    |  target   | class0... | class_thr |
    -------------------------------------------------
    |  1        |  1.479e+0 |  4.371    |  14.31    |
    |  2        |  1.569e+0 |  7.588    |  9.381    |
    |  3        |  1.893e+0 |  2.404    |  3.184    |
    |  4        | -1.667e+0 |  5.649    |  1.077    |
    |  5        |  1.772e+0 |  2.392    |  3.294    |
    |  6        | -6.204e+0 |  1.25     |  1.719    |
    |  7        |  1.797e+0 |  3.629    |  3.1      |
    |  8        |  1.599e+0 |  6.677    |  10.78    |
    |  9        |  1.41e+04 |  8.388    |  10.92    |
    |  10       |  1.755e+0 |  5.765    |  9.208    |
    |  11       |  1.72e+04 |  4.752    |  10.7     |
    |  12       |  1.646e+0 |  5.375    |  12.49    |
    |  13       |  1.691e+0 |  3.519    |  12.33    |
    =================================================

## Retrieving the Results

**We can access the best configuration via**

```python
optimizer.max
```

    {'target': 18934.0,
     'params': {'class0_weight': 2.4041677639819286,
      'class_thr': 3.183923284706837}}

Once we know the configuration, we can reconstruct the corresponding model:

```python
class_thr4 = optimizer.max['params']['class_thr']
class0_weight4 = optimizer.max['params']['class0_weight']
nn4.set_weights(bo_weights[class_thr4, class0_weight4])
```

* We do this by accessing the weights in the `bo_weights` dictionary

## Evaluation

**Finally, we can obtain the predictions**

```python
tr_pred4 = np.round(nn4.predict(tr_s[dt_in]).ravel())
ts_pred4 = np.round(nn4.predict(ts_s[dt_in]).ravel())
```

...And we can evaluate the optimized model:

```python
tr_c4, tr_f4, tr_s4 = cmodel.cost(tr['machine'].values, tr_pred4, 0.5, return_margin=True)
ts_c4, ts_f4, ts_s4 = cmodel.cost(ts['machine'].values, ts_pred4, 0.5, return_margin=True)
print(f'Cost: {tr_c4} (training), {ts_c4} (test)')
print(f'Avg. fails: {tr_f4/len(tr_mcn)} (training), {ts_f4/len(ts_mcn)} (test)')
print(f'Avg. slack: {tr_s4/len(tr_mcn):.2f} (training), {ts_s4/len(ts_mcn):.2f} (test)')
```

    Cost: -18934 (training), -6745 (test)
    Avg. fails: 0.005376344086021506 (training), 0.015873015873015872 (test)
    Avg. slack: 10.78 (training), 7.68 (test)

* We are now on-par with the best regression based approaches!
* Actual results may be slightly better or worse due to randomness

## Considerations

**Why choosing a classification-based approach?**

* In some cases, classification can be _easier_ than regression
* The threshold affects training, which is computationally heavier
* ...But it means that training can _adapt to the threshold_

**We we tackle the prediction and cost optimization stages in isolation:**

* The two stages are "glued" by the ML loss (MSE or cross-entropy)
* ....But the loss does not penalize mistakes based on their (economic) cost!
* Ideally, an end-to-end cost optimization approach should work better

**Bayesian/surrogate-based optimization can have many uses**

* Hyper-parameter optimization
* Fitting simulation models
* Optimizing equipment setpoints (e.g. Air Conditioning), based on simulation
* ...

3. RUL Prediction as Classification.ipynb